diff options
88 files changed, 1698 insertions, 5563 deletions
diff --git a/Makefile b/Makefile index 834f84b286bf..45a3338ed21c 100644 --- a/Makefile +++ b/Makefile @@ -5,17 +5,7 @@ makefiles = \ src/libmain/local.mk \ src/libexpr/local.mk \ src/nix/local.mk \ - src/nix-store/local.mk \ - src/nix-instantiate/local.mk \ - src/nix-env/local.mk \ - src/nix-daemon/local.mk \ - src/nix-collect-garbage/local.mk \ - src/nix-copy-closure/local.mk \ - src/nix-prefetch-url/local.mk \ src/resolve-system-dependencies/local.mk \ - src/nix-channel/local.mk \ - src/nix-build/local.mk \ - src/build-remote/local.mk \ scripts/local.mk \ corepkgs/local.mk \ misc/systemd/local.mk \ diff --git a/Makefile.config.in b/Makefile.config.in index 08edcb863ef6..fad077d869be 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -15,6 +15,7 @@ SODIUM_LIBS = @SODIUM_LIBS@ LIBLZMA_LIBS = @LIBLZMA_LIBS@ SQLITE3_LIBS = @SQLITE3_LIBS@ LIBBROTLI_LIBS = @LIBBROTLI_LIBS@ +EDITLINE_LIBS = @EDITLINE_LIBS@ bash = @bash@ bindir = @bindir@ brotli = @brotli@ diff --git a/configure.ac b/configure.ac index 9c85182efbf9..85fec77a29c9 100644 --- a/configure.ac +++ b/configure.ac @@ -164,6 +164,8 @@ PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CX # Look for libcurl, a required dependency. PKG_CHECK_MODULES([LIBCURL], [libcurl], [CXXFLAGS="$LIBCURL_CFLAGS $CXXFLAGS"]) +# Look for editline, a required dependency. +PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLAGS"]) # Look for libsodium, an optional dependency. PKG_CHECK_MODULES([SODIUM], [libsodium], diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index fd09883be402..e9947ebc673f 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -757,6 +757,33 @@ password <replaceable>my-password</replaceable> </varlistentry> + <varlistentry xml:id="conf-system-features"><term><literal>system-features</literal></term> + + <listitem><para>A set of system “features” supported by this + machine, e.g. <literal>kvm</literal>. Derivations can express a + dependency on such features through the derivation attribute + <varname>requiredSystemFeatures</varname>. For example, the + attribute + +<programlisting> +requiredSystemFeatures = [ "kvm" ]; +</programlisting> + + ensures that the derivation can only be built on a machine with + the <literal>kvm</literal> feature.</para> + + <para>This setting by default includes <literal>kvm</literal> if + <filename>/dev/kvm</filename> is accessible, and the + pseudo-features <literal>nixos-test</literal>, + <literal>benchmark</literal> and <literal>big-parallel</literal> + that are used in Nixpkgs to route builds to specific + machines.</para> + + </listitem> + + </varlistentry> + + <varlistentry xml:id="conf-timeout"><term><literal>timeout</literal></term> <listitem> diff --git a/doc/manual/command-ref/nix-shell.xml b/doc/manual/command-ref/nix-shell.xml index 5c44c4a8f446..cb443c888d3d 100644 --- a/doc/manual/command-ref/nix-shell.xml +++ b/doc/manual/command-ref/nix-shell.xml @@ -317,13 +317,28 @@ while (my $token = $p->get_tag("a")) { </para> -<para>Finally, the following Haskell script uses a specific branch of -Nixpkgs/NixOS (the 14.12 stable branch): +<para>Sometimes you need to pass a simple Nix expression to customize +a package like Terraform: + +<programlisting><![CDATA[ +#! /usr/bin/env nix-shell +#! nix-shell -i bash -p "terraform.withPlugins (plugins: [ plugins.openstack ])" + +terraform apply +]]></programlisting> + +<note><para>You must use double quotes (<literal>"</literal>) when +passing a simple Nix expression in a nix-shell shebang.</para></note> +</para> + +<para>Finally, using the merging of multiple nix-shell shebangs the +following Haskell script uses a specific branch of Nixpkgs/NixOS (the +18.03 stable branch): <programlisting><![CDATA[ #! /usr/bin/env nix-shell #! nix-shell -i runghc -p haskellPackages.ghc haskellPackages.HTTP haskellPackages.tagsoup -#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz +#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-18.03.tar.gz import Network.HTTP import Text.HTML.TagSoup diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml index c827d85b3815..41a04f265d7c 100644 --- a/doc/manual/command-ref/nix-store.xml +++ b/doc/manual/command-ref/nix-store.xml @@ -679,6 +679,18 @@ query is applied to the target of the symlink.</para> </varlistentry> + <varlistentry><term><option>--graphml</option></term> + + <listitem><para>Prints the references graph of the store paths + <replaceable>paths</replaceable> in the <link + xlink:href="http://graphml.graphdrawing.org/">GraphML</link> file format. + This can be used to visualise dependency graphs. To obtain a + build-time dependency graph, apply this to a store derivation. To + obtain a runtime dependency graph, apply it to an output + path.</para></listitem> + + </varlistentry> + <varlistentry><term><option>--binding</option> <replaceable>name</replaceable></term> <term><option>-b</option> <replaceable>name</replaceable></term> diff --git a/doc/manual/expressions/advanced-attributes.xml b/doc/manual/expressions/advanced-attributes.xml index 9422e82ff362..07b0d97d3f7d 100644 --- a/doc/manual/expressions/advanced-attributes.xml +++ b/doc/manual/expressions/advanced-attributes.xml @@ -216,7 +216,7 @@ fetchurl { <programlisting> { stdenv, curl }: # The <command>curl</command> program is used for downloading. -{ url, md5 }: +{ url, sha256 }: stdenv.mkDerivation { name = baseNameOf (toString url); @@ -224,10 +224,10 @@ stdenv.mkDerivation { buildInputs = [ curl ]; # This is a fixed-output derivation; the output must be a regular - # file with MD5 hash <varname>md5</varname>. + # file with SHA256 hash <varname>sha256</varname>. outputHashMode = "flat"; - outputHashAlgo = "md5"; - outputHash = md5; + outputHashAlgo = "sha256"; + outputHash = sha256; inherit url; } @@ -237,8 +237,8 @@ stdenv.mkDerivation { <para>The <varname>outputHashAlgo</varname> attribute specifies the hash algorithm used to compute the hash. It can currently be - <literal>"md5"</literal>, <literal>"sha1"</literal> or - <literal>"sha256"</literal>.</para> + <literal>"sha1"</literal>, <literal>"sha256"</literal> or + <literal>"sha512"</literal>.</para> <para>The <varname>outputHashMode</varname> attribute determines how the hash is computed. It must be one of the following two @@ -251,7 +251,7 @@ stdenv.mkDerivation { <listitem><para>The output must be a non-executable regular file. If it isn’t, the build fails. The hash is simply computed over the contents of that file (so it’s equal to what - Unix commands like <command>md5sum</command> or + Unix commands like <command>sha256sum</command> or <command>sha1sum</command> produce).</para> <para>This is the default.</para></listitem> @@ -312,9 +312,7 @@ big = "a very long string"; <varlistentry><term><varname>preferLocalBuild</varname></term> <listitem><para>If this attribute is set to - <literal>true</literal>, it has two effects. First, the - derivation will always be built, not substituted, even if a - substitute is available. Second, if <link + <literal>true</literal> and <link linkend="chap-distributed-builds">distributed building is enabled</link>, then, if possible, the derivaton will be built locally instead of forwarded to a remote machine. This is @@ -324,6 +322,19 @@ big = "a very long string"; </varlistentry> + + <varlistentry><term><varname>allowSubstitutes</varname></term> + + <listitem><para>If this attribute is set to + <literal>false</literal>, then Nix will always build this + derivation; it will not try to substitute its outputs. This is + useful for very trivial derivations (such as + <function>writeText</function> in Nixpkgs) that are cheaper to + build than to substitute from a binary cache.</para></listitem> + + </varlistentry> + + </variablelist> </section> diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 873f30b062ee..412622714d40 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -21,7 +21,8 @@ available as <function>builtins.derivation</function>.</para> <variablelist> - <varlistentry><term><function>abort</function> <replaceable>s</replaceable></term> + <varlistentry xml:id='builtin-abort'> + <term><function>abort</function> <replaceable>s</replaceable></term> <listitem><para>Abort Nix expression evaluation, print error message <replaceable>s</replaceable>.</para></listitem> @@ -29,8 +30,10 @@ available as <function>builtins.derivation</function>.</para> </varlistentry> - <varlistentry><term><function>builtins.add</function> - <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + <varlistentry xml:id='builtin-add'> + <term><function>builtins.add</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable> + </term> <listitem><para>Return the sum of the numbers <replaceable>e1</replaceable> and @@ -39,8 +42,9 @@ available as <function>builtins.derivation</function>.</para> </varlistentry> - <varlistentry><term><function>builtins.all</function> - <replaceable>pred</replaceable> <replaceable>list</replaceable></term> + <varlistentry xml:id='builtin-all'> + <term><function>builtins.all</function> + <replaceable>pred</replaceable> <replaceable>list</replaceable></term> <listitem><para>Return <literal>true</literal> if the function <replaceable>pred</replaceable> returns <literal>true</literal> @@ -50,8 +54,9 @@ available as <function>builtins.derivation</function>.</para> </varlistentry> - <varlistentry><term><function>builtins.any</function> - <replaceable>pred</replaceable> <replaceable>list</replaceable></term> + <varlistentry xml:id='builtin-any'> + <term><function>builtins.any</function> + <replaceable>pred</replaceable> <replaceable>list</replaceable></term> <listitem><para>Return <literal>true</literal> if the function <replaceable>pred</replaceable> returns <literal>true</literal> @@ -61,8 +66,9 @@ available as <function>builtins.derivation</function>.</para> </varlistentry> - <varlistentry><term><function>builtins.attrNames</function> - <replaceable>set</replaceable></term> + <varlistentry xml:id='builtin-attrNames'> + <term><function>builtins.attrNames</function> + <replaceable>set</replaceable></term> <listitem><para>Return the names of the attributes in the set <replaceable>set</replaceable> in an alphabetically sorted list. For instance, @@ -72,8 +78,9 @@ available as <function>builtins.derivation</function>.</para> </varlistentry> - <varlistentry><term><function>builtins.attrValues</function> - <replaceable>set</replaceable></term> + <varlistentry xml:id='builtin-attrValues'> + <term><function>builtins.attrValues</function> + <replaceable>set</replaceable></term> <listitem><para>Return the values of the attributes in the set <replaceable>set</replaceable> in the order corresponding to the @@ -82,7 +89,8 @@ available as <function>builtins.derivation</function>.</para> </varlistentry> - <varlistentry><term><function>baseNameOf</function> <replaceable>s</replaceable></term> + <varlistentry xml:id='builtin-baseNameOf'> + <term><function>baseNameOf</function> <replaceable>s</replaceable></term> <listitem><para>Return the <emphasis>base name</emphasis> of the string <replaceable>s</replaceable>, that is, everything following @@ -92,8 +100,9 @@ available as <function>builtins.derivation</function>.</para> </varlistentry> - <varlistentry><term><function>builtins.bitAnd</function> - <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + <varlistentry xml:id='builtin-bitAnd'> + <term><function>builtins.bitAnd</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> <listitem><para>Return the bitwise AND of the integers <replaceable>e1</replaceable> and @@ -102,8 +111,9 @@ available as <function>builtins.derivation</function>.</para> </varlistentry> - <varlistentry><term><function>builtins.bitOr</function> - <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + <varlistentry xml:id='builtin-bitOr'> + <term><function>builtins.bitOr</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> <listitem><para>Return the bitwise OR of the integers <replaceable>e1</replaceable> and @@ -112,8 +122,9 @@ available as <function>builtins.derivation</function>.</para> </varlistentry> - <varlistentry><term><function>builtins.bitXor</function> - <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + <varlistentry xml:id='builtin-bitXor'> + <term><function>builtins.bitXor</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> <listitem><para>Return the bitwise XOR of the integers <replaceable>e1</replaceable> and @@ -122,7 +133,8 @@ available as <function>builtins.derivation</function>.</para> </varlistentry> - <varlistentry><term><varname>builtins</varname></term> + <varlistentry xml:id='builtin-builtins'> + <term><varname>builtins</varname></term> <listitem><para>The set <varname>builtins</varname> contains all the built-in functions and values. You can use @@ -139,8 +151,9 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting> </varlistentry> - <varlistentry><term><function>builtins.compareVersions</function> - <replaceable>s1</replaceable> <replaceable>s2</replaceable></term> + <varlistentry xml:id='builtin-compareVersions'> + <term><function>builtins.compareVersions</function> + <replaceable>s1</replaceable> <replaceable>s2</replaceable></term> <listitem><para>Compare two strings representing versions and return <literal>-1</literal> if version @@ -156,8 +169,9 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting> </varlistentry> - <varlistentry><term><function>builtins.splitVersion</function> - <replaceable>s</replaceable></term> + <varlistentry xml:id='builtin-splitVersion'> + <term><function>builtins.splitVersion</function> + <replaceable>s</replaceable></term> <listitem><para>Split a string representing a version into its components, by the same version splitting logic underlying the @@ -167,16 +181,18 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting> </varlistentry> - <varlistentry><term><function>builtins.concatLists</function> - <replaceable>lists</replaceable></term> + <varlistentry xml:id='builtin-concatLists'> + <term><function>builtins.concatLists</function> + <replaceable>lists</replaceable></term> <listitem><para>Concatenate a list of lists into a single list.</para></listitem> </varlistentry> - <varlistentry><term><function>builtins.concatStringsSep</function> - <replaceable>separator</replaceable> <replaceable>list</replaceable></term> + <varlistentry xml:id='builtin-concatStringsSep'> + <term><function>builtins.concatStringsSep</function> + <replaceable>separator</replaceable> <replaceable>list</replaceable></term> <listitem><para>Concatenate a list of strings with a separator between each element, e.g. <literal>concatStringsSep "/" @@ -184,8 +200,8 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting> </varlistentry> - <varlistentry - xml:id='builtin-currentSystem'><term><varname>builtins.currentSystem</varname></term> + <varlistentry xml:id='builtin-currentSystem'> + <term><varname>builtins.currentSystem</varname></term> <listitem><para>The built-in value <varname>currentSystem</varname> evaluates to the Nix platform identifier for the Nix installation @@ -218,8 +234,9 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting> --> - <varlistentry><term><function>builtins.deepSeq</function> - <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + <varlistentry xml:id='builtin-deepSeq'> + <term><function>builtins.deepSeq</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> <listitem><para>This is like <literal>seq <replaceable>e1</replaceable> @@ -231,8 +248,9 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting> </varlistentry> - <varlistentry><term><function>derivation</function> - <replaceable>attrs</replaceable></term> + <varlistentry xml:id='builtin-derivation'> + <term><function>derivation</function> + <replaceable>attrs</replaceable></term> <listitem><para><function>derivation</function> is described in <xref linkend='ssec-derivation' />.</para></listitem> @@ -240,7 +258,8 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting> </varlistentry> - <varlistentry><term><function>dirOf</function> <replaceable>s</replaceable></term> + <varlistentry xml:id='builtin-dirOf'> + <term><function>dirOf</function> <replaceable>s</replaceable></term> <listitem><para>Return the directory part of the string <replaceable>s</replaceable>, that is, everything before the final @@ -250,8 +269,9 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting> </varlistentry> - <varlistentry><term><function>builtins.div</function> - <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + <varlistentry xml:id='builtin-div'> + <term><function>builtins.div</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> <listitem><para>Return the quotient of the numbers <replaceable>e1</replaceable> and @@ -259,8 +279,9 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting> </varlistentry> - <varlistentry><term><function>builtins.elem</function> - <replaceable>x</replaceable> <replaceable>xs</replaceable></term> + <varlistentry xml:id='builtin-elem'> + <term><function>builtins.elem</function> + <replaceable>x</replaceable> <replaceable>xs</replaceable></term> <listitem><para>Return <literal>true</literal> if a value equal to <replaceable>x</replaceable> occurs in the list @@ -270,8 +291,9 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting> </varlistentry> - <varlistentry><term><function>builtins.elemAt</function> - <replaceable>xs</replaceable> <replaceable>n</replaceable></term> + <varlistentry xml:id='builtin-elemAt'> + <term><function>builtins.elemAt</function> + <replaceable>xs</replaceable> <replaceable>n</replaceable></term> <listitem><para>Return element <replaceable>n</replaceable> from the list <replaceable>xs</replaceable>. Elements are counted @@ -281,8 +303,9 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting> </varlistentry> - <varlistentry><term><function>builtins.fetchurl</function> - <replaceable>url</replaceable></term> + <varlistentry xml:id='builtin-fetchurl'> + <term><function>builtins.fetchurl</function> + <replaceable>url</replaceable></term> <listitem><para>Download the specified URL and return the path of the downloaded file. This function is not available if <link @@ -292,8 +315,9 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting> </varlistentry> - <varlistentry><term><function>fetchTarball</function> - <replaceable>url</replaceable></term> + <varlistentry xml:id='builtin-fetchTarball'> + <term><function>fetchTarball</function> + <replaceable>url</replaceable></term> <listitem><para>Download the specified URL, unpack it and return the path of the unpacked tree. The file must be a tape archive @@ -346,7 +370,7 @@ stdenv.mkDerivation { … } </varlistentry> - <varlistentry> + <varlistentry xml:id='builtin-fetchGit'> <term> <function>builtins.fetchGit</function> <replaceable>args</replaceable> @@ -412,7 +436,7 @@ stdenv.mkDerivation { … } <title>Fetching a repository's specific commit on an arbitrary branch</title> <para> If the revision you're looking for is in the default branch - of the gift repository you don't strictly need to specify + of the git repository you don't strictly need to specify the branch name in the <varname>ref</varname> attribute. </para> <para> @@ -441,7 +465,7 @@ stdenv.mkDerivation { … } <title>Fetching a repository's specific commit on the default branch</title> <para> If the revision you're looking for is in the default branch - of the gift repository you may omit the + of the git repository you may omit the <varname>ref</varname> attribute. </para> <programlisting>builtins.fetchGit { @@ -546,7 +570,8 @@ stdenv.mkDerivation { </varlistentry> - <varlistentry><term><function>builtins.foldl’</function> + <varlistentry xml:id='builtin-foldl-prime'> + <term><function>builtins.foldl’</function> <replaceable>op</replaceable> <replaceable>nul</replaceable> <replaceable>list</replaceable></term> <listitem><para>Reduce a list by applying a binary operator, from @@ -559,7 +584,8 @@ stdenv.mkDerivation { </varlistentry> - <varlistentry><term><function>builtins.functionArgs</function> + <varlistentry xml:id='builtin-functionArgs'> + <term><function>builtins.functionArgs</function> <replaceable>f</replaceable></term> <listitem><para> @@ -577,7 +603,8 @@ stdenv.mkDerivation { </varlistentry> - <varlistentry><term><function>builtins.fromJSON</function> <replaceable>e</replaceable></term> + <varlistentry xml:id='builtin-fromJSON'> + <term><function>builtins.fromJSON</function> <replaceable>e</replaceable></term> <listitem><para>Convert a JSON string to a Nix value. For example, @@ -592,8 +619,9 @@ builtins.fromJSON ''{"x": [1, 2, 3], "y": null}'' </varlistentry> - <varlistentry><term><function>builtins.genList</function> - <replaceable>generator</replaceable> <replaceable>length</replaceable></term> + <varlistentry xml:id='builtin-genList'> + <term><function>builtins.genList</function> + <replaceable>generator</replaceable> <replaceable>length</replaceable></term> <listitem><para>Generate list of size <replaceable>length</replaceable>, with each element @@ -610,8 +638,9 @@ builtins.genList (x: x * x) 5 </varlistentry> - <varlistentry><term><function>builtins.getAttr</function> - <replaceable>s</replaceable> <replaceable>set</replaceable></term> + <varlistentry xml:id='builtin-getAttr'> + <term><function>builtins.getAttr</function> + <replaceable>s</replaceable> <replaceable>set</replaceable></term> <listitem><para><function>getAttr</function> returns the attribute named <replaceable>s</replaceable> from @@ -623,8 +652,9 @@ builtins.genList (x: x * x) 5 </varlistentry> - <varlistentry><term><function>builtins.getEnv</function> - <replaceable>s</replaceable></term> + <varlistentry xml:id='builtin-getEnv'> + <term><function>builtins.getEnv</function> + <replaceable>s</replaceable></term> <listitem><para><function>getEnv</function> returns the value of the environment variable <replaceable>s</replaceable>, or an empty @@ -641,8 +671,9 @@ builtins.genList (x: x * x) 5 </varlistentry> - <varlistentry><term><function>builtins.hasAttr</function> - <replaceable>s</replaceable> <replaceable>set</replaceable></term> + <varlistentry xml:id='builtin-hasAttr'> + <term><function>builtins.hasAttr</function> + <replaceable>s</replaceable> <replaceable>set</replaceable></term> <listitem><para><function>hasAttr</function> returns <literal>true</literal> if <replaceable>set</replaceable> has an @@ -655,8 +686,9 @@ builtins.genList (x: x * x) 5 </varlistentry> - <varlistentry><term><function>builtins.hashString</function> - <replaceable>type</replaceable> <replaceable>s</replaceable></term> + <varlistentry xml:id='builtin-hashString'> + <term><function>builtins.hashString</function> + <replaceable>type</replaceable> <replaceable>s</replaceable></term> <listitem><para>Return a base-16 representation of the cryptographic hash of string <replaceable>s</replaceable>. The @@ -667,8 +699,9 @@ builtins.genList (x: x * x) 5 </varlistentry> - <varlistentry><term><function>builtins.head</function> - <replaceable>list</replaceable></term> + <varlistentry xml:id='builtin-head'> + <term><function>builtins.head</function> + <replaceable>list</replaceable></term> <listitem><para>Return the first element of a list; abort evaluation if the argument isn’t a list or is an empty list. You @@ -678,8 +711,9 @@ builtins.genList (x: x * x) 5 </varlistentry> - <varlistentry><term><function>import</function> - <replaceable>path</replaceable></term> + <varlistentry xml:id='builtin-import'> + <term><function>import</function> + <replaceable>path</replaceable></term> <listitem><para>Load, parse and return the Nix expression in the file <replaceable>path</replaceable>. If <replaceable>path @@ -733,8 +767,9 @@ x: x + 456</programlisting> </varlistentry> - <varlistentry><term><function>builtins.intersectAttrs</function> - <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + <varlistentry xml:id='builtin-intersectAttrs'> + <term><function>builtins.intersectAttrs</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> <listitem><para>Return a set consisting of the attributes in the set <replaceable>e2</replaceable> that also exist in the set @@ -743,8 +778,9 @@ x: x + 456</programlisting> </varlistentry> - <varlistentry><term><function>builtins.isAttrs</function> - <replaceable>e</replaceable></term> + <varlistentry xml:id='builtin-isAttrs'> + <term><function>builtins.isAttrs</function> + <replaceable>e</replaceable></term> <listitem><para>Return <literal>true</literal> if <replaceable>e</replaceable> evaluates to a set, and @@ -753,8 +789,9 @@ x: x + 456</programlisting> </varlistentry> - <varlistentry><term><function>builtins.isList</function> - <replaceable>e</replaceable></term> + <varlistentry xml:id='builtin-isList'> + <term><function>builtins.isList</function> + <replaceable>e</replaceable></term> <listitem><para>Return <literal>true</literal> if <replaceable>e</replaceable> evaluates to a list, and @@ -763,7 +800,7 @@ x: x + 456</programlisting> </varlistentry> - <varlistentry><term><function>builtins.isFunction</function> + <varlistentry xml:id='builtin-isFunction'><term><function>builtins.isFunction</function> <replaceable>e</replaceable></term> <listitem><para>Return <literal>true</literal> if @@ -773,8 +810,9 @@ x: x + 456</programlisting> </varlistentry> - <varlistentry><term><function>builtins.isString</function> - <replaceable>e</replaceable></term> + <varlistentry xml:id='builtin-isString'> + <term><function>builtins.isString</function> + <replaceable>e</replaceable></term> <listitem><para>Return <literal>true</literal> if <replaceable>e</replaceable> evaluates to a string, and @@ -783,8 +821,9 @@ x: x + 456</programlisting> </varlistentry> - <varlistentry><term><function>builtins.isInt</function> - <replaceable>e</replaceable></term> + <varlistentry xml:id='builtin-isInt'> + <term><function>builtins.isInt</function> + <replaceable>e</replaceable></term> <listitem><para>Return <literal>true</literal> if <replaceable>e</replaceable> evaluates to an int, and @@ -793,8 +832,9 @@ x: x + 456</programlisting> </varlistentry> - <varlistentry><term><function>builtins.isFloat</function> - <replaceable>e</replaceable></term> + <varlistentry xml:id='builtin-isFloat'> + <term><function>builtins.isFloat</function> + <replaceable>e</replaceable></term> <listitem><para>Return <literal>true</literal> if <replaceable>e</replaceable> evaluates to a float, and @@ -803,8 +843,9 @@ x: x + 456</programlisting> </varlistentry> - <varlistentry><term><function>builtins.isBool</function> - <replaceable>e</replaceable></term> + <varlistentry xml:id='builtin-isBool'> + <term><function>builtins.isBool</function> + <replaceable>e</replaceable></term> <listitem><para>Return <literal>true</literal> if <replaceable>e</replaceable> evaluates to a bool, and @@ -813,8 +854,9 @@ x: x + 456</programlisting> </varlistentry> - <varlistentry><term><function>isNull</function> - <replaceable>e</replaceable></term> + <varlistentry xml:id='builtin-isNull'> + <term><function>isNull</function> + <replaceable>e</replaceable></term> <listitem><para>Return <literal>true</literal> if <replaceable>e</replaceable> evaluates to <literal>null</literal>, @@ -828,8 +870,9 @@ x: x + 456</programlisting> </varlistentry> - <varlistentry><term><function>builtins.length</function> - <replaceable>e</replaceable></term> + <varlistentry xml:id='builtin-length'> + <term><function>builtins.length</function> + <replaceable>e</replaceable></term> <listitem><para>Return the length of the list <replaceable>e</replaceable>.</para></listitem> @@ -837,8 +880,9 @@ x: x + 456</programlisting> </varlistentry> - <varlistentry><term><function>builtins.lessThan</function> - <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + <varlistentry xml:id='builtin-lessThan'> + <term><function>builtins.lessThan</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> <listitem><para>Return <literal>true</literal> if the number <replaceable>e1</replaceable> is less than the number @@ -850,8 +894,9 @@ x: x + 456</programlisting> </varlistentry> - <varlistentry><term><function>builtins.listToAttrs</function> - <replaceable>e</replaceable></term> + <varlistentry xml:id='builtin-listToAttrs'> + <term><function>builtins.listToAttrs</function> + <replaceable>e</replaceable></term> <listitem><para>Construct a set from a list specifying the names and values of each attribute. Each element of the list should be @@ -877,8 +922,9 @@ builtins.listToAttrs </varlistentry> - <varlistentry><term><function>map</function> - <replaceable>f</replaceable> <replaceable>list</replaceable></term> + <varlistentry xml:id='builtin-map'> + <term><function>map</function> + <replaceable>f</replaceable> <replaceable>list</replaceable></term> <listitem><para>Apply the function <replaceable>f</replaceable> to each element in the list <replaceable>list</replaceable>. For @@ -893,14 +939,15 @@ map (x: "foo" + x) [ "bar" "bla" "abc" ]</programlisting> </varlistentry> - <varlistentry><term><function>builtins.match</function> - <replaceable>regex</replaceable> <replaceable>str</replaceable></term> + <varlistentry xml:id='builtin-match'> + <term><function>builtins.match</function> + <replaceable>regex</replaceable> <replaceable>str</replaceable></term> - <listitem><para>Returns a list if the <link - xlink:href="http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04">extended - POSIX regular expression</link> <replaceable>regex</replaceable> - matches <replaceable>str</replaceable> precisely, otherwise returns - <literal>null</literal>. Each item in the list is a regex group. + <listitem><para>Returns a list if the <link + xlink:href="http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04">extended + POSIX regular expression</link> <replaceable>regex</replaceable> + matches <replaceable>str</replaceable> precisely, otherwise returns + <literal>null</literal>. Each item in the list is a regex group. <programlisting> builtins.match "ab" "abc" @@ -926,11 +973,12 @@ builtins.match "[[:space:]]+([[:upper:]]+)[[:space:]]+" " FOO " Evaluates to <literal>[ "foo" ]</literal>. - </para></listitem> + </para></listitem> </varlistentry> - <varlistentry><term><function>builtins.mul</function> - <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + <varlistentry xml:id='builtin-mul'> + <term><function>builtins.mul</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> <listitem><para>Return the product of the numbers <replaceable>e1</replaceable> and @@ -939,8 +987,9 @@ Evaluates to <literal>[ "foo" ]</literal>. </varlistentry> - <varlistentry><term><function>builtins.parseDrvName</function> - <replaceable>s</replaceable></term> + <varlistentry xml:id='builtin-parseDrvName'> + <term><function>builtins.parseDrvName</function> + <replaceable>s</replaceable></term> <listitem><para>Split the string <replaceable>s</replaceable> into a package name and version. The package name is everything up to @@ -953,7 +1002,7 @@ Evaluates to <literal>[ "foo" ]</literal>. </varlistentry> - <varlistentry> + <varlistentry xml:id='builtin-path'> <term> <function>builtins.path</function> <replaceable>args</replaceable> @@ -1023,32 +1072,20 @@ Evaluates to <literal>[ "foo" ]</literal>. </listitem> </varlistentry> - <varlistentry><term><function>builtins.pathExists</function> - <replaceable>path</replaceable></term> + <varlistentry xml:id='builtin-pathExists'> + <term><function>builtins.pathExists</function> + <replaceable>path</replaceable></term> <listitem><para>Return <literal>true</literal> if the path - <replaceable>path</replaceable> exists, and - <literal>false</literal> otherwise. One application of this - function is to conditionally include a Nix expression containing - user configuration: - -<programlisting> -let - fileName = builtins.getEnv "CONFIG_FILE"; - config = - if fileName != "" && builtins.pathExists (builtins.toPath fileName) - then import (builtins.toPath fileName) - else { someSetting = false; }; <lineannotation># default configuration</lineannotation> -in config.someSetting</programlisting> - - (Note that <envar>CONFIG_FILE</envar> must be an absolute path for - this to work.)</para></listitem> + <replaceable>path</replaceable> exists at evaluation time, and + <literal>false</literal> otherwise.</para></listitem> </varlistentry> - <varlistentry><term><function>builtins.readDir</function> - <replaceable>path</replaceable></term> + <varlistentry xml:id='builtin-readDir'> + <term><function>builtins.readDir</function> + <replaceable>path</replaceable></term> <listitem><para>Return the contents of the directory <replaceable>path</replaceable> as a set mapping directory entries @@ -1069,8 +1106,9 @@ in config.someSetting</programlisting> </varlistentry> - <varlistentry><term><function>builtins.readFile</function> - <replaceable>path</replaceable></term> + <varlistentry xml:id='builtin-readFile'> + <term><function>builtins.readFile</function> + <replaceable>path</replaceable></term> <listitem><para>Return the contents of the file <replaceable>path</replaceable> as a string.</para></listitem> @@ -1078,8 +1116,9 @@ in config.someSetting</programlisting> </varlistentry> - <varlistentry><term><function>removeAttrs</function> - <replaceable>set</replaceable> <replaceable>list</replaceable></term> + <varlistentry xml:id='builtin-removeAttrs'> + <term><function>removeAttrs</function> + <replaceable>set</replaceable> <replaceable>list</replaceable></term> <listitem><para>Remove the attributes listed in <replaceable>list</replaceable> from @@ -1094,8 +1133,9 @@ removeAttrs { x = 1; y = 2; z = 3; } [ "a" "x" "z" ]</programlisting> </varlistentry> - <varlistentry><term><function>builtins.replaceStrings</function> - <replaceable>from</replaceable> <replaceable>to</replaceable> <replaceable>s</replaceable></term> + <varlistentry xml:id='builtin-replaceStrings'> + <term><function>builtins.replaceStrings</function> + <replaceable>from</replaceable> <replaceable>to</replaceable> <replaceable>s</replaceable></term> <listitem><para>Given string <replaceable>s</replaceable>, replace every occurrence of the strings in <replaceable>from</replaceable> @@ -1111,8 +1151,9 @@ builtins.replaceStrings ["oo" "a"] ["a" "i"] "foobar" </varlistentry> - <varlistentry><term><function>builtins.seq</function> - <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + <varlistentry xml:id='builtin-seq'> + <term><function>builtins.seq</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> <listitem><para>Evaluate <replaceable>e1</replaceable>, then evaluate and return <replaceable>e2</replaceable>. This ensures @@ -1122,8 +1163,9 @@ builtins.replaceStrings ["oo" "a"] ["a" "i"] "foobar" </varlistentry> - <varlistentry><term><function>builtins.sort</function> - <replaceable>comparator</replaceable> <replaceable>list</replaceable></term> + <varlistentry xml:id='builtin-sort'> + <term><function>builtins.sort</function> + <replaceable>comparator</replaceable> <replaceable>list</replaceable></term> <listitem><para>Return <replaceable>list</replaceable> in sorted order. It repeatedly calls the function @@ -1145,15 +1187,16 @@ builtins.sort builtins.lessThan [ 483 249 526 147 42 77 ] </varlistentry> - <varlistentry><term><function>builtins.split</function> - <replaceable>regex</replaceable> <replaceable>str</replaceable></term> + <varlistentry xml:id='builtin-split'> + <term><function>builtins.split</function> + <replaceable>regex</replaceable> <replaceable>str</replaceable></term> - <listitem><para>Returns a list composed of non matched strings interleaved - with the lists of the <link - xlink:href="http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04">extended - POSIX regular expression</link> <replaceable>regex</replaceable> matches - of <replaceable>str</replaceable>. Each item in the lists of matched - sequences is a regex group. + <listitem><para>Returns a list composed of non matched strings interleaved + with the lists of the <link + xlink:href="http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04">extended + POSIX regular expression</link> <replaceable>regex</replaceable> matches + of <replaceable>str</replaceable>. Each item in the lists of matched + sequences is a regex group. <programlisting> builtins.split "(a)b" "abc" @@ -1179,11 +1222,12 @@ builtins.split "([[:upper:]]+)" " FOO " Evaluates to <literal>[ " " [ "FOO" ] " " ]</literal>. - </para></listitem> + </para></listitem> </varlistentry> - <varlistentry><term><function>builtins.stringLength</function> - <replaceable>e</replaceable></term> + <varlistentry xml:id='builtin-stringLength'> + <term><function>builtins.stringLength</function> + <replaceable>e</replaceable></term> <listitem><para>Return the length of the string <replaceable>e</replaceable>. If <replaceable>e</replaceable> is @@ -1192,8 +1236,9 @@ Evaluates to <literal>[ " " [ "FOO" ] " " ]</literal>. </varlistentry> - <varlistentry><term><function>builtins.sub</function> - <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + <varlistentry xml:id='builtin-sub'> + <term><function>builtins.sub</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> <listitem><para>Return the difference between the numbers <replaceable>e1</replaceable> and @@ -1202,9 +1247,10 @@ Evaluates to <literal>[ " " [ "FOO" ] " " ]</literal>. </varlistentry> - <varlistentry><term><function>builtins.substring</function> - <replaceable>start</replaceable> <replaceable>len</replaceable> - <replaceable>s</replaceable></term> + <varlistentry xml:id='builtin-substring'> + <term><function>builtins.substring</function> + <replaceable>start</replaceable> <replaceable>len</replaceable> + <replaceable>s</replaceable></term> <listitem><para>Return the substring of <replaceable>s</replaceable> from character position @@ -1227,8 +1273,9 @@ builtins.substring 0 3 "nixos" </varlistentry> - <varlistentry><term><function>builtins.tail</function> - <replaceable>list</replaceable></term> + <varlistentry xml:id='builtin-tail'> + <term><function>builtins.tail</function> + <replaceable>list</replaceable></term> <listitem><para>Return the second to last elements of a list; abort evaluation if the argument isn’t a list or is an empty @@ -1237,8 +1284,9 @@ builtins.substring 0 3 "nixos" </varlistentry> - <varlistentry><term><function>throw</function> - <replaceable>s</replaceable></term> + <varlistentry xml:id='builtin-throw'> + <term><function>throw</function> + <replaceable>s</replaceable></term> <listitem><para>Throw an error message <replaceable>s</replaceable>. This usually aborts Nix expression @@ -1251,9 +1299,10 @@ builtins.substring 0 3 "nixos" </varlistentry> - <varlistentry - xml:id='builtin-toFile'><term><function>builtins.toFile</function> - <replaceable>name</replaceable> <replaceable>s</replaceable></term> + <varlistentry xml:id='builtin-toFile'> + <term><function>builtins.toFile</function> + <replaceable>name</replaceable> + <replaceable>s</replaceable></term> <listitem><para>Store the string <replaceable>s</replaceable> in a file in the Nix store and return its path. The file has suffix @@ -1329,7 +1378,8 @@ in foo</programlisting> </varlistentry> - <varlistentry><term><function>builtins.toJSON</function> <replaceable>e</replaceable></term> + <varlistentry xml:id='builtin-toJSON'> + <term><function>builtins.toJSON</function> <replaceable>e</replaceable></term> <listitem><para>Return a string containing a JSON representation of <replaceable>e</replaceable>. Strings, integers, floats, booleans, @@ -1342,20 +1392,19 @@ in foo</programlisting> </varlistentry> - <varlistentry><term><function>builtins.toPath</function> <replaceable>s</replaceable></term> + <varlistentry xml:id='builtin-toPath'> + <term><function>builtins.toPath</function> <replaceable>s</replaceable></term> - <listitem><para>Convert the string value - <replaceable>s</replaceable> into a path value. The string - <replaceable>s</replaceable> must represent an absolute path - (i.e., must start with <literal>/</literal>). The path need not - exist. The resulting path is canonicalised, e.g., - <literal>builtins.toPath "//foo/xyzzy/../bar/"</literal> returns - <literal>/foo/bar</literal>.</para></listitem> + <listitem><para> DEPRECATED. Use <literal>/. + "/path"</literal> + to convert a string into an absolute path. For relative paths, + use <literal>./. + "/path"</literal>. + </para></listitem> </varlistentry> - <varlistentry><term><function>toString</function> <replaceable>e</replaceable></term> + <varlistentry xml:id='builtin-toString'> + <term><function>toString</function> <replaceable>e</replaceable></term> <listitem><para>Convert the expression <replaceable>e</replaceable> to a string. @@ -1374,7 +1423,8 @@ in foo</programlisting> </varlistentry> - <varlistentry xml:id='builtin-toXML'><term><function>builtins.toXML</function> <replaceable>e</replaceable></term> + <varlistentry xml:id='builtin-toXML'> + <term><function>builtins.toXML</function> <replaceable>e</replaceable></term> <listitem><para>Return a string containing an XML representation of <replaceable>e</replaceable>. The main application for @@ -1430,7 +1480,7 @@ stdenv.mkDerivation (rec { builder = builtins.toFile "builder.sh" " source $stdenv/setup mkdir $out - echo $servlets | xsltproc ${stylesheet} - > $out/server-conf.xml]]> <co xml:id='ex-toxml-co-apply' /> <![CDATA[ + echo "$servlets" | xsltproc ${stylesheet} - > $out/server-conf.xml]]> <co xml:id='ex-toxml-co-apply' /> <![CDATA[ "; stylesheet = builtins.toFile "stylesheet.xsl"]]> <co xml:id='ex-toxml-co-stylesheet' /> <![CDATA[ @@ -1489,8 +1539,9 @@ stdenv.mkDerivation (rec { </varlistentry> - <varlistentry><term><function>builtins.trace</function> - <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + <varlistentry xml:id='builtin-trace'> + <term><function>builtins.trace</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> <listitem><para>Evaluate <replaceable>e1</replaceable> and print its abstract syntax representation on standard error. Then return @@ -1499,8 +1550,9 @@ stdenv.mkDerivation (rec { </varlistentry> - <varlistentry><term><function>builtins.tryEval</function> - <replaceable>e</replaceable></term> + <varlistentry xml:id='builtin-tryEval'> + <term><function>builtins.tryEval</function> + <replaceable>e</replaceable></term> <listitem><para>Try to evaluate <replaceable>e</replaceable>. Return a set containing the attributes <literal>success</literal> @@ -1513,8 +1565,9 @@ stdenv.mkDerivation (rec { </varlistentry> - <varlistentry><term><function>builtins.typeOf</function> - <replaceable>e</replaceable></term> + <varlistentry xml:id='builtin-typeOf'> + <term><function>builtins.typeOf</function> + <replaceable>e</replaceable></term> <listitem><para>Return a string representing the type of the value <replaceable>e</replaceable>, namely <literal>"int"</literal>, diff --git a/doc/manual/installation/upgrading.xml b/doc/manual/installation/upgrading.xml index a3f86ade95cd..30670d7fec9c 100644 --- a/doc/manual/installation/upgrading.xml +++ b/doc/manual/installation/upgrading.xml @@ -7,15 +7,16 @@ <title>Upgrading Nix</title> <para> - Multi-user Nix users on macOS can upgrade Nix by running - <command>sudo -i sh -c 'nix-channel --update && nix-env - -iA nixpkgs.nix'; sudo launchctl stop org.nixos.nix-daemon; sudo - launchctl start org.nixos.nix-daemon</command>. + Multi-user Nix users on macOS can upgrade Nix by running: + <command>sudo -i sh -c 'nix-channel --update && + nix-env -iA nixpkgs.nix && + launchctl remove org.nixos.nix-daemon && + launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist'</command> </para> + <para> - Single-user installations of Nix should run <command>nix-channel - --update; nix-env -iA nixpkgs.nix</command>. + Single-user installations of Nix should run this: + <command>nix-channel --update; nix-env -iA nixpkgs.nix</command> </para> - </chapter> diff --git a/doc/manual/introduction/about-nix.xml b/doc/manual/introduction/about-nix.xml index e8c0a29753a1..c21ed34ddc74 100644 --- a/doc/manual/introduction/about-nix.xml +++ b/doc/manual/introduction/about-nix.xml @@ -262,12 +262,6 @@ xlink:href="http://nixos.org/">NixOS homepage</link>.</para> xlink:href="http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html">GNU LGPLv2.1 or (at your option) any later version</link>.</para> -<para>Nix uses the <link -xlink:href="https://github.com/arangodb/linenoise-ng">linenoise-ng -library</link>, which has the following license:</para> - -<programlisting><xi:include href="../../../src/linenoise/LICENSE" parse="text" /></programlisting> - </simplesect> diff --git a/doc/manual/packages/s3-substituter.xml b/doc/manual/packages/s3-substituter.xml index bcd91cfdbccd..ea654392c6b1 100644 --- a/doc/manual/packages/s3-substituter.xml +++ b/doc/manual/packages/s3-substituter.xml @@ -12,8 +12,49 @@ from Amazon S3 and S3 compatible services. This uses the same <emphasis>binary</emphasis> cache mechanism that Nix usually uses to fetch prebuilt binaries from <uri>cache.nixos.org</uri>.</para> +<para>The following options can be specified as URL parameters to +the S3 URL:</para> + +<variablelist> + <varlistentry><term><literal>profile</literal></term> + <listitem> + <para> + The name of the AWS configuration profile to use. By default + Nix will use the <literal>default</literal> profile. + </para> + </listitem> + </varlistentry> + + <varlistentry><term><literal>region</literal></term> + <listitem> + <para> + The region of the S3 bucket. <literal>us–east-1</literal> by + default. + </para> + + <para> + If your bucket is not in <literal>us–east-1</literal>, you + should always explicitly specify the region parameter. + </para> + </listitem> + </varlistentry> + + <varlistentry><term><literal>endpoint</literal></term> + <listitem> + <para> + The URL to your S3-compatible service, for when not using + Amazon S3. Do not specify this value if you're using Amazon + S3. + </para> + <note><para>This endpoint must support HTTPS and will use + path-based addressing instead of virtual host based + addressing.</para></note> + </listitem> + </varlistentry> +</variablelist> + <para>In this example we will use the bucket named -<literal>example-bucket</literal>.</para> +<literal>example-nix-cache</literal>.</para> <section xml:id="ssec-s3-substituter-anonymous-reads"> <title>Anonymous Reads to your S3-compatible binary cache</title> @@ -24,65 +65,56 @@ fetch prebuilt binaries from <uri>cache.nixos.org</uri>.</para> cache.</para> <para>For AWS S3 the binary cache URL for example bucket will be - exactly <uri>https://example-bucket.s3.amazonaws.com</uri>. For S3 - compatible binary caches ago have to consult your software's - documentation.</para> + exactly <uri>https://example-nix-cache.s3.amazonaws.com</uri> or + <uri>s3://example-nix-cache</uri>. For S3 compatible binary caches, + consult that cache's documentation.</para> <para>Your bucket will need the following bucket policy:</para> - <programlisting> -<![CDATA[ + <programlisting><![CDATA[ { - "Id": "DirectReads", - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "AlowDirectReads", - "Action": [ - "s3:GetObject" - ], - "Effect": "Allow", - "Resource": "arn:aws:s3:::example-bucket/*", - "Principal": "*" - } - ] + "Id": "DirectReads", + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AlowDirectReads", + "Action": [ + "s3:GetObject", + "s3:GetBucketLocation" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:s3:::example-nix-cache", + "arn:aws:s3:::example-nix-cache/*" + ], + "Principal": "*" + } + ] } -]]> -</programlisting> +]]></programlisting> </section> <section xml:id="ssec-s3-substituter-authenticated-reads"> <title>Authenticated Reads to your S3 binary cache</title> <para>For AWS S3 the binary cache URL for example bucket will be - exactly <uri>s3://example-bucket</uri>.</para> + exactly <uri>s3://example-nix-cache</uri>.</para> <para>Nix will use the <link xlink:href="https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default.">default credential provider chain</link> for authenticating requests to Amazon S3.</para> - <para>Nix supports authenticated writes to S3 compatible binary - caches but only supports Authenticated reads from Amazon S3. - Additionally, the following limitations are in place for - authenticated reads:</para> - - <itemizedlist> - <listitem><para>The bucket must actually be hosted by Amazon S3 and - <emphasis>not</emphasis> an S3 compatible - service.</para></listitem> - - <listitem><para>The bucket must be within the - <literal>us-east-1</literal> region.</para></listitem> - - <listitem><para>The Amazon credentials, if stored in a credential - profile, must be stored in the <literal>default</literal> - profile.</para></listitem> - </itemizedlist> + <para>Nix supports authenticated reads from Amazon S3 and S3 + compatible binary caches.</para> <para>Your bucket will need a bucket policy allowing the desired - users to perform the <literal>s3:GetObject</literal> action on all - objects in the bucket.</para> + users to perform the <literal>s3:GetObject</literal> and + <literal>s3:GetBucketLocation</literal> action on all objects in the + bucket. The anonymous policy in <xref + linkend="ssec-s3-substituter-anonymous-reads" /> can be updated to + have a restricted <literal>Principal</literal> to support + this.</para> </section> @@ -91,69 +123,49 @@ fetch prebuilt binaries from <uri>cache.nixos.org</uri>.</para> <para>Nix support fully supports writing to Amazon S3 and S3 compatible buckets. The binary cache URL for our example bucket will - be <uri>s3://example-bucket</uri>.</para> + be <uri>s3://example-nix-cache</uri>.</para> <para>Nix will use the <link xlink:href="https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default.">default credential provider chain</link> for authenticating requests to Amazon S3.</para> - <para>The following options can be specified as URL parameters to - the S3 URL:</para> - <variablelist> - <varlistentry><term><literal>profile</literal></term> - <listitem> - <para> - The name of the AWS configuration profile to use. By default - Nix will use the <literal>default</literal> profile. - </para> - </listitem> - </varlistentry> - - <varlistentry><term><literal>region</literal></term> - <listitem> - <para> - The region of the S3 bucket. <literal>us–east-1</literal> by - default. - </para> - </listitem> - </varlistentry> - - <varlistentry><term><literal>endpoint</literal></term> - <listitem> - <para> - The URL to your S3-compatible service, for when not using - Amazon S3. Do not specify this value if you're using Amazon - S3. - </para> - <note><para>This endpoint must support HTTPS and will use - path-based addressing instead of virtual host based - addressing.</para></note> - </listitem> - </varlistentry> - </variablelist> - - <example><title>Uploading with non-default credential profile for Amazon S3</title> - <para><command>nix copy --to ssh://machine nixpkgs.hello s3://example-bucket?profile=cache-upload</command></para> + <para>Your account will need the following IAM policy to + upload to the cache:</para> + + <programlisting><![CDATA[ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "UploadToCache", + "Effect": "Allow", + "Action": [ + "s3:AbortMultipartUpload", + "s3:GetBucketLocation", + "s3:GetObject", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:ListMultipartUploadParts", + "s3:ListObjects", + "s3:PutObject" + ], + "Resource": [ + "arn:aws:s3:::example-nix-cache", + "arn:aws:s3:::example-nix-cache/*" + ] + } + ] +} +]]></programlisting> + + + <example><title>Uploading with a specific credential profile for Amazon S3</title> + <para><command>nix copy --to 's3://example-nix-cache?profile=cache-upload&region=eu-west-2' nixpkgs.hello</command></para> </example> <example><title>Uploading to an S3-Compatible Binary Cache</title> - <para><command>nix copy --to ssh://machine nixpkgs.hello s3://example-bucket?profile=cache-upload&endpoint=minio.example.com</command></para> + <para><command>nix copy --to 's3://example-nix-cache?profile=cache-upload&endpoint=minio.example.com' nixpkgs.hello</command></para> </example> - - <para>The user writing to the bucket will need to perform the - following actions against the bucket:</para> - - <itemizedlist> - <listitem><para><literal>s3:ListBucket</literal></para></listitem> - <listitem><para><literal>s3:GetBucketLocation</literal></para></listitem> - <listitem><para><literal>s3:ListObjects</literal></para></listitem> - <listitem><para><literal>s3:GetObject</literal></para></listitem> - <listitem><para><literal>s3:PutObject</literal></para></listitem> - <listitem><para><literal>s3:ListBucketMultipartUploads</literal></para></listitem> - <listitem><para><literal>s3:CreateMultipartUpload</literal></para></listitem> - <listitem><para><literal>s3:ListMultipartUploadParts</literal></para></listitem> - <listitem><para><literal>s3:AbortMultipartUpload</literal></para></listitem> - </itemizedlist> </section> </section> diff --git a/doc/manual/release-notes/release-notes.xml b/doc/manual/release-notes/release-notes.xml index ff4085cb792d..e8ff586fa43f 100644 --- a/doc/manual/release-notes/release-notes.xml +++ b/doc/manual/release-notes/release-notes.xml @@ -12,6 +12,7 @@ </partintro> --> +<xi:include href="rl-2.2.xml" /> <xi:include href="rl-2.1.xml" /> <xi:include href="rl-2.0.xml" /> <xi:include href="rl-1.11.10.xml" /> diff --git a/doc/manual/release-notes/rl-2.2.xml b/doc/manual/release-notes/rl-2.2.xml new file mode 100644 index 000000000000..abe9b49adadd --- /dev/null +++ b/doc/manual/release-notes/rl-2.2.xml @@ -0,0 +1,29 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-relnotes-2.2"> + +<title>Release 2.2 (201?-??-??)</title> + +<para>This release has the following changes:</para> + +<itemizedlist> + + <listitem> + <para>The derivation attribute + <varname>requiredSystemFeatures</varname> is now enforced for + local builds, and not just to route builds to remote builders. + The supported features of a machine can be specified through the + configuration setting <varname>system-features</varname>. + </para> + </listitem> + + <listitem> + <para>Sandbox builds are now enabled by default on Linux.</para> + </listitem> + +</itemizedlist> + +</section> + diff --git a/local.mk b/local.mk index 5d7e0fb2e428..4b380176f2ec 100644 --- a/local.mk +++ b/local.mk @@ -6,7 +6,7 @@ dist-files += configure config.h.in nix.spec perl/configure clean-files += Makefile.config -GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr +GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr -I src/nix $(foreach i, config.h $(call rwildcard, src/lib*, *.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) diff --git a/release-common.nix b/release-common.nix index ace2a4f9b91f..4c5565985267 100644 --- a/release-common.nix +++ b/release-common.nix @@ -49,7 +49,7 @@ rec { buildDeps = [ curl - bzip2 xz brotli + bzip2 xz brotli editline openssl pkgconfig sqlite boehmgc boost @@ -64,10 +64,12 @@ rec { apis = ["s3" "transfer"]; customMemoryManagement = false; }).overrideDerivation (args: { + /* patches = args.patches or [] ++ [ (fetchpatch { url = https://github.com/edolstra/aws-sdk-cpp/commit/3e07e1f1aae41b4c8b340735ff9e8c735f0c063f.patch; sha256 = "1pij0v449p166f9l29x7ppzk8j7g9k9mp15ilh5qxp29c7fnvxy2"; }) ]; + */ })); perlDeps = diff --git a/release.nix b/release.nix index e359ebcb2ce3..415c87da7cb3 100644 --- a/release.nix +++ b/release.nix @@ -1,5 +1,5 @@ { nix ? builtins.fetchGit ./. -, nixpkgs ? builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-18.03"; } +, nixpkgs ? builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-18.09"; } , officialRelease ? false , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ] }: diff --git a/scripts/install.in b/scripts/install.in index 26ab85ba0992..7bff7b216d9e 100644 --- a/scripts/install.in +++ b/scripts/install.in @@ -11,14 +11,14 @@ oops() { } tmpDir="$(mktemp -d -t nix-binary-tarball-unpack.XXXXXXXXXX || \ - oops "Can\'t create temporary directory for downloading the Nix binary tarball")" + oops "Can't create temporary directory for downloading the Nix binary tarball")" cleanup() { rm -rf "$tmpDir" } trap cleanup EXIT INT QUIT TERM require_util() { - type "$1" > /dev/null 2>&1 || which "$1" > /dev/null 2>&1 || + type "$1" > /dev/null 2>&1 || command -v "$1" > /dev/null 2>&1 || oops "you do not have '$1' installed, which I need to $2" } diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index 567a543d95ed..6940969cca7b 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -71,4 +71,4 @@ else fi export NIX_PATH="nixpkgs=@localstatedir@/nix/profiles/per-user/root/channels/nixpkgs:@localstatedir@/nix/profiles/per-user/root/channels" -export PATH="$HOME/.nix-profile/bin:$HOME/.nix-profile/lib/kde4/libexec:@localstatedir@/nix/profiles/default/bin:@localstatedir@/nix/profiles/default:@localstatedir@/nix/profiles/default/lib/kde4/libexec:$PATH" +export PATH="$HOME/.nix-profile/bin:@localstatedir@/nix/profiles/default/bin:$PATH" diff --git a/shell.nix b/shell.nix index c04bcd151309..817684b7646e 100644 --- a/shell.nix +++ b/shell.nix @@ -1,6 +1,6 @@ { useClang ? false }: -with import (builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-18.03"; }) {}; +with import (builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-18.09"; }) {}; with import ./release-common.nix { inherit pkgs; }; diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 38dbe3e58b26..abf3669b5b35 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -17,6 +17,7 @@ #include "store-api.hh" #include "derivations.hh" #include "local-store.hh" +#include "legacy.hh" using namespace nix; using std::cin; @@ -37,11 +38,9 @@ static AutoCloseFD openSlotLock(const Machine & m, unsigned long long slot) return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true); } -int main (int argc, char * * argv) +static int _main(int argc, char * * argv) { - return handleExceptions(argv[0], [&]() { - initNix(); - + { logger = makeJSONLogger(*logger); /* Ensure we don't get any SSH passphrase or host key popups. */ @@ -80,7 +79,7 @@ int main (int argc, char * * argv) if (machines.empty()) { std::cerr << "# decline-permanently\n"; - return; + return 0; } string drvPath; @@ -90,8 +89,8 @@ int main (int argc, char * * argv) try { auto s = readString(source); - if (s != "try") return; - } catch (EndOfFile &) { return; } + if (s != "try") return 0; + } catch (EndOfFile &) { return 0; } auto amWilling = readInt(source); auto neededSystem = readString(source); @@ -253,6 +252,8 @@ connected: copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, NoSubstitute); } - return; - }); + return 0; + } } + +static RegisterLegacyCommand s1("build-remote", _main); diff --git a/src/build-remote/local.mk b/src/build-remote/local.mk deleted file mode 100644 index 50b0409d1886..000000000000 --- a/src/build-remote/local.mk +++ /dev/null @@ -1,9 +0,0 @@ -programs += build-remote - -build-remote_DIR := $(d) - -build-remote_INSTALL_DIR := $(libexecdir)/nix - -build-remote_LIBS = libmain libformat libstore libutil - -build-remote_SOURCES := $(d)/build-remote.cc diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh index 4d9128e3f448..daaa635fe1b1 100644 --- a/src/libexpr/get-drvs.hh +++ b/src/libexpr/get-drvs.hh @@ -44,7 +44,7 @@ public: string queryDrvPath() const; string queryOutPath() const; string queryOutputName() const; - /** Return the list of outputs. The "outputs to install" are determined by `mesa.outputsToInstall`. */ + /** Return the list of outputs. The "outputs to install" are determined by `meta.outputsToInstall`. */ Outputs queryOutputs(bool onlyOutputsToInstall = false); StringSet queryMetaNames(); diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index a052447d3dce..c34e5c383923 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -6,9 +6,9 @@ %option nounput noyy_top_state +%s DEFAULT %x STRING %x IND_STRING -%x INSIDE_DOLLAR_CURLY %{ @@ -99,8 +99,6 @@ URI [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~ %% -<INITIAL,INSIDE_DOLLAR_CURLY>{ - if { return IF; } then { return THEN; } @@ -140,17 +138,19 @@ or { return OR_KW; } return FLOAT; } -\$\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; } -} +\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; } -\} { return '}'; } -<INSIDE_DOLLAR_CURLY>\} { POP_STATE(); return '}'; } -\{ { return '{'; } -<INSIDE_DOLLAR_CURLY>\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return '{'; } +\} { /* State INITIAL only exists at the bottom of the stack and is + used as a marker. DEFAULT replaces it everywhere else. + Popping when in INITIAL state causes an empty stack exception, + so don't */ + if (YYSTATE != INITIAL) + POP_STATE(); + return '}'; + } +\{ { PUSH_STATE(DEFAULT); return '{'; } -<INITIAL,INSIDE_DOLLAR_CURLY>\" { - PUSH_STATE(STRING); return '"'; - } +\" { PUSH_STATE(STRING); return '"'; } <STRING>([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})*\$/\" | <STRING>([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})+ { /* It is impossible to match strings ending with '$' with one @@ -159,7 +159,7 @@ or { return OR_KW; } yylval->e = unescapeStr(data->symbols, yytext, yyleng); return STR; } -<STRING>\$\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; } +<STRING>\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; } <STRING>\" { POP_STATE(); return '"'; } <STRING>\$|\\|\$\\ { /* This can only occur when we reach EOF, otherwise the above @@ -169,7 +169,7 @@ or { return OR_KW; } return STR; } -<INITIAL,INSIDE_DOLLAR_CURLY>\'\'(\ *\n)? { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; } +\'\'(\ *\n)? { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; } <IND_STRING>([^\$\']|\$[^\{\']|\'[^\'\$])+ { yylval->e = new ExprIndStr(yytext); return IND_STR; @@ -187,14 +187,13 @@ or { return OR_KW; } yylval->e = unescapeStr(data->symbols, yytext + 2, yyleng - 2); return IND_STR; } -<IND_STRING>\$\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; } +<IND_STRING>\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; } <IND_STRING>\'\' { POP_STATE(); return IND_STRING_CLOSE; } <IND_STRING>\' { yylval->e = new ExprIndStr("'"); return IND_STR; } -<INITIAL,INSIDE_DOLLAR_CURLY>{ {PATH} { if (yytext[yyleng-1] == '/') throw ParseError("path '%s' has a trailing slash", yytext); @@ -219,7 +218,5 @@ or { return OR_KW; } return (unsigned char) yytext[0]; } -} - %% diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 1402bd097c35..676ad5856b13 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -11,6 +11,7 @@ #include "compression.hh" #include "json.hh" #include "nar-info.hh" +#include "parsed-derivations.hh" #include <algorithm> #include <iostream> @@ -20,6 +21,7 @@ #include <future> #include <chrono> #include <regex> +#include <queue> #include <limits.h> #include <sys/time.h> @@ -740,6 +742,8 @@ private: /* The derivation stored at drvPath. */ std::unique_ptr<BasicDerivation> drv; + std::unique_ptr<ParsedDerivation> parsedDrv; + /* The remainder is state held during the build. */ /* Locks on the output paths. */ @@ -854,7 +858,7 @@ private: building multiple times. Since this contains the hash, it allows us to compare whether two rounds produced the same result. */ - ValidPathInfos prevInfos; + std::map<Path, ValidPathInfo> prevInfos; const uid_t sandboxUid = 1000; const gid_t sandboxGid = 100; @@ -935,6 +939,11 @@ private: as valid. */ void registerOutputs(); + /* Check that an output meets the requirements specified by the + 'outputChecks' attribute (or the legacy + '{allowed,disallowed}{References,Requisites}' attributes). */ + void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs); + /* Open a log file and a pipe to it. */ Path openLogFile(); @@ -1139,6 +1148,8 @@ void DerivationGoal::haveDerivation() return; } + parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv); + /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build them. */ @@ -1395,7 +1406,7 @@ void DerivationGoal::tryToBuild() /* Don't do a remote build if the derivation has the attribute `preferLocalBuild' set. Also, check and repair modes are only supported for local builds. */ - bool buildLocally = buildMode != bmNormal || drv->willBuildLocally(); + bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(); auto started = [&]() { auto msg = fmt( @@ -1641,19 +1652,13 @@ HookReply DerivationGoal::tryBuildHook() try { - /* Tell the hook about system features (beyond the system type) - required from the build machine. (The hook could parse the - drv file itself, but this is easier.) */ - Strings features = tokenizeString<Strings>(get(drv->env, "requiredSystemFeatures")); - for (auto & i : features) checkStoreName(i); /* !!! abuse */ - /* Send the request to the hook. */ worker.hook->sink << "try" << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0) << drv->platform << drvPath - << features; + << parsedDrv->getRequiredSystemFeatures(); worker.hook->sink.flush(); /* Read the first line of input, which should be a word indicating @@ -1793,23 +1798,26 @@ static void preloadNSS() { void DerivationGoal::startBuilder() { /* Right platform? */ - if (!drv->canBuildLocally()) { - throw Error( - format("a '%1%' is required to build '%3%', but I am a '%2%'") - % drv->platform % settings.thisSystem % drvPath); - } + if (!parsedDrv->canBuildLocally()) + throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}", + drv->platform, + concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), + drvPath, + settings.thisSystem, + concatStringsSep(", ", settings.systemFeatures)); if (drv->isBuiltin()) preloadNSS(); #if __APPLE__ - additionalSandboxProfile = get(drv->env, "__sandboxProfile"); + additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or(""); #endif /* Are we doing a chroot build? */ { + auto noChroot = parsedDrv->getBoolAttr("__noChroot"); if (settings.sandboxMode == smEnabled) { - if (get(drv->env, "__noChroot") == "1") + if (noChroot) throw Error(format("derivation '%1%' has '__noChroot' set, " "but that's not allowed when 'sandbox' is 'true'") % drvPath); #if __APPLE__ @@ -1822,7 +1830,7 @@ void DerivationGoal::startBuilder() else if (settings.sandboxMode == smDisabled) useChroot = false; else if (settings.sandboxMode == smRelaxed) - useChroot = !fixedOutput && get(drv->env, "__noChroot") != "1"; + useChroot = !fixedOutput && !noChroot; } if (worker.store.storeDir != worker.store.realStoreDir) { @@ -1873,7 +1881,7 @@ void DerivationGoal::startBuilder() writeStructuredAttrs(); /* Handle exportReferencesGraph(), if set. */ - if (!drv->env.count("__json")) { + if (!parsedDrv->getStructuredAttrs()) { /* The `exportReferencesGraph' feature allows the references graph to be passed to a builder. This attribute should be a list of pairs [name1 path1 name2 path2 ...]. The references graph of @@ -1938,7 +1946,7 @@ void DerivationGoal::startBuilder() PathSet allowedPaths = settings.allowedImpureHostPrefixes; /* This works like the above, except on a per-derivation level */ - Strings impurePaths = tokenizeString<Strings>(get(drv->env, "__impureHostDeps")); + auto impurePaths = parsedDrv->getStringsAttr("__impureHostDeps").value_or(Strings()); for (auto & i : impurePaths) { bool found = false; @@ -2306,7 +2314,7 @@ void DerivationGoal::initEnv() passAsFile is ignored in structure mode because it's not needed (attributes are not passed through the environment, so there is no size constraint). */ - if (!drv->env.count("__json")) { + if (!parsedDrv->getStructuredAttrs()) { StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile")); int fileNr = 0; @@ -2353,8 +2361,8 @@ void DerivationGoal::initEnv() fixed-output derivations is by definition pure (since we already know the cryptographic hash of the output). */ if (fixedOutput) { - Strings varNames = tokenizeString<Strings>(get(drv->env, "impureEnvVars")); - for (auto & i : varNames) env[i] = getEnv(i); + for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) + env[i] = getEnv(i); } /* Currently structured log messages piggyback on stderr, but we @@ -2369,111 +2377,103 @@ static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); void DerivationGoal::writeStructuredAttrs() { - auto jsonAttr = drv->env.find("__json"); - if (jsonAttr == drv->env.end()) return; + auto & structuredAttrs = parsedDrv->getStructuredAttrs(); + if (!structuredAttrs) return; - try { + auto json = *structuredAttrs; - auto jsonStr = rewriteStrings(jsonAttr->second, inputRewrites); - - auto json = nlohmann::json::parse(jsonStr); - - /* Add an "outputs" object containing the output paths. */ - nlohmann::json outputs; - for (auto & i : drv->outputs) - outputs[i.first] = rewriteStrings(i.second.path, inputRewrites); - json["outputs"] = outputs; - - /* Handle exportReferencesGraph. */ - auto e = json.find("exportReferencesGraph"); - if (e != json.end() && e->is_object()) { - for (auto i = e->begin(); i != e->end(); ++i) { - std::ostringstream str; - { - JSONPlaceholder jsonRoot(str, true); - PathSet storePaths; - for (auto & p : *i) - storePaths.insert(p.get<std::string>()); - worker.store.pathInfoToJSON(jsonRoot, - exportReferences(storePaths), false, true); - } - json[i.key()] = nlohmann::json::parse(str.str()); // urgh + /* Add an "outputs" object containing the output paths. */ + nlohmann::json outputs; + for (auto & i : drv->outputs) + outputs[i.first] = rewriteStrings(i.second.path, inputRewrites); + json["outputs"] = outputs; + + /* Handle exportReferencesGraph. */ + auto e = json.find("exportReferencesGraph"); + if (e != json.end() && e->is_object()) { + for (auto i = e->begin(); i != e->end(); ++i) { + std::ostringstream str; + { + JSONPlaceholder jsonRoot(str, true); + PathSet storePaths; + for (auto & p : *i) + storePaths.insert(p.get<std::string>()); + worker.store.pathInfoToJSON(jsonRoot, + exportReferences(storePaths), false, true); } + json[i.key()] = nlohmann::json::parse(str.str()); // urgh } + } - writeFile(tmpDir + "/.attrs.json", json.dump()); - - /* As a convenience to bash scripts, write a shell file that - maps all attributes that are representable in bash - - namely, strings, integers, nulls, Booleans, and arrays and - objects consisting entirely of those values. (So nested - arrays or objects are not supported.) */ + writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); - auto handleSimpleType = [](const nlohmann::json & value) -> std::experimental::optional<std::string> { - if (value.is_string()) - return shellEscape(value); + /* As a convenience to bash scripts, write a shell file that + maps all attributes that are representable in bash - + namely, strings, integers, nulls, Booleans, and arrays and + objects consisting entirely of those values. (So nested + arrays or objects are not supported.) */ - if (value.is_number()) { - auto f = value.get<float>(); - if (std::ceil(f) == f) - return std::to_string(value.get<int>()); - } + auto handleSimpleType = [](const nlohmann::json & value) -> std::experimental::optional<std::string> { + if (value.is_string()) + return shellEscape(value); - if (value.is_null()) - return std::string("''"); + if (value.is_number()) { + auto f = value.get<float>(); + if (std::ceil(f) == f) + return std::to_string(value.get<int>()); + } - if (value.is_boolean()) - return value.get<bool>() ? std::string("1") : std::string(""); + if (value.is_null()) + return std::string("''"); - return {}; - }; + if (value.is_boolean()) + return value.get<bool>() ? std::string("1") : std::string(""); - std::string jsonSh; + return {}; + }; - for (auto i = json.begin(); i != json.end(); ++i) { + std::string jsonSh; - if (!std::regex_match(i.key(), shVarName)) continue; + for (auto i = json.begin(); i != json.end(); ++i) { - auto & value = i.value(); + if (!std::regex_match(i.key(), shVarName)) continue; - auto s = handleSimpleType(value); - if (s) - jsonSh += fmt("declare %s=%s\n", i.key(), *s); + auto & value = i.value(); - else if (value.is_array()) { - std::string s2; - bool good = true; + auto s = handleSimpleType(value); + if (s) + jsonSh += fmt("declare %s=%s\n", i.key(), *s); - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); - if (!s3) { good = false; break; } - s2 += *s3; s2 += ' '; - } + else if (value.is_array()) { + std::string s2; + bool good = true; - if (good) - jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2); + for (auto i = value.begin(); i != value.end(); ++i) { + auto s3 = handleSimpleType(i.value()); + if (!s3) { good = false; break; } + s2 += *s3; s2 += ' '; } - else if (value.is_object()) { - std::string s2; - bool good = true; + if (good) + jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2); + } - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); - if (!s3) { good = false; break; } - s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3); - } + else if (value.is_object()) { + std::string s2; + bool good = true; - if (good) - jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2); + for (auto i = value.begin(); i != value.end(); ++i) { + auto s3 = handleSimpleType(i.value()); + if (!s3) { good = false; break; } + s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3); } - } - - writeFile(tmpDir + "/.attrs.sh", jsonSh); - } catch (std::exception & e) { - throw Error("cannot process __json attribute of '%s': %s", drvPath, e.what()); + if (good) + jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2); + } } + + writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); } @@ -2628,7 +2628,7 @@ void DerivationGoal::runChild() createDirs(chrootRootDir + "/dev/shm"); createDirs(chrootRootDir + "/dev/pts"); ss.push_back("/dev/full"); - if (pathExists("/dev/kvm")) + if (settings.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) ss.push_back("/dev/kvm"); ss.push_back("/dev/null"); ss.push_back("/dev/random"); @@ -2917,7 +2917,7 @@ void DerivationGoal::runChild() writeFile(sandboxFile, sandboxProfile); - bool allowLocalNetworking = get(drv->env, "__darwinAllowLocalNetworking") == "1"; + bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking"); /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */ @@ -2989,10 +2989,9 @@ void DerivationGoal::runChild() /* Parse a list of reference specifiers. Each element must either be a store path, or the symbolic name of the output of the derivation (such as `out'). */ -PathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv, string attr) +PathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv, const Strings & paths) { PathSet result; - Paths paths = tokenizeString<Paths>(attr); for (auto & i : paths) { if (store.isStorePath(i)) result.insert(i); @@ -3017,7 +3016,7 @@ void DerivationGoal::registerOutputs() if (allValid) return; } - ValidPathInfos infos; + std::map<std::string, ValidPathInfo> infos; /* Set of inodes seen during calls to canonicalisePathMetaData() for this build's outputs. This needs to be shared between @@ -3121,7 +3120,7 @@ void DerivationGoal::registerOutputs() the derivation to its content-addressed location. */ Hash h2 = recursive ? hashPath(h.type, actualPath).first : hashFile(h.type, actualPath); - Path dest = worker.store.makeFixedOutputPath(recursive, h2, drv->env["name"]); + Path dest = worker.store.makeFixedOutputPath(recursive, h2, storePathToName(path)); if (h != h2) { @@ -3202,48 +3201,6 @@ void DerivationGoal::registerOutputs() debug(format("referenced input: '%1%'") % i); } - /* Enforce `allowedReferences' and friends. */ - auto checkRefs = [&](const string & attrName, bool allowed, bool recursive) { - if (drv->env.find(attrName) == drv->env.end()) return; - - PathSet spec = parseReferenceSpecifiers(worker.store, *drv, get(drv->env, attrName)); - - PathSet used; - if (recursive) { - /* Our requisites are the union of the closures of our references. */ - for (auto & i : references) - /* Don't call computeFSClosure on ourselves. */ - if (path != i) - worker.store.computeFSClosure(i, used); - } else - used = references; - - PathSet badPaths; - - for (auto & i : used) - if (allowed) { - if (spec.find(i) == spec.end()) - badPaths.insert(i); - } else { - if (spec.find(i) != spec.end()) - badPaths.insert(i); - } - - if (!badPaths.empty()) { - string badPathsStr; - for (auto & i : badPaths) { - badPathsStr += "\n\t"; - badPathsStr += i; - } - throw BuildError(format("output '%1%' is not allowed to refer to the following paths:%2%") % actualPath % badPathsStr); - } - }; - - checkRefs("allowedReferences", true, false); - checkRefs("allowedRequisites", true, true); - checkRefs("disallowedReferences", false, false); - checkRefs("disallowedRequisites", false, true); - if (curRound == nrRounds) { worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences() worker.markContentsGood(path); @@ -3259,11 +3216,14 @@ void DerivationGoal::registerOutputs() if (!info.references.empty()) info.ca.clear(); - infos.push_back(info); + infos[i.first] = info; } if (buildMode == bmCheck) return; + /* Apply output checks. */ + checkOutputs(infos); + /* Compare the result with the previous round, and report which path is different, if any.*/ if (curRound > 1 && prevInfos != infos) { @@ -3271,16 +3231,16 @@ void DerivationGoal::registerOutputs() for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j) if (!(*i == *j)) { result.isNonDeterministic = true; - Path prev = i->path + checkSuffix; + Path prev = i->second.path + checkSuffix; bool prevExists = keepPreviousRound && pathExists(prev); auto msg = prevExists - ? fmt("output '%1%' of '%2%' differs from '%3%' from previous round", i->path, drvPath, prev) - : fmt("output '%1%' of '%2%' differs from previous round", i->path, drvPath); + ? fmt("output '%1%' of '%2%' differs from '%3%' from previous round", i->second.path, drvPath, prev) + : fmt("output '%1%' of '%2%' differs from previous round", i->second.path, drvPath); auto diffHook = settings.diffHook; if (prevExists && diffHook != "" && runDiffHook) { try { - auto diff = runProgram(diffHook, true, {prev, i->path}); + auto diff = runProgram(diffHook, true, {prev, i->second.path}); if (diff != "") printError(chomp(diff)); } catch (Error & error) { @@ -3325,7 +3285,11 @@ void DerivationGoal::registerOutputs() /* Register each output path as valid, and register the sets of paths referenced by each of them. If there are cycles in the outputs, this will fail. */ - worker.store.registerValidPaths(infos); + { + ValidPathInfos infos2; + for (auto & i : infos) infos2.push_back(i.second); + worker.store.registerValidPaths(infos2); + } /* In case of a fixed-output derivation hash mismatch, throw an exception now that we have registered the output as valid. */ @@ -3334,6 +3298,158 @@ void DerivationGoal::registerOutputs() } +void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs) +{ + std::map<Path, const ValidPathInfo &> outputsByPath; + for (auto & output : outputs) + outputsByPath.emplace(output.second.path, output.second); + + for (auto & output : outputs) { + auto & outputName = output.first; + auto & info = output.second; + + struct Checks + { + bool ignoreSelfRefs = false; + std::experimental::optional<uint64_t> maxSize, maxClosureSize; + std::experimental::optional<Strings> allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites; + }; + + /* Compute the closure and closure size of some output. This + is slightly tricky because some of its references (namely + other outputs) may not be valid yet. */ + auto getClosure = [&](const Path & path) + { + uint64_t closureSize = 0; + PathSet pathsDone; + std::queue<Path> pathsLeft; + pathsLeft.push(path); + + while (!pathsLeft.empty()) { + auto path = pathsLeft.front(); + pathsLeft.pop(); + if (!pathsDone.insert(path).second) continue; + + auto i = outputsByPath.find(path); + if (i != outputsByPath.end()) { + closureSize += i->second.narSize; + for (auto & ref : i->second.references) + pathsLeft.push(ref); + } else { + auto info = worker.store.queryPathInfo(path); + closureSize += info->narSize; + for (auto & ref : info->references) + pathsLeft.push(ref); + } + } + + return std::make_pair(pathsDone, closureSize); + }; + + auto applyChecks = [&](const Checks & checks) + { + if (checks.maxSize && info.narSize > *checks.maxSize) + throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes", + info.path, info.narSize, *checks.maxSize); + + if (checks.maxClosureSize) { + uint64_t closureSize = getClosure(info.path).second; + if (closureSize > *checks.maxClosureSize) + throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes", + info.path, closureSize, *checks.maxClosureSize); + } + + auto checkRefs = [&](const std::experimental::optional<Strings> & value, bool allowed, bool recursive) + { + if (!value) return; + + PathSet spec = parseReferenceSpecifiers(worker.store, *drv, *value); + + PathSet used = recursive ? getClosure(info.path).first : info.references; + + if (recursive && checks.ignoreSelfRefs) + used.erase(info.path); + + PathSet badPaths; + + for (auto & i : used) + if (allowed) { + if (!spec.count(i)) + badPaths.insert(i); + } else { + if (spec.count(i)) + badPaths.insert(i); + } + + if (!badPaths.empty()) { + string badPathsStr; + for (auto & i : badPaths) { + badPathsStr += "\n "; + badPathsStr += i; + } + throw BuildError("output '%s' is not allowed to refer to the following paths:%s", info.path, badPathsStr); + } + }; + + checkRefs(checks.allowedReferences, true, false); + checkRefs(checks.allowedRequisites, true, true); + checkRefs(checks.disallowedReferences, false, false); + checkRefs(checks.disallowedRequisites, false, true); + }; + + if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { + auto outputChecks = structuredAttrs->find("outputChecks"); + if (outputChecks != structuredAttrs->end()) { + auto output = outputChecks->find(outputName); + + if (output != outputChecks->end()) { + Checks checks; + + auto maxSize = output->find("maxSize"); + if (maxSize != output->end()) + checks.maxSize = maxSize->get<uint64_t>(); + + auto maxClosureSize = output->find("maxClosureSize"); + if (maxClosureSize != output->end()) + checks.maxClosureSize = maxClosureSize->get<uint64_t>(); + + auto get = [&](const std::string & name) -> std::experimental::optional<Strings> { + auto i = output->find(name); + if (i != output->end()) { + Strings res; + for (auto j = i->begin(); j != i->end(); ++j) { + if (!j->is_string()) + throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath); + res.push_back(j->get<std::string>()); + } + checks.disallowedRequisites = res; + return res; + } + return {}; + }; + + checks.allowedReferences = get("allowedReferences"); + checks.allowedRequisites = get("allowedRequisites"); + checks.disallowedReferences = get("disallowedReferences"); + checks.disallowedRequisites = get("disallowedRequisites"); + + applyChecks(checks); + } + } + } else { + // legacy non-structured-attributes case + Checks checks; + checks.ignoreSelfRefs = true; + checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences"); + checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites"); + checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences"); + checks.disallowedRequisites = parsedDrv->getStringsAttr("disallowedRequisites"); + applyChecks(checks); + } + } +} + + Path DerivationGoal::openLogFile() { logSize = 0; diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 1e187ec5e954..3961126fff9c 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -36,12 +36,6 @@ Path BasicDerivation::findOutput(const string & id) const } -bool BasicDerivation::willBuildLocally() const -{ - return get(env, "preferLocalBuild") == "1" && canBuildLocally(); -} - - bool BasicDerivation::substitutesAllowed() const { return get(env, "allowSubstitutes", "1") == "1"; @@ -54,14 +48,6 @@ bool BasicDerivation::isBuiltin() const } -bool BasicDerivation::canBuildLocally() const -{ - return platform == settings.thisSystem - || settings.extraPlatforms.get().count(platform) > 0 - || isBuiltin(); -} - - Path writeDerivation(ref<Store> store, const Derivation & drv, const string & name, RepairFlag repair) { diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 7b97730d3bf2..9753e796db5f 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -56,14 +56,10 @@ struct BasicDerivation the given derivation. */ Path findOutput(const string & id) const; - bool willBuildLocally() const; - bool substitutesAllowed() const; bool isBuiltin() const; - bool canBuildLocally() const; - /* Return true iff this is a fixed-output derivation. */ bool isFixedOutput() const; diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 13913d031daa..f44f1836b31e 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -710,11 +710,12 @@ void Downloader::download(DownloadRequest && request, Sink & sink) /* If the buffer is full, then go to sleep until the calling thread wakes us up (i.e. when it has removed data from the - buffer). Note: this does stall the download thread. */ - while (state->data.size() > 1024 * 1024) { - if (state->quit) return; + buffer). We don't wait forever to prevent stalling the + download thread. (Hopefully sleeping will throttle the + sender.) */ + if (state->data.size() > 1024 * 1024) { debug("download buffer is full; going to sleep"); - state.wait(state->request); + state.wait_for(state->request, std::chrono::seconds(10)); } /* Append data to the buffer and wake up the calling @@ -736,30 +737,36 @@ void Downloader::download(DownloadRequest && request, Sink & sink) state->request.notify_one(); }}); - auto state(_state->lock()); - while (true) { checkInterrupt(); - /* If no data is available, then wait for the download thread - to wake us up. */ - if (state->data.empty()) { + std::string chunk; + + /* Grab data if available, otherwise wait for the download + thread to wake us up. */ + { + auto state(_state->lock()); + + while (state->data.empty()) { - if (state->quit) { - if (state->exc) std::rethrow_exception(state->exc); - break; + if (state->quit) { + if (state->exc) std::rethrow_exception(state->exc); + return; + } + + state.wait(state->avail); } - state.wait(state->avail); - } + chunk = std::move(state->data); - /* If data is available, then flush it to the sink and wake up - the download thread if it's blocked on a full buffer. */ - if (!state->data.empty()) { - sink((unsigned char *) state->data.data(), state->data.size()); - state->data.clear(); state->request.notify_one(); } + + /* Flush the data to the sink and wake up the download thread + if it's blocked on a full buffer. We don't hold the state + lock while doing this to prevent blocking the download + thread if sink() takes a long time. */ + sink((unsigned char *) chunk.data(), chunk.size()); } } diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index d95db56726cb..1c2c08715a14 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -78,7 +78,11 @@ void loadConfFile() ~/.nix/nix.conf or the command line. */ globalConfig.resetOverriden(); - globalConfig.applyConfigFile(getConfigDir() + "/nix/nix.conf"); + auto dirs = getConfigDirs(); + // Iterate over them in reverse so that the ones appearing first in the path take priority + for (auto dir = dirs.rbegin(); dir != dirs.rend(); dir++) { + globalConfig.applyConfigFile(*dir + "/nix/nix.conf"); + } } unsigned int Settings::getDefaultCores() @@ -86,6 +90,21 @@ unsigned int Settings::getDefaultCores() return std::max(1U, std::thread::hardware_concurrency()); } +StringSet Settings::getDefaultSystemFeatures() +{ + /* For backwards compatibility, accept some "features" that are + used in Nixpkgs to route builds to certain machines but don't + actually require anything special on the machines. */ + StringSet features{"nixos-test", "benchmark", "big-parallel"}; + + #if __linux__ + if (access("/dev/kvm", R_OK | W_OK) == 0) + features.insert("kvm"); + #endif + + return features; +} + const string nixVersion = PACKAGE_VERSION; template<> void BaseSetting<SandboxMode>::set(const std::string & str) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index f589078dbb98..53efc6a90fb6 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -32,6 +32,8 @@ class Settings : public Config { unsigned int getDefaultCores(); + StringSet getDefaultSystemFeatures(); + public: Settings(); @@ -80,9 +82,9 @@ public: /* Whether to show build log output in real time. */ bool verboseBuild = true; - /* If verboseBuild is false, the number of lines of the tail of - the log to show if a build fails. */ - size_t logLines = 10; + Setting<size_t> logLines{this, 10, "log-lines", + "If verbose-build is false, the number of lines of the tail of " + "the log to show if a build fails."}; MaxBuildJobsSetting maxBuildJobs{this, 1, "max-jobs", "Maximum number of parallel build jobs. \"auto\" means use number of cores.", @@ -193,7 +195,13 @@ public: Setting<bool> showTrace{this, false, "show-trace", "Whether to show a stack trace on evaluation errors."}; - Setting<SandboxMode> sandboxMode{this, smDisabled, "sandbox", + Setting<SandboxMode> sandboxMode{this, + #if __linux__ + smEnabled + #else + smDisabled + #endif + , "sandbox", "Whether to enable sandboxed builds. Can be \"true\", \"false\" or \"relaxed\".", {"build-use-chroot", "build-use-sandbox"}}; @@ -261,6 +269,10 @@ public: "These may be supported natively (e.g. armv7 on some aarch64 CPUs " "or using hacks like qemu-user."}; + Setting<StringSet> systemFeatures{this, getDefaultSystemFeatures(), + "system-features", + "Optional features that this system implements (like \"kvm\")."}; + Setting<Strings> substituters{this, nixStore == "/nix/store" ? Strings{"https://cache.nixos.org/"} : Strings(), "substituters", diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 88d2574e86ef..26e1851981db 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -303,6 +303,12 @@ struct LegacySSHStore : public Store { auto conn(connections->get()); } + + unsigned int getProtocol() override + { + auto conn(connections->get()); + return conn->remoteVersion; + } }; static RegisterStoreImplementation regStore([]( diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 197b9d78995b..e1cb423d151f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -366,6 +366,8 @@ void LocalStore::makeStoreWritable() throw SysError("getting info about the Nix store mount point"); if (stat.f_flag & ST_RDONLY) { + saveMountNamespace(); + if (unshare(CLONE_NEWNS) == -1) throw SysError("setting up a private mount namespace"); @@ -1338,6 +1340,12 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store, } +unsigned int LocalStore::getProtocol() +{ + return PROTOCOL_VERSION; +} + + #if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && defined(FS_IMMUTABLE_FL) static void makeMutable(const Path & path) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 746bdbeed793..fce963433a5e 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -209,6 +209,8 @@ public: void registerValidPaths(const ValidPathInfos & infos); + unsigned int getProtocol() override; + void vacuumDB(); /* Repair the contents of the given path by redownloading it using diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc new file mode 100644 index 000000000000..dc3286482736 --- /dev/null +++ b/src/libstore/parsed-derivations.cc @@ -0,0 +1,111 @@ +#include "parsed-derivations.hh" + +namespace nix { + +ParsedDerivation::ParsedDerivation(const Path & drvPath, BasicDerivation & drv) + : drvPath(drvPath), drv(drv) +{ + /* Parse the __json attribute, if any. */ + auto jsonAttr = drv.env.find("__json"); + if (jsonAttr != drv.env.end()) { + try { + structuredAttrs = nlohmann::json::parse(jsonAttr->second); + } catch (std::exception & e) { + throw Error("cannot process __json attribute of '%s': %s", drvPath, e.what()); + } + } +} + +std::experimental::optional<std::string> ParsedDerivation::getStringAttr(const std::string & name) const +{ + if (structuredAttrs) { + auto i = structuredAttrs->find(name); + if (i == structuredAttrs->end()) + return {}; + else { + if (!i->is_string()) + throw Error("attribute '%s' of derivation '%s' must be a string", name, drvPath); + return i->get<std::string>(); + } + } else { + auto i = drv.env.find(name); + if (i == drv.env.end()) + return {}; + else + return i->second; + } +} + +bool ParsedDerivation::getBoolAttr(const std::string & name, bool def) const +{ + if (structuredAttrs) { + auto i = structuredAttrs->find(name); + if (i == structuredAttrs->end()) + return def; + else { + if (!i->is_boolean()) + throw Error("attribute '%s' of derivation '%s' must be a Boolean", name, drvPath); + return i->get<bool>(); + } + } else { + auto i = drv.env.find(name); + if (i == drv.env.end()) + return def; + else + return i->second == "1"; + } +} + +std::experimental::optional<Strings> ParsedDerivation::getStringsAttr(const std::string & name) const +{ + if (structuredAttrs) { + auto i = structuredAttrs->find(name); + if (i == structuredAttrs->end()) + return {}; + else { + if (!i->is_array()) + throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath); + Strings res; + for (auto j = i->begin(); j != i->end(); ++j) { + if (!j->is_string()) + throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath); + res.push_back(j->get<std::string>()); + } + return res; + } + } else { + auto i = drv.env.find(name); + if (i == drv.env.end()) + return {}; + else + return tokenizeString<Strings>(i->second); + } +} + +StringSet ParsedDerivation::getRequiredSystemFeatures() const +{ + StringSet res; + for (auto & i : getStringsAttr("requiredSystemFeatures").value_or(Strings())) + res.insert(i); + return res; +} + +bool ParsedDerivation::canBuildLocally() const +{ + if (drv.platform != settings.thisSystem.get() + && !settings.extraPlatforms.get().count(drv.platform) + && !drv.isBuiltin()) + return false; + + for (auto & feature : getRequiredSystemFeatures()) + if (!settings.systemFeatures.get().count(feature)) return false; + + return true; +} + +bool ParsedDerivation::willBuildLocally() const +{ + return getBoolAttr("preferLocalBuild") && canBuildLocally(); +} + +} diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh new file mode 100644 index 000000000000..0a82c146172b --- /dev/null +++ b/src/libstore/parsed-derivations.hh @@ -0,0 +1,35 @@ +#include "derivations.hh" + +#include <nlohmann/json.hpp> + +namespace nix { + +class ParsedDerivation +{ + Path drvPath; + BasicDerivation & drv; + std::experimental::optional<nlohmann::json> structuredAttrs; + +public: + + ParsedDerivation(const Path & drvPath, BasicDerivation & drv); + + const std::experimental::optional<nlohmann::json> & getStructuredAttrs() const + { + return structuredAttrs; + } + + std::experimental::optional<std::string> getStringAttr(const std::string & name) const; + + bool getBoolAttr(const std::string & name, bool def = false) const; + + std::experimental::optional<Strings> getStringsAttr(const std::string & name) const; + + StringSet getRequiredSystemFeatures() const; + + bool canBuildLocally() const; + + bool willBuildLocally() const; +}; + +} diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index ea86ef052f53..def140cfbe18 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -161,7 +161,8 @@ void RemoteStore::initConnection(Connection & conn) if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 11) conn.to << false; - conn.processStderr(); + auto ex = conn.processStderr(); + if (ex) std::rethrow_exception(ex); } catch (Error & e) { throw Error("cannot open connection to remote store '%s': %s", getUri(), e.what()); @@ -195,22 +196,68 @@ void RemoteStore::setOptions(Connection & conn) conn.to << i.first << i.second.value; } - conn.processStderr(); + auto ex = conn.processStderr(); + if (ex) std::rethrow_exception(ex); +} + + +/* A wrapper around Pool<RemoteStore::Connection>::Handle that marks + the connection as bad (causing it to be closed) if a non-daemon + exception is thrown before the handle is closed. Such an exception + causes a deviation from the expected protocol and therefore a + desynchronization between the client and daemon. */ +struct ConnectionHandle +{ + Pool<RemoteStore::Connection>::Handle handle; + bool daemonException = false; + + ConnectionHandle(Pool<RemoteStore::Connection>::Handle && handle) + : handle(std::move(handle)) + { } + + ConnectionHandle(ConnectionHandle && h) + : handle(std::move(h.handle)) + { } + + ~ConnectionHandle() + { + if (!daemonException && std::uncaught_exception()) { + handle.markBad(); + debug("closing daemon connection because of an exception"); + } + } + + RemoteStore::Connection * operator -> () { return &*handle; } + + void processStderr(Sink * sink = 0, Source * source = 0) + { + auto ex = handle->processStderr(sink, source); + if (ex) { + daemonException = true; + std::rethrow_exception(ex); + } + } +}; + + +ConnectionHandle RemoteStore::getConnection() +{ + return ConnectionHandle(connections->get()); } bool RemoteStore::isValidPathUncached(const Path & path) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopIsValidPath << path; - conn->processStderr(); + conn.processStderr(); return readInt(conn->from); } PathSet RemoteStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubstitute) { - auto conn(connections->get()); + auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { PathSet res; for (auto & i : paths) @@ -218,7 +265,7 @@ PathSet RemoteStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybe return res; } else { conn->to << wopQueryValidPaths << paths; - conn->processStderr(); + conn.processStderr(); return readStorePaths<PathSet>(*this, conn->from); } } @@ -226,27 +273,27 @@ PathSet RemoteStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybe PathSet RemoteStore::queryAllValidPaths() { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopQueryAllValidPaths; - conn->processStderr(); + conn.processStderr(); return readStorePaths<PathSet>(*this, conn->from); } PathSet RemoteStore::querySubstitutablePaths(const PathSet & paths) { - auto conn(connections->get()); + auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { PathSet res; for (auto & i : paths) { conn->to << wopHasSubstitutes << i; - conn->processStderr(); + conn.processStderr(); if (readInt(conn->from)) res.insert(i); } return res; } else { conn->to << wopQuerySubstitutablePaths << paths; - conn->processStderr(); + conn.processStderr(); return readStorePaths<PathSet>(*this, conn->from); } } @@ -257,14 +304,14 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths, { if (paths.empty()) return; - auto conn(connections->get()); + auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { for (auto & i : paths) { SubstitutablePathInfo info; conn->to << wopQuerySubstitutablePathInfo << i; - conn->processStderr(); + conn.processStderr(); unsigned int reply = readInt(conn->from); if (reply == 0) continue; info.deriver = readString(conn->from); @@ -278,7 +325,7 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths, } else { conn->to << wopQuerySubstitutablePathInfos << paths; - conn->processStderr(); + conn.processStderr(); size_t count = readNum<size_t>(conn->from); for (size_t n = 0; n < count; n++) { Path path = readStorePath(*this, conn->from); @@ -300,10 +347,10 @@ void RemoteStore::queryPathInfoUncached(const Path & path, try { std::shared_ptr<ValidPathInfo> info; { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopQueryPathInfo << path; try { - conn->processStderr(); + conn.processStderr(); } catch (Error & e) { // Ugly backwards compatibility hack. if (e.msg().find("is not valid") != std::string::npos) @@ -335,9 +382,9 @@ void RemoteStore::queryPathInfoUncached(const Path & path, void RemoteStore::queryReferrers(const Path & path, PathSet & referrers) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopQueryReferrers << path; - conn->processStderr(); + conn.processStderr(); PathSet referrers2 = readStorePaths<PathSet>(*this, conn->from); referrers.insert(referrers2.begin(), referrers2.end()); } @@ -345,36 +392,36 @@ void RemoteStore::queryReferrers(const Path & path, PathSet RemoteStore::queryValidDerivers(const Path & path) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopQueryValidDerivers << path; - conn->processStderr(); + conn.processStderr(); return readStorePaths<PathSet>(*this, conn->from); } PathSet RemoteStore::queryDerivationOutputs(const Path & path) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopQueryDerivationOutputs << path; - conn->processStderr(); + conn.processStderr(); return readStorePaths<PathSet>(*this, conn->from); } PathSet RemoteStore::queryDerivationOutputNames(const Path & path) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopQueryDerivationOutputNames << path; - conn->processStderr(); + conn.processStderr(); return readStrings<PathSet>(conn->from); } Path RemoteStore::queryPathFromHashPart(const string & hashPart) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopQueryPathFromHashPart << hashPart; - conn->processStderr(); + conn.processStderr(); Path path = readString(conn->from); if (!path.empty()) assertStorePath(path); return path; @@ -384,7 +431,7 @@ Path RemoteStore::queryPathFromHashPart(const string & hashPart) void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) { - auto conn(connections->get()); + auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) { conn->to << wopImportPaths; @@ -403,7 +450,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, ; }); - conn->processStderr(0, source2.get()); + conn.processStderr(0, source2.get()); auto importedPaths = readStorePaths<PathSet>(*this, conn->from); assert(importedPaths.size() <= 1); @@ -417,7 +464,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << repair << !checkSigs; bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21; if (!tunnel) copyNAR(source, conn->to); - conn->processStderr(0, tunnel ? &source : nullptr); + conn.processStderr(0, tunnel ? &source : nullptr); } } @@ -427,7 +474,7 @@ Path RemoteStore::addToStore(const string & name, const Path & _srcPath, { if (repair) throw Error("repairing is not supported when building through the Nix daemon"); - auto conn(connections->get()); + auto conn(getConnection()); Path srcPath(absPath(_srcPath)); @@ -445,13 +492,13 @@ Path RemoteStore::addToStore(const string & name, const Path & _srcPath, dumpPath(srcPath, conn->to, filter); } conn->to.warn = false; - conn->processStderr(); + conn.processStderr(); } catch (SysError & e) { /* Daemon closed while we were sending the path. Probably OOM or I/O error. */ if (e.errNo == EPIPE) try { - conn->processStderr(); + conn.processStderr(); } catch (EndOfFile & e) { } throw; } @@ -465,17 +512,17 @@ Path RemoteStore::addTextToStore(const string & name, const string & s, { if (repair) throw Error("repairing is not supported when building through the Nix daemon"); - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopAddTextToStore << name << s << references; - conn->processStderr(); + conn.processStderr(); return readStorePath(*this, conn->from); } void RemoteStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopBuildPaths; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13) { conn->to << drvPaths; @@ -494,7 +541,7 @@ void RemoteStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) drvPaths2.insert(string(i, 0, i.find('!'))); conn->to << drvPaths2; } - conn->processStderr(); + conn.processStderr(); readInt(conn->from); } @@ -502,9 +549,9 @@ void RemoteStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) BuildResult RemoteStore::buildDerivation(const Path & drvPath, const BasicDerivation & drv, BuildMode buildMode) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopBuildDerivation << drvPath << drv << buildMode; - conn->processStderr(); + conn.processStderr(); BuildResult res; unsigned int status; conn->from >> status >> res.errorMsg; @@ -515,45 +562,45 @@ BuildResult RemoteStore::buildDerivation(const Path & drvPath, const BasicDeriva void RemoteStore::ensurePath(const Path & path) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopEnsurePath << path; - conn->processStderr(); + conn.processStderr(); readInt(conn->from); } void RemoteStore::addTempRoot(const Path & path) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopAddTempRoot << path; - conn->processStderr(); + conn.processStderr(); readInt(conn->from); } void RemoteStore::addIndirectRoot(const Path & path) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopAddIndirectRoot << path; - conn->processStderr(); + conn.processStderr(); readInt(conn->from); } void RemoteStore::syncWithGC() { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopSyncWithGC; - conn->processStderr(); + conn.processStderr(); readInt(conn->from); } Roots RemoteStore::findRoots() { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopFindRoots; - conn->processStderr(); + conn.processStderr(); size_t count = readNum<size_t>(conn->from); Roots result; while (count--) { @@ -567,7 +614,7 @@ Roots RemoteStore::findRoots() void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopCollectGarbage << options.action << options.pathsToDelete << options.ignoreLiveness @@ -575,7 +622,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) /* removed options */ << 0 << 0 << 0; - conn->processStderr(); + conn.processStderr(); results.paths = readStrings<PathSet>(conn->from); results.bytesFreed = readLongLong(conn->from); @@ -590,27 +637,27 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) void RemoteStore::optimiseStore() { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopOptimiseStore; - conn->processStderr(); + conn.processStderr(); readInt(conn->from); } bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopVerifyStore << checkContents << repair; - conn->processStderr(); + conn.processStderr(); return readInt(conn->from); } void RemoteStore::addSignatures(const Path & storePath, const StringSet & sigs) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopAddSignatures << storePath << sigs; - conn->processStderr(); + conn.processStderr(); readInt(conn->from); } @@ -620,13 +667,13 @@ void RemoteStore::queryMissing(const PathSet & targets, unsigned long long & downloadSize, unsigned long long & narSize) { { - auto conn(connections->get()); + auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 19) // Don't hold the connection handle in the fallback case // to prevent a deadlock. goto fallback; conn->to << wopQueryMissing << targets; - conn->processStderr(); + conn.processStderr(); willBuild = readStorePaths<PathSet>(*this, conn->from); willSubstitute = readStorePaths<PathSet>(*this, conn->from); unknown = readStorePaths<PathSet>(*this, conn->from); @@ -642,7 +689,14 @@ void RemoteStore::queryMissing(const PathSet & targets, void RemoteStore::connect() { + auto conn(getConnection()); +} + + +unsigned int RemoteStore::getProtocol() +{ auto conn(connections->get()); + return conn->daemonVersion; } @@ -679,7 +733,7 @@ static Logger::Fields readFields(Source & from) } -void RemoteStore::Connection::processStderr(Sink * sink, Source * source) +std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source) { to.flush(); @@ -704,7 +758,7 @@ void RemoteStore::Connection::processStderr(Sink * sink, Source * source) else if (msg == STDERR_ERROR) { string error = readString(from); unsigned int status = readInt(from); - throw Error(status, error); + return std::make_exception_ptr(Error(status, error)); } else if (msg == STDERR_NEXT) @@ -738,6 +792,8 @@ void RemoteStore::Connection::processStderr(Sink * sink, Source * source) else throw Error("got unknown message type %x from Nix daemon", msg); } + + return nullptr; } static std::string uriScheme = "unix://"; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index b488e34ce263..4f554b5980e8 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -14,6 +14,7 @@ class Pid; struct FdSink; struct FdSource; template<typename T> class Pool; +struct ConnectionHandle; /* FIXME: RemoteStore is a misnomer - should be something like @@ -97,12 +98,15 @@ public: void connect() override; + unsigned int getProtocol() override; + void flushBadConnections(); protected: struct Connection { + AutoCloseFD fd; FdSink to; FdSource from; unsigned int daemonVersion; @@ -110,7 +114,7 @@ protected: virtual ~Connection(); - void processStderr(Sink * sink = 0, Source * source = 0); + std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0); }; ref<Connection> openConnectionWrapper(); @@ -123,6 +127,10 @@ protected: virtual void setOptions(Connection & conn); + ConnectionHandle getConnection(); + + friend struct ConnectionHandle; + private: std::atomic_bool failed{false}; @@ -140,11 +148,6 @@ public: private: - struct Connection : RemoteStore::Connection - { - AutoCloseFD fd; - }; - ref<RemoteStore::Connection> openConnection() override; std::experimental::optional<std::string> path; }; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 7711388f05a9..4f1e23198ffe 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -19,8 +19,6 @@ #include <aws/core/utils/logging/LogMacros.h> #include <aws/core/utils/threading/Executor.h> #include <aws/s3/S3Client.h> -#include <aws/s3/model/CreateBucketRequest.h> -#include <aws/s3/model/GetBucketLocationRequest.h> #include <aws/s3/model/GetObjectRequest.h> #include <aws/s3/model/HeadObjectRequest.h> #include <aws/s3/model/ListObjectsRequest.h> @@ -175,6 +173,8 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"}; const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"}; const Setting<std::string> logCompression{this, "", "log-compression", "compression method for log/* files"}; + const Setting<bool> multipartUpload{ + this, false, "multipart-upload", "whether to use multi-part uploads"}; const Setting<uint64_t> bufferSize{ this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"}; @@ -202,32 +202,6 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore { if (!diskCache->cacheExists(getUri(), wantMassQuery_, priority)) { - /* Create the bucket if it doesn't already exists. */ - // FIXME: HeadBucket would be more appropriate, but doesn't return - // an easily parsed 404 message. - auto res = s3Helper.client->GetBucketLocation( - Aws::S3::Model::GetBucketLocationRequest().WithBucket(bucketName)); - - if (!res.IsSuccess()) { - if (res.GetError().GetErrorType() != Aws::S3::S3Errors::NO_SUCH_BUCKET) - throw Error(format("AWS error checking bucket '%s': %s") % bucketName % res.GetError().GetMessage()); - - printInfo("creating S3 bucket '%s'...", bucketName); - - // Stupid S3 bucket locations. - auto bucketConfig = Aws::S3::Model::CreateBucketConfiguration(); - if (s3Helper.config->region != "us-east-1") - bucketConfig.SetLocationConstraint( - Aws::S3::Model::BucketLocationConstraintMapper::GetBucketLocationConstraintForName( - s3Helper.config->region)); - - checkAws(format("AWS error creating bucket '%s'") % bucketName, - s3Helper.client->CreateBucket( - Aws::S3::Model::CreateBucketRequest() - .WithBucket(bucketName) - .WithCreateBucketConfiguration(bucketConfig))); - } - BinaryCacheStore::init(); diskCache->createCache(getUri(), storeDir, wantMassQuery_, priority); @@ -289,48 +263,73 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor> executor = std::make_shared<Aws::Utils::Threading::PooledThreadExecutor>(maxThreads); - std::call_once(transferManagerCreated, [&]() { + std::call_once(transferManagerCreated, [&]() + { + if (multipartUpload) { + TransferManagerConfiguration transferConfig(executor.get()); + + transferConfig.s3Client = s3Helper.client; + transferConfig.bufferSize = bufferSize; + + transferConfig.uploadProgressCallback = + [](const TransferManager *transferManager, + const std::shared_ptr<const TransferHandle> + &transferHandle) + { + //FIXME: find a way to properly abort the multipart upload. + //checkInterrupt(); + debug("upload progress ('%s'): '%d' of '%d' bytes", + transferHandle->GetKey(), + transferHandle->GetBytesTransferred(), + transferHandle->GetBytesTotalSize()); + }; + + transferManager = TransferManager::Create(transferConfig); + } + }); - TransferManagerConfiguration transferConfig(executor.get()); + auto now1 = std::chrono::steady_clock::now(); - transferConfig.s3Client = s3Helper.client; - transferConfig.bufferSize = bufferSize; + if (transferManager) { - transferConfig.uploadProgressCallback = - [&](const TransferManager *transferManager, - const std::shared_ptr<const TransferHandle> - &transferHandle) - { - //FIXME: find a way to properly abort the multipart upload. - //checkInterrupt(); - debug("upload progress ('%s'): '%d' of '%d' bytes", - path, - transferHandle->GetBytesTransferred(), - transferHandle->GetBytesTotalSize()); - }; + if (contentEncoding != "") + throw Error("setting a content encoding is not supported with S3 multi-part uploads"); - transferManager = TransferManager::Create(transferConfig); - }); + std::shared_ptr<TransferHandle> transferHandle = + transferManager->UploadFile( + stream, bucketName, path, mimeType, + Aws::Map<Aws::String, Aws::String>(), + nullptr /*, contentEncoding */); - auto now1 = std::chrono::steady_clock::now(); + transferHandle->WaitUntilFinished(); + + if (transferHandle->GetStatus() == TransferStatus::FAILED) + throw Error("AWS error: failed to upload 's3://%s/%s': %s", + bucketName, path, transferHandle->GetLastError().GetMessage()); + + if (transferHandle->GetStatus() != TransferStatus::COMPLETED) + throw Error("AWS error: transfer status of 's3://%s/%s' in unexpected state", + bucketName, path); - std::shared_ptr<TransferHandle> transferHandle = - transferManager->UploadFile( - stream, bucketName, path, mimeType, - Aws::Map<Aws::String, Aws::String>(), - nullptr, contentEncoding); + } else { - transferHandle->WaitUntilFinished(); + auto request = + Aws::S3::Model::PutObjectRequest() + .WithBucket(bucketName) + .WithKey(path); - if (transferHandle->GetStatus() == TransferStatus::FAILED) - throw Error("AWS error: failed to upload 's3://%s/%s': %s", - bucketName, path, transferHandle->GetLastError().GetMessage()); + request.SetContentType(mimeType); - if (transferHandle->GetStatus() != TransferStatus::COMPLETED) - throw Error("AWS error: transfer status of 's3://%s/%s' in unexpected state", - bucketName, path); + if (contentEncoding != "") + request.SetContentEncoding(contentEncoding); - printTalkative("upload of '%s' completed", path); + auto stream = std::make_shared<istringstream_nocopy>(data); + + request.SetBody(stream); + + auto result = checkAws(fmt("AWS error uploading '%s'", path), + s3Helper.client->PutObject(request)); + } auto now2 = std::chrono::steady_clock::now(); diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 5e0e44935cca..cf133b57cb20 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -1,4 +1,5 @@ #include "ssh.hh" +#include "affinity.hh" namespace nix { @@ -34,7 +35,9 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string auto conn = std::make_unique<Connection>(); conn->sshPid = startProcess([&]() { + restoreAffinity(); restoreSignals(); + restoreMountNamespace(); close(in.writeSide.get()); close(out.readSide.get()); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 1f42097fccfb..dc54c735fdb1 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -320,6 +320,8 @@ ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath) void Store::queryPathInfo(const Path & storePath, Callback<ref<ValidPathInfo>> callback) { + assertStorePath(storePath); + auto hashPart = storePathToHash(storePath); try { @@ -586,15 +588,19 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, uint64_t total = 0; - // FIXME -#if 0 if (!info->narHash) { + StringSink sink; + srcStore->narFromPath({storePath}, sink); auto info2 = make_ref<ValidPathInfo>(*info); info2->narHash = hashString(htSHA256, *sink.s); if (!info->narSize) info2->narSize = sink.s->size(); + if (info->ultimate) info2->ultimate = false; info = info2; + + StringSource source(*sink.s); + dstStore->addToStore(*info, source, repair, checkSigs); + return; } -#endif if (info->ultimate) { auto info2 = make_ref<ValidPathInfo>(*info); @@ -610,7 +616,7 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, }); srcStore->narFromPath({storePath}, wrapperSink); }, [&]() { - throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri()); + throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri()); }); dstStore->addToStore(*info, *source, repair, checkSigs); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 099818ed6f69..106b2be5e6b2 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -599,6 +599,12 @@ public: a notion of connection. Otherwise this is a no-op. */ virtual void connect() { }; + /* Get the protocol version of this store or it's connection. */ + virtual unsigned int getProtocol() + { + return 0; + }; + /* Get the priority of the store, used to order substituters. In particular, binary caches can specify a priority field in their "nix-cache-info" file. Lower value means higher priority. */ diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 1be8934a2eba..bb68e82886d0 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -283,7 +283,7 @@ void parseDump(ParseSink & sink, Source & source) { string version; try { - version = readString(source); + version = readString(source, narVersionMagic1.size()); } catch (SerialisationError & e) { /* This generally means the integer at the start couldn't be decoded. Ignore and throw the exception below. */ diff --git a/src/libutil/pool.hh b/src/libutil/pool.hh index 0b142b0597c7..d49067bb95dc 100644 --- a/src/libutil/pool.hh +++ b/src/libutil/pool.hh @@ -97,6 +97,7 @@ public: private: Pool & pool; std::shared_ptr<R> r; + bool bad = false; friend Pool; @@ -112,7 +113,8 @@ public: if (!r) return; { auto state_(pool.state.lock()); - state_->idle.push_back(ref<R>(r)); + if (!bad) + state_->idle.push_back(ref<R>(r)); assert(state_->inUse); state_->inUse--; } @@ -121,6 +123,8 @@ public: R * operator -> () { return &*r; } R & operator * () { return *r; } + + void markBad() { bad = true; } }; Handle get() diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 17448f70efb6..0e75eeec2bfe 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -169,17 +169,13 @@ std::unique_ptr<Source> sinkToSource( { typedef boost::coroutines2::coroutine<std::string> coro_t; + std::function<void(Sink &)> fun; std::function<void()> eof; - coro_t::pull_type coro; + std::experimental::optional<coro_t::pull_type> coro; + bool started = false; SinkToSource(std::function<void(Sink &)> fun, std::function<void()> eof) - : eof(eof) - , coro([&](coro_t::push_type & yield) { - LambdaSink sink([&](const unsigned char * data, size_t len) { - if (len) yield(std::string((const char *) data, len)); - }); - fun(sink); - }) + : fun(fun), eof(eof) { } @@ -188,11 +184,19 @@ std::unique_ptr<Source> sinkToSource( size_t read(unsigned char * data, size_t len) override { - if (!coro) { eof(); abort(); } + if (!coro) + coro = coro_t::pull_type([&](coro_t::push_type & yield) { + LambdaSink sink([&](const unsigned char * data, size_t len) { + if (len) yield(std::string((const char *) data, len)); + }); + fun(sink); + }); + + if (!*coro) { eof(); abort(); } if (pos == cur.size()) { - if (!cur.empty()) coro(); - cur = coro.get(); + if (!cur.empty()) (*coro)(); + cur = coro->get(); pos = 0; } @@ -268,16 +272,17 @@ void readPadding(size_t len, Source & source) size_t readString(unsigned char * buf, size_t max, Source & source) { auto len = readNum<size_t>(source); - if (len > max) throw Error("string is too long"); + if (len > max) throw SerialisationError("string is too long"); source(buf, len); readPadding(len, source); return len; } -string readString(Source & source) +string readString(Source & source, size_t max) { auto len = readNum<size_t>(source); + if (len > max) throw SerialisationError("string is too long"); std::string res(len, 0); source((unsigned char*) res.data(), len); readPadding(len, source); diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 4b6ad5da5b9c..969e4dff383d 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -284,7 +284,7 @@ inline uint64_t readLongLong(Source & source) void readPadding(size_t len, Source & source); size_t readString(unsigned char * buf, size_t max, Source & source); -string readString(Source & source); +string readString(Source & source, size_t max = std::numeric_limits<size_t>::max()); template<class T> T readStrings(Source & source); Source & operator >> (Source & in, string & s); diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 03f0be705c1d..6e4536e6e4ea 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -496,6 +496,15 @@ Path getConfigDir() return configDir; } +std::vector<Path> getConfigDirs() +{ + Path configHome = getConfigDir(); + string configDirs = getEnv("XDG_CONFIG_DIRS"); + std::vector<Path> result = tokenizeString<std::vector<string>>(configDirs, ":"); + result.insert(result.begin(), configHome); + return result; +} + Path getDataDir() { @@ -927,6 +936,7 @@ pid_t startProcess(std::function<void()> fun, const ProcessOptions & options) throw SysError("setting death signal"); #endif restoreAffinity(); + restoreMountNamespace(); fun(); } catch (std::exception & e) { try { @@ -1495,4 +1505,26 @@ std::unique_ptr<InterruptCallback> createInterruptCallback(std::function<void()> return std::unique_ptr<InterruptCallback>(res.release()); } +static AutoCloseFD fdSavedMountNamespace; + +void saveMountNamespace() +{ +#if __linux__ + std::once_flag done; + std::call_once(done, []() { + fdSavedMountNamespace = open("/proc/self/ns/mnt", O_RDONLY); + if (!fdSavedMountNamespace) + throw SysError("saving parent mount namespace"); + }); +#endif +} + +void restoreMountNamespace() +{ +#if __linux__ + if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) + throw SysError("restoring parent mount namespace"); +#endif +} + } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index fc25d27758c7..2689cbd8b412 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -131,6 +131,9 @@ Path getCacheDir(); /* Return $XDG_CONFIG_HOME or $HOME/.config. */ Path getConfigDir(); +/* Return the directories to search for user configuration files */ +std::vector<Path> getConfigDirs(); + /* Return $XDG_DATA_HOME or $HOME/.local/share. */ Path getDataDir(); @@ -511,4 +514,13 @@ typedef std::function<bool(const Path & path)> PathFilter; extern PathFilter defaultPathFilter; +/* Save the current mount namespace. Ignored if called more than + once. */ +void saveMountNamespace(); + +/* Restore the mount namespace saved by saveMountNamespace(). Ignored + if saveMountNamespace() was never called. */ +void restoreMountNamespace(); + + } diff --git a/src/linenoise/ConvertUTF.cpp b/src/linenoise/ConvertUTF.cpp deleted file mode 100644 index f7e5915d5e8f..000000000000 --- a/src/linenoise/ConvertUTF.cpp +++ /dev/null @@ -1,542 +0,0 @@ -/* - * Copyright 2001-2004 Unicode, Inc. - * - * Disclaimer - * - * This source code is provided as is by Unicode, Inc. No claims are - * made as to fitness for any particular purpose. No warranties of any - * kind are expressed or implied. The recipient agrees to determine - * applicability of information provided. If this file has been - * purchased on magnetic or optical media from Unicode, Inc., the - * sole remedy for any claim will be exchange of defective media - * within 90 days of receipt. - * - * Limitations on Rights to Redistribute This Code - * - * Unicode, Inc. hereby grants the right to freely use the information - * supplied in this file in the creation of products supporting the - * Unicode Standard, and to make copies of this file in any form - * for internal or external distribution as long as this notice - * remains attached. - */ - -/* --------------------------------------------------------------------- - - Conversions between UTF32, UTF-16, and UTF-8. Source code file. - Author: Mark E. Davis, 1994. - Rev History: Rick McGowan, fixes & updates May 2001. - Sept 2001: fixed const & error conditions per - mods suggested by S. Parent & A. Lillich. - June 2002: Tim Dodd added detection and handling of incomplete - source sequences, enhanced error detection, added casts - to eliminate compiler warnings. - July 2003: slight mods to back out aggressive FFFE detection. - Jan 2004: updated switches in from-UTF8 conversions. - Oct 2004: updated to use UNI_MAX_LEGAL_UTF32 in UTF-32 conversions. - - See the header file "ConvertUTF.h" for complete documentation. - ------------------------------------------------------------------------- */ - -#include "ConvertUTF.h" -#ifdef CVTUTF_DEBUG -#include <stdio.h> -#endif - -namespace linenoise_ng { - -static const int halfShift = 10; /* used for shifting by 10 bits */ - -static const UTF32 halfBase = 0x0010000UL; -static const UTF32 halfMask = 0x3FFUL; - -#define UNI_SUR_HIGH_START (UTF32)0xD800 -#define UNI_SUR_HIGH_END (UTF32)0xDBFF -#define UNI_SUR_LOW_START (UTF32)0xDC00 -#define UNI_SUR_LOW_END (UTF32)0xDFFF -#define false 0 -#define true 1 - -/* --------------------------------------------------------------------- */ - -ConversionResult ConvertUTF32toUTF16 ( - const UTF32** sourceStart, const UTF32* sourceEnd, - char16_t** targetStart, char16_t* targetEnd, ConversionFlags flags) { - ConversionResult result = conversionOK; - const UTF32* source = *sourceStart; - char16_t* target = *targetStart; - while (source < sourceEnd) { - UTF32 ch; - if (target >= targetEnd) { - result = targetExhausted; break; - } - ch = *source++; - if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */ - /* UTF-16 surrogate values are illegal in UTF-32; 0xffff or 0xfffe are both reserved values */ - if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { - if (flags == strictConversion) { - --source; /* return to the illegal value itself */ - result = sourceIllegal; - break; - } else { - *target++ = UNI_REPLACEMENT_CHAR; - } - } else { - *target++ = (UTF16)ch; /* normal case */ - } - } else if (ch > UNI_MAX_LEGAL_UTF32) { - if (flags == strictConversion) { - result = sourceIllegal; - } else { - *target++ = UNI_REPLACEMENT_CHAR; - } - } else { - /* target is a character in range 0xFFFF - 0x10FFFF. */ - if (target + 1 >= targetEnd) { - --source; /* Back up source pointer! */ - result = targetExhausted; break; - } - ch -= halfBase; - *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START); - *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START); - } - } - *sourceStart = source; - *targetStart = target; - return result; -} - -/* --------------------------------------------------------------------- */ - -ConversionResult ConvertUTF16toUTF32 ( - const UTF16** sourceStart, const UTF16* sourceEnd, - UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) { - ConversionResult result = conversionOK; - const UTF16* source = *sourceStart; - UTF32* target = *targetStart; - UTF32 ch, ch2; - while (source < sourceEnd) { - const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */ - ch = *source++; - /* If we have a surrogate pair, convert to UTF32 first. */ - if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) { - /* If the 16 bits following the high surrogate are in the source buffer... */ - if (source < sourceEnd) { - ch2 = *source; - /* If it's a low surrogate, convert to UTF32. */ - if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) { - ch = ((ch - UNI_SUR_HIGH_START) << halfShift) - + (ch2 - UNI_SUR_LOW_START) + halfBase; - ++source; - } else if (flags == strictConversion) { /* it's an unpaired high surrogate */ - --source; /* return to the illegal value itself */ - result = sourceIllegal; - break; - } - } else { /* We don't have the 16 bits following the high surrogate. */ - --source; /* return to the high surrogate */ - result = sourceExhausted; - break; - } - } else if (flags == strictConversion) { - /* UTF-16 surrogate values are illegal in UTF-32 */ - if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) { - --source; /* return to the illegal value itself */ - result = sourceIllegal; - break; - } - } - if (target >= targetEnd) { - source = oldSource; /* Back up source pointer! */ - result = targetExhausted; break; - } - *target++ = ch; - } - *sourceStart = source; - *targetStart = target; -#ifdef CVTUTF_DEBUG -if (result == sourceIllegal) { - fprintf(stderr, "ConvertUTF16toUTF32 illegal seq 0x%04x,%04x\n", ch, ch2); - fflush(stderr); -} -#endif - return result; -} - -/* --------------------------------------------------------------------- */ - -/* - * Index into the table below with the first byte of a UTF-8 sequence to - * get the number of trailing bytes that are supposed to follow it. - * Note that *legal* UTF-8 values can't have 4 or 5-bytes. The table is - * left as-is for anyone who may want to do such conversion, which was - * allowed in earlier algorithms. - */ -static const char trailingBytesForUTF8[256] = { - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, - 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5 -}; - -/* - * Magic values subtracted from a buffer value during UTF8 conversion. - * This table contains as many values as there might be trailing bytes - * in a UTF-8 sequence. - */ -static const UTF32 offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL, - 0x03C82080UL, 0xFA082080UL, 0x82082080UL }; - -/* - * Once the bits are split out into bytes of UTF-8, this is a mask OR-ed - * into the first byte, depending on how many bytes follow. There are - * as many entries in this table as there are UTF-8 sequence types. - * (I.e., one byte sequence, two byte... etc.). Remember that sequencs - * for *legal* UTF-8 will be 4 or fewer bytes total. - */ -static const UTF8 firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC }; - -/* --------------------------------------------------------------------- */ - -/* The interface converts a whole buffer to avoid function-call overhead. - * Constants have been gathered. Loops & conditionals have been removed as - * much as possible for efficiency, in favor of drop-through switches. - * (See "Note A" at the bottom of the file for equivalent code.) - * If your compiler supports it, the "isLegalUTF8" call can be turned - * into an inline function. - */ - -/* --------------------------------------------------------------------- */ - -ConversionResult ConvertUTF16toUTF8 ( - const UTF16** sourceStart, const UTF16* sourceEnd, - UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) { - ConversionResult result = conversionOK; - const UTF16* source = *sourceStart; - UTF8* target = *targetStart; - while (source < sourceEnd) { - UTF32 ch; - unsigned short bytesToWrite = 0; - const UTF32 byteMask = 0xBF; - const UTF32 byteMark = 0x80; - const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */ - ch = *source++; - /* If we have a surrogate pair, convert to UTF32 first. */ - if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) { - /* If the 16 bits following the high surrogate are in the source buffer... */ - if (source < sourceEnd) { - UTF32 ch2 = *source; - /* If it's a low surrogate, convert to UTF32. */ - if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) { - ch = ((ch - UNI_SUR_HIGH_START) << halfShift) - + (ch2 - UNI_SUR_LOW_START) + halfBase; - ++source; - } else if (flags == strictConversion) { /* it's an unpaired high surrogate */ - --source; /* return to the illegal value itself */ - result = sourceIllegal; - break; - } - } else { /* We don't have the 16 bits following the high surrogate. */ - --source; /* return to the high surrogate */ - result = sourceExhausted; - break; - } - } else if (flags == strictConversion) { - /* UTF-16 surrogate values are illegal in UTF-32 */ - if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) { - --source; /* return to the illegal value itself */ - result = sourceIllegal; - break; - } - } - /* Figure out how many bytes the result will require */ - if (ch < (UTF32)0x80) { bytesToWrite = 1; - } else if (ch < (UTF32)0x800) { bytesToWrite = 2; - } else if (ch < (UTF32)0x10000) { bytesToWrite = 3; - } else if (ch < (UTF32)0x110000) { bytesToWrite = 4; - } else { bytesToWrite = 3; - ch = UNI_REPLACEMENT_CHAR; - } - - target += bytesToWrite; - if (target > targetEnd) { - source = oldSource; /* Back up source pointer! */ - target -= bytesToWrite; result = targetExhausted; break; - } - switch (bytesToWrite) { /* note: everything falls through. */ - case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; - case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; - case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; - case 1: *--target = (UTF8)(ch | firstByteMark[bytesToWrite]); - } - target += bytesToWrite; - } - *sourceStart = source; - *targetStart = target; - return result; -} - -/* --------------------------------------------------------------------- */ - -/* - * Utility routine to tell whether a sequence of bytes is legal UTF-8. - * This must be called with the length pre-determined by the first byte. - * If not calling this from ConvertUTF8to*, then the length can be set by: - * length = trailingBytesForUTF8[*source]+1; - * and the sequence is illegal right away if there aren't that many bytes - * available. - * If presented with a length > 4, this returns false. The Unicode - * definition of UTF-8 goes up to 4-byte sequences. - */ - -static Boolean isLegalUTF8(const UTF8 *source, int length) { - UTF8 a; - const UTF8 *srcptr = source+length; - switch (length) { - default: return false; - /* Everything else falls through when "true"... */ - case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false; - case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false; - case 2: if ((a = (*--srcptr)) > 0xBF) return false; - - switch (*source) { - /* no fall-through in this inner switch */ - case 0xE0: if (a < 0xA0) return false; break; - case 0xED: if (a > 0x9F) return false; break; - case 0xF0: if (a < 0x90) return false; break; - case 0xF4: if (a > 0x8F) return false; break; - default: if (a < 0x80) return false; - } - - case 1: if (*source >= 0x80 && *source < 0xC2) return false; - } - if (*source > 0xF4) return false; - return true; -} - -/* --------------------------------------------------------------------- */ - -/* - * Exported function to return whether a UTF-8 sequence is legal or not. - * This is not used here; it's just exported. - */ -Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd) { - int length = trailingBytesForUTF8[*source]+1; - if (source+length > sourceEnd) { - return false; - } - return isLegalUTF8(source, length); -} - -/* --------------------------------------------------------------------- */ - -ConversionResult ConvertUTF8toUTF16 ( - const UTF8** sourceStart, const UTF8* sourceEnd, - UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) { - ConversionResult result = conversionOK; - const UTF8* source = *sourceStart; - UTF16* target = *targetStart; - while (source < sourceEnd) { - UTF32 ch = 0; - unsigned short extraBytesToRead = trailingBytesForUTF8[*source]; - if (source + extraBytesToRead >= sourceEnd) { - result = sourceExhausted; break; - } - /* Do this check whether lenient or strict */ - if (! isLegalUTF8(source, extraBytesToRead+1)) { - result = sourceIllegal; - break; - } - /* - * The cases all fall through. See "Note A" below. - */ - switch (extraBytesToRead) { - case 5: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */ - case 4: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */ - case 3: ch += *source++; ch <<= 6; - case 2: ch += *source++; ch <<= 6; - case 1: ch += *source++; ch <<= 6; - case 0: ch += *source++; - } - ch -= offsetsFromUTF8[extraBytesToRead]; - - if (target >= targetEnd) { - source -= (extraBytesToRead+1); /* Back up source pointer! */ - result = targetExhausted; break; - } - if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */ - /* UTF-16 surrogate values are illegal in UTF-32 */ - if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { - if (flags == strictConversion) { - source -= (extraBytesToRead+1); /* return to the illegal value itself */ - result = sourceIllegal; - break; - } else { - *target++ = UNI_REPLACEMENT_CHAR; - } - } else { - *target++ = (UTF16)ch; /* normal case */ - } - } else if (ch > UNI_MAX_UTF16) { - if (flags == strictConversion) { - result = sourceIllegal; - source -= (extraBytesToRead+1); /* return to the start */ - break; /* Bail out; shouldn't continue */ - } else { - *target++ = UNI_REPLACEMENT_CHAR; - } - } else { - /* target is a character in range 0xFFFF - 0x10FFFF. */ - if (target + 1 >= targetEnd) { - source -= (extraBytesToRead+1); /* Back up source pointer! */ - result = targetExhausted; break; - } - ch -= halfBase; - *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START); - *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START); - } - } - *sourceStart = source; - *targetStart = target; - return result; -} - -/* --------------------------------------------------------------------- */ - -ConversionResult ConvertUTF32toUTF8 ( - const UTF32** sourceStart, const UTF32* sourceEnd, - UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) { - ConversionResult result = conversionOK; - const UTF32* source = *sourceStart; - UTF8* target = *targetStart; - while (source < sourceEnd) { - UTF32 ch; - unsigned short bytesToWrite = 0; - const UTF32 byteMask = 0xBF; - const UTF32 byteMark = 0x80; - ch = *source++; - if (flags == strictConversion ) { - /* UTF-16 surrogate values are illegal in UTF-32 */ - if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { - --source; /* return to the illegal value itself */ - result = sourceIllegal; - break; - } - } - /* - * Figure out how many bytes the result will require. Turn any - * illegally large UTF32 things (> Plane 17) into replacement chars. - */ - if (ch < (UTF32)0x80) { bytesToWrite = 1; - } else if (ch < (UTF32)0x800) { bytesToWrite = 2; - } else if (ch < (UTF32)0x10000) { bytesToWrite = 3; - } else if (ch <= UNI_MAX_LEGAL_UTF32) { bytesToWrite = 4; - } else { bytesToWrite = 3; - ch = UNI_REPLACEMENT_CHAR; - result = sourceIllegal; - } - - target += bytesToWrite; - if (target > targetEnd) { - --source; /* Back up source pointer! */ - target -= bytesToWrite; result = targetExhausted; break; - } - switch (bytesToWrite) { /* note: everything falls through. */ - case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; - case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; - case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; - case 1: *--target = (UTF8) (ch | firstByteMark[bytesToWrite]); - } - target += bytesToWrite; - } - *sourceStart = source; - *targetStart = target; - return result; -} - -/* --------------------------------------------------------------------- */ - -ConversionResult ConvertUTF8toUTF32 ( - const UTF8** sourceStart, const UTF8* sourceEnd, - UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) { - ConversionResult result = conversionOK; - const UTF8* source = *sourceStart; - UTF32* target = *targetStart; - while (source < sourceEnd) { - UTF32 ch = 0; - unsigned short extraBytesToRead = trailingBytesForUTF8[*source]; - if (source + extraBytesToRead >= sourceEnd) { - result = sourceExhausted; break; - } - /* Do this check whether lenient or strict */ - if (! isLegalUTF8(source, extraBytesToRead+1)) { - result = sourceIllegal; - break; - } - /* - * The cases all fall through. See "Note A" below. - */ - switch (extraBytesToRead) { - case 5: ch += *source++; ch <<= 6; - case 4: ch += *source++; ch <<= 6; - case 3: ch += *source++; ch <<= 6; - case 2: ch += *source++; ch <<= 6; - case 1: ch += *source++; ch <<= 6; - case 0: ch += *source++; - } - ch -= offsetsFromUTF8[extraBytesToRead]; - - if (target >= targetEnd) { - source -= (extraBytesToRead+1); /* Back up the source pointer! */ - result = targetExhausted; break; - } - if (ch <= UNI_MAX_LEGAL_UTF32) { - /* - * UTF-16 surrogate values are illegal in UTF-32, and anything - * over Plane 17 (> 0x10FFFF) is illegal. - */ - if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { - if (flags == strictConversion) { - source -= (extraBytesToRead+1); /* return to the illegal value itself */ - result = sourceIllegal; - break; - } else { - *target++ = UNI_REPLACEMENT_CHAR; - } - } else { - *target++ = ch; - } - } else { /* i.e., ch > UNI_MAX_LEGAL_UTF32 */ - result = sourceIllegal; - *target++ = UNI_REPLACEMENT_CHAR; - } - } - *sourceStart = source; - *targetStart = target; - return result; -} - -} - -/* --------------------------------------------------------------------- - - Note A. - The fall-through switches in UTF-8 reading code save a - temp variable, some decrements & conditionals. The switches - are equivalent to the following loop: - { - int tmpBytesToRead = extraBytesToRead+1; - do { - ch += *source++; - --tmpBytesToRead; - if (tmpBytesToRead) ch <<= 6; - } while (tmpBytesToRead > 0); - } - In UTF-8 writing code, the switches on "bytesToWrite" are - similarly unrolled loops. - - --------------------------------------------------------------------- */ diff --git a/src/linenoise/ConvertUTF.h b/src/linenoise/ConvertUTF.h deleted file mode 100755 index 8a296235dcd9..000000000000 --- a/src/linenoise/ConvertUTF.h +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Copyright 2001-2004 Unicode, Inc. - * - * Disclaimer - * - * This source code is provided as is by Unicode, Inc. No claims are - * made as to fitness for any particular purpose. No warranties of any - * kind are expressed or implied. The recipient agrees to determine - * applicability of information provided. If this file has been - * purchased on magnetic or optical media from Unicode, Inc., the - * sole remedy for any claim will be exchange of defective media - * within 90 days of receipt. - * - * Limitations on Rights to Redistribute This Code - * - * Unicode, Inc. hereby grants the right to freely use the information - * supplied in this file in the creation of products supporting the - * Unicode Standard, and to make copies of this file in any form - * for internal or external distribution as long as this notice - * remains attached. - */ - -/* --------------------------------------------------------------------- - - Conversions between UTF32, UTF-16, and UTF-8. Header file. - - Several funtions are included here, forming a complete set of - conversions between the three formats. UTF-7 is not included - here, but is handled in a separate source file. - - Each of these routines takes pointers to input buffers and output - buffers. The input buffers are const. - - Each routine converts the text between *sourceStart and sourceEnd, - putting the result into the buffer between *targetStart and - targetEnd. Note: the end pointers are *after* the last item: e.g. - *(sourceEnd - 1) is the last item. - - The return result indicates whether the conversion was successful, - and if not, whether the problem was in the source or target buffers. - (Only the first encountered problem is indicated.) - - After the conversion, *sourceStart and *targetStart are both - updated to point to the end of last text successfully converted in - the respective buffers. - - Input parameters: - sourceStart - pointer to a pointer to the source buffer. - The contents of this are modified on return so that - it points at the next thing to be converted. - targetStart - similarly, pointer to pointer to the target buffer. - sourceEnd, targetEnd - respectively pointers to the ends of the - two buffers, for overflow checking only. - - These conversion functions take a ConversionFlags argument. When this - flag is set to strict, both irregular sequences and isolated surrogates - will cause an error. When the flag is set to lenient, both irregular - sequences and isolated surrogates are converted. - - Whether the flag is strict or lenient, all illegal sequences will cause - an error return. This includes sequences such as: <F4 90 80 80>, <C0 80>, - or <A0> in UTF-8, and values above 0x10FFFF in UTF-32. Conformant code - must check for illegal sequences. - - When the flag is set to lenient, characters over 0x10FFFF are converted - to the replacement character; otherwise (when the flag is set to strict) - they constitute an error. - - Output parameters: - The value "sourceIllegal" is returned from some routines if the input - sequence is malformed. When "sourceIllegal" is returned, the source - value will point to the illegal value that caused the problem. E.g., - in UTF-8 when a sequence is malformed, it points to the start of the - malformed sequence. - - Author: Mark E. Davis, 1994. - Rev History: Rick McGowan, fixes & updates May 2001. - Fixes & updates, Sept 2001. - ------------------------------------------------------------------------- */ - -/* --------------------------------------------------------------------- - The following 4 definitions are compiler-specific. - The C standard does not guarantee that wchar_t has at least - 16 bits, so wchar_t is no less portable than unsigned short! - All should be unsigned values to avoid sign extension during - bit mask & shift operations. ------------------------------------------------------------------------- */ - -#if 0 -typedef unsigned long UTF32; /* at least 32 bits */ -typedef unsigned short UTF16; /* at least 16 bits */ -typedef unsigned char UTF8; /* typically 8 bits */ -#endif - -#include <stdint.h> -#include <string> - -namespace linenoise_ng { - -typedef uint32_t UTF32; -typedef uint16_t UTF16; -typedef uint8_t UTF8; -typedef unsigned char Boolean; /* 0 or 1 */ - -/* Some fundamental constants */ -#define UNI_REPLACEMENT_CHAR (UTF32)0x0000FFFD -#define UNI_MAX_BMP (UTF32)0x0000FFFF -#define UNI_MAX_UTF16 (UTF32)0x0010FFFF -#define UNI_MAX_UTF32 (UTF32)0x7FFFFFFF -#define UNI_MAX_LEGAL_UTF32 (UTF32)0x0010FFFF - -typedef enum { - conversionOK, /* conversion successful */ - sourceExhausted, /* partial character in source, but hit end */ - targetExhausted, /* insuff. room in target for conversion */ - sourceIllegal /* source sequence is illegal/malformed */ -} ConversionResult; - -typedef enum { - strictConversion = 0, - lenientConversion -} ConversionFlags; - -// /* This is for C++ and does no harm in C */ -// #ifdef __cplusplus -// extern "C" { -// #endif - -ConversionResult ConvertUTF8toUTF16 ( - const UTF8** sourceStart, const UTF8* sourceEnd, - UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags); - -ConversionResult ConvertUTF16toUTF8 ( - const UTF16** sourceStart, const UTF16* sourceEnd, - UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags); - -ConversionResult ConvertUTF8toUTF32 ( - const UTF8** sourceStart, const UTF8* sourceEnd, - UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags); - -ConversionResult ConvertUTF32toUTF8 ( - const UTF32** sourceStart, const UTF32* sourceEnd, - UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags); - -ConversionResult ConvertUTF16toUTF32 ( - const UTF16** sourceStart, const UTF16* sourceEnd, - UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags); - -ConversionResult ConvertUTF32toUTF16 ( - const UTF32** sourceStart, const UTF32* sourceEnd, - char16_t** targetStart, char16_t* targetEnd, ConversionFlags flags); - -Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd); - -// #ifdef __cplusplus -// } -// #endif - -} - -/* --------------------------------------------------------------------- */ diff --git a/src/linenoise/LICENSE b/src/linenoise/LICENSE deleted file mode 100644 index b7c58c445860..000000000000 --- a/src/linenoise/LICENSE +++ /dev/null @@ -1,66 +0,0 @@ -linenoise.cpp -============= - -Copyright (c) 2010, Salvatore Sanfilippo <antirez at gmail dot com> -Copyright (c) 2010, Pieter Noordhuis <pcnoordhuis at gmail dot com> - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of Redis nor the names of its contributors may be used - to endorse or promote products derived from this software without - specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. - - -wcwidth.cpp -=========== - -Markus Kuhn -- 2007-05-26 (Unicode 5.0) - -Permission to use, copy, modify, and distribute this software -for any purpose and without fee is hereby granted. The author -disclaims all warranties with regard to this software. - - - -ConvertUTF.cpp -============== - -Copyright 2001-2004 Unicode, Inc. - -Disclaimer - -This source code is provided as is by Unicode, Inc. No claims are -made as to fitness for any particular purpose. No warranties of any -kind are expressed or implied. The recipient agrees to determine -applicability of information provided. If this file has been -purchased on magnetic or optical media from Unicode, Inc., the -sole remedy for any claim will be exchange of defective media -within 90 days of receipt. - -Limitations on Rights to Redistribute This Code - -Unicode, Inc. hereby grants the right to freely use the information -supplied in this file in the creation of products supporting the -Unicode Standard, and to make copies of this file in any form -for internal or external distribution as long as this notice -remains attached. diff --git a/src/linenoise/linenoise.cpp b/src/linenoise/linenoise.cpp deleted file mode 100644 index c57505d2fa97..000000000000 --- a/src/linenoise/linenoise.cpp +++ /dev/null @@ -1,3450 +0,0 @@ -/* linenoise.c -- guerrilla line editing library against the idea that a - * line editing lib needs to be 20,000 lines of C code. - * - * Copyright (c) 2010, Salvatore Sanfilippo <antirez at gmail dot com> - * Copyright (c) 2010, Pieter Noordhuis <pcnoordhuis at gmail dot com> - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Redis nor the names of its contributors may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * - * line editing lib needs to be 20,000 lines of C code. - * - * You can find the latest source code at: - * - * http://github.com/antirez/linenoise - * - * Does a number of crazy assumptions that happen to be true in 99.9999% of - * the 2010 UNIX computers around. - * - * References: - * - http://invisible-island.net/xterm/ctlseqs/ctlseqs.html - * - http://www.3waylabs.com/nw/WWW/products/wizcon/vt220.html - * - * Todo list: - * - Switch to gets() if $TERM is something we can't support. - * - Filter bogus Ctrl+<char> combinations. - * - Win32 support - * - * Bloat: - * - Completion? - * - History search like Ctrl+r in readline? - * - * List of escape sequences used by this program, we do everything just - * with three sequences. In order to be so cheap we may have some - * flickering effect with some slow terminal, but the lesser sequences - * the more compatible. - * - * CHA (Cursor Horizontal Absolute) - * Sequence: ESC [ n G - * Effect: moves cursor to column n (1 based) - * - * EL (Erase Line) - * Sequence: ESC [ n K - * Effect: if n is 0 or missing, clear from cursor to end of line - * Effect: if n is 1, clear from beginning of line to cursor - * Effect: if n is 2, clear entire line - * - * CUF (Cursor Forward) - * Sequence: ESC [ n C - * Effect: moves cursor forward of n chars - * - * The following are used to clear the screen: ESC [ H ESC [ 2 J - * This is actually composed of two sequences: - * - * cursorhome - * Sequence: ESC [ H - * Effect: moves the cursor to upper left corner - * - * ED2 (Clear entire screen) - * Sequence: ESC [ 2 J - * Effect: clear the whole screen - * - */ - -#ifdef _WIN32 - -#include <conio.h> -#include <windows.h> -#include <io.h> - -#if defined(_MSC_VER) && _MSC_VER < 1900 -#define snprintf _snprintf // Microsoft headers use underscores in some names -#endif - -#if !defined GNUC -#define strcasecmp _stricmp -#endif - -#define strdup _strdup -#define isatty _isatty -#define write _write -#define STDIN_FILENO 0 - -#else /* _WIN32 */ - -#include <signal.h> -#include <termios.h> -#include <unistd.h> -#include <stdlib.h> -#include <string.h> -#include <sys/types.h> -#include <sys/ioctl.h> -#include <cctype> -#include <wctype.h> - -#endif /* _WIN32 */ - -#include <stdio.h> -#include <errno.h> -#include <fcntl.h> - -#include "linenoise.h" -#include "ConvertUTF.h" - -#include <string> -#include <vector> -#include <memory> - -using std::string; -using std::vector; -using std::unique_ptr; -using namespace linenoise_ng; - -typedef unsigned char char8_t; - -static ConversionResult copyString8to32(char32_t* dst, size_t dstSize, - size_t& dstCount, const char* src) { - const UTF8* sourceStart = reinterpret_cast<const UTF8*>(src); - const UTF8* sourceEnd = sourceStart + strlen(src); - UTF32* targetStart = reinterpret_cast<UTF32*>(dst); - UTF32* targetEnd = targetStart + dstSize; - - ConversionResult res = ConvertUTF8toUTF32( - &sourceStart, sourceEnd, &targetStart, targetEnd, lenientConversion); - - if (res == conversionOK) { - dstCount = targetStart - reinterpret_cast<UTF32*>(dst); - - if (dstCount < dstSize) { - *targetStart = 0; - } - } - - return res; -} - -static ConversionResult copyString8to32(char32_t* dst, size_t dstSize, - size_t& dstCount, const char8_t* src) { - return copyString8to32(dst, dstSize, dstCount, - reinterpret_cast<const char*>(src)); -} - -static size_t strlen32(const char32_t* str) { - const char32_t* ptr = str; - - while (*ptr) { - ++ptr; - } - - return ptr - str; -} - -static size_t strlen8(const char8_t* str) { - return strlen(reinterpret_cast<const char*>(str)); -} - -static char8_t* strdup8(const char* src) { - return reinterpret_cast<char8_t*>(strdup(src)); -} - -#ifdef _WIN32 -static const int FOREGROUND_WHITE = - FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE; -static const int BACKGROUND_WHITE = - BACKGROUND_RED | BACKGROUND_GREEN | BACKGROUND_BLUE; -static const int INTENSITY = FOREGROUND_INTENSITY | BACKGROUND_INTENSITY; - -class WinAttributes { - public: - WinAttributes() { - CONSOLE_SCREEN_BUFFER_INFO info; - GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &info); - _defaultAttribute = info.wAttributes & INTENSITY; - _defaultColor = info.wAttributes & FOREGROUND_WHITE; - _defaultBackground = info.wAttributes & BACKGROUND_WHITE; - - _consoleAttribute = _defaultAttribute; - _consoleColor = _defaultColor | _defaultBackground; - } - - public: - int _defaultAttribute; - int _defaultColor; - int _defaultBackground; - - int _consoleAttribute; - int _consoleColor; -}; - -static WinAttributes WIN_ATTR; - -static void copyString32to16(char16_t* dst, size_t dstSize, size_t* dstCount, - const char32_t* src, size_t srcSize) { - const UTF32* sourceStart = reinterpret_cast<const UTF32*>(src); - const UTF32* sourceEnd = sourceStart + srcSize; - char16_t* targetStart = reinterpret_cast<char16_t*>(dst); - char16_t* targetEnd = targetStart + dstSize; - - ConversionResult res = ConvertUTF32toUTF16( - &sourceStart, sourceEnd, &targetStart, targetEnd, lenientConversion); - - if (res == conversionOK) { - *dstCount = targetStart - reinterpret_cast<char16_t*>(dst); - - if (*dstCount < dstSize) { - *targetStart = 0; - } - } -} -#endif - -static void copyString32to8(char* dst, size_t dstSize, size_t* dstCount, - const char32_t* src, size_t srcSize) { - const UTF32* sourceStart = reinterpret_cast<const UTF32*>(src); - const UTF32* sourceEnd = sourceStart + srcSize; - UTF8* targetStart = reinterpret_cast<UTF8*>(dst); - UTF8* targetEnd = targetStart + dstSize; - - ConversionResult res = ConvertUTF32toUTF8( - &sourceStart, sourceEnd, &targetStart, targetEnd, lenientConversion); - - if (res == conversionOK) { - *dstCount = targetStart - reinterpret_cast<UTF8*>(dst); - - if (*dstCount < dstSize) { - *targetStart = 0; - } - } -} - -static void copyString32to8(char* dst, size_t dstLen, const char32_t* src) { - size_t dstCount = 0; - copyString32to8(dst, dstLen, &dstCount, src, strlen32(src)); -} - -static void copyString32(char32_t* dst, const char32_t* src, size_t len) { - while (0 < len && *src) { - *dst++ = *src++; - --len; - } - - *dst = 0; -} - -static int strncmp32(const char32_t* left, const char32_t* right, size_t len) { - while (0 < len && *left) { - if (*left != *right) { - return *left - *right; - } - - ++left; - ++right; - --len; - } - - return 0; -} - -#ifdef _WIN32 -#include <iostream> - -static size_t OutputWin(char16_t* text16, char32_t* text32, size_t len32) { - size_t count16 = 0; - - copyString32to16(text16, len32, &count16, text32, len32); - WriteConsoleW(GetStdHandle(STD_OUTPUT_HANDLE), text16, - static_cast<DWORD>(count16), nullptr, nullptr); - - return count16; -} - -static char32_t* HandleEsc(char32_t* p, char32_t* end) { - if (*p == '[') { - int code = 0; - - for (++p; p < end; ++p) { - char32_t c = *p; - - if ('0' <= c && c <= '9') { - code = code * 10 + (c - '0'); - } else if (c == 'm' || c == ';') { - switch (code) { - case 0: - WIN_ATTR._consoleAttribute = WIN_ATTR._defaultAttribute; - WIN_ATTR._consoleColor = - WIN_ATTR._defaultColor | WIN_ATTR._defaultBackground; - break; - - case 1: // BOLD - case 5: // BLINK - WIN_ATTR._consoleAttribute = - (WIN_ATTR._defaultAttribute ^ FOREGROUND_INTENSITY) & INTENSITY; - break; - - case 30: - WIN_ATTR._consoleColor = BACKGROUND_WHITE; - break; - - case 31: - WIN_ATTR._consoleColor = - FOREGROUND_RED | WIN_ATTR._defaultBackground; - break; - - case 32: - WIN_ATTR._consoleColor = - FOREGROUND_GREEN | WIN_ATTR._defaultBackground; - break; - - case 33: - WIN_ATTR._consoleColor = - FOREGROUND_RED | FOREGROUND_GREEN | WIN_ATTR._defaultBackground; - break; - - case 34: - WIN_ATTR._consoleColor = - FOREGROUND_BLUE | WIN_ATTR._defaultBackground; - break; - - case 35: - WIN_ATTR._consoleColor = - FOREGROUND_BLUE | FOREGROUND_RED | WIN_ATTR._defaultBackground; - break; - - case 36: - WIN_ATTR._consoleColor = FOREGROUND_BLUE | FOREGROUND_GREEN | - WIN_ATTR._defaultBackground; - break; - - case 37: - WIN_ATTR._consoleColor = FOREGROUND_GREEN | FOREGROUND_RED | - FOREGROUND_BLUE | - WIN_ATTR._defaultBackground; - break; - } - - code = 0; - } - - if (*p == 'm') { - ++p; - break; - } - } - } else { - ++p; - } - - auto handle = GetStdHandle(STD_OUTPUT_HANDLE); - SetConsoleTextAttribute(handle, - WIN_ATTR._consoleAttribute | WIN_ATTR._consoleColor); - - return p; -} - -static size_t WinWrite32(char16_t* text16, char32_t* text32, size_t len32) { - char32_t* p = text32; - char32_t* q = p; - char32_t* e = text32 + len32; - size_t count16 = 0; - - while (p < e) { - if (*p == 27) { - if (q < p) { - count16 += OutputWin(text16, q, p - q); - } - - q = p = HandleEsc(p + 1, e); - } else { - ++p; - } - } - - if (q < p) { - count16 += OutputWin(text16, q, p - q); - } - - return count16; -} -#endif - -static int write32(int fd, char32_t* text32, int len32) { -#ifdef _WIN32 - if (isatty(fd)) { - size_t len16 = 2 * len32 + 1; - unique_ptr<char16_t[]> text16(new char16_t[len16]); - size_t count16 = WinWrite32(text16.get(), text32, len32); - - return static_cast<int>(count16); - } else { - size_t len8 = 4 * len32 + 1; - unique_ptr<char[]> text8(new char[len8]); - size_t count8 = 0; - - copyString32to8(text8.get(), len8, &count8, text32, len32); - - return write(fd, text8.get(), static_cast<unsigned int>(count8)); - } -#else - size_t len8 = 4 * len32 + 1; - unique_ptr<char[]> text8(new char[len8]); - size_t count8 = 0; - - copyString32to8(text8.get(), len8, &count8, text32, len32); - - return write(fd, text8.get(), count8); -#endif -} - -class Utf32String { - public: - Utf32String() : _length(0), _data(nullptr) { - // note: parens intentional, _data must be properly initialized - _data = new char32_t[1](); - } - - explicit Utf32String(const char* src) : _length(0), _data(nullptr) { - size_t len = strlen(src); - // note: parens intentional, _data must be properly initialized - _data = new char32_t[len + 1](); - copyString8to32(_data, len + 1, _length, src); - } - - explicit Utf32String(const char8_t* src) : _length(0), _data(nullptr) { - size_t len = strlen(reinterpret_cast<const char*>(src)); - // note: parens intentional, _data must be properly initialized - _data = new char32_t[len + 1](); - copyString8to32(_data, len + 1, _length, src); - } - - explicit Utf32String(const char32_t* src) : _length(0), _data(nullptr) { - for (_length = 0; src[_length] != 0; ++_length) { - } - - // note: parens intentional, _data must be properly initialized - _data = new char32_t[_length + 1](); - memcpy(_data, src, _length * sizeof(char32_t)); - } - - explicit Utf32String(const char32_t* src, int len) : _length(len), _data(nullptr) { - // note: parens intentional, _data must be properly initialized - _data = new char32_t[len + 1](); - memcpy(_data, src, len * sizeof(char32_t)); - } - - explicit Utf32String(int len) : _length(0), _data(nullptr) { - // note: parens intentional, _data must be properly initialized - _data = new char32_t[len](); - } - - explicit Utf32String(const Utf32String& that) : _length(that._length), _data(nullptr) { - // note: parens intentional, _data must be properly initialized - _data = new char32_t[_length + 1](); - memcpy(_data, that._data, sizeof(char32_t) * _length); - } - - Utf32String& operator=(const Utf32String& that) { - if (this != &that) { - delete[] _data; - _data = new char32_t[that._length](); - _length = that._length; - memcpy(_data, that._data, sizeof(char32_t) * _length); - } - - return *this; - } - - ~Utf32String() { delete[] _data; } - - public: - char32_t* get() const { return _data; } - - size_t length() const { return _length; } - - size_t chars() const { return _length; } - - void initFromBuffer() { - for (_length = 0; _data[_length] != 0; ++_length) { - } - } - - const char32_t& operator[](size_t pos) const { return _data[pos]; } - - char32_t& operator[](size_t pos) { return _data[pos]; } - - private: - size_t _length; - char32_t* _data; -}; - -class Utf8String { - Utf8String(const Utf8String&) = delete; - Utf8String& operator=(const Utf8String&) = delete; - - public: - explicit Utf8String(const Utf32String& src) { - size_t len = src.length() * 4 + 1; - _data = new char[len]; - copyString32to8(_data, len, src.get()); - } - - ~Utf8String() { delete[] _data; } - - public: - char* get() const { return _data; } - - private: - char* _data; -}; - -struct linenoiseCompletions { - vector<Utf32String> completionStrings; -}; - -#define LINENOISE_DEFAULT_HISTORY_MAX_LEN 100 -#define LINENOISE_MAX_LINE 4096 - -// make control-characters more readable -#define ctrlChar(upperCaseASCII) (upperCaseASCII - 0x40) - -/** - * Recompute widths of all characters in a char32_t buffer - * @param text input buffer of Unicode characters - * @param widths output buffer of character widths - * @param charCount number of characters in buffer - */ -namespace linenoise_ng { -int mk_wcwidth(char32_t ucs); -} - -static void recomputeCharacterWidths(const char32_t* text, char* widths, - int charCount) { - for (int i = 0; i < charCount; ++i) { - widths[i] = mk_wcwidth(text[i]); - } -} - -/** - * Calculate a new screen position given a starting position, screen width and - * character count - * @param x initial x position (zero-based) - * @param y initial y position (zero-based) - * @param screenColumns screen column count - * @param charCount character positions to advance - * @param xOut returned x position (zero-based) - * @param yOut returned y position (zero-based) - */ -static void calculateScreenPosition(int x, int y, int screenColumns, - int charCount, int& xOut, int& yOut) { - xOut = x; - yOut = y; - int charsRemaining = charCount; - while (charsRemaining > 0) { - int charsThisRow = (x + charsRemaining < screenColumns) ? charsRemaining - : screenColumns - x; - xOut = x + charsThisRow; - yOut = y; - charsRemaining -= charsThisRow; - x = 0; - ++y; - } - if (xOut == screenColumns) { // we have to special-case line wrap - xOut = 0; - ++yOut; - } -} - -/** - * Calculate a column width using mk_wcswidth() - * @param buf32 text to calculate - * @param len length of text to calculate - */ -namespace linenoise_ng { -int mk_wcswidth(const char32_t* pwcs, size_t n); -} - -static int calculateColumnPosition(char32_t* buf32, int len) { - int width = mk_wcswidth(reinterpret_cast<const char32_t*>(buf32), len); - if (width == -1) - return len; - else - return width; -} - -static bool isControlChar(char32_t testChar) { - return (testChar < ' ') || // C0 controls - (testChar >= 0x7F && testChar <= 0x9F); // DEL and C1 controls -} - -struct PromptBase { // a convenience struct for grouping prompt info - Utf32String promptText; // our copy of the prompt text, edited - char* promptCharWidths; // character widths from mk_wcwidth() - int promptChars; // chars in promptText - int promptBytes; // bytes in promptText - int promptExtraLines; // extra lines (beyond 1) occupied by prompt - int promptIndentation; // column offset to end of prompt - int promptLastLinePosition; // index into promptText where last line begins - int promptPreviousInputLen; // promptChars of previous input line, for - // clearing - int promptCursorRowOffset; // where the cursor is relative to the start of - // the prompt - int promptScreenColumns; // width of screen in columns - int promptPreviousLen; // help erasing - int promptErrorCode; // error code (invalid UTF-8) or zero - - PromptBase() : promptPreviousInputLen(0) {} - - bool write() { - if (write32(1, promptText.get(), promptBytes) == -1) return false; - - return true; - } -}; - -struct PromptInfo : public PromptBase { - PromptInfo(const char* textPtr, int columns) { - promptExtraLines = 0; - promptLastLinePosition = 0; - promptPreviousLen = 0; - promptScreenColumns = columns; - Utf32String tempUnicode(textPtr); - - // strip control characters from the prompt -- we do allow newline - char32_t* pIn = tempUnicode.get(); - char32_t* pOut = pIn; - - int len = 0; - int x = 0; - - bool const strip = (isatty(1) == 0); - - while (*pIn) { - char32_t c = *pIn; - if ('\n' == c || !isControlChar(c)) { - *pOut = c; - ++pOut; - ++pIn; - ++len; - if ('\n' == c || ++x >= promptScreenColumns) { - x = 0; - ++promptExtraLines; - promptLastLinePosition = len; - } - } else if (c == '\x1b') { - if (strip) { - // jump over control chars - ++pIn; - if (*pIn == '[') { - ++pIn; - while (*pIn && ((*pIn == ';') || ((*pIn >= '0' && *pIn <= '9')))) { - ++pIn; - } - if (*pIn == 'm') { - ++pIn; - } - } - } else { - // copy control chars - *pOut = *pIn; - ++pOut; - ++pIn; - if (*pIn == '[') { - *pOut = *pIn; - ++pOut; - ++pIn; - while (*pIn && ((*pIn == ';') || ((*pIn >= '0' && *pIn <= '9')))) { - *pOut = *pIn; - ++pOut; - ++pIn; - } - if (*pIn == 'm') { - *pOut = *pIn; - ++pOut; - ++pIn; - } - } - } - } else { - ++pIn; - } - } - *pOut = 0; - promptChars = len; - promptBytes = static_cast<int>(pOut - tempUnicode.get()); - promptText = tempUnicode; - - promptIndentation = len - promptLastLinePosition; - promptCursorRowOffset = promptExtraLines; - } -}; - -// Used with DynamicPrompt (history search) -// -static const Utf32String forwardSearchBasePrompt("(i-search)`"); -static const Utf32String reverseSearchBasePrompt("(reverse-i-search)`"); -static const Utf32String endSearchBasePrompt("': "); -static Utf32String - previousSearchText; // remembered across invocations of linenoise() - -// changing prompt for "(reverse-i-search)`text':" etc. -// -struct DynamicPrompt : public PromptBase { - Utf32String searchText; // text we are searching for - char* searchCharWidths; // character widths from mk_wcwidth() - int searchTextLen; // chars in searchText - int direction; // current search direction, 1=forward, -1=reverse - - DynamicPrompt(PromptBase& pi, int initialDirection) - : searchTextLen(0), direction(initialDirection) { - promptScreenColumns = pi.promptScreenColumns; - promptCursorRowOffset = 0; - Utf32String emptyString(1); - searchText = emptyString; - const Utf32String* basePrompt = - (direction > 0) ? &forwardSearchBasePrompt : &reverseSearchBasePrompt; - size_t promptStartLength = basePrompt->length(); - promptChars = - static_cast<int>(promptStartLength + endSearchBasePrompt.length()); - promptBytes = promptChars; - promptLastLinePosition = promptChars; // TODO fix this, we are asssuming - // that the history prompt won't wrap - // (!) - promptPreviousLen = promptChars; - Utf32String tempUnicode(promptChars + 1); - memcpy(tempUnicode.get(), basePrompt->get(), - sizeof(char32_t) * promptStartLength); - memcpy(&tempUnicode[promptStartLength], endSearchBasePrompt.get(), - sizeof(char32_t) * (endSearchBasePrompt.length() + 1)); - tempUnicode.initFromBuffer(); - promptText = tempUnicode; - calculateScreenPosition(0, 0, pi.promptScreenColumns, promptChars, - promptIndentation, promptExtraLines); - } - - void updateSearchPrompt(void) { - const Utf32String* basePrompt = - (direction > 0) ? &forwardSearchBasePrompt : &reverseSearchBasePrompt; - size_t promptStartLength = basePrompt->length(); - promptChars = static_cast<int>(promptStartLength + searchTextLen + - endSearchBasePrompt.length()); - promptBytes = promptChars; - Utf32String tempUnicode(promptChars + 1); - memcpy(tempUnicode.get(), basePrompt->get(), - sizeof(char32_t) * promptStartLength); - memcpy(&tempUnicode[promptStartLength], searchText.get(), - sizeof(char32_t) * searchTextLen); - size_t endIndex = promptStartLength + searchTextLen; - memcpy(&tempUnicode[endIndex], endSearchBasePrompt.get(), - sizeof(char32_t) * (endSearchBasePrompt.length() + 1)); - tempUnicode.initFromBuffer(); - promptText = tempUnicode; - } - - void updateSearchText(const char32_t* textPtr) { - Utf32String tempUnicode(textPtr); - searchTextLen = static_cast<int>(tempUnicode.chars()); - searchText = tempUnicode; - updateSearchPrompt(); - } -}; - -class KillRing { - static const int capacity = 10; - int size; - int index; - char indexToSlot[10]; - vector<Utf32String> theRing; - - public: - enum action { actionOther, actionKill, actionYank }; - action lastAction; - size_t lastYankSize; - - KillRing() : size(0), index(0), lastAction(actionOther) { - theRing.reserve(capacity); - } - - void kill(const char32_t* text, int textLen, bool forward) { - if (textLen == 0) { - return; - } - Utf32String killedText(text, textLen); - if (lastAction == actionKill && size > 0) { - int slot = indexToSlot[0]; - int currentLen = static_cast<int>(theRing[slot].length()); - int resultLen = currentLen + textLen; - Utf32String temp(resultLen + 1); - if (forward) { - memcpy(temp.get(), theRing[slot].get(), currentLen * sizeof(char32_t)); - memcpy(&temp[currentLen], killedText.get(), textLen * sizeof(char32_t)); - } else { - memcpy(temp.get(), killedText.get(), textLen * sizeof(char32_t)); - memcpy(&temp[textLen], theRing[slot].get(), - currentLen * sizeof(char32_t)); - } - temp[resultLen] = 0; - temp.initFromBuffer(); - theRing[slot] = temp; - } else { - if (size < capacity) { - if (size > 0) { - memmove(&indexToSlot[1], &indexToSlot[0], size); - } - indexToSlot[0] = size; - size++; - theRing.push_back(killedText); - } else { - int slot = indexToSlot[capacity - 1]; - theRing[slot] = killedText; - memmove(&indexToSlot[1], &indexToSlot[0], capacity - 1); - indexToSlot[0] = slot; - } - index = 0; - } - } - - Utf32String* yank() { return (size > 0) ? &theRing[indexToSlot[index]] : 0; } - - Utf32String* yankPop() { - if (size == 0) { - return 0; - } - ++index; - if (index == size) { - index = 0; - } - return &theRing[indexToSlot[index]]; - } -}; - -class InputBuffer { - char32_t* buf32; // input buffer - char* charWidths; // character widths from mk_wcwidth() - int buflen; // buffer size in characters - int len; // length of text in input buffer - int pos; // character position in buffer ( 0 <= pos <= len ) - - void clearScreen(PromptBase& pi); - int incrementalHistorySearch(PromptBase& pi, int startChar); - int completeLine(PromptBase& pi); - void refreshLine(PromptBase& pi); - - public: - InputBuffer(char32_t* buffer, char* widthArray, int bufferLen) - : buf32(buffer), - charWidths(widthArray), - buflen(bufferLen - 1), - len(0), - pos(0) { - buf32[0] = 0; - } - void preloadBuffer(const char* preloadText) { - size_t ucharCount = 0; - copyString8to32(buf32, buflen + 1, ucharCount, preloadText); - recomputeCharacterWidths(buf32, charWidths, static_cast<int>(ucharCount)); - len = static_cast<int>(ucharCount); - pos = static_cast<int>(ucharCount); - } - int getInputLine(PromptBase& pi); - int length(void) const { return len; } -}; - -// Special codes for keyboard input: -// -// Between Windows and the various Linux "terminal" programs, there is some -// pretty diverse behavior in the "scan codes" and escape sequences we are -// presented with. So ... we'll translate them all into our own pidgin -// pseudocode, trying to stay out of the way of UTF-8 and international -// characters. Here's the general plan. -// -// "User input keystrokes" (key chords, whatever) will be encoded as a single -// value. -// The low 21 bits are reserved for Unicode characters. Popular function-type -// keys -// get their own codes in the range 0x10200000 to (if needed) 0x1FE00000, -// currently -// just arrow keys, Home, End and Delete. Keypresses with Ctrl get ORed with -// 0x20000000, with Alt get ORed with 0x40000000. So, Ctrl+Alt+Home is encoded -// as 0x20000000 + 0x40000000 + 0x10A00000 == 0x70A00000. To keep things -// complicated, -// the Alt key is equivalent to prefixing the keystroke with ESC, so ESC -// followed by -// D is treated the same as Alt + D ... we'll just use Emacs terminology and -// call -// this "Meta". So, we will encode both ESC followed by D and Alt held down -// while D -// is pressed the same, as Meta-D, encoded as 0x40000064. -// -// Here are the definitions of our component constants: -// -// Maximum unsigned 32-bit value = 0xFFFFFFFF; // For reference, max 32-bit -// value -// Highest allocated Unicode char = 0x001FFFFF; // For reference, max -// Unicode value -static const int META = 0x40000000; // Meta key combination -static const int CTRL = 0x20000000; // Ctrl key combination -// static const int SPECIAL_KEY = 0x10000000; // Common bit for all special -// keys -static const int UP_ARROW_KEY = 0x10200000; // Special keys -static const int DOWN_ARROW_KEY = 0x10400000; -static const int RIGHT_ARROW_KEY = 0x10600000; -static const int LEFT_ARROW_KEY = 0x10800000; -static const int HOME_KEY = 0x10A00000; -static const int END_KEY = 0x10C00000; -static const int DELETE_KEY = 0x10E00000; -static const int PAGE_UP_KEY = 0x11000000; -static const int PAGE_DOWN_KEY = 0x11200000; - -static const char* unsupported_term[] = {"dumb", "cons25", "emacs", NULL}; -static linenoiseCompletionCallback* completionCallback = NULL; - -#ifdef _WIN32 -static HANDLE console_in, console_out; -static DWORD oldMode; -static WORD oldDisplayAttribute; -#else -static struct termios orig_termios; /* in order to restore at exit */ -#endif - -static KillRing killRing; - -static int rawmode = 0; /* for atexit() function to check if restore is needed*/ -static int atexit_registered = 0; /* register atexit just 1 time */ -static int historyMaxLen = LINENOISE_DEFAULT_HISTORY_MAX_LEN; -static int historyLen = 0; -static int historyIndex = 0; -static char8_t** history = NULL; - -// used to emulate Windows command prompt on down-arrow after a recall -// we use -2 as our "not set" value because we add 1 to the previous index on -// down-arrow, -// and zero is a valid index (so -1 is a valid "previous index") -static int historyPreviousIndex = -2; -static bool historyRecallMostRecent = false; - -static void linenoiseAtExit(void); - -static bool isUnsupportedTerm(void) { - char* term = getenv("TERM"); - if (term == NULL) return false; - for (int j = 0; unsupported_term[j]; ++j) - if (!strcasecmp(term, unsupported_term[j])) { - return true; - } - return false; -} - -static void beep() { - fprintf(stderr, "\x7"); // ctrl-G == bell/beep - fflush(stderr); -} - -void linenoiseHistoryFree(void) { - if (history) { - for (int j = 0; j < historyLen; ++j) free(history[j]); - historyLen = 0; - free(history); - history = 0; - } -} - -static int enableRawMode(void) { -#ifdef _WIN32 - if (!console_in) { - console_in = GetStdHandle(STD_INPUT_HANDLE); - console_out = GetStdHandle(STD_OUTPUT_HANDLE); - - GetConsoleMode(console_in, &oldMode); - SetConsoleMode(console_in, oldMode & - ~(ENABLE_LINE_INPUT | ENABLE_ECHO_INPUT | - ENABLE_PROCESSED_INPUT)); - } - return 0; -#else - struct termios raw; - - if (!isatty(STDIN_FILENO)) goto fatal; - if (!atexit_registered) { - atexit(linenoiseAtExit); - atexit_registered = 1; - } - if (tcgetattr(0, &orig_termios) == -1) goto fatal; - - raw = orig_termios; /* modify the original mode */ - /* input modes: no break, no CR to NL, no parity check, no strip char, - * no start/stop output control. */ - raw.c_iflag &= ~(BRKINT | ICRNL | INPCK | ISTRIP | IXON); - /* output modes - disable post processing */ - // this is wrong, we don't want raw output, it turns newlines into straight - // linefeeds - // raw.c_oflag &= ~(OPOST); - /* control modes - set 8 bit chars */ - raw.c_cflag |= (CS8); - /* local modes - echoing off, canonical off, no extended functions, - * no signal chars (^Z,^C) */ - raw.c_lflag &= ~(ECHO | ICANON | IEXTEN | ISIG); - /* control chars - set return condition: min number of bytes and timer. - * We want read to return every single byte, without timeout. */ - raw.c_cc[VMIN] = 1; - raw.c_cc[VTIME] = 0; /* 1 byte, no timer */ - - /* put terminal in raw mode after flushing */ - if (tcsetattr(0, TCSADRAIN, &raw) < 0) goto fatal; - rawmode = 1; - return 0; - -fatal: - errno = ENOTTY; - return -1; -#endif -} - -static void disableRawMode(void) { -#ifdef _WIN32 - SetConsoleMode(console_in, oldMode); - console_in = 0; - console_out = 0; -#else - if (rawmode && tcsetattr(0, TCSADRAIN, &orig_termios) != -1) rawmode = 0; -#endif -} - -// At exit we'll try to fix the terminal to the initial conditions -static void linenoiseAtExit(void) { disableRawMode(); } - -static int getScreenColumns(void) { - int cols; -#ifdef _WIN32 - CONSOLE_SCREEN_BUFFER_INFO inf; - GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &inf); - cols = inf.dwSize.X; -#else - struct winsize ws; - cols = (ioctl(1, TIOCGWINSZ, &ws) == -1) ? 80 : ws.ws_col; -#endif - // cols is 0 in certain circumstances like inside debugger, which creates - // further issues - return (cols > 0) ? cols : 80; -} - -static int getScreenRows(void) { - int rows; -#ifdef _WIN32 - CONSOLE_SCREEN_BUFFER_INFO inf; - GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &inf); - rows = 1 + inf.srWindow.Bottom - inf.srWindow.Top; -#else - struct winsize ws; - rows = (ioctl(1, TIOCGWINSZ, &ws) == -1) ? 24 : ws.ws_row; -#endif - return (rows > 0) ? rows : 24; -} - -static void setDisplayAttribute(bool enhancedDisplay, bool error) { -#ifdef _WIN32 - if (enhancedDisplay) { - CONSOLE_SCREEN_BUFFER_INFO inf; - GetConsoleScreenBufferInfo(console_out, &inf); - oldDisplayAttribute = inf.wAttributes; - BYTE oldLowByte = oldDisplayAttribute & 0xFF; - BYTE newLowByte; - switch (oldLowByte) { - case 0x07: - // newLowByte = FOREGROUND_BLUE | FOREGROUND_INTENSITY; // too dim - // newLowByte = FOREGROUND_BLUE; // even dimmer - newLowByte = FOREGROUND_BLUE | - FOREGROUND_GREEN; // most similar to xterm appearance - break; - case 0x70: - newLowByte = BACKGROUND_BLUE | BACKGROUND_INTENSITY; - break; - default: - newLowByte = oldLowByte ^ 0xFF; // default to inverse video - break; - } - inf.wAttributes = (inf.wAttributes & 0xFF00) | newLowByte; - SetConsoleTextAttribute(console_out, inf.wAttributes); - } else { - SetConsoleTextAttribute(console_out, oldDisplayAttribute); - } -#else - if (enhancedDisplay) { - char const* p = (error ? "\x1b[1;31m" : "\x1b[1;34m"); - if (write(1, p, 7) == -1) - return; /* bright blue (visible with both B&W bg) */ - } else { - if (write(1, "\x1b[0m", 4) == -1) return; /* reset */ - } -#endif -} - -/** - * Display the dynamic incremental search prompt and the current user input - * line. - * @param pi PromptBase struct holding information about the prompt and our - * screen position - * @param buf32 input buffer to be displayed - * @param len count of characters in the buffer - * @param pos current cursor position within the buffer (0 <= pos <= len) - */ -static void dynamicRefresh(PromptBase& pi, char32_t* buf32, int len, int pos) { - // calculate the position of the end of the prompt - int xEndOfPrompt, yEndOfPrompt; - calculateScreenPosition(0, 0, pi.promptScreenColumns, pi.promptChars, - xEndOfPrompt, yEndOfPrompt); - pi.promptIndentation = xEndOfPrompt; - - // calculate the position of the end of the input line - int xEndOfInput, yEndOfInput; - calculateScreenPosition(xEndOfPrompt, yEndOfPrompt, pi.promptScreenColumns, - calculateColumnPosition(buf32, len), xEndOfInput, - yEndOfInput); - - // calculate the desired position of the cursor - int xCursorPos, yCursorPos; - calculateScreenPosition(xEndOfPrompt, yEndOfPrompt, pi.promptScreenColumns, - calculateColumnPosition(buf32, pos), xCursorPos, - yCursorPos); - -#ifdef _WIN32 - // position at the start of the prompt, clear to end of previous input - CONSOLE_SCREEN_BUFFER_INFO inf; - GetConsoleScreenBufferInfo(console_out, &inf); - inf.dwCursorPosition.X = 0; - inf.dwCursorPosition.Y -= pi.promptCursorRowOffset /*- pi.promptExtraLines*/; - SetConsoleCursorPosition(console_out, inf.dwCursorPosition); - DWORD count; - FillConsoleOutputCharacterA(console_out, ' ', - pi.promptPreviousLen + pi.promptPreviousInputLen, - inf.dwCursorPosition, &count); - pi.promptPreviousLen = pi.promptIndentation; - pi.promptPreviousInputLen = len; - - // display the prompt - if (!pi.write()) return; - - // display the input line - if (write32(1, buf32, len) == -1) return; - - // position the cursor - GetConsoleScreenBufferInfo(console_out, &inf); - inf.dwCursorPosition.X = xCursorPos; // 0-based on Win32 - inf.dwCursorPosition.Y -= yEndOfInput - yCursorPos; - SetConsoleCursorPosition(console_out, inf.dwCursorPosition); -#else // _WIN32 - char seq[64]; - int cursorRowMovement = pi.promptCursorRowOffset - pi.promptExtraLines; - if (cursorRowMovement > 0) { // move the cursor up as required - snprintf(seq, sizeof seq, "\x1b[%dA", cursorRowMovement); - if (write(1, seq, strlen(seq)) == -1) return; - } - // position at the start of the prompt, clear to end of screen - snprintf(seq, sizeof seq, "\x1b[1G\x1b[J"); // 1-based on VT100 - if (write(1, seq, strlen(seq)) == -1) return; - - // display the prompt - if (!pi.write()) return; - - // display the input line - if (write32(1, buf32, len) == -1) return; - - // we have to generate our own newline on line wrap - if (xEndOfInput == 0 && yEndOfInput > 0) - if (write(1, "\n", 1) == -1) return; - - // position the cursor - cursorRowMovement = yEndOfInput - yCursorPos; - if (cursorRowMovement > 0) { // move the cursor up as required - snprintf(seq, sizeof seq, "\x1b[%dA", cursorRowMovement); - if (write(1, seq, strlen(seq)) == -1) return; - } - // position the cursor within the line - snprintf(seq, sizeof seq, "\x1b[%dG", xCursorPos + 1); // 1-based on VT100 - if (write(1, seq, strlen(seq)) == -1) return; -#endif - - pi.promptCursorRowOffset = - pi.promptExtraLines + yCursorPos; // remember row for next pass -} - -/** - * Refresh the user's input line: the prompt is already onscreen and is not - * redrawn here - * @param pi PromptBase struct holding information about the prompt and our - * screen position - */ -void InputBuffer::refreshLine(PromptBase& pi) { - // check for a matching brace/bracket/paren, remember its position if found - int highlight = -1; - bool indicateError = false; - if (pos < len) { - /* this scans for a brace matching buf32[pos] to highlight */ - unsigned char part1, part2; - int scanDirection = 0; - if (strchr("}])", buf32[pos])) { - scanDirection = -1; /* backwards */ - if (buf32[pos] == '}') { - part1 = '}'; part2 = '{'; - } else if (buf32[pos] == ']') { - part1 = ']'; part2 = '['; - } else { - part1 = ')'; part2 = '('; - } - } - else if (strchr("{[(", buf32[pos])) { - scanDirection = 1; /* forwards */ - if (buf32[pos] == '{') { - //part1 = '{'; part2 = '}'; - part1 = '}'; part2 = '{'; - } else if (buf32[pos] == '[') { - //part1 = '['; part2 = ']'; - part1 = ']'; part2 = '['; - } else { - //part1 = '('; part2 = ')'; - part1 = ')'; part2 = '('; - } - } - - if (scanDirection) { - int unmatched = scanDirection; - int unmatchedOther = 0; - for (int i = pos + scanDirection; i >= 0 && i < len; i += scanDirection) { - /* TODO: the right thing when inside a string */ - if (strchr("}])", buf32[i])) { - if (buf32[i] == part1) { - --unmatched; - } else { - --unmatchedOther; - } - } else if (strchr("{[(", buf32[i])) { - if (buf32[i] == part2) { - ++unmatched; - } else { - ++unmatchedOther; - } - } -/* - if (strchr("}])", buf32[i])) - --unmatched; - else if (strchr("{[(", buf32[i])) - ++unmatched; -*/ - if (unmatched == 0) { - highlight = i; - indicateError = (unmatchedOther != 0); - break; - } - } - } - } - - // calculate the position of the end of the input line - int xEndOfInput, yEndOfInput; - calculateScreenPosition(pi.promptIndentation, 0, pi.promptScreenColumns, - calculateColumnPosition(buf32, len), xEndOfInput, - yEndOfInput); - - // calculate the desired position of the cursor - int xCursorPos, yCursorPos; - calculateScreenPosition(pi.promptIndentation, 0, pi.promptScreenColumns, - calculateColumnPosition(buf32, pos), xCursorPos, - yCursorPos); - -#ifdef _WIN32 - // position at the end of the prompt, clear to end of previous input - CONSOLE_SCREEN_BUFFER_INFO inf; - GetConsoleScreenBufferInfo(console_out, &inf); - inf.dwCursorPosition.X = pi.promptIndentation; // 0-based on Win32 - inf.dwCursorPosition.Y -= pi.promptCursorRowOffset - pi.promptExtraLines; - SetConsoleCursorPosition(console_out, inf.dwCursorPosition); - DWORD count; - if (len < pi.promptPreviousInputLen) - FillConsoleOutputCharacterA(console_out, ' ', pi.promptPreviousInputLen, - inf.dwCursorPosition, &count); - pi.promptPreviousInputLen = len; - - // display the input line - if (highlight == -1) { - if (write32(1, buf32, len) == -1) return; - } else { - if (write32(1, buf32, highlight) == -1) return; - setDisplayAttribute(true, indicateError); /* bright blue (visible with both B&W bg) */ - if (write32(1, &buf32[highlight], 1) == -1) return; - setDisplayAttribute(false, indicateError); - if (write32(1, buf32 + highlight + 1, len - highlight - 1) == -1) return; - } - - // position the cursor - GetConsoleScreenBufferInfo(console_out, &inf); - inf.dwCursorPosition.X = xCursorPos; // 0-based on Win32 - inf.dwCursorPosition.Y -= yEndOfInput - yCursorPos; - SetConsoleCursorPosition(console_out, inf.dwCursorPosition); -#else // _WIN32 - char seq[64]; - int cursorRowMovement = pi.promptCursorRowOffset - pi.promptExtraLines; - if (cursorRowMovement > 0) { // move the cursor up as required - snprintf(seq, sizeof seq, "\x1b[%dA", cursorRowMovement); - if (write(1, seq, strlen(seq)) == -1) return; - } - // position at the end of the prompt, clear to end of screen - snprintf(seq, sizeof seq, "\x1b[%dG\x1b[J", - pi.promptIndentation + 1); // 1-based on VT100 - if (write(1, seq, strlen(seq)) == -1) return; - - if (highlight == -1) { // write unhighlighted text - if (write32(1, buf32, len) == -1) return; - } else { // highlight the matching brace/bracket/parenthesis - if (write32(1, buf32, highlight) == -1) return; - setDisplayAttribute(true, indicateError); - if (write32(1, &buf32[highlight], 1) == -1) return; - setDisplayAttribute(false, indicateError); - if (write32(1, buf32 + highlight + 1, len - highlight - 1) == -1) return; - } - - // we have to generate our own newline on line wrap - if (xEndOfInput == 0 && yEndOfInput > 0) - if (write(1, "\n", 1) == -1) return; - - // position the cursor - cursorRowMovement = yEndOfInput - yCursorPos; - if (cursorRowMovement > 0) { // move the cursor up as required - snprintf(seq, sizeof seq, "\x1b[%dA", cursorRowMovement); - if (write(1, seq, strlen(seq)) == -1) return; - } - // position the cursor within the line - snprintf(seq, sizeof seq, "\x1b[%dG", xCursorPos + 1); // 1-based on VT100 - if (write(1, seq, strlen(seq)) == -1) return; -#endif - - pi.promptCursorRowOffset = - pi.promptExtraLines + yCursorPos; // remember row for next pass -} - -#ifndef _WIN32 - -/** - * Read a UTF-8 sequence from the non-Windows keyboard and return the Unicode - * (char32_t) character it - * encodes - * - * @return char32_t Unicode character - */ -static char32_t readUnicodeCharacter(void) { - static char8_t utf8String[5]; - static size_t utf8Count = 0; - while (true) { - char8_t c; - - /* Continue reading if interrupted by signal. */ - ssize_t nread; - do { - nread = read(0, &c, 1); - } while ((nread == -1) && (errno == EINTR)); - - if (nread <= 0) return 0; - if (c <= 0x7F) { // short circuit ASCII - utf8Count = 0; - return c; - } else if (utf8Count < sizeof(utf8String) - 1) { - utf8String[utf8Count++] = c; - utf8String[utf8Count] = 0; - char32_t unicodeChar[2]; - size_t ucharCount; - ConversionResult res = - copyString8to32(unicodeChar, 2, ucharCount, utf8String); - if (res == conversionOK && ucharCount) { - utf8Count = 0; - return unicodeChar[0]; - } - } else { - utf8Count = - 0; // this shouldn't happen: got four bytes but no UTF-8 character - } - } -} - -namespace EscapeSequenceProcessing { // move these out of global namespace - -// This chunk of code does parsing of the escape sequences sent by various Linux -// terminals. -// -// It handles arrow keys, Home, End and Delete keys by interpreting the -// sequences sent by -// gnome terminal, xterm, rxvt, konsole, aterm and yakuake including the Alt and -// Ctrl key -// combinations that are understood by linenoise. -// -// The parsing uses tables, a bunch of intermediate dispatch routines and a -// doDispatch -// loop that reads the tables and sends control to "deeper" routines to continue -// the -// parsing. The starting call to doDispatch( c, initialDispatch ) will -// eventually return -// either a character (with optional CTRL and META bits set), or -1 if parsing -// fails, or -// zero if an attempt to read from the keyboard fails. -// -// This is rather sloppy escape sequence processing, since we're not paying -// attention to what the -// actual TERM is set to and are processing all key sequences for all terminals, -// but it works with -// the most common keystrokes on the most common terminals. It's intricate, but -// the nested 'if' -// statements required to do it directly would be worse. This way has the -// advantage of allowing -// changes and extensions without having to touch a lot of code. - -// This is a typedef for the routine called by doDispatch(). It takes the -// current character -// as input, does any required processing including reading more characters and -// calling other -// dispatch routines, then eventually returns the final (possibly extended or -// special) character. -// -typedef char32_t (*CharacterDispatchRoutine)(char32_t); - -// This structure is used by doDispatch() to hold a list of characters to test -// for and -// a list of routines to call if the character matches. The dispatch routine -// list is one -// longer than the character list; the final entry is used if no character -// matches. -// -struct CharacterDispatch { - unsigned int len; // length of the chars list - const char* chars; // chars to test - CharacterDispatchRoutine* dispatch; // array of routines to call -}; - -// This dispatch routine is given a dispatch table and then farms work out to -// routines -// listed in the table based on the character it is called with. The dispatch -// routines can -// read more input characters to decide what should eventually be returned. -// Eventually, -// a called routine returns either a character or -1 to indicate parsing -// failure. -// -static char32_t doDispatch(char32_t c, CharacterDispatch& dispatchTable) { - for (unsigned int i = 0; i < dispatchTable.len; ++i) { - if (static_cast<unsigned char>(dispatchTable.chars[i]) == c) { - return dispatchTable.dispatch[i](c); - } - } - return dispatchTable.dispatch[dispatchTable.len](c); -} - -static char32_t thisKeyMetaCtrl = - 0; // holds pre-set Meta and/or Ctrl modifiers - -// Final dispatch routines -- return something -// -static char32_t normalKeyRoutine(char32_t c) { return thisKeyMetaCtrl | c; } -static char32_t upArrowKeyRoutine(char32_t) { - return thisKeyMetaCtrl | UP_ARROW_KEY; -} -static char32_t downArrowKeyRoutine(char32_t) { - return thisKeyMetaCtrl | DOWN_ARROW_KEY; -} -static char32_t rightArrowKeyRoutine(char32_t) { - return thisKeyMetaCtrl | RIGHT_ARROW_KEY; -} -static char32_t leftArrowKeyRoutine(char32_t) { - return thisKeyMetaCtrl | LEFT_ARROW_KEY; -} -static char32_t homeKeyRoutine(char32_t) { return thisKeyMetaCtrl | HOME_KEY; } -static char32_t endKeyRoutine(char32_t) { return thisKeyMetaCtrl | END_KEY; } -static char32_t pageUpKeyRoutine(char32_t) { - return thisKeyMetaCtrl | PAGE_UP_KEY; -} -static char32_t pageDownKeyRoutine(char32_t) { - return thisKeyMetaCtrl | PAGE_DOWN_KEY; -} -static char32_t deleteCharRoutine(char32_t) { - return thisKeyMetaCtrl | ctrlChar('H'); -} // key labeled Backspace -static char32_t deleteKeyRoutine(char32_t) { - return thisKeyMetaCtrl | DELETE_KEY; -} // key labeled Delete -static char32_t ctrlUpArrowKeyRoutine(char32_t) { - return thisKeyMetaCtrl | CTRL | UP_ARROW_KEY; -} -static char32_t ctrlDownArrowKeyRoutine(char32_t) { - return thisKeyMetaCtrl | CTRL | DOWN_ARROW_KEY; -} -static char32_t ctrlRightArrowKeyRoutine(char32_t) { - return thisKeyMetaCtrl | CTRL | RIGHT_ARROW_KEY; -} -static char32_t ctrlLeftArrowKeyRoutine(char32_t) { - return thisKeyMetaCtrl | CTRL | LEFT_ARROW_KEY; -} -static char32_t escFailureRoutine(char32_t) { - beep(); - return -1; -} - -// Handle ESC [ 1 ; 3 (or 5) <more stuff> escape sequences -// -static CharacterDispatchRoutine escLeftBracket1Semicolon3or5Routines[] = { - upArrowKeyRoutine, downArrowKeyRoutine, rightArrowKeyRoutine, - leftArrowKeyRoutine, escFailureRoutine}; -static CharacterDispatch escLeftBracket1Semicolon3or5Dispatch = { - 4, "ABCD", escLeftBracket1Semicolon3or5Routines}; - -// Handle ESC [ 1 ; <more stuff> escape sequences -// -static char32_t escLeftBracket1Semicolon3Routine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - thisKeyMetaCtrl |= META; - return doDispatch(c, escLeftBracket1Semicolon3or5Dispatch); -} -static char32_t escLeftBracket1Semicolon5Routine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - thisKeyMetaCtrl |= CTRL; - return doDispatch(c, escLeftBracket1Semicolon3or5Dispatch); -} -static CharacterDispatchRoutine escLeftBracket1SemicolonRoutines[] = { - escLeftBracket1Semicolon3Routine, escLeftBracket1Semicolon5Routine, - escFailureRoutine}; -static CharacterDispatch escLeftBracket1SemicolonDispatch = { - 2, "35", escLeftBracket1SemicolonRoutines}; - -// Handle ESC [ 1 <more stuff> escape sequences -// -static char32_t escLeftBracket1SemicolonRoutine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - return doDispatch(c, escLeftBracket1SemicolonDispatch); -} -static CharacterDispatchRoutine escLeftBracket1Routines[] = { - homeKeyRoutine, escLeftBracket1SemicolonRoutine, escFailureRoutine}; -static CharacterDispatch escLeftBracket1Dispatch = {2, "~;", - escLeftBracket1Routines}; - -// Handle ESC [ 3 <more stuff> escape sequences -// -static CharacterDispatchRoutine escLeftBracket3Routines[] = {deleteKeyRoutine, - escFailureRoutine}; -static CharacterDispatch escLeftBracket3Dispatch = {1, "~", - escLeftBracket3Routines}; - -// Handle ESC [ 4 <more stuff> escape sequences -// -static CharacterDispatchRoutine escLeftBracket4Routines[] = {endKeyRoutine, - escFailureRoutine}; -static CharacterDispatch escLeftBracket4Dispatch = {1, "~", - escLeftBracket4Routines}; - -// Handle ESC [ 5 <more stuff> escape sequences -// -static CharacterDispatchRoutine escLeftBracket5Routines[] = {pageUpKeyRoutine, - escFailureRoutine}; -static CharacterDispatch escLeftBracket5Dispatch = {1, "~", - escLeftBracket5Routines}; - -// Handle ESC [ 6 <more stuff> escape sequences -// -static CharacterDispatchRoutine escLeftBracket6Routines[] = {pageDownKeyRoutine, - escFailureRoutine}; -static CharacterDispatch escLeftBracket6Dispatch = {1, "~", - escLeftBracket6Routines}; - -// Handle ESC [ 7 <more stuff> escape sequences -// -static CharacterDispatchRoutine escLeftBracket7Routines[] = {homeKeyRoutine, - escFailureRoutine}; -static CharacterDispatch escLeftBracket7Dispatch = {1, "~", - escLeftBracket7Routines}; - -// Handle ESC [ 8 <more stuff> escape sequences -// -static CharacterDispatchRoutine escLeftBracket8Routines[] = {endKeyRoutine, - escFailureRoutine}; -static CharacterDispatch escLeftBracket8Dispatch = {1, "~", - escLeftBracket8Routines}; - -// Handle ESC [ <digit> escape sequences -// -static char32_t escLeftBracket0Routine(char32_t c) { - return escFailureRoutine(c); -} -static char32_t escLeftBracket1Routine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - return doDispatch(c, escLeftBracket1Dispatch); -} -static char32_t escLeftBracket2Routine(char32_t c) { - return escFailureRoutine(c); // Insert key, unused -} -static char32_t escLeftBracket3Routine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - return doDispatch(c, escLeftBracket3Dispatch); -} -static char32_t escLeftBracket4Routine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - return doDispatch(c, escLeftBracket4Dispatch); -} -static char32_t escLeftBracket5Routine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - return doDispatch(c, escLeftBracket5Dispatch); -} -static char32_t escLeftBracket6Routine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - return doDispatch(c, escLeftBracket6Dispatch); -} -static char32_t escLeftBracket7Routine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - return doDispatch(c, escLeftBracket7Dispatch); -} -static char32_t escLeftBracket8Routine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - return doDispatch(c, escLeftBracket8Dispatch); -} -static char32_t escLeftBracket9Routine(char32_t c) { - return escFailureRoutine(c); -} - -// Handle ESC [ <more stuff> escape sequences -// -static CharacterDispatchRoutine escLeftBracketRoutines[] = { - upArrowKeyRoutine, downArrowKeyRoutine, rightArrowKeyRoutine, - leftArrowKeyRoutine, homeKeyRoutine, endKeyRoutine, - escLeftBracket0Routine, escLeftBracket1Routine, escLeftBracket2Routine, - escLeftBracket3Routine, escLeftBracket4Routine, escLeftBracket5Routine, - escLeftBracket6Routine, escLeftBracket7Routine, escLeftBracket8Routine, - escLeftBracket9Routine, escFailureRoutine}; -static CharacterDispatch escLeftBracketDispatch = {16, "ABCDHF0123456789", - escLeftBracketRoutines}; - -// Handle ESC O <char> escape sequences -// -static CharacterDispatchRoutine escORoutines[] = { - upArrowKeyRoutine, downArrowKeyRoutine, rightArrowKeyRoutine, - leftArrowKeyRoutine, homeKeyRoutine, endKeyRoutine, - ctrlUpArrowKeyRoutine, ctrlDownArrowKeyRoutine, ctrlRightArrowKeyRoutine, - ctrlLeftArrowKeyRoutine, escFailureRoutine}; -static CharacterDispatch escODispatch = {10, "ABCDHFabcd", escORoutines}; - -// Initial ESC dispatch -- could be a Meta prefix or the start of an escape -// sequence -// -static char32_t escLeftBracketRoutine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - return doDispatch(c, escLeftBracketDispatch); -} -static char32_t escORoutine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - return doDispatch(c, escODispatch); -} -static char32_t setMetaRoutine(char32_t c); // need forward reference -static CharacterDispatchRoutine escRoutines[] = {escLeftBracketRoutine, - escORoutine, setMetaRoutine}; -static CharacterDispatch escDispatch = {2, "[O", escRoutines}; - -// Initial dispatch -- we are not in the middle of anything yet -// -static char32_t escRoutine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - return doDispatch(c, escDispatch); -} -static CharacterDispatchRoutine initialRoutines[] = { - escRoutine, deleteCharRoutine, normalKeyRoutine}; -static CharacterDispatch initialDispatch = {2, "\x1B\x7F", initialRoutines}; - -// Special handling for the ESC key because it does double duty -// -static char32_t setMetaRoutine(char32_t c) { - thisKeyMetaCtrl = META; - if (c == 0x1B) { // another ESC, stay in ESC processing mode - c = readUnicodeCharacter(); - if (c == 0) return 0; - return doDispatch(c, escDispatch); - } - return doDispatch(c, initialDispatch); -} - -} // namespace EscapeSequenceProcessing // move these out of global namespace - -#endif // #ifndef _WIN32 - -// linenoiseReadChar -- read a keystroke or keychord from the keyboard, and -// translate it -// into an encoded "keystroke". When convenient, extended keys are translated -// into their -// simpler Emacs keystrokes, so an unmodified "left arrow" becomes Ctrl-B. -// -// A return value of zero means "no input available", and a return value of -1 -// means "invalid key". -// -static char32_t linenoiseReadChar(void) { -#ifdef _WIN32 - - INPUT_RECORD rec; - DWORD count; - int modifierKeys = 0; - bool escSeen = false; - while (true) { - ReadConsoleInputW(console_in, &rec, 1, &count); -#if 0 // helper for debugging keystrokes, display info in the debug "Output" - // window in the debugger - { - if ( rec.EventType == KEY_EVENT ) { - //if ( rec.Event.KeyEvent.uChar.UnicodeChar ) { - char buf[1024]; - sprintf( - buf, - "Unicode character 0x%04X, repeat count %d, virtual keycode 0x%04X, " - "virtual scancode 0x%04X, key %s%s%s%s%s\n", - rec.Event.KeyEvent.uChar.UnicodeChar, - rec.Event.KeyEvent.wRepeatCount, - rec.Event.KeyEvent.wVirtualKeyCode, - rec.Event.KeyEvent.wVirtualScanCode, - rec.Event.KeyEvent.bKeyDown ? "down" : "up", - (rec.Event.KeyEvent.dwControlKeyState & LEFT_CTRL_PRESSED) ? - " L-Ctrl" : "", - (rec.Event.KeyEvent.dwControlKeyState & RIGHT_CTRL_PRESSED) ? - " R-Ctrl" : "", - (rec.Event.KeyEvent.dwControlKeyState & LEFT_ALT_PRESSED) ? - " L-Alt" : "", - (rec.Event.KeyEvent.dwControlKeyState & RIGHT_ALT_PRESSED) ? - " R-Alt" : "" - ); - OutputDebugStringA( buf ); - //} - } - } -#endif - if (rec.EventType != KEY_EVENT) { - continue; - } - // Windows provides for entry of characters that are not on your keyboard by - // sending the - // Unicode characters as a "key up" with virtual keycode 0x12 (VK_MENU == - // Alt key) ... - // accept these characters, otherwise only process characters on "key down" - if (!rec.Event.KeyEvent.bKeyDown && - rec.Event.KeyEvent.wVirtualKeyCode != VK_MENU) { - continue; - } - modifierKeys = 0; - // AltGr is encoded as ( LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED ), so don't - // treat this - // combination as either CTRL or META we just turn off those two bits, so it - // is still - // possible to combine CTRL and/or META with an AltGr key by using - // right-Ctrl and/or - // left-Alt - if ((rec.Event.KeyEvent.dwControlKeyState & - (LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED)) == - (LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED)) { - rec.Event.KeyEvent.dwControlKeyState &= - ~(LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED); - } - if (rec.Event.KeyEvent.dwControlKeyState & - (RIGHT_CTRL_PRESSED | LEFT_CTRL_PRESSED)) { - modifierKeys |= CTRL; - } - if (rec.Event.KeyEvent.dwControlKeyState & - (RIGHT_ALT_PRESSED | LEFT_ALT_PRESSED)) { - modifierKeys |= META; - } - if (escSeen) { - modifierKeys |= META; - } - if (rec.Event.KeyEvent.uChar.UnicodeChar == 0) { - switch (rec.Event.KeyEvent.wVirtualKeyCode) { - case VK_LEFT: - return modifierKeys | LEFT_ARROW_KEY; - case VK_RIGHT: - return modifierKeys | RIGHT_ARROW_KEY; - case VK_UP: - return modifierKeys | UP_ARROW_KEY; - case VK_DOWN: - return modifierKeys | DOWN_ARROW_KEY; - case VK_DELETE: - return modifierKeys | DELETE_KEY; - case VK_HOME: - return modifierKeys | HOME_KEY; - case VK_END: - return modifierKeys | END_KEY; - case VK_PRIOR: - return modifierKeys | PAGE_UP_KEY; - case VK_NEXT: - return modifierKeys | PAGE_DOWN_KEY; - default: - continue; // in raw mode, ReadConsoleInput shows shift, ctrl ... - } // ... ignore them - } else if (rec.Event.KeyEvent.uChar.UnicodeChar == - ctrlChar('[')) { // ESC, set flag for later - escSeen = true; - continue; - } else { - // we got a real character, return it - return modifierKeys | rec.Event.KeyEvent.uChar.UnicodeChar; - } - } - -#else - char32_t c; - c = readUnicodeCharacter(); - if (c == 0) return 0; - -// If _DEBUG_LINUX_KEYBOARD is set, then ctrl-^ puts us into a keyboard -// debugging mode -// where we print out decimal and decoded values for whatever the "terminal" -// program -// gives us on different keystrokes. Hit ctrl-C to exit this mode. -// -#define _DEBUG_LINUX_KEYBOARD -#if defined(_DEBUG_LINUX_KEYBOARD) - if (c == ctrlChar('^')) { // ctrl-^, special debug mode, prints all keys hit, - // ctrl-C to get out - printf( - "\nEntering keyboard debugging mode (on ctrl-^), press ctrl-C to exit " - "this mode\n"); - while (true) { - unsigned char keys[10]; - int ret = read(0, keys, 10); - - if (ret <= 0) { - printf("\nret: %d\n", ret); - } - for (int i = 0; i < ret; ++i) { - char32_t key = static_cast<char32_t>(keys[i]); - char* friendlyTextPtr; - char friendlyTextBuf[10]; - const char* prefixText = (key < 0x80) ? "" : "0x80+"; - char32_t keyCopy = (key < 0x80) ? key : key - 0x80; - if (keyCopy >= '!' && keyCopy <= '~') { // printable - friendlyTextBuf[0] = '\''; - friendlyTextBuf[1] = keyCopy; - friendlyTextBuf[2] = '\''; - friendlyTextBuf[3] = 0; - friendlyTextPtr = friendlyTextBuf; - } else if (keyCopy == ' ') { - friendlyTextPtr = const_cast<char*>("space"); - } else if (keyCopy == 27) { - friendlyTextPtr = const_cast<char*>("ESC"); - } else if (keyCopy == 0) { - friendlyTextPtr = const_cast<char*>("NUL"); - } else if (keyCopy == 127) { - friendlyTextPtr = const_cast<char*>("DEL"); - } else { - friendlyTextBuf[0] = '^'; - friendlyTextBuf[1] = keyCopy + 0x40; - friendlyTextBuf[2] = 0; - friendlyTextPtr = friendlyTextBuf; - } - printf("%d x%02X (%s%s) ", key, key, prefixText, friendlyTextPtr); - } - printf("\x1b[1G\n"); // go to first column of new line - - // drop out of this loop on ctrl-C - if (keys[0] == ctrlChar('C')) { - printf("Leaving keyboard debugging mode (on ctrl-C)\n"); - fflush(stdout); - return -2; - } - } - } -#endif // _DEBUG_LINUX_KEYBOARD - - EscapeSequenceProcessing::thisKeyMetaCtrl = - 0; // no modifiers yet at initialDispatch - return EscapeSequenceProcessing::doDispatch( - c, EscapeSequenceProcessing::initialDispatch); -#endif // #_WIN32 -} - -/** - * Free memory used in a recent command completion session - * - * @param lc pointer to a linenoiseCompletions struct - */ -static void freeCompletions(linenoiseCompletions* lc) { - lc->completionStrings.clear(); -} - -/** - * convert {CTRL + 'A'}, {CTRL + 'a'} and {CTRL + ctrlChar( 'A' )} into - * ctrlChar( 'A' ) - * leave META alone - * - * @param c character to clean up - * @return cleaned-up character - */ -static int cleanupCtrl(int c) { - if (c & CTRL) { - int d = c & 0x1FF; - if (d >= 'a' && d <= 'z') { - c = (c + ('a' - ctrlChar('A'))) & ~CTRL; - } - if (d >= 'A' && d <= 'Z') { - c = (c + ('A' - ctrlChar('A'))) & ~CTRL; - } - if (d >= ctrlChar('A') && d <= ctrlChar('Z')) { - c = c & ~CTRL; - } - } - return c; -} - -// break characters that may precede items to be completed -static const char breakChars[] = " =+-/\\*?\"'`&<>;|@{([])}"; - -// maximum number of completions to display without asking -static const size_t completionCountCutoff = 100; - -/** - * Handle command completion, using a completionCallback() routine to provide - * possible substitutions - * This routine handles the mechanics of updating the user's input buffer with - * possible replacement - * of text as the user selects a proposed completion string, or cancels the - * completion attempt. - * @param pi PromptBase struct holding information about the prompt and our - * screen position - */ -int InputBuffer::completeLine(PromptBase& pi) { - linenoiseCompletions lc; - char32_t c = 0; - - // completionCallback() expects a parsable entity, so find the previous break - // character and - // extract a copy to parse. we also handle the case where tab is hit while - // not at end-of-line. - int startIndex = pos; - while (--startIndex >= 0) { - if (strchr(breakChars, buf32[startIndex])) { - break; - } - } - ++startIndex; - int itemLength = pos - startIndex; - Utf32String unicodeCopy(&buf32[startIndex], itemLength); - Utf8String parseItem(unicodeCopy); - - // get a list of completions - completionCallback(parseItem.get(), &lc); - - // if no completions, we are done - if (lc.completionStrings.size() == 0) { - beep(); - freeCompletions(&lc); - return 0; - } - - // at least one completion - int longestCommonPrefix = 0; - int displayLength = 0; - if (lc.completionStrings.size() == 1) { - longestCommonPrefix = static_cast<int>(lc.completionStrings[0].length()); - } else { - bool keepGoing = true; - while (keepGoing) { - for (size_t j = 0; j < lc.completionStrings.size() - 1; ++j) { - char32_t c1 = lc.completionStrings[j][longestCommonPrefix]; - char32_t c2 = lc.completionStrings[j + 1][longestCommonPrefix]; - if ((0 == c1) || (0 == c2) || (c1 != c2)) { - keepGoing = false; - break; - } - } - if (keepGoing) { - ++longestCommonPrefix; - } - } - } - if (lc.completionStrings.size() != 1) { // beep if ambiguous - beep(); - } - - // if we can extend the item, extend it and return to main loop - if (longestCommonPrefix > itemLength) { - displayLength = len + longestCommonPrefix - itemLength; - if (displayLength > buflen) { - longestCommonPrefix -= displayLength - buflen; // don't overflow buffer - displayLength = buflen; // truncate the insertion - beep(); // and make a noise - } - Utf32String displayText(displayLength + 1); - memcpy(displayText.get(), buf32, sizeof(char32_t) * startIndex); - memcpy(&displayText[startIndex], &lc.completionStrings[0][0], - sizeof(char32_t) * longestCommonPrefix); - int tailIndex = startIndex + longestCommonPrefix; - memcpy(&displayText[tailIndex], &buf32[pos], - sizeof(char32_t) * (displayLength - tailIndex + 1)); - copyString32(buf32, displayText.get(), displayLength); - pos = startIndex + longestCommonPrefix; - len = displayLength; - refreshLine(pi); - return 0; - } - - // we can't complete any further, wait for second tab - do { - c = linenoiseReadChar(); - c = cleanupCtrl(c); - } while (c == static_cast<char32_t>(-1)); - - // if any character other than tab, pass it to the main loop - if (c != ctrlChar('I')) { - freeCompletions(&lc); - return c; - } - - // we got a second tab, maybe show list of possible completions - bool showCompletions = true; - bool onNewLine = false; - if (lc.completionStrings.size() > completionCountCutoff) { - int savePos = - pos; // move cursor to EOL to avoid overwriting the command line - pos = len; - refreshLine(pi); - pos = savePos; - printf("\nDisplay all %u possibilities? (y or n)", - static_cast<unsigned int>(lc.completionStrings.size())); - fflush(stdout); - onNewLine = true; - while (c != 'y' && c != 'Y' && c != 'n' && c != 'N' && c != ctrlChar('C')) { - do { - c = linenoiseReadChar(); - c = cleanupCtrl(c); - } while (c == static_cast<char32_t>(-1)); - } - switch (c) { - case 'n': - case 'N': - showCompletions = false; - freeCompletions(&lc); - break; - case ctrlChar('C'): - showCompletions = false; - freeCompletions(&lc); - if (write(1, "^C", 2) == -1) return -1; // Display the ^C we got - c = 0; - break; - } - } - - // if showing the list, do it the way readline does it - bool stopList = false; - if (showCompletions) { - int longestCompletion = 0; - for (size_t j = 0; j < lc.completionStrings.size(); ++j) { - itemLength = static_cast<int>(lc.completionStrings[j].length()); - if (itemLength > longestCompletion) { - longestCompletion = itemLength; - } - } - longestCompletion += 2; - int columnCount = pi.promptScreenColumns / longestCompletion; - if (columnCount < 1) { - columnCount = 1; - } - if (!onNewLine) { // skip this if we showed "Display all %d possibilities?" - int savePos = - pos; // move cursor to EOL to avoid overwriting the command line - pos = len; - refreshLine(pi); - pos = savePos; - } - size_t pauseRow = getScreenRows() - 1; - size_t rowCount = - (lc.completionStrings.size() + columnCount - 1) / columnCount; - for (size_t row = 0; row < rowCount; ++row) { - if (row == pauseRow) { - printf("\n--More--"); - fflush(stdout); - c = 0; - bool doBeep = false; - while (c != ' ' && c != '\r' && c != '\n' && c != 'y' && c != 'Y' && - c != 'n' && c != 'N' && c != 'q' && c != 'Q' && - c != ctrlChar('C')) { - if (doBeep) { - beep(); - } - doBeep = true; - do { - c = linenoiseReadChar(); - c = cleanupCtrl(c); - } while (c == static_cast<char32_t>(-1)); - } - switch (c) { - case ' ': - case 'y': - case 'Y': - printf("\r \r"); - pauseRow += getScreenRows() - 1; - break; - case '\r': - case '\n': - printf("\r \r"); - ++pauseRow; - break; - case 'n': - case 'N': - case 'q': - case 'Q': - printf("\r \r"); - stopList = true; - break; - case ctrlChar('C'): - if (write(1, "^C", 2) == -1) return -1; // Display the ^C we got - stopList = true; - break; - } - } else { - printf("\n"); - } - if (stopList) { - break; - } - for (int column = 0; column < columnCount; ++column) { - size_t index = (column * rowCount) + row; - if (index < lc.completionStrings.size()) { - itemLength = static_cast<int>(lc.completionStrings[index].length()); - fflush(stdout); - if (write32(1, lc.completionStrings[index].get(), itemLength) == -1) - return -1; - if (((column + 1) * rowCount) + row < lc.completionStrings.size()) { - for (int k = itemLength; k < longestCompletion; ++k) { - printf(" "); - } - } - } - } - } - fflush(stdout); - freeCompletions(&lc); - } - - // display the prompt on a new line, then redisplay the input buffer - if (!stopList || c == ctrlChar('C')) { - if (write(1, "\n", 1) == -1) return 0; - } - if (!pi.write()) return 0; -#ifndef _WIN32 - // we have to generate our own newline on line wrap on Linux - if (pi.promptIndentation == 0 && pi.promptExtraLines > 0) - if (write(1, "\n", 1) == -1) return 0; -#endif - pi.promptCursorRowOffset = pi.promptExtraLines; - refreshLine(pi); - return 0; -} - -/** - * Clear the screen ONLY (no redisplay of anything) - */ -void linenoiseClearScreen(void) { -#ifdef _WIN32 - COORD coord = {0, 0}; - CONSOLE_SCREEN_BUFFER_INFO inf; - HANDLE screenHandle = GetStdHandle(STD_OUTPUT_HANDLE); - GetConsoleScreenBufferInfo(screenHandle, &inf); - SetConsoleCursorPosition(screenHandle, coord); - DWORD count; - FillConsoleOutputCharacterA(screenHandle, ' ', inf.dwSize.X * inf.dwSize.Y, - coord, &count); -#else - if (write(1, "\x1b[H\x1b[2J", 7) <= 0) return; -#endif -} - -void InputBuffer::clearScreen(PromptBase& pi) { - linenoiseClearScreen(); - if (!pi.write()) return; -#ifndef _WIN32 - // we have to generate our own newline on line wrap on Linux - if (pi.promptIndentation == 0 && pi.promptExtraLines > 0) - if (write(1, "\n", 1) == -1) return; -#endif - pi.promptCursorRowOffset = pi.promptExtraLines; - refreshLine(pi); -} - -/** - * Incremental history search -- take over the prompt and keyboard as the user - * types a search - * string, deletes characters from it, changes direction, and either accepts the - * found line (for - * execution orediting) or cancels. - * @param pi PromptBase struct holding information about the (old, - * static) prompt and our - * screen position - * @param startChar the character that began the search, used to set the initial - * direction - */ -int InputBuffer::incrementalHistorySearch(PromptBase& pi, int startChar) { - size_t bufferSize; - size_t ucharCount = 0; - - // if not already recalling, add the current line to the history list so we - // don't have to - // special case it - if (historyIndex == historyLen - 1) { - free(history[historyLen - 1]); - bufferSize = sizeof(char32_t) * len + 1; - unique_ptr<char[]> tempBuffer(new char[bufferSize]); - copyString32to8(tempBuffer.get(), bufferSize, buf32); - history[historyLen - 1] = strdup8(tempBuffer.get()); - } - int historyLineLength = len; - int historyLinePosition = pos; - char32_t emptyBuffer[1]; - char emptyWidths[1]; - InputBuffer empty(emptyBuffer, emptyWidths, 1); - empty.refreshLine(pi); // erase the old input first - DynamicPrompt dp(pi, (startChar == ctrlChar('R')) ? -1 : 1); - - dp.promptPreviousLen = pi.promptPreviousLen; - dp.promptPreviousInputLen = pi.promptPreviousInputLen; - dynamicRefresh(dp, buf32, historyLineLength, - historyLinePosition); // draw user's text with our prompt - - // loop until we get an exit character - int c = 0; - bool keepLooping = true; - bool useSearchedLine = true; - bool searchAgain = false; - char32_t* activeHistoryLine = 0; - while (keepLooping) { - c = linenoiseReadChar(); - c = cleanupCtrl(c); // convert CTRL + <char> into normal ctrl - - switch (c) { - // these characters keep the selected text but do not execute it - case ctrlChar('A'): // ctrl-A, move cursor to start of line - case HOME_KEY: - case ctrlChar('B'): // ctrl-B, move cursor left by one character - case LEFT_ARROW_KEY: - case META + 'b': // meta-B, move cursor left by one word - case META + 'B': - case CTRL + LEFT_ARROW_KEY: - case META + LEFT_ARROW_KEY: // Emacs allows Meta, bash & readline don't - case ctrlChar('D'): - case META + 'd': // meta-D, kill word to right of cursor - case META + 'D': - case ctrlChar('E'): // ctrl-E, move cursor to end of line - case END_KEY: - case ctrlChar('F'): // ctrl-F, move cursor right by one character - case RIGHT_ARROW_KEY: - case META + 'f': // meta-F, move cursor right by one word - case META + 'F': - case CTRL + RIGHT_ARROW_KEY: - case META + RIGHT_ARROW_KEY: // Emacs allows Meta, bash & readline don't - case META + ctrlChar('H'): - case ctrlChar('J'): - case ctrlChar('K'): // ctrl-K, kill from cursor to end of line - case ctrlChar('M'): - case ctrlChar('N'): // ctrl-N, recall next line in history - case ctrlChar('P'): // ctrl-P, recall previous line in history - case DOWN_ARROW_KEY: - case UP_ARROW_KEY: - case ctrlChar('T'): // ctrl-T, transpose characters - case ctrlChar( - 'U'): // ctrl-U, kill all characters to the left of the cursor - case ctrlChar('W'): - case META + 'y': // meta-Y, "yank-pop", rotate popped text - case META + 'Y': - case 127: - case DELETE_KEY: - case META + '<': // start of history - case PAGE_UP_KEY: - case META + '>': // end of history - case PAGE_DOWN_KEY: - keepLooping = false; - break; - - // these characters revert the input line to its previous state - case ctrlChar('C'): // ctrl-C, abort this line - case ctrlChar('G'): - case ctrlChar('L'): // ctrl-L, clear screen and redisplay line - keepLooping = false; - useSearchedLine = false; - if (c != ctrlChar('L')) { - c = -1; // ctrl-C and ctrl-G just abort the search and do nothing - // else - } - break; - - // these characters stay in search mode and update the display - case ctrlChar('S'): - case ctrlChar('R'): - if (dp.searchTextLen == - 0) { // if no current search text, recall previous text - if (previousSearchText.length()) { - dp.updateSearchText(previousSearchText.get()); - } - } - if ((dp.direction == 1 && c == ctrlChar('R')) || - (dp.direction == -1 && c == ctrlChar('S'))) { - dp.direction = 0 - dp.direction; // reverse direction - dp.updateSearchPrompt(); // change the prompt - } else { - searchAgain = true; // same direction, search again - } - break; - -// job control is its own thing -#ifndef _WIN32 - case ctrlChar('Z'): // ctrl-Z, job control - disableRawMode(); // Returning to Linux (whatever) shell, leave raw - // mode - raise(SIGSTOP); // Break out in mid-line - enableRawMode(); // Back from Linux shell, re-enter raw mode - { - bufferSize = historyLineLength + 1; - unique_ptr<char32_t[]> tempUnicode(new char32_t[bufferSize]); - copyString8to32(tempUnicode.get(), bufferSize, ucharCount, - history[historyIndex]); - dynamicRefresh(dp, tempUnicode.get(), historyLineLength, - historyLinePosition); - } - continue; - break; -#endif - - // these characters update the search string, and hence the selected input - // line - case ctrlChar('H'): // backspace/ctrl-H, delete char to left of cursor - if (dp.searchTextLen > 0) { - unique_ptr<char32_t[]> tempUnicode(new char32_t[dp.searchTextLen]); - --dp.searchTextLen; - dp.searchText[dp.searchTextLen] = 0; - copyString32(tempUnicode.get(), dp.searchText.get(), - dp.searchTextLen); - dp.updateSearchText(tempUnicode.get()); - } else { - beep(); - } - break; - - case ctrlChar('Y'): // ctrl-Y, yank killed text - break; - - default: - if (!isControlChar(c) && c <= 0x0010FFFF) { // not an action character - unique_ptr<char32_t[]> tempUnicode( - new char32_t[dp.searchTextLen + 2]); - copyString32(tempUnicode.get(), dp.searchText.get(), - dp.searchTextLen); - tempUnicode[dp.searchTextLen] = c; - tempUnicode[dp.searchTextLen + 1] = 0; - dp.updateSearchText(tempUnicode.get()); - } else { - beep(); - } - } // switch - - // if we are staying in search mode, search now - if (keepLooping) { - bufferSize = historyLineLength + 1; - if (activeHistoryLine) { - delete[] activeHistoryLine; - activeHistoryLine = nullptr; - } - activeHistoryLine = new char32_t[bufferSize]; - copyString8to32(activeHistoryLine, bufferSize, ucharCount, - history[historyIndex]); - if (dp.searchTextLen > 0) { - bool found = false; - int historySearchIndex = historyIndex; - int lineLength = static_cast<int>(ucharCount); - int lineSearchPos = historyLinePosition; - if (searchAgain) { - lineSearchPos += dp.direction; - } - searchAgain = false; - while (true) { - while ((dp.direction > 0) ? (lineSearchPos < lineLength) - : (lineSearchPos >= 0)) { - if (strncmp32(dp.searchText.get(), - &activeHistoryLine[lineSearchPos], - dp.searchTextLen) == 0) { - found = true; - break; - } - lineSearchPos += dp.direction; - } - if (found) { - historyIndex = historySearchIndex; - historyLineLength = lineLength; - historyLinePosition = lineSearchPos; - break; - } else if ((dp.direction > 0) ? (historySearchIndex < historyLen - 1) - : (historySearchIndex > 0)) { - historySearchIndex += dp.direction; - bufferSize = strlen8(history[historySearchIndex]) + 1; - delete[] activeHistoryLine; - activeHistoryLine = nullptr; - activeHistoryLine = new char32_t[bufferSize]; - copyString8to32(activeHistoryLine, bufferSize, ucharCount, - history[historySearchIndex]); - lineLength = static_cast<int>(ucharCount); - lineSearchPos = - (dp.direction > 0) ? 0 : (lineLength - dp.searchTextLen); - } else { - beep(); - break; - } - }; // while - } - if (activeHistoryLine) { - delete[] activeHistoryLine; - activeHistoryLine = nullptr; - } - bufferSize = historyLineLength + 1; - activeHistoryLine = new char32_t[bufferSize]; - copyString8to32(activeHistoryLine, bufferSize, ucharCount, - history[historyIndex]); - dynamicRefresh(dp, activeHistoryLine, historyLineLength, - historyLinePosition); // draw user's text with our prompt - } - } // while - - // leaving history search, restore previous prompt, maybe make searched line - // current - PromptBase pb; - pb.promptChars = pi.promptIndentation; - pb.promptBytes = pi.promptBytes; - Utf32String tempUnicode(pb.promptBytes + 1); - - copyString32(tempUnicode.get(), &pi.promptText[pi.promptLastLinePosition], - pb.promptBytes - pi.promptLastLinePosition); - tempUnicode.initFromBuffer(); - pb.promptText = tempUnicode; - pb.promptExtraLines = 0; - pb.promptIndentation = pi.promptIndentation; - pb.promptLastLinePosition = 0; - pb.promptPreviousInputLen = historyLineLength; - pb.promptCursorRowOffset = dp.promptCursorRowOffset; - pb.promptScreenColumns = pi.promptScreenColumns; - pb.promptPreviousLen = dp.promptChars; - if (useSearchedLine && activeHistoryLine) { - historyRecallMostRecent = true; - copyString32(buf32, activeHistoryLine, buflen + 1); - len = historyLineLength; - pos = historyLinePosition; - } - if (activeHistoryLine) { - delete[] activeHistoryLine; - activeHistoryLine = nullptr; - } - dynamicRefresh(pb, buf32, len, - pos); // redraw the original prompt with current input - pi.promptPreviousInputLen = len; - pi.promptCursorRowOffset = pi.promptExtraLines + pb.promptCursorRowOffset; - previousSearchText = - dp.searchText; // save search text for possible reuse on ctrl-R ctrl-R - return c; // pass a character or -1 back to main loop -} - -static bool isCharacterAlphanumeric(char32_t testChar) { -#ifdef _WIN32 - return (iswalnum((wint_t)testChar) != 0 ? true : false); -#else - return (iswalnum(testChar) != 0 ? true : false); -#endif -} - -#ifndef _WIN32 -static bool gotResize = false; -#endif -static int keyType = 0; - -int InputBuffer::getInputLine(PromptBase& pi) { - keyType = 0; - - // The latest history entry is always our current buffer - if (len > 0) { - size_t bufferSize = sizeof(char32_t) * len + 1; - unique_ptr<char[]> tempBuffer(new char[bufferSize]); - copyString32to8(tempBuffer.get(), bufferSize, buf32); - linenoiseHistoryAdd(tempBuffer.get()); - } else { - linenoiseHistoryAdd(""); - } - historyIndex = historyLen - 1; - historyRecallMostRecent = false; - - // display the prompt - if (!pi.write()) return -1; - -#ifndef _WIN32 - // we have to generate our own newline on line wrap on Linux - if (pi.promptIndentation == 0 && pi.promptExtraLines > 0) - if (write(1, "\n", 1) == -1) return -1; -#endif - - // the cursor starts out at the end of the prompt - pi.promptCursorRowOffset = pi.promptExtraLines; - - // kill and yank start in "other" mode - killRing.lastAction = KillRing::actionOther; - - // when history search returns control to us, we execute its terminating - // keystroke - int terminatingKeystroke = -1; - - // if there is already text in the buffer, display it first - if (len > 0) { - refreshLine(pi); - } - - // loop collecting characters, respond to line editing characters - while (true) { - int c; - if (terminatingKeystroke == -1) { - c = linenoiseReadChar(); // get a new keystroke - - keyType = 0; - if (c != 0) { - // set flag that we got some input - if (c == ctrlChar('C')) { - keyType = 1; - } else if (c == ctrlChar('D')) { - keyType = 2; - } - } - -#ifndef _WIN32 - if (c == 0 && gotResize) { - // caught a window resize event - // now redraw the prompt and line - gotResize = false; - pi.promptScreenColumns = getScreenColumns(); - dynamicRefresh(pi, buf32, len, - pos); // redraw the original prompt with current input - continue; - } -#endif - } else { - c = terminatingKeystroke; // use the terminating keystroke from search - terminatingKeystroke = -1; // clear it once we've used it - } - - c = cleanupCtrl(c); // convert CTRL + <char> into normal ctrl - - if (c == 0) { - return len; - } - - if (c == -1) { - refreshLine(pi); - continue; - } - - if (c == -2) { - if (!pi.write()) return -1; - refreshLine(pi); - continue; - } - - // ctrl-I/tab, command completion, needs to be before switch statement - if (c == ctrlChar('I') && completionCallback) { - killRing.lastAction = KillRing::actionOther; - historyRecallMostRecent = false; - - // completeLine does the actual completion and replacement - c = completeLine(pi); - - if (c < 0) // return on error - return len; - - if (c == 0) // read next character when 0 - continue; - - // deliberate fall-through here, so we use the terminating character - } - - switch (c) { - case ctrlChar('A'): // ctrl-A, move cursor to start of line - case HOME_KEY: - killRing.lastAction = KillRing::actionOther; - pos = 0; - refreshLine(pi); - break; - - case ctrlChar('B'): // ctrl-B, move cursor left by one character - case LEFT_ARROW_KEY: - killRing.lastAction = KillRing::actionOther; - if (pos > 0) { - --pos; - refreshLine(pi); - } - break; - - case META + 'b': // meta-B, move cursor left by one word - case META + 'B': - case CTRL + LEFT_ARROW_KEY: - case META + LEFT_ARROW_KEY: // Emacs allows Meta, bash & readline don't - killRing.lastAction = KillRing::actionOther; - if (pos > 0) { - while (pos > 0 && !isCharacterAlphanumeric(buf32[pos - 1])) { - --pos; - } - while (pos > 0 && isCharacterAlphanumeric(buf32[pos - 1])) { - --pos; - } - refreshLine(pi); - } - break; - - case ctrlChar('C'): // ctrl-C, abort this line - killRing.lastAction = KillRing::actionOther; - historyRecallMostRecent = false; - errno = EAGAIN; - --historyLen; - free(history[historyLen]); - // we need one last refresh with the cursor at the end of the line - // so we don't display the next prompt over the previous input line - pos = len; // pass len as pos for EOL - refreshLine(pi); - if (write(1, "^C", 2) == -1) return -1; // Display the ^C we got - return -1; - - case META + 'c': // meta-C, give word initial Cap - case META + 'C': - killRing.lastAction = KillRing::actionOther; - historyRecallMostRecent = false; - if (pos < len) { - while (pos < len && !isCharacterAlphanumeric(buf32[pos])) { - ++pos; - } - if (pos < len && isCharacterAlphanumeric(buf32[pos])) { - if (buf32[pos] >= 'a' && buf32[pos] <= 'z') { - buf32[pos] += 'A' - 'a'; - } - ++pos; - } - while (pos < len && isCharacterAlphanumeric(buf32[pos])) { - if (buf32[pos] >= 'A' && buf32[pos] <= 'Z') { - buf32[pos] += 'a' - 'A'; - } - ++pos; - } - refreshLine(pi); - } - break; - - // ctrl-D, delete the character under the cursor - // on an empty line, exit the shell - case ctrlChar('D'): - killRing.lastAction = KillRing::actionOther; - if (len > 0 && pos < len) { - historyRecallMostRecent = false; - memmove(buf32 + pos, buf32 + pos + 1, sizeof(char32_t) * (len - pos)); - --len; - refreshLine(pi); - } else if (len == 0) { - --historyLen; - free(history[historyLen]); - return -1; - } - break; - - case META + 'd': // meta-D, kill word to right of cursor - case META + 'D': - if (pos < len) { - historyRecallMostRecent = false; - int endingPos = pos; - while (endingPos < len && - !isCharacterAlphanumeric(buf32[endingPos])) { - ++endingPos; - } - while (endingPos < len && isCharacterAlphanumeric(buf32[endingPos])) { - ++endingPos; - } - killRing.kill(&buf32[pos], endingPos - pos, true); - memmove(buf32 + pos, buf32 + endingPos, - sizeof(char32_t) * (len - endingPos + 1)); - len -= endingPos - pos; - refreshLine(pi); - } - killRing.lastAction = KillRing::actionKill; - break; - - case ctrlChar('E'): // ctrl-E, move cursor to end of line - case END_KEY: - killRing.lastAction = KillRing::actionOther; - pos = len; - refreshLine(pi); - break; - - case ctrlChar('F'): // ctrl-F, move cursor right by one character - case RIGHT_ARROW_KEY: - killRing.lastAction = KillRing::actionOther; - if (pos < len) { - ++pos; - refreshLine(pi); - } - break; - - case META + 'f': // meta-F, move cursor right by one word - case META + 'F': - case CTRL + RIGHT_ARROW_KEY: - case META + RIGHT_ARROW_KEY: // Emacs allows Meta, bash & readline don't - killRing.lastAction = KillRing::actionOther; - if (pos < len) { - while (pos < len && !isCharacterAlphanumeric(buf32[pos])) { - ++pos; - } - while (pos < len && isCharacterAlphanumeric(buf32[pos])) { - ++pos; - } - refreshLine(pi); - } - break; - - case ctrlChar('H'): // backspace/ctrl-H, delete char to left of cursor - killRing.lastAction = KillRing::actionOther; - if (pos > 0) { - historyRecallMostRecent = false; - memmove(buf32 + pos - 1, buf32 + pos, - sizeof(char32_t) * (1 + len - pos)); - --pos; - --len; - refreshLine(pi); - } - break; - - // meta-Backspace, kill word to left of cursor - case META + ctrlChar('H'): - if (pos > 0) { - historyRecallMostRecent = false; - int startingPos = pos; - while (pos > 0 && !isCharacterAlphanumeric(buf32[pos - 1])) { - --pos; - } - while (pos > 0 && isCharacterAlphanumeric(buf32[pos - 1])) { - --pos; - } - killRing.kill(&buf32[pos], startingPos - pos, false); - memmove(buf32 + pos, buf32 + startingPos, - sizeof(char32_t) * (len - startingPos + 1)); - len -= startingPos - pos; - refreshLine(pi); - } - killRing.lastAction = KillRing::actionKill; - break; - - case ctrlChar('J'): // ctrl-J/linefeed/newline, accept line - case ctrlChar('M'): // ctrl-M/return/enter - killRing.lastAction = KillRing::actionOther; - // we need one last refresh with the cursor at the end of the line - // so we don't display the next prompt over the previous input line - pos = len; // pass len as pos for EOL - refreshLine(pi); - historyPreviousIndex = historyRecallMostRecent ? historyIndex : -2; - --historyLen; - free(history[historyLen]); - return len; - - case ctrlChar('K'): // ctrl-K, kill from cursor to end of line - killRing.kill(&buf32[pos], len - pos, true); - buf32[pos] = '\0'; - len = pos; - refreshLine(pi); - killRing.lastAction = KillRing::actionKill; - historyRecallMostRecent = false; - break; - - case ctrlChar('L'): // ctrl-L, clear screen and redisplay line - clearScreen(pi); - break; - - case META + 'l': // meta-L, lowercase word - case META + 'L': - killRing.lastAction = KillRing::actionOther; - if (pos < len) { - historyRecallMostRecent = false; - while (pos < len && !isCharacterAlphanumeric(buf32[pos])) { - ++pos; - } - while (pos < len && isCharacterAlphanumeric(buf32[pos])) { - if (buf32[pos] >= 'A' && buf32[pos] <= 'Z') { - buf32[pos] += 'a' - 'A'; - } - ++pos; - } - refreshLine(pi); - } - break; - - case ctrlChar('N'): // ctrl-N, recall next line in history - case ctrlChar('P'): // ctrl-P, recall previous line in history - case DOWN_ARROW_KEY: - case UP_ARROW_KEY: - killRing.lastAction = KillRing::actionOther; - // if not already recalling, add the current line to the history list so - // we don't - // have to special case it - if (historyIndex == historyLen - 1) { - free(history[historyLen - 1]); - size_t tempBufferSize = sizeof(char32_t) * len + 1; - unique_ptr<char[]> tempBuffer(new char[tempBufferSize]); - copyString32to8(tempBuffer.get(), tempBufferSize, buf32); - history[historyLen - 1] = strdup8(tempBuffer.get()); - } - if (historyLen > 1) { - if (c == UP_ARROW_KEY) { - c = ctrlChar('P'); - } - if (historyPreviousIndex != -2 && c != ctrlChar('P')) { - historyIndex = - 1 + historyPreviousIndex; // emulate Windows down-arrow - } else { - historyIndex += (c == ctrlChar('P')) ? -1 : 1; - } - historyPreviousIndex = -2; - if (historyIndex < 0) { - historyIndex = 0; - break; - } else if (historyIndex >= historyLen) { - historyIndex = historyLen - 1; - break; - } - historyRecallMostRecent = true; - size_t ucharCount = 0; - copyString8to32(buf32, buflen, ucharCount, history[historyIndex]); - len = pos = static_cast<int>(ucharCount); - refreshLine(pi); - } - break; - - case ctrlChar('R'): // ctrl-R, reverse history search - case ctrlChar('S'): // ctrl-S, forward history search - terminatingKeystroke = incrementalHistorySearch(pi, c); - break; - - case ctrlChar('T'): // ctrl-T, transpose characters - killRing.lastAction = KillRing::actionOther; - if (pos > 0 && len > 1) { - historyRecallMostRecent = false; - size_t leftCharPos = (pos == len) ? pos - 2 : pos - 1; - char32_t aux = buf32[leftCharPos]; - buf32[leftCharPos] = buf32[leftCharPos + 1]; - buf32[leftCharPos + 1] = aux; - if (pos != len) ++pos; - refreshLine(pi); - } - break; - - case ctrlChar( - 'U'): // ctrl-U, kill all characters to the left of the cursor - if (pos > 0) { - historyRecallMostRecent = false; - killRing.kill(&buf32[0], pos, false); - len -= pos; - memmove(buf32, buf32 + pos, sizeof(char32_t) * (len + 1)); - pos = 0; - refreshLine(pi); - } - killRing.lastAction = KillRing::actionKill; - break; - - case META + 'u': // meta-U, uppercase word - case META + 'U': - killRing.lastAction = KillRing::actionOther; - if (pos < len) { - historyRecallMostRecent = false; - while (pos < len && !isCharacterAlphanumeric(buf32[pos])) { - ++pos; - } - while (pos < len && isCharacterAlphanumeric(buf32[pos])) { - if (buf32[pos] >= 'a' && buf32[pos] <= 'z') { - buf32[pos] += 'A' - 'a'; - } - ++pos; - } - refreshLine(pi); - } - break; - - // ctrl-W, kill to whitespace (not word) to left of cursor - case ctrlChar('W'): - if (pos > 0) { - historyRecallMostRecent = false; - int startingPos = pos; - while (pos > 0 && buf32[pos - 1] == ' ') { - --pos; - } - while (pos > 0 && buf32[pos - 1] != ' ') { - --pos; - } - killRing.kill(&buf32[pos], startingPos - pos, false); - memmove(buf32 + pos, buf32 + startingPos, - sizeof(char32_t) * (len - startingPos + 1)); - len -= startingPos - pos; - refreshLine(pi); - } - killRing.lastAction = KillRing::actionKill; - break; - - case ctrlChar('Y'): // ctrl-Y, yank killed text - historyRecallMostRecent = false; - { - Utf32String* restoredText = killRing.yank(); - if (restoredText) { - bool truncated = false; - size_t ucharCount = restoredText->length(); - if (ucharCount > static_cast<size_t>(buflen - len)) { - ucharCount = buflen - len; - truncated = true; - } - memmove(buf32 + pos + ucharCount, buf32 + pos, - sizeof(char32_t) * (len - pos + 1)); - memmove(buf32 + pos, restoredText->get(), - sizeof(char32_t) * ucharCount); - pos += static_cast<int>(ucharCount); - len += static_cast<int>(ucharCount); - refreshLine(pi); - killRing.lastAction = KillRing::actionYank; - killRing.lastYankSize = ucharCount; - if (truncated) { - beep(); - } - } else { - beep(); - } - } - break; - - case META + 'y': // meta-Y, "yank-pop", rotate popped text - case META + 'Y': - if (killRing.lastAction == KillRing::actionYank) { - historyRecallMostRecent = false; - Utf32String* restoredText = killRing.yankPop(); - if (restoredText) { - bool truncated = false; - size_t ucharCount = restoredText->length(); - if (ucharCount > - static_cast<size_t>(killRing.lastYankSize + buflen - len)) { - ucharCount = killRing.lastYankSize + buflen - len; - truncated = true; - } - if (ucharCount > killRing.lastYankSize) { - memmove(buf32 + pos + ucharCount - killRing.lastYankSize, - buf32 + pos, sizeof(char32_t) * (len - pos + 1)); - memmove(buf32 + pos - killRing.lastYankSize, restoredText->get(), - sizeof(char32_t) * ucharCount); - } else { - memmove(buf32 + pos - killRing.lastYankSize, restoredText->get(), - sizeof(char32_t) * ucharCount); - memmove(buf32 + pos + ucharCount - killRing.lastYankSize, - buf32 + pos, sizeof(char32_t) * (len - pos + 1)); - } - pos += static_cast<int>(ucharCount - killRing.lastYankSize); - len += static_cast<int>(ucharCount - killRing.lastYankSize); - killRing.lastYankSize = ucharCount; - refreshLine(pi); - if (truncated) { - beep(); - } - break; - } - } - beep(); - break; - -#ifndef _WIN32 - case ctrlChar('Z'): // ctrl-Z, job control - disableRawMode(); // Returning to Linux (whatever) shell, leave raw - // mode - raise(SIGSTOP); // Break out in mid-line - enableRawMode(); // Back from Linux shell, re-enter raw mode - if (!pi.write()) break; // Redraw prompt - refreshLine(pi); // Refresh the line - break; -#endif - - // DEL, delete the character under the cursor - case 127: - case DELETE_KEY: - killRing.lastAction = KillRing::actionOther; - if (len > 0 && pos < len) { - historyRecallMostRecent = false; - memmove(buf32 + pos, buf32 + pos + 1, sizeof(char32_t) * (len - pos)); - --len; - refreshLine(pi); - } - break; - - case META + '<': // meta-<, beginning of history - case PAGE_UP_KEY: // Page Up, beginning of history - case META + '>': // meta->, end of history - case PAGE_DOWN_KEY: // Page Down, end of history - killRing.lastAction = KillRing::actionOther; - // if not already recalling, add the current line to the history list so - // we don't - // have to special case it - if (historyIndex == historyLen - 1) { - free(history[historyLen - 1]); - size_t tempBufferSize = sizeof(char32_t) * len + 1; - unique_ptr<char[]> tempBuffer(new char[tempBufferSize]); - copyString32to8(tempBuffer.get(), tempBufferSize, buf32); - history[historyLen - 1] = strdup8(tempBuffer.get()); - } - if (historyLen > 1) { - historyIndex = - (c == META + '<' || c == PAGE_UP_KEY) ? 0 : historyLen - 1; - historyPreviousIndex = -2; - historyRecallMostRecent = true; - size_t ucharCount = 0; - copyString8to32(buf32, buflen, ucharCount, history[historyIndex]); - len = pos = static_cast<int>(ucharCount); - refreshLine(pi); - } - break; - - // not one of our special characters, maybe insert it in the buffer - default: - killRing.lastAction = KillRing::actionOther; - historyRecallMostRecent = false; - if (c & (META | CTRL)) { // beep on unknown Ctrl and/or Meta keys - beep(); - break; - } - if (len < buflen) { - if (isControlChar(c)) { // don't insert control characters - beep(); - break; - } - if (len == pos) { // at end of buffer - buf32[pos] = c; - ++pos; - ++len; - buf32[len] = '\0'; - int inputLen = calculateColumnPosition(buf32, len); - if (pi.promptIndentation + inputLen < pi.promptScreenColumns) { - if (inputLen > pi.promptPreviousInputLen) - pi.promptPreviousInputLen = inputLen; - /* Avoid a full update of the line in the - * trivial case. */ - if (write32(1, reinterpret_cast<char32_t*>(&c), 1) == -1) - return -1; - } else { - refreshLine(pi); - } - } else { // not at end of buffer, have to move characters to our - // right - memmove(buf32 + pos + 1, buf32 + pos, - sizeof(char32_t) * (len - pos)); - buf32[pos] = c; - ++len; - ++pos; - buf32[len] = '\0'; - refreshLine(pi); - } - } else { - beep(); // buffer is full, beep on new characters - } - break; - } - } - return len; -} - -static string preloadedBufferContents; // used with linenoisePreloadBuffer -static string preloadErrorMessage; - -/** - * linenoisePreloadBuffer provides text to be inserted into the command buffer - * - * the provided text will be processed to be usable and will be used to preload - * the input buffer on the next call to linenoise() - * - * @param preloadText text to begin with on the next call to linenoise() - */ -void linenoisePreloadBuffer(const char* preloadText) { - if (!preloadText) { - return; - } - int bufferSize = static_cast<int>(strlen(preloadText) + 1); - unique_ptr<char[]> tempBuffer(new char[bufferSize]); - strncpy(&tempBuffer[0], preloadText, bufferSize); - - // remove characters that won't display correctly - char* pIn = &tempBuffer[0]; - char* pOut = pIn; - bool controlsStripped = false; - bool whitespaceSeen = false; - while (*pIn) { - unsigned char c = - *pIn++; // we need unsigned so chars 0x80 and above are allowed - if ('\r' == c) { // silently skip CR - continue; - } - if ('\n' == c || '\t' == c) { // note newline or tab - whitespaceSeen = true; - continue; - } - if (isControlChar( - c)) { // remove other control characters, flag for message - controlsStripped = true; - *pOut++ = ' '; - continue; - } - if (whitespaceSeen) { // convert whitespace to a single space - *pOut++ = ' '; - whitespaceSeen = false; - } - *pOut++ = c; - } - *pOut = 0; - int processedLength = static_cast<int>(pOut - tempBuffer.get()); - bool lineTruncated = false; - if (processedLength > (LINENOISE_MAX_LINE - 1)) { - lineTruncated = true; - tempBuffer[LINENOISE_MAX_LINE - 1] = 0; - } - preloadedBufferContents = tempBuffer.get(); - if (controlsStripped) { - preloadErrorMessage += - " [Edited line: control characters were converted to spaces]\n"; - } - if (lineTruncated) { - preloadErrorMessage += " [Edited line: the line length was reduced from "; - char buf[128]; - snprintf(buf, sizeof(buf), "%d to %d]\n", processedLength, - (LINENOISE_MAX_LINE - 1)); - preloadErrorMessage += buf; - } -} - -/** - * linenoise is a readline replacement. - * - * call it with a prompt to display and it will return a line of input from the - * user - * - * @param prompt text of prompt to display to the user - * @return the returned string belongs to the caller on return and must be - * freed to prevent - * memory leaks - */ -char* linenoise(const char* prompt) { -#ifndef _WIN32 - gotResize = false; -#endif - if (isatty(STDIN_FILENO)) { // input is from a terminal - char32_t buf32[LINENOISE_MAX_LINE]; - char charWidths[LINENOISE_MAX_LINE]; - if (!preloadErrorMessage.empty()) { - printf("%s", preloadErrorMessage.c_str()); - fflush(stdout); - preloadErrorMessage.clear(); - } - PromptInfo pi(prompt, getScreenColumns()); - if (isUnsupportedTerm()) { - if (!pi.write()) return 0; - fflush(stdout); - if (preloadedBufferContents.empty()) { - unique_ptr<char[]> buf8(new char[LINENOISE_MAX_LINE]); - if (fgets(buf8.get(), LINENOISE_MAX_LINE, stdin) == NULL) { - return NULL; - } - size_t len = strlen(buf8.get()); - while (len && (buf8[len - 1] == '\n' || buf8[len - 1] == '\r')) { - --len; - buf8[len] = '\0'; - } - return strdup(buf8.get()); // caller must free buffer - } else { - char* buf8 = strdup(preloadedBufferContents.c_str()); - preloadedBufferContents.clear(); - return buf8; // caller must free buffer - } - } else { - if (enableRawMode() == -1) { - return NULL; - } - InputBuffer ib(buf32, charWidths, LINENOISE_MAX_LINE); - if (!preloadedBufferContents.empty()) { - ib.preloadBuffer(preloadedBufferContents.c_str()); - preloadedBufferContents.clear(); - } - int count = ib.getInputLine(pi); - disableRawMode(); - printf("\n"); - if (count == -1) { - return NULL; - } - size_t bufferSize = sizeof(char32_t) * ib.length() + 1; - unique_ptr<char[]> buf8(new char[bufferSize]); - copyString32to8(buf8.get(), bufferSize, buf32); - return strdup(buf8.get()); // caller must free buffer - } - } else { // input not from a terminal, we should work with piped input, i.e. - // redirected stdin - unique_ptr<char[]> buf8(new char[LINENOISE_MAX_LINE]); - if (fgets(buf8.get(), LINENOISE_MAX_LINE, stdin) == NULL) { - return NULL; - } - - // if fgets() gave us the newline, remove it - int count = static_cast<int>(strlen(buf8.get())); - if (count > 0 && buf8[count - 1] == '\n') { - --count; - buf8[count] = '\0'; - } - return strdup(buf8.get()); // caller must free buffer - } -} - -/* Register a callback function to be called for tab-completion. */ -void linenoiseSetCompletionCallback(linenoiseCompletionCallback* fn) { - completionCallback = fn; -} - -void linenoiseAddCompletion(linenoiseCompletions* lc, const char* str) { - lc->completionStrings.push_back(Utf32String(str)); -} - -int linenoiseHistoryAdd(const char* line) { - if (historyMaxLen == 0) { - return 0; - } - if (history == NULL) { - history = - reinterpret_cast<char8_t**>(malloc(sizeof(char8_t*) * historyMaxLen)); - if (history == NULL) { - return 0; - } - memset(history, 0, (sizeof(char*) * historyMaxLen)); - } - char8_t* linecopy = strdup8(line); - if (!linecopy) { - return 0; - } - - // convert newlines in multi-line code to spaces before storing - char8_t* p = linecopy; - while (*p) { - if (*p == '\n') { - *p = ' '; - } - ++p; - } - - // prevent duplicate history entries - if (historyLen > 0 && history[historyLen - 1] != nullptr && - strcmp(reinterpret_cast<char const*>(history[historyLen - 1]), - reinterpret_cast<char const*>(linecopy)) == 0) { - free(linecopy); - return 0; - } - - if (historyLen == historyMaxLen) { - free(history[0]); - memmove(history, history + 1, sizeof(char*) * (historyMaxLen - 1)); - --historyLen; - if (--historyPreviousIndex < -1) { - historyPreviousIndex = -2; - } - } - - history[historyLen] = linecopy; - ++historyLen; - return 1; -} - -int linenoiseHistorySetMaxLen(int len) { - if (len < 1) { - return 0; - } - if (history) { - int tocopy = historyLen; - char8_t** newHistory = - reinterpret_cast<char8_t**>(malloc(sizeof(char8_t*) * len)); - if (newHistory == NULL) { - return 0; - } - if (len < tocopy) { - tocopy = len; - } - memcpy(newHistory, history + historyMaxLen - tocopy, - sizeof(char8_t*) * tocopy); - free(history); - history = newHistory; - } - historyMaxLen = len; - if (historyLen > historyMaxLen) { - historyLen = historyMaxLen; - } - return 1; -} - -/* Fetch a line of the history by (zero-based) index. If the requested - * line does not exist, NULL is returned. The return value is a heap-allocated - * copy of the line, and the caller is responsible for de-allocating it. */ -char* linenoiseHistoryLine(int index) { - if (index < 0 || index >= historyLen) return NULL; - - return strdup(reinterpret_cast<char const*>(history[index])); -} - -/* Save the history in the specified file. On success 0 is returned - * otherwise -1 is returned. */ -int linenoiseHistorySave(const char* filename) { -#if _WIN32 - FILE* fp = fopen(filename, "wt"); -#else - int fd = open(filename, O_CREAT | O_TRUNC | O_WRONLY, S_IRUSR | S_IWUSR); - - if (fd < 0) { - return -1; - } - - FILE* fp = fdopen(fd, "wt"); -#endif - - if (fp == NULL) { - return -1; - } - - for (int j = 0; j < historyLen; ++j) { - if (history[j][0] != '\0') { - fprintf(fp, "%s\n", history[j]); - } - } - - fclose(fp); - - return 0; -} - -/* Load the history from the specified file. If the file does not exist - * zero is returned and no operation is performed. - * - * If the file exists and the operation succeeded 0 is returned, otherwise - * on error -1 is returned. */ -int linenoiseHistoryLoad(const char* filename) { - FILE* fp = fopen(filename, "rt"); - if (fp == NULL) { - return -1; - } - - char buf[LINENOISE_MAX_LINE]; - while (fgets(buf, LINENOISE_MAX_LINE, fp) != NULL) { - char* p = strchr(buf, '\r'); - if (!p) { - p = strchr(buf, '\n'); - } - if (p) { - *p = '\0'; - } - if (p != buf) { - linenoiseHistoryAdd(buf); - } - } - fclose(fp); - return 0; -} - -/* Set if to use or not the multi line mode. */ -/* note that this is a stub only, as linenoise-ng always multi-line */ -void linenoiseSetMultiLine(int) {} - -/* This special mode is used by linenoise in order to print scan codes - * on screen for debugging / development purposes. It is implemented - * by the linenoise_example program using the --keycodes option. */ -void linenoisePrintKeyCodes(void) { - char quit[4]; - - printf( - "Linenoise key codes debugging mode.\n" - "Press keys to see scan codes. Type 'quit' at any time to exit.\n"); - if (enableRawMode() == -1) return; - memset(quit, ' ', 4); - while (1) { - char c; - int nread; - -#if _WIN32 - nread = _read(STDIN_FILENO, &c, 1); -#else - nread = read(STDIN_FILENO, &c, 1); -#endif - if (nread <= 0) continue; - memmove(quit, quit + 1, sizeof(quit) - 1); /* shift string to left. */ - quit[sizeof(quit) - 1] = c; /* Insert current char on the right. */ - if (memcmp(quit, "quit", sizeof(quit)) == 0) break; - - printf("'%c' %02x (%d) (type quit to exit)\n", isprint(c) ? c : '?', (int)c, - (int)c); - printf("\r"); /* Go left edge manually, we are in raw mode. */ - fflush(stdout); - } - disableRawMode(); -} - -#ifndef _WIN32 -static void WindowSizeChanged(int) { - // do nothing here but setting this flag - gotResize = true; -} -#endif - -int linenoiseInstallWindowChangeHandler(void) { -#ifndef _WIN32 - struct sigaction sa; - sigemptyset(&sa.sa_mask); - sa.sa_flags = 0; - sa.sa_handler = &WindowSizeChanged; - - if (sigaction(SIGWINCH, &sa, nullptr) == -1) { - return errno; - } -#endif - return 0; -} - -int linenoiseKeyType(void) { - return keyType; -} diff --git a/src/linenoise/linenoise.h b/src/linenoise/linenoise.h deleted file mode 100644 index 3a8eb9f7ee63..000000000000 --- a/src/linenoise/linenoise.h +++ /dev/null @@ -1,73 +0,0 @@ -/* linenoise.h -- guerrilla line editing library against the idea that a - * line editing lib needs to be 20,000 lines of C code. - * - * See linenoise.c for more information. - * - * Copyright (c) 2010, Salvatore Sanfilippo <antirez at gmail dot com> - * Copyright (c) 2010, Pieter Noordhuis <pcnoordhuis at gmail dot com> - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Redis nor the names of its contributors may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef __LINENOISE_H -#define __LINENOISE_H - -#define LINENOISE_VERSION "1.0.0" -#define LINENOISE_VERSION_MAJOR 1 -#define LINENOISE_VERSION_MINOR 1 - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct linenoiseCompletions linenoiseCompletions; - -typedef void(linenoiseCompletionCallback)(const char*, linenoiseCompletions*); -void linenoiseSetCompletionCallback(linenoiseCompletionCallback* fn); -void linenoiseAddCompletion(linenoiseCompletions* lc, const char* str); - -char* linenoise(const char* prompt); -void linenoisePreloadBuffer(const char* preloadText); -int linenoiseHistoryAdd(const char* line); -int linenoiseHistorySetMaxLen(int len); -char* linenoiseHistoryLine(int index); -int linenoiseHistorySave(const char* filename); -int linenoiseHistoryLoad(const char* filename); -void linenoiseHistoryFree(void); -void linenoiseClearScreen(void); -void linenoiseSetMultiLine(int ml); -void linenoisePrintKeyCodes(void); -/* the following are extensions to the original linenoise API */ -int linenoiseInstallWindowChangeHandler(void); -/* returns type of key pressed: 1 = CTRL-C, 2 = CTRL-D, 0 = other */ -int linenoiseKeyType(void); - -#ifdef __cplusplus -} -#endif - -#endif /* __LINENOISE_H */ diff --git a/src/linenoise/wcwidth.cpp b/src/linenoise/wcwidth.cpp deleted file mode 100644 index deec0ba6b57f..000000000000 --- a/src/linenoise/wcwidth.cpp +++ /dev/null @@ -1,315 +0,0 @@ -/* - * This is an implementation of wcwidth() and wcswidth() (defined in - * IEEE Std 1002.1-2001) for Unicode. - * - * http://www.opengroup.org/onlinepubs/007904975/functions/wcwidth.html - * http://www.opengroup.org/onlinepubs/007904975/functions/wcswidth.html - * - * In fixed-width output devices, Latin characters all occupy a single - * "cell" position of equal width, whereas ideographic CJK characters - * occupy two such cells. Interoperability between terminal-line - * applications and (teletype-style) character terminals using the - * UTF-8 encoding requires agreement on which character should advance - * the cursor by how many cell positions. No established formal - * standards exist at present on which Unicode character shall occupy - * how many cell positions on character terminals. These routines are - * a first attempt of defining such behavior based on simple rules - * applied to data provided by the Unicode Consortium. - * - * For some graphical characters, the Unicode standard explicitly - * defines a character-cell width via the definition of the East Asian - * FullWidth (F), Wide (W), Half-width (H), and Narrow (Na) classes. - * In all these cases, there is no ambiguity about which width a - * terminal shall use. For characters in the East Asian Ambiguous (A) - * class, the width choice depends purely on a preference of backward - * compatibility with either historic CJK or Western practice. - * Choosing single-width for these characters is easy to justify as - * the appropriate long-term solution, as the CJK practice of - * displaying these characters as double-width comes from historic - * implementation simplicity (8-bit encoded characters were displayed - * single-width and 16-bit ones double-width, even for Greek, - * Cyrillic, etc.) and not any typographic considerations. - * - * Much less clear is the choice of width for the Not East Asian - * (Neutral) class. Existing practice does not dictate a width for any - * of these characters. It would nevertheless make sense - * typographically to allocate two character cells to characters such - * as for instance EM SPACE or VOLUME INTEGRAL, which cannot be - * represented adequately with a single-width glyph. The following - * routines at present merely assign a single-cell width to all - * neutral characters, in the interest of simplicity. This is not - * entirely satisfactory and should be reconsidered before - * establishing a formal standard in this area. At the moment, the - * decision which Not East Asian (Neutral) characters should be - * represented by double-width glyphs cannot yet be answered by - * applying a simple rule from the Unicode database content. Setting - * up a proper standard for the behavior of UTF-8 character terminals - * will require a careful analysis not only of each Unicode character, - * but also of each presentation form, something the author of these - * routines has avoided to do so far. - * - * http://www.unicode.org/unicode/reports/tr11/ - * - * Markus Kuhn -- 2007-05-26 (Unicode 5.0) - * - * Permission to use, copy, modify, and distribute this software - * for any purpose and without fee is hereby granted. The author - * disclaims all warranties with regard to this software. - * - * Latest version: http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c - */ - -#include <wchar.h> -#include <string> -#include <memory> - -namespace linenoise_ng { - -struct interval { - char32_t first; - char32_t last; -}; - -/* auxiliary function for binary search in interval table */ -static int bisearch(char32_t ucs, const struct interval *table, int max) { - int min = 0; - int mid; - - if (ucs < table[0].first || ucs > table[max].last) - return 0; - while (max >= min) { - mid = (min + max) / 2; - if (ucs > table[mid].last) - min = mid + 1; - else if (ucs < table[mid].first) - max = mid - 1; - else - return 1; - } - - return 0; -} - - -/* The following two functions define the column width of an ISO 10646 - * character as follows: - * - * - The null character (U+0000) has a column width of 0. - * - * - Other C0/C1 control characters and DEL will lead to a return - * value of -1. - * - * - Non-spacing and enclosing combining characters (general - * category code Mn or Me in the Unicode database) have a - * column width of 0. - * - * - SOFT HYPHEN (U+00AD) has a column width of 1. - * - * - Other format characters (general category code Cf in the Unicode - * database) and ZERO WIDTH SPACE (U+200B) have a column width of 0. - * - * - Hangul Jamo medial vowels and final consonants (U+1160-U+11FF) - * have a column width of 0. - * - * - Spacing characters in the East Asian Wide (W) or East Asian - * Full-width (F) category as defined in Unicode Technical - * Report #11 have a column width of 2. - * - * - All remaining characters (including all printable - * ISO 8859-1 and WGL4 characters, Unicode control characters, - * etc.) have a column width of 1. - * - * This implementation assumes that wchar_t characters are encoded - * in ISO 10646. - */ - -int mk_wcwidth(char32_t ucs) -{ - /* sorted list of non-overlapping intervals of non-spacing characters */ - /* generated by "uniset +cat=Me +cat=Mn +cat=Cf -00AD +1160-11FF +200B c" */ - static const struct interval combining[] = { - { 0x0300, 0x036F }, { 0x0483, 0x0486 }, { 0x0488, 0x0489 }, - { 0x0591, 0x05BD }, { 0x05BF, 0x05BF }, { 0x05C1, 0x05C2 }, - { 0x05C4, 0x05C5 }, { 0x05C7, 0x05C7 }, { 0x0600, 0x0603 }, - { 0x0610, 0x0615 }, { 0x064B, 0x065E }, { 0x0670, 0x0670 }, - { 0x06D6, 0x06E4 }, { 0x06E7, 0x06E8 }, { 0x06EA, 0x06ED }, - { 0x070F, 0x070F }, { 0x0711, 0x0711 }, { 0x0730, 0x074A }, - { 0x07A6, 0x07B0 }, { 0x07EB, 0x07F3 }, { 0x0901, 0x0902 }, - { 0x093C, 0x093C }, { 0x0941, 0x0948 }, { 0x094D, 0x094D }, - { 0x0951, 0x0954 }, { 0x0962, 0x0963 }, { 0x0981, 0x0981 }, - { 0x09BC, 0x09BC }, { 0x09C1, 0x09C4 }, { 0x09CD, 0x09CD }, - { 0x09E2, 0x09E3 }, { 0x0A01, 0x0A02 }, { 0x0A3C, 0x0A3C }, - { 0x0A41, 0x0A42 }, { 0x0A47, 0x0A48 }, { 0x0A4B, 0x0A4D }, - { 0x0A70, 0x0A71 }, { 0x0A81, 0x0A82 }, { 0x0ABC, 0x0ABC }, - { 0x0AC1, 0x0AC5 }, { 0x0AC7, 0x0AC8 }, { 0x0ACD, 0x0ACD }, - { 0x0AE2, 0x0AE3 }, { 0x0B01, 0x0B01 }, { 0x0B3C, 0x0B3C }, - { 0x0B3F, 0x0B3F }, { 0x0B41, 0x0B43 }, { 0x0B4D, 0x0B4D }, - { 0x0B56, 0x0B56 }, { 0x0B82, 0x0B82 }, { 0x0BC0, 0x0BC0 }, - { 0x0BCD, 0x0BCD }, { 0x0C3E, 0x0C40 }, { 0x0C46, 0x0C48 }, - { 0x0C4A, 0x0C4D }, { 0x0C55, 0x0C56 }, { 0x0CBC, 0x0CBC }, - { 0x0CBF, 0x0CBF }, { 0x0CC6, 0x0CC6 }, { 0x0CCC, 0x0CCD }, - { 0x0CE2, 0x0CE3 }, { 0x0D41, 0x0D43 }, { 0x0D4D, 0x0D4D }, - { 0x0DCA, 0x0DCA }, { 0x0DD2, 0x0DD4 }, { 0x0DD6, 0x0DD6 }, - { 0x0E31, 0x0E31 }, { 0x0E34, 0x0E3A }, { 0x0E47, 0x0E4E }, - { 0x0EB1, 0x0EB1 }, { 0x0EB4, 0x0EB9 }, { 0x0EBB, 0x0EBC }, - { 0x0EC8, 0x0ECD }, { 0x0F18, 0x0F19 }, { 0x0F35, 0x0F35 }, - { 0x0F37, 0x0F37 }, { 0x0F39, 0x0F39 }, { 0x0F71, 0x0F7E }, - { 0x0F80, 0x0F84 }, { 0x0F86, 0x0F87 }, { 0x0F90, 0x0F97 }, - { 0x0F99, 0x0FBC }, { 0x0FC6, 0x0FC6 }, { 0x102D, 0x1030 }, - { 0x1032, 0x1032 }, { 0x1036, 0x1037 }, { 0x1039, 0x1039 }, - { 0x1058, 0x1059 }, { 0x1160, 0x11FF }, { 0x135F, 0x135F }, - { 0x1712, 0x1714 }, { 0x1732, 0x1734 }, { 0x1752, 0x1753 }, - { 0x1772, 0x1773 }, { 0x17B4, 0x17B5 }, { 0x17B7, 0x17BD }, - { 0x17C6, 0x17C6 }, { 0x17C9, 0x17D3 }, { 0x17DD, 0x17DD }, - { 0x180B, 0x180D }, { 0x18A9, 0x18A9 }, { 0x1920, 0x1922 }, - { 0x1927, 0x1928 }, { 0x1932, 0x1932 }, { 0x1939, 0x193B }, - { 0x1A17, 0x1A18 }, { 0x1B00, 0x1B03 }, { 0x1B34, 0x1B34 }, - { 0x1B36, 0x1B3A }, { 0x1B3C, 0x1B3C }, { 0x1B42, 0x1B42 }, - { 0x1B6B, 0x1B73 }, { 0x1DC0, 0x1DCA }, { 0x1DFE, 0x1DFF }, - { 0x200B, 0x200F }, { 0x202A, 0x202E }, { 0x2060, 0x2063 }, - { 0x206A, 0x206F }, { 0x20D0, 0x20EF }, { 0x302A, 0x302F }, - { 0x3099, 0x309A }, { 0xA806, 0xA806 }, { 0xA80B, 0xA80B }, - { 0xA825, 0xA826 }, { 0xFB1E, 0xFB1E }, { 0xFE00, 0xFE0F }, - { 0xFE20, 0xFE23 }, { 0xFEFF, 0xFEFF }, { 0xFFF9, 0xFFFB }, - { 0x10A01, 0x10A03 }, { 0x10A05, 0x10A06 }, { 0x10A0C, 0x10A0F }, - { 0x10A38, 0x10A3A }, { 0x10A3F, 0x10A3F }, { 0x1D167, 0x1D169 }, - { 0x1D173, 0x1D182 }, { 0x1D185, 0x1D18B }, { 0x1D1AA, 0x1D1AD }, - { 0x1D242, 0x1D244 }, { 0xE0001, 0xE0001 }, { 0xE0020, 0xE007F }, - { 0xE0100, 0xE01EF } - }; - - /* test for 8-bit control characters */ - if (ucs == 0) - return 0; - if (ucs < 32 || (ucs >= 0x7f && ucs < 0xa0)) - return -1; - - /* binary search in table of non-spacing characters */ - if (bisearch(ucs, combining, - sizeof(combining) / sizeof(struct interval) - 1)) - return 0; - - /* if we arrive here, ucs is not a combining or C0/C1 control character */ - - return 1 + - (ucs >= 0x1100 && - (ucs <= 0x115f || /* Hangul Jamo init. consonants */ - ucs == 0x2329 || ucs == 0x232a || - (ucs >= 0x2e80 && ucs <= 0xa4cf && - ucs != 0x303f) || /* CJK ... Yi */ - (ucs >= 0xac00 && ucs <= 0xd7a3) || /* Hangul Syllables */ - (ucs >= 0xf900 && ucs <= 0xfaff) || /* CJK Compatibility Ideographs */ - (ucs >= 0xfe10 && ucs <= 0xfe19) || /* Vertical forms */ - (ucs >= 0xfe30 && ucs <= 0xfe6f) || /* CJK Compatibility Forms */ - (ucs >= 0xff00 && ucs <= 0xff60) || /* Fullwidth Forms */ - (ucs >= 0xffe0 && ucs <= 0xffe6) || - (ucs >= 0x20000 && ucs <= 0x2fffd) || - (ucs >= 0x30000 && ucs <= 0x3fffd))); -} - - -int mk_wcswidth(const char32_t* pwcs, size_t n) -{ - int w, width = 0; - - for (;*pwcs && n-- > 0; pwcs++) - if ((w = mk_wcwidth(*pwcs)) < 0) - return -1; - else - width += w; - - return width; -} - - -/* - * The following functions are the same as mk_wcwidth() and - * mk_wcswidth(), except that spacing characters in the East Asian - * Ambiguous (A) category as defined in Unicode Technical Report #11 - * have a column width of 2. This variant might be useful for users of - * CJK legacy encodings who want to migrate to UCS without changing - * the traditional terminal character-width behaviour. It is not - * otherwise recommended for general use. - */ -int mk_wcwidth_cjk(wchar_t ucs) -{ - /* sorted list of non-overlapping intervals of East Asian Ambiguous - * characters, generated by "uniset +WIDTH-A -cat=Me -cat=Mn -cat=Cf c" */ - static const struct interval ambiguous[] = { - { 0x00A1, 0x00A1 }, { 0x00A4, 0x00A4 }, { 0x00A7, 0x00A8 }, - { 0x00AA, 0x00AA }, { 0x00AE, 0x00AE }, { 0x00B0, 0x00B4 }, - { 0x00B6, 0x00BA }, { 0x00BC, 0x00BF }, { 0x00C6, 0x00C6 }, - { 0x00D0, 0x00D0 }, { 0x00D7, 0x00D8 }, { 0x00DE, 0x00E1 }, - { 0x00E6, 0x00E6 }, { 0x00E8, 0x00EA }, { 0x00EC, 0x00ED }, - { 0x00F0, 0x00F0 }, { 0x00F2, 0x00F3 }, { 0x00F7, 0x00FA }, - { 0x00FC, 0x00FC }, { 0x00FE, 0x00FE }, { 0x0101, 0x0101 }, - { 0x0111, 0x0111 }, { 0x0113, 0x0113 }, { 0x011B, 0x011B }, - { 0x0126, 0x0127 }, { 0x012B, 0x012B }, { 0x0131, 0x0133 }, - { 0x0138, 0x0138 }, { 0x013F, 0x0142 }, { 0x0144, 0x0144 }, - { 0x0148, 0x014B }, { 0x014D, 0x014D }, { 0x0152, 0x0153 }, - { 0x0166, 0x0167 }, { 0x016B, 0x016B }, { 0x01CE, 0x01CE }, - { 0x01D0, 0x01D0 }, { 0x01D2, 0x01D2 }, { 0x01D4, 0x01D4 }, - { 0x01D6, 0x01D6 }, { 0x01D8, 0x01D8 }, { 0x01DA, 0x01DA }, - { 0x01DC, 0x01DC }, { 0x0251, 0x0251 }, { 0x0261, 0x0261 }, - { 0x02C4, 0x02C4 }, { 0x02C7, 0x02C7 }, { 0x02C9, 0x02CB }, - { 0x02CD, 0x02CD }, { 0x02D0, 0x02D0 }, { 0x02D8, 0x02DB }, - { 0x02DD, 0x02DD }, { 0x02DF, 0x02DF }, { 0x0391, 0x03A1 }, - { 0x03A3, 0x03A9 }, { 0x03B1, 0x03C1 }, { 0x03C3, 0x03C9 }, - { 0x0401, 0x0401 }, { 0x0410, 0x044F }, { 0x0451, 0x0451 }, - { 0x2010, 0x2010 }, { 0x2013, 0x2016 }, { 0x2018, 0x2019 }, - { 0x201C, 0x201D }, { 0x2020, 0x2022 }, { 0x2024, 0x2027 }, - { 0x2030, 0x2030 }, { 0x2032, 0x2033 }, { 0x2035, 0x2035 }, - { 0x203B, 0x203B }, { 0x203E, 0x203E }, { 0x2074, 0x2074 }, - { 0x207F, 0x207F }, { 0x2081, 0x2084 }, { 0x20AC, 0x20AC }, - { 0x2103, 0x2103 }, { 0x2105, 0x2105 }, { 0x2109, 0x2109 }, - { 0x2113, 0x2113 }, { 0x2116, 0x2116 }, { 0x2121, 0x2122 }, - { 0x2126, 0x2126 }, { 0x212B, 0x212B }, { 0x2153, 0x2154 }, - { 0x215B, 0x215E }, { 0x2160, 0x216B }, { 0x2170, 0x2179 }, - { 0x2190, 0x2199 }, { 0x21B8, 0x21B9 }, { 0x21D2, 0x21D2 }, - { 0x21D4, 0x21D4 }, { 0x21E7, 0x21E7 }, { 0x2200, 0x2200 }, - { 0x2202, 0x2203 }, { 0x2207, 0x2208 }, { 0x220B, 0x220B }, - { 0x220F, 0x220F }, { 0x2211, 0x2211 }, { 0x2215, 0x2215 }, - { 0x221A, 0x221A }, { 0x221D, 0x2220 }, { 0x2223, 0x2223 }, - { 0x2225, 0x2225 }, { 0x2227, 0x222C }, { 0x222E, 0x222E }, - { 0x2234, 0x2237 }, { 0x223C, 0x223D }, { 0x2248, 0x2248 }, - { 0x224C, 0x224C }, { 0x2252, 0x2252 }, { 0x2260, 0x2261 }, - { 0x2264, 0x2267 }, { 0x226A, 0x226B }, { 0x226E, 0x226F }, - { 0x2282, 0x2283 }, { 0x2286, 0x2287 }, { 0x2295, 0x2295 }, - { 0x2299, 0x2299 }, { 0x22A5, 0x22A5 }, { 0x22BF, 0x22BF }, - { 0x2312, 0x2312 }, { 0x2460, 0x24E9 }, { 0x24EB, 0x254B }, - { 0x2550, 0x2573 }, { 0x2580, 0x258F }, { 0x2592, 0x2595 }, - { 0x25A0, 0x25A1 }, { 0x25A3, 0x25A9 }, { 0x25B2, 0x25B3 }, - { 0x25B6, 0x25B7 }, { 0x25BC, 0x25BD }, { 0x25C0, 0x25C1 }, - { 0x25C6, 0x25C8 }, { 0x25CB, 0x25CB }, { 0x25CE, 0x25D1 }, - { 0x25E2, 0x25E5 }, { 0x25EF, 0x25EF }, { 0x2605, 0x2606 }, - { 0x2609, 0x2609 }, { 0x260E, 0x260F }, { 0x2614, 0x2615 }, - { 0x261C, 0x261C }, { 0x261E, 0x261E }, { 0x2640, 0x2640 }, - { 0x2642, 0x2642 }, { 0x2660, 0x2661 }, { 0x2663, 0x2665 }, - { 0x2667, 0x266A }, { 0x266C, 0x266D }, { 0x266F, 0x266F }, - { 0x273D, 0x273D }, { 0x2776, 0x277F }, { 0xE000, 0xF8FF }, - { 0xFFFD, 0xFFFD }, { 0xF0000, 0xFFFFD }, { 0x100000, 0x10FFFD } - }; - - /* binary search in table of non-spacing characters */ - if (bisearch(ucs, ambiguous, - sizeof(ambiguous) / sizeof(struct interval) - 1)) - return 2; - - return mk_wcwidth(ucs); -} - - -int mk_wcswidth_cjk(const wchar_t *pwcs, size_t n) -{ - int w, width = 0; - - for (;*pwcs && n-- > 0; pwcs++) - if ((w = mk_wcwidth_cjk(*pwcs)) < 0) - return -1; - else - width += w; - - return width; -} - -} diff --git a/src/nix-build/local.mk b/src/nix-build/local.mk deleted file mode 100644 index a2d1c91dfd9d..000000000000 --- a/src/nix-build/local.mk +++ /dev/null @@ -1,9 +0,0 @@ -programs += nix-build - -nix-build_DIR := $(d) - -nix-build_SOURCES := $(d)/nix-build.cc - -nix-build_LIBS = libmain libexpr libstore libutil libformat - -$(eval $(call install-symlink, nix-build, $(bindir)/nix-shell)) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 94d3a27560fe..11ea3b1f7ae1 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -16,6 +16,7 @@ #include "get-drvs.hh" #include "common-eval-args.hh" #include "attr-path.hh" +#include "legacy.hh" using namespace nix; using namespace std::string_literals; @@ -66,11 +67,8 @@ std::vector<string> shellwords(const string & s) return res; } -void mainWrapped(int argc, char * * argv) +static void _main(int argc, char * * argv) { - initNix(); - initGC(); - auto dryRun = false; auto runEnv = std::regex_search(argv[0], std::regex("nix-shell$")); auto pure = false; @@ -403,8 +401,6 @@ void mainWrapped(int argc, char * * argv) } else env[var.first] = var.second; - restoreAffinity(); - /* Run a shell using the derivation's environment. For convenience, source $stdenv/setup to setup additional environment variables and shell functions. Also don't @@ -417,16 +413,20 @@ void mainWrapped(int argc, char * * argv) "dontAddDisableDepTrack=1; " "[ -e $stdenv/setup ] && source $stdenv/setup; " "%3%" + "PATH=\"%4%:$PATH\"; " + "SHELL=%5%; " "set +e; " R"s([ -n "$PS1" ] && PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s" "if [ \"$(type -t runHook)\" = function ]; then runHook shellHook; fi; " "unset NIX_ENFORCE_PURITY; " "shopt -u nullglob; " - "unset TZ; %4%" - "%5%", + "unset TZ; %6%" + "%7%", (Path) tmpDir, (pure ? "" : "p=$PATH; "), (pure ? "" : "PATH=$PATH:$p; unset p; "), + dirOf(shell), + shell, (getenv("TZ") ? (string("export TZ='") + getenv("TZ") + "'; ") : ""), envCommand)); @@ -444,7 +444,9 @@ void mainWrapped(int argc, char * * argv) auto argPtrs = stringsToCharPtrs(args); + restoreAffinity(); restoreSignals(); + restoreMountNamespace(); execvp(shell.c_str(), argPtrs.data()); @@ -500,9 +502,5 @@ void mainWrapped(int argc, char * * argv) } } -int main(int argc, char * * argv) -{ - return handleExceptions(argv[0], [&]() { - return mainWrapped(argc, argv); - }); -} +static RegisterLegacyCommand s1("nix-build", _main); +static RegisterLegacyCommand s2("nix-shell", _main); diff --git a/src/nix-channel/local.mk b/src/nix-channel/local.mk deleted file mode 100644 index c14e8c359ca0..000000000000 --- a/src/nix-channel/local.mk +++ /dev/null @@ -1,7 +0,0 @@ -programs += nix-channel - -nix-channel_DIR := $(d) - -nix-channel_LIBS = libmain libformat libstore libutil - -nix-channel_SOURCES := $(d)/nix-channel.cc diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 2083d3df5cab..8b66cc7e314e 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -1,9 +1,11 @@ #include "shared.hh" #include "globals.hh" #include "download.hh" +#include "store-api.hh" +#include "legacy.hh" + #include <fcntl.h> #include <regex> -#include "store-api.hh" #include <pwd.h> using namespace nix; @@ -157,11 +159,9 @@ static void update(const StringSet & channelNames) replaceSymlink(profile, channelLink); } -int main(int argc, char ** argv) +static int _main(int argc, char ** argv) { - return handleExceptions(argv[0], [&]() { - initNix(); - + { // Figure out the name of the `.nix-channels' file to use auto home = getHome(); channelsList = home + "/.nix-channels"; @@ -255,5 +255,9 @@ int main(int argc, char ** argv) runProgram(settings.nixBinDir + "/nix-env", false, envArgs); break; } - }); + + return 0; + } } + +static RegisterLegacyCommand s1("nix-channel", _main); diff --git a/src/nix-collect-garbage/local.mk b/src/nix-collect-garbage/local.mk deleted file mode 100644 index 02d14cf62199..000000000000 --- a/src/nix-collect-garbage/local.mk +++ /dev/null @@ -1,7 +0,0 @@ -programs += nix-collect-garbage - -nix-collect-garbage_DIR := $(d) - -nix-collect-garbage_SOURCES := $(d)/nix-collect-garbage.cc - -nix-collect-garbage_LIBS = libmain libstore libutil libformat diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index 37fe22f48134..d4060ac937fc 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -2,6 +2,7 @@ #include "profiles.hh" #include "shared.hh" #include "globals.hh" +#include "legacy.hh" #include <iostream> #include <cerrno> @@ -48,12 +49,10 @@ void removeOldGenerations(std::string dir) } } -int main(int argc, char * * argv) +static int _main(int argc, char * * argv) { - bool removeOld = false; - - return handleExceptions(argv[0], [&]() { - initNix(); + { + bool removeOld = false; GCOptions options; @@ -90,5 +89,9 @@ int main(int argc, char * * argv) PrintFreed freed(true, results); store->collectGarbage(options, results); } - }); + + return 0; + } } + +static RegisterLegacyCommand s1("nix-collect-garbage", _main); diff --git a/src/nix-copy-closure/local.mk b/src/nix-copy-closure/local.mk deleted file mode 100644 index 5018ab975b44..000000000000 --- a/src/nix-copy-closure/local.mk +++ /dev/null @@ -1,7 +0,0 @@ -programs += nix-copy-closure - -nix-copy-closure_DIR := $(d) - -nix-copy-closure_LIBS = libmain libformat libstore libutil - -nix-copy-closure_SOURCES := $(d)/nix-copy-closure.cc diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc index dfb1b8fc5dc4..fdcde8b076b5 100755 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -1,13 +1,12 @@ #include "shared.hh" #include "store-api.hh" +#include "legacy.hh" using namespace nix; -int main(int argc, char ** argv) +static int _main(int argc, char ** argv) { - return handleExceptions(argv[0], [&]() { - initNix(); - + { auto gzip = false; auto toMode = true; auto includeOutputs = false; @@ -61,5 +60,9 @@ int main(int argc, char ** argv) from->computeFSClosure(storePaths2, closure, false, includeOutputs); copyPaths(from, to, closure, NoRepair, NoCheckSigs, useSubstitutes); - }); + + return 0; + } } + +static RegisterLegacyCommand s1("nix-copy-closure", _main); diff --git a/src/nix-daemon/local.mk b/src/nix-daemon/local.mk deleted file mode 100644 index 5a4474465b3c..000000000000 --- a/src/nix-daemon/local.mk +++ /dev/null @@ -1,13 +0,0 @@ -programs += nix-daemon - -nix-daemon_DIR := $(d) - -nix-daemon_SOURCES := $(d)/nix-daemon.cc - -nix-daemon_LIBS = libmain libstore libutil libformat - -nix-daemon_LDFLAGS = -pthread - -ifeq ($(OS), SunOS) - nix-daemon_LDFLAGS += -lsocket -endif diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 644fa6681de3..8368c3266142 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -9,6 +9,7 @@ #include "monitor-fd.hh" #include "derivations.hh" #include "finally.hh" +#include "legacy.hh" #include <algorithm> @@ -557,7 +558,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store, ; else if (trusted || name == settings.buildTimeout.name - || name == "connect-timeout") + || name == "connect-timeout" + || (name == "builders" && value == "")) settings.set(name, value); else if (setSubstituters(settings.substituters)) ; @@ -708,7 +710,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, logger->startWork(); // FIXME: race if addToStore doesn't read source? - store.cast<Store>()->addToStore(info, *source, (RepairFlag) repair, + store->addToStore(info, *source, (RepairFlag) repair, dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr); logger->stopWork(); @@ -1057,11 +1059,9 @@ static void daemonLoop(char * * argv) } -int main(int argc, char * * argv) +static int _main(int argc, char * * argv) { - return handleExceptions(argv[0], [&]() { - initNix(); - + { auto stdio = false; parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { @@ -1121,7 +1121,7 @@ int main(int argc, char * * argv) if (res == -1) throw SysError("splicing data from stdin to daemon socket"); else if (res == 0) - return; + return 0; } } } else { @@ -1130,5 +1130,9 @@ int main(int argc, char * * argv) } else { daemonLoop(argv); } - }); + + return 0; + } } + +static RegisterLegacyCommand s1("nix-daemon", _main); diff --git a/src/nix-env/local.mk b/src/nix-env/local.mk deleted file mode 100644 index e80719cd76f7..000000000000 --- a/src/nix-env/local.mk +++ /dev/null @@ -1,7 +0,0 @@ -programs += nix-env - -nix-env_DIR := $(d) - -nix-env_SOURCES := $(wildcard $(d)/*.cc) - -nix-env_LIBS = libexpr libmain libstore libutil libformat diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index f9c8a8d313e7..56ed75daee44 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -13,6 +13,7 @@ #include "json.hh" #include "value-to-json.hh" #include "xml-writer.hh" +#include "legacy.hh" #include <cerrno> #include <ctime> @@ -1311,12 +1312,9 @@ static void opVersion(Globals & globals, Strings opFlags, Strings opArgs) } -int main(int argc, char * * argv) +static int _main(int argc, char * * argv) { - return handleExceptions(argv[0], [&]() { - initNix(); - initGC(); - + { Strings opFlags, opArgs; Operation op = 0; RepairFlag repair = NoRepair; @@ -1428,5 +1426,9 @@ int main(int argc, char * * argv) op(globals, opFlags, opArgs); globals.state->printStats(); - }); + + return 0; + } } + +static RegisterLegacyCommand s1("nix-env", _main); diff --git a/src/nix-instantiate/local.mk b/src/nix-instantiate/local.mk deleted file mode 100644 index 7d1bc5ec9dfb..000000000000 --- a/src/nix-instantiate/local.mk +++ /dev/null @@ -1,7 +0,0 @@ -programs += nix-instantiate - -nix-instantiate_DIR := $(d) - -nix-instantiate_SOURCES := $(d)/nix-instantiate.cc - -nix-instantiate_LIBS = libexpr libmain libstore libutil libformat diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index eb6d34dd8219..a736caa8f056 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -9,6 +9,7 @@ #include "util.hh" #include "store-api.hh" #include "common-eval-args.hh" +#include "legacy.hh" #include <map> #include <iostream> @@ -83,12 +84,9 @@ void processExpr(EvalState & state, const Strings & attrPaths, } -int main(int argc, char * * argv) +static int _main(int argc, char * * argv) { - return handleExceptions(argv[0], [&]() { - initNix(); - initGC(); - + { Strings files; bool readStdin = false; bool fromArgs = false; @@ -171,7 +169,7 @@ int main(int argc, char * * argv) if (p == "") throw Error(format("unable to find '%1%'") % i); std::cout << p << std::endl; } - return; + return 0; } if (readStdin) { @@ -190,5 +188,9 @@ int main(int argc, char * * argv) } state->printStats(); - }); + + return 0; + } } + +static RegisterLegacyCommand s1("nix-instantiate", _main); diff --git a/src/nix-prefetch-url/local.mk b/src/nix-prefetch-url/local.mk deleted file mode 100644 index 3e7735406af0..000000000000 --- a/src/nix-prefetch-url/local.mk +++ /dev/null @@ -1,7 +0,0 @@ -programs += nix-prefetch-url - -nix-prefetch-url_DIR := $(d) - -nix-prefetch-url_SOURCES := $(d)/nix-prefetch-url.cc - -nix-prefetch-url_LIBS = libmain libexpr libstore libutil libformat diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index a3b025723cf1..f54706a8a011 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -6,6 +6,9 @@ #include "eval-inline.hh" #include "common-eval-args.hh" #include "attr-path.hh" +#include "legacy.hh" +#include "finally.hh" +#include "progress-bar.hh" #include <iostream> @@ -44,12 +47,9 @@ string resolveMirrorUri(EvalState & state, string uri) } -int main(int argc, char * * argv) +static int _main(int argc, char * * argv) { - return handleExceptions(argv[0], [&]() { - initNix(); - initGC(); - + { HashType ht = htSHA256; std::vector<string> args; bool printPath = getEnv("PRINT_PATH") != ""; @@ -98,6 +98,11 @@ int main(int argc, char * * argv) if (args.size() > 2) throw UsageError("too many arguments"); + Finally f([]() { stopProgressBar(); }); + + if (isatty(STDERR_FILENO)) + startProgressBar(); + auto store = openStore(); auto state = std::make_unique<EvalState>(myArgs.searchPath, store); @@ -215,11 +220,17 @@ int main(int argc, char * * argv) assert(storePath == store->makeFixedOutputPath(unpack, hash, name)); } + stopProgressBar(); + if (!printPath) printInfo(format("path is '%1%'") % storePath); std::cout << printHash16or32(hash) << std::endl; if (printPath) std::cout << storePath << std::endl; - }); + + return 0; + } } + +static RegisterLegacyCommand s1("nix-prefetch-url", _main); diff --git a/src/nix-store/graphml.cc b/src/nix-store/graphml.cc new file mode 100644 index 000000000000..670fbe227a4c --- /dev/null +++ b/src/nix-store/graphml.cc @@ -0,0 +1,90 @@ +#include "graphml.hh" +#include "util.hh" +#include "store-api.hh" +#include "derivations.hh" + +#include <iostream> + + +using std::cout; + +namespace nix { + + +static inline const string & xmlQuote(const string & s) +{ + // Luckily, store paths shouldn't contain any character that needs to be + // quoted. + return s; +} + + +static string symbolicName(const string & path) +{ + string p = baseNameOf(path); + return string(p, p.find('-') + 1); +} + + +static string makeEdge(const string & src, const string & dst) +{ + return fmt(" <edge source=\"%1%\" target=\"%2%\"/>\n", + xmlQuote(src), xmlQuote(dst)); +} + + +static string makeNode(const ValidPathInfo & info) +{ + return fmt( + " <node id=\"%1%\">\n" + " <data key=\"narSize\">%2%</data>\n" + " <data key=\"name\">%3%</data>\n" + " <data key=\"type\">%4%</data>\n" + " </node>\n", + info.path, + info.narSize, + symbolicName(info.path), + (isDerivation(info.path) ? "derivation" : "output-path")); +} + + +void printGraphML(ref<Store> store, const PathSet & roots) +{ + PathSet workList(roots); + PathSet doneSet; + std::pair<PathSet::iterator,bool> ret; + + cout << "<?xml version='1.0' encoding='utf-8'?>\n" + << "<graphml xmlns='http://graphml.graphdrawing.org/xmlns'\n" + << " xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'\n" + << " xsi:schemaLocation='http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd'>\n" + << "<key id='narSize' for='node' attr.name='narSize' attr.type='int'/>" + << "<key id='name' for='node' attr.name='name' attr.type='string'/>" + << "<key id='type' for='node' attr.name='type' attr.type='string'/>" + << "<graph id='G' edgedefault='directed'>\n"; + + while (!workList.empty()) { + Path path = *(workList.begin()); + workList.erase(path); + + ret = doneSet.insert(path); + if (ret.second == false) continue; + + ValidPathInfo info = *(store->queryPathInfo(path)); + cout << makeNode(info); + + for (auto & p : store->queryPathInfo(path)->references) { + if (p != path) { + workList.insert(p); + cout << makeEdge(path, p); + } + } + + } + + cout << "</graph>\n"; + cout << "</graphml>\n"; +} + + +} diff --git a/src/nix-store/xmlgraph.hh b/src/nix-store/graphml.hh index a6e7d4e2805a..b78df1e49a67 100644 --- a/src/nix-store/xmlgraph.hh +++ b/src/nix-store/graphml.hh @@ -6,6 +6,6 @@ namespace nix { class Store; -void printXmlGraph(ref<Store> store, const PathSet & roots); +void printGraphML(ref<Store> store, const PathSet & roots); } diff --git a/src/nix-store/local.mk b/src/nix-store/local.mk deleted file mode 100644 index ade0b233adf3..000000000000 --- a/src/nix-store/local.mk +++ /dev/null @@ -1,9 +0,0 @@ -programs += nix-store - -nix-store_DIR := $(d) - -nix-store_SOURCES := $(wildcard $(d)/*.cc) - -nix-store_LIBS = libmain libstore libutil libformat - -nix-store_LDFLAGS = -lbz2 -pthread $(SODIUM_LIBS) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index fe68f681ae28..a9ad14762e62 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -8,7 +8,8 @@ #include "shared.hh" #include "util.hh" #include "worker-protocol.hh" -#include "xmlgraph.hh" +#include "graphml.hh" +#include "legacy.hh" #include <iostream> #include <algorithm> @@ -273,7 +274,7 @@ static void opQuery(Strings opFlags, Strings opArgs) enum QueryType { qDefault, qOutputs, qRequisites, qReferences, qReferrers , qReferrersClosure, qDeriver, qBinding, qHash, qSize - , qTree, qGraph, qXml, qResolve, qRoots }; + , qTree, qGraph, qGraphML, qResolve, qRoots }; QueryType query = qDefault; bool useOutput = false; bool includeOutputs = false; @@ -299,7 +300,7 @@ static void opQuery(Strings opFlags, Strings opArgs) else if (i == "--size") query = qSize; else if (i == "--tree") query = qTree; else if (i == "--graph") query = qGraph; - else if (i == "--xml") query = qXml; + else if (i == "--graphml") query = qGraphML; else if (i == "--resolve") query = qResolve; else if (i == "--roots") query = qRoots; else if (i == "--use-output" || i == "-u") useOutput = true; @@ -403,13 +404,13 @@ static void opQuery(Strings opFlags, Strings opArgs) break; } - case qXml: { + case qGraphML: { PathSet roots; for (auto & i : opArgs) { PathSet paths = maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise); roots.insert(paths.begin(), paths.end()); } - printXmlGraph(ref<Store>(store), roots); + printGraphML(ref<Store>(store), roots); break; } @@ -993,11 +994,9 @@ static void opVersion(Strings opFlags, Strings opArgs) /* Scan the arguments; find the operation, set global flags, put all other flags in a list, and put all other arguments in another list. */ -int main(int argc, char * * argv) +static int _main(int argc, char * * argv) { - return handleExceptions(argv[0], [&]() { - initNix(); - + { Strings opFlags, opArgs; Operation op = 0; @@ -1084,5 +1083,9 @@ int main(int argc, char * * argv) store = openStore(); op(opFlags, opArgs); - }); + + return 0; + } } + +static RegisterLegacyCommand s1("nix-store", _main); diff --git a/src/nix-store/xmlgraph.cc b/src/nix-store/xmlgraph.cc deleted file mode 100644 index 0f7be7f7a02d..000000000000 --- a/src/nix-store/xmlgraph.cc +++ /dev/null @@ -1,66 +0,0 @@ -#include "xmlgraph.hh" -#include "util.hh" -#include "store-api.hh" - -#include <iostream> - - -using std::cout; - -namespace nix { - - -static inline const string & xmlQuote(const string & s) -{ - // Luckily, store paths shouldn't contain any character that needs to be - // quoted. - return s; -} - - -static string makeEdge(const string & src, const string & dst) -{ - format f = format(" <edge src=\"%1%\" dst=\"%2%\"/>\n") - % xmlQuote(src) % xmlQuote(dst); - return f.str(); -} - - -static string makeNode(const string & id) -{ - format f = format(" <node name=\"%1%\"/>\n") % xmlQuote(id); - return f.str(); -} - - -void printXmlGraph(ref<Store> store, const PathSet & roots) -{ - PathSet workList(roots); - PathSet doneSet; - - cout << "<?xml version='1.0' encoding='utf-8'?>\n" - << "<nix>\n"; - - while (!workList.empty()) { - Path path = *(workList.begin()); - workList.erase(path); - - if (doneSet.find(path) != doneSet.end()) continue; - doneSet.insert(path); - - cout << makeNode(path); - - for (auto & p : store->queryPathInfo(path)->references) { - if (p != path) { - workList.insert(p); - cout << makeEdge(p, path); - } - } - - } - - cout << "</nix>\n"; -} - - -} diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 91711c8b46da..96bd453d87b4 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -69,12 +69,12 @@ struct CmdCopy : StorePathsCommand }, #ifdef ENABLE_S3 Example{ - "To populate the current folder build output to a S3 binary cache:", - "nix copy --to s3://my-bucket?region=eu-west-1" + "To copy Hello to an S3 binary cache:", + "nix copy --to s3://my-bucket?region=eu-west-1 nixpkgs.hello" }, Example{ - "To populate the current folder build output to an S3-compatible binary cache:", - "nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com" + "To copy Hello to an S3-compatible binary cache:", + "nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com nixpkgs.hello" }, #endif }; diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc new file mode 100644 index 000000000000..7b5444619470 --- /dev/null +++ b/src/nix/doctor.cc @@ -0,0 +1,124 @@ +#include "command.hh" +#include "serve-protocol.hh" +#include "shared.hh" +#include "store-api.hh" +#include "worker-protocol.hh" + +using namespace nix; + +std::string formatProtocol(unsigned int proto) +{ + if (proto) { + auto major = GET_PROTOCOL_MAJOR(proto) >> 8; + auto minor = GET_PROTOCOL_MINOR(proto); + return (format("%1%.%2%") % major % minor).str(); + } + return "unknown"; +} + +struct CmdDoctor : StoreCommand +{ + bool success = true; + + std::string name() override + { + return "doctor"; + } + + std::string description() override + { + return "check your system for potential problems"; + } + + void run(ref<Store> store) override + { + std::cout << "Store uri: " << store->getUri() << std::endl; + std::cout << std::endl; + + auto type = getStoreType(); + + if (type < tOther) { + success &= checkNixInPath(); + success &= checkProfileRoots(store); + } + success &= checkStoreProtocol(store->getProtocol()); + + if (!success) + throw Exit(2); + } + + bool checkNixInPath() + { + PathSet dirs; + + for (auto & dir : tokenizeString<Strings>(getEnv("PATH"), ":")) + if (pathExists(dir + "/nix-env")) + dirs.insert(dirOf(canonPath(dir + "/nix-env", true))); + + if (dirs.size() != 1) { + std::cout << "Warning: multiple versions of nix found in PATH." << std::endl; + std::cout << std::endl; + for (auto & dir : dirs) + std::cout << " " << dir << std::endl; + std::cout << std::endl; + return false; + } + + return true; + } + + bool checkProfileRoots(ref<Store> store) + { + PathSet dirs; + + for (auto & dir : tokenizeString<Strings>(getEnv("PATH"), ":")) { + Path profileDir = dirOf(dir); + try { + Path userEnv = canonPath(profileDir, true); + + if (store->isStorePath(userEnv) && hasSuffix(userEnv, "user-environment")) { + while (profileDir.find("/profiles/") == std::string::npos && isLink(profileDir)) + profileDir = absPath(readLink(profileDir), dirOf(profileDir)); + + if (profileDir.find("/profiles/") == std::string::npos) + dirs.insert(dir); + } + } catch (SysError &) {} + } + + if (!dirs.empty()) { + std::cout << "Warning: found profiles outside of " << settings.nixStateDir << "/profiles." << std::endl; + std::cout << "The generation this profile points to might not have a gcroot and could be" << std::endl; + std::cout << "garbage collected, resulting in broken symlinks." << std::endl; + std::cout << std::endl; + for (auto & dir : dirs) + std::cout << " " << dir << std::endl; + std::cout << std::endl; + return false; + } + + return true; + } + + bool checkStoreProtocol(unsigned int storeProto) + { + unsigned int clientProto = GET_PROTOCOL_MAJOR(SERVE_PROTOCOL_VERSION) == GET_PROTOCOL_MAJOR(storeProto) + ? SERVE_PROTOCOL_VERSION + : PROTOCOL_VERSION; + + if (clientProto != storeProto) { + std::cout << "Warning: protocol version of this client does not match the store." << std::endl; + std::cout << "While this is not necessarily a problem it's recommended to keep the client in" << std::endl; + std::cout << "sync with the daemon." << std::endl; + std::cout << std::endl; + std::cout << "Client protocol: " << formatProtocol(clientProto) << std::endl; + std::cout << "Store protocol: " << formatProtocol(storeProto) << std::endl; + std::cout << std::endl; + return false; + } + + return true; + } +}; + +static RegisterCommand r1(make_ref<CmdDoctor>()); diff --git a/src/nix/edit.cc b/src/nix/edit.cc index c9671f76d0fa..d8d5895bd867 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -3,6 +3,7 @@ #include "eval.hh" #include "attr-path.hh" #include "progress-bar.hh" +#include "affinity.hh" #include <unistd.h> @@ -72,6 +73,10 @@ struct CmdEdit : InstallableCommand stopProgressBar(); + restoreAffinity(); + restoreSignals(); + restoreMountNamespace(); + execvp(args.front().c_str(), stringsToCharPtrs(args).data()); throw SysError("cannot run editor '%s'", editor); diff --git a/src/nix/local.mk b/src/nix/local.mk index f76da194467c..40a0e8d6bde1 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -2,10 +2,24 @@ programs += nix nix_DIR := $(d) -nix_SOURCES := $(wildcard $(d)/*.cc) $(wildcard src/linenoise/*.cpp) +nix_SOURCES := \ + $(wildcard $(d)/*.cc) \ + $(wildcard src/build-remote/*.cc) \ + $(wildcard src/nix-build/*.cc) \ + $(wildcard src/nix-channel/*.cc) \ + $(wildcard src/nix-collect-garbage/*.cc) \ + $(wildcard src/nix-copy-closure/*.cc) \ + $(wildcard src/nix-daemon/*.cc) \ + $(wildcard src/nix-env/*.cc) \ + $(wildcard src/nix-instantiate/*.cc) \ + $(wildcard src/nix-prefetch-url/*.cc) \ + $(wildcard src/nix-store/*.cc) \ nix_LIBS = libexpr libmain libstore libutil libformat -nix_LDFLAGS = -pthread +nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) -$(eval $(call install-symlink, nix, $(bindir)/nix-hash)) +$(foreach name, \ + nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \ + $(eval $(call install-symlink, nix, $(bindir)/$(name)))) +$(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote)) diff --git a/src/nix/main.cc b/src/nix/main.cc index 69791e223c22..64c1dc35787c 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -67,9 +67,6 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs void mainWrapped(int argc, char * * argv) { - verbosity = lvlError; - settings.verboseBuild = false; - /* The chroot helper needs to be run before any threads have been started. */ if (argc > 0 && argv[0] == chrootHelperName) { @@ -88,6 +85,9 @@ void mainWrapped(int argc, char * * argv) if (legacy) return legacy(argc, argv); } + verbosity = lvlError; + settings.verboseBuild = false; + NixArgs args; args.parseCmdline(argvToStrings(argc, argv)); diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 1bbe256b2d8b..ff8477865b5f 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -1,8 +1,12 @@ #include <iostream> #include <cstdlib> +#include <cstring> +#include <climits> #include <setjmp.h> +#include <editline.h> + #include "shared.hh" #include "eval.hh" #include "eval-inline.hh" @@ -15,8 +19,6 @@ #include "command.hh" #include "finally.hh" -#include "src/linenoise/linenoise.h" - namespace nix { #define ESC_RED "\033[31m" @@ -118,17 +120,71 @@ NixRepl::NixRepl(const Strings & searchPath, nix::ref<Store> store) NixRepl::~NixRepl() { - linenoiseHistorySave(historyFile.c_str()); + write_history(historyFile.c_str()); } - static NixRepl * curRepl; // ugly -static void completionCallback(const char * s, linenoiseCompletions *lc) -{ - /* Otherwise, return all symbols that start with the prefix. */ - for (auto & c : curRepl->completePrefix(s)) - linenoiseAddCompletion(lc, c.c_str()); +static char * completionCallback(char * s, int *match) { + auto possible = curRepl->completePrefix(s); + if (possible.size() == 1) { + *match = 1; + auto *res = strdup(possible.begin()->c_str() + strlen(s)); + if (!res) throw Error("allocation failure"); + return res; + } else if (possible.size() > 1) { + auto checkAllHaveSameAt = [&](size_t pos) { + auto &first = *possible.begin(); + for (auto &p : possible) { + if (p.size() <= pos || p[pos] != first[pos]) + return false; + } + return true; + }; + size_t start = strlen(s); + size_t len = 0; + while (checkAllHaveSameAt(start + len)) ++len; + if (len > 0) { + *match = 1; + auto *res = strdup(std::string(*possible.begin(), start, len).c_str()); + if (!res) throw Error("allocation failure"); + return res; + } + } + + *match = 0; + return nullptr; +} + +static int listPossibleCallback(char *s, char ***avp) { + auto possible = curRepl->completePrefix(s); + + if (possible.size() > (INT_MAX / sizeof(char*))) + throw Error("too many completions"); + + int ac = 0; + char **vp = nullptr; + + auto check = [&](auto *p) { + if (!p) { + if (vp) { + while (--ac >= 0) + free(vp[ac]); + free(vp); + } + throw Error("allocation failure"); + } + return p; + }; + + vp = check((char **)malloc(possible.size() * sizeof(char*))); + + for (auto & p : possible) + vp[ac++] = check(strdup(p.c_str())); + + *avp = vp; + + return ac; } @@ -143,12 +199,16 @@ void NixRepl::mainLoop(const std::vector<std::string> & files) reloadFiles(); if (!loadedFiles.empty()) std::cout << std::endl; + // Allow nix-repl specific settings in .inputrc + rl_readline_name = "nix-repl"; createDirs(dirOf(historyFile)); - linenoiseHistorySetMaxLen(1000); - linenoiseHistoryLoad(historyFile.c_str()); - + el_hist_size = 1000; + read_history(historyFile.c_str()); + // rl_initialize(); + // linenoiseSetCompletionCallback(completionCallback); curRepl = this; - linenoiseSetCompletionCallback(completionCallback); + rl_set_complete_func(completionCallback); + rl_set_list_possib_func(listPossibleCallback); std::string input; @@ -174,12 +234,6 @@ void NixRepl::mainLoop(const std::vector<std::string> & files) printMsg(lvlError, format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg()); } - if (input.size() > 0) { - // Remove trailing newline before adding to history - input.erase(input.size() - 1); - linenoiseHistoryAdd(input.c_str()); - } - // We handled the current input fully, so we should clear it // and read brand new input. input.clear(); @@ -190,19 +244,10 @@ void NixRepl::mainLoop(const std::vector<std::string> & files) bool NixRepl::getLine(string & input, const std::string &prompt) { - char * s = linenoise(prompt.c_str()); + char * s = readline(prompt.c_str()); Finally doFree([&]() { free(s); }); - if (!s) { - switch (auto type = linenoiseKeyType()) { - case 1: // ctrl-C - input = ""; - return true; - case 2: // ctrl-D - return false; - default: - throw Error(format("Unexpected linenoise keytype: %1%") % type); - } - } + if (!s) + return false; input += s; input += '\n'; return true; @@ -285,6 +330,8 @@ static int runProgram(const string & program, const Strings & args) if (pid == -1) throw SysError("forking"); if (pid == 0) { restoreAffinity(); + restoreSignals(); + restoreMountNamespace(); execvp(program.c_str(), stringsToCharPtrs(args2).data()); _exit(1); } diff --git a/src/nix/run.cc b/src/nix/run.cc index 35b763345872..1297072989b9 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -153,9 +153,9 @@ struct CmdRun : InstallablesCommand stopProgressBar(); - restoreSignals(); - restoreAffinity(); + restoreSignals(); + restoreMountNamespace(); /* If this is a diverted store (i.e. its "logical" location (typically /nix/store) differs from its "physical" location diff --git a/src/nix/search.cc b/src/nix/search.cc index 4cb1efa7955b..e086de2260a6 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -173,10 +173,12 @@ struct CmdSearch : SourceExprCommand, MixJSON jsonElem.attr("description", description); } else { + auto name = hilite(parsed.name, nameMatch, "\e[0;2m") + + std::string(parsed.fullName, parsed.name.length()); results[attrPath] = fmt( "* %s (%s)\n %s\n", wrap("\e[0;1m", hilite(attrPath, attrPathMatch, "\e[0;1m")), - wrap("\e[0;2m", hilite(parsed.fullName, nameMatch, "\e[0;2m")), + wrap("\e[0;2m", hilite(name, nameMatch, "\e[0;2m")), hilite(description, descriptionMatch, ANSI_NORMAL)); } } diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 6540208a8a2c..7ef571561a0e 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -120,7 +120,7 @@ struct CmdVerify : StorePathsCommand for (auto sig : sigs) { if (sigsSeen.count(sig)) continue; sigsSeen.insert(sig); - if (info->checkSignature(publicKeys, sig)) + if (validSigs < ValidPathInfo::maxSigs && info->checkSignature(publicKeys, sig)) validSigs++; } }; diff --git a/tests/build-remote.sh b/tests/build-remote.sh index 9bca0f4a3856..ddd68f327a15 100644 --- a/tests/build-remote.sh +++ b/tests/build-remote.sh @@ -11,7 +11,8 @@ rm -rf $TEST_ROOT/store0 $TEST_ROOT/store1 nix build -f build-hook.nix -o $TEST_ROOT/result --max-jobs 0 \ --sandbox-paths /nix/store --sandbox-build-dir /build-tmp \ - --builders "$TEST_ROOT/store0; $TEST_ROOT/store1 - - 1 1 foo" + --builders "$TEST_ROOT/store0; $TEST_ROOT/store1 - - 1 1 foo" \ + --system-features foo outPath=$TEST_ROOT/result diff --git a/tests/check-refs.sh b/tests/check-refs.sh index 34ee22cfc8f7..16bbabc40985 100644 --- a/tests/check-refs.sh +++ b/tests/check-refs.sh @@ -1,5 +1,7 @@ source common.sh +clearStore + RESULT=$TEST_ROOT/result dep=$(nix-build -o $RESULT check-refs.nix -A dep) diff --git a/tests/check-reqs.sh b/tests/check-reqs.sh index 77689215def1..e9f65fc2a6d3 100644 --- a/tests/check-reqs.sh +++ b/tests/check-reqs.sh @@ -1,5 +1,7 @@ source common.sh +clearStore + RESULT=$TEST_ROOT/result nix-build -o $RESULT check-reqs.nix -A test1 diff --git a/tests/common.sh.in b/tests/common.sh.in index 2ee2f589dae4..6a523ca9d832 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -85,16 +85,13 @@ killDaemon() { trap "" EXIT } -canUseSandbox() { - if [[ $(uname) != Linux ]]; then return 1; fi - - if [ ! -L /proc/self/ns/user ]; then - echo "Kernel doesn't support user namespaces, skipping this test..." - return 1 - fi +if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then + _canUseSandbox=1 +fi - if ! unshare --user true ; then - echo "Unprivileged user namespaces disabled by sysctl, skipping this test..." +canUseSandbox() { + if [[ ! $_canUseSandbox ]]; then + echo "Sandboxing not supported, skipping this test..." return 1 fi diff --git a/tests/init.sh b/tests/init.sh index e5353598bcc4..19a12c1e2d9e 100644 --- a/tests/init.sh +++ b/tests/init.sh @@ -16,6 +16,7 @@ mkdir "$NIX_CONF_DIR" cat > "$NIX_CONF_DIR"/nix.conf <<EOF build-users-group = keep-derivations = false +sandbox = false include nix.conf.extra EOF diff --git a/tests/signing.sh b/tests/signing.sh index 46929639199d..9e29e3fbf063 100644 --- a/tests/signing.sh +++ b/tests/signing.sh @@ -62,6 +62,10 @@ outPathCA=$(IMPURE_VAR1=foo IMPURE_VAR2=bar nix-build ./fixed.nix -A good.0 --no nix verify $outPathCA nix verify $outPathCA --sigs-needed 1000 +# Check that signing a content-addressed path doesn't overflow validSigs +nix sign-paths --key-file $TEST_ROOT/sk1 $outPathCA +nix verify -r $outPathCA --sigs-needed 1000 --trusted-public-keys $pk1 + # Copy to a binary cache. nix copy --to file://$cacheDir $outPath2 |