diff options
238 files changed, 19593 insertions, 11555 deletions
diff --git a/.gitignore b/.gitignore index 0f2f3ddeec14..b75c5d489050 100644 --- a/.gitignore +++ b/.gitignore @@ -81,6 +81,9 @@ perl/Makefile.config /tests/common.sh /tests/dummy /tests/result* +/tests/restricted-innocent +/tests/shell +/tests/shell.drv # /tests/lang/ /tests/lang/*.out diff --git a/.version b/.version new file mode 100644 index 000000000000..c0943d3e98da --- /dev/null +++ b/.version @@ -0,0 +1 @@ +2.3 \ No newline at end of file diff --git a/Makefile b/Makefile index 834f84b286bf..45a3338ed21c 100644 --- a/Makefile +++ b/Makefile @@ -5,17 +5,7 @@ makefiles = \ src/libmain/local.mk \ src/libexpr/local.mk \ src/nix/local.mk \ - src/nix-store/local.mk \ - src/nix-instantiate/local.mk \ - src/nix-env/local.mk \ - src/nix-daemon/local.mk \ - src/nix-collect-garbage/local.mk \ - src/nix-copy-closure/local.mk \ - src/nix-prefetch-url/local.mk \ src/resolve-system-dependencies/local.mk \ - src/nix-channel/local.mk \ - src/nix-build/local.mk \ - src/build-remote/local.mk \ scripts/local.mk \ corepkgs/local.mk \ misc/systemd/local.mk \ diff --git a/Makefile.config.in b/Makefile.config.in index a9785dc73955..59730b646387 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -1,4 +1,6 @@ +AR = @AR@ BDW_GC_LIBS = @BDW_GC_LIBS@ +BUILD_SHARED_LIBS = @BUILD_SHARED_LIBS@ CC = @CC@ CFLAGS = @CFLAGS@ CXX = @CXX@ @@ -6,7 +8,6 @@ CXXFLAGS = @CXXFLAGS@ ENABLE_S3 = @ENABLE_S3@ HAVE_SODIUM = @HAVE_SODIUM@ HAVE_READLINE = @HAVE_READLINE@ -HAVE_BROTLI = @HAVE_BROTLI@ HAVE_SECCOMP = @HAVE_SECCOMP@ LIBCURL_LIBS = @LIBCURL_LIBS@ OPENSSL_LIBS = @OPENSSL_LIBS@ @@ -16,9 +17,9 @@ SODIUM_LIBS = @SODIUM_LIBS@ LIBLZMA_LIBS = @LIBLZMA_LIBS@ SQLITE3_LIBS = @SQLITE3_LIBS@ LIBBROTLI_LIBS = @LIBBROTLI_LIBS@ +EDITLINE_LIBS = @EDITLINE_LIBS@ bash = @bash@ bindir = @bindir@ -brotli = @brotli@ lsof = @lsof@ datadir = @datadir@ datarootdir = @datarootdir@ diff --git a/config/config.guess b/config/config.guess index 137bedf2e28b..d4fb3213ec7a 100755 --- a/config/config.guess +++ b/config/config.guess @@ -1,14 +1,12 @@ #! /bin/sh # Attempt to guess a canonical system name. -# Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, -# 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, -# 2011, 2012 Free Software Foundation, Inc. +# Copyright 1992-2018 Free Software Foundation, Inc. -timestamp='2012-08-14' +timestamp='2018-08-02' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or +# the Free Software Foundation; either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, but @@ -17,24 +15,22 @@ timestamp='2012-08-14' # General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with this program; if not, see <http://www.gnu.org/licenses/>. +# along with this program; if not, see <https://www.gnu.org/licenses/>. # # As a special exception to the GNU General Public License, if you # distribute this file as part of a program that contains a # configuration script generated by Autoconf, you may include it under -# the same distribution terms that you use for the rest of that program. - - -# Originally written by Per Bothner. Please send patches (context -# diff format) to <config-patches@gnu.org> and include a ChangeLog -# entry. +# the same distribution terms that you use for the rest of that +# program. This Exception is an additional permission under section 7 +# of the GNU General Public License, version 3 ("GPLv3"). # -# This script attempts to guess a canonical system name similar to -# config.sub. If it succeeds, it prints the system name on stdout, and -# exits with 0. Otherwise, it exits with 1. +# Originally written by Per Bothner; maintained since 2000 by Ben Elliston. # # You can get the latest version of this script from: -# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD +# https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess +# +# Please send patches to <config-patches@gnu.org>. + me=`echo "$0" | sed -e 's,.*/,,'` @@ -43,7 +39,7 @@ Usage: $0 [OPTION] Output the configuration name of the system \`$me' is run on. -Operation modes: +Options: -h, --help print this help, then exit -t, --time-stamp print date of last modification, then exit -v, --version print version number, then exit @@ -54,9 +50,7 @@ version="\ GNU config.guess ($timestamp) Originally written by Per Bothner. -Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, -2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 -Free Software Foundation, Inc. +Copyright 1992-2018 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -90,8 +84,6 @@ if test $# != 0; then exit 1 fi -trap 'exit 1' 1 2 15 - # CC_FOR_BUILD -- compiler used by this script. Note that the use of a # compiler to aid in system detection is discouraged as it requires # temporary files to be created and, as you can see below, it is a @@ -102,34 +94,39 @@ trap 'exit 1' 1 2 15 # Portable tmp directory creation inspired by the Autoconf team. -set_cc_for_build=' -trap "exitcode=\$?; (rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null) && exit \$exitcode" 0 ; -trap "rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null; exit 1" 1 2 13 15 ; -: ${TMPDIR=/tmp} ; - { tmp=`(umask 077 && mktemp -d "$TMPDIR/cgXXXXXX") 2>/dev/null` && test -n "$tmp" && test -d "$tmp" ; } || - { test -n "$RANDOM" && tmp=$TMPDIR/cg$$-$RANDOM && (umask 077 && mkdir $tmp) ; } || - { tmp=$TMPDIR/cg-$$ && (umask 077 && mkdir $tmp) && echo "Warning: creating insecure temp directory" >&2 ; } || - { echo "$me: cannot create a temporary directory in $TMPDIR" >&2 ; exit 1 ; } ; -dummy=$tmp/dummy ; -tmpfiles="$dummy.c $dummy.o $dummy.rel $dummy" ; -case $CC_FOR_BUILD,$HOST_CC,$CC in - ,,) echo "int x;" > $dummy.c ; - for c in cc gcc c89 c99 ; do - if ($c -c -o $dummy.o $dummy.c) >/dev/null 2>&1 ; then - CC_FOR_BUILD="$c"; break ; - fi ; - done ; - if test x"$CC_FOR_BUILD" = x ; then - CC_FOR_BUILD=no_compiler_found ; - fi - ;; - ,,*) CC_FOR_BUILD=$CC ;; - ,*,*) CC_FOR_BUILD=$HOST_CC ;; -esac ; set_cc_for_build= ;' +tmp= +# shellcheck disable=SC2172 +trap 'test -z "$tmp" || rm -fr "$tmp"' 1 2 13 15 +trap 'exitcode=$?; test -z "$tmp" || rm -fr "$tmp"; exit $exitcode' 0 + +set_cc_for_build() { + : "${TMPDIR=/tmp}" + # shellcheck disable=SC2039 + { tmp=`(umask 077 && mktemp -d "$TMPDIR/cgXXXXXX") 2>/dev/null` && test -n "$tmp" && test -d "$tmp" ; } || + { test -n "$RANDOM" && tmp=$TMPDIR/cg$$-$RANDOM && (umask 077 && mkdir "$tmp" 2>/dev/null) ; } || + { tmp=$TMPDIR/cg-$$ && (umask 077 && mkdir "$tmp" 2>/dev/null) && echo "Warning: creating insecure temp directory" >&2 ; } || + { echo "$me: cannot create a temporary directory in $TMPDIR" >&2 ; exit 1 ; } + dummy=$tmp/dummy + case ${CC_FOR_BUILD-},${HOST_CC-},${CC-} in + ,,) echo "int x;" > "$dummy.c" + for driver in cc gcc c89 c99 ; do + if ($driver -c -o "$dummy.o" "$dummy.c") >/dev/null 2>&1 ; then + CC_FOR_BUILD="$driver" + break + fi + done + if test x"$CC_FOR_BUILD" = x ; then + CC_FOR_BUILD=no_compiler_found + fi + ;; + ,,*) CC_FOR_BUILD=$CC ;; + ,*,*) CC_FOR_BUILD=$HOST_CC ;; + esac +} # This is needed to find uname on a Pyramid OSx when run in the BSD universe. # (ghazi@noc.rutgers.edu 1994-08-24) -if (test -f /.attbin/uname) >/dev/null 2>&1 ; then +if test -f /.attbin/uname ; then PATH=$PATH:/.attbin ; export PATH fi @@ -138,9 +135,37 @@ UNAME_RELEASE=`(uname -r) 2>/dev/null` || UNAME_RELEASE=unknown UNAME_SYSTEM=`(uname -s) 2>/dev/null` || UNAME_SYSTEM=unknown UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown +case "$UNAME_SYSTEM" in +Linux|GNU|GNU/*) + # If the system lacks a compiler, then just pick glibc. + # We could probably try harder. + LIBC=gnu + + set_cc_for_build + cat <<-EOF > "$dummy.c" + #include <features.h> + #if defined(__UCLIBC__) + LIBC=uclibc + #elif defined(__dietlibc__) + LIBC=dietlibc + #else + LIBC=gnu + #endif + EOF + eval "`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^LIBC' | sed 's, ,,g'`" + + # If ldd exists, use it to detect musl libc. + if command -v ldd >/dev/null && \ + ldd --version 2>&1 | grep -q ^musl + then + LIBC=musl + fi + ;; +esac + # Note: order is significant - the case branches are not exclusive. -case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in +case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in *:NetBSD:*:*) # NetBSD (nbsd) targets should (where applicable) match one or # more of the tuples: *-*-netbsdelf*, *-*-netbsdaout*, @@ -153,21 +178,31 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in # Note: NetBSD doesn't particularly care about the vendor # portion of the name. We always set it to "unknown". sysctl="sysctl -n hw.machine_arch" - UNAME_MACHINE_ARCH=`(/sbin/$sysctl 2>/dev/null || \ - /usr/sbin/$sysctl 2>/dev/null || echo unknown)` - case "${UNAME_MACHINE_ARCH}" in + UNAME_MACHINE_ARCH=`(uname -p 2>/dev/null || \ + "/sbin/$sysctl" 2>/dev/null || \ + "/usr/sbin/$sysctl" 2>/dev/null || \ + echo unknown)` + case "$UNAME_MACHINE_ARCH" in armeb) machine=armeb-unknown ;; arm*) machine=arm-unknown ;; sh3el) machine=shl-unknown ;; sh3eb) machine=sh-unknown ;; sh5el) machine=sh5le-unknown ;; - *) machine=${UNAME_MACHINE_ARCH}-unknown ;; + earmv*) + arch=`echo "$UNAME_MACHINE_ARCH" | sed -e 's,^e\(armv[0-9]\).*$,\1,'` + endian=`echo "$UNAME_MACHINE_ARCH" | sed -ne 's,^.*\(eb\)$,\1,p'` + machine="${arch}${endian}"-unknown + ;; + *) machine="$UNAME_MACHINE_ARCH"-unknown ;; esac # The Operating System including object format, if it has switched - # to ELF recently, or will in the future. - case "${UNAME_MACHINE_ARCH}" in + # to ELF recently (or will in the future) and ABI. + case "$UNAME_MACHINE_ARCH" in + earm*) + os=netbsdelf + ;; arm*|i386|m68k|ns32k|sh3*|sparc|vax) - eval $set_cc_for_build + set_cc_for_build if echo __ELF__ | $CC_FOR_BUILD -E - 2>/dev/null \ | grep -q __ELF__ then @@ -182,44 +217,67 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in os=netbsd ;; esac + # Determine ABI tags. + case "$UNAME_MACHINE_ARCH" in + earm*) + expr='s/^earmv[0-9]/-eabi/;s/eb$//' + abi=`echo "$UNAME_MACHINE_ARCH" | sed -e "$expr"` + ;; + esac # The OS release # Debian GNU/NetBSD machines have a different userland, and # thus, need a distinct triplet. However, they do not need # kernel version information, so it can be replaced with a # suitable tag, in the style of linux-gnu. - case "${UNAME_VERSION}" in + case "$UNAME_VERSION" in Debian*) release='-gnu' ;; *) - release=`echo ${UNAME_RELEASE}|sed -e 's/[-_].*/\./'` + release=`echo "$UNAME_RELEASE" | sed -e 's/[-_].*//' | cut -d. -f1,2` ;; esac # Since CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM: # contains redundant information, the shorter form: # CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used. - echo "${machine}-${os}${release}" + echo "$machine-${os}${release}${abi-}" exit ;; *:Bitrig:*:*) UNAME_MACHINE_ARCH=`arch | sed 's/Bitrig.//'` - echo ${UNAME_MACHINE_ARCH}-unknown-bitrig${UNAME_RELEASE} + echo "$UNAME_MACHINE_ARCH"-unknown-bitrig"$UNAME_RELEASE" exit ;; *:OpenBSD:*:*) UNAME_MACHINE_ARCH=`arch | sed 's/OpenBSD.//'` - echo ${UNAME_MACHINE_ARCH}-unknown-openbsd${UNAME_RELEASE} + echo "$UNAME_MACHINE_ARCH"-unknown-openbsd"$UNAME_RELEASE" + exit ;; + *:LibertyBSD:*:*) + UNAME_MACHINE_ARCH=`arch | sed 's/^.*BSD\.//'` + echo "$UNAME_MACHINE_ARCH"-unknown-libertybsd"$UNAME_RELEASE" + exit ;; + *:MidnightBSD:*:*) + echo "$UNAME_MACHINE"-unknown-midnightbsd"$UNAME_RELEASE" exit ;; *:ekkoBSD:*:*) - echo ${UNAME_MACHINE}-unknown-ekkobsd${UNAME_RELEASE} + echo "$UNAME_MACHINE"-unknown-ekkobsd"$UNAME_RELEASE" exit ;; *:SolidBSD:*:*) - echo ${UNAME_MACHINE}-unknown-solidbsd${UNAME_RELEASE} + echo "$UNAME_MACHINE"-unknown-solidbsd"$UNAME_RELEASE" exit ;; macppc:MirBSD:*:*) - echo powerpc-unknown-mirbsd${UNAME_RELEASE} + echo powerpc-unknown-mirbsd"$UNAME_RELEASE" exit ;; *:MirBSD:*:*) - echo ${UNAME_MACHINE}-unknown-mirbsd${UNAME_RELEASE} + echo "$UNAME_MACHINE"-unknown-mirbsd"$UNAME_RELEASE" + exit ;; + *:Sortix:*:*) + echo "$UNAME_MACHINE"-unknown-sortix + exit ;; + *:Redox:*:*) + echo "$UNAME_MACHINE"-unknown-redox exit ;; + mips:OSF1:*.*) + echo mips-dec-osf1 + exit ;; alpha:OSF1:*:*) case $UNAME_RELEASE in *4.0) @@ -236,63 +294,54 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in ALPHA_CPU_TYPE=`/usr/sbin/psrinfo -v | sed -n -e 's/^ The alpha \(.*\) processor.*$/\1/p' | head -n 1` case "$ALPHA_CPU_TYPE" in "EV4 (21064)") - UNAME_MACHINE="alpha" ;; + UNAME_MACHINE=alpha ;; "EV4.5 (21064)") - UNAME_MACHINE="alpha" ;; + UNAME_MACHINE=alpha ;; "LCA4 (21066/21068)") - UNAME_MACHINE="alpha" ;; + UNAME_MACHINE=alpha ;; "EV5 (21164)") - UNAME_MACHINE="alphaev5" ;; + UNAME_MACHINE=alphaev5 ;; "EV5.6 (21164A)") - UNAME_MACHINE="alphaev56" ;; + UNAME_MACHINE=alphaev56 ;; "EV5.6 (21164PC)") - UNAME_MACHINE="alphapca56" ;; + UNAME_MACHINE=alphapca56 ;; "EV5.7 (21164PC)") - UNAME_MACHINE="alphapca57" ;; + UNAME_MACHINE=alphapca57 ;; "EV6 (21264)") - UNAME_MACHINE="alphaev6" ;; + UNAME_MACHINE=alphaev6 ;; "EV6.7 (21264A)") - UNAME_MACHINE="alphaev67" ;; + UNAME_MACHINE=alphaev67 ;; "EV6.8CB (21264C)") - UNAME_MACHINE="alphaev68" ;; + UNAME_MACHINE=alphaev68 ;; "EV6.8AL (21264B)") - UNAME_MACHINE="alphaev68" ;; + UNAME_MACHINE=alphaev68 ;; "EV6.8CX (21264D)") - UNAME_MACHINE="alphaev68" ;; + UNAME_MACHINE=alphaev68 ;; "EV6.9A (21264/EV69A)") - UNAME_MACHINE="alphaev69" ;; + UNAME_MACHINE=alphaev69 ;; "EV7 (21364)") - UNAME_MACHINE="alphaev7" ;; + UNAME_MACHINE=alphaev7 ;; "EV7.9 (21364A)") - UNAME_MACHINE="alphaev79" ;; + UNAME_MACHINE=alphaev79 ;; esac # A Pn.n version is a patched version. # A Vn.n version is a released version. # A Tn.n version is a released field test version. # A Xn.n version is an unreleased experimental baselevel. # 1.2 uses "1.2" for uname -r. - echo ${UNAME_MACHINE}-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^[PVTX]//' | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'` + echo "$UNAME_MACHINE"-dec-osf"`echo "$UNAME_RELEASE" | sed -e 's/^[PVTX]//' | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz`" # Reset EXIT trap before exiting to avoid spurious non-zero exit code. exitcode=$? trap '' 0 exit $exitcode ;; - Alpha\ *:Windows_NT*:*) - # How do we know it's Interix rather than the generic POSIX subsystem? - # Should we change UNAME_MACHINE based on the output of uname instead - # of the specific Alpha model? - echo alpha-pc-interix - exit ;; - 21064:Windows_NT:50:3) - echo alpha-dec-winnt3.5 - exit ;; Amiga*:UNIX_System_V:4.0:*) echo m68k-unknown-sysv4 exit ;; *:[Aa]miga[Oo][Ss]:*:*) - echo ${UNAME_MACHINE}-unknown-amigaos + echo "$UNAME_MACHINE"-unknown-amigaos exit ;; *:[Mm]orph[Oo][Ss]:*:*) - echo ${UNAME_MACHINE}-unknown-morphos + echo "$UNAME_MACHINE"-unknown-morphos exit ;; *:OS/390:*:*) echo i370-ibm-openedition @@ -304,9 +353,9 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in echo powerpc-ibm-os400 exit ;; arm:RISC*:1.[012]*:*|arm:riscix:1.[012]*:*) - echo arm-acorn-riscix${UNAME_RELEASE} + echo arm-acorn-riscix"$UNAME_RELEASE" exit ;; - arm:riscos:*:*|arm:RISCOS:*:*) + arm*:riscos:*:*|arm*:RISCOS:*:*) echo arm-unknown-riscos exit ;; SR2?01:HI-UX/MPP:*:* | SR8000:HI-UX/MPP:*:*) @@ -331,38 +380,33 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in sparc) echo sparc-icl-nx7; exit ;; esac ;; s390x:SunOS:*:*) - echo ${UNAME_MACHINE}-ibm-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + echo "$UNAME_MACHINE"-ibm-solaris2"`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'`" exit ;; sun4H:SunOS:5.*:*) - echo sparc-hal-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + echo sparc-hal-solaris2"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`" exit ;; sun4*:SunOS:5.*:* | tadpole*:SunOS:5.*:*) - echo sparc-sun-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + echo sparc-sun-solaris2"`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'`" exit ;; i86pc:AuroraUX:5.*:* | i86xen:AuroraUX:5.*:*) - echo i386-pc-auroraux${UNAME_RELEASE} + echo i386-pc-auroraux"$UNAME_RELEASE" exit ;; i86pc:SunOS:5.*:* | i86xen:SunOS:5.*:*) - eval $set_cc_for_build - SUN_ARCH="i386" - # If there is a compiler, see if it is configured for 64-bit objects. - # Note that the Sun cc does not turn __LP64__ into 1 like gcc does. - # This test works for both compilers. - if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then - if (echo '#ifdef __amd64'; echo IS_64BIT_ARCH; echo '#endif') | \ - (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \ - grep IS_64BIT_ARCH >/dev/null - then - SUN_ARCH="x86_64" - fi - fi - echo ${SUN_ARCH}-pc-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + UNAME_REL="`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'`" + case `isainfo -b` in + 32) + echo i386-pc-solaris2"$UNAME_REL" + ;; + 64) + echo x86_64-pc-solaris2"$UNAME_REL" + ;; + esac exit ;; sun4*:SunOS:6*:*) # According to config.sub, this is the proper way to canonicalize # SunOS6. Hard to guess exactly what SunOS6 will be like, but # it's likely to be more like Solaris than SunOS4. - echo sparc-sun-solaris3`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + echo sparc-sun-solaris3"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`" exit ;; sun4*:SunOS:*:*) case "`/usr/bin/arch -k`" in @@ -371,25 +415,25 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in ;; esac # Japanese Language versions have a version number like `4.1.3-JL'. - echo sparc-sun-sunos`echo ${UNAME_RELEASE}|sed -e 's/-/_/'` + echo sparc-sun-sunos"`echo "$UNAME_RELEASE"|sed -e 's/-/_/'`" exit ;; sun3*:SunOS:*:*) - echo m68k-sun-sunos${UNAME_RELEASE} + echo m68k-sun-sunos"$UNAME_RELEASE" exit ;; sun*:*:4.2BSD:*) UNAME_RELEASE=`(sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null` - test "x${UNAME_RELEASE}" = "x" && UNAME_RELEASE=3 + test "x$UNAME_RELEASE" = x && UNAME_RELEASE=3 case "`/bin/arch`" in sun3) - echo m68k-sun-sunos${UNAME_RELEASE} + echo m68k-sun-sunos"$UNAME_RELEASE" ;; sun4) - echo sparc-sun-sunos${UNAME_RELEASE} + echo sparc-sun-sunos"$UNAME_RELEASE" ;; esac exit ;; aushp:SunOS:*:*) - echo sparc-auspex-sunos${UNAME_RELEASE} + echo sparc-auspex-sunos"$UNAME_RELEASE" exit ;; # The situation for MiNT is a little confusing. The machine name # can be virtually everything (everything which is not @@ -400,44 +444,44 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in # MiNT. But MiNT is downward compatible to TOS, so this should # be no problem. atarist[e]:*MiNT:*:* | atarist[e]:*mint:*:* | atarist[e]:*TOS:*:*) - echo m68k-atari-mint${UNAME_RELEASE} + echo m68k-atari-mint"$UNAME_RELEASE" exit ;; atari*:*MiNT:*:* | atari*:*mint:*:* | atarist[e]:*TOS:*:*) - echo m68k-atari-mint${UNAME_RELEASE} + echo m68k-atari-mint"$UNAME_RELEASE" exit ;; *falcon*:*MiNT:*:* | *falcon*:*mint:*:* | *falcon*:*TOS:*:*) - echo m68k-atari-mint${UNAME_RELEASE} + echo m68k-atari-mint"$UNAME_RELEASE" exit ;; milan*:*MiNT:*:* | milan*:*mint:*:* | *milan*:*TOS:*:*) - echo m68k-milan-mint${UNAME_RELEASE} + echo m68k-milan-mint"$UNAME_RELEASE" exit ;; hades*:*MiNT:*:* | hades*:*mint:*:* | *hades*:*TOS:*:*) - echo m68k-hades-mint${UNAME_RELEASE} + echo m68k-hades-mint"$UNAME_RELEASE" exit ;; *:*MiNT:*:* | *:*mint:*:* | *:*TOS:*:*) - echo m68k-unknown-mint${UNAME_RELEASE} + echo m68k-unknown-mint"$UNAME_RELEASE" exit ;; m68k:machten:*:*) - echo m68k-apple-machten${UNAME_RELEASE} + echo m68k-apple-machten"$UNAME_RELEASE" exit ;; powerpc:machten:*:*) - echo powerpc-apple-machten${UNAME_RELEASE} + echo powerpc-apple-machten"$UNAME_RELEASE" exit ;; RISC*:Mach:*:*) echo mips-dec-mach_bsd4.3 exit ;; RISC*:ULTRIX:*:*) - echo mips-dec-ultrix${UNAME_RELEASE} + echo mips-dec-ultrix"$UNAME_RELEASE" exit ;; VAX*:ULTRIX*:*:*) - echo vax-dec-ultrix${UNAME_RELEASE} + echo vax-dec-ultrix"$UNAME_RELEASE" exit ;; 2020:CLIX:*:* | 2430:CLIX:*:*) - echo clipper-intergraph-clix${UNAME_RELEASE} + echo clipper-intergraph-clix"$UNAME_RELEASE" exit ;; mips:*:*:UMIPS | mips:*:*:RISCos) - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c + set_cc_for_build + sed 's/^ //' << EOF > "$dummy.c" #ifdef __cplusplus #include <stdio.h> /* for printf() prototype */ int main (int argc, char *argv[]) { @@ -446,23 +490,23 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in #endif #if defined (host_mips) && defined (MIPSEB) #if defined (SYSTYPE_SYSV) - printf ("mips-mips-riscos%ssysv\n", argv[1]); exit (0); + printf ("mips-mips-riscos%ssysv\\n", argv[1]); exit (0); #endif #if defined (SYSTYPE_SVR4) - printf ("mips-mips-riscos%ssvr4\n", argv[1]); exit (0); + printf ("mips-mips-riscos%ssvr4\\n", argv[1]); exit (0); #endif #if defined (SYSTYPE_BSD43) || defined(SYSTYPE_BSD) - printf ("mips-mips-riscos%sbsd\n", argv[1]); exit (0); + printf ("mips-mips-riscos%sbsd\\n", argv[1]); exit (0); #endif #endif exit (-1); } EOF - $CC_FOR_BUILD -o $dummy $dummy.c && - dummyarg=`echo "${UNAME_RELEASE}" | sed -n 's/\([0-9]*\).*/\1/p'` && - SYSTEM_NAME=`$dummy $dummyarg` && + $CC_FOR_BUILD -o "$dummy" "$dummy.c" && + dummyarg=`echo "$UNAME_RELEASE" | sed -n 's/\([0-9]*\).*/\1/p'` && + SYSTEM_NAME=`"$dummy" "$dummyarg"` && { echo "$SYSTEM_NAME"; exit; } - echo mips-mips-riscos${UNAME_RELEASE} + echo mips-mips-riscos"$UNAME_RELEASE" exit ;; Motorola:PowerMAX_OS:*:*) echo powerpc-motorola-powermax @@ -488,17 +532,17 @@ EOF AViiON:dgux:*:*) # DG/UX returns AViiON for all architectures UNAME_PROCESSOR=`/usr/bin/uname -p` - if [ $UNAME_PROCESSOR = mc88100 ] || [ $UNAME_PROCESSOR = mc88110 ] + if [ "$UNAME_PROCESSOR" = mc88100 ] || [ "$UNAME_PROCESSOR" = mc88110 ] then - if [ ${TARGET_BINARY_INTERFACE}x = m88kdguxelfx ] || \ - [ ${TARGET_BINARY_INTERFACE}x = x ] + if [ "$TARGET_BINARY_INTERFACE"x = m88kdguxelfx ] || \ + [ "$TARGET_BINARY_INTERFACE"x = x ] then - echo m88k-dg-dgux${UNAME_RELEASE} + echo m88k-dg-dgux"$UNAME_RELEASE" else - echo m88k-dg-dguxbcs${UNAME_RELEASE} + echo m88k-dg-dguxbcs"$UNAME_RELEASE" fi else - echo i586-dg-dgux${UNAME_RELEASE} + echo i586-dg-dgux"$UNAME_RELEASE" fi exit ;; M88*:DolphinOS:*:*) # DolphinOS (SVR3) @@ -515,7 +559,7 @@ EOF echo m68k-tektronix-bsd exit ;; *:IRIX*:*:*) - echo mips-sgi-irix`echo ${UNAME_RELEASE}|sed -e 's/-/_/g'` + echo mips-sgi-irix"`echo "$UNAME_RELEASE"|sed -e 's/-/_/g'`" exit ;; ????????:AIX?:[12].1:2) # AIX 2.2.1 or AIX 2.1.1 is RT/PC AIX. echo romp-ibm-aix # uname -m gives an 8 hex-code CPU id @@ -527,14 +571,14 @@ EOF if [ -x /usr/bin/oslevel ] ; then IBM_REV=`/usr/bin/oslevel` else - IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE} + IBM_REV="$UNAME_VERSION.$UNAME_RELEASE" fi - echo ${UNAME_MACHINE}-ibm-aix${IBM_REV} + echo "$UNAME_MACHINE"-ibm-aix"$IBM_REV" exit ;; *:AIX:2:3) if grep bos325 /usr/include/stdio.h >/dev/null 2>&1; then - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c + set_cc_for_build + sed 's/^ //' << EOF > "$dummy.c" #include <sys/systemcfg.h> main() @@ -545,7 +589,7 @@ EOF exit(0); } EOF - if $CC_FOR_BUILD -o $dummy $dummy.c && SYSTEM_NAME=`$dummy` + if $CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=`"$dummy"` then echo "$SYSTEM_NAME" else @@ -559,26 +603,27 @@ EOF exit ;; *:AIX:*:[4567]) IBM_CPU_ID=`/usr/sbin/lsdev -C -c processor -S available | sed 1q | awk '{ print $1 }'` - if /usr/sbin/lsattr -El ${IBM_CPU_ID} | grep ' POWER' >/dev/null 2>&1; then + if /usr/sbin/lsattr -El "$IBM_CPU_ID" | grep ' POWER' >/dev/null 2>&1; then IBM_ARCH=rs6000 else IBM_ARCH=powerpc fi - if [ -x /usr/bin/oslevel ] ; then - IBM_REV=`/usr/bin/oslevel` + if [ -x /usr/bin/lslpp ] ; then + IBM_REV=`/usr/bin/lslpp -Lqc bos.rte.libc | + awk -F: '{ print $3 }' | sed s/[0-9]*$/0/` else - IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE} + IBM_REV="$UNAME_VERSION.$UNAME_RELEASE" fi - echo ${IBM_ARCH}-ibm-aix${IBM_REV} + echo "$IBM_ARCH"-ibm-aix"$IBM_REV" exit ;; *:AIX:*:*) echo rs6000-ibm-aix exit ;; - ibmrt:4.4BSD:*|romp-ibm:BSD:*) + ibmrt:4.4BSD:*|romp-ibm:4.4BSD:*) echo romp-ibm-bsd4.4 exit ;; ibmrt:*BSD:*|romp-ibm:BSD:*) # covers RT/PC BSD and - echo romp-ibm-bsd${UNAME_RELEASE} # 4.3 with uname added to + echo romp-ibm-bsd"$UNAME_RELEASE" # 4.3 with uname added to exit ;; # report: romp-ibm BSD 4.3 *:BOSX:*:*) echo rs6000-bull-bosx @@ -593,28 +638,28 @@ EOF echo m68k-hp-bsd4.4 exit ;; 9000/[34678]??:HP-UX:*:*) - HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'` - case "${UNAME_MACHINE}" in - 9000/31? ) HP_ARCH=m68000 ;; - 9000/[34]?? ) HP_ARCH=m68k ;; + HPUX_REV=`echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//'` + case "$UNAME_MACHINE" in + 9000/31?) HP_ARCH=m68000 ;; + 9000/[34]??) HP_ARCH=m68k ;; 9000/[678][0-9][0-9]) if [ -x /usr/bin/getconf ]; then sc_cpu_version=`/usr/bin/getconf SC_CPU_VERSION 2>/dev/null` sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null` - case "${sc_cpu_version}" in - 523) HP_ARCH="hppa1.0" ;; # CPU_PA_RISC1_0 - 528) HP_ARCH="hppa1.1" ;; # CPU_PA_RISC1_1 + case "$sc_cpu_version" in + 523) HP_ARCH=hppa1.0 ;; # CPU_PA_RISC1_0 + 528) HP_ARCH=hppa1.1 ;; # CPU_PA_RISC1_1 532) # CPU_PA_RISC2_0 - case "${sc_kernel_bits}" in - 32) HP_ARCH="hppa2.0n" ;; - 64) HP_ARCH="hppa2.0w" ;; - '') HP_ARCH="hppa2.0" ;; # HP-UX 10.20 + case "$sc_kernel_bits" in + 32) HP_ARCH=hppa2.0n ;; + 64) HP_ARCH=hppa2.0w ;; + '') HP_ARCH=hppa2.0 ;; # HP-UX 10.20 esac ;; esac fi - if [ "${HP_ARCH}" = "" ]; then - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c + if [ "$HP_ARCH" = "" ]; then + set_cc_for_build + sed 's/^ //' << EOF > "$dummy.c" #define _HPUX_SOURCE #include <stdlib.h> @@ -647,13 +692,13 @@ EOF exit (0); } EOF - (CCOPTS= $CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null) && HP_ARCH=`$dummy` + (CCOPTS="" $CC_FOR_BUILD -o "$dummy" "$dummy.c" 2>/dev/null) && HP_ARCH=`"$dummy"` test -z "$HP_ARCH" && HP_ARCH=hppa fi ;; esac - if [ ${HP_ARCH} = "hppa2.0w" ] + if [ "$HP_ARCH" = hppa2.0w ] then - eval $set_cc_for_build + set_cc_for_build # hppa2.0w-hp-hpux* has a 64-bit kernel and a compiler generating # 32-bit code. hppa64-hp-hpux* has the same kernel and a compiler @@ -664,23 +709,23 @@ EOF # $ CC_FOR_BUILD="cc +DA2.0w" ./config.guess # => hppa64-hp-hpux11.23 - if echo __LP64__ | (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | + if echo __LP64__ | (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | grep -q __LP64__ then - HP_ARCH="hppa2.0w" + HP_ARCH=hppa2.0w else - HP_ARCH="hppa64" + HP_ARCH=hppa64 fi fi - echo ${HP_ARCH}-hp-hpux${HPUX_REV} + echo "$HP_ARCH"-hp-hpux"$HPUX_REV" exit ;; ia64:HP-UX:*:*) - HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'` - echo ia64-hp-hpux${HPUX_REV} + HPUX_REV=`echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//'` + echo ia64-hp-hpux"$HPUX_REV" exit ;; 3050*:HI-UX:*:*) - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c + set_cc_for_build + sed 's/^ //' << EOF > "$dummy.c" #include <unistd.h> int main () @@ -705,11 +750,11 @@ EOF exit (0); } EOF - $CC_FOR_BUILD -o $dummy $dummy.c && SYSTEM_NAME=`$dummy` && + $CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=`"$dummy"` && { echo "$SYSTEM_NAME"; exit; } echo unknown-hitachi-hiuxwe2 exit ;; - 9000/7??:4.3bsd:*:* | 9000/8?[79]:4.3bsd:*:* ) + 9000/7??:4.3bsd:*:* | 9000/8?[79]:4.3bsd:*:*) echo hppa1.1-hp-bsd exit ;; 9000/8??:4.3bsd:*:*) @@ -718,7 +763,7 @@ EOF *9??*:MPE/iX:*:* | *3000*:MPE/iX:*:*) echo hppa1.0-hp-mpeix exit ;; - hp7??:OSF1:*:* | hp8?[79]:OSF1:*:* ) + hp7??:OSF1:*:* | hp8?[79]:OSF1:*:*) echo hppa1.1-hp-osf exit ;; hp8??:OSF1:*:*) @@ -726,9 +771,9 @@ EOF exit ;; i*86:OSF1:*:*) if [ -x /usr/sbin/sysversion ] ; then - echo ${UNAME_MACHINE}-unknown-osf1mk + echo "$UNAME_MACHINE"-unknown-osf1mk else - echo ${UNAME_MACHINE}-unknown-osf1 + echo "$UNAME_MACHINE"-unknown-osf1 fi exit ;; parisc*:Lites*:*:*) @@ -753,127 +798,120 @@ EOF echo c4-convex-bsd exit ;; CRAY*Y-MP:*:*:*) - echo ymp-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + echo ymp-cray-unicos"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' exit ;; CRAY*[A-Z]90:*:*:*) - echo ${UNAME_MACHINE}-cray-unicos${UNAME_RELEASE} \ + echo "$UNAME_MACHINE"-cray-unicos"$UNAME_RELEASE" \ | sed -e 's/CRAY.*\([A-Z]90\)/\1/' \ -e y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefghijklmnopqrstuvwxyz/ \ -e 's/\.[^.]*$/.X/' exit ;; CRAY*TS:*:*:*) - echo t90-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + echo t90-cray-unicos"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' exit ;; CRAY*T3E:*:*:*) - echo alphaev5-cray-unicosmk${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + echo alphaev5-cray-unicosmk"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' exit ;; CRAY*SV1:*:*:*) - echo sv1-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + echo sv1-cray-unicos"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' exit ;; *:UNICOS/mp:*:*) - echo craynv-cray-unicosmp${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + echo craynv-cray-unicosmp"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' exit ;; F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*) - FUJITSU_PROC=`uname -m | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'` - FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` - FUJITSU_REL=`echo ${UNAME_RELEASE} | sed -e 's/ /_/'` + FUJITSU_PROC=`uname -m | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz` + FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'` + FUJITSU_REL=`echo "$UNAME_RELEASE" | sed -e 's/ /_/'` echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" exit ;; 5000:UNIX_System_V:4.*:*) - FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` - FUJITSU_REL=`echo ${UNAME_RELEASE} | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/ /_/'` + FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'` + FUJITSU_REL=`echo "$UNAME_RELEASE" | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/ /_/'` echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" exit ;; i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*) - echo ${UNAME_MACHINE}-pc-bsdi${UNAME_RELEASE} + echo "$UNAME_MACHINE"-pc-bsdi"$UNAME_RELEASE" exit ;; sparc*:BSD/OS:*:*) - echo sparc-unknown-bsdi${UNAME_RELEASE} + echo sparc-unknown-bsdi"$UNAME_RELEASE" exit ;; *:BSD/OS:*:*) - echo ${UNAME_MACHINE}-unknown-bsdi${UNAME_RELEASE} + echo "$UNAME_MACHINE"-unknown-bsdi"$UNAME_RELEASE" + exit ;; + arm*:FreeBSD:*:*) + UNAME_PROCESSOR=`uname -p` + set_cc_for_build + if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \ + | grep -q __ARM_PCS_VFP + then + echo "${UNAME_PROCESSOR}"-unknown-freebsd"`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`"-gnueabi + else + echo "${UNAME_PROCESSOR}"-unknown-freebsd"`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`"-gnueabihf + fi exit ;; *:FreeBSD:*:*) UNAME_PROCESSOR=`/usr/bin/uname -p` - case ${UNAME_PROCESSOR} in + case "$UNAME_PROCESSOR" in amd64) - echo x86_64-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;; - *) - echo ${UNAME_PROCESSOR}-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;; + UNAME_PROCESSOR=x86_64 ;; + i386) + UNAME_PROCESSOR=i586 ;; esac + echo "$UNAME_PROCESSOR"-unknown-freebsd"`echo "$UNAME_RELEASE"|sed -e 's/[-(].*//'`" exit ;; i*:CYGWIN*:*) - echo ${UNAME_MACHINE}-pc-cygwin + echo "$UNAME_MACHINE"-pc-cygwin exit ;; *:MINGW64*:*) - echo ${UNAME_MACHINE}-pc-mingw64 + echo "$UNAME_MACHINE"-pc-mingw64 exit ;; *:MINGW*:*) - echo ${UNAME_MACHINE}-pc-mingw32 + echo "$UNAME_MACHINE"-pc-mingw32 exit ;; - i*:MSYS*:*) - echo ${UNAME_MACHINE}-pc-msys - exit ;; - i*:windows32*:*) - # uname -m includes "-pc" on this system. - echo ${UNAME_MACHINE}-mingw32 + *:MSYS*:*) + echo "$UNAME_MACHINE"-pc-msys exit ;; i*:PW*:*) - echo ${UNAME_MACHINE}-pc-pw32 + echo "$UNAME_MACHINE"-pc-pw32 exit ;; *:Interix*:*) - case ${UNAME_MACHINE} in + case "$UNAME_MACHINE" in x86) - echo i586-pc-interix${UNAME_RELEASE} + echo i586-pc-interix"$UNAME_RELEASE" exit ;; authenticamd | genuineintel | EM64T) - echo x86_64-unknown-interix${UNAME_RELEASE} + echo x86_64-unknown-interix"$UNAME_RELEASE" exit ;; IA64) - echo ia64-unknown-interix${UNAME_RELEASE} + echo ia64-unknown-interix"$UNAME_RELEASE" exit ;; esac ;; - [345]86:Windows_95:* | [345]86:Windows_98:* | [345]86:Windows_NT:*) - echo i${UNAME_MACHINE}-pc-mks - exit ;; - 8664:Windows_NT:*) - echo x86_64-pc-mks - exit ;; - i*:Windows_NT*:* | Pentium*:Windows_NT*:*) - # How do we know it's Interix rather than the generic POSIX subsystem? - # It also conflicts with pre-2.0 versions of AT&T UWIN. Should we - # UNAME_MACHINE based on the output of uname instead of i386? - echo i586-pc-interix - exit ;; i*:UWIN*:*) - echo ${UNAME_MACHINE}-pc-uwin + echo "$UNAME_MACHINE"-pc-uwin exit ;; amd64:CYGWIN*:*:* | x86_64:CYGWIN*:*:*) echo x86_64-unknown-cygwin exit ;; - p*:CYGWIN*:*) - echo powerpcle-unknown-cygwin - exit ;; prep*:SunOS:5.*:*) - echo powerpcle-unknown-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + echo powerpcle-unknown-solaris2"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`" exit ;; *:GNU:*:*) # the GNU system - echo `echo ${UNAME_MACHINE}|sed -e 's,[-/].*$,,'`-unknown-gnu`echo ${UNAME_RELEASE}|sed -e 's,/.*$,,'` + echo "`echo "$UNAME_MACHINE"|sed -e 's,[-/].*$,,'`-unknown-$LIBC`echo "$UNAME_RELEASE"|sed -e 's,/.*$,,'`" exit ;; *:GNU/*:*:*) # other systems with GNU libc and userland - echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr '[A-Z]' '[a-z]'``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-gnu + echo "$UNAME_MACHINE-unknown-`echo "$UNAME_SYSTEM" | sed 's,^[^/]*/,,' | tr "[:upper:]" "[:lower:]"``echo "$UNAME_RELEASE"|sed -e 's/[-(].*//'`-$LIBC" exit ;; - i*86:Minix:*:*) - echo ${UNAME_MACHINE}-pc-minix + *:Minix:*:*) + echo "$UNAME_MACHINE"-unknown-minix exit ;; aarch64:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; aarch64_be:Linux:*:*) UNAME_MACHINE=aarch64_be - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; alpha:Linux:*:*) case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in @@ -886,63 +924,64 @@ EOF EV68*) UNAME_MACHINE=alphaev68 ;; esac objdump --private-headers /bin/sh | grep -q ld.so.1 - if test "$?" = 0 ; then LIBC="libc1" ; else LIBC="" ; fi - echo ${UNAME_MACHINE}-unknown-linux-gnu${LIBC} + if test "$?" = 0 ; then LIBC=gnulibc1 ; fi + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" + exit ;; + arc:Linux:*:* | arceb:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; arm*:Linux:*:*) - eval $set_cc_for_build + set_cc_for_build if echo __ARM_EABI__ | $CC_FOR_BUILD -E - 2>/dev/null \ | grep -q __ARM_EABI__ then - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" else if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \ | grep -q __ARM_PCS_VFP then - echo ${UNAME_MACHINE}-unknown-linux-gnueabi + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"eabi else - echo ${UNAME_MACHINE}-unknown-linux-gnueabihf + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"eabihf fi fi exit ;; avr32*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; cris:Linux:*:*) - echo ${UNAME_MACHINE}-axis-linux-gnu + echo "$UNAME_MACHINE"-axis-linux-"$LIBC" exit ;; crisv32:Linux:*:*) - echo ${UNAME_MACHINE}-axis-linux-gnu + echo "$UNAME_MACHINE"-axis-linux-"$LIBC" + exit ;; + e2k:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; frv:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; hexagon:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; i*86:Linux:*:*) - LIBC=gnu - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c - #ifdef __dietlibc__ - LIBC=dietlibc - #endif -EOF - eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^LIBC'` - echo "${UNAME_MACHINE}-pc-linux-${LIBC}" + echo "$UNAME_MACHINE"-pc-linux-"$LIBC" exit ;; ia64:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" + exit ;; + k1om:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; m32r*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; m68*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; mips:Linux:*:* | mips64:Linux:*:*) - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c + set_cc_for_build + sed 's/^ //' << EOF > "$dummy.c" #undef CPU #undef ${UNAME_MACHINE} #undef ${UNAME_MACHINE}el @@ -956,55 +995,70 @@ EOF #endif #endif EOF - eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^CPU'` - test x"${CPU}" != x && { echo "${CPU}-unknown-linux-gnu"; exit; } + eval "`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^CPU'`" + test "x$CPU" != x && { echo "$CPU-unknown-linux-$LIBC"; exit; } ;; - or32:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + mips64el:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" + exit ;; + openrisc*:Linux:*:*) + echo or1k-unknown-linux-"$LIBC" + exit ;; + or32:Linux:*:* | or1k*:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; padre:Linux:*:*) - echo sparc-unknown-linux-gnu + echo sparc-unknown-linux-"$LIBC" exit ;; parisc64:Linux:*:* | hppa64:Linux:*:*) - echo hppa64-unknown-linux-gnu + echo hppa64-unknown-linux-"$LIBC" exit ;; parisc:Linux:*:* | hppa:Linux:*:*) # Look for CPU level case `grep '^cpu[^a-z]*:' /proc/cpuinfo 2>/dev/null | cut -d' ' -f2` in - PA7*) echo hppa1.1-unknown-linux-gnu ;; - PA8*) echo hppa2.0-unknown-linux-gnu ;; - *) echo hppa-unknown-linux-gnu ;; + PA7*) echo hppa1.1-unknown-linux-"$LIBC" ;; + PA8*) echo hppa2.0-unknown-linux-"$LIBC" ;; + *) echo hppa-unknown-linux-"$LIBC" ;; esac exit ;; ppc64:Linux:*:*) - echo powerpc64-unknown-linux-gnu + echo powerpc64-unknown-linux-"$LIBC" exit ;; ppc:Linux:*:*) - echo powerpc-unknown-linux-gnu + echo powerpc-unknown-linux-"$LIBC" + exit ;; + ppc64le:Linux:*:*) + echo powerpc64le-unknown-linux-"$LIBC" + exit ;; + ppcle:Linux:*:*) + echo powerpcle-unknown-linux-"$LIBC" + exit ;; + riscv32:Linux:*:* | riscv64:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; s390:Linux:*:* | s390x:Linux:*:*) - echo ${UNAME_MACHINE}-ibm-linux + echo "$UNAME_MACHINE"-ibm-linux-"$LIBC" exit ;; sh64*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; sh*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; sparc:Linux:*:* | sparc64:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; tile*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; vax:Linux:*:*) - echo ${UNAME_MACHINE}-dec-linux-gnu + echo "$UNAME_MACHINE"-dec-linux-"$LIBC" exit ;; x86_64:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo "$UNAME_MACHINE"-pc-linux-"$LIBC" exit ;; xtensa*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; i*86:DYNIX/ptx:4*:*) # ptx 4.0 does uname -s correctly, with DYNIX/ptx in there. @@ -1018,34 +1072,34 @@ EOF # I am not positive that other SVR4 systems won't match this, # I just have to hope. -- rms. # Use sysv4.2uw... so that sysv4* matches it. - echo ${UNAME_MACHINE}-pc-sysv4.2uw${UNAME_VERSION} + echo "$UNAME_MACHINE"-pc-sysv4.2uw"$UNAME_VERSION" exit ;; i*86:OS/2:*:*) # If we were able to find `uname', then EMX Unix compatibility # is probably installed. - echo ${UNAME_MACHINE}-pc-os2-emx + echo "$UNAME_MACHINE"-pc-os2-emx exit ;; i*86:XTS-300:*:STOP) - echo ${UNAME_MACHINE}-unknown-stop + echo "$UNAME_MACHINE"-unknown-stop exit ;; i*86:atheos:*:*) - echo ${UNAME_MACHINE}-unknown-atheos + echo "$UNAME_MACHINE"-unknown-atheos exit ;; i*86:syllable:*:*) - echo ${UNAME_MACHINE}-pc-syllable + echo "$UNAME_MACHINE"-pc-syllable exit ;; i*86:LynxOS:2.*:* | i*86:LynxOS:3.[01]*:* | i*86:LynxOS:4.[02]*:*) - echo i386-unknown-lynxos${UNAME_RELEASE} + echo i386-unknown-lynxos"$UNAME_RELEASE" exit ;; i*86:*DOS:*:*) - echo ${UNAME_MACHINE}-pc-msdosdjgpp + echo "$UNAME_MACHINE"-pc-msdosdjgpp exit ;; - i*86:*:4.*:* | i*86:SYSTEM_V:4.*:*) - UNAME_REL=`echo ${UNAME_RELEASE} | sed 's/\/MP$//'` + i*86:*:4.*:*) + UNAME_REL=`echo "$UNAME_RELEASE" | sed 's/\/MP$//'` if grep Novell /usr/include/link.h >/dev/null 2>/dev/null; then - echo ${UNAME_MACHINE}-univel-sysv${UNAME_REL} + echo "$UNAME_MACHINE"-univel-sysv"$UNAME_REL" else - echo ${UNAME_MACHINE}-pc-sysv${UNAME_REL} + echo "$UNAME_MACHINE"-pc-sysv"$UNAME_REL" fi exit ;; i*86:*:5:[678]*) @@ -1055,12 +1109,12 @@ EOF *Pentium) UNAME_MACHINE=i586 ;; *Pent*|*Celeron) UNAME_MACHINE=i686 ;; esac - echo ${UNAME_MACHINE}-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}${UNAME_VERSION} + echo "$UNAME_MACHINE-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}{$UNAME_VERSION}" exit ;; i*86:*:3.2:*) if test -f /usr/options/cb.name; then UNAME_REL=`sed -n 's/.*Version //p' </usr/options/cb.name` - echo ${UNAME_MACHINE}-pc-isc$UNAME_REL + echo "$UNAME_MACHINE"-pc-isc"$UNAME_REL" elif /bin/uname -X 2>/dev/null >/dev/null ; then UNAME_REL=`(/bin/uname -X|grep Release|sed -e 's/.*= //')` (/bin/uname -X|grep i80486 >/dev/null) && UNAME_MACHINE=i486 @@ -1070,9 +1124,9 @@ EOF && UNAME_MACHINE=i686 (/bin/uname -X|grep '^Machine.*Pentium Pro' >/dev/null) \ && UNAME_MACHINE=i686 - echo ${UNAME_MACHINE}-pc-sco$UNAME_REL + echo "$UNAME_MACHINE"-pc-sco"$UNAME_REL" else - echo ${UNAME_MACHINE}-pc-sysv32 + echo "$UNAME_MACHINE"-pc-sysv32 fi exit ;; pc:*:*:*) @@ -1080,7 +1134,7 @@ EOF # uname -m prints for DJGPP always 'pc', but it prints nothing about # the processor, so we play safe by assuming i586. # Note: whatever this is, it MUST be the same as what config.sub - # prints for the "djgpp" host, or else GDB configury will decide that + # prints for the "djgpp" host, or else GDB configure will decide that # this is a cross-build. echo i586-pc-msdosdjgpp exit ;; @@ -1092,9 +1146,9 @@ EOF exit ;; i860:*:4.*:*) # i860-SVR4 if grep Stardent /usr/include/sys/uadmin.h >/dev/null 2>&1 ; then - echo i860-stardent-sysv${UNAME_RELEASE} # Stardent Vistra i860-SVR4 + echo i860-stardent-sysv"$UNAME_RELEASE" # Stardent Vistra i860-SVR4 else # Add other i860-SVR4 vendors below as they are discovered. - echo i860-unknown-sysv${UNAME_RELEASE} # Unknown i860-SVR4 + echo i860-unknown-sysv"$UNAME_RELEASE" # Unknown i860-SVR4 fi exit ;; mini*:CTIX:SYS*5:*) @@ -1114,9 +1168,9 @@ EOF test -r /etc/.relid \ && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid` /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ - && { echo i486-ncr-sysv4.3${OS_REL}; exit; } + && { echo i486-ncr-sysv4.3"$OS_REL"; exit; } /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ - && { echo i586-ncr-sysv4.3${OS_REL}; exit; } ;; + && { echo i586-ncr-sysv4.3"$OS_REL"; exit; } ;; 3[34]??:*:4.0:* | 3[34]??,*:*:4.0:*) /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ && { echo i486-ncr-sysv4; exit; } ;; @@ -1125,28 +1179,28 @@ EOF test -r /etc/.relid \ && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid` /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ - && { echo i486-ncr-sysv4.3${OS_REL}; exit; } + && { echo i486-ncr-sysv4.3"$OS_REL"; exit; } /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ - && { echo i586-ncr-sysv4.3${OS_REL}; exit; } + && { echo i586-ncr-sysv4.3"$OS_REL"; exit; } /bin/uname -p 2>/dev/null | /bin/grep pteron >/dev/null \ - && { echo i586-ncr-sysv4.3${OS_REL}; exit; } ;; + && { echo i586-ncr-sysv4.3"$OS_REL"; exit; } ;; m68*:LynxOS:2.*:* | m68*:LynxOS:3.0*:*) - echo m68k-unknown-lynxos${UNAME_RELEASE} + echo m68k-unknown-lynxos"$UNAME_RELEASE" exit ;; mc68030:UNIX_System_V:4.*:*) echo m68k-atari-sysv4 exit ;; TSUNAMI:LynxOS:2.*:*) - echo sparc-unknown-lynxos${UNAME_RELEASE} + echo sparc-unknown-lynxos"$UNAME_RELEASE" exit ;; rs6000:LynxOS:2.*:*) - echo rs6000-unknown-lynxos${UNAME_RELEASE} + echo rs6000-unknown-lynxos"$UNAME_RELEASE" exit ;; PowerPC:LynxOS:2.*:* | PowerPC:LynxOS:3.[01]*:* | PowerPC:LynxOS:4.[02]*:*) - echo powerpc-unknown-lynxos${UNAME_RELEASE} + echo powerpc-unknown-lynxos"$UNAME_RELEASE" exit ;; SM[BE]S:UNIX_SV:*:*) - echo mips-dde-sysv${UNAME_RELEASE} + echo mips-dde-sysv"$UNAME_RELEASE" exit ;; RM*:ReliantUNIX-*:*:*) echo mips-sni-sysv4 @@ -1157,7 +1211,7 @@ EOF *:SINIX-*:*:*) if uname -p 2>/dev/null >/dev/null ; then UNAME_MACHINE=`(uname -p) 2>/dev/null` - echo ${UNAME_MACHINE}-sni-sysv4 + echo "$UNAME_MACHINE"-sni-sysv4 else echo ns32k-sni-sysv fi @@ -1177,23 +1231,23 @@ EOF exit ;; i*86:VOS:*:*) # From Paul.Green@stratus.com. - echo ${UNAME_MACHINE}-stratus-vos + echo "$UNAME_MACHINE"-stratus-vos exit ;; *:VOS:*:*) # From Paul.Green@stratus.com. echo hppa1.1-stratus-vos exit ;; mc68*:A/UX:*:*) - echo m68k-apple-aux${UNAME_RELEASE} + echo m68k-apple-aux"$UNAME_RELEASE" exit ;; news*:NEWS-OS:6*:*) echo mips-sony-newsos6 exit ;; R[34]000:*System_V*:*:* | R4000:UNIX_SYSV:*:* | R*000:UNIX_SV:*:*) if [ -d /usr/nec ]; then - echo mips-nec-sysv${UNAME_RELEASE} + echo mips-nec-sysv"$UNAME_RELEASE" else - echo mips-unknown-sysv${UNAME_RELEASE} + echo mips-unknown-sysv"$UNAME_RELEASE" fi exit ;; BeBox:BeOS:*:*) # BeOS running on hardware made by Be, PPC only. @@ -1212,65 +1266,93 @@ EOF echo x86_64-unknown-haiku exit ;; SX-4:SUPER-UX:*:*) - echo sx4-nec-superux${UNAME_RELEASE} + echo sx4-nec-superux"$UNAME_RELEASE" exit ;; SX-5:SUPER-UX:*:*) - echo sx5-nec-superux${UNAME_RELEASE} + echo sx5-nec-superux"$UNAME_RELEASE" exit ;; SX-6:SUPER-UX:*:*) - echo sx6-nec-superux${UNAME_RELEASE} + echo sx6-nec-superux"$UNAME_RELEASE" exit ;; SX-7:SUPER-UX:*:*) - echo sx7-nec-superux${UNAME_RELEASE} + echo sx7-nec-superux"$UNAME_RELEASE" exit ;; SX-8:SUPER-UX:*:*) - echo sx8-nec-superux${UNAME_RELEASE} + echo sx8-nec-superux"$UNAME_RELEASE" exit ;; SX-8R:SUPER-UX:*:*) - echo sx8r-nec-superux${UNAME_RELEASE} + echo sx8r-nec-superux"$UNAME_RELEASE" + exit ;; + SX-ACE:SUPER-UX:*:*) + echo sxace-nec-superux"$UNAME_RELEASE" exit ;; Power*:Rhapsody:*:*) - echo powerpc-apple-rhapsody${UNAME_RELEASE} + echo powerpc-apple-rhapsody"$UNAME_RELEASE" exit ;; *:Rhapsody:*:*) - echo ${UNAME_MACHINE}-apple-rhapsody${UNAME_RELEASE} + echo "$UNAME_MACHINE"-apple-rhapsody"$UNAME_RELEASE" exit ;; *:Darwin:*:*) UNAME_PROCESSOR=`uname -p` || UNAME_PROCESSOR=unknown - case $UNAME_PROCESSOR in - i386) - eval $set_cc_for_build - if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then - if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \ - (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \ - grep IS_64BIT_ARCH >/dev/null - then - UNAME_PROCESSOR="x86_64" - fi - fi ;; - unknown) UNAME_PROCESSOR=powerpc ;; - esac - echo ${UNAME_PROCESSOR}-apple-darwin${UNAME_RELEASE} + set_cc_for_build + if test "$UNAME_PROCESSOR" = unknown ; then + UNAME_PROCESSOR=powerpc + fi + if test "`echo "$UNAME_RELEASE" | sed -e 's/\..*//'`" -le 10 ; then + if [ "$CC_FOR_BUILD" != no_compiler_found ]; then + if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \ + (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_64BIT_ARCH >/dev/null + then + case $UNAME_PROCESSOR in + i386) UNAME_PROCESSOR=x86_64 ;; + powerpc) UNAME_PROCESSOR=powerpc64 ;; + esac + fi + # On 10.4-10.6 one might compile for PowerPC via gcc -arch ppc + if (echo '#ifdef __POWERPC__'; echo IS_PPC; echo '#endif') | \ + (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_PPC >/dev/null + then + UNAME_PROCESSOR=powerpc + fi + fi + elif test "$UNAME_PROCESSOR" = i386 ; then + # Avoid executing cc on OS X 10.9, as it ships with a stub + # that puts up a graphical alert prompting to install + # developer tools. Any system running Mac OS X 10.7 or + # later (Darwin 11 and later) is required to have a 64-bit + # processor. This is not true of the ARM version of Darwin + # that Apple uses in portable devices. + UNAME_PROCESSOR=x86_64 + fi + echo "$UNAME_PROCESSOR"-apple-darwin"$UNAME_RELEASE" exit ;; *:procnto*:*:* | *:QNX:[0123456789]*:*) UNAME_PROCESSOR=`uname -p` - if test "$UNAME_PROCESSOR" = "x86"; then + if test "$UNAME_PROCESSOR" = x86; then UNAME_PROCESSOR=i386 UNAME_MACHINE=pc fi - echo ${UNAME_PROCESSOR}-${UNAME_MACHINE}-nto-qnx${UNAME_RELEASE} + echo "$UNAME_PROCESSOR"-"$UNAME_MACHINE"-nto-qnx"$UNAME_RELEASE" exit ;; *:QNX:*:4*) echo i386-pc-qnx exit ;; - NEO-?:NONSTOP_KERNEL:*:*) - echo neo-tandem-nsk${UNAME_RELEASE} + NEO-*:NONSTOP_KERNEL:*:*) + echo neo-tandem-nsk"$UNAME_RELEASE" exit ;; NSE-*:NONSTOP_KERNEL:*:*) - echo nse-tandem-nsk${UNAME_RELEASE} + echo nse-tandem-nsk"$UNAME_RELEASE" + exit ;; + NSR-*:NONSTOP_KERNEL:*:*) + echo nsr-tandem-nsk"$UNAME_RELEASE" exit ;; - NSR-?:NONSTOP_KERNEL:*:*) - echo nsr-tandem-nsk${UNAME_RELEASE} + NSV-*:NONSTOP_KERNEL:*:*) + echo nsv-tandem-nsk"$UNAME_RELEASE" + exit ;; + NSX-*:NONSTOP_KERNEL:*:*) + echo nsx-tandem-nsk"$UNAME_RELEASE" exit ;; *:NonStop-UX:*:*) echo mips-compaq-nonstopux @@ -1279,18 +1361,19 @@ EOF echo bs2000-siemens-sysv exit ;; DS/*:UNIX_System_V:*:*) - echo ${UNAME_MACHINE}-${UNAME_SYSTEM}-${UNAME_RELEASE} + echo "$UNAME_MACHINE"-"$UNAME_SYSTEM"-"$UNAME_RELEASE" exit ;; *:Plan9:*:*) # "uname -m" is not consistent, so use $cputype instead. 386 # is converted to i386 for consistency with other x86 # operating systems. - if test "$cputype" = "386"; then + # shellcheck disable=SC2154 + if test "$cputype" = 386; then UNAME_MACHINE=i386 else UNAME_MACHINE="$cputype" fi - echo ${UNAME_MACHINE}-unknown-plan9 + echo "$UNAME_MACHINE"-unknown-plan9 exit ;; *:TOPS-10:*:*) echo pdp10-unknown-tops10 @@ -1311,14 +1394,14 @@ EOF echo pdp10-unknown-its exit ;; SEI:*:*:SEIUX) - echo mips-sei-seiux${UNAME_RELEASE} + echo mips-sei-seiux"$UNAME_RELEASE" exit ;; *:DragonFly:*:*) - echo ${UNAME_MACHINE}-unknown-dragonfly`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` + echo "$UNAME_MACHINE"-unknown-dragonfly"`echo "$UNAME_RELEASE"|sed -e 's/[-(].*//'`" exit ;; *:*VMS:*:*) UNAME_MACHINE=`(uname -p) 2>/dev/null` - case "${UNAME_MACHINE}" in + case "$UNAME_MACHINE" in A*) echo alpha-dec-vms ; exit ;; I*) echo ia64-dec-vms ; exit ;; V*) echo vax-dec-vms ; exit ;; @@ -1327,182 +1410,48 @@ EOF echo i386-pc-xenix exit ;; i*86:skyos:*:*) - echo ${UNAME_MACHINE}-pc-skyos`echo ${UNAME_RELEASE}` | sed -e 's/ .*$//' + echo "$UNAME_MACHINE"-pc-skyos"`echo "$UNAME_RELEASE" | sed -e 's/ .*$//'`" exit ;; i*86:rdos:*:*) - echo ${UNAME_MACHINE}-pc-rdos + echo "$UNAME_MACHINE"-pc-rdos exit ;; i*86:AROS:*:*) - echo ${UNAME_MACHINE}-pc-aros + echo "$UNAME_MACHINE"-pc-aros exit ;; x86_64:VMkernel:*:*) - echo ${UNAME_MACHINE}-unknown-esx + echo "$UNAME_MACHINE"-unknown-esx + exit ;; + amd64:Isilon\ OneFS:*:*) + echo x86_64-unknown-onefs exit ;; esac -eval $set_cc_for_build -cat >$dummy.c <<EOF -#ifdef _SEQUENT_ -# include <sys/types.h> -# include <sys/utsname.h> -#endif -main () -{ -#if defined (sony) -#if defined (MIPSEB) - /* BFD wants "bsd" instead of "newsos". Perhaps BFD should be changed, - I don't know.... */ - printf ("mips-sony-bsd\n"); exit (0); -#else -#include <sys/param.h> - printf ("m68k-sony-newsos%s\n", -#ifdef NEWSOS4 - "4" -#else - "" -#endif - ); exit (0); -#endif -#endif - -#if defined (__arm) && defined (__acorn) && defined (__unix) - printf ("arm-acorn-riscix\n"); exit (0); -#endif +echo "$0: unable to guess system type" >&2 -#if defined (hp300) && !defined (hpux) - printf ("m68k-hp-bsd\n"); exit (0); -#endif +case "$UNAME_MACHINE:$UNAME_SYSTEM" in + mips:Linux | mips64:Linux) + # If we got here on MIPS GNU/Linux, output extra information. + cat >&2 <<EOF -#if defined (NeXT) -#if !defined (__ARCHITECTURE__) -#define __ARCHITECTURE__ "m68k" -#endif - int version; - version=`(hostinfo | sed -n 's/.*NeXT Mach \([0-9]*\).*/\1/p') 2>/dev/null`; - if (version < 4) - printf ("%s-next-nextstep%d\n", __ARCHITECTURE__, version); - else - printf ("%s-next-openstep%d\n", __ARCHITECTURE__, version); - exit (0); -#endif - -#if defined (MULTIMAX) || defined (n16) -#if defined (UMAXV) - printf ("ns32k-encore-sysv\n"); exit (0); -#else -#if defined (CMU) - printf ("ns32k-encore-mach\n"); exit (0); -#else - printf ("ns32k-encore-bsd\n"); exit (0); -#endif -#endif -#endif - -#if defined (__386BSD__) - printf ("i386-pc-bsd\n"); exit (0); -#endif - -#if defined (sequent) -#if defined (i386) - printf ("i386-sequent-dynix\n"); exit (0); -#endif -#if defined (ns32000) - printf ("ns32k-sequent-dynix\n"); exit (0); -#endif -#endif - -#if defined (_SEQUENT_) - struct utsname un; - - uname(&un); - - if (strncmp(un.version, "V2", 2) == 0) { - printf ("i386-sequent-ptx2\n"); exit (0); - } - if (strncmp(un.version, "V1", 2) == 0) { /* XXX is V1 correct? */ - printf ("i386-sequent-ptx1\n"); exit (0); - } - printf ("i386-sequent-ptx\n"); exit (0); - -#endif - -#if defined (vax) -# if !defined (ultrix) -# include <sys/param.h> -# if defined (BSD) -# if BSD == 43 - printf ("vax-dec-bsd4.3\n"); exit (0); -# else -# if BSD == 199006 - printf ("vax-dec-bsd4.3reno\n"); exit (0); -# else - printf ("vax-dec-bsd\n"); exit (0); -# endif -# endif -# else - printf ("vax-dec-bsd\n"); exit (0); -# endif -# else - printf ("vax-dec-ultrix\n"); exit (0); -# endif -#endif - -#if defined (alliant) && defined (i860) - printf ("i860-alliant-bsd\n"); exit (0); -#endif - - exit (1); -} +NOTE: MIPS GNU/Linux systems require a C compiler to fully recognize +the system type. Please install a C compiler and try again. EOF - -$CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null && SYSTEM_NAME=`$dummy` && - { echo "$SYSTEM_NAME"; exit; } - -# Apollos put the system type in the environment. - -test -d /usr/apollo && { echo ${ISP}-apollo-${SYSTYPE}; exit; } - -# Convex versions that predate uname can use getsysinfo(1) - -if [ -x /usr/convex/getsysinfo ] -then - case `getsysinfo -f cpu_type` in - c1*) - echo c1-convex-bsd - exit ;; - c2*) - if getsysinfo -f scalar_acc - then echo c32-convex-bsd - else echo c2-convex-bsd - fi - exit ;; - c34*) - echo c34-convex-bsd - exit ;; - c38*) - echo c38-convex-bsd - exit ;; - c4*) - echo c4-convex-bsd - exit ;; - esac -fi + ;; +esac cat >&2 <<EOF -$0: unable to guess system type -This script, last modified $timestamp, has failed to recognize -the operating system you are using. It is advised that you -download the most up to date version of the config scripts from +This script (version $timestamp), has failed to recognize the +operating system you are using. If your script is old, overwrite *all* +copies of config.guess and config.sub with the latest versions from: - http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD + https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess and - http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD + https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub -If the version you run ($0) is already up to date, please -send the following data and any information you think might be -pertinent to <config-patches@gnu.org> in order to provide the needed -information to handle your system. +If $0 has already been updated, send the following data and any +information you think might be pertinent to config-patches@gnu.org to +provide the necessary information to handle your system. config.guess timestamp = $timestamp @@ -1521,16 +1470,16 @@ hostinfo = `(hostinfo) 2>/dev/null` /usr/bin/oslevel = `(/usr/bin/oslevel) 2>/dev/null` /usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null` -UNAME_MACHINE = ${UNAME_MACHINE} -UNAME_RELEASE = ${UNAME_RELEASE} -UNAME_SYSTEM = ${UNAME_SYSTEM} -UNAME_VERSION = ${UNAME_VERSION} +UNAME_MACHINE = "$UNAME_MACHINE" +UNAME_RELEASE = "$UNAME_RELEASE" +UNAME_SYSTEM = "$UNAME_SYSTEM" +UNAME_VERSION = "$UNAME_VERSION" EOF exit 1 # Local variables: -# eval: (add-hook 'write-file-hooks 'time-stamp) +# eval: (add-hook 'before-save-hook 'time-stamp) # time-stamp-start: "timestamp='" # time-stamp-format: "%:y-%02m-%02d" # time-stamp-end: "'" diff --git a/config/config.sub b/config/config.sub index bdda9e4a32c1..c19e671805a8 100755 --- a/config/config.sub +++ b/config/config.sub @@ -1,36 +1,31 @@ #! /bin/sh # Configuration validation subroutine script. -# Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, -# 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, -# 2011, 2012 Free Software Foundation, Inc. +# Copyright 1992-2018 Free Software Foundation, Inc. -timestamp='2012-08-18' +timestamp='2018-08-13' -# This file is (in principle) common to ALL GNU software. -# The presence of a machine in this file suggests that SOME GNU software -# can handle that machine. It does not imply ALL GNU software can. -# -# This file is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or # (at your option) any later version. # -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with this program; if not, see <http://www.gnu.org/licenses/>. +# along with this program; if not, see <https://www.gnu.org/licenses/>. # # As a special exception to the GNU General Public License, if you # distribute this file as part of a program that contains a # configuration script generated by Autoconf, you may include it under -# the same distribution terms that you use for the rest of that program. +# the same distribution terms that you use for the rest of that +# program. This Exception is an additional permission under section 7 +# of the GNU General Public License, version 3 ("GPLv3"). -# Please send patches to <config-patches@gnu.org>. Submit a context -# diff and a properly formatted GNU ChangeLog entry. +# Please send patches to <config-patches@gnu.org>. # # Configuration subroutine to validate and canonicalize a configuration type. # Supply the specified configuration type as an argument. @@ -38,7 +33,7 @@ timestamp='2012-08-18' # Otherwise, we print the canonical config type on stdout and succeed. # You can get the latest version of this script from: -# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD +# https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub # This file is supposed to be the same for all GNU packages # and recognize all the CPU types, system types and aliases @@ -58,12 +53,11 @@ timestamp='2012-08-18' me=`echo "$0" | sed -e 's,.*/,,'` usage="\ -Usage: $0 [OPTION] CPU-MFR-OPSYS - $0 [OPTION] ALIAS +Usage: $0 [OPTION] CPU-MFR-OPSYS or ALIAS Canonicalize a configuration name. -Operation modes: +Options: -h, --help print this help, then exit -t, --time-stamp print date of last modification, then exit -v, --version print version number, then exit @@ -73,9 +67,7 @@ Report bugs and patches to <config-patches@gnu.org>." version="\ GNU config.sub ($timestamp) -Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, -2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 -Free Software Foundation, Inc. +Copyright 1992-2018 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -102,7 +94,7 @@ while test $# -gt 0 ; do *local*) # First pass through any local machine types. - echo $1 + echo "$1" exit ;; * ) @@ -118,162 +110,596 @@ case $# in exit 1;; esac -# Separate what the user gave into CPU-COMPANY and OS or KERNEL-OS (if any). -# Here we must recognize all the valid KERNEL-OS combinations. -maybe_os=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\2/'` -case $maybe_os in - nto-qnx* | linux-gnu* | linux-android* | linux-dietlibc | linux-newlib* | \ - linux-musl* | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | \ - knetbsd*-gnu* | netbsd*-gnu* | \ - kopensolaris*-gnu* | \ - storm-chaos* | os2-emx* | rtmk-nova*) - os=-$maybe_os - basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'` - ;; - android-linux) - os=-linux-android - basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'`-unknown - ;; - *) - basic_machine=`echo $1 | sed 's/-[^-]*$//'` - if [ $basic_machine != $1 ] - then os=`echo $1 | sed 's/.*-/-/'` - else os=; fi - ;; -esac +# Split fields of configuration type +IFS="-" read -r field1 field2 field3 field4 <<EOF +$1 +EOF -### Let's recognize common machines as not being operating systems so -### that things like config.sub decstation-3100 work. We also -### recognize some manufacturers as not being operating systems, so we -### can provide default operating systems below. -case $os in - -sun*os*) - # Prevent following clause from handling this invalid input. - ;; - -dec* | -mips* | -sequent* | -encore* | -pc532* | -sgi* | -sony* | \ - -att* | -7300* | -3300* | -delta* | -motorola* | -sun[234]* | \ - -unicom* | -ibm* | -next | -hp | -isi* | -apollo | -altos* | \ - -convergent* | -ncr* | -news | -32* | -3600* | -3100* | -hitachi* |\ - -c[123]* | -convex* | -sun | -crds | -omron* | -dg | -ultra | -tti* | \ - -harris | -dolphin | -highlevel | -gould | -cbm | -ns | -masscomp | \ - -apple | -axis | -knuth | -cray | -microblaze) - os= - basic_machine=$1 - ;; - -bluegene*) - os=-cnk - ;; - -sim | -cisco | -oki | -wec | -winbond) - os= - basic_machine=$1 - ;; - -scout) +# Separate into logical components for further validation +case $1 in + *-*-*-*-*) + echo Invalid configuration \`"$1"\': more than four components >&2 + exit 1 ;; - -wrs) - os=-vxworks - basic_machine=$1 + *-*-*-*) + basic_machine=$field1-$field2 + os=$field3-$field4 ;; - -chorusos*) - os=-chorusos - basic_machine=$1 + *-*-*) + # Ambiguous whether COMPANY is present, or skipped and KERNEL-OS is two + # parts + maybe_os=$field2-$field3 + case $maybe_os in + nto-qnx* | linux-gnu* | linux-android* | linux-dietlibc \ + | linux-newlib* | linux-musl* | linux-uclibc* | uclinux-uclibc* \ + | uclinux-gnu* | kfreebsd*-gnu* | knetbsd*-gnu* | netbsd*-gnu* \ + | netbsd*-eabi* | kopensolaris*-gnu* | cloudabi*-eabi* \ + | storm-chaos* | os2-emx* | rtmk-nova*) + basic_machine=$field1 + os=$maybe_os + ;; + android-linux) + basic_machine=$field1-unknown + os=linux-android + ;; + *) + basic_machine=$field1-$field2 + os=$field3 + ;; + esac ;; - -chorusrdb) - os=-chorusrdb - basic_machine=$1 + *-*) + # Second component is usually, but not always the OS + case $field2 in + # Prevent following clause from handling this valid os + sun*os*) + basic_machine=$field1 + os=$field2 + ;; + # Manufacturers + dec* | mips* | sequent* | encore* | pc532* | sgi* | sony* \ + | att* | 7300* | 3300* | delta* | motorola* | sun[234]* \ + | unicom* | ibm* | next | hp | isi* | apollo | altos* \ + | convergent* | ncr* | news | 32* | 3600* | 3100* | hitachi* \ + | c[123]* | convex* | sun | crds | omron* | dg | ultra | tti* \ + | harris | dolphin | highlevel | gould | cbm | ns | masscomp \ + | apple | axis | knuth | cray | microblaze* \ + | sim | cisco | oki | wec | wrs | winbond) + basic_machine=$field1-$field2 + os= + ;; + *) + basic_machine=$field1 + os=$field2 + ;; + esac ;; - -hiux*) - os=-hiuxwe2 + *) + # Convert single-component short-hands not valid as part of + # multi-component configurations. + case $field1 in + 386bsd) + basic_machine=i386-pc + os=bsd + ;; + a29khif) + basic_machine=a29k-amd + os=udi + ;; + adobe68k) + basic_machine=m68010-adobe + os=scout + ;; + alliant) + basic_machine=fx80-alliant + os= + ;; + altos | altos3068) + basic_machine=m68k-altos + os= + ;; + am29k) + basic_machine=a29k-none + os=bsd + ;; + amdahl) + basic_machine=580-amdahl + os=sysv + ;; + amigaos | amigados) + basic_machine=m68k-unknown + os=amigaos + ;; + amigaunix | amix) + basic_machine=m68k-unknown + os=sysv4 + ;; + apollo68) + basic_machine=m68k-apollo + os=sysv + ;; + apollo68bsd) + basic_machine=m68k-apollo + os=bsd + ;; + aros) + basic_machine=i386-pc + os=aros + ;; + aux) + basic_machine=m68k-apple + os=aux + ;; + balance) + basic_machine=ns32k-sequent + os=dynix + ;; + blackfin) + basic_machine=bfin-unknown + os=linux + ;; + cegcc) + basic_machine=arm-unknown + os=cegcc + ;; + convex-c1) + basic_machine=c1-convex + os=bsd + ;; + convex-c2) + basic_machine=c2-convex + os=bsd + ;; + convex-c32) + basic_machine=c32-convex + os=bsd + ;; + convex-c34) + basic_machine=c34-convex + os=bsd + ;; + convex-c38) + basic_machine=c38-convex + os=bsd + ;; + cray) + basic_machine=j90-cray + os=unicos + ;; + crds | unos) + basic_machine=m68k-crds + os= + ;; + delta88) + basic_machine=m88k-motorola + os=sysv3 + ;; + dicos) + basic_machine=i686-pc + os=dicos + ;; + djgpp) + basic_machine=i586-pc + os=msdosdjgpp + ;; + ebmon29k) + basic_machine=a29k-amd + os=ebmon + ;; + es1800 | OSE68k | ose68k | ose | OSE) + basic_machine=m68k-ericsson + os=ose + ;; + gmicro) + basic_machine=tron-gmicro + os=sysv + ;; + go32) + basic_machine=i386-pc + os=go32 + ;; + h8300hms) + basic_machine=h8300-hitachi + os=hms + ;; + h8300xray) + basic_machine=h8300-hitachi + os=xray + ;; + h8500hms) + basic_machine=h8500-hitachi + os=hms + ;; + harris) + basic_machine=m88k-harris + os=sysv3 + ;; + hp300bsd) + basic_machine=m68k-hp + os=bsd + ;; + hp300hpux) + basic_machine=m68k-hp + os=hpux + ;; + hppaosf) + basic_machine=hppa1.1-hp + os=osf + ;; + hppro) + basic_machine=hppa1.1-hp + os=proelf + ;; + i386mach) + basic_machine=i386-mach + os=mach + ;; + vsta) + basic_machine=i386-pc + os=vsta + ;; + isi68 | isi) + basic_machine=m68k-isi + os=sysv + ;; + m68knommu) + basic_machine=m68k-unknown + os=linux + ;; + magnum | m3230) + basic_machine=mips-mips + os=sysv + ;; + merlin) + basic_machine=ns32k-utek + os=sysv + ;; + mingw64) + basic_machine=x86_64-pc + os=mingw64 + ;; + mingw32) + basic_machine=i686-pc + os=mingw32 + ;; + mingw32ce) + basic_machine=arm-unknown + os=mingw32ce + ;; + monitor) + basic_machine=m68k-rom68k + os=coff + ;; + morphos) + basic_machine=powerpc-unknown + os=morphos + ;; + moxiebox) + basic_machine=moxie-unknown + os=moxiebox + ;; + msdos) + basic_machine=i386-pc + os=msdos + ;; + msys) + basic_machine=i686-pc + os=msys + ;; + mvs) + basic_machine=i370-ibm + os=mvs + ;; + nacl) + basic_machine=le32-unknown + os=nacl + ;; + ncr3000) + basic_machine=i486-ncr + os=sysv4 + ;; + netbsd386) + basic_machine=i386-pc + os=netbsd + ;; + netwinder) + basic_machine=armv4l-rebel + os=linux + ;; + news | news700 | news800 | news900) + basic_machine=m68k-sony + os=newsos + ;; + news1000) + basic_machine=m68030-sony + os=newsos + ;; + necv70) + basic_machine=v70-nec + os=sysv + ;; + nh3000) + basic_machine=m68k-harris + os=cxux + ;; + nh[45]000) + basic_machine=m88k-harris + os=cxux + ;; + nindy960) + basic_machine=i960-intel + os=nindy + ;; + mon960) + basic_machine=i960-intel + os=mon960 + ;; + nonstopux) + basic_machine=mips-compaq + os=nonstopux + ;; + os400) + basic_machine=powerpc-ibm + os=os400 + ;; + OSE68000 | ose68000) + basic_machine=m68000-ericsson + os=ose + ;; + os68k) + basic_machine=m68k-none + os=os68k + ;; + paragon) + basic_machine=i860-intel + os=osf + ;; + parisc) + basic_machine=hppa-unknown + os=linux + ;; + pw32) + basic_machine=i586-unknown + os=pw32 + ;; + rdos | rdos64) + basic_machine=x86_64-pc + os=rdos + ;; + rdos32) + basic_machine=i386-pc + os=rdos + ;; + rom68k) + basic_machine=m68k-rom68k + os=coff + ;; + sa29200) + basic_machine=a29k-amd + os=udi + ;; + sei) + basic_machine=mips-sei + os=seiux + ;; + sps7) + basic_machine=m68k-bull + os=sysv2 + ;; + st2000) + basic_machine=m68k-tandem + os= + ;; + stratus) + basic_machine=i860-stratus + os=sysv4 + ;; + sun2) + basic_machine=m68000-sun + os= + ;; + sun2os3) + basic_machine=m68000-sun + os=sunos3 + ;; + sun2os4) + basic_machine=m68000-sun + os=sunos4 + ;; + sun3) + basic_machine=m68k-sun + os= + ;; + sun3os3) + basic_machine=m68k-sun + os=sunos3 + ;; + sun3os4) + basic_machine=m68k-sun + os=sunos4 + ;; + sun4) + basic_machine=sparc-sun + os= + ;; + sun4os3) + basic_machine=sparc-sun + os=sunos3 + ;; + sun4os4) + basic_machine=sparc-sun + os=sunos4 + ;; + sun4sol2) + basic_machine=sparc-sun + os=solaris2 + ;; + sun386 | sun386i | roadrunner) + basic_machine=i386-sun + os= + ;; + sv1) + basic_machine=sv1-cray + os=unicos + ;; + symmetry) + basic_machine=i386-sequent + os=dynix + ;; + t3e) + basic_machine=alphaev5-cray + os=unicos + ;; + t90) + basic_machine=t90-cray + os=unicos + ;; + toad1) + basic_machine=pdp10-xkl + os=tops20 + ;; + tpf) + basic_machine=s390x-ibm + os=tpf + ;; + udi29k) + basic_machine=a29k-amd + os=udi + ;; + ultra3) + basic_machine=a29k-nyu + os=sym1 + ;; + v810 | necv810) + basic_machine=v810-nec + os=none + ;; + vaxv) + basic_machine=vax-dec + os=sysv + ;; + vms) + basic_machine=vax-dec + os=vms + ;; + vxworks960) + basic_machine=i960-wrs + os=vxworks + ;; + vxworks68) + basic_machine=m68k-wrs + os=vxworks + ;; + vxworks29k) + basic_machine=a29k-wrs + os=vxworks + ;; + xbox) + basic_machine=i686-pc + os=mingw32 + ;; + ymp) + basic_machine=ymp-cray + os=unicos + ;; + *) + basic_machine=$1 + os= + ;; + esac ;; - -sco6) - os=-sco5v6 - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` +esac + +# Decode aliases for certain CPU-COMPANY combinations. +case $basic_machine in + # Here we handle the default manufacturer of certain CPU types. It is in + # some cases the only manufacturer, in others, it is the most popular. + craynv) + basic_machine=craynv-cray + os=${os:-unicosmp} ;; - -sco5) - os=-sco3.2v5 - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + fx80) + basic_machine=fx80-alliant ;; - -sco4) - os=-sco3.2v4 - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + w89k) + basic_machine=hppa1.1-winbond ;; - -sco3.2.[4-9]*) - os=`echo $os | sed -e 's/sco3.2./sco3.2v/'` - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + op50n) + basic_machine=hppa1.1-oki ;; - -sco3.2v[4-9]*) - # Don't forget version if it is 3.2v4 or newer. - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + op60c) + basic_machine=hppa1.1-oki ;; - -sco5v6*) - # Don't forget version if it is 3.2v4 or newer. - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + romp) + basic_machine=romp-ibm ;; - -sco*) - os=-sco3.2v2 - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + mmix) + basic_machine=mmix-knuth ;; - -udk*) - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + rs6000) + basic_machine=rs6000-ibm ;; - -isc) - os=-isc2.2 - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + vax) + basic_machine=vax-dec ;; - -clix*) - basic_machine=clipper-intergraph + pdp11) + basic_machine=pdp11-dec ;; - -isc*) - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + we32k) + basic_machine=we32k-att ;; - -lynx*178) - os=-lynxos178 + cydra) + basic_machine=cydra-cydrome ;; - -lynx*5) - os=-lynxos5 + i370-ibm* | ibm*) + basic_machine=i370-ibm ;; - -lynx*) - os=-lynxos + orion) + basic_machine=orion-highlevel ;; - -ptx*) - basic_machine=`echo $1 | sed -e 's/86-.*/86-sequent/'` + orion105) + basic_machine=clipper-highlevel ;; - -windowsnt*) - os=`echo $os | sed -e 's/windowsnt/winnt/'` + mac | mpw | mac-mpw) + basic_machine=m68k-apple ;; - -psos*) - os=-psos + pmac | pmac-mpw) + basic_machine=powerpc-apple ;; - -mint | -mint[0-9]*) - basic_machine=m68k-atari - os=-mint + xps | xps100) + basic_machine=xps100-honeywell ;; -esac -# Decode aliases for certain CPU-COMPANY combinations. -case $basic_machine in # Recognize the basic CPU types without company name. # Some are omitted here because they have special meanings below. 1750a | 580 \ | a29k \ | aarch64 | aarch64_be \ + | abacus \ | alpha | alphaev[4-8] | alphaev56 | alphaev6[78] | alphapca5[67] \ | alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] | alpha64pca5[67] \ | am33_2.0 \ - | arc | arm | arm[bl]e | arme[lb] | armv[2345] | armv[345][lb] | avr | avr32 \ - | be32 | be64 \ + | arc | arceb \ + | arm | arm[bl]e | arme[lb] | armv[2-8] | armv[3-8][lb] | armv6m | armv[78][arm] \ + | avr | avr32 \ + | asmjs \ + | ba \ + | be32 | be64 \ | bfin \ - | c4x | clipper \ + | c4x | c8051 | clipper | csky \ | d10v | d30v | dlx | dsp16xx \ - | epiphany \ - | fido | fr30 | frv \ + | e2k | epiphany \ + | fido | fr30 | frv | ft32 \ | h8300 | h8500 | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \ | hexagon \ - | i370 | i860 | i960 | ia64 \ + | i370 | i860 | i960 | ia16 | ia64 \ | ip2k | iq2000 \ + | k1om \ | le32 | le64 \ | lm32 \ | m32c | m32r | m32rle | m68000 | m68k | m88k \ - | maxq | mb | microblaze | mcore | mep | metag \ + | m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x | nvptx | picochip \ + | maxq | mb | microblaze | microblazeel | mcore | mep | metag \ | mips | mipsbe | mipseb | mipsel | mipsle \ | mips16 \ | mips64 | mips64el \ @@ -287,26 +713,31 @@ case $basic_machine in | mips64vr5900 | mips64vr5900el \ | mipsisa32 | mipsisa32el \ | mipsisa32r2 | mipsisa32r2el \ + | mipsisa32r6 | mipsisa32r6el \ | mipsisa64 | mipsisa64el \ | mipsisa64r2 | mipsisa64r2el \ + | mipsisa64r6 | mipsisa64r6el \ | mipsisa64sb1 | mipsisa64sb1el \ | mipsisa64sr71k | mipsisa64sr71kel \ + | mipsr5900 | mipsr5900el \ | mipstx39 | mipstx39el \ | mn10200 | mn10300 \ | moxie \ | mt \ | msp430 \ | nds32 | nds32le | nds32be \ - | nios | nios2 \ + | nfp \ + | nios | nios2 | nios2eb | nios2el \ | ns16k | ns32k \ - | open8 \ - | or32 \ - | pdp10 | pdp11 | pj | pjl \ + | open8 | or1k | or1knd | or32 \ + | pdp10 | pj | pjl \ | powerpc | powerpc64 | powerpc64le | powerpcle \ + | pru \ | pyramid \ + | riscv | riscv32 | riscv64 \ | rl78 | rx \ | score \ - | sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[34]eb | sheb | shbe | shle | sh[1234]le | sh3ele \ + | sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[234]eb | sheb | shbe | shle | sh[1234]le | sh[23]ele \ | sh64 | sh64le \ | sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet | sparclite \ | sparcv8 | sparcv9 | sparcv9b | sparcv9v \ @@ -314,8 +745,9 @@ case $basic_machine in | tahoe | tic4x | tic54x | tic55x | tic6x | tic80 | tron \ | ubicom32 \ | v850 | v850e | v850e1 | v850e2 | v850es | v850e2v3 \ - | we32k \ - | x86 | xc16x | xstormy16 | xtensa \ + | visium \ + | wasm32 \ + | x86 | xc16x | xstormy16 | xgate | xtensa \ | z8k | z80) basic_machine=$basic_machine-unknown ;; @@ -328,23 +760,23 @@ case $basic_machine in c6x) basic_machine=tic6x-unknown ;; - m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x | picochip) - basic_machine=$basic_machine-unknown - os=-none + leon|leon[3-9]) + basic_machine=sparc-$basic_machine + ;; + m88110 | m680[12346]0 | m683?2 | m68360 | m5200 | v70 | w65) + ;; + m9s12z | m68hcs12z | hcs12z | s12z) + basic_machine=s12z-unknown ;; - m88110 | m680[12346]0 | m683?2 | m68360 | m5200 | v70 | w65 | z8k) + m9s12z-* | m68hcs12z-* | hcs12z-* | s12z-*) + basic_machine=s12z-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; ms1) basic_machine=mt-unknown ;; - strongarm | thumb | xscale) basic_machine=arm-unknown ;; - xgate) - basic_machine=$basic_machine-unknown - os=-none - ;; xscaleeb) basic_machine=armeb-unknown ;; @@ -359,37 +791,40 @@ case $basic_machine in i*86 | x86_64) basic_machine=$basic_machine-pc ;; - # Object if more than one company name word. - *-*-*) - echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2 - exit 1 - ;; # Recognize the basic CPU types with company name. - 580-* \ + 1750a-* | 580-* \ | a29k-* \ | aarch64-* | aarch64_be-* \ + | abacus-* \ | alpha-* | alphaev[4-8]-* | alphaev56-* | alphaev6[78]-* \ | alpha64-* | alpha64ev[4-8]-* | alpha64ev56-* | alpha64ev6[78]-* \ - | alphapca5[67]-* | alpha64pca5[67]-* | arc-* \ - | arm-* | armbe-* | armle-* | armeb-* | armv*-* \ + | alphapca5[67]-* | alpha64pca5[67]-* \ + | am33_2.0-* \ + | arc-* | arceb-* \ + | arm-* | arm[lb]e-* | arme[lb]-* | armv*-* \ | avr-* | avr32-* \ + | asmjs-* \ + | ba-* \ | be32-* | be64-* \ | bfin-* | bs2000-* \ | c[123]* | c30-* | [cjt]90-* | c4x-* \ - | clipper-* | craynv-* | cydra-* \ - | d10v-* | d30v-* | dlx-* \ - | elxsi-* \ - | f30[01]-* | f700-* | fido-* | fr30-* | frv-* | fx80-* \ + | c8051-* | clipper-* | craynv-* | csky-* | cydra-* \ + | d10v-* | d30v-* | dlx-* | dsp16xx-* \ + | e2k-* | elxsi-* | epiphany-* \ + | f30[01]-* | f700-* | fido-* | fr30-* | frv-* | ft32-* | fx80-* \ | h8300-* | h8500-* \ | hppa-* | hppa1.[01]-* | hppa2.0-* | hppa2.0[nw]-* | hppa64-* \ | hexagon-* \ - | i*86-* | i860-* | i960-* | ia64-* \ + | i370-* | i*86-* | i860-* | i960-* | ia16-* | ia64-* \ | ip2k-* | iq2000-* \ + | k1om-* \ | le32-* | le64-* \ | lm32-* \ | m32c-* | m32r-* | m32rle-* \ - | m68000-* | m680[012346]0-* | m68360-* | m683?2-* | m68k-* \ - | m88110-* | m88k-* | maxq-* | mcore-* | metag-* | microblaze-* \ + | m5200-* | m68000-* | m680[012346]0-* | m68360-* | m683?2-* | m68k-* | v70-* | w65-* \ + | m6811-* | m68hc11-* | m6812-* | m68hc12-* | m68hcs12x-* | nvptx-* | picochip-* \ + | m88110-* | m88k-* | maxq-* | mb-* | mcore-* | mep-* | metag-* \ + | microblaze-* | microblazeel-* \ | mips-* | mipsbe-* | mipseb-* | mipsel-* | mipsle-* \ | mips16-* \ | mips64-* | mips64el-* \ @@ -403,37 +838,50 @@ case $basic_machine in | mips64vr5900-* | mips64vr5900el-* \ | mipsisa32-* | mipsisa32el-* \ | mipsisa32r2-* | mipsisa32r2el-* \ + | mipsisa32r6-* | mipsisa32r6el-* \ | mipsisa64-* | mipsisa64el-* \ | mipsisa64r2-* | mipsisa64r2el-* \ + | mipsisa64r6-* | mipsisa64r6el-* \ | mipsisa64sb1-* | mipsisa64sb1el-* \ | mipsisa64sr71k-* | mipsisa64sr71kel-* \ + | mipsr5900-* | mipsr5900el-* \ | mipstx39-* | mipstx39el-* \ | mmix-* \ + | mn10200-* | mn10300-* \ + | moxie-* \ | mt-* \ | msp430-* \ | nds32-* | nds32le-* | nds32be-* \ - | nios-* | nios2-* \ + | nfp-* \ + | nios-* | nios2-* | nios2eb-* | nios2el-* \ | none-* | np1-* | ns16k-* | ns32k-* \ | open8-* \ + | or1k*-* \ + | or32-* \ | orion-* \ | pdp10-* | pdp11-* | pj-* | pjl-* | pn-* | power-* \ | powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* \ + | pru-* \ | pyramid-* \ + | riscv-* | riscv32-* | riscv64-* \ | rl78-* | romp-* | rs6000-* | rx-* \ - | sh-* | sh[1234]-* | sh[24]a-* | sh[24]aeb-* | sh[23]e-* | sh[34]eb-* | sheb-* | shbe-* \ - | shle-* | sh[1234]le-* | sh3ele-* | sh64-* | sh64le-* \ + | score-* \ + | sh-* | sh[1234]-* | sh[24]a-* | sh[24]ae[lb]-* | sh[23]e-* | she[lb]-* | sh[lb]e-* \ + | sh[1234]e[lb]-* | sh[12345][lb]e-* | sh[23]ele-* | sh64-* | sh64le-* \ | sparc-* | sparc64-* | sparc64b-* | sparc64v-* | sparc86x-* | sparclet-* \ | sparclite-* \ - | sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx?-* \ + | sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx*-* \ + | spu-* \ | tahoe-* \ | tic30-* | tic4x-* | tic54x-* | tic55x-* | tic6x-* | tic80-* \ - | tile*-* \ | tron-* \ | ubicom32-* \ | v850-* | v850e-* | v850e1-* | v850es-* | v850e2-* | v850e2v3-* \ | vax-* \ + | visium-* \ + | wasm32-* \ | we32k-* \ - | x86-* | x86_64-* | xc16x-* | xps100-* \ + | x86-* | x86_64-* | xc16x-* | xgate-* | xps100-* \ | xstormy16-* | xtensa*-* \ | ymp-* \ | z8k-* | z80-*) @@ -444,141 +892,45 @@ case $basic_machine in ;; # Recognize the various machine names and aliases which stand # for a CPU type and a company and sometimes even an OS. - 386bsd) - basic_machine=i386-unknown - os=-bsd - ;; 3b1 | 7300 | 7300-att | att-7300 | pc7300 | safari | unixpc) basic_machine=m68000-att ;; 3b*) basic_machine=we32k-att ;; - a29khif) - basic_machine=a29k-amd - os=-udi - ;; - abacus) - basic_machine=abacus-unknown - ;; - adobe68k) - basic_machine=m68010-adobe - os=-scout - ;; - alliant | fx80) - basic_machine=fx80-alliant - ;; - altos | altos3068) - basic_machine=m68k-altos - ;; - am29k) - basic_machine=a29k-none - os=-bsd - ;; amd64) basic_machine=x86_64-pc ;; amd64-*) - basic_machine=x86_64-`echo $basic_machine | sed 's/^[^-]*-//'` - ;; - amdahl) - basic_machine=580-amdahl - os=-sysv + basic_machine=x86_64-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; amiga | amiga-*) basic_machine=m68k-unknown ;; - amigaos | amigados) - basic_machine=m68k-unknown - os=-amigaos - ;; - amigaunix | amix) - basic_machine=m68k-unknown - os=-sysv4 - ;; - apollo68) - basic_machine=m68k-apollo - os=-sysv - ;; - apollo68bsd) - basic_machine=m68k-apollo - os=-bsd - ;; - aros) - basic_machine=i386-pc - os=-aros - ;; - aux) - basic_machine=m68k-apple - os=-aux - ;; - balance) - basic_machine=ns32k-sequent - os=-dynix - ;; - blackfin) - basic_machine=bfin-unknown - os=-linux - ;; blackfin-*) - basic_machine=bfin-`echo $basic_machine | sed 's/^[^-]*-//'` - os=-linux + basic_machine=bfin-`echo "$basic_machine" | sed 's/^[^-]*-//'` + os=linux ;; bluegene*) basic_machine=powerpc-ibm - os=-cnk + os=cnk ;; c54x-*) - basic_machine=tic54x-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=tic54x-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; c55x-*) - basic_machine=tic55x-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=tic55x-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; c6x-*) - basic_machine=tic6x-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=tic6x-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; c90) basic_machine=c90-cray - os=-unicos - ;; - cegcc) - basic_machine=arm-unknown - os=-cegcc - ;; - convex-c1) - basic_machine=c1-convex - os=-bsd - ;; - convex-c2) - basic_machine=c2-convex - os=-bsd - ;; - convex-c32) - basic_machine=c32-convex - os=-bsd - ;; - convex-c34) - basic_machine=c34-convex - os=-bsd - ;; - convex-c38) - basic_machine=c38-convex - os=-bsd - ;; - cray | j90) - basic_machine=j90-cray - os=-unicos - ;; - craynv) - basic_machine=craynv-cray - os=-unicosmp + os=${os:-unicos} ;; cr16 | cr16-*) basic_machine=cr16-unknown - os=-elf - ;; - crds | unos) - basic_machine=m68k-crds + os=${os:-elf} ;; crisv32 | crisv32-* | etraxfs*) basic_machine=crisv32-axis @@ -588,7 +940,7 @@ case $basic_machine in ;; crx) basic_machine=crx-unknown - os=-elf + os=${os:-elf} ;; da30 | da30-*) basic_machine=m68k-da30 @@ -598,50 +950,38 @@ case $basic_machine in ;; decsystem10* | dec10*) basic_machine=pdp10-dec - os=-tops10 + os=tops10 ;; decsystem20* | dec20*) basic_machine=pdp10-dec - os=-tops20 + os=tops20 ;; delta | 3300 | motorola-3300 | motorola-delta \ | 3300-motorola | delta-motorola) basic_machine=m68k-motorola ;; - delta88) - basic_machine=m88k-motorola - os=-sysv3 - ;; - dicos) - basic_machine=i686-pc - os=-dicos - ;; - djgpp) - basic_machine=i586-pc - os=-msdosdjgpp - ;; dpx20 | dpx20-*) basic_machine=rs6000-bull - os=-bosx + os=${os:-bosx} ;; - dpx2* | dpx2*-bull) + dpx2*) basic_machine=m68k-bull - os=-sysv3 + os=sysv3 ;; - ebmon29k) - basic_machine=a29k-amd - os=-ebmon + e500v[12]) + basic_machine=powerpc-unknown + os=$os"spe" ;; - elxsi) - basic_machine=elxsi-elxsi - os=-bsd + e500v[12]-*) + basic_machine=powerpc-`echo "$basic_machine" | sed 's/^[^-]*-//'` + os=$os"spe" ;; encore | umax | mmax) basic_machine=ns32k-encore ;; - es1800 | OSE68k | ose68k | ose | OSE) - basic_machine=m68k-ericsson - os=-ose + elxsi) + basic_machine=elxsi-elxsi + os=${os:-bsd} ;; fx2800) basic_machine=i860-alliant @@ -649,45 +989,13 @@ case $basic_machine in genix) basic_machine=ns32k-ns ;; - gmicro) - basic_machine=tron-gmicro - os=-sysv - ;; - go32) - basic_machine=i386-pc - os=-go32 - ;; h3050r* | hiux*) basic_machine=hppa1.1-hitachi - os=-hiuxwe2 - ;; - h8300hms) - basic_machine=h8300-hitachi - os=-hms - ;; - h8300xray) - basic_machine=h8300-hitachi - os=-xray - ;; - h8500hms) - basic_machine=h8500-hitachi - os=-hms - ;; - harris) - basic_machine=m88k-harris - os=-sysv3 + os=hiuxwe2 ;; hp300-*) basic_machine=m68k-hp ;; - hp300bsd) - basic_machine=m68k-hp - os=-bsd - ;; - hp300hpux) - basic_machine=m68k-hp - os=-hpux - ;; hp3k9[0-9][0-9] | hp9[0-9][0-9]) basic_machine=hppa1.0-hp ;; @@ -717,193 +1025,79 @@ case $basic_machine in hp9k8[0-9][0-9] | hp8[0-9][0-9]) basic_machine=hppa1.0-hp ;; - hppa-next) - os=-nextstep3 - ;; - hppaosf) - basic_machine=hppa1.1-hp - os=-osf - ;; - hppro) - basic_machine=hppa1.1-hp - os=-proelf - ;; - i370-ibm* | ibm*) - basic_machine=i370-ibm - ;; i*86v32) - basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` - os=-sysv32 + basic_machine=`echo "$1" | sed -e 's/86.*/86-pc/'` + os=sysv32 ;; i*86v4*) - basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` - os=-sysv4 + basic_machine=`echo "$1" | sed -e 's/86.*/86-pc/'` + os=sysv4 ;; i*86v) - basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` - os=-sysv + basic_machine=`echo "$1" | sed -e 's/86.*/86-pc/'` + os=sysv ;; i*86sol2) - basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` - os=-solaris2 - ;; - i386mach) - basic_machine=i386-mach - os=-mach + basic_machine=`echo "$1" | sed -e 's/86.*/86-pc/'` + os=solaris2 ;; - i386-vsta | vsta) - basic_machine=i386-unknown - os=-vsta + j90 | j90-cray) + basic_machine=j90-cray + os=${os:-unicos} ;; iris | iris4d) basic_machine=mips-sgi case $os in - -irix*) + irix*) ;; *) - os=-irix4 + os=irix4 ;; esac ;; - isi68 | isi) - basic_machine=m68k-isi - os=-sysv - ;; - m68knommu) - basic_machine=m68k-unknown - os=-linux + leon-*|leon[3-9]-*) + basic_machine=sparc-`echo "$basic_machine" | sed 's/-.*//'` ;; m68knommu-*) - basic_machine=m68k-`echo $basic_machine | sed 's/^[^-]*-//'` - os=-linux - ;; - m88k-omron*) - basic_machine=m88k-omron - ;; - magnum | m3230) - basic_machine=mips-mips - os=-sysv + basic_machine=m68k-`echo "$basic_machine" | sed 's/^[^-]*-//'` + os=linux ;; - merlin) - basic_machine=ns32k-utek - os=-sysv - ;; - microblaze) + microblaze*) basic_machine=microblaze-xilinx ;; - mingw64) - basic_machine=x86_64-pc - os=-mingw64 - ;; - mingw32) - basic_machine=i386-pc - os=-mingw32 - ;; - mingw32ce) - basic_machine=arm-unknown - os=-mingw32ce - ;; miniframe) basic_machine=m68000-convergent ;; - *mint | -mint[0-9]* | *MiNT | *MiNT[0-9]*) + *mint | mint[0-9]* | *MiNT | *MiNT[0-9]*) basic_machine=m68k-atari - os=-mint + os=mint ;; mips3*-*) - basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'` + basic_machine=`echo "$basic_machine" | sed -e 's/mips3/mips64/'` ;; mips3*) - basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`-unknown - ;; - monitor) - basic_machine=m68k-rom68k - os=-coff - ;; - morphos) - basic_machine=powerpc-unknown - os=-morphos - ;; - msdos) - basic_machine=i386-pc - os=-msdos + basic_machine=`echo "$basic_machine" | sed -e 's/mips3/mips64/'`-unknown ;; ms1-*) - basic_machine=`echo $basic_machine | sed -e 's/ms1-/mt-/'` - ;; - msys) - basic_machine=i386-pc - os=-msys - ;; - mvs) - basic_machine=i370-ibm - os=-mvs - ;; - nacl) - basic_machine=le32-unknown - os=-nacl - ;; - ncr3000) - basic_machine=i486-ncr - os=-sysv4 - ;; - netbsd386) - basic_machine=i386-unknown - os=-netbsd - ;; - netwinder) - basic_machine=armv4l-rebel - os=-linux - ;; - news | news700 | news800 | news900) - basic_machine=m68k-sony - os=-newsos - ;; - news1000) - basic_machine=m68030-sony - os=-newsos + basic_machine=`echo "$basic_machine" | sed -e 's/ms1-/mt-/'` ;; news-3600 | risc-news) basic_machine=mips-sony - os=-newsos - ;; - necv70) - basic_machine=v70-nec - os=-sysv + os=newsos ;; - next | m*-next ) + next | m*-next) basic_machine=m68k-next case $os in - -nextstep* ) + nextstep* ) ;; - -ns2*) - os=-nextstep2 + ns2*) + os=nextstep2 ;; *) - os=-nextstep3 + os=nextstep3 ;; esac ;; - nh3000) - basic_machine=m68k-harris - os=-cxux - ;; - nh[45]000) - basic_machine=m88k-harris - os=-cxux - ;; - nindy960) - basic_machine=i960-intel - os=-nindy - ;; - mon960) - basic_machine=i960-intel - os=-mon960 - ;; - nonstopux) - basic_machine=mips-compaq - os=-nonstopux - ;; np1) basic_machine=np1-gould ;; @@ -916,40 +1110,26 @@ case $basic_machine in nsr-tandem) basic_machine=nsr-tandem ;; + nsv-tandem) + basic_machine=nsv-tandem + ;; + nsx-tandem) + basic_machine=nsx-tandem + ;; op50n-* | op60c-*) basic_machine=hppa1.1-oki - os=-proelf + os=proelf ;; openrisc | openrisc-*) basic_machine=or32-unknown ;; - os400) - basic_machine=powerpc-ibm - os=-os400 - ;; - OSE68000 | ose68000) - basic_machine=m68000-ericsson - os=-ose - ;; - os68k) - basic_machine=m68k-none - os=-os68k - ;; pa-hitachi) basic_machine=hppa1.1-hitachi - os=-hiuxwe2 - ;; - paragon) - basic_machine=i860-intel - os=-osf - ;; - parisc) - basic_machine=hppa-unknown - os=-linux + os=hiuxwe2 ;; parisc-*) - basic_machine=hppa-`echo $basic_machine | sed 's/^[^-]*-//'` - os=-linux + basic_machine=hppa-`echo "$basic_machine" | sed 's/^[^-]*-//'` + os=linux ;; pbd) basic_machine=sparc-tti @@ -964,7 +1144,7 @@ case $basic_machine in basic_machine=i386-pc ;; pc98-*) - basic_machine=i386-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=i386-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; pentium | p5 | k5 | k6 | nexgen | viac3) basic_machine=i586-pc @@ -979,16 +1159,16 @@ case $basic_machine in basic_machine=i786-pc ;; pentium-* | p5-* | k5-* | k6-* | nexgen-* | viac3-*) - basic_machine=i586-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=i586-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; pentiumpro-* | p6-* | 6x86-* | athlon-*) - basic_machine=i686-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=i686-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; pentiumii-* | pentium2-* | pentiumiii-* | pentium3-*) - basic_machine=i686-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=i686-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; pentium4-*) - basic_machine=i786-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=i786-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; pn) basic_machine=pn-gould @@ -998,39 +1178,27 @@ case $basic_machine in ppc | ppcbe) basic_machine=powerpc-unknown ;; ppc-* | ppcbe-*) - basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=powerpc-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; - ppcle | powerpclittle | ppc-le | powerpc-little) + ppcle | powerpclittle) basic_machine=powerpcle-unknown ;; ppcle-* | powerpclittle-*) - basic_machine=powerpcle-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=powerpcle-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; ppc64) basic_machine=powerpc64-unknown ;; - ppc64-*) basic_machine=powerpc64-`echo $basic_machine | sed 's/^[^-]*-//'` + ppc64-*) basic_machine=powerpc64-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; - ppc64le | powerpc64little | ppc64-le | powerpc64-little) + ppc64le | powerpc64little) basic_machine=powerpc64le-unknown ;; ppc64le-* | powerpc64little-*) - basic_machine=powerpc64le-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=powerpc64le-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; ps2) basic_machine=i386-ibm ;; - pw32) - basic_machine=i586-unknown - os=-pw32 - ;; - rdos) - basic_machine=i386-pc - os=-rdos - ;; - rom68k) - basic_machine=m68k-rom68k - os=-coff - ;; rm[46]00) basic_machine=mips-siemens ;; @@ -1043,10 +1211,6 @@ case $basic_machine in s390x | s390x-*) basic_machine=s390x-ibm ;; - sa29200) - basic_machine=a29k-amd - os=-udi - ;; sb1) basic_machine=mipsisa64sb1-unknown ;; @@ -1055,105 +1219,32 @@ case $basic_machine in ;; sde) basic_machine=mipsisa32-sde - os=-elf - ;; - sei) - basic_machine=mips-sei - os=-seiux + os=${os:-elf} ;; sequent) basic_machine=i386-sequent ;; - sh) - basic_machine=sh-hitachi - os=-hms - ;; sh5el) basic_machine=sh5le-unknown ;; - sh64) - basic_machine=sh64-unknown + sh5el-*) + basic_machine=sh5le-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; - sparclite-wrs | simso-wrs) + simso-wrs) basic_machine=sparclite-wrs - os=-vxworks - ;; - sps7) - basic_machine=m68k-bull - os=-sysv2 + os=vxworks ;; spur) basic_machine=spur-unknown ;; - st2000) - basic_machine=m68k-tandem - ;; - stratus) - basic_machine=i860-stratus - os=-sysv4 - ;; strongarm-* | thumb-*) - basic_machine=arm-`echo $basic_machine | sed 's/^[^-]*-//'` - ;; - sun2) - basic_machine=m68000-sun - ;; - sun2os3) - basic_machine=m68000-sun - os=-sunos3 - ;; - sun2os4) - basic_machine=m68000-sun - os=-sunos4 + basic_machine=arm-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; - sun3os3) - basic_machine=m68k-sun - os=-sunos3 - ;; - sun3os4) - basic_machine=m68k-sun - os=-sunos4 - ;; - sun4os3) - basic_machine=sparc-sun - os=-sunos3 - ;; - sun4os4) - basic_machine=sparc-sun - os=-sunos4 - ;; - sun4sol2) - basic_machine=sparc-sun - os=-solaris2 - ;; - sun3 | sun3-*) - basic_machine=m68k-sun - ;; - sun4) - basic_machine=sparc-sun - ;; - sun386 | sun386i | roadrunner) - basic_machine=i386-sun - ;; - sv1) - basic_machine=sv1-cray - os=-unicos - ;; - symmetry) - basic_machine=i386-sequent - os=-dynix - ;; - t3e) - basic_machine=alphaev5-cray - os=-unicos - ;; - t90) - basic_machine=t90-cray - os=-unicos + tile*-*) ;; tile*) basic_machine=$basic_machine-unknown - os=-linux-gnu + os=${os:-linux-gnu} ;; tx39) basic_machine=mipstx39-unknown @@ -1161,146 +1252,32 @@ case $basic_machine in tx39el) basic_machine=mipstx39el-unknown ;; - toad1) - basic_machine=pdp10-xkl - os=-tops20 - ;; tower | tower-32) basic_machine=m68k-ncr ;; - tpf) - basic_machine=s390x-ibm - os=-tpf - ;; - udi29k) - basic_machine=a29k-amd - os=-udi - ;; - ultra3) - basic_machine=a29k-nyu - os=-sym1 - ;; - v810 | necv810) - basic_machine=v810-nec - os=-none - ;; - vaxv) - basic_machine=vax-dec - os=-sysv - ;; - vms) - basic_machine=vax-dec - os=-vms - ;; vpp*|vx|vx-*) basic_machine=f301-fujitsu ;; - vxworks960) - basic_machine=i960-wrs - os=-vxworks - ;; - vxworks68) - basic_machine=m68k-wrs - os=-vxworks - ;; - vxworks29k) - basic_machine=a29k-wrs - os=-vxworks - ;; w65*) basic_machine=w65-wdc - os=-none + os=none ;; w89k-*) basic_machine=hppa1.1-winbond - os=-proelf - ;; - xbox) - basic_machine=i686-pc - os=-mingw32 + os=proelf ;; - xps | xps100) - basic_machine=xps100-honeywell + x64) + basic_machine=x86_64-pc ;; xscale-* | xscalee[bl]-*) - basic_machine=`echo $basic_machine | sed 's/^xscale/arm/'` - ;; - ymp) - basic_machine=ymp-cray - os=-unicos - ;; - z8k-*-coff) - basic_machine=z8k-unknown - os=-sim - ;; - z80-*-coff) - basic_machine=z80-unknown - os=-sim + basic_machine=`echo "$basic_machine" | sed 's/^xscale/arm/'` ;; none) basic_machine=none-none - os=-none ;; -# Here we handle the default manufacturer of certain CPU types. It is in -# some cases the only manufacturer, in others, it is the most popular. - w89k) - basic_machine=hppa1.1-winbond - ;; - op50n) - basic_machine=hppa1.1-oki - ;; - op60c) - basic_machine=hppa1.1-oki - ;; - romp) - basic_machine=romp-ibm - ;; - mmix) - basic_machine=mmix-knuth - ;; - rs6000) - basic_machine=rs6000-ibm - ;; - vax) - basic_machine=vax-dec - ;; - pdp10) - # there are many clones, so DEC is not a safe bet - basic_machine=pdp10-unknown - ;; - pdp11) - basic_machine=pdp11-dec - ;; - we32k) - basic_machine=we32k-att - ;; - sh[1234] | sh[24]a | sh[24]aeb | sh[34]eb | sh[1234]le | sh[23]ele) - basic_machine=sh-unknown - ;; - sparc | sparcv8 | sparcv9 | sparcv9b | sparcv9v) - basic_machine=sparc-sun - ;; - cydra) - basic_machine=cydra-cydrome - ;; - orion) - basic_machine=orion-highlevel - ;; - orion105) - basic_machine=clipper-highlevel - ;; - mac | mpw | mac-mpw) - basic_machine=m68k-apple - ;; - pmac | pmac-mpw) - basic_machine=powerpc-apple - ;; - *-unknown) - # Make sure to match an already-canonicalized machine name. - ;; *) - echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2 + echo Invalid configuration \`"$1"\': machine \`"$basic_machine"\' not recognized 1>&2 exit 1 ;; esac @@ -1308,10 +1285,10 @@ esac # Here we canonicalize certain aliases for manufacturers. case $basic_machine in *-digital*) - basic_machine=`echo $basic_machine | sed 's/digital.*/dec/'` + basic_machine=`echo "$basic_machine" | sed 's/digital.*/dec/'` ;; *-commodore*) - basic_machine=`echo $basic_machine | sed 's/commodore.*/cbm/'` + basic_machine=`echo "$basic_machine" | sed 's/commodore.*/cbm/'` ;; *) ;; @@ -1319,200 +1296,246 @@ esac # Decode manufacturer-specific aliases for certain operating systems. -if [ x"$os" != x"" ] +if [ x$os != x ] then case $os in - # First match some system type aliases - # that might get confused with valid system types. - # -solaris* is a basic system type, with this one exception. - -auroraux) - os=-auroraux + # First match some system type aliases that might get confused + # with valid system types. + # solaris* is a basic system type, with this one exception. + auroraux) + os=auroraux ;; - -solaris1 | -solaris1.*) - os=`echo $os | sed -e 's|solaris1|sunos4|'` + bluegene*) + os=cnk ;; - -solaris) - os=-solaris2 + solaris1 | solaris1.*) + os=`echo $os | sed -e 's|solaris1|sunos4|'` ;; - -svr4*) - os=-sysv4 + solaris) + os=solaris2 ;; - -unixware*) - os=-sysv4.2uw + unixware*) + os=sysv4.2uw ;; - -gnu/linux*) + gnu/linux*) os=`echo $os | sed -e 's|gnu/linux|linux-gnu|'` ;; - # First accept the basic system types. + # es1800 is here to avoid being matched by es* (a different OS) + es1800*) + os=ose + ;; + # Some version numbers need modification + chorusos*) + os=chorusos + ;; + isc) + os=isc2.2 + ;; + sco6) + os=sco5v6 + ;; + sco5) + os=sco3.2v5 + ;; + sco4) + os=sco3.2v4 + ;; + sco3.2.[4-9]*) + os=`echo $os | sed -e 's/sco3.2./sco3.2v/'` + ;; + sco3.2v[4-9]* | sco5v6*) + # Don't forget version if it is 3.2v4 or newer. + ;; + scout) + # Don't match below + ;; + sco*) + os=sco3.2v2 + ;; + psos*) + os=psos + ;; + # Now accept the basic system types. # The portable systems comes first. - # Each alternative MUST END IN A *, to match a version number. - # -sysv* is not here because it comes later, after sysvr4. - -gnu* | -bsd* | -mach* | -minix* | -genix* | -ultrix* | -irix* \ - | -*vms* | -sco* | -esix* | -isc* | -aix* | -cnk* | -sunos | -sunos[34]*\ - | -hpux* | -unos* | -osf* | -luna* | -dgux* | -auroraux* | -solaris* \ - | -sym* | -kopensolaris* \ - | -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \ - | -aos* | -aros* \ - | -nindy* | -vxsim* | -vxworks* | -ebmon* | -hms* | -mvs* \ - | -clix* | -riscos* | -uniplus* | -iris* | -rtu* | -xenix* \ - | -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* \ - | -bitrig* | -openbsd* | -solidbsd* \ - | -ekkobsd* | -kfreebsd* | -freebsd* | -riscix* | -lynxos* \ - | -bosx* | -nextstep* | -cxux* | -aout* | -elf* | -oabi* \ - | -ptx* | -coff* | -ecoff* | -winnt* | -domain* | -vsta* \ - | -udi* | -eabi* | -lites* | -ieee* | -go32* | -aux* \ - | -chorusos* | -chorusrdb* | -cegcc* \ - | -cygwin* | -msys* | -pe* | -psos* | -moss* | -proelf* | -rtems* \ - | -mingw32* | -mingw64* | -linux-gnu* | -linux-android* \ - | -linux-newlib* | -linux-musl* | -linux-uclibc* \ - | -uxpv* | -beos* | -mpeix* | -udk* \ - | -interix* | -uwin* | -mks* | -rhapsody* | -darwin* | -opened* \ - | -openstep* | -oskit* | -conix* | -pw32* | -nonstopux* \ - | -storm-chaos* | -tops10* | -tenex* | -tops20* | -its* \ - | -os2* | -vos* | -palmos* | -uclinux* | -nucleus* \ - | -morphos* | -superux* | -rtmk* | -rtmk-nova* | -windiss* \ - | -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly* \ - | -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es*) + # Each alternative MUST end in a * to match a version number. + # sysv* is not here because it comes later, after sysvr4. + gnu* | bsd* | mach* | minix* | genix* | ultrix* | irix* \ + | *vms* | esix* | aix* | cnk* | sunos | sunos[34]*\ + | hpux* | unos* | osf* | luna* | dgux* | auroraux* | solaris* \ + | sym* | kopensolaris* | plan9* \ + | amigaos* | amigados* | msdos* | newsos* | unicos* | aof* \ + | aos* | aros* | cloudabi* | sortix* \ + | nindy* | vxsim* | vxworks* | ebmon* | hms* | mvs* \ + | clix* | riscos* | uniplus* | iris* | isc* | rtu* | xenix* \ + | knetbsd* | mirbsd* | netbsd* \ + | bitrig* | openbsd* | solidbsd* | libertybsd* \ + | ekkobsd* | kfreebsd* | freebsd* | riscix* | lynxos* \ + | bosx* | nextstep* | cxux* | aout* | elf* | oabi* \ + | ptx* | coff* | ecoff* | winnt* | domain* | vsta* \ + | udi* | eabi* | lites* | ieee* | go32* | aux* | hcos* \ + | chorusrdb* | cegcc* | glidix* \ + | cygwin* | msys* | pe* | moss* | proelf* | rtems* \ + | midipix* | mingw32* | mingw64* | linux-gnu* | linux-android* \ + | linux-newlib* | linux-musl* | linux-uclibc* \ + | uxpv* | beos* | mpeix* | udk* | moxiebox* \ + | interix* | uwin* | mks* | rhapsody* | darwin* \ + | openstep* | oskit* | conix* | pw32* | nonstopux* \ + | storm-chaos* | tops10* | tenex* | tops20* | its* \ + | os2* | vos* | palmos* | uclinux* | nucleus* \ + | morphos* | superux* | rtmk* | windiss* \ + | powermax* | dnix* | nx6 | nx7 | sei* | dragonfly* \ + | skyos* | haiku* | rdos* | toppers* | drops* | es* \ + | onefs* | tirtos* | phoenix* | fuchsia* | redox* | bme* \ + | midnightbsd*) # Remember, each alternative MUST END IN *, to match a version number. ;; - -qnx*) + qnx*) case $basic_machine in x86-* | i*86-*) ;; *) - os=-nto$os + os=nto-$os ;; esac ;; - -nto-qnx*) + hiux*) + os=hiuxwe2 ;; - -nto*) - os=`echo $os | sed -e 's|nto|nto-qnx|'` + nto-qnx*) ;; - -sim | -es1800* | -hms* | -xray | -os68k* | -none* | -v88r* \ - | -windows* | -osx | -abug | -netware* | -os9* | -beos* | -haiku* \ - | -macos* | -mpw* | -magic* | -mmixware* | -mon960* | -lnews*) + nto*) + os=`echo $os | sed -e 's|nto|nto-qnx|'` ;; - -mac*) - os=`echo $os | sed -e 's|mac|macos|'` + sim | xray | os68k* | v88r* \ + | windows* | osx | abug | netware* | os9* \ + | macos* | mpw* | magic* | mmixware* | mon960* | lnews*) ;; - -linux-dietlibc) - os=-linux-dietlibc + linux-dietlibc) + os=linux-dietlibc ;; - -linux*) + linux*) os=`echo $os | sed -e 's|linux|linux-gnu|'` ;; - -sunos5*) - os=`echo $os | sed -e 's|sunos5|solaris2|'` + lynx*178) + os=lynxos178 ;; - -sunos6*) - os=`echo $os | sed -e 's|sunos6|solaris3|'` + lynx*5) + os=lynxos5 ;; - -opened*) - os=-openedition + lynx*) + os=lynxos ;; - -os400*) - os=-os400 + mac*) + os=`echo "$os" | sed -e 's|mac|macos|'` ;; - -wince*) - os=-wince + opened*) + os=openedition ;; - -osfrose*) - os=-osfrose + os400*) + os=os400 ;; - -osf*) - os=-osf + sunos5*) + os=`echo "$os" | sed -e 's|sunos5|solaris2|'` ;; - -utek*) - os=-bsd + sunos6*) + os=`echo "$os" | sed -e 's|sunos6|solaris3|'` ;; - -dynix*) - os=-bsd + wince*) + os=wince ;; - -acis*) - os=-aos + utek*) + os=bsd ;; - -atheos*) - os=-atheos + dynix*) + os=bsd ;; - -syllable*) - os=-syllable + acis*) + os=aos ;; - -386bsd) - os=-bsd + atheos*) + os=atheos ;; - -ctix* | -uts*) - os=-sysv + syllable*) + os=syllable ;; - -nova*) - os=-rtmk-nova + 386bsd) + os=bsd + ;; + ctix* | uts*) + os=sysv + ;; + nova*) + os=rtmk-nova ;; - -ns2 ) - os=-nextstep2 + ns2) + os=nextstep2 ;; - -nsk*) - os=-nsk + nsk*) + os=nsk ;; # Preserve the version number of sinix5. - -sinix5.*) + sinix5.*) os=`echo $os | sed -e 's|sinix|sysv|'` ;; - -sinix*) - os=-sysv4 + sinix*) + os=sysv4 ;; - -tpf*) - os=-tpf + tpf*) + os=tpf ;; - -triton*) - os=-sysv3 + triton*) + os=sysv3 ;; - -oss*) - os=-sysv3 + oss*) + os=sysv3 ;; - -svr4) - os=-sysv4 + svr4*) + os=sysv4 ;; - -svr3) - os=-sysv3 + svr3) + os=sysv3 ;; - -sysvr4) - os=-sysv4 + sysvr4) + os=sysv4 ;; - # This must come after -sysvr4. - -sysv*) + # This must come after sysvr4. + sysv*) ;; - -ose*) - os=-ose + ose*) + os=ose ;; - -es1800*) - os=-ose + *mint | mint[0-9]* | *MiNT | MiNT[0-9]*) + os=mint ;; - -xenix) - os=-xenix + zvmoe) + os=zvmoe ;; - -*mint | -mint[0-9]* | -*MiNT | -MiNT[0-9]*) - os=-mint + dicos*) + os=dicos ;; - -aros*) - os=-aros - ;; - -kaos*) - os=-kaos + pikeos*) + # Until real need of OS specific support for + # particular features comes up, bare metal + # configurations are quite functional. + case $basic_machine in + arm*) + os=eabi + ;; + *) + os=elf + ;; + esac ;; - -zvmoe) - os=-zvmoe + nacl*) ;; - -dicos*) - os=-dicos + ios) ;; - -nacl*) + none) ;; - -none) + *-eabi) ;; *) - # Get rid of the `-' at the beginning of $os. - os=`echo $os | sed 's/[^-]*-//'` - echo Invalid configuration \`$1\': system \`$os\' not recognized 1>&2 + echo Invalid configuration \`"$1"\': system \`"$os"\' not recognized 1>&2 exit 1 ;; esac @@ -1530,173 +1553,179 @@ else case $basic_machine in score-*) - os=-elf + os=elf ;; spu-*) - os=-elf + os=elf ;; *-acorn) - os=-riscix1.2 + os=riscix1.2 ;; arm*-rebel) - os=-linux + os=linux ;; arm*-semi) - os=-aout + os=aout ;; c4x-* | tic4x-*) - os=-coff + os=coff + ;; + c8051-*) + os=elf + ;; + clipper-intergraph) + os=clix ;; hexagon-*) - os=-elf + os=elf ;; tic54x-*) - os=-coff + os=coff ;; tic55x-*) - os=-coff + os=coff ;; tic6x-*) - os=-coff + os=coff ;; # This must come before the *-dec entry. pdp10-*) - os=-tops20 + os=tops20 ;; pdp11-*) - os=-none + os=none ;; *-dec | vax-*) - os=-ultrix4.2 + os=ultrix4.2 ;; m68*-apollo) - os=-domain + os=domain ;; i386-sun) - os=-sunos4.0.2 + os=sunos4.0.2 ;; m68000-sun) - os=-sunos3 + os=sunos3 ;; m68*-cisco) - os=-aout + os=aout ;; mep-*) - os=-elf + os=elf ;; mips*-cisco) - os=-elf + os=elf ;; mips*-*) - os=-elf + os=elf ;; or32-*) - os=-coff + os=coff ;; *-tti) # must be before sparc entry or we get the wrong os. - os=-sysv3 + os=sysv3 ;; sparc-* | *-sun) - os=-sunos4.1.1 + os=sunos4.1.1 ;; - *-be) - os=-beos + pru-*) + os=elf ;; - *-haiku) - os=-haiku + *-be) + os=beos ;; *-ibm) - os=-aix + os=aix ;; *-knuth) - os=-mmixware + os=mmixware ;; *-wec) - os=-proelf + os=proelf ;; *-winbond) - os=-proelf + os=proelf ;; *-oki) - os=-proelf + os=proelf ;; *-hp) - os=-hpux + os=hpux ;; *-hitachi) - os=-hiux + os=hiux ;; i860-* | *-att | *-ncr | *-altos | *-motorola | *-convergent) - os=-sysv + os=sysv ;; *-cbm) - os=-amigaos + os=amigaos ;; *-dg) - os=-dgux + os=dgux ;; *-dolphin) - os=-sysv3 + os=sysv3 ;; m68k-ccur) - os=-rtu + os=rtu ;; m88k-omron*) - os=-luna + os=luna ;; - *-next ) - os=-nextstep + *-next) + os=nextstep ;; *-sequent) - os=-ptx + os=ptx ;; *-crds) - os=-unos + os=unos ;; *-ns) - os=-genix + os=genix ;; i370-*) - os=-mvs - ;; - *-next) - os=-nextstep3 + os=mvs ;; *-gould) - os=-sysv + os=sysv ;; *-highlevel) - os=-bsd + os=bsd ;; *-encore) - os=-bsd + os=bsd ;; *-sgi) - os=-irix + os=irix ;; *-siemens) - os=-sysv4 + os=sysv4 ;; *-masscomp) - os=-rtu + os=rtu ;; f30[01]-fujitsu | f700-fujitsu) - os=-uxpv + os=uxpv ;; *-rom68k) - os=-coff + os=coff ;; *-*bug) - os=-coff + os=coff ;; *-apple) - os=-macos + os=macos ;; *-atari*) - os=-mint + os=mint + ;; + *-wrs) + os=vxworks ;; *) - os=-none + os=none ;; esac fi @@ -1707,79 +1736,82 @@ vendor=unknown case $basic_machine in *-unknown) case $os in - -riscix*) + riscix*) vendor=acorn ;; - -sunos*) + sunos*) vendor=sun ;; - -cnk*|-aix*) + cnk*|-aix*) vendor=ibm ;; - -beos*) + beos*) vendor=be ;; - -hpux*) + hpux*) vendor=hp ;; - -mpeix*) + mpeix*) vendor=hp ;; - -hiux*) + hiux*) vendor=hitachi ;; - -unos*) + unos*) vendor=crds ;; - -dgux*) + dgux*) vendor=dg ;; - -luna*) + luna*) vendor=omron ;; - -genix*) + genix*) vendor=ns ;; - -mvs* | -opened*) + clix*) + vendor=intergraph + ;; + mvs* | opened*) vendor=ibm ;; - -os400*) + os400*) vendor=ibm ;; - -ptx*) + ptx*) vendor=sequent ;; - -tpf*) + tpf*) vendor=ibm ;; - -vxsim* | -vxworks* | -windiss*) + vxsim* | vxworks* | windiss*) vendor=wrs ;; - -aux*) + aux*) vendor=apple ;; - -hms*) + hms*) vendor=hitachi ;; - -mpw* | -macos*) + mpw* | macos*) vendor=apple ;; - -*mint | -mint[0-9]* | -*MiNT | -MiNT[0-9]*) + *mint | mint[0-9]* | *MiNT | MiNT[0-9]*) vendor=atari ;; - -vos*) + vos*) vendor=stratus ;; esac - basic_machine=`echo $basic_machine | sed "s/unknown/$vendor/"` + basic_machine=`echo "$basic_machine" | sed "s/unknown/$vendor/"` ;; esac -echo $basic_machine$os +echo "$basic_machine-$os" exit # Local variables: -# eval: (add-hook 'write-file-hooks 'time-stamp) +# eval: (add-hook 'before-save-hook 'time-stamp) # time-stamp-start: "timestamp='" # time-stamp-format: "%:y-%02m-%02d" # time-stamp-end: "'" diff --git a/configure.ac b/configure.ac index c41a83c97646..f5b1614f19f1 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT(nix, m4_esyscmd([bash -c "echo -n $(cat ./version)$VERSION_SUFFIX"])) +AC_INIT(nix, m4_esyscmd([bash -c "echo -n $(cat ./.version)$VERSION_SUFFIX"])) AC_CONFIG_SRCDIR(README.md) AC_CONFIG_AUX_DIR(config) @@ -62,8 +62,9 @@ CXXFLAGS= AC_PROG_CC AC_PROG_CXX AC_PROG_CPP -AX_CXX_COMPILE_STDCXX_14 +AX_CXX_COMPILE_STDCXX_17 +AC_CHECK_TOOL([AR], [ar]) # Use 64-bit file system calls so that we can support files > 2 GiB. AC_SYS_LARGEFILE @@ -127,8 +128,6 @@ NEED_PROG(bzip2, bzip2) NEED_PROG(gzip, gzip) NEED_PROG(xz, xz) AC_PATH_PROG(dot, dot) -AC_PATH_PROG(pv, pv, pv) -AC_PATH_PROGS(brotli, brotli bro, bro) AC_PATH_PROG(lsof, lsof, lsof) @@ -152,9 +151,9 @@ PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"] # Look for libbz2, a required dependency. AC_CHECK_LIB([bz2], [BZ2_bzWriteOpen], [true], - [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See http://www.bzip.org/.])]) + [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])]) AC_CHECK_HEADERS([bzlib.h], [true], - [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See http://www.bzip.org/.])]) + [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])]) # Look for SQLite, a required dependency. @@ -164,6 +163,8 @@ PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CX # Look for libcurl, a required dependency. PKG_CHECK_MODULES([LIBCURL], [libcurl], [CXXFLAGS="$LIBCURL_CFLAGS $CXXFLAGS"]) +# Look for editline, a required dependency. +PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLAGS"]) # Look for libsodium, an optional dependency. PKG_CHECK_MODULES([SODIUM], [libsodium], @@ -179,12 +180,9 @@ AC_CHECK_LIB([lzma], [lzma_stream_encoder_mt], [AC_DEFINE([HAVE_LZMA_MT], [1], [xz multithreaded compression support])]) -# Look for libbrotli{enc,dec}, optional dependencies -PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], - [AC_DEFINE([HAVE_BROTLI], [1], [Whether to use libbrotli.]) - CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"] - have_brotli=1], [have_brotli=]) -AC_SUBST(HAVE_BROTLI, [$have_brotli]) +# Look for libbrotli{enc,dec}. +PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"]) + # Look for libseccomp, required for Linux sandboxing. if test "$sys_name" = linux; then @@ -232,12 +230,6 @@ if test "$gc" = yes; then fi -AC_ARG_ENABLE(init-state, AC_HELP_STRING([--disable-init-state], - [do not initialise DB etc. in `make install']), - init_state=$enableval, init_state=yes) -#AM_CONDITIONAL(INIT_STATE, test "$init_state" = "yes") - - # documentation generation switch AC_ARG_ENABLE(doc-gen, AC_HELP_STRING([--disable-doc-gen], [disable documentation generation]), @@ -260,11 +252,6 @@ if test "$(uname)" = "Darwin"; then fi -# Figure out the extension of dynamic libraries. -eval dynlib_suffix=$shrext_cmds -AC_SUBST(dynlib_suffix) - - # Do we have GNU tar? AC_MSG_CHECKING([if you have a recent GNU tar]) if $tar --version 2> /dev/null | grep -q GNU && tar cvf /dev/null --warning=no-timestamp ./config.log > /dev/null; then @@ -281,6 +268,15 @@ AC_ARG_WITH(sandbox-shell, AC_HELP_STRING([--with-sandbox-shell=PATH], sandbox_shell=$withval) AC_SUBST(sandbox_shell) +AC_ARG_ENABLE(shared, AC_HELP_STRING([--enable-shared], + [Build shared libraries for Nix [default=yes]]), + shared=$enableval, shared=yes) +if test "$shared" = yes; then + AC_SUBST(BUILD_SHARED_LIBS, 1, [Whether to build shared libraries.]) +else + AC_SUBST(BUILD_SHARED_LIBS, 0, [Whether to build shared libraries.]) +fi + # Expand all variables in config.status. test "$prefix" = NONE && prefix=$ac_default_prefix diff --git a/corepkgs/fetchurl.nix b/corepkgs/fetchurl.nix index 0ce1bab112f3..a84777f57448 100644 --- a/corepkgs/fetchurl.nix +++ b/corepkgs/fetchurl.nix @@ -1,10 +1,14 @@ { system ? "" # obsolete , url +, hash ? "" # an SRI ash + +# Legacy hash specification , md5 ? "", sha1 ? "", sha256 ? "", sha512 ? "" , outputHash ? - if sha512 != "" then sha512 else if sha1 != "" then sha1 else if md5 != "" then md5 else sha256 + if hash != "" then hash else if sha512 != "" then sha512 else if sha1 != "" then sha1 else if md5 != "" then md5 else sha256 , outputHashAlgo ? - if sha512 != "" then "sha512" else if sha1 != "" then "sha1" else if md5 != "" then "md5" else "sha256" + if hash != "" then "" else if sha512 != "" then "sha512" else if sha1 != "" then "sha1" else if md5 != "" then "md5" else "sha256" + , executable ? false , unpack ? false , name ? baseNameOf (toString url) diff --git a/doc/manual/advanced-topics/advanced-topics.xml b/doc/manual/advanced-topics/advanced-topics.xml index 338aa6f3a238..c304367aaf8a 100644 --- a/doc/manual/advanced-topics/advanced-topics.xml +++ b/doc/manual/advanced-topics/advanced-topics.xml @@ -1,10 +1,12 @@ <part xmlns="http://docbook.org/ns/docbook" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xi="http://www.w3.org/2001/XInclude" + xml:id="part-advanced-topics" version="5.0"> <title>Advanced Topics</title> <xi:include href="distributed-builds.xml" /> +<xi:include href="diff-hook.xml" /> </part> diff --git a/doc/manual/advanced-topics/diff-hook.xml b/doc/manual/advanced-topics/diff-hook.xml new file mode 100644 index 000000000000..fb4bf819f94b --- /dev/null +++ b/doc/manual/advanced-topics/diff-hook.xml @@ -0,0 +1,205 @@ +<chapter xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + xml:id="chap-diff-hook" + version="5.0" + > + +<title>Verifying Build Reproducibility with <option linkend="conf-diff-hook">diff-hook</option></title> + +<subtitle>Check build reproducibility by running builds multiple times +and comparing their results.</subtitle> + +<para>Specify a program with Nix's <xref linkend="conf-diff-hook" /> to +compare build results when two builds produce different results. Note: +this hook is only executed if the results are not the same, this hook +is not used for determining if the results are the same.</para> + +<para>For purposes of demonstration, we'll use the following Nix file, +<filename>deterministic.nix</filename> for testing:</para> + +<programlisting> +let + inherit (import <nixpkgs> {}) runCommand; +in { + stable = runCommand "stable" {} '' + touch $out + ''; + + unstable = runCommand "unstable" {} '' + echo $RANDOM > $out + ''; +} +</programlisting> + +<para>Additionally, <filename>nix.conf</filename> contains: + +<programlisting> +diff-hook = /etc/nix/my-diff-hook +run-diff-hook = true +</programlisting> + +where <filename>/etc/nix/my-diff-hook</filename> is an executable +file containing: + +<programlisting> +#!/bin/sh +exec >&2 +echo "For derivation $3:" +/run/current-system/sw/bin/diff -r "$1" "$2" +</programlisting> + +</para> + +<para>The diff hook is executed by the same user and group who ran the +build. However, the diff hook does not have write access to the store +path just built.</para> + +<section> + <title> + Spot-Checking Build Determinism + </title> + + <para> + Verify a path which already exists in the Nix store by passing + <option>--check</option> to the build command. + </para> + + <para>If the build passes and is deterministic, Nix will exit with a + status code of 0:</para> + + <screen> +$ nix-build ./deterministic.nix -A stable +these derivations will be built: + /nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv +building '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'... +/nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable + +$ nix-build ./deterministic.nix -A stable --check +checking outputs of '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'... +/nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable +</screen> + + <para>If the build is not deterministic, Nix will exit with a status + code of 1:</para> + + <screen> +$ nix-build ./deterministic.nix -A unstable +these derivations will be built: + /nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv +building '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'... +/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable + +$ nix-build ./deterministic.nix -A unstable --check +checking outputs of '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'... +error: derivation '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv' may not be deterministic: output '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable' differs +</screen> + +<para>In the Nix daemon's log, we will now see: +<screen> +For derivation /nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv: +1c1 +< 8108 +--- +> 30204 +</screen> +</para> + + <para>Using <option>--check</option> with <option>--keep-failed</option> + will cause Nix to keep the second build's output in a special, + <literal>.check</literal> path:</para> + + <screen> +$ nix-build ./deterministic.nix -A unstable --check --keep-failed +checking outputs of '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'... +note: keeping build directory '/tmp/nix-build-unstable.drv-0' +error: derivation '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv' may not be deterministic: output '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable' differs from '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable.check' +</screen> + + <para>In particular, notice the + <literal>/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable.check</literal> + output. Nix has copied the build results to that directory where you + can examine it.</para> + + <note xml:id="check-dirs-are-unregistered"> + <title><literal>.check</literal> paths are not registered store paths</title> + + <para>Check paths are not protected against garbage collection, + and this path will be deleted on the next garbage collection.</para> + + <para>The path is guaranteed to be alive for the duration of + <xref linkend="conf-diff-hook" />'s execution, but may be deleted + any time after.</para> + + <para>If the comparison is performed as part of automated tooling, + please use the diff-hook or author your tooling to handle the case + where the build was not deterministic and also a check path does + not exist.</para> + </note> + + <para> + <option>--check</option> is only usable if the derivation has + been built on the system already. If the derivation has not been + built Nix will fail with the error: + <screen> +error: some outputs of '/nix/store/hzi1h60z2qf0nb85iwnpvrai3j2w7rr6-unstable.drv' are not valid, so checking is not possible +</screen> + + Run the build without <option>--check</option>, and then try with + <option>--check</option> again. + </para> +</section> + +<section> + <title> + Automatic and Optionally Enforced Determinism Verification + </title> + + <para> + Automatically verify every build at build time by executing the + build multiple times. + </para> + + <para> + Setting <xref linkend="conf-repeat" /> and + <xref linkend="conf-enforce-determinism" /> in your + <filename>nix.conf</filename> permits the automated verification + of every build Nix performs. + </para> + + <para> + The following configuration will run each build three times, and + will require the build to be deterministic: + + <programlisting> +enforce-determinism = true +repeat = 2 +</programlisting> + </para> + + <para> + Setting <xref linkend="conf-enforce-determinism" /> to false as in + the following configuration will run the build multiple times, + execute the build hook, but will allow the build to succeed even + if it does not build reproducibly: + + <programlisting> +enforce-determinism = false +repeat = 1 +</programlisting> + </para> + + <para> + An example output of this configuration: + <screen> +$ nix-build ./test.nix -A unstable +these derivations will be built: + /nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv +building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 1/2)... +building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 2/2)... +output '/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable' of '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' differs from '/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable.check' from previous round +/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable +</screen> + </para> +</section> +</chapter> diff --git a/doc/manual/advanced-topics/distributed-builds.xml b/doc/manual/advanced-topics/distributed-builds.xml index 20fd6a0cfb0d..9ac4a92cd5b1 100644 --- a/doc/manual/advanced-topics/distributed-builds.xml +++ b/doc/manual/advanced-topics/distributed-builds.xml @@ -81,6 +81,7 @@ or a newline, e.g. <para>Each machine specification consists of the following elements, separated by spaces. Only the first element is required. +To leave a field at its default, set it to <literal>-</literal>. <orderedlist> @@ -179,4 +180,11 @@ builders = @/etc/nix/machines causes the list of machines in <filename>/etc/nix/machines</filename> to be included. (This is the default.)</para> +<para>If you want the builders to use caches, you likely want to set +the option <link linkend='conf-builders-use-substitutes'><literal>builders-use-substitutes</literal></link> +in your local <filename>nix.conf</filename>.</para> + +<para>To build only on remote builders and disable building on the local machine, +you can use the option <option>--max-jobs 0</option>.</para> + </chapter> diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index 431c0e6d3570..24fbf28cff25 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -1,7 +1,9 @@ +<?xml version="1.0" encoding="utf-8"?> <refentry xmlns="http://docbook.org/ns/docbook" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xi="http://www.w3.org/2001/XInclude" - xml:id="sec-conf-file"> + xml:id="sec-conf-file" + version="5"> <refmeta> <refentrytitle>nix.conf</refentrytitle> @@ -135,7 +137,6 @@ false</literal>.</para> </varlistentry> - <varlistentry xml:id="conf-builders"> <term><literal>builders</literal></term> <listitem> @@ -159,7 +160,6 @@ false</literal>.</para> </varlistentry> - <varlistentry xml:id="conf-build-users-group"><term><literal>build-users-group</literal></term> <listitem><para>This options specifies the Unix group containing @@ -210,7 +210,6 @@ false</literal>.</para> </varlistentry> - <varlistentry xml:id="conf-connect-timeout"><term><literal>connect-timeout</literal></term> <listitem> @@ -243,6 +242,70 @@ false</literal>.</para> </varlistentry> + <varlistentry xml:id="conf-diff-hook"><term><literal>diff-hook</literal></term> + <listitem> + <para> + Absolute path to an executable capable of diffing build results. + The hook executes if <xref linkend="conf-run-diff-hook" /> is + true, and the output of a build is known to not be the same. + This program is not executed to determine if two results are the + same. + </para> + + <para> + The diff hook is executed by the same user and group who ran the + build. However, the diff hook does not have write access to the + store path just built. + </para> + + <para>The diff hook program receives three parameters:</para> + + <orderedlist> + <listitem> + <para> + A path to the previous build's results + </para> + </listitem> + + <listitem> + <para> + A path to the current build's results + </para> + </listitem> + + <listitem> + <para> + The path to the build's derivation + </para> + </listitem> + + <listitem> + <para> + The path to the build's scratch directory. This directory + will exist only if the build was run with + <option>--keep-failed</option>. + </para> + </listitem> + </orderedlist> + + <para> + The stderr and stdout output from the diff hook will not be + displayed to the user. Instead, it will print to the nix-daemon's + log. + </para> + + <para>When using the Nix daemon, <literal>diff-hook</literal> must + be set in the <filename>nix.conf</filename> configuration file, and + cannot be passed at the command line. + </para> + </listitem> + </varlistentry> + + <varlistentry xml:id="conf-enforce-determinism"> + <term><literal>enforce-determinism</literal></term> + + <listitem><para>See <xref linkend="conf-repeat" />.</para></listitem> + </varlistentry> <varlistentry xml:id="conf-extra-sandbox-paths"> <term><literal>extra-sandbox-paths</literal></term> @@ -254,6 +317,25 @@ false</literal>.</para> </varlistentry> + <varlistentry xml:id="conf-extra-platforms"><term><literal>extra-platforms</literal></term> + + <listitem><para>Platforms other than the native one which + this machine is capable of building for. This can be useful for + supporting additional architectures on compatible machines: + i686-linux can be built on x86_64-linux machines (and the default + for this setting reflects this); armv7 is backwards-compatible with + armv6 and armv5tel; some aarch64 machines can also natively run + 32-bit ARM code; and qemu-user may be used to support non-native + platforms (though this may be slow and buggy). Most values for this + are not enabled by default because build systems will often + misdetect the target platform and generate incompatible code, so you + may wish to cross-check the results of using this option against + proper natively-built versions of your + derivations.</para></listitem> + + </varlistentry> + + <varlistentry xml:id="conf-extra-substituters"><term><literal>extra-substituters</literal></term> <listitem><para>Additional binary caches appended to those @@ -264,7 +346,6 @@ false</literal>.</para> </varlistentry> - <varlistentry xml:id="conf-fallback"><term><literal>fallback</literal></term> <listitem><para>If set to <literal>true</literal>, Nix will fall @@ -274,7 +355,6 @@ false</literal>.</para> </varlistentry> - <varlistentry xml:id="conf-fsync-metadata"><term><literal>fsync-metadata</literal></term> <listitem><para>If set to <literal>true</literal>, changes to the @@ -285,7 +365,6 @@ false</literal>.</para> </varlistentry> - <varlistentry xml:id="conf-hashed-mirrors"><term><literal>hashed-mirrors</literal></term> <listitem><para>A list of web servers used by @@ -348,10 +427,8 @@ builtins.fetchurl { options a store path was built), so by default this option is on. Turn it off to save a bit of disk space (or a lot if <literal>keep-outputs</literal> is also turned on).</para></listitem> - </varlistentry> - <varlistentry xml:id="conf-keep-env-derivations"><term><literal>keep-env-derivations</literal></term> <listitem><para>If <literal>false</literal> (default), derivations @@ -375,7 +452,6 @@ builtins.fetchurl { </varlistentry> - <varlistentry xml:id="conf-keep-outputs"><term><literal>keep-outputs</literal></term> <listitem><para>If <literal>true</literal>, the garbage collector @@ -389,10 +465,8 @@ builtins.fetchurl { only at build time (e.g., the C compiler, or source tarballs downloaded from the network). To prevent it from doing so, set this option to <literal>true</literal>.</para></listitem> - </varlistentry> - <varlistentry xml:id="conf-max-build-log-size"><term><literal>max-build-log-size</literal></term> <listitem> @@ -418,14 +492,15 @@ builtins.fetchurl { <listitem><para>This option defines the maximum number of jobs that Nix will try to build in parallel. The default is <literal>1</literal>. The special value <literal>auto</literal> - causes Nix to use the number of CPUs in your system. It can be + causes Nix to use the number of CPUs in your system. <literal>0</literal> + is useful when using remote builders to prevent any local builds (except for + <literal>preferLocalBuild</literal> derivation attribute which executes locally + regardless). It can be overridden using the <option linkend='opt-max-jobs'>--max-jobs</option> (<option>-j</option>) command line switch.</para></listitem> - </varlistentry> - <varlistentry xml:id="conf-max-silent-time"><term><literal>max-silent-time</literal></term> <listitem> @@ -505,7 +580,12 @@ password <replaceable>my-password</replaceable> For the exact syntax, see <link xlink:href="https://ec.haxx.se/usingcurl-netrc.html">the - <literal>curl</literal> documentation.</link></para></listitem> + <literal>curl</literal> documentation.</link></para> + + <note><para>This must be an absolute path, and <literal>~</literal> + is not resolved. For example, <filename>~/.netrc</filename> won't + resolve to your home directory's <filename>.netrc</filename>.</para></note> + </listitem> </varlistentry> @@ -576,19 +656,17 @@ password <replaceable>my-password</replaceable> </varlistentry> - <varlistentry xml:id="conf-repeat"><term><literal>repeat</literal></term> <listitem><para>How many times to repeat builds to check whether they are deterministic. The default value is 0. If the value is non-zero, every build is repeated the specified number of times. If the contents of any of the runs differs from the - previous ones, the build is rejected and the resulting store paths - are not registered as “valid” in Nix’s database.</para></listitem> - + previous ones and <xref linkend="conf-enforce-determinism" /> is + true, the build is rejected and the resulting store paths are not + registered as “valid” in Nix’s database.</para></listitem> </varlistentry> - <varlistentry xml:id="conf-require-sigs"><term><literal>require-sigs</literal></term> <listitem><para>If set to <literal>true</literal> (the default), @@ -617,6 +695,19 @@ password <replaceable>my-password</replaceable> </varlistentry> + <varlistentry xml:id="conf-run-diff-hook"><term><literal>run-diff-hook</literal></term> + <listitem> + <para> + If true, enable the execution of <xref linkend="conf-diff-hook" />. + </para> + + <para> + When using the Nix daemon, <literal>run-diff-hook</literal> must + be set in the <filename>nix.conf</filename> configuration file, + and cannot be passed at the command line. + </para> + </listitem> + </varlistentry> <varlistentry xml:id="conf-sandbox"><term><literal>sandbox</literal></term> @@ -646,13 +737,13 @@ password <replaceable>my-password</replaceable> <varname>__noChroot</varname> attribute set to <literal>true</literal> do not run in sandboxes.</para> - <para>The default is <literal>false</literal>.</para> + <para>The default is <literal>true</literal> on Linux and + <literal>false</literal> on all other platforms.</para> </listitem> </varlistentry> - <varlistentry xml:id="conf-sandbox-dev-shm-size"><term><literal>sandbox-dev-shm-size</literal></term> <listitem><para>This option determines the maximum size of the @@ -718,7 +809,6 @@ password <replaceable>my-password</replaceable> </varlistentry> - <varlistentry xml:id="conf-substituters"><term><literal>substituters</literal></term> <listitem><para>A list of URLs of substituters, separated by @@ -727,7 +817,6 @@ password <replaceable>my-password</replaceable> </varlistentry> - <varlistentry xml:id="conf-system"><term><literal>system</literal></term> <listitem><para>This option specifies the canonical Nix system @@ -749,6 +838,33 @@ password <replaceable>my-password</replaceable> </varlistentry> + <varlistentry xml:id="conf-system-features"><term><literal>system-features</literal></term> + + <listitem><para>A set of system “features” supported by this + machine, e.g. <literal>kvm</literal>. Derivations can express a + dependency on such features through the derivation attribute + <varname>requiredSystemFeatures</varname>. For example, the + attribute + +<programlisting> +requiredSystemFeatures = [ "kvm" ]; +</programlisting> + + ensures that the derivation can only be built on a machine with + the <literal>kvm</literal> feature.</para> + + <para>This setting by default includes <literal>kvm</literal> if + <filename>/dev/kvm</filename> is accessible, and the + pseudo-features <literal>nixos-test</literal>, + <literal>benchmark</literal> and <literal>big-parallel</literal> + that are used in Nixpkgs to route builds to specific + machines.</para> + + </listitem> + + </varlistentry> + + <varlistentry xml:id="conf-timeout"><term><literal>timeout</literal></term> <listitem> @@ -768,7 +884,6 @@ password <replaceable>my-password</replaceable> </varlistentry> - <varlistentry xml:id="conf-trusted-public-keys"><term><literal>trusted-public-keys</literal></term> <listitem><para>A whitespace-separated list of public keys. When @@ -779,7 +894,6 @@ password <replaceable>my-password</replaceable> </varlistentry> - <varlistentry xml:id="conf-trusted-substituters"><term><literal>trusted-substituters</literal></term> <listitem><para>A list of URLs of substituters, separated by @@ -792,7 +906,6 @@ password <replaceable>my-password</replaceable> </varlistentry> - <varlistentry xml:id="conf-trusted-users"><term><literal>trusted-users</literal></term> <listitem> @@ -818,8 +931,177 @@ password <replaceable>my-password</replaceable> </varlistentry> </variablelist> +</para> + +<refsection> + <title>Deprecated Settings</title> + +<para> + +<variablelist> + + <varlistentry xml:id="conf-binary-caches"> + <term><literal>binary-caches</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>binary-caches</literal> is now an alias to + <xref linkend="conf-substituters" />.</para></listitem> + </varlistentry> + <varlistentry xml:id="conf-binary-cache-public-keys"> + <term><literal>binary-cache-public-keys</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>binary-cache-public-keys</literal> is now an alias to + <xref linkend="conf-trusted-public-keys" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-build-compress-log"> + <term><literal>build-compress-log</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>build-compress-log</literal> is now an alias to + <xref linkend="conf-compress-build-log" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-build-cores"> + <term><literal>build-cores</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>build-cores</literal> is now an alias to + <xref linkend="conf-cores" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-build-extra-chroot-dirs"> + <term><literal>build-extra-chroot-dirs</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>build-extra-chroot-dirs</literal> is now an alias to + <xref linkend="conf-extra-sandbox-paths" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-build-extra-sandbox-paths"> + <term><literal>build-extra-sandbox-paths</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>build-extra-sandbox-paths</literal> is now an alias to + <xref linkend="conf-extra-sandbox-paths" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-build-fallback"> + <term><literal>build-fallback</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>build-fallback</literal> is now an alias to + <xref linkend="conf-fallback" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-build-max-jobs"> + <term><literal>build-max-jobs</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>build-max-jobs</literal> is now an alias to + <xref linkend="conf-max-jobs" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-build-max-log-size"> + <term><literal>build-max-log-size</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>build-max-log-size</literal> is now an alias to + <xref linkend="conf-max-build-log-size" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-build-max-silent-time"> + <term><literal>build-max-silent-time</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>build-max-silent-time</literal> is now an alias to + <xref linkend="conf-max-silent-time" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-build-repeat"> + <term><literal>build-repeat</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>build-repeat</literal> is now an alias to + <xref linkend="conf-repeat" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-build-timeout"> + <term><literal>build-timeout</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>build-timeout</literal> is now an alias to + <xref linkend="conf-timeout" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-build-use-chroot"> + <term><literal>build-use-chroot</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>build-use-chroot</literal> is now an alias to + <xref linkend="conf-sandbox" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-build-use-sandbox"> + <term><literal>build-use-sandbox</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>build-use-sandbox</literal> is now an alias to + <xref linkend="conf-sandbox" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-build-use-substitutes"> + <term><literal>build-use-substitutes</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>build-use-substitutes</literal> is now an alias to + <xref linkend="conf-substitute" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-gc-keep-derivations"> + <term><literal>gc-keep-derivations</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>gc-keep-derivations</literal> is now an alias to + <xref linkend="conf-keep-derivations" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-gc-keep-outputs"> + <term><literal>gc-keep-outputs</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>gc-keep-outputs</literal> is now an alias to + <xref linkend="conf-keep-outputs" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-env-keep-derivations"> + <term><literal>env-keep-derivations</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>env-keep-derivations</literal> is now an alias to + <xref linkend="conf-keep-env-derivations" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-extra-binary-caches"> + <term><literal>extra-binary-caches</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>extra-binary-caches</literal> is now an alias to + <xref linkend="conf-extra-substituters" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-trusted-binary-caches"> + <term><literal>trusted-binary-caches</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>trusted-binary-caches</literal> is now an alias to + <xref linkend="conf-trusted-substituters" />.</para></listitem> + </varlistentry> +</variablelist> </para> +</refsection> </refsection> diff --git a/doc/manual/command-ref/env-common.xml b/doc/manual/command-ref/env-common.xml index 361d3e2b0330..6a3aaae717e2 100644 --- a/doc/manual/command-ref/env-common.xml +++ b/doc/manual/command-ref/env-common.xml @@ -14,7 +14,8 @@ <varlistentry><term><envar>IN_NIX_SHELL</envar></term> <listitem><para>Indicator that tells if the current environment was set up by - <command>nix-shell</command>.</para></listitem> + <command>nix-shell</command>. Since Nix 2.0 the values are + <literal>"pure"</literal> and <literal>"impure"</literal></para></listitem> </varlistentry> @@ -52,10 +53,15 @@ nixpkgs=/home/eelco/Dev/nixpkgs-branch:/etc/nixos</screen> <envar>NIX_PATH</envar> to <screen> -nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz</screen> +nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-15.09.tar.gz</screen> tells Nix to download the latest revision in the Nixpkgs/NixOS - 14.12 channel.</para> + 15.09 channel.</para> + + <para>A following shorthand can be used to refer to the official channels: + + <screen>nixpkgs=channel:nixos-15.09</screen> + </para> <para>The search path can be extended using the <option linkend="opt-I">-I</option> option, which takes precedence over diff --git a/doc/manual/command-ref/nix-collect-garbage.xml b/doc/manual/command-ref/nix-collect-garbage.xml index 35a78c5b2015..43e06879691c 100644 --- a/doc/manual/command-ref/nix-collect-garbage.xml +++ b/doc/manual/command-ref/nix-collect-garbage.xml @@ -22,12 +22,6 @@ <arg><option>--delete-old</option></arg> <arg><option>-d</option></arg> <arg><option>--delete-older-than</option> <replaceable>period</replaceable></arg> - <group choice='opt'> - <arg choice='plain'><option>--print-roots</option></arg> - <arg choice='plain'><option>--print-live</option></arg> - <arg choice='plain'><option>--print-dead</option></arg> - <arg choice='plain'><option>--delete</option></arg> - </group> <arg><option>--max-freed</option> <replaceable>bytes</replaceable></arg> <arg><option>--dry-run</option></arg> </cmdsynopsis> diff --git a/doc/manual/command-ref/nix-copy-closure.xml b/doc/manual/command-ref/nix-copy-closure.xml index 800e1db6ab0f..e6dcf180ad69 100644 --- a/doc/manual/command-ref/nix-copy-closure.xml +++ b/doc/manual/command-ref/nix-copy-closure.xml @@ -95,15 +95,6 @@ those paths. If this bothers you, use </varlistentry> - <!-- - <varlistentry><term><option>- -show-progress</option></term> - - <listitem><para>Show the progress of each path's transfer as it's made. - This requires the <command>pv</command> utility to be in <envar>PATH</envar>.</para></listitem> - - </varlistentry> - --> - <varlistentry><term><option>--include-outputs</option></term> <listitem><para>Also copy the outputs of store derivations diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml index eac7739558be..56c466268ea0 100644 --- a/doc/manual/command-ref/nix-env.xml +++ b/doc/manual/command-ref/nix-env.xml @@ -1346,11 +1346,12 @@ $ nix-env --list-generations <para>This operation deletes the specified generations of the current profile. The generations can be a list of generation numbers, the special value <literal>old</literal> to delete all non-current -generations, or a value such as <literal>30d</literal> to delete all +generations, a value such as <literal>30d</literal> to delete all generations older than the specified number of days (except for the -generation that was active at that point in time). -Periodically deleting old generations is important to make garbage -collection effective.</para> +generation that was active at that point in time), or a value such as. +<literal>+5</literal> to only keep the specified items older than the +current generation. Periodically deleting old generations is important +to make garbage collection effective.</para> </refsection> @@ -1359,6 +1360,8 @@ collection effective.</para> <screen> $ nix-env --delete-generations 3 4 8 +$ nix-env --delete-generations +5 + $ nix-env --delete-generations 30d $ nix-env -p other_profile --delete-generations old</screen> @@ -1458,7 +1461,7 @@ error: no generation older than the current (91) exists</screen> <refsection condition="manpage"><title>Environment variables</title> <variablelist> - + <varlistentry><term><envar>NIX_PROFILE</envar></term> <listitem><para>Location of the Nix profile. Defaults to the @@ -1472,6 +1475,6 @@ error: no generation older than the current (91) exists</screen> </variablelist> </refsection> - + </refentry> diff --git a/doc/manual/command-ref/nix-instantiate.xml b/doc/manual/command-ref/nix-instantiate.xml index 39c1282fcc36..53f06aed1241 100644 --- a/doc/manual/command-ref/nix-instantiate.xml +++ b/doc/manual/command-ref/nix-instantiate.xml @@ -154,7 +154,9 @@ input.</para> <listitem><para>When used with <option>--eval</option>, perform evaluation in read/write mode so nix language features that require it will still work (at the cost of needing to do - instantiation of every evaluated derivation).</para> + instantiation of every evaluated derivation). If this option is + not enabled, there may be uninstantiated store paths in the final + output.</para> </listitem> diff --git a/doc/manual/command-ref/nix-shell.xml b/doc/manual/command-ref/nix-shell.xml index 62d026ac238e..bb4a4e420122 100644 --- a/doc/manual/command-ref/nix-shell.xml +++ b/doc/manual/command-ref/nix-shell.xml @@ -32,6 +32,7 @@ <arg><option>--run</option> <replaceable>cmd</replaceable></arg> <arg><option>--exclude</option> <replaceable>regexp</replaceable></arg> <arg><option>--pure</option></arg> + <arg><option>--keep</option> <replaceable>name</replaceable></arg> <group choice='req'> <arg choice='plain'> <group choice='req'> @@ -165,6 +166,13 @@ also <xref linkend="sec-common-options" />.</phrase></para> </listitem></varlistentry> + <varlistentry><term><option>--keep</option> <replaceable>name</replaceable></term> + + <listitem><para>When a <option>--pure</option> shell is started, + keep the listed environment variables.</para></listitem> + + </varlistentry> + </variablelist> <para>The following common options are supported:</para> @@ -309,13 +317,28 @@ while (my $token = $p->get_tag("a")) { </para> -<para>Finally, the following Haskell script uses a specific branch of -Nixpkgs/NixOS (the 14.12 stable branch): +<para>Sometimes you need to pass a simple Nix expression to customize +a package like Terraform: + +<programlisting><![CDATA[ +#! /usr/bin/env nix-shell +#! nix-shell -i bash -p "terraform.withPlugins (plugins: [ plugins.openstack ])" + +terraform apply +]]></programlisting> + +<note><para>You must use double quotes (<literal>"</literal>) when +passing a simple Nix expression in a nix-shell shebang.</para></note> +</para> + +<para>Finally, using the merging of multiple nix-shell shebangs the +following Haskell script uses a specific branch of Nixpkgs/NixOS (the +18.03 stable branch): <programlisting><![CDATA[ #! /usr/bin/env nix-shell -#! nix-shell -i runghc -p haskellPackages.ghc haskellPackages.HTTP haskellPackages.tagsoup -#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz +#! nix-shell -i runghc -p "haskellPackages.ghcWithPackages (ps: [ps.HTTP ps.tagsoup])" +#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-18.03.tar.gz import Network.HTTP import Text.HTML.TagSoup diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml index f2dc6ed8540d..d73cb92ee223 100644 --- a/doc/manual/command-ref/nix-store.xml +++ b/doc/manual/command-ref/nix-store.xml @@ -204,7 +204,7 @@ printed.)</para> with <option>-K</option>, if an output path is not identical to the corresponding output from the previous build, the new output path is left in - <filename>/nix/store/<replaceable>name</replaceable>-check.</filename></para> + <filename>/nix/store/<replaceable>name</replaceable>.check.</filename></para> <para>See also the <option>build-repeat</option> configuration option, which repeats a derivation a number of times and prevents @@ -275,7 +275,7 @@ as a means of providing Nix store access to a restricted ssh user. <listitem><para>Allow the connected client to request the realization of derivations. In effect, this can be used to make the host act - as a build slave.</para></listitem> + as a remote builder.</para></listitem> </varlistentry> @@ -679,6 +679,18 @@ query is applied to the target of the symlink.</para> </varlistentry> + <varlistentry><term><option>--graphml</option></term> + + <listitem><para>Prints the references graph of the store paths + <replaceable>paths</replaceable> in the <link + xlink:href="http://graphml.graphdrawing.org/">GraphML</link> file format. + This can be used to visualise dependency graphs. To obtain a + build-time dependency graph, apply this to a store derivation. To + obtain a runtime dependency graph, apply it to an output + path.</para></listitem> + + </varlistentry> + <varlistentry><term><option>--binding</option> <replaceable>name</replaceable></term> <term><option>-b</option> <replaceable>name</replaceable></term> @@ -1270,6 +1282,7 @@ ktorrent-2.2.1/NEWS <cmdsynopsis> <command>nix-store</command> <arg choice='plain'><option>--dump-db</option></arg> + <arg rep='repeat'><replaceable>paths</replaceable></arg> </cmdsynopsis> </refsection> @@ -1280,6 +1293,13 @@ Nix database to standard output. It can be loaded into an empty Nix store using <option>--load-db</option>. This is useful for making backups and when migrating to different database schemas.</para> +<para>By default, <option>--dump-db</option> will dump the entire Nix +database. When one or more store paths is passed, only the subset of +the Nix database for those store paths is dumped. As with +<option>--export</option>, the user is responsible for passing all the +store paths for a closure. See <option>--export</option> for an +example.</para> + </refsection> </refsection> diff --git a/doc/manual/command-ref/opt-common-syn.xml b/doc/manual/command-ref/opt-common-syn.xml index 168bef080f4f..b610b54b9620 100644 --- a/doc/manual/command-ref/opt-common-syn.xml +++ b/doc/manual/command-ref/opt-common-syn.xml @@ -9,6 +9,9 @@ </group> </arg> <arg> + <arg choice='plain'><option>--quiet</option></arg> +</arg> +<arg> <group choice='plain'> <arg choice='plain'><option>--no-build-output</option></arg> <arg choice='plain'><option>-Q</option></arg> diff --git a/doc/manual/command-ref/opt-common.xml b/doc/manual/command-ref/opt-common.xml index bcb60b30125c..b8a2f260e8fe 100644 --- a/doc/manual/command-ref/opt-common.xml +++ b/doc/manual/command-ref/opt-common.xml @@ -75,6 +75,23 @@ </varlistentry> +<varlistentry><term><option>--quiet</option></term> + + <listitem> + + <para>Decreases the level of verbosity of diagnostic messages + printed on standard error. This is the inverse option to + <option>-v</option> / <option>--verbose</option>. + </para> + + <para>This option may be specified repeatedly. See the previous + verbosity levels list.</para> + + </listitem> + +</varlistentry> + + <varlistentry><term><option>--no-build-output</option> / <option>-Q</option></term> <listitem><para>By default, output written by builders to standard @@ -90,14 +107,22 @@ <varlistentry xml:id="opt-max-jobs"><term><option>--max-jobs</option> / <option>-j</option> <replaceable>number</replaceable></term> - <listitem><para>Sets the maximum number of build jobs that Nix will + <listitem> + + <para>Sets the maximum number of build jobs that Nix will perform in parallel to the specified number. Specify <literal>auto</literal> to use the number of CPUs in the system. The default is specified by the <link linkend='conf-max-jobs'><literal>max-jobs</literal></link> configuration setting, which itself defaults to <literal>1</literal>. A higher value is useful on SMP systems or to - exploit I/O latency.</para></listitem> + exploit I/O latency.</para> + + <para> Setting it to <literal>0</literal> disallows building on the local + machine, which is useful when you want builds to happen only on remote + builders.</para> + + </listitem> </varlistentry> diff --git a/doc/manual/expressions/advanced-attributes.xml b/doc/manual/expressions/advanced-attributes.xml index f3cf98371302..07b0d97d3f7d 100644 --- a/doc/manual/expressions/advanced-attributes.xml +++ b/doc/manual/expressions/advanced-attributes.xml @@ -50,6 +50,40 @@ allowedRequisites = [ foobar ]; </varlistentry> + <varlistentry><term><varname>disallowedReferences</varname></term> + + <listitem><para>The optional attribute + <varname>disallowedReferences</varname> specifies a list of illegal + references (dependencies) of the output of the builder. For + example, + +<programlisting> +disallowedReferences = [ foo ]; +</programlisting> + + enforces that the output of a derivation cannot have a direct runtime + dependencies on the derivation <varname>foo</varname>.</para></listitem> + + </varlistentry> + + + <varlistentry><term><varname>disallowedRequisites</varname></term> + + <listitem><para>This attribute is similar to + <varname>disallowedReferences</varname>, but it specifies illegal + requisites for the whole closure, so all the dependencies + recursively. For example, + +<programlisting> +disallowedRequisites = [ foobar ]; +</programlisting> + + enforces that the output of a derivation cannot have any + runtime dependency on <varname>foobar</varname> or any other derivation + depending recursively on <varname>foobar</varname>.</para></listitem> + + </varlistentry> + <varlistentry><term><varname>exportReferencesGraph</varname></term> @@ -112,7 +146,13 @@ impureEnvVars = [ "http_proxy" "https_proxy" <replaceable>...</replaceable> ]; linkend="fixed-output-drvs">fixed-output derivations</link>, where impurities such as these are okay since (the hash of) the output is known in advance. It is ignored for all other - derivations.</para></listitem> + derivations.</para> + + <warning><para><varname>impureEnvVars</varname> implementation takes + environment variables from the current builder process. When a daemon is + building its environmental variables are used. Without the daemon, the + environmental variables come from the environment of the + <command>nix-build</command>.</para></warning></listitem> </varlistentry> @@ -176,7 +216,7 @@ fetchurl { <programlisting> { stdenv, curl }: # The <command>curl</command> program is used for downloading. -{ url, md5 }: +{ url, sha256 }: stdenv.mkDerivation { name = baseNameOf (toString url); @@ -184,10 +224,10 @@ stdenv.mkDerivation { buildInputs = [ curl ]; # This is a fixed-output derivation; the output must be a regular - # file with MD5 hash <varname>md5</varname>. + # file with SHA256 hash <varname>sha256</varname>. outputHashMode = "flat"; - outputHashAlgo = "md5"; - outputHash = md5; + outputHashAlgo = "sha256"; + outputHash = sha256; inherit url; } @@ -197,8 +237,8 @@ stdenv.mkDerivation { <para>The <varname>outputHashAlgo</varname> attribute specifies the hash algorithm used to compute the hash. It can currently be - <literal>"md5"</literal>, <literal>"sha1"</literal> or - <literal>"sha256"</literal>.</para> + <literal>"sha1"</literal>, <literal>"sha256"</literal> or + <literal>"sha512"</literal>.</para> <para>The <varname>outputHashMode</varname> attribute determines how the hash is computed. It must be one of the following two @@ -211,7 +251,7 @@ stdenv.mkDerivation { <listitem><para>The output must be a non-executable regular file. If it isn’t, the build fails. The hash is simply computed over the contents of that file (so it’s equal to what - Unix commands like <command>md5sum</command> or + Unix commands like <command>sha256sum</command> or <command>sha1sum</command> produce).</para> <para>This is the default.</para></listitem> @@ -272,9 +312,7 @@ big = "a very long string"; <varlistentry><term><varname>preferLocalBuild</varname></term> <listitem><para>If this attribute is set to - <literal>true</literal>, it has two effects. First, the - derivation will always be built, not substituted, even if a - substitute is available. Second, if <link + <literal>true</literal> and <link linkend="chap-distributed-builds">distributed building is enabled</link>, then, if possible, the derivaton will be built locally instead of forwarded to a remote machine. This is @@ -284,6 +322,19 @@ big = "a very long string"; </varlistentry> + + <varlistentry><term><varname>allowSubstitutes</varname></term> + + <listitem><para>If this attribute is set to + <literal>false</literal>, then Nix will always build this + derivation; it will not try to substitute its outputs. This is + useful for very trivial derivations (such as + <function>writeText</function> in Nixpkgs) that are cheaper to + build than to substitute from a binary cache.</para></listitem> + + </varlistentry> + + </variablelist> </section> diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index ac1fe7e2fafe..a87639a075a5 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -21,7 +21,9 @@ available as <function>builtins.derivation</function>.</para> <variablelist> - <varlistentry><term><function>abort</function> <replaceable>s</replaceable></term> + <varlistentry xml:id='builtin-abort'> + <term><function>abort</function> <replaceable>s</replaceable></term> + <term><function>builtins.abort</function> <replaceable>s</replaceable></term> <listitem><para>Abort Nix expression evaluation, print error message <replaceable>s</replaceable>.</para></listitem> @@ -29,8 +31,10 @@ available as <function>builtins.derivation</function>.</para> </varlistentry> - <varlistentry><term><function>builtins.add</function> - <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + <varlistentry xml:id='builtin-add'> + <term><function>builtins.add</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable> + </term> <listitem><para>Return the sum of the numbers <replaceable>e1</replaceable> and @@ -39,8 +43,9 @@ available as <function>builtins.derivation</function>.</para> </varlistentry> - <varlistentry><term><function>builtins.all</function> - <replaceable>pred</replaceable> <replaceable>list</replaceable></term> + <varlistentry xml:id='builtin-all'> + <term><function>builtins.all</function> + <replaceable>pred</replaceable> <replaceable>list</replaceable></term> <listitem><para>Return <literal>true</literal> if the function <replaceable>pred</replaceable> returns <literal>true</literal> @@ -50,8 +55,9 @@ available as <function>builtins.derivation</function>.</para> </varlistentry> - <varlistentry><term><function>builtins.any</function> - <replaceable>pred</replaceable> <replaceable>list</replaceable></term> + <varlistentry xml:id='builtin-any'> + <term><function>builtins.any</function> + <replaceable>pred</replaceable> <replaceable>list</replaceable></term> <listitem><para>Return <literal>true</literal> if the function <replaceable>pred</replaceable> returns <literal>true</literal> @@ -61,8 +67,9 @@ available as <function>builtins.derivation</function>.</para> </varlistentry> - <varlistentry><term><function>builtins.attrNames</function> - <replaceable>set</replaceable></term> + <varlistentry xml:id='builtin-attrNames'> + <term><function>builtins.attrNames</function> + <replaceable>set</replaceable></term> <listitem><para>Return the names of the attributes in the set <replaceable>set</replaceable> in an alphabetically sorted list. For instance, @@ -72,8 +79,9 @@ available as <function>builtins.derivation</function>.</para> </varlistentry> - <varlistentry><term><function>builtins.attrValues</function> - <replaceable>set</replaceable></term> + <varlistentry xml:id='builtin-attrValues'> + <term><function>builtins.attrValues</function> + <replaceable>set</replaceable></term> <listitem><para>Return the values of the attributes in the set <replaceable>set</replaceable> in the order corresponding to the @@ -82,7 +90,8 @@ available as <function>builtins.derivation</function>.</para> </varlistentry> - <varlistentry><term><function>baseNameOf</function> <replaceable>s</replaceable></term> + <varlistentry xml:id='builtin-baseNameOf'> + <term><function>baseNameOf</function> <replaceable>s</replaceable></term> <listitem><para>Return the <emphasis>base name</emphasis> of the string <replaceable>s</replaceable>, that is, everything following @@ -92,7 +101,41 @@ available as <function>builtins.derivation</function>.</para> </varlistentry> - <varlistentry><term><varname>builtins</varname></term> + <varlistentry xml:id='builtin-bitAnd'> + <term><function>builtins.bitAnd</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + + <listitem><para>Return the bitwise AND of the integers + <replaceable>e1</replaceable> and + <replaceable>e2</replaceable>.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-bitOr'> + <term><function>builtins.bitOr</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + + <listitem><para>Return the bitwise OR of the integers + <replaceable>e1</replaceable> and + <replaceable>e2</replaceable>.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-bitXor'> + <term><function>builtins.bitXor</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + + <listitem><para>Return the bitwise XOR of the integers + <replaceable>e1</replaceable> and + <replaceable>e2</replaceable>.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-builtins'> + <term><varname>builtins</varname></term> <listitem><para>The set <varname>builtins</varname> contains all the built-in functions and values. You can use @@ -109,8 +152,9 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting> </varlistentry> - <varlistentry><term><function>builtins.compareVersions</function> - <replaceable>s1</replaceable> <replaceable>s2</replaceable></term> + <varlistentry xml:id='builtin-compareVersions'> + <term><function>builtins.compareVersions</function> + <replaceable>s1</replaceable> <replaceable>s2</replaceable></term> <listitem><para>Compare two strings representing versions and return <literal>-1</literal> if version @@ -126,8 +170,9 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting> </varlistentry> - <varlistentry><term><function>builtins.splitVersion</function> - <replaceable>s</replaceable></term> + <varlistentry xml:id='builtin-splitVersion'> + <term><function>builtins.splitVersion</function> + <replaceable>s</replaceable></term> <listitem><para>Split a string representing a version into its components, by the same version splitting logic underlying the @@ -137,16 +182,18 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting> </varlistentry> - <varlistentry><term><function>builtins.concatLists</function> - <replaceable>lists</replaceable></term> + <varlistentry xml:id='builtin-concatLists'> + <term><function>builtins.concatLists</function> + <replaceable>lists</replaceable></term> <listitem><para>Concatenate a list of lists into a single list.</para></listitem> </varlistentry> - <varlistentry><term><function>builtins.concatStringsSep</function> - <replaceable>separator</replaceable> <replaceable>list</replaceable></term> + <varlistentry xml:id='builtin-concatStringsSep'> + <term><function>builtins.concatStringsSep</function> + <replaceable>separator</replaceable> <replaceable>list</replaceable></term> <listitem><para>Concatenate a list of strings with a separator between each element, e.g. <literal>concatStringsSep "/" @@ -154,8 +201,8 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting> </varlistentry> - <varlistentry - xml:id='builtin-currentSystem'><term><varname>builtins.currentSystem</varname></term> + <varlistentry xml:id='builtin-currentSystem'> + <term><varname>builtins.currentSystem</varname></term> <listitem><para>The built-in value <varname>currentSystem</varname> evaluates to the Nix platform identifier for the Nix installation @@ -188,8 +235,9 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting> --> - <varlistentry><term><function>builtins.deepSeq</function> - <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + <varlistentry xml:id='builtin-deepSeq'> + <term><function>builtins.deepSeq</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> <listitem><para>This is like <literal>seq <replaceable>e1</replaceable> @@ -201,8 +249,11 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting> </varlistentry> - <varlistentry><term><function>derivation</function> - <replaceable>attrs</replaceable></term> + <varlistentry xml:id='builtin-derivation'> + <term><function>derivation</function> + <replaceable>attrs</replaceable></term> + <term><function>builtins.derivation</function> + <replaceable>attrs</replaceable></term> <listitem><para><function>derivation</function> is described in <xref linkend='ssec-derivation' />.</para></listitem> @@ -210,7 +261,9 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting> </varlistentry> - <varlistentry><term><function>dirOf</function> <replaceable>s</replaceable></term> + <varlistentry xml:id='builtin-dirOf'> + <term><function>dirOf</function> <replaceable>s</replaceable></term> + <term><function>builtins.dirOf</function> <replaceable>s</replaceable></term> <listitem><para>Return the directory part of the string <replaceable>s</replaceable>, that is, everything before the final @@ -220,8 +273,9 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting> </varlistentry> - <varlistentry><term><function>builtins.div</function> - <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + <varlistentry xml:id='builtin-div'> + <term><function>builtins.div</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> <listitem><para>Return the quotient of the numbers <replaceable>e1</replaceable> and @@ -229,8 +283,9 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting> </varlistentry> - <varlistentry><term><function>builtins.elem</function> - <replaceable>x</replaceable> <replaceable>xs</replaceable></term> + <varlistentry xml:id='builtin-elem'> + <term><function>builtins.elem</function> + <replaceable>x</replaceable> <replaceable>xs</replaceable></term> <listitem><para>Return <literal>true</literal> if a value equal to <replaceable>x</replaceable> occurs in the list @@ -240,8 +295,9 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting> </varlistentry> - <varlistentry><term><function>builtins.elemAt</function> - <replaceable>xs</replaceable> <replaceable>n</replaceable></term> + <varlistentry xml:id='builtin-elemAt'> + <term><function>builtins.elemAt</function> + <replaceable>xs</replaceable> <replaceable>n</replaceable></term> <listitem><para>Return element <replaceable>n</replaceable> from the list <replaceable>xs</replaceable>. Elements are counted @@ -251,8 +307,9 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting> </varlistentry> - <varlistentry><term><function>builtins.fetchurl</function> - <replaceable>url</replaceable></term> + <varlistentry xml:id='builtin-fetchurl'> + <term><function>builtins.fetchurl</function> + <replaceable>url</replaceable></term> <listitem><para>Download the specified URL and return the path of the downloaded file. This function is not available if <link @@ -262,8 +319,11 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting> </varlistentry> - <varlistentry><term><function>fetchTarball</function> - <replaceable>url</replaceable></term> + <varlistentry xml:id='builtin-fetchTarball'> + <term><function>fetchTarball</function> + <replaceable>url</replaceable></term> + <term><function>builtins.fetchTarball</function> + <replaceable>url</replaceable></term> <listitem><para>Download the specified URL, unpack it and return the path of the unpacked tree. The file must be a tape archive @@ -280,8 +340,17 @@ with import (fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixo stdenv.mkDerivation { … } </programlisting> + </para> + + <para>The fetched tarball is cached for a certain amount of time + (1 hour by default) in <filename>~/.cache/nix/tarballs/</filename>. + You can change the cache timeout either on the command line with + <option>--option tarball-ttl <replaceable>number of seconds</replaceable></option> or + in the Nix configuration file with this option: + <literal>tarball-ttl <replaceable>number of seconds to cache</replaceable></literal>. + </para> - Note that when obtaining the hash with <varname>nix-prefetch-url + <para>Note that when obtaining the hash with <varname>nix-prefetch-url </varname> the option <varname>--unpack</varname> is required. </para> @@ -307,7 +376,7 @@ stdenv.mkDerivation { … } </varlistentry> - <varlistentry> + <varlistentry xml:id='builtin-fetchGit'> <term> <function>builtins.fetchGit</function> <replaceable>args</replaceable> @@ -359,6 +428,84 @@ stdenv.mkDerivation { … } </listitem> </varlistentry> </variablelist> + + <example> + <title>Fetching a private repository over SSH</title> + <programlisting>builtins.fetchGit { + url = "git@github.com:my-secret/repository.git"; + ref = "master"; + rev = "adab8b916a45068c044658c4158d81878f9ed1c3"; +}</programlisting> + </example> + + <example> + <title>Fetching a repository's specific commit on an arbitrary branch</title> + <para> + If the revision you're looking for is in the default branch + of the git repository you don't strictly need to specify + the branch name in the <varname>ref</varname> attribute. + </para> + <para> + However, if the revision you're looking for is in a future + branch for the non-default branch you will need to specify + the the <varname>ref</varname> attribute as well. + </para> + <programlisting>builtins.fetchGit { + url = "https://github.com/nixos/nix.git"; + rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452"; + ref = "1.11-maintenance"; +}</programlisting> + <note> + <para> + It is nice to always specify the branch which a revision + belongs to. Without the branch being specified, the + fetcher might fail if the default branch changes. + Additionally, it can be confusing to try a commit from a + non-default branch and see the fetch fail. If the branch + is specified the fault is much more obvious. + </para> + </note> + </example> + + <example> + <title>Fetching a repository's specific commit on the default branch</title> + <para> + If the revision you're looking for is in the default branch + of the git repository you may omit the + <varname>ref</varname> attribute. + </para> + <programlisting>builtins.fetchGit { + url = "https://github.com/nixos/nix.git"; + rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452"; +}</programlisting> + </example> + + <example> + <title>Fetching a tag</title> + <programlisting>builtins.fetchGit { + url = "https://github.com/nixos/nix.git"; + ref = "tags/1.9"; +}</programlisting> + <note><para>Due to a bug (<link + xlink:href="https://github.com/NixOS/nix/issues/2385">#2385</link>), + only non-annotated tags can be fetched.</para></note> + </example> + + <example> + <title>Fetching the latest version of a remote branch</title> + <para> + <function>builtins.fetchGit</function> can behave impurely + fetch the latest version of a remote branch. + </para> + <note><para>Nix will refetch the branch in accordance to + <option>tarball-ttl</option>.</para></note> + <note><para>This behavior is disabled in + <emphasis>Pure evaluation mode</emphasis>.</para></note> + <programlisting>builtins.fetchGit { + url = "ssh://git@github.com/nixos/nix.git"; + ref = "master"; +}</programlisting> + </example> </listitem> </varlistentry> @@ -420,14 +567,17 @@ stdenv.mkDerivation { <literal>"unknown"</literal> (for other kinds of files such as device nodes or fifos — but note that those cannot be copied to the Nix store, so if the predicate returns - <literal>true</literal> for them, the copy will fail).</para> + <literal>true</literal> for them, the copy will fail). If you + exclude a directory, the entire corresponding subtree of + <replaceable>e2</replaceable> will be excluded.</para> </listitem> </varlistentry> - <varlistentry><term><function>builtins.foldl’</function> + <varlistentry xml:id='builtin-foldl-prime'> + <term><function>builtins.foldl’</function> <replaceable>op</replaceable> <replaceable>nul</replaceable> <replaceable>list</replaceable></term> <listitem><para>Reduce a list by applying a binary operator, from @@ -440,7 +590,8 @@ stdenv.mkDerivation { </varlistentry> - <varlistentry><term><function>builtins.functionArgs</function> + <varlistentry xml:id='builtin-functionArgs'> + <term><function>builtins.functionArgs</function> <replaceable>f</replaceable></term> <listitem><para> @@ -458,7 +609,8 @@ stdenv.mkDerivation { </varlistentry> - <varlistentry><term><function>builtins.fromJSON</function> <replaceable>e</replaceable></term> + <varlistentry xml:id='builtin-fromJSON'> + <term><function>builtins.fromJSON</function> <replaceable>e</replaceable></term> <listitem><para>Convert a JSON string to a Nix value. For example, @@ -468,14 +620,14 @@ builtins.fromJSON ''{"x": [1, 2, 3], "y": null}'' </programlisting> returns the value <literal>{ x = [ 1 2 3 ]; y = null; - }</literal>. Floating point numbers are not - supported.</para></listitem> + }</literal>.</para></listitem> </varlistentry> - <varlistentry><term><function>builtins.genList</function> - <replaceable>generator</replaceable> <replaceable>length</replaceable></term> + <varlistentry xml:id='builtin-genList'> + <term><function>builtins.genList</function> + <replaceable>generator</replaceable> <replaceable>length</replaceable></term> <listitem><para>Generate list of size <replaceable>length</replaceable>, with each element @@ -492,8 +644,9 @@ builtins.genList (x: x * x) 5 </varlistentry> - <varlistentry><term><function>builtins.getAttr</function> - <replaceable>s</replaceable> <replaceable>set</replaceable></term> + <varlistentry xml:id='builtin-getAttr'> + <term><function>builtins.getAttr</function> + <replaceable>s</replaceable> <replaceable>set</replaceable></term> <listitem><para><function>getAttr</function> returns the attribute named <replaceable>s</replaceable> from @@ -505,8 +658,9 @@ builtins.genList (x: x * x) 5 </varlistentry> - <varlistentry><term><function>builtins.getEnv</function> - <replaceable>s</replaceable></term> + <varlistentry xml:id='builtin-getEnv'> + <term><function>builtins.getEnv</function> + <replaceable>s</replaceable></term> <listitem><para><function>getEnv</function> returns the value of the environment variable <replaceable>s</replaceable>, or an empty @@ -523,8 +677,9 @@ builtins.genList (x: x * x) 5 </varlistentry> - <varlistentry><term><function>builtins.hasAttr</function> - <replaceable>s</replaceable> <replaceable>set</replaceable></term> + <varlistentry xml:id='builtin-hasAttr'> + <term><function>builtins.hasAttr</function> + <replaceable>s</replaceable> <replaceable>set</replaceable></term> <listitem><para><function>hasAttr</function> returns <literal>true</literal> if <replaceable>set</replaceable> has an @@ -537,20 +692,35 @@ builtins.genList (x: x * x) 5 </varlistentry> - <varlistentry><term><function>builtins.hashString</function> - <replaceable>type</replaceable> <replaceable>s</replaceable></term> + <varlistentry xml:id='builtin-hashString'> + <term><function>builtins.hashString</function> + <replaceable>type</replaceable> <replaceable>s</replaceable></term> <listitem><para>Return a base-16 representation of the cryptographic hash of string <replaceable>s</replaceable>. The hash algorithm specified by <replaceable>type</replaceable> must - be one of <literal>"md5"</literal>, <literal>"sha1"</literal> or - <literal>"sha256"</literal>.</para></listitem> + be one of <literal>"md5"</literal>, <literal>"sha1"</literal>, + <literal>"sha256"</literal> or <literal>"sha512"</literal>.</para></listitem> </varlistentry> - <varlistentry><term><function>builtins.head</function> - <replaceable>list</replaceable></term> + <varlistentry xml:id='builtin-hashFile'> + <term><function>builtins.hashFile</function> + <replaceable>type</replaceable> <replaceable>p</replaceable></term> + + <listitem><para>Return a base-16 representation of the + cryptographic hash of the file at path <replaceable>p</replaceable>. The + hash algorithm specified by <replaceable>type</replaceable> must + be one of <literal>"md5"</literal>, <literal>"sha1"</literal>, + <literal>"sha256"</literal> or <literal>"sha512"</literal>.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-head'> + <term><function>builtins.head</function> + <replaceable>list</replaceable></term> <listitem><para>Return the first element of a list; abort evaluation if the argument isn’t a list or is an empty list. You @@ -560,8 +730,11 @@ builtins.genList (x: x * x) 5 </varlistentry> - <varlistentry><term><function>import</function> - <replaceable>path</replaceable></term> + <varlistentry xml:id='builtin-import'> + <term><function>import</function> + <replaceable>path</replaceable></term> + <term><function>builtins.import</function> + <replaceable>path</replaceable></term> <listitem><para>Load, parse and return the Nix expression in the file <replaceable>path</replaceable>. If <replaceable>path @@ -615,8 +788,9 @@ x: x + 456</programlisting> </varlistentry> - <varlistentry><term><function>builtins.intersectAttrs</function> - <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + <varlistentry xml:id='builtin-intersectAttrs'> + <term><function>builtins.intersectAttrs</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> <listitem><para>Return a set consisting of the attributes in the set <replaceable>e2</replaceable> that also exist in the set @@ -625,8 +799,9 @@ x: x + 456</programlisting> </varlistentry> - <varlistentry><term><function>builtins.isAttrs</function> - <replaceable>e</replaceable></term> + <varlistentry xml:id='builtin-isAttrs'> + <term><function>builtins.isAttrs</function> + <replaceable>e</replaceable></term> <listitem><para>Return <literal>true</literal> if <replaceable>e</replaceable> evaluates to a set, and @@ -635,8 +810,9 @@ x: x + 456</programlisting> </varlistentry> - <varlistentry><term><function>builtins.isList</function> - <replaceable>e</replaceable></term> + <varlistentry xml:id='builtin-isList'> + <term><function>builtins.isList</function> + <replaceable>e</replaceable></term> <listitem><para>Return <literal>true</literal> if <replaceable>e</replaceable> evaluates to a list, and @@ -645,7 +821,7 @@ x: x + 456</programlisting> </varlistentry> - <varlistentry><term><function>builtins.isFunction</function> + <varlistentry xml:id='builtin-isFunction'><term><function>builtins.isFunction</function> <replaceable>e</replaceable></term> <listitem><para>Return <literal>true</literal> if @@ -655,8 +831,9 @@ x: x + 456</programlisting> </varlistentry> - <varlistentry><term><function>builtins.isString</function> - <replaceable>e</replaceable></term> + <varlistentry xml:id='builtin-isString'> + <term><function>builtins.isString</function> + <replaceable>e</replaceable></term> <listitem><para>Return <literal>true</literal> if <replaceable>e</replaceable> evaluates to a string, and @@ -665,8 +842,9 @@ x: x + 456</programlisting> </varlistentry> - <varlistentry><term><function>builtins.isInt</function> - <replaceable>e</replaceable></term> + <varlistentry xml:id='builtin-isInt'> + <term><function>builtins.isInt</function> + <replaceable>e</replaceable></term> <listitem><para>Return <literal>true</literal> if <replaceable>e</replaceable> evaluates to an int, and @@ -675,8 +853,9 @@ x: x + 456</programlisting> </varlistentry> - <varlistentry><term><function>builtins.isFloat</function> - <replaceable>e</replaceable></term> + <varlistentry xml:id='builtin-isFloat'> + <term><function>builtins.isFloat</function> + <replaceable>e</replaceable></term> <listitem><para>Return <literal>true</literal> if <replaceable>e</replaceable> evaluates to a float, and @@ -685,8 +864,9 @@ x: x + 456</programlisting> </varlistentry> - <varlistentry><term><function>builtins.isBool</function> - <replaceable>e</replaceable></term> + <varlistentry xml:id='builtin-isBool'> + <term><function>builtins.isBool</function> + <replaceable>e</replaceable></term> <listitem><para>Return <literal>true</literal> if <replaceable>e</replaceable> evaluates to a bool, and @@ -694,11 +874,22 @@ x: x + 456</programlisting> </varlistentry> - - <varlistentry><term><function>isNull</function> + <varlistentry><term><function>builtins.isPath</function> <replaceable>e</replaceable></term> <listitem><para>Return <literal>true</literal> if + <replaceable>e</replaceable> evaluates to a path, and + <literal>false</literal> otherwise.</para></listitem> + + </varlistentry> + + <varlistentry xml:id='builtin-isNull'> + <term><function>isNull</function> + <replaceable>e</replaceable></term> + <term><function>builtins.isNull</function> + <replaceable>e</replaceable></term> + + <listitem><para>Return <literal>true</literal> if <replaceable>e</replaceable> evaluates to <literal>null</literal>, and <literal>false</literal> otherwise.</para> @@ -710,8 +901,9 @@ x: x + 456</programlisting> </varlistentry> - <varlistentry><term><function>builtins.length</function> - <replaceable>e</replaceable></term> + <varlistentry xml:id='builtin-length'> + <term><function>builtins.length</function> + <replaceable>e</replaceable></term> <listitem><para>Return the length of the list <replaceable>e</replaceable>.</para></listitem> @@ -719,8 +911,9 @@ x: x + 456</programlisting> </varlistentry> - <varlistentry><term><function>builtins.lessThan</function> - <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + <varlistentry xml:id='builtin-lessThan'> + <term><function>builtins.lessThan</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> <listitem><para>Return <literal>true</literal> if the number <replaceable>e1</replaceable> is less than the number @@ -732,8 +925,9 @@ x: x + 456</programlisting> </varlistentry> - <varlistentry><term><function>builtins.listToAttrs</function> - <replaceable>e</replaceable></term> + <varlistentry xml:id='builtin-listToAttrs'> + <term><function>builtins.listToAttrs</function> + <replaceable>e</replaceable></term> <listitem><para>Construct a set from a list specifying the names and values of each attribute. Each element of the list should be @@ -759,8 +953,11 @@ builtins.listToAttrs </varlistentry> - <varlistentry><term><function>map</function> - <replaceable>f</replaceable> <replaceable>list</replaceable></term> + <varlistentry xml:id='builtin-map'> + <term><function>map</function> + <replaceable>f</replaceable> <replaceable>list</replaceable></term> + <term><function>builtins.map</function> + <replaceable>f</replaceable> <replaceable>list</replaceable></term> <listitem><para>Apply the function <replaceable>f</replaceable> to each element in the list <replaceable>list</replaceable>. For @@ -775,14 +972,15 @@ map (x: "foo" + x) [ "bar" "bla" "abc" ]</programlisting> </varlistentry> - <varlistentry><term><function>builtins.match</function> - <replaceable>regex</replaceable> <replaceable>str</replaceable></term> + <varlistentry xml:id='builtin-match'> + <term><function>builtins.match</function> + <replaceable>regex</replaceable> <replaceable>str</replaceable></term> - <listitem><para>Returns a list if the <link - xlink:href="http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04">extended - POSIX regular expression</link> <replaceable>regex</replaceable> - matches <replaceable>str</replaceable> precisely, otherwise returns - <literal>null</literal>. Each item in the list is a regex group. + <listitem><para>Returns a list if the <link + xlink:href="http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04">extended + POSIX regular expression</link> <replaceable>regex</replaceable> + matches <replaceable>str</replaceable> precisely, otherwise returns + <literal>null</literal>. Each item in the list is a regex group. <programlisting> builtins.match "ab" "abc" @@ -808,11 +1006,12 @@ builtins.match "[[:space:]]+([[:upper:]]+)[[:space:]]+" " FOO " Evaluates to <literal>[ "foo" ]</literal>. - </para></listitem> + </para></listitem> </varlistentry> - <varlistentry><term><function>builtins.mul</function> - <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + <varlistentry xml:id='builtin-mul'> + <term><function>builtins.mul</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> <listitem><para>Return the product of the numbers <replaceable>e1</replaceable> and @@ -821,8 +1020,9 @@ Evaluates to <literal>[ "foo" ]</literal>. </varlistentry> - <varlistentry><term><function>builtins.parseDrvName</function> - <replaceable>s</replaceable></term> + <varlistentry xml:id='builtin-parseDrvName'> + <term><function>builtins.parseDrvName</function> + <replaceable>s</replaceable></term> <listitem><para>Split the string <replaceable>s</replaceable> into a package name and version. The package name is everything up to @@ -835,7 +1035,7 @@ Evaluates to <literal>[ "foo" ]</literal>. </varlistentry> - <varlistentry> + <varlistentry xml:id='builtin-path'> <term> <function>builtins.path</function> <replaceable>args</replaceable> @@ -905,32 +1105,20 @@ Evaluates to <literal>[ "foo" ]</literal>. </listitem> </varlistentry> - <varlistentry><term><function>builtins.pathExists</function> - <replaceable>path</replaceable></term> + <varlistentry xml:id='builtin-pathExists'> + <term><function>builtins.pathExists</function> + <replaceable>path</replaceable></term> <listitem><para>Return <literal>true</literal> if the path - <replaceable>path</replaceable> exists, and - <literal>false</literal> otherwise. One application of this - function is to conditionally include a Nix expression containing - user configuration: - -<programlisting> -let - fileName = builtins.getEnv "CONFIG_FILE"; - config = - if fileName != "" && builtins.pathExists (builtins.toPath fileName) - then import (builtins.toPath fileName) - else { someSetting = false; }; <lineannotation># default configuration</lineannotation> -in config.someSetting</programlisting> - - (Note that <envar>CONFIG_FILE</envar> must be an absolute path for - this to work.)</para></listitem> + <replaceable>path</replaceable> exists at evaluation time, and + <literal>false</literal> otherwise.</para></listitem> </varlistentry> - <varlistentry><term><function>builtins.readDir</function> - <replaceable>path</replaceable></term> + <varlistentry xml:id='builtin-readDir'> + <term><function>builtins.readDir</function> + <replaceable>path</replaceable></term> <listitem><para>Return the contents of the directory <replaceable>path</replaceable> as a set mapping directory entries @@ -951,8 +1139,9 @@ in config.someSetting</programlisting> </varlistentry> - <varlistentry><term><function>builtins.readFile</function> - <replaceable>path</replaceable></term> + <varlistentry xml:id='builtin-readFile'> + <term><function>builtins.readFile</function> + <replaceable>path</replaceable></term> <listitem><para>Return the contents of the file <replaceable>path</replaceable> as a string.</para></listitem> @@ -960,8 +1149,11 @@ in config.someSetting</programlisting> </varlistentry> - <varlistentry><term><function>removeAttrs</function> - <replaceable>set</replaceable> <replaceable>list</replaceable></term> + <varlistentry xml:id='builtin-removeAttrs'> + <term><function>removeAttrs</function> + <replaceable>set</replaceable> <replaceable>list</replaceable></term> + <term><function>builtins.removeAttrs</function> + <replaceable>set</replaceable> <replaceable>list</replaceable></term> <listitem><para>Remove the attributes listed in <replaceable>list</replaceable> from @@ -976,8 +1168,9 @@ removeAttrs { x = 1; y = 2; z = 3; } [ "a" "x" "z" ]</programlisting> </varlistentry> - <varlistentry><term><function>builtins.replaceStrings</function> - <replaceable>from</replaceable> <replaceable>to</replaceable> <replaceable>s</replaceable></term> + <varlistentry xml:id='builtin-replaceStrings'> + <term><function>builtins.replaceStrings</function> + <replaceable>from</replaceable> <replaceable>to</replaceable> <replaceable>s</replaceable></term> <listitem><para>Given string <replaceable>s</replaceable>, replace every occurrence of the strings in <replaceable>from</replaceable> @@ -993,8 +1186,9 @@ builtins.replaceStrings ["oo" "a"] ["a" "i"] "foobar" </varlistentry> - <varlistentry><term><function>builtins.seq</function> - <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + <varlistentry xml:id='builtin-seq'> + <term><function>builtins.seq</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> <listitem><para>Evaluate <replaceable>e1</replaceable>, then evaluate and return <replaceable>e2</replaceable>. This ensures @@ -1004,8 +1198,9 @@ builtins.replaceStrings ["oo" "a"] ["a" "i"] "foobar" </varlistentry> - <varlistentry><term><function>builtins.sort</function> - <replaceable>comparator</replaceable> <replaceable>list</replaceable></term> + <varlistentry xml:id='builtin-sort'> + <term><function>builtins.sort</function> + <replaceable>comparator</replaceable> <replaceable>list</replaceable></term> <listitem><para>Return <replaceable>list</replaceable> in sorted order. It repeatedly calls the function @@ -1027,15 +1222,16 @@ builtins.sort builtins.lessThan [ 483 249 526 147 42 77 ] </varlistentry> - <varlistentry><term><function>builtins.split</function> - <replaceable>regex</replaceable> <replaceable>str</replaceable></term> + <varlistentry xml:id='builtin-split'> + <term><function>builtins.split</function> + <replaceable>regex</replaceable> <replaceable>str</replaceable></term> - <listitem><para>Returns a list composed of non matched strings interleaved - with the lists of the <link - xlink:href="http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04">extended - POSIX regular expression</link> <replaceable>regex</replaceable> matches - of <replaceable>str</replaceable>. Each item in the lists of matched - sequences is a regex group. + <listitem><para>Returns a list composed of non matched strings interleaved + with the lists of the <link + xlink:href="http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04">extended + POSIX regular expression</link> <replaceable>regex</replaceable> matches + of <replaceable>str</replaceable>. Each item in the lists of matched + sequences is a regex group. <programlisting> builtins.split "(a)b" "abc" @@ -1061,11 +1257,12 @@ builtins.split "([[:upper:]]+)" " FOO " Evaluates to <literal>[ " " [ "FOO" ] " " ]</literal>. - </para></listitem> + </para></listitem> </varlistentry> - <varlistentry><term><function>builtins.stringLength</function> - <replaceable>e</replaceable></term> + <varlistentry xml:id='builtin-stringLength'> + <term><function>builtins.stringLength</function> + <replaceable>e</replaceable></term> <listitem><para>Return the length of the string <replaceable>e</replaceable>. If <replaceable>e</replaceable> is @@ -1074,8 +1271,9 @@ Evaluates to <literal>[ " " [ "FOO" ] " " ]</literal>. </varlistentry> - <varlistentry><term><function>builtins.sub</function> - <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + <varlistentry xml:id='builtin-sub'> + <term><function>builtins.sub</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> <listitem><para>Return the difference between the numbers <replaceable>e1</replaceable> and @@ -1084,9 +1282,10 @@ Evaluates to <literal>[ " " [ "FOO" ] " " ]</literal>. </varlistentry> - <varlistentry><term><function>builtins.substring</function> - <replaceable>start</replaceable> <replaceable>len</replaceable> - <replaceable>s</replaceable></term> + <varlistentry xml:id='builtin-substring'> + <term><function>builtins.substring</function> + <replaceable>start</replaceable> <replaceable>len</replaceable> + <replaceable>s</replaceable></term> <listitem><para>Return the substring of <replaceable>s</replaceable> from character position @@ -1109,8 +1308,9 @@ builtins.substring 0 3 "nixos" </varlistentry> - <varlistentry><term><function>builtins.tail</function> - <replaceable>list</replaceable></term> + <varlistentry xml:id='builtin-tail'> + <term><function>builtins.tail</function> + <replaceable>list</replaceable></term> <listitem><para>Return the second to last elements of a list; abort evaluation if the argument isn’t a list or is an empty @@ -1119,8 +1319,11 @@ builtins.substring 0 3 "nixos" </varlistentry> - <varlistentry><term><function>throw</function> - <replaceable>s</replaceable></term> + <varlistentry xml:id='builtin-throw'> + <term><function>throw</function> + <replaceable>s</replaceable></term> + <term><function>builtins.throw</function> + <replaceable>s</replaceable></term> <listitem><para>Throw an error message <replaceable>s</replaceable>. This usually aborts Nix expression @@ -1133,9 +1336,10 @@ builtins.substring 0 3 "nixos" </varlistentry> - <varlistentry - xml:id='builtin-toFile'><term><function>builtins.toFile</function> - <replaceable>name</replaceable> <replaceable>s</replaceable></term> + <varlistentry xml:id='builtin-toFile'> + <term><function>builtins.toFile</function> + <replaceable>name</replaceable> + <replaceable>s</replaceable></term> <listitem><para>Store the string <replaceable>s</replaceable> in a file in the Nix store and return its path. The file has suffix @@ -1204,14 +1408,15 @@ in foo</programlisting> This is not allowed because it would cause a cyclic dependency in the computation of the cryptographic hashes for <varname>foo</varname> and <varname>bar</varname>.</para> - <para>It is also not possible to reference the result of a derivation. - If you are using Nixpkgs, the <literal>writeTextFile</literal> function is able to + <para>It is also not possible to reference the result of a derivation. + If you are using Nixpkgs, the <literal>writeTextFile</literal> function is able to do that.</para></listitem> </varlistentry> - <varlistentry><term><function>builtins.toJSON</function> <replaceable>e</replaceable></term> + <varlistentry xml:id='builtin-toJSON'> + <term><function>builtins.toJSON</function> <replaceable>e</replaceable></term> <listitem><para>Return a string containing a JSON representation of <replaceable>e</replaceable>. Strings, integers, floats, booleans, @@ -1224,20 +1429,20 @@ in foo</programlisting> </varlistentry> - <varlistentry><term><function>builtins.toPath</function> <replaceable>s</replaceable></term> + <varlistentry xml:id='builtin-toPath'> + <term><function>builtins.toPath</function> <replaceable>s</replaceable></term> - <listitem><para>Convert the string value - <replaceable>s</replaceable> into a path value. The string - <replaceable>s</replaceable> must represent an absolute path - (i.e., must start with <literal>/</literal>). The path need not - exist. The resulting path is canonicalised, e.g., - <literal>builtins.toPath "//foo/xyzzy/../bar/"</literal> returns - <literal>/foo/bar</literal>.</para></listitem> + <listitem><para> DEPRECATED. Use <literal>/. + "/path"</literal> + to convert a string into an absolute path. For relative paths, + use <literal>./. + "/path"</literal>. + </para></listitem> </varlistentry> - <varlistentry><term><function>toString</function> <replaceable>e</replaceable></term> + <varlistentry xml:id='builtin-toString'> + <term><function>toString</function> <replaceable>e</replaceable></term> + <term><function>builtins.toString</function> <replaceable>e</replaceable></term> <listitem><para>Convert the expression <replaceable>e</replaceable> to a string. @@ -1256,7 +1461,8 @@ in foo</programlisting> </varlistentry> - <varlistentry xml:id='builtin-toXML'><term><function>builtins.toXML</function> <replaceable>e</replaceable></term> + <varlistentry xml:id='builtin-toXML'> + <term><function>builtins.toXML</function> <replaceable>e</replaceable></term> <listitem><para>Return a string containing an XML representation of <replaceable>e</replaceable>. The main application for @@ -1312,7 +1518,7 @@ stdenv.mkDerivation (rec { builder = builtins.toFile "builder.sh" " source $stdenv/setup mkdir $out - echo $servlets | xsltproc ${stylesheet} - > $out/server-conf.xml]]> <co xml:id='ex-toxml-co-apply' /> <![CDATA[ + echo "$servlets" | xsltproc ${stylesheet} - > $out/server-conf.xml]]> <co xml:id='ex-toxml-co-apply' /> <![CDATA[ "; stylesheet = builtins.toFile "stylesheet.xsl"]]> <co xml:id='ex-toxml-co-stylesheet' /> <![CDATA[ @@ -1371,8 +1577,9 @@ stdenv.mkDerivation (rec { </varlistentry> - <varlistentry><term><function>builtins.trace</function> - <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + <varlistentry xml:id='builtin-trace'> + <term><function>builtins.trace</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> <listitem><para>Evaluate <replaceable>e1</replaceable> and print its abstract syntax representation on standard error. Then return @@ -1381,8 +1588,9 @@ stdenv.mkDerivation (rec { </varlistentry> - <varlistentry><term><function>builtins.tryEval</function> - <replaceable>e</replaceable></term> + <varlistentry xml:id='builtin-tryEval'> + <term><function>builtins.tryEval</function> + <replaceable>e</replaceable></term> <listitem><para>Try to evaluate <replaceable>e</replaceable>. Return a set containing the attributes <literal>success</literal> @@ -1395,8 +1603,9 @@ stdenv.mkDerivation (rec { </varlistentry> - <varlistentry><term><function>builtins.typeOf</function> - <replaceable>e</replaceable></term> + <varlistentry xml:id='builtin-typeOf'> + <term><function>builtins.typeOf</function> + <replaceable>e</replaceable></term> <listitem><para>Return a string representing the type of the value <replaceable>e</replaceable>, namely <literal>"int"</literal>, diff --git a/doc/manual/expressions/language-constructs.xml b/doc/manual/expressions/language-constructs.xml index 47d95f8a13e3..923b5d8c4011 100644 --- a/doc/manual/expressions/language-constructs.xml +++ b/doc/manual/expressions/language-constructs.xml @@ -41,7 +41,7 @@ encountered</quote>).</para></footnote>.</para> </simplesect> -<simplesect><title>Let-expressions</title> +<simplesect xml:id="sect-let-expressions"><title>Let-expressions</title> <para>A let-expression allows you define local variables for an expression. For instance, @@ -217,7 +217,25 @@ but can also be written as: ellipsis(<literal>...</literal>) as you can access attribute names as <literal>a</literal>, using <literal>args.a</literal>, which was given as an additional attribute to the function. - </para></listitem> + </para> + + <warning> + <para> + The <literal>args@</literal> expression is bound to the argument passed to the function which + means that attributes with defaults that aren't explicitly specified in the function call + won't cause an evaluation error, but won't exist in <literal>args</literal>. + </para> + <para> + For instance +<programlisting> +let + function = args@{ a ? 23, ... }: args; +in + function {} +</programlisting> + will evaluate to an empty attribute set. + </para> + </warning></listitem> </itemizedlist> diff --git a/doc/manual/glossary/glossary.xml b/doc/manual/glossary/glossary.xml index 4977825578f1..e3162ed8d469 100644 --- a/doc/manual/glossary/glossary.xml +++ b/doc/manual/glossary/glossary.xml @@ -1,5 +1,6 @@ <appendix xmlns="http://docbook.org/ns/docbook" - xmlns:xlink="http://www.w3.org/1999/xlink"> + xmlns:xlink="http://www.w3.org/1999/xlink" + xml:id="part-glossary"> <title>Glossary</title> diff --git a/doc/manual/hacking.xml b/doc/manual/hacking.xml index 183aed7adff2..b671811d3a30 100644 --- a/doc/manual/hacking.xml +++ b/doc/manual/hacking.xml @@ -30,7 +30,7 @@ To build Nix itself in this shell: [nix-shell]$ configurePhase [nix-shell]$ make </screen> -To install it in <literal>$(pwd)/nix</literal> and test it: +To install it in <literal>$(pwd)/inst</literal> and test it: <screen> [nix-shell]$ make install [nix-shell]$ make installcheck diff --git a/doc/manual/installation/env-variables.xml b/doc/manual/installation/env-variables.xml index fc39cdd9dfef..d1ee0bb2e096 100644 --- a/doc/manual/installation/env-variables.xml +++ b/doc/manual/installation/env-variables.xml @@ -21,4 +21,51 @@ in your <filename>~/.profile</filename> (or similar), like this:</para> <screen> source <replaceable>prefix</replaceable>/etc/profile.d/nix.sh</screen> -</chapter> \ No newline at end of file +<section xml:id="sec-nix-ssl-cert-file"> + +<title><envar>NIX_SSL_CERT_FILE</envar></title> + +<para>If you need to specify a custom certificate bundle to account +for an HTTPS-intercepting man in the middle proxy, you must specify +the path to the certificate bundle in the environment variable +<envar>NIX_SSL_CERT_FILE</envar>.</para> + + +<para>If you don't specify a <envar>NIX_SSL_CERT_FILE</envar> +manually, Nix will install and use its own certificate +bundle.</para> + +<procedure> + <step><para>Set the environment variable and install Nix</para> + <screen> +$ export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt +$ sh <(curl https://nixos.org/nix/install) +</screen></step> + + <step><para>In the shell profile and rc files (for example, + <filename>/etc/bashrc</filename>, <filename>/etc/zshrc</filename>), + add the following line:</para> +<programlisting> +export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt +</programlisting> +</step> +</procedure> + +<note><para>You must not add the export and then do the install, as +the Nix installer will detect the presense of Nix configuration, and +abort.</para></note> + +<section xml:id="sec-nix-ssl-cert-file-with-nix-daemon-and-macos"> +<title><envar>NIX_SSL_CERT_FILE</envar> with macOS and the Nix daemon</title> + +<para>On macOS you must specify the environment variable for the Nix +daemon service, then restart it:</para> + +<screen> +$ sudo launchctl setenv NIX_SSL_CERT_FILE /etc/ssl/my-certificate-bundle.crt +$ sudo launchctl kickstart -k system/org.nixos.nix-daemon +</screen> +</section> + +</section> +</chapter> diff --git a/doc/manual/installation/installing-binary.xml b/doc/manual/installation/installing-binary.xml index 7e8dfb0db3d4..394d8053b942 100644 --- a/doc/manual/installation/installing-binary.xml +++ b/doc/manual/installation/installing-binary.xml @@ -6,13 +6,30 @@ <title>Installing a Binary Distribution</title> -<para>If you are using Linux or macOS, the easiest way to install -Nix is to run the following command: +<para>If you are using Linux or macOS, the easiest way to install Nix +is to run the following command: <screen> -$ bash <(curl https://nixos.org/nix/install) + $ sh <(curl https://nixos.org/nix/install) </screen> +As of Nix 2.1.0, the Nix installer will always default to creating a +single-user installation, however opting in to the multi-user +installation is highly recommended. +</para> + +<section xml:id="sect-single-user-installation"> + <title>Single User Installation</title> + + <para> + To explicitly select a single-user installation on your system: + + <screen> + sh <(curl https://nixos.org/nix/install) --no-daemon +</screen> + </para> + +<para> This will perform a single-user installation of Nix, meaning that <filename>/nix</filename> is owned by the invoking user. You should run this under your usual user account, <emphasis>not</emphasis> as @@ -33,58 +50,141 @@ and <filename>.profile</filename> to source the <command>NIX_INSTALLER_NO_MODIFY_PROFILE</command> environment variable before executing the install script to disable this behaviour. - </para> -<!-- -<para>You can also manually download and install a binary package. -Binary packages of the latest stable release are available for Fedora, -Debian, Ubuntu, macOS and various other systems from the <link -xlink:href="http://nixos.org/nix/download.html">Nix homepage</link>. -You can also get builds of the latest development release from our -<link -xlink:href="http://hydra.nixos.org/job/nix/master/release/latest-finished#tabs-constituents">continuous -build system</link>.</para> -<para>For Fedora, RPM packages are available. These can be installed -or upgraded using <command>rpm -U</command>. For example, +<para>You can uninstall Nix simply by running: <screen> -$ rpm -U nix-1.8-1.i386.rpm</screen> +$ rm -rf /nix +</screen> </para> - -<para>For Debian and Ubuntu, you can download a Deb package and -install it like this: +</section> + +<section xml:id="sect-multi-user-installation"> + <title>Multi User Installation</title> + <para> + The multi-user Nix installation creates system users, and a system + service for the Nix daemon. + </para> + + <itemizedlist> + <title>Supported Systems</title> + + <listitem> + <para>Linux running systemd, with SELinux disabled</para> + </listitem> + <listitem><para>macOS</para></listitem> + </itemizedlist> + + <para> + You can instruct the installer to perform a multi-user + installation on your system: + + <screen> + sh <(curl https://nixos.org/nix/install) --daemon +</screen> + </para> + + <para> + The multi-user installation of Nix will create build users between + the user IDs 30001 and 30032, and a group with the group ID 30000. + + You should run this under your usual user account, + <emphasis>not</emphasis> as root. The script will invoke + <command>sudo</command> as needed. + </para> + + <note><para> + If you need Nix to use a different group ID or user ID set, you + will have to download the tarball manually and <link + linkend="sect-nix-install-binary-tarball">edit the install + script</link>. + </para></note> + + <para> + The installer will modify <filename>/etc/bashrc</filename>, and + <filename>/etc/zshrc</filename> if they exist. The installer will + first back up these files with a + <literal>.backup-before-nix</literal> extension. The installer + will also create <filename>/etc/profile.d/nix.sh</filename>. + </para> + + <para>You can uninstall Nix with the following commands: <screen> -$ dpkg -i nix_1.8-1_amd64.deb</screen> +sudo rm -rf /etc/profile/nix.sh /etc/nix /nix ~root/.nix-profile ~root/.nix-defexpr ~root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels + +# If you are on Linux with systemd, you will need to run: +sudo systemctl stop nix-daemon.socket +sudo systemctl stop nix-daemon.service +sudo systemctl disable nix-daemon.socket +sudo systemctl disable nix-daemon.service +sudo systemctl daemon-reload + +# If you are on macOS, you will need to run: +sudo launchctl unload /Library/LaunchDaemons/org.nixos.nix-daemon.plist +sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist +</screen> -</para> ---> + There may also be references to Nix in + <filename>/etc/profile</filename>, + <filename>/etc/bashrc</filename>, and + <filename>/etc/zshrc</filename> which you may remove. + </para> -<para>You can also download a binary tarball that contains Nix and all -its dependencies. (This is what the install script at -<uri>https://nixos.org/nix/install</uri> does automatically.) You -should unpack it somewhere (e.g. in <filename>/tmp</filename>), and -then run the script named <command>install</command> inside the binary -tarball: +</section> -<screen> -alice$ cd /tmp -alice$ tar xfj nix-1.8-x86_64-darwin.tar.bz2 -alice$ cd nix-1.8-x86_64-darwin -alice$ ./install -</screen> +<section xml:id="sect-nix-install-pinned-version-url"> + <title>Installing a pinned Nix version from a URL</title> -</para> + <para> + NixOS.org hosts version-specific installation URLs for all Nix + versions since 1.11.16, at + <literal>https://nixos.org/releases/nix/nix-VERSION/install</literal>. + </para> -<para>You can uninstall Nix simply by running: + <para> + These install scripts can be used the same as the main + NixOS.org installation script: -<screen> -$ rm -rf /nix + <screen> + sh <(curl https://nixos.org/nix/install) </screen> + </para> -</para> + <para> + In the same directory of the install script are sha256 sums, and + gpg signature files. + </para> +</section> + +<section xml:id="sect-nix-install-binary-tarball"> + <title>Installing from a binary tarball</title> + <para> + You can also download a binary tarball that contains Nix and all + its dependencies. (This is what the install script at + <uri>https://nixos.org/nix/install</uri> does automatically.) You + should unpack it somewhere (e.g. in <filename>/tmp</filename>), + and then run the script named <command>install</command> inside + the binary tarball: + + +<screen> +alice$ cd /tmp +alice$ tar xfj nix-1.8-x86_64-darwin.tar.bz2 +alice$ cd nix-1.8-x86_64-darwin +alice$ ./install +</screen> + </para> + + <para> + If you need to edit the multi-user installation script to use + different group ID or a different user ID range, modify the + variables set in the file named + <filename>install-multi-user</filename>. + </para> +</section> </chapter> diff --git a/doc/manual/installation/prerequisites-source.xml b/doc/manual/installation/prerequisites-source.xml index 01e9688d635f..e87d0de21ef6 100644 --- a/doc/manual/installation/prerequisites-source.xml +++ b/doc/manual/installation/prerequisites-source.xml @@ -25,11 +25,18 @@ If your distribution does not provide it, you can get it from <link xlink:href="https://www.openssl.org"/>.</para></listitem> + <listitem><para>The <literal>libbrotlienc</literal> and + <literal>libbrotlidec</literal> libraries to provide implementation + of the Brotli compression algorithm. They are available for download + from the official repository <link + xlink:href="https://github.com/google/brotli" />.</para></listitem> + <listitem><para>The bzip2 compressor program and the <literal>libbz2</literal> library. Thus you must have bzip2 installed, including development headers and libraries. If your distribution does not provide these, you can obtain bzip2 from <link - xlink:href="http://www.bzip.org/"/>.</para></listitem> + xlink:href="https://web.archive.org/web/20180624184756/http://www.bzip.org/" + />.</para></listitem> <listitem><para><literal>liblzma</literal>, which is provided by XZ Utils. If your distribution does not provide this, you can @@ -51,6 +58,10 @@ pass the flag <option>--enable-gc</option> to <command>configure</command>.</para></listitem> + <listitem><para>The <literal>boost</literal> library of version + 1.66.0 or higher. It can be obtained from the official web site + <link xlink:href="https://www.boost.org/" />.</para></listitem> + <listitem><para>The <command>xmllint</command> and <command>xsltproc</command> programs to build this manual and the man-pages. These are part of the <literal>libxml2</literal> and @@ -76,6 +87,15 @@ modify the parser or when you are building from the Git repository.</para></listitem> + <listitem><para>The <literal>libseccomp</literal> is used to provide + syscall filtering on Linux. This is an optional dependency and can + be disabled passing a <option>--disable-seccomp-sandboxing</option> + option to the <command>configure</command> script (Not recommended + unless your system doesn't support + <literal>libseccomp</literal>). To get the library, visit <link + xlink:href="https://github.com/seccomp/libseccomp" + />.</para></listitem> + </itemizedlist> </section> diff --git a/doc/manual/installation/supported-platforms.xml b/doc/manual/installation/supported-platforms.xml index 6858573ff407..3e74be49d1f7 100644 --- a/doc/manual/installation/supported-platforms.xml +++ b/doc/manual/installation/supported-platforms.xml @@ -10,7 +10,7 @@ <itemizedlist> - <listitem><para>Linux (i686, x86_64).</para></listitem> + <listitem><para>Linux (i686, x86_64, aarch64).</para></listitem> <listitem><para>macOS (x86_64).</para></listitem> diff --git a/doc/manual/installation/upgrading.xml b/doc/manual/installation/upgrading.xml new file mode 100644 index 000000000000..30670d7fec9c --- /dev/null +++ b/doc/manual/installation/upgrading.xml @@ -0,0 +1,22 @@ +<chapter xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ch-upgrading-nix"> + + <title>Upgrading Nix</title> + + <para> + Multi-user Nix users on macOS can upgrade Nix by running: + <command>sudo -i sh -c 'nix-channel --update && + nix-env -iA nixpkgs.nix && + launchctl remove org.nixos.nix-daemon && + launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist'</command> + </para> + + + <para> + Single-user installations of Nix should run this: + <command>nix-channel --update; nix-env -iA nixpkgs.nix</command> + </para> +</chapter> diff --git a/doc/manual/introduction/about-nix.xml b/doc/manual/introduction/about-nix.xml index e8c0a29753a1..c21ed34ddc74 100644 --- a/doc/manual/introduction/about-nix.xml +++ b/doc/manual/introduction/about-nix.xml @@ -262,12 +262,6 @@ xlink:href="http://nixos.org/">NixOS homepage</link>.</para> xlink:href="http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html">GNU LGPLv2.1 or (at your option) any later version</link>.</para> -<para>Nix uses the <link -xlink:href="https://github.com/arangodb/linenoise-ng">linenoise-ng -library</link>, which has the following license:</para> - -<programlisting><xi:include href="../../../src/linenoise/LICENSE" parse="text" /></programlisting> - </simplesect> diff --git a/doc/manual/manual.xml b/doc/manual/manual.xml index b408b6817727..87d9de28ab14 100644 --- a/doc/manual/manual.xml +++ b/doc/manual/manual.xml @@ -32,6 +32,7 @@ <xi:include href="introduction/introduction.xml" /> <xi:include href="installation/installation.xml" /> + <xi:include href="installation/upgrading.xml" /> <xi:include href="packages/package-management.xml" /> <xi:include href="expressions/writing-nix-expressions.xml" /> <xi:include href="advanced-topics/advanced-topics.xml" /> diff --git a/doc/manual/packages/basic-package-mgmt.xml b/doc/manual/packages/basic-package-mgmt.xml index e8d1419da093..0f21297f31b9 100644 --- a/doc/manual/packages/basic-package-mgmt.xml +++ b/doc/manual/packages/basic-package-mgmt.xml @@ -24,11 +24,11 @@ symlinks to the files of the active applications. </para> <para>Components are installed from a set of <emphasis>Nix expressions</emphasis> that tell Nix how to build those packages, including, if necessary, their dependencies. There is a collection of -Nix expressions called the Nix Package collection that contains +Nix expressions called the Nixpkgs package collection that contains packages ranging from basic development stuff such as GCC and Glibc, to end-user applications like Mozilla Firefox. (Nix is however not -tied to the Nix Package collection; you could write your own Nix -expressions based on it, or completely new ones.)</para> +tied to the Nixpkgs package collection; you could write your own Nix +expressions based on Nixpkgs, or completely new ones.)</para> <para>You can manually download the latest version of Nixpkgs from <link xlink:href='http://nixos.org/nixpkgs/download.html'/>. However, diff --git a/doc/manual/packages/s3-substituter.xml b/doc/manual/packages/s3-substituter.xml new file mode 100644 index 000000000000..e7589ffdb034 --- /dev/null +++ b/doc/manual/packages/s3-substituter.xml @@ -0,0 +1,183 @@ +<?xml version="1.0" encoding="utf-8"?> +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-s3-substituter"> + +<title>Serving a Nix store via AWS S3 or S3-compatible Service</title> + +<para>Nix has built-in support for storing and fetching store paths +from Amazon S3 and S3 compatible services. This uses the same +<emphasis>binary</emphasis> cache mechanism that Nix usually uses to +fetch prebuilt binaries from <uri>cache.nixos.org</uri>.</para> + +<para>The following options can be specified as URL parameters to +the S3 URL:</para> + +<variablelist> + <varlistentry><term><literal>profile</literal></term> + <listitem> + <para> + The name of the AWS configuration profile to use. By default + Nix will use the <literal>default</literal> profile. + </para> + </listitem> + </varlistentry> + + <varlistentry><term><literal>region</literal></term> + <listitem> + <para> + The region of the S3 bucket. <literal>us–east-1</literal> by + default. + </para> + + <para> + If your bucket is not in <literal>us–east-1</literal>, you + should always explicitly specify the region parameter. + </para> + </listitem> + </varlistentry> + + <varlistentry><term><literal>endpoint</literal></term> + <listitem> + <para> + The URL to your S3-compatible service, for when not using + Amazon S3. Do not specify this value if you're using Amazon + S3. + </para> + <note><para>This endpoint must support HTTPS and will use + path-based addressing instead of virtual host based + addressing.</para></note> + </listitem> + </varlistentry> + + <varlistentry><term><literal>scheme</literal></term> + <listitem> + <para> + The scheme used for S3 requests, <literal>https</literal> + (default) or <literal>http</literal>. This option allows you to + disable HTTPS for binary caches which don't support it. + </para> + <note><para>HTTPS should be used if the cache might contain + sensitive information.</para></note> + </listitem> + </varlistentry> +</variablelist> + +<para>In this example we will use the bucket named +<literal>example-nix-cache</literal>.</para> + +<section xml:id="ssec-s3-substituter-anonymous-reads"> + <title>Anonymous Reads to your S3-compatible binary cache</title> + + <para>If your binary cache is publicly accessible and does not + require authentication, the simplest and easiest way to use Nix with + your S3 compatible binary cache is to use the HTTP URL for that + cache.</para> + + <para>For AWS S3 the binary cache URL for example bucket will be + exactly <uri>https://example-nix-cache.s3.amazonaws.com</uri> or + <uri>s3://example-nix-cache</uri>. For S3 compatible binary caches, + consult that cache's documentation.</para> + + <para>Your bucket will need the following bucket policy:</para> + + <programlisting><![CDATA[ +{ + "Id": "DirectReads", + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowDirectReads", + "Action": [ + "s3:GetObject", + "s3:GetBucketLocation" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:s3:::example-nix-cache", + "arn:aws:s3:::example-nix-cache/*" + ], + "Principal": "*" + } + ] +} +]]></programlisting> +</section> + +<section xml:id="ssec-s3-substituter-authenticated-reads"> + <title>Authenticated Reads to your S3 binary cache</title> + + <para>For AWS S3 the binary cache URL for example bucket will be + exactly <uri>s3://example-nix-cache</uri>.</para> + + <para>Nix will use the <link + xlink:href="https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default.">default + credential provider chain</link> for authenticating requests to + Amazon S3.</para> + + <para>Nix supports authenticated reads from Amazon S3 and S3 + compatible binary caches.</para> + + <para>Your bucket will need a bucket policy allowing the desired + users to perform the <literal>s3:GetObject</literal> and + <literal>s3:GetBucketLocation</literal> action on all objects in the + bucket. The anonymous policy in <xref + linkend="ssec-s3-substituter-anonymous-reads" /> can be updated to + have a restricted <literal>Principal</literal> to support + this.</para> +</section> + + +<section xml:id="ssec-s3-substituter-authenticated-writes"> + <title>Authenticated Writes to your S3-compatible binary cache</title> + + <para>Nix support fully supports writing to Amazon S3 and S3 + compatible buckets. The binary cache URL for our example bucket will + be <uri>s3://example-nix-cache</uri>.</para> + + <para>Nix will use the <link + xlink:href="https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default.">default + credential provider chain</link> for authenticating requests to + Amazon S3.</para> + + <para>Your account will need the following IAM policy to + upload to the cache:</para> + + <programlisting><![CDATA[ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "UploadToCache", + "Effect": "Allow", + "Action": [ + "s3:AbortMultipartUpload", + "s3:GetBucketLocation", + "s3:GetObject", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:ListMultipartUploadParts", + "s3:ListObjects", + "s3:PutObject" + ], + "Resource": [ + "arn:aws:s3:::example-nix-cache", + "arn:aws:s3:::example-nix-cache/*" + ] + } + ] +} +]]></programlisting> + + + <example><title>Uploading with a specific credential profile for Amazon S3</title> + <para><command>nix copy --to 's3://example-nix-cache?profile=cache-upload&region=eu-west-2' nixpkgs.hello</command></para> + </example> + + <example><title>Uploading to an S3-Compatible Binary Cache</title> + <para><command>nix copy --to 's3://example-nix-cache?profile=cache-upload&scheme=https&endpoint=minio.example.com' nixpkgs.hello</command></para> + </example> +</section> +</section> diff --git a/doc/manual/packages/sharing-packages.xml b/doc/manual/packages/sharing-packages.xml index 8465c182ee72..bb6c52b8f8c1 100644 --- a/doc/manual/packages/sharing-packages.xml +++ b/doc/manual/packages/sharing-packages.xml @@ -15,5 +15,6 @@ packages between machines.</para> <xi:include href="binary-cache-substituter.xml" /> <xi:include href="copy-closure.xml" /> <xi:include href="ssh-substituter.xml" /> +<xi:include href="s3-substituter.xml" /> </chapter> diff --git a/doc/manual/release-notes/release-notes.xml b/doc/manual/release-notes/release-notes.xml index b8392a647af9..e8ff586fa43f 100644 --- a/doc/manual/release-notes/release-notes.xml +++ b/doc/manual/release-notes/release-notes.xml @@ -12,6 +12,8 @@ </partintro> --> +<xi:include href="rl-2.2.xml" /> +<xi:include href="rl-2.1.xml" /> <xi:include href="rl-2.0.xml" /> <xi:include href="rl-1.11.10.xml" /> <xi:include href="rl-1.11.xml" /> diff --git a/doc/manual/release-notes/rl-2.1.xml b/doc/manual/release-notes/rl-2.1.xml new file mode 100644 index 000000000000..16c243fc191a --- /dev/null +++ b/doc/manual/release-notes/rl-2.1.xml @@ -0,0 +1,133 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-relnotes-2.1"> + +<title>Release 2.1 (2018-09-02)</title> + +<para>This is primarily a bug fix release. It also reduces memory +consumption in certain situations. In addition, it has the following +new features:</para> + +<itemizedlist> + + <listitem> + <para>The Nix installer will no longer default to the Multi-User + installation for macOS. You can still <link + linkend="sect-multi-user-installation">instruct the installer to + run in multi-user mode</link>. + </para> + </listitem> + + <listitem> + <para>The Nix installer now supports performing a Multi-User + installation for Linux computers which are running systemd. You + can <link + linkend="sect-multi-user-installation">select a Multi-User installation</link> by passing the + <option>--daemon</option> flag to the installer: <command>sh <(curl + https://nixos.org/nix/install) --daemon</command>. + </para> + + <para>The multi-user installer cannot handle systems with SELinux. + If your system has SELinux enabled, you can <link + linkend="sect-single-user-installation">force the installer to run + in single-user mode</link>.</para> + </listitem> + + <listitem> + <para>New builtin functions: + <literal>builtins.bitAnd</literal>, + <literal>builtins.bitOr</literal>, + <literal>builtins.bitXor</literal>, + <literal>builtins.fromTOML</literal>, + <literal>builtins.concatMap</literal>, + <literal>builtins.mapAttrs</literal>. + </para> + </listitem> + + <listitem> + <para>The S3 binary cache store now supports uploading NARs larger + than 5 GiB.</para> + </listitem> + + <listitem> + <para>The S3 binary cache store now supports uploading to + S3-compatible services with the <literal>endpoint</literal> + option.</para> + </listitem> + + <listitem> + <para>The flag <option>--fallback</option> is no longer required + to recover from disappeared NARs in binary caches.</para> + </listitem> + + <listitem> + <para><command>nix-daemon</command> now respects + <option>--store</option>.</para> + </listitem> + + <listitem> + <para><command>nix run</command> now respects + <varname>nix-support/propagated-user-env-packages</varname>.</para> + </listitem> + +</itemizedlist> + +<para>This release has contributions from + +Adrien Devresse, +Aleksandr Pashkov, +Alexandre Esteves, +Amine Chikhaoui, +Andrew Dunham, +Asad Saeeduddin, +aszlig, +Ben Challenor, +Ben Gamari, +Benjamin Hipple, +Bogdan Seniuc, +Corey O'Connor, +Daiderd Jordan, +Daniel Peebles, +Daniel Poelzleithner, +Danylo Hlynskyi, +Dmitry Kalinkin, +Domen Kožar, +Doug Beardsley, +Eelco Dolstra, +Erik Arvstedt, +Félix Baylac-Jacqué, +Gleb Peregud, +Graham Christensen, +Guillaume Maudoux, +Ivan Kozik, +John Arnold, +Justin Humm, +Linus Heckemann, +Lorenzo Manacorda, +Matthew Justin Bauer, +Matthew O'Gorman, +Maximilian Bosch, +Michael Bishop, +Michael Fiano, +Michael Mercier, +Michael Raskin, +Michael Weiss, +Nicolas Dudebout, +Peter Simons, +Ryan Trinkle, +Samuel Dionne-Riel, +Sean Seefried, +Shea Levy, +Symphorien Gibol, +Tim Engler, +Tim Sears, +Tuomas Tynkkynen, +volth, +Will Dietz, +Yorick van Pelt and +zimbatm. +</para> + +</section> diff --git a/doc/manual/release-notes/rl-2.2.xml b/doc/manual/release-notes/rl-2.2.xml new file mode 100644 index 000000000000..d29eb87e82c8 --- /dev/null +++ b/doc/manual/release-notes/rl-2.2.xml @@ -0,0 +1,143 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-relnotes-2.2"> + +<title>Release 2.2 (2019-01-11)</title> + +<para>This is primarily a bug fix release. It also has the following +changes:</para> + +<itemizedlist> + + <listitem> + <para>In derivations that use structured attributes (i.e. that + specify set the <varname>__structuredAttrs</varname> attribute to + <literal>true</literal> to cause all attributes to be passed to + the builder in JSON format), you can now specify closure checks + per output, e.g.: + +<programlisting> +outputChecks."out" = { + # The closure of 'out' must not be larger than 256 MiB. + maxClosureSize = 256 * 1024 * 1024; + + # It must not refer to C compiler or to the 'dev' output. + disallowedRequisites = [ stdenv.cc "dev" ]; +}; + +outputChecks."dev" = { + # The 'dev' output must not be larger than 128 KiB. + maxSize = 128 * 1024; +}; +</programlisting> + + </para> + </listitem> + + + <listitem> + <para>The derivation attribute + <varname>requiredSystemFeatures</varname> is now enforced for + local builds, and not just to route builds to remote builders. + The supported features of a machine can be specified through the + configuration setting <varname>system-features</varname>.</para> + + <para>By default, <varname>system-features</varname> includes + <literal>kvm</literal> if <filename>/dev/kvm</filename> + exists. For compatibility, it also includes the pseudo-features + <literal>nixos-test</literal>, <literal>benchmark</literal> and + <literal>big-parallel</literal> which are used by Nixpkgs to route + builds to particular Hydra build machines.</para> + + </listitem> + + <listitem> + <para>Sandbox builds are now enabled by default on Linux.</para> + </listitem> + + <listitem> + <para>The new command <command>nix doctor</command> shows + potential issues with your Nix installation.</para> + </listitem> + + <listitem> + <para>The <literal>fetchGit</literal> builtin function now uses a + caching scheme that puts different remote repositories in distinct + local repositories, rather than a single shared repository. This + may require more disk space but is faster.</para> + </listitem> + + <listitem> + <para>The <literal>dirOf</literal> builtin function now works on + relative paths.</para> + </listitem> + + <listitem> + <para>Nix now supports <link + xlink:href="https://www.w3.org/TR/SRI/">SRI hashes</link>, + allowing the hash algorithm and hash to be specified in a single + string. For example, you can write: + +<programlisting> +import <nix/fetchurl.nix> { + url = https://nixos.org/releases/nix/nix-2.1.3/nix-2.1.3.tar.xz; + hash = "sha256-XSLa0FjVyADWWhFfkZ2iKTjFDda6mMXjoYMXLRSYQKQ="; +}; +</programlisting> + + instead of + +<programlisting> +import <nix/fetchurl.nix> { + url = https://nixos.org/releases/nix/nix-2.1.3/nix-2.1.3.tar.xz; + sha256 = "5d22dad058d5c800d65a115f919da22938c50dd6ba98c5e3a183172d149840a4"; +}; +</programlisting> + + </para> + + <para>In fixed-output derivations, the + <varname>outputHashAlgo</varname> attribute is no longer mandatory + if <varname>outputHash</varname> specifies the hash.</para> + + <para><command>nix hash-file</command> and <command>nix + hash-path</command> now print hashes in SRI format by + default. They also use SHA-256 by default instead of SHA-512 + because that's what we use most of the time in Nixpkgs.</para> + </listitem> + + <listitem> + <para>Integers are now 64 bits on all platforms.</para> + </listitem> + + <listitem> + <para>The evaluator now prints profiling statistics (enabled via + the <envar>NIX_SHOW_STATS</envar> and + <envar>NIX_COUNT_CALLS</envar> environment variables) in JSON + format.</para> + </listitem> + + <listitem> + <para>The option <option>--xml</option> in <command>nix-store + --query</command> has been removed. Instead, there now is an + option <option>--graphml</option> to output the dependency graph + in GraphML format.</para> + </listitem> + + <listitem> + <para>All <filename>nix-*</filename> commands are now symlinks to + <filename>nix</filename>. This saves a bit of disk space.</para> + </listitem> + + <listitem> + <para><command>nix repl</command> now uses + <literal>libeditline</literal> or + <literal>libreadline</literal>.</para> + </listitem> + +</itemizedlist> + +</section> + diff --git a/local.mk b/local.mk index 5d7e0fb2e428..4b380176f2ec 100644 --- a/local.mk +++ b/local.mk @@ -6,7 +6,7 @@ dist-files += configure config.h.in nix.spec perl/configure clean-files += Makefile.config -GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr +GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr -I src/nix $(foreach i, config.h $(call rwildcard, src/lib*, *.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index aa7633a709c1..1cdf5ed16dcd 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -67,24 +67,31 @@ sub downloadFile { } my $sha256_expected = $buildInfo->{buildproducts}->{$productNr}->{sha256hash} or die; - my $sha256_actual = `nix hash-file --type sha256 '$dstFile'`; + my $sha256_actual = `nix hash-file --base16 --type sha256 '$dstFile'`; chomp $sha256_actual; if ($sha256_expected ne $sha256_actual) { - print STDERR "file $dstFile is corrupt\n"; + print STDERR "file $dstFile is corrupt, got $sha256_actual, expected $sha256_expected\n"; exit 1; } write_file("$dstFile.sha256", $sha256_expected); + if (! -e "$dstFile.asc") { + system("gpg2 --detach-sign --armor $dstFile") == 0 or die "unable to sign $dstFile\n"; + } + return ($dstFile, $sha256_expected); } downloadFile("tarball", "2"); # .tar.bz2 my ($tarball, $tarballHash) = downloadFile("tarball", "3"); # .tar.xz -my ($tarball_i686_linux, $tarball_i686_linux_hash) = downloadFile("binaryTarball.i686-linux", "1"); -my ($tarball_x86_64_linux, $tarball_x86_64_linux_hash) = downloadFile("binaryTarball.x86_64-linux", "1"); -my ($tarball_aarch64_linux, $tarball_aarch64_linux_hash) = downloadFile("binaryTarball.aarch64-linux", "1"); -my ($tarball_x86_64_darwin, $tarball_x86_64_darwin_hash) = downloadFile("binaryTarball.x86_64-darwin", "1"); +downloadFile("binaryTarball.i686-linux", "1"); +downloadFile("binaryTarball.x86_64-linux", "1"); +downloadFile("binaryTarball.aarch64-linux", "1"); +downloadFile("binaryTarball.x86_64-darwin", "1"); +downloadFile("installerScript", "1"); + +exit if $version =~ /pre/; # Update Nixpkgs in a very hacky way. system("cd $nixpkgsDir && git pull") == 0 or die; @@ -144,17 +151,6 @@ system("cd $siteDir && git pull") == 0 or die; write_file("$siteDir/nix-release.tt", "[%-\n" . "latestNixVersion = \"$version\"\n" . - "nix_hash_i686_linux = \"$tarball_i686_linux_hash\"\n" . - "nix_hash_x86_64_linux = \"$tarball_x86_64_linux_hash\"\n" . - "nix_hash_aarch64_linux = \"$tarball_aarch64_linux_hash\"\n" . - "nix_hash_x86_64_darwin = \"$tarball_x86_64_darwin_hash\"\n" . "-%]\n"); -system("cd $siteDir && nix-shell --run 'make nix/install nix/install.sig'") == 0 or die; - -copy("$siteDir/nix/install", "$siteDir/nix/install-$version") or die; -copy("$siteDir/nix/install.sig", "$siteDir/nix/install-$version.sig") or die; - -system("cd $siteDir && git add nix/install-$version nix/install-$version.sig") == 0 or die; - system("cd $siteDir && git commit -a -m 'Nix $version released'") == 0 or die; diff --git a/misc/docker/Dockerfile b/misc/docker/Dockerfile deleted file mode 100644 index 2f8e3dd7a679..000000000000 --- a/misc/docker/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -FROM alpine - -# Enable HTTPS support in wget. -RUN apk add --update openssl - -# Download Nix and install it into the system. -RUN wget https://nixos.org/releases/nix/nix-2.0/nix-2.0-x86_64-linux.tar.bz2 \ - && echo "6312837aee33306cdbb351b75ba1638b89d21b30f0caf0346f9a742425f197ee nix-2.0-x86_64-linux.tar.bz2" | sha256sum -c \ - && tar xjf nix-*-x86_64-linux.tar.bz2 \ - && addgroup -g 30000 -S nixbld \ - && for i in $(seq 1 30); do adduser -S -D -h /var/empty -g "Nix build user $i" -u $((30000 + i)) -G nixbld nixbld$i ; done \ - && mkdir -m 0755 /nix && USER=root sh nix-*-x86_64-linux/install \ - && ln -s /nix/var/nix/profiles/default/etc/profile.d/nix.sh /etc/profile.d/ \ - && rm -r /nix-*-x86_64-linux \ - && rm -r /var/cache/apk/* - -ONBUILD ENV \ - ENV=/etc/profile \ - PATH=/nix/var/nix/profiles/default/bin:/nix/var/nix/profiles/default/sbin:/bin:/sbin:/usr/bin:/usr/sbin \ - GIT_SSL_CAINFO=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt \ - NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt - -ENV \ - ENV=/etc/profile \ - PATH=/nix/var/nix/profiles/default/bin:/nix/var/nix/profiles/default/sbin:/bin:/sbin:/usr/bin:/usr/sbin \ - GIT_SSL_CAINFO=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt \ - NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt \ - NIX_PATH=/nix/var/nix/profiles/per-user/root/channels diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in index 549619a57d56..92ed12fa5090 100644 --- a/misc/launchd/org.nixos.nix-daemon.plist.in +++ b/misc/launchd/org.nixos.nix-daemon.plist.in @@ -2,6 +2,11 @@ <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> <plist version="1.0"> <dict> + <key>EnvironmentVariables</key> + <dict> + <key>OBJC_DISABLE_INITIALIZE_FORK_SAFETY</key> + <string>YES</string> + </dict> <key>Label</key> <string>org.nixos.nix-daemon</string> <key>KeepAlive</key> diff --git a/mk/libraries.mk b/mk/libraries.mk index 14c95fa91cf6..3953446cba32 100644 --- a/mk/libraries.mk +++ b/mk/libraries.mk @@ -125,7 +125,7 @@ define build-library $(1)_PATH := $$(_d)/$$($(1)_NAME).a $$($(1)_PATH): $$($(1)_OBJS) | $$(_d)/ - $(trace-ar) ar crs $$@ $$? + $(trace-ar) $(AR) crs $$@ $$? $(1)_LDFLAGS_USE += $$($(1)_PATH) $$($(1)_LDFLAGS) diff --git a/nix.spec.in b/nix.spec.in index d962bcc857b9..477768c6a68c 100644 --- a/nix.spec.in +++ b/nix.spec.in @@ -3,33 +3,47 @@ %global nixbld_user "nix-builder-" %global nixbld_group "nixbld" +# NOTE: BUILD on EL7 requires +# - Centos / RHEL7 software collection repository +# yum install centos-release-scl +# +# - Recent boost backport +# curl https://copr.fedorainfracloud.org/coprs/whosthere/boost/repo/epel-7/whosthere-boost-epel-7.repo -o /etc/yum.repos.d/whosthere-boost-epel-7.repo +# + +# Disable documentation generation +# necessary on some platforms +%bcond_without docgen + Summary: The Nix software deployment system Name: nix Version: @PACKAGE_VERSION@ Release: 2%{?dist} License: LGPLv2+ -%if 0%{?rhel} && 0%{?rhel} < 7 Group: Applications/System -%endif URL: http://nixos.org/ Source0: %{name}-%{version}.tar.bz2 -%if 0%{?el5} -BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) -%endif + Requires: curl Requires: bzip2 Requires: gzip Requires: xz -Requires: libseccomp -Requires: boost-context +BuildRequires: bison +BuildRequires: boost-devel >= 1.60 BuildRequires: bzip2-devel -BuildRequires: sqlite-devel + +# for RHEL <= 7, we need software collections for a C++14 compatible compatible compiler +%if 0%{?rhel} +BuildRequires: devtoolset-7-gcc +BuildRequires: devtoolset-7-gcc-c++ +%endif + +BuildRequires: flex BuildRequires: libcurl-devel BuildRequires: libseccomp-devel -BuildRequires: boost-devel - -# Hack to make that shitty RPM scanning hack shut up. -Provides: perl(Nix::SSH) +BuildRequires: openssl-devel +BuildRequires: sqlite-devel +BuildRequires: xz-devel %description Nix is a purely functional package manager. It allows multiple @@ -41,9 +55,6 @@ it can be used equally well under other Unix systems. %package devel Summary: Development files for %{name} -%if 0%{?rhel} && 0%{?rhel} < 7 -Group: Development/Libraries -%endif Requires: %{name}%{?_isa} = %{version}-%{release} %description devel @@ -53,9 +64,6 @@ developing applications that use %{name}. %package doc Summary: Documentation files for %{name} -%if 0%{?rhel} && 0%{?rhel} < 7 -Group: Documentation -%endif BuildArch: noarch Requires: %{name} = %{version}-%{release} @@ -67,20 +75,25 @@ The %{name}-doc package contains documentation files for %{name}. %build +%if 0%{?rhel} +source /opt/rh/devtoolset-7/enable +%endif extraFlags= # - override docdir so large documentation files are owned by the # -doc subpackage # - set localstatedir by hand to the preferred nix value %configure --localstatedir=/nix/var \ + %{!?without_docgen:--disable-doc-gen} \ --docdir=%{_defaultdocdir}/%{name}-doc-%{version} \ $extraFlags -make -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES +make V=1 %{?_smp_mflags} %install -%if 0%{?el5} -rm -rf $RPM_BUILD_ROOT +%if 0%{?rhel} +source /opt/rh/devtoolset-7/enable %endif + make DESTDIR=$RPM_BUILD_ROOT install find $RPM_BUILD_ROOT -name '*.la' -exec rm -f {} ';' @@ -130,6 +143,7 @@ systemctl start nix-daemon.socket %endif %files +%license COPYING %{_bindir}/nix* %{_libdir}/*.so %{_prefix}/libexec/* @@ -138,9 +152,11 @@ systemctl start nix-daemon.socket %{_prefix}/lib/systemd/system/nix-daemon.service %endif %{_datadir}/nix -%{_mandir}/man1/*.1* -%{_mandir}/man5/*.5* -%{_mandir}/man8/*.8* +#%if ! %{without docgen} +#%{_mandir}/man1/*.1* +#%{_mandir}/man5/*.5* +#%{_mandir}/man8/*.8* +#%endif %config(noreplace) %{_sysconfdir}/profile.d/nix.sh %config(noreplace) %{_sysconfdir}/profile.d/nix-daemon.sh /nix @@ -149,6 +165,9 @@ systemctl start nix-daemon.socket %{_includedir}/nix %{_prefix}/lib/pkgconfig/*.pc -%files doc -%docdir %{_defaultdocdir}/%{name}-doc-%{version} -%{_defaultdocdir}/%{name}-doc-%{version} + +#%if ! %{without docgen} +#%files doc +#%docdir %{_defaultdocdir}/%{name}-doc-%{version} +#%{_defaultdocdir}/%{name}-doc-%{version} +#%endif diff --git a/perl/configure.ac b/perl/configure.ac index 9f49db4d2816..966700695ff5 100644 --- a/perl/configure.ac +++ b/perl/configure.ac @@ -1,4 +1,4 @@ -AC_INIT(nix-perl, m4_esyscmd([bash -c "echo -n $(cat ../version)$VERSION_SUFFIX"])) +AC_INIT(nix-perl, m4_esyscmd([bash -c "echo -n $(cat ../.version)$VERSION_SUFFIX"])) AC_CONFIG_SRCDIR(MANIFEST) AC_CONFIG_AUX_DIR(../config) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index bbfb2934315b..ce553bb53ebc 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -27,7 +27,7 @@ static ref<Store> store() static std::shared_ptr<Store> _store; if (!_store) { try { - settings.loadConfFile(); + loadConfFile(); settings.lockCPU = false; _store = openStore(); } catch (Error & e) { diff --git a/release-common.nix b/release-common.nix index 0c12bc7ceb38..4c5565985267 100644 --- a/release-common.nix +++ b/release-common.nix @@ -30,7 +30,7 @@ rec { }); configureFlags = - [ "--disable-init-state" + [ "--enable-gc" ] ++ lib.optionals stdenv.isLinux [ "--with-sandbox-shell=${sh}/bin/busybox" @@ -49,7 +49,7 @@ rec { buildDeps = [ curl - bzip2 xz brotli + bzip2 xz brotli editline openssl pkgconfig sqlite boehmgc boost @@ -57,13 +57,20 @@ rec { git mercurial ] - ++ lib.optional stdenv.isLinux libseccomp + ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal] ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) - (aws-sdk-cpp.override { - apis = ["s3"]; + ((aws-sdk-cpp.override { + apis = ["s3" "transfer"]; customMemoryManagement = false; - }); + }).overrideDerivation (args: { + /* + patches = args.patches or [] ++ [ (fetchpatch { + url = https://github.com/edolstra/aws-sdk-cpp/commit/3e07e1f1aae41b4c8b340735ff9e8c735f0c063f.patch; + sha256 = "1pij0v449p166f9l29x7ppzk8j7g9k9mp15ilh5qxp29c7fnvxy2"; + }) ]; + */ + })); perlDeps = [ perl diff --git a/release.nix b/release.nix index 9778b2db68a3..78b39108f85e 100644 --- a/release.nix +++ b/release.nix @@ -1,5 +1,5 @@ { nix ? builtins.fetchGit ./. -, nixpkgs ? builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-18.03"; } +, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.03.tar.gz , officialRelease ? false , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ] }: @@ -18,7 +18,7 @@ let releaseTools.sourceTarball { name = "nix-tarball"; - version = builtins.readFile ./version; + version = builtins.readFile ./.version; versionSuffix = if officialRelease then "" else "pre${toString nix.revCount}_${nix.shortRev}"; src = nix; inherit officialRelease; @@ -67,6 +67,14 @@ let buildInputs = buildDeps; + preConfigure = + # Copy libboost_context so we don't get all of Boost in our closure. + # https://github.com/NixOS/nixpkgs/issues/45462 + '' + mkdir -p $out/lib + cp ${boost}/lib/libboost_context* $out/lib + ''; + configureFlags = configureFlags ++ [ "--sysconfdir=/etc" ]; @@ -74,8 +82,6 @@ let makeFlags = "profiledir=$(out)/etc/profile.d"; - preBuild = "unset NIX_INDENT_MAKE"; - installFlags = "sysconfdir=$(out)/etc"; doInstallCheck = true; @@ -103,8 +109,6 @@ let enableParallelBuilding = true; postUnpack = "sourceRoot=$sourceRoot/perl"; - - preBuild = "unset NIX_INDENT_MAKE"; }); @@ -189,10 +193,6 @@ let buildInputs = buildDeps; - configureFlags = '' - --disable-init-state - ''; - dontInstall = false; doInstallCheck = true; @@ -206,14 +206,14 @@ let }; - rpm_fedora27x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora27x86_64) [ ]; + #rpm_fedora27x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora27x86_64) [ ]; #deb_debian8i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian8i386) [ "libsodium-dev" ] [ "libsodium13" ]; #deb_debian8x86_64 = makeDeb_x86_64 (diskImageFunsFun: diskImageFunsFun.debian8x86_64) [ "libsodium-dev" ] [ "libsodium13" ]; - deb_ubuntu1710i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1710i386) [ ] [ "libsodium18" ]; - deb_ubuntu1710x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1710x86_64) [ ] [ "libsodium18" "libboost-context1.62.0" ]; + #deb_ubuntu1710i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1710i386) [ ] [ "libsodium18" ]; + #deb_ubuntu1710x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1710x86_64) [ ] [ "libsodium18" "libboost-context1.62.0" ]; # System tests. @@ -241,6 +241,7 @@ let { diskImage = vmTools.diskImages.ubuntu1204x86_64; } '' + set -x useradd -m alice su - alice -c 'tar xf ${binaryTarball.x86_64-linux}/*.tar.*' mkdir /dest-nix @@ -249,6 +250,17 @@ let su - alice -c '_NIX_INSTALLER_TEST=1 ./nix-*/install' su - alice -c 'nix-store --verify' su - alice -c 'PAGER= nix-store -qR ${build.x86_64-linux}' + + # Check whether 'nix upgrade-nix' works. + cat > /tmp/paths.nix <<EOF + { + x86_64-linux = "${build.x86_64-linux}"; + } + EOF + su - alice -c 'nix upgrade-nix -vvv --nix-store-paths-url file:///tmp/paths.nix' + (! [ -L /home/alice/.profile-1-link ]) + su - alice -c 'PAGER= nix-store -qR ${build.x86_64-linux}' + mkdir -p $out/nix-support touch $out/nix-support/hydra-build-products umount /nix @@ -266,14 +278,32 @@ let pkgs.runCommand "eval-nixos" { buildInputs = [ build.x86_64-linux ]; } '' export NIX_STATE_DIR=$TMPDIR - nix-store --init - nix-instantiate ${nixpkgs}/nixos/release-combined.nix -A tested --dry-run + nix-instantiate ${nixpkgs}/nixos/release-combined.nix -A tested --dry-run \ + --arg nixpkgs '{ outPath = ${nixpkgs}; revCount = 123; shortRev = "abcdefgh"; }' touch $out ''; + installerScript = + pkgs.runCommand "installer-script" + { buildInputs = [ build.x86_64-linux ]; + } + '' + mkdir -p $out/nix-support + + substitute ${./scripts/install.in} $out/install \ + ${pkgs.lib.concatMapStrings + (system: "--replace '@binaryTarball_${system}@' $(nix hash-file --base16 --type sha256 ${binaryTarball.${system}}/*.tar.bz2) ") + [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ] + } \ + --replace '@nixVersion@' ${build.x86_64-linux.src.version} + + echo "file installer $out/install" >> $out/nix-support/hydra-build-products + ''; + + # Aggregate job containing the release-critical jobs. release = pkgs.releaseTools.aggregate { name = "nix-${tarball.version}"; @@ -283,14 +313,17 @@ let build.i686-linux build.x86_64-darwin build.x86_64-linux + build.aarch64-linux binaryTarball.i686-linux binaryTarball.x86_64-darwin binaryTarball.x86_64-linux + binaryTarball.aarch64-linux tests.remoteBuilds tests.nix-copy-closure tests.binaryTarball tests.evalNixpkgs tests.evalNixOS + installerScript ]; }; @@ -310,7 +343,7 @@ let src = jobs.tarball; diskImage = (diskImageFun vmTools.diskImageFuns) { extraPackages = - [ "sqlite" "sqlite-devel" "bzip2-devel" "libcurl-devel" "openssl-devel" "xz-devel" "libseccomp-devel" "libsodium-devel" "boost-devel" ] + [ "sqlite" "sqlite-devel" "bzip2-devel" "libcurl-devel" "openssl-devel" "xz-devel" "libseccomp-devel" "libsodium-devel" "boost-devel" "bison" "flex" ] ++ extraPackages; }; # At most 2047MB can be simulated in qemu-system-i386 memSize = 2047; diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 5f6542355e0c..c89e18b5116d 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -240,10 +240,16 @@ EOF } trap finish_fail EXIT +channel_update_failed=0 function finish_success { finish_cleanup ok "Alright! We're done!" + if [ "x$channel_update_failed" = x1 ]; then + echo "" + echo "But fetching the nixpkgs channel failed. (Are you offline?)" + echo "To try again later, run \"sudo -i nix-channel --update nixpkgs\"." + fi cat <<EOF Before Nix will work in your existing shells, you'll need to close @@ -674,9 +680,6 @@ $NIX_INSTALLED_NIX. EOF fi - _sudo "to initialize the Nix Database" \ - $NIX_INSTALLED_NIX/bin/nix-store --init - cat ./.reginfo \ | _sudo "to load data for the first time in to the Nix Database" \ "$NIX_INSTALLED_NIX/bin/nix-store" --load-db @@ -727,11 +730,19 @@ setup_default_profile() { _sudo "to installing a bootstrapping Nix in to the default Profile" \ HOME="$ROOT_HOME" "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_NIX" - _sudo "to installing a bootstrapping SSL certificate just for Nix in to the default Profile" \ - HOME="$ROOT_HOME" "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_CACERT" + if [ -z "${NIX_SSL_CERT_FILE:-}" ] || ! [ -f "${NIX_SSL_CERT_FILE:-}" ]; then + _sudo "to installing a bootstrapping SSL certificate just for Nix in to the default Profile" \ + HOME="$ROOT_HOME" "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_CACERT" + export NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt + fi + # Have to explicitly pass NIX_SSL_CERT_FILE as part of the sudo call, + # otherwise it will be lost in environments where sudo doesn't pass + # all the environment variables by default. _sudo "to update the default channel in the default profile" \ - HOME="$ROOT_HOME" NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs + HOME="$ROOT_HOME" NIX_SSL_CERT_FILE="$NIX_SSL_CERT_FILE" "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs \ + || channel_update_failed=1 + } @@ -741,7 +752,6 @@ build-users-group = $NIX_BUILD_GROUP_NAME max-jobs = $NIX_USER_COUNT cores = 1 -sandbox = false EOF _sudo "to place the default nix daemon configuration (part 2)" \ install -m 0664 "$SCRATCH/nix.conf" /etc/nix/nix.conf diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index 8416fb9673b5..fc999d336d1f 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -22,20 +22,50 @@ if [ -z "$HOME" ]; then exit 1 fi -# macOS support for 10.10 or higher +# macOS support for 10.12.6 or higher if [ "$(uname -s)" = "Darwin" ]; then - if [ $(($(sw_vers -productVersion | cut -d '.' -f 2))) -lt 10 ]; then - echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.10 or higher" + macos_major=$(sw_vers -productVersion | cut -d '.' -f 2) + macos_minor=$(sw_vers -productVersion | cut -d '.' -f 3) + if [ "$macos_major" -lt 12 ] || { [ "$macos_major" -eq 12 ] && [ "$macos_minor" -lt 6 ]; }; then + echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.12.6 or higher" exit 1 fi - - printf '\e[1;31mSwitching to the Daemon-based Installer\e[0m\n' - exec "$self/install-multi-user" - exit 0 fi -# Linux & Systemd support -if [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then +# Determine if we could use the multi-user installer or not +if [ "$(uname -s)" = "Darwin" ]; then + echo "Note: a multi-user installation is possible. See https://nixos.org/nix/manual/#sect-multi-user-installation" >&2 +elif [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then + echo "Note: a multi-user installation is possible. See https://nixos.org/nix/manual/#sect-multi-user-installation" >&2 +fi + +INSTALL_MODE=no-daemon +# Trivially handle the --daemon / --no-daemon options +if [ "x${1:-}" = "x--no-daemon" ]; then + INSTALL_MODE=no-daemon +elif [ "x${1:-}" = "x--daemon" ]; then + INSTALL_MODE=daemon +elif [ "x${1:-}" != "x" ]; then + ( + echo "Nix Installer [--daemon|--no-daemon]" + + echo "Choose installation method." + echo "" + echo " --daemon: Installs and configures a background daemon that manages the store," + echo " providing multi-user support and better isolation for local builds." + echo " Both for security and reproducibility, this method is recommended if" + echo " supported on your platform." + echo " See https://nixos.org/nix/manual/#sect-multi-user-installation" + echo "" + echo " --no-daemon: Simple, single-user installation that does not require root and is" + echo " trivial to uninstall." + echo " (default)" + echo "" + ) >&2 + exit +fi + +if [ "$INSTALL_MODE" = "daemon" ]; then printf '\e[1;31mSwitching to the Daemon-based Installer\e[0m\n' exec "$self/install-multi-user" exit 0 @@ -81,12 +111,6 @@ for i in $(cd "$self/store" >/dev/null && echo ./*); do done echo "" >&2 -echo "initialising Nix database..." >&2 -if ! $nix/bin/nix-store --init; then - echo "$0: failed to initialize the Nix database" >&2 - exit 1 -fi - if ! "$nix/bin/nix-store" --load-db < "$self/.reginfo"; then echo "$0: unable to register valid paths" >&2 exit 1 @@ -110,7 +134,10 @@ if ! $nix/bin/nix-channel --list | grep -q "^nixpkgs "; then $nix/bin/nix-channel --add https://nixos.org/channels/nixpkgs-unstable fi if [ -z "$_NIX_INSTALLER_TEST" ]; then - $nix/bin/nix-channel --update nixpkgs + if ! $nix/bin/nix-channel --update nixpkgs; then + echo "Fetching the nixpkgs channel failed. (Are you offline?)" + echo "To try again later, run \"nix-channel --update nixpkgs\"." + fi fi added= diff --git a/scripts/install.in b/scripts/install.in new file mode 100644 index 000000000000..7bff7b216d9e --- /dev/null +++ b/scripts/install.in @@ -0,0 +1,67 @@ +#!/bin/sh + +# This script installs the Nix package manager on your system by +# downloading a binary distribution and running its installer script +# (which in turn creates and populates /nix). + +{ # Prevent execution if this script was only partially downloaded +oops() { + echo "$0:" "$@" >&2 + exit 1 +} + +tmpDir="$(mktemp -d -t nix-binary-tarball-unpack.XXXXXXXXXX || \ + oops "Can't create temporary directory for downloading the Nix binary tarball")" +cleanup() { + rm -rf "$tmpDir" +} +trap cleanup EXIT INT QUIT TERM + +require_util() { + type "$1" > /dev/null 2>&1 || command -v "$1" > /dev/null 2>&1 || + oops "you do not have '$1' installed, which I need to $2" +} + +case "$(uname -s).$(uname -m)" in + Linux.x86_64) system=x86_64-linux; hash=@binaryTarball_x86_64-linux@;; + Linux.i?86) system=i686-linux; hash=@binaryTarball_i686-linux@;; + Linux.aarch64) system=aarch64-linux; hash=@binaryTarball_aarch64-linux@;; + Darwin.x86_64) system=x86_64-darwin; hash=@binaryTarball_x86_64-darwin@;; + *) oops "sorry, there is no binary distribution of Nix for your platform";; +esac + +url="https://nixos.org/releases/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.bz2" + +tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.bz2")" + +require_util curl "download the binary tarball" +require_util bzcat "decompress the binary tarball" +require_util tar "unpack the binary tarball" + +echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..." +curl -L "$url" -o "$tarball" || oops "failed to download '$url'" + +if type sha256sum > /dev/null 2>&1; then + hash2="$(sha256sum -b "$tarball" | cut -c1-64)" +elif type shasum > /dev/null 2>&1; then + hash2="$(shasum -a 256 -b "$tarball" | cut -c1-64)" +elif type openssl > /dev/null 2>&1; then + hash2="$(openssl dgst -r -sha256 "$tarball" | cut -c1-64)" +else + oops "cannot verify the SHA-256 hash of '$url'; you need one of 'shasum', 'sha256sum', or 'openssl'" +fi + +if [ "$hash" != "$hash2" ]; then + oops "SHA-256 hash mismatch in '$url'; expected $hash, got $hash2" +fi + +unpack=$tmpDir/unpack +mkdir -p "$unpack" +< "$tarball" bzcat | tar -xf - -C "$unpack" || oops "failed to unpack '$url'" + +script=$(echo "$unpack"/*/install) + +[ -e "$script" ] || oops "installation script is missing from the binary tarball!" +"$script" "$@" + +} # End of wrapping diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index 43c7156062de..23da5e8559eb 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -1,13 +1,7 @@ # Only execute this file once per shell. -if [ -n "$__ETC_PROFILE_NIX_SOURCED" ]; then return; fi +if [ -n "${__ETC_PROFILE_NIX_SOURCED:-}" ]; then return; fi __ETC_PROFILE_NIX_SOURCED=1 -# Set up secure multi-user builds: non-root users build through the -# Nix daemon. -if [ "$USER" != root -o ! -w @localstatedir@/nix/db ]; then - export NIX_REMOTE=daemon -fi - export NIX_USER_PROFILE_DIR="@localstatedir@/nix/profiles/per-user/$USER" export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile" @@ -49,6 +43,26 @@ if test -w $HOME; then fi fi -export NIX_SSL_CERT_FILE="@localstatedir@/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" -export NIX_PATH="@localstatedir@/nix/profiles/per-user/root/channels" -export PATH="$HOME/.nix-profile/bin:$HOME/.nix-profile/lib/kde4/libexec:@localstatedir@/nix/profiles/default/bin:@localstatedir@/nix/profiles/default:@localstatedir@/nix/profiles/default/lib/kde4/libexec:$PATH" + +# Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. +if [ ! -z "${NIX_SSL_CERT_FILE:-}" ]; then + : # Allow users to override the NIX_SSL_CERT_FILE +elif [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch + export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt +elif [ -e /etc/ssl/ca-bundle.pem ]; then # openSUSE Tumbleweed + export NIX_SSL_CERT_FILE=/etc/ssl/ca-bundle.pem +elif [ -e /etc/ssl/certs/ca-bundle.crt ]; then # Old NixOS + export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt +elif [ -e /etc/pki/tls/certs/ca-bundle.crt ]; then # Fedora, CentOS + export NIX_SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt +else + # Fall back to what is in the nix profiles, favouring whatever is defined last. + for i in $NIX_PROFILES; do + if [ -e $i/etc/ssl/certs/ca-bundle.crt ]; then + export NIX_SSL_CERT_FILE=$i/etc/ssl/certs/ca-bundle.crt + fi + done +fi + +export NIX_PATH="nixpkgs=@localstatedir@/nix/profiles/per-user/root/channels/nixpkgs:@localstatedir@/nix/profiles/per-user/root/channels" +export PATH="$HOME/.nix-profile/bin:@localstatedir@/nix/profiles/default/bin:$PATH" diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index a5f52274fc70..85f1d6e5dae2 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -51,14 +51,13 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then unset __nix_defexpr fi - # Append ~/.nix-defexpr/channels/nixpkgs to $NIX_PATH so that - # <nixpkgs> paths work when the user has fetched the Nixpkgs - # channel. - export NIX_PATH="${NIX_PATH:+$NIX_PATH:}nixpkgs=$HOME/.nix-defexpr/channels/nixpkgs" + # Append ~/.nix-defexpr/channels to $NIX_PATH so that <nixpkgs> + # paths work when the user has fetched the Nixpkgs channel. + export NIX_PATH=${NIX_PATH:+$NIX_PATH:}$HOME/.nix-defexpr/channels # Set up environment. # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix - NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_USER_PROFILE_DIR" + export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile" # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. if [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch @@ -75,10 +74,10 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then export NIX_SSL_CERT_FILE="$NIX_LINK/etc/ca-bundle.crt" fi - if [ -n "${MANPATH}" ]; then + if [ -n "${MANPATH-}" ]; then export MANPATH="$NIX_LINK/share/man:$MANPATH" fi export PATH="$NIX_LINK/bin:$__savedpath" - unset __savedpath NIX_LINK NIX_USER_PROFILE_DIR NIX_PROFILES + unset __savedpath NIX_LINK NIX_USER_PROFILE_DIR fi diff --git a/shell.nix b/shell.nix index c04bcd151309..8167f87a2929 100644 --- a/shell.nix +++ b/shell.nix @@ -1,6 +1,6 @@ { useClang ? false }: -with import (builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-18.03"; }) {}; +with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.03.tar.gz) {}; with import ./release-common.nix { inherit pkgs; }; diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 9cd01bb61bf5..279ae62f69cc 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -17,6 +17,7 @@ #include "store-api.hh" #include "derivations.hh" #include "local-store.hh" +#include "legacy.hh" using namespace nix; using std::cin; @@ -37,11 +38,15 @@ static AutoCloseFD openSlotLock(const Machine & m, unsigned long long slot) return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true); } -int main (int argc, char * * argv) -{ - return handleExceptions(argv[0], [&]() { - initNix(); +static bool allSupportedLocally(const std::set<std::string>& requiredFeatures) { + for (auto & feature : requiredFeatures) + if (!settings.systemFeatures.get().count(feature)) return false; + return true; +} +static int _main(int argc, char * * argv) +{ + { logger = makeJSONLogger(*logger); /* Ensure we don't get any SSH passphrase or host key popups. */ @@ -80,7 +85,7 @@ int main (int argc, char * * argv) if (machines.empty()) { std::cerr << "# decline-permanently\n"; - return; + return 0; } string drvPath; @@ -90,15 +95,18 @@ int main (int argc, char * * argv) try { auto s = readString(source); - if (s != "try") return; - } catch (EndOfFile &) { return; } + if (s != "try") return 0; + } catch (EndOfFile &) { return 0; } auto amWilling = readInt(source); auto neededSystem = readString(source); source >> drvPath; auto requiredFeatures = readStrings<std::set<std::string>>(source); - auto canBuildLocally = amWilling && (neededSystem == settings.thisSystem); + auto canBuildLocally = amWilling + && ( neededSystem == settings.thisSystem + || settings.extraPlatforms.get().count(neededSystem) > 0) + && allSupportedLocally(requiredFeatures); /* Error ignored here, will be caught later */ mkdir(currentLoad.c_str(), 0777); @@ -251,6 +259,8 @@ connected: copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, NoSubstitute); } - return; - }); + return 0; + } } + +static RegisterLegacyCommand s1("build-remote", _main); diff --git a/src/build-remote/local.mk b/src/build-remote/local.mk deleted file mode 100644 index 64368a43ff73..000000000000 --- a/src/build-remote/local.mk +++ /dev/null @@ -1,9 +0,0 @@ -programs += build-remote - -build-remote_DIR := $(d) - -build-remote_INSTALL_DIR := $(libexecdir)/nix - -build-remote_LIBS = libmain libutil libformat libstore - -build-remote_SOURCES := $(d)/build-remote.cc diff --git a/src/cpptoml/LICENSE b/src/cpptoml/LICENSE new file mode 100644 index 000000000000..8802c4fa5a36 --- /dev/null +++ b/src/cpptoml/LICENSE @@ -0,0 +1,18 @@ +Copyright (c) 2014 Chase Geigle + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/src/cpptoml/cpptoml.h b/src/cpptoml/cpptoml.h new file mode 100644 index 000000000000..07de010b1520 --- /dev/null +++ b/src/cpptoml/cpptoml.h @@ -0,0 +1,3457 @@ +/** + * @file cpptoml.h + * @author Chase Geigle + * @date May 2013 + */ + +#ifndef _CPPTOML_H_ +#define _CPPTOML_H_ + +#include <algorithm> +#include <cassert> +#include <clocale> +#include <cstdint> +#include <cstring> +#include <fstream> +#include <iomanip> +#include <map> +#include <memory> +#include <sstream> +#include <stdexcept> +#include <string> +#include <unordered_map> +#include <vector> + +#if __cplusplus > 201103L +#define CPPTOML_DEPRECATED(reason) [[deprecated(reason)]] +#elif defined(__clang__) +#define CPPTOML_DEPRECATED(reason) __attribute__((deprecated(reason))) +#elif defined(__GNUG__) +#define CPPTOML_DEPRECATED(reason) __attribute__((deprecated)) +#elif defined(_MSC_VER) +#if _MSC_VER < 1910 +#define CPPTOML_DEPRECATED(reason) __declspec(deprecated) +#else +#define CPPTOML_DEPRECATED(reason) [[deprecated(reason)]] +#endif +#endif + +namespace cpptoml +{ +class writer; // forward declaration +class base; // forward declaration +#if defined(CPPTOML_USE_MAP) +// a std::map will ensure that entries a sorted, albeit at a slight +// performance penalty relative to the (default) unordered_map +using string_to_base_map = std::map<std::string, std::shared_ptr<base>>; +#else +// by default an unordered_map is used for best performance as the +// toml specification does not require entries to be sorted +using string_to_base_map + = std::unordered_map<std::string, std::shared_ptr<base>>; +#endif + +// if defined, `base` will retain type information in form of an enum class +// such that static_cast can be used instead of dynamic_cast +// #define CPPTOML_NO_RTTI + +template <class T> +class option +{ + public: + option() : empty_{true} + { + // nothing + } + + option(T value) : empty_{false}, value_(std::move(value)) + { + // nothing + } + + explicit operator bool() const + { + return !empty_; + } + + const T& operator*() const + { + return value_; + } + + const T* operator->() const + { + return &value_; + } + + const T& value_or(const T& alternative) const + { + if (!empty_) + return value_; + return alternative; + } + + private: + bool empty_; + T value_; +}; + +struct local_date +{ + int year = 0; + int month = 0; + int day = 0; +}; + +struct local_time +{ + int hour = 0; + int minute = 0; + int second = 0; + int microsecond = 0; +}; + +struct zone_offset +{ + int hour_offset = 0; + int minute_offset = 0; +}; + +struct local_datetime : local_date, local_time +{ +}; + +struct offset_datetime : local_datetime, zone_offset +{ + static inline struct offset_datetime from_zoned(const struct tm& t) + { + offset_datetime dt; + dt.year = t.tm_year + 1900; + dt.month = t.tm_mon + 1; + dt.day = t.tm_mday; + dt.hour = t.tm_hour; + dt.minute = t.tm_min; + dt.second = t.tm_sec; + + char buf[16]; + strftime(buf, 16, "%z", &t); + + int offset = std::stoi(buf); + dt.hour_offset = offset / 100; + dt.minute_offset = offset % 100; + return dt; + } + + CPPTOML_DEPRECATED("from_local has been renamed to from_zoned") + static inline struct offset_datetime from_local(const struct tm& t) + { + return from_zoned(t); + } + + static inline struct offset_datetime from_utc(const struct tm& t) + { + offset_datetime dt; + dt.year = t.tm_year + 1900; + dt.month = t.tm_mon + 1; + dt.day = t.tm_mday; + dt.hour = t.tm_hour; + dt.minute = t.tm_min; + dt.second = t.tm_sec; + return dt; + } +}; + +CPPTOML_DEPRECATED("datetime has been renamed to offset_datetime") +typedef offset_datetime datetime; + +class fill_guard +{ + public: + fill_guard(std::ostream& os) : os_(os), fill_{os.fill()} + { + // nothing + } + + ~fill_guard() + { + os_.fill(fill_); + } + + private: + std::ostream& os_; + std::ostream::char_type fill_; +}; + +inline std::ostream& operator<<(std::ostream& os, const local_date& dt) +{ + fill_guard g{os}; + os.fill('0'); + + using std::setw; + os << setw(4) << dt.year << "-" << setw(2) << dt.month << "-" << setw(2) + << dt.day; + + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const local_time& ltime) +{ + fill_guard g{os}; + os.fill('0'); + + using std::setw; + os << setw(2) << ltime.hour << ":" << setw(2) << ltime.minute << ":" + << setw(2) << ltime.second; + + if (ltime.microsecond > 0) + { + os << "."; + int power = 100000; + for (int curr_us = ltime.microsecond; curr_us; power /= 10) + { + auto num = curr_us / power; + os << num; + curr_us -= num * power; + } + } + + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const zone_offset& zo) +{ + fill_guard g{os}; + os.fill('0'); + + using std::setw; + + if (zo.hour_offset != 0 || zo.minute_offset != 0) + { + if (zo.hour_offset > 0) + { + os << "+"; + } + else + { + os << "-"; + } + os << setw(2) << std::abs(zo.hour_offset) << ":" << setw(2) + << std::abs(zo.minute_offset); + } + else + { + os << "Z"; + } + + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const local_datetime& dt) +{ + return os << static_cast<const local_date&>(dt) << "T" + << static_cast<const local_time&>(dt); +} + +inline std::ostream& operator<<(std::ostream& os, const offset_datetime& dt) +{ + return os << static_cast<const local_datetime&>(dt) + << static_cast<const zone_offset&>(dt); +} + +template <class T, class... Ts> +struct is_one_of; + +template <class T, class V> +struct is_one_of<T, V> : std::is_same<T, V> +{ +}; + +template <class T, class V, class... Ts> +struct is_one_of<T, V, Ts...> +{ + const static bool value + = std::is_same<T, V>::value || is_one_of<T, Ts...>::value; +}; + +template <class T> +class value; + +template <class T> +struct valid_value + : is_one_of<T, std::string, int64_t, double, bool, local_date, local_time, + local_datetime, offset_datetime> +{ +}; + +template <class T, class Enable = void> +struct value_traits; + +template <class T> +struct valid_value_or_string_convertible +{ + + const static bool value = valid_value<typename std::decay<T>::type>::value + || std::is_convertible<T, std::string>::value; +}; + +template <class T> +struct value_traits<T, typename std:: + enable_if<valid_value_or_string_convertible<T>:: + value>::type> +{ + using value_type = typename std:: + conditional<valid_value<typename std::decay<T>::type>::value, + typename std::decay<T>::type, std::string>::type; + + using type = value<value_type>; + + static value_type construct(T&& val) + { + return value_type(val); + } +}; + +template <class T> +struct value_traits<T, + typename std:: + enable_if<!valid_value_or_string_convertible<T>::value + && std::is_floating_point< + typename std::decay<T>::type>::value>:: + type> +{ + using value_type = typename std::decay<T>::type; + + using type = value<double>; + + static value_type construct(T&& val) + { + return value_type(val); + } +}; + +template <class T> +struct value_traits<T, + typename std:: + enable_if<!valid_value_or_string_convertible<T>::value + && std::is_signed<typename std::decay<T>:: + type>::value>::type> +{ + using value_type = int64_t; + + using type = value<int64_t>; + + static value_type construct(T&& val) + { + if (val < (std::numeric_limits<int64_t>::min)()) + throw std::underflow_error{"constructed value cannot be " + "represented by a 64-bit signed " + "integer"}; + + if (val > (std::numeric_limits<int64_t>::max)()) + throw std::overflow_error{"constructed value cannot be represented " + "by a 64-bit signed integer"}; + + return static_cast<int64_t>(val); + } +}; + +template <class T> +struct value_traits<T, + typename std:: + enable_if<!valid_value_or_string_convertible<T>::value + && std::is_unsigned<typename std::decay<T>:: + type>::value>::type> +{ + using value_type = int64_t; + + using type = value<int64_t>; + + static value_type construct(T&& val) + { + if (val > static_cast<uint64_t>((std::numeric_limits<int64_t>::max)())) + throw std::overflow_error{"constructed value cannot be represented " + "by a 64-bit signed integer"}; + + return static_cast<int64_t>(val); + } +}; + +class array; +class table; +class table_array; + +template <class T> +struct array_of_trait +{ + using return_type = option<std::vector<T>>; +}; + +template <> +struct array_of_trait<array> +{ + using return_type = option<std::vector<std::shared_ptr<array>>>; +}; + +template <class T> +inline std::shared_ptr<typename value_traits<T>::type> make_value(T&& val); +inline std::shared_ptr<array> make_array(); +template <class T> +inline std::shared_ptr<T> make_element(); +inline std::shared_ptr<table> make_table(); +inline std::shared_ptr<table_array> make_table_array(); + +#if defined(CPPTOML_NO_RTTI) +/// Base type used to store underlying data type explicitly if RTTI is disabled +enum class base_type +{ + NONE, + STRING, + LOCAL_TIME, + LOCAL_DATE, + LOCAL_DATETIME, + OFFSET_DATETIME, + INT, + FLOAT, + BOOL, + TABLE, + ARRAY, + TABLE_ARRAY +}; + +/// Type traits class to convert C++ types to enum member +template <class T> +struct base_type_traits; + +template <> +struct base_type_traits<std::string> +{ + static const base_type type = base_type::STRING; +}; + +template <> +struct base_type_traits<local_time> +{ + static const base_type type = base_type::LOCAL_TIME; +}; + +template <> +struct base_type_traits<local_date> +{ + static const base_type type = base_type::LOCAL_DATE; +}; + +template <> +struct base_type_traits<local_datetime> +{ + static const base_type type = base_type::LOCAL_DATETIME; +}; + +template <> +struct base_type_traits<offset_datetime> +{ + static const base_type type = base_type::OFFSET_DATETIME; +}; + +template <> +struct base_type_traits<int64_t> +{ + static const base_type type = base_type::INT; +}; + +template <> +struct base_type_traits<double> +{ + static const base_type type = base_type::FLOAT; +}; + +template <> +struct base_type_traits<bool> +{ + static const base_type type = base_type::BOOL; +}; + +template <> +struct base_type_traits<table> +{ + static const base_type type = base_type::TABLE; +}; + +template <> +struct base_type_traits<array> +{ + static const base_type type = base_type::ARRAY; +}; + +template <> +struct base_type_traits<table_array> +{ + static const base_type type = base_type::TABLE_ARRAY; +}; +#endif + +/** + * A generic base TOML value used for type erasure. + */ +class base : public std::enable_shared_from_this<base> +{ + public: + virtual ~base() = default; + + virtual std::shared_ptr<base> clone() const = 0; + + /** + * Determines if the given TOML element is a value. + */ + virtual bool is_value() const + { + return false; + } + + /** + * Determines if the given TOML element is a table. + */ + virtual bool is_table() const + { + return false; + } + + /** + * Converts the TOML element into a table. + */ + std::shared_ptr<table> as_table() + { + if (is_table()) + return std::static_pointer_cast<table>(shared_from_this()); + return nullptr; + } + /** + * Determines if the TOML element is an array of "leaf" elements. + */ + virtual bool is_array() const + { + return false; + } + + /** + * Converts the TOML element to an array. + */ + std::shared_ptr<array> as_array() + { + if (is_array()) + return std::static_pointer_cast<array>(shared_from_this()); + return nullptr; + } + + /** + * Determines if the given TOML element is an array of tables. + */ + virtual bool is_table_array() const + { + return false; + } + + /** + * Converts the TOML element into a table array. + */ + std::shared_ptr<table_array> as_table_array() + { + if (is_table_array()) + return std::static_pointer_cast<table_array>(shared_from_this()); + return nullptr; + } + + /** + * Attempts to coerce the TOML element into a concrete TOML value + * of type T. + */ + template <class T> + std::shared_ptr<value<T>> as(); + + template <class T> + std::shared_ptr<const value<T>> as() const; + + template <class Visitor, class... Args> + void accept(Visitor&& visitor, Args&&... args) const; + +#if defined(CPPTOML_NO_RTTI) + base_type type() const + { + return type_; + } + + protected: + base(const base_type t) : type_(t) + { + // nothing + } + + private: + const base_type type_ = base_type::NONE; + +#else + protected: + base() + { + // nothing + } +#endif +}; + +/** + * A concrete TOML value representing the "leaves" of the "tree". + */ +template <class T> +class value : public base +{ + struct make_shared_enabler + { + // nothing; this is a private key accessible only to friends + }; + + template <class U> + friend std::shared_ptr<typename value_traits<U>::type> + cpptoml::make_value(U&& val); + + public: + static_assert(valid_value<T>::value, "invalid value type"); + + std::shared_ptr<base> clone() const override; + + value(const make_shared_enabler&, const T& val) : value(val) + { + // nothing; note that users cannot actually invoke this function + // because they lack access to the make_shared_enabler. + } + + bool is_value() const override + { + return true; + } + + /** + * Gets the data associated with this value. + */ + T& get() + { + return data_; + } + + /** + * Gets the data associated with this value. Const version. + */ + const T& get() const + { + return data_; + } + + private: + T data_; + + /** + * Constructs a value from the given data. + */ +#if defined(CPPTOML_NO_RTTI) + value(const T& val) : base(base_type_traits<T>::type), data_(val) + { + } +#else + value(const T& val) : data_(val) + { + } +#endif + + value(const value& val) = delete; + value& operator=(const value& val) = delete; +}; + +template <class T> +std::shared_ptr<typename value_traits<T>::type> make_value(T&& val) +{ + using value_type = typename value_traits<T>::type; + using enabler = typename value_type::make_shared_enabler; + return std::make_shared<value_type>( + enabler{}, value_traits<T>::construct(std::forward<T>(val))); +} + +template <class T> +inline std::shared_ptr<value<T>> base::as() +{ +#if defined(CPPTOML_NO_RTTI) + if (type() == base_type_traits<T>::type) + return std::static_pointer_cast<value<T>>(shared_from_this()); + else + return nullptr; +#else + return std::dynamic_pointer_cast<value<T>>(shared_from_this()); +#endif +} + +// special case value<double> to allow getting an integer parameter as a +// double value +template <> +inline std::shared_ptr<value<double>> base::as() +{ +#if defined(CPPTOML_NO_RTTI) + if (type() == base_type::FLOAT) + return std::static_pointer_cast<value<double>>(shared_from_this()); + + if (type() == base_type::INT) + { + auto v = std::static_pointer_cast<value<int64_t>>(shared_from_this()); + return make_value<double>(static_cast<double>(v->get()));; + } +#else + if (auto v = std::dynamic_pointer_cast<value<double>>(shared_from_this())) + return v; + + if (auto v = std::dynamic_pointer_cast<value<int64_t>>(shared_from_this())) + return make_value<double>(static_cast<double>(v->get())); +#endif + + return nullptr; +} + +template <class T> +inline std::shared_ptr<const value<T>> base::as() const +{ +#if defined(CPPTOML_NO_RTTI) + if (type() == base_type_traits<T>::type) + return std::static_pointer_cast<const value<T>>(shared_from_this()); + else + return nullptr; +#else + return std::dynamic_pointer_cast<const value<T>>(shared_from_this()); +#endif +} + +// special case value<double> to allow getting an integer parameter as a +// double value +template <> +inline std::shared_ptr<const value<double>> base::as() const +{ +#if defined(CPPTOML_NO_RTTI) + if (type() == base_type::FLOAT) + return std::static_pointer_cast<const value<double>>(shared_from_this()); + + if (type() == base_type::INT) + { + auto v = as<int64_t>(); + // the below has to be a non-const value<double> due to a bug in + // libc++: https://llvm.org/bugs/show_bug.cgi?id=18843 + return make_value<double>(static_cast<double>(v->get())); + } +#else + if (auto v + = std::dynamic_pointer_cast<const value<double>>(shared_from_this())) + return v; + + if (auto v = as<int64_t>()) + { + // the below has to be a non-const value<double> due to a bug in + // libc++: https://llvm.org/bugs/show_bug.cgi?id=18843 + return make_value<double>(static_cast<double>(v->get())); + } +#endif + + return nullptr; +} + +/** + * Exception class for array insertion errors. + */ +class array_exception : public std::runtime_error +{ + public: + array_exception(const std::string& err) : std::runtime_error{err} + { + } +}; + +class array : public base +{ + public: + friend std::shared_ptr<array> make_array(); + + std::shared_ptr<base> clone() const override; + + virtual bool is_array() const override + { + return true; + } + + using size_type = std::size_t; + + /** + * arrays can be iterated over + */ + using iterator = std::vector<std::shared_ptr<base>>::iterator; + + /** + * arrays can be iterated over. Const version. + */ + using const_iterator = std::vector<std::shared_ptr<base>>::const_iterator; + + iterator begin() + { + return values_.begin(); + } + + const_iterator begin() const + { + return values_.begin(); + } + + iterator end() + { + return values_.end(); + } + + const_iterator end() const + { + return values_.end(); + } + + /** + * Obtains the array (vector) of base values. + */ + std::vector<std::shared_ptr<base>>& get() + { + return values_; + } + + /** + * Obtains the array (vector) of base values. Const version. + */ + const std::vector<std::shared_ptr<base>>& get() const + { + return values_; + } + + std::shared_ptr<base> at(size_t idx) const + { + return values_.at(idx); + } + + /** + * Obtains an array of value<T>s. Note that elements may be + * nullptr if they cannot be converted to a value<T>. + */ + template <class T> + std::vector<std::shared_ptr<value<T>>> array_of() const + { + std::vector<std::shared_ptr<value<T>>> result(values_.size()); + + std::transform(values_.begin(), values_.end(), result.begin(), + [&](std::shared_ptr<base> v) { return v->as<T>(); }); + + return result; + } + + /** + * Obtains a option<vector<T>>. The option will be empty if the array + * contains values that are not of type T. + */ + template <class T> + inline typename array_of_trait<T>::return_type get_array_of() const + { + std::vector<T> result; + result.reserve(values_.size()); + + for (const auto& val : values_) + { + if (auto v = val->as<T>()) + result.push_back(v->get()); + else + return {}; + } + + return {std::move(result)}; + } + + /** + * Obtains an array of arrays. Note that elements may be nullptr + * if they cannot be converted to a array. + */ + std::vector<std::shared_ptr<array>> nested_array() const + { + std::vector<std::shared_ptr<array>> result(values_.size()); + + std::transform(values_.begin(), values_.end(), result.begin(), + [&](std::shared_ptr<base> v) -> std::shared_ptr<array> { + if (v->is_array()) + return std::static_pointer_cast<array>(v); + return std::shared_ptr<array>{}; + }); + + return result; + } + + /** + * Add a value to the end of the array + */ + template <class T> + void push_back(const std::shared_ptr<value<T>>& val) + { + if (values_.empty() || values_[0]->as<T>()) + { + values_.push_back(val); + } + else + { + throw array_exception{"Arrays must be homogenous."}; + } + } + + /** + * Add an array to the end of the array + */ + void push_back(const std::shared_ptr<array>& val) + { + if (values_.empty() || values_[0]->is_array()) + { + values_.push_back(val); + } + else + { + throw array_exception{"Arrays must be homogenous."}; + } + } + + /** + * Convenience function for adding a simple element to the end + * of the array. + */ + template <class T> + void push_back(T&& val, typename value_traits<T>::type* = 0) + { + push_back(make_value(std::forward<T>(val))); + } + + /** + * Insert a value into the array + */ + template <class T> + iterator insert(iterator position, const std::shared_ptr<value<T>>& value) + { + if (values_.empty() || values_[0]->as<T>()) + { + return values_.insert(position, value); + } + else + { + throw array_exception{"Arrays must be homogenous."}; + } + } + + /** + * Insert an array into the array + */ + iterator insert(iterator position, const std::shared_ptr<array>& value) + { + if (values_.empty() || values_[0]->is_array()) + { + return values_.insert(position, value); + } + else + { + throw array_exception{"Arrays must be homogenous."}; + } + } + + /** + * Convenience function for inserting a simple element in the array + */ + template <class T> + iterator insert(iterator position, T&& val, + typename value_traits<T>::type* = 0) + { + return insert(position, make_value(std::forward<T>(val))); + } + + /** + * Erase an element from the array + */ + iterator erase(iterator position) + { + return values_.erase(position); + } + + /** + * Clear the array + */ + void clear() + { + values_.clear(); + } + + /** + * Reserve space for n values. + */ + void reserve(size_type n) + { + values_.reserve(n); + } + + private: +#if defined(CPPTOML_NO_RTTI) + array() : base(base_type::ARRAY) + { + // empty + } +#else + array() = default; +#endif + + template <class InputIterator> + array(InputIterator begin, InputIterator end) : values_{begin, end} + { + // nothing + } + + array(const array& obj) = delete; + array& operator=(const array& obj) = delete; + + std::vector<std::shared_ptr<base>> values_; +}; + +inline std::shared_ptr<array> make_array() +{ + struct make_shared_enabler : public array + { + make_shared_enabler() + { + // nothing + } + }; + + return std::make_shared<make_shared_enabler>(); +} + +template <> +inline std::shared_ptr<array> make_element<array>() +{ + return make_array(); +} + +/** + * Obtains a option<vector<T>>. The option will be empty if the array + * contains values that are not of type T. + */ +template <> +inline typename array_of_trait<array>::return_type +array::get_array_of<array>() const +{ + std::vector<std::shared_ptr<array>> result; + result.reserve(values_.size()); + + for (const auto& val : values_) + { + if (auto v = val->as_array()) + result.push_back(v); + else + return {}; + } + + return {std::move(result)}; +} + +class table; + +class table_array : public base +{ + friend class table; + friend std::shared_ptr<table_array> make_table_array(); + + public: + std::shared_ptr<base> clone() const override; + + using size_type = std::size_t; + + /** + * arrays can be iterated over + */ + using iterator = std::vector<std::shared_ptr<table>>::iterator; + + /** + * arrays can be iterated over. Const version. + */ + using const_iterator = std::vector<std::shared_ptr<table>>::const_iterator; + + iterator begin() + { + return array_.begin(); + } + + const_iterator begin() const + { + return array_.begin(); + } + + iterator end() + { + return array_.end(); + } + + const_iterator end() const + { + return array_.end(); + } + + virtual bool is_table_array() const override + { + return true; + } + + std::vector<std::shared_ptr<table>>& get() + { + return array_; + } + + const std::vector<std::shared_ptr<table>>& get() const + { + return array_; + } + + /** + * Add a table to the end of the array + */ + void push_back(const std::shared_ptr<table>& val) + { + array_.push_back(val); + } + + /** + * Insert a table into the array + */ + iterator insert(iterator position, const std::shared_ptr<table>& value) + { + return array_.insert(position, value); + } + + /** + * Erase an element from the array + */ + iterator erase(iterator position) + { + return array_.erase(position); + } + + /** + * Clear the array + */ + void clear() + { + array_.clear(); + } + + /** + * Reserve space for n tables. + */ + void reserve(size_type n) + { + array_.reserve(n); + } + + private: +#if defined(CPPTOML_NO_RTTI) + table_array() : base(base_type::TABLE_ARRAY) + { + // nothing + } +#else + table_array() + { + // nothing + } +#endif + + table_array(const table_array& obj) = delete; + table_array& operator=(const table_array& rhs) = delete; + + std::vector<std::shared_ptr<table>> array_; +}; + +inline std::shared_ptr<table_array> make_table_array() +{ + struct make_shared_enabler : public table_array + { + make_shared_enabler() + { + // nothing + } + }; + + return std::make_shared<make_shared_enabler>(); +} + +template <> +inline std::shared_ptr<table_array> make_element<table_array>() +{ + return make_table_array(); +} + +// The below are overloads for fetching specific value types out of a value +// where special casting behavior (like bounds checking) is desired + +template <class T> +typename std::enable_if<!std::is_floating_point<T>::value + && std::is_signed<T>::value, + option<T>>::type +get_impl(const std::shared_ptr<base>& elem) +{ + if (auto v = elem->as<int64_t>()) + { + if (v->get() < (std::numeric_limits<T>::min)()) + throw std::underflow_error{ + "T cannot represent the value requested in get"}; + + if (v->get() > (std::numeric_limits<T>::max)()) + throw std::overflow_error{ + "T cannot represent the value requested in get"}; + + return {static_cast<T>(v->get())}; + } + else + { + return {}; + } +} + +template <class T> +typename std::enable_if<!std::is_same<T, bool>::value + && std::is_unsigned<T>::value, + option<T>>::type +get_impl(const std::shared_ptr<base>& elem) +{ + if (auto v = elem->as<int64_t>()) + { + if (v->get() < 0) + throw std::underflow_error{"T cannot store negative value in get"}; + + if (static_cast<uint64_t>(v->get()) > (std::numeric_limits<T>::max)()) + throw std::overflow_error{ + "T cannot represent the value requested in get"}; + + return {static_cast<T>(v->get())}; + } + else + { + return {}; + } +} + +template <class T> +typename std::enable_if<!std::is_integral<T>::value + || std::is_same<T, bool>::value, + option<T>>::type +get_impl(const std::shared_ptr<base>& elem) +{ + if (auto v = elem->as<T>()) + { + return {v->get()}; + } + else + { + return {}; + } +} + +/** + * Represents a TOML keytable. + */ +class table : public base +{ + public: + friend class table_array; + friend std::shared_ptr<table> make_table(); + + std::shared_ptr<base> clone() const override; + + /** + * tables can be iterated over. + */ + using iterator = string_to_base_map::iterator; + + /** + * tables can be iterated over. Const version. + */ + using const_iterator = string_to_base_map::const_iterator; + + iterator begin() + { + return map_.begin(); + } + + const_iterator begin() const + { + return map_.begin(); + } + + iterator end() + { + return map_.end(); + } + + const_iterator end() const + { + return map_.end(); + } + + bool is_table() const override + { + return true; + } + + bool empty() const + { + return map_.empty(); + } + + /** + * Determines if this key table contains the given key. + */ + bool contains(const std::string& key) const + { + return map_.find(key) != map_.end(); + } + + /** + * Determines if this key table contains the given key. Will + * resolve "qualified keys". Qualified keys are the full access + * path separated with dots like "grandparent.parent.child". + */ + bool contains_qualified(const std::string& key) const + { + return resolve_qualified(key); + } + + /** + * Obtains the base for a given key. + * @throw std::out_of_range if the key does not exist + */ + std::shared_ptr<base> get(const std::string& key) const + { + return map_.at(key); + } + + /** + * Obtains the base for a given key. Will resolve "qualified + * keys". Qualified keys are the full access path separated with + * dots like "grandparent.parent.child". + * + * @throw std::out_of_range if the key does not exist + */ + std::shared_ptr<base> get_qualified(const std::string& key) const + { + std::shared_ptr<base> p; + resolve_qualified(key, &p); + return p; + } + + /** + * Obtains a table for a given key, if possible. + */ + std::shared_ptr<table> get_table(const std::string& key) const + { + if (contains(key) && get(key)->is_table()) + return std::static_pointer_cast<table>(get(key)); + return nullptr; + } + + /** + * Obtains a table for a given key, if possible. Will resolve + * "qualified keys". + */ + std::shared_ptr<table> get_table_qualified(const std::string& key) const + { + if (contains_qualified(key) && get_qualified(key)->is_table()) + return std::static_pointer_cast<table>(get_qualified(key)); + return nullptr; + } + + /** + * Obtains an array for a given key. + */ + std::shared_ptr<array> get_array(const std::string& key) const + { + if (!contains(key)) + return nullptr; + return get(key)->as_array(); + } + + /** + * Obtains an array for a given key. Will resolve "qualified keys". + */ + std::shared_ptr<array> get_array_qualified(const std::string& key) const + { + if (!contains_qualified(key)) + return nullptr; + return get_qualified(key)->as_array(); + } + + /** + * Obtains a table_array for a given key, if possible. + */ + std::shared_ptr<table_array> get_table_array(const std::string& key) const + { + if (!contains(key)) + return nullptr; + return get(key)->as_table_array(); + } + + /** + * Obtains a table_array for a given key, if possible. Will resolve + * "qualified keys". + */ + std::shared_ptr<table_array> + get_table_array_qualified(const std::string& key) const + { + if (!contains_qualified(key)) + return nullptr; + return get_qualified(key)->as_table_array(); + } + + /** + * Helper function that attempts to get a value corresponding + * to the template parameter from a given key. + */ + template <class T> + option<T> get_as(const std::string& key) const + { + try + { + return get_impl<T>(get(key)); + } + catch (const std::out_of_range&) + { + return {}; + } + } + + /** + * Helper function that attempts to get a value corresponding + * to the template parameter from a given key. Will resolve "qualified + * keys". + */ + template <class T> + option<T> get_qualified_as(const std::string& key) const + { + try + { + return get_impl<T>(get_qualified(key)); + } + catch (const std::out_of_range&) + { + return {}; + } + } + + /** + * Helper function that attempts to get an array of values of a given + * type corresponding to the template parameter for a given key. + * + * If the key doesn't exist, doesn't exist as an array type, or one or + * more keys inside the array type are not of type T, an empty option + * is returned. Otherwise, an option containing a vector of the values + * is returned. + */ + template <class T> + inline typename array_of_trait<T>::return_type + get_array_of(const std::string& key) const + { + if (auto v = get_array(key)) + { + std::vector<T> result; + result.reserve(v->get().size()); + + for (const auto& b : v->get()) + { + if (auto val = b->as<T>()) + result.push_back(val->get()); + else + return {}; + } + return {std::move(result)}; + } + + return {}; + } + + /** + * Helper function that attempts to get an array of values of a given + * type corresponding to the template parameter for a given key. Will + * resolve "qualified keys". + * + * If the key doesn't exist, doesn't exist as an array type, or one or + * more keys inside the array type are not of type T, an empty option + * is returned. Otherwise, an option containing a vector of the values + * is returned. + */ + template <class T> + inline typename array_of_trait<T>::return_type + get_qualified_array_of(const std::string& key) const + { + if (auto v = get_array_qualified(key)) + { + std::vector<T> result; + result.reserve(v->get().size()); + + for (const auto& b : v->get()) + { + if (auto val = b->as<T>()) + result.push_back(val->get()); + else + return {}; + } + return {std::move(result)}; + } + + return {}; + } + + /** + * Adds an element to the keytable. + */ + void insert(const std::string& key, const std::shared_ptr<base>& value) + { + map_[key] = value; + } + + /** + * Convenience shorthand for adding a simple element to the + * keytable. + */ + template <class T> + void insert(const std::string& key, T&& val, + typename value_traits<T>::type* = 0) + { + insert(key, make_value(std::forward<T>(val))); + } + + /** + * Removes an element from the table. + */ + void erase(const std::string& key) + { + map_.erase(key); + } + + private: +#if defined(CPPTOML_NO_RTTI) + table() : base(base_type::TABLE) + { + // nothing + } +#else + table() + { + // nothing + } +#endif + + table(const table& obj) = delete; + table& operator=(const table& rhs) = delete; + + std::vector<std::string> split(const std::string& value, + char separator) const + { + std::vector<std::string> result; + std::string::size_type p = 0; + std::string::size_type q; + while ((q = value.find(separator, p)) != std::string::npos) + { + result.emplace_back(value, p, q - p); + p = q + 1; + } + result.emplace_back(value, p); + return result; + } + + // If output parameter p is specified, fill it with the pointer to the + // specified entry and throw std::out_of_range if it couldn't be found. + // + // Otherwise, just return true if the entry could be found or false + // otherwise and do not throw. + bool resolve_qualified(const std::string& key, + std::shared_ptr<base>* p = nullptr) const + { + auto parts = split(key, '.'); + auto last_key = parts.back(); + parts.pop_back(); + + auto cur_table = this; + for (const auto& part : parts) + { + cur_table = cur_table->get_table(part).get(); + if (!cur_table) + { + if (!p) + return false; + + throw std::out_of_range{key + " is not a valid key"}; + } + } + + if (!p) + return cur_table->map_.count(last_key) != 0; + + *p = cur_table->map_.at(last_key); + return true; + } + + string_to_base_map map_; +}; + +/** + * Helper function that attempts to get an array of arrays for a given + * key. + * + * If the key doesn't exist, doesn't exist as an array type, or one or + * more keys inside the array type are not of type T, an empty option + * is returned. Otherwise, an option containing a vector of the values + * is returned. + */ +template <> +inline typename array_of_trait<array>::return_type +table::get_array_of<array>(const std::string& key) const +{ + if (auto v = get_array(key)) + { + std::vector<std::shared_ptr<array>> result; + result.reserve(v->get().size()); + + for (const auto& b : v->get()) + { + if (auto val = b->as_array()) + result.push_back(val); + else + return {}; + } + + return {std::move(result)}; + } + + return {}; +} + +/** + * Helper function that attempts to get an array of arrays for a given + * key. Will resolve "qualified keys". + * + * If the key doesn't exist, doesn't exist as an array type, or one or + * more keys inside the array type are not of type T, an empty option + * is returned. Otherwise, an option containing a vector of the values + * is returned. + */ +template <> +inline typename array_of_trait<array>::return_type +table::get_qualified_array_of<array>(const std::string& key) const +{ + if (auto v = get_array_qualified(key)) + { + std::vector<std::shared_ptr<array>> result; + result.reserve(v->get().size()); + + for (const auto& b : v->get()) + { + if (auto val = b->as_array()) + result.push_back(val); + else + return {}; + } + + return {std::move(result)}; + } + + return {}; +} + +std::shared_ptr<table> make_table() +{ + struct make_shared_enabler : public table + { + make_shared_enabler() + { + // nothing + } + }; + + return std::make_shared<make_shared_enabler>(); +} + +template <> +inline std::shared_ptr<table> make_element<table>() +{ + return make_table(); +} + +template <class T> +std::shared_ptr<base> value<T>::clone() const +{ + return make_value(data_); +} + +inline std::shared_ptr<base> array::clone() const +{ + auto result = make_array(); + result->reserve(values_.size()); + for (const auto& ptr : values_) + result->values_.push_back(ptr->clone()); + return result; +} + +inline std::shared_ptr<base> table_array::clone() const +{ + auto result = make_table_array(); + result->reserve(array_.size()); + for (const auto& ptr : array_) + result->array_.push_back(ptr->clone()->as_table()); + return result; +} + +inline std::shared_ptr<base> table::clone() const +{ + auto result = make_table(); + for (const auto& pr : map_) + result->insert(pr.first, pr.second->clone()); + return result; +} + +/** + * Exception class for all TOML parsing errors. + */ +class parse_exception : public std::runtime_error +{ + public: + parse_exception(const std::string& err) : std::runtime_error{err} + { + } + + parse_exception(const std::string& err, std::size_t line_number) + : std::runtime_error{err + " at line " + std::to_string(line_number)} + { + } +}; + +inline bool is_number(char c) +{ + return c >= '0' && c <= '9'; +} + +/** + * Helper object for consuming expected characters. + */ +template <class OnError> +class consumer +{ + public: + consumer(std::string::iterator& it, const std::string::iterator& end, + OnError&& on_error) + : it_(it), end_(end), on_error_(std::forward<OnError>(on_error)) + { + // nothing + } + + void operator()(char c) + { + if (it_ == end_ || *it_ != c) + on_error_(); + ++it_; + } + + template <std::size_t N> + void operator()(const char (&str)[N]) + { + std::for_each(std::begin(str), std::end(str) - 1, + [&](char c) { (*this)(c); }); + } + + int eat_digits(int len) + { + int val = 0; + for (int i = 0; i < len; ++i) + { + if (!is_number(*it_) || it_ == end_) + on_error_(); + val = 10 * val + (*it_++ - '0'); + } + return val; + } + + void error() + { + on_error_(); + } + + private: + std::string::iterator& it_; + const std::string::iterator& end_; + OnError on_error_; +}; + +template <class OnError> +consumer<OnError> make_consumer(std::string::iterator& it, + const std::string::iterator& end, + OnError&& on_error) +{ + return consumer<OnError>(it, end, std::forward<OnError>(on_error)); +} + +// replacement for std::getline to handle incorrectly line-ended files +// https://stackoverflow.com/questions/6089231/getting-std-ifstream-to-handle-lf-cr-and-crlf +namespace detail +{ +inline std::istream& getline(std::istream& input, std::string& line) +{ + line.clear(); + + std::istream::sentry sentry{input, true}; + auto sb = input.rdbuf(); + + while (true) + { + auto c = sb->sbumpc(); + if (c == '\r') + { + if (sb->sgetc() == '\n') + c = sb->sbumpc(); + } + + if (c == '\n') + return input; + + if (c == std::istream::traits_type::eof()) + { + if (line.empty()) + input.setstate(std::ios::eofbit); + return input; + } + + line.push_back(static_cast<char>(c)); + } +} +} + +/** + * The parser class. + */ +class parser +{ + public: + /** + * Parsers are constructed from streams. + */ + parser(std::istream& stream) : input_(stream) + { + // nothing + } + + parser& operator=(const parser& parser) = delete; + + /** + * Parses the stream this parser was created on until EOF. + * @throw parse_exception if there are errors in parsing + */ + std::shared_ptr<table> parse() + { + std::shared_ptr<table> root = make_table(); + + table* curr_table = root.get(); + + while (detail::getline(input_, line_)) + { + line_number_++; + auto it = line_.begin(); + auto end = line_.end(); + consume_whitespace(it, end); + if (it == end || *it == '#') + continue; + if (*it == '[') + { + curr_table = root.get(); + parse_table(it, end, curr_table); + } + else + { + parse_key_value(it, end, curr_table); + consume_whitespace(it, end); + eol_or_comment(it, end); + } + } + return root; + } + + private: +#if defined _MSC_VER + __declspec(noreturn) +#elif defined __GNUC__ + __attribute__((noreturn)) +#endif + void throw_parse_exception(const std::string& err) + { + throw parse_exception{err, line_number_}; + } + + void parse_table(std::string::iterator& it, + const std::string::iterator& end, table*& curr_table) + { + // remove the beginning keytable marker + ++it; + if (it == end) + throw_parse_exception("Unexpected end of table"); + if (*it == '[') + parse_table_array(it, end, curr_table); + else + parse_single_table(it, end, curr_table); + } + + void parse_single_table(std::string::iterator& it, + const std::string::iterator& end, + table*& curr_table) + { + if (it == end || *it == ']') + throw_parse_exception("Table name cannot be empty"); + + std::string full_table_name; + bool inserted = false; + while (it != end && *it != ']') + { + auto part = parse_key(it, end, + [](char c) { return c == '.' || c == ']'; }); + + if (part.empty()) + throw_parse_exception("Empty component of table name"); + + if (!full_table_name.empty()) + full_table_name += "."; + full_table_name += part; + + if (curr_table->contains(part)) + { + auto b = curr_table->get(part); + if (b->is_table()) + curr_table = static_cast<table*>(b.get()); + else if (b->is_table_array()) + curr_table = std::static_pointer_cast<table_array>(b) + ->get() + .back() + .get(); + else + throw_parse_exception("Key " + full_table_name + + "already exists as a value"); + } + else + { + inserted = true; + curr_table->insert(part, make_table()); + curr_table = static_cast<table*>(curr_table->get(part).get()); + } + consume_whitespace(it, end); + if (it != end && *it == '.') + ++it; + consume_whitespace(it, end); + } + + if (it == end) + throw_parse_exception( + "Unterminated table declaration; did you forget a ']'?"); + + // table already existed + if (!inserted) + { + auto is_value + = [](const std::pair<const std::string&, + const std::shared_ptr<base>&>& p) { + return p.second->is_value(); + }; + + // if there are any values, we can't add values to this table + // since it has already been defined. If there aren't any + // values, then it was implicitly created by something like + // [a.b] + if (curr_table->empty() || std::any_of(curr_table->begin(), + curr_table->end(), is_value)) + { + throw_parse_exception("Redefinition of table " + + full_table_name); + } + } + + ++it; + consume_whitespace(it, end); + eol_or_comment(it, end); + } + + void parse_table_array(std::string::iterator& it, + const std::string::iterator& end, table*& curr_table) + { + ++it; + if (it == end || *it == ']') + throw_parse_exception("Table array name cannot be empty"); + + std::string full_ta_name; + while (it != end && *it != ']') + { + auto part = parse_key(it, end, + [](char c) { return c == '.' || c == ']'; }); + + if (part.empty()) + throw_parse_exception("Empty component of table array name"); + + if (!full_ta_name.empty()) + full_ta_name += "."; + full_ta_name += part; + + consume_whitespace(it, end); + if (it != end && *it == '.') + ++it; + consume_whitespace(it, end); + + if (curr_table->contains(part)) + { + auto b = curr_table->get(part); + + // if this is the end of the table array name, add an + // element to the table array that we just looked up + if (it != end && *it == ']') + { + if (!b->is_table_array()) + throw_parse_exception("Key " + full_ta_name + + " is not a table array"); + auto v = b->as_table_array(); + v->get().push_back(make_table()); + curr_table = v->get().back().get(); + } + // otherwise, just keep traversing down the key name + else + { + if (b->is_table()) + curr_table = static_cast<table*>(b.get()); + else if (b->is_table_array()) + curr_table = std::static_pointer_cast<table_array>(b) + ->get() + .back() + .get(); + else + throw_parse_exception("Key " + full_ta_name + + " already exists as a value"); + } + } + else + { + // if this is the end of the table array name, add a new + // table array and a new table inside that array for us to + // add keys to next + if (it != end && *it == ']') + { + curr_table->insert(part, make_table_array()); + auto arr = std::static_pointer_cast<table_array>( + curr_table->get(part)); + arr->get().push_back(make_table()); + curr_table = arr->get().back().get(); + } + // otherwise, create the implicitly defined table and move + // down to it + else + { + curr_table->insert(part, make_table()); + curr_table + = static_cast<table*>(curr_table->get(part).get()); + } + } + } + + // consume the last "]]" + if (it == end) + throw_parse_exception("Unterminated table array name"); + ++it; + if (it == end) + throw_parse_exception("Unterminated table array name"); + ++it; + + consume_whitespace(it, end); + eol_or_comment(it, end); + } + + void parse_key_value(std::string::iterator& it, std::string::iterator& end, + table* curr_table) + { + auto key = parse_key(it, end, [](char c) { return c == '='; }); + if (curr_table->contains(key)) + throw_parse_exception("Key " + key + " already present"); + if (it == end || *it != '=') + throw_parse_exception("Value must follow after a '='"); + ++it; + consume_whitespace(it, end); + curr_table->insert(key, parse_value(it, end)); + consume_whitespace(it, end); + } + + template <class Function> + std::string parse_key(std::string::iterator& it, + const std::string::iterator& end, Function&& fun) + { + consume_whitespace(it, end); + if (*it == '"') + { + return parse_quoted_key(it, end); + } + else + { + auto bke = std::find_if(it, end, std::forward<Function>(fun)); + return parse_bare_key(it, bke); + } + } + + std::string parse_bare_key(std::string::iterator& it, + const std::string::iterator& end) + { + if (it == end) + { + throw_parse_exception("Bare key missing name"); + } + + auto key_end = end; + --key_end; + consume_backwards_whitespace(key_end, it); + ++key_end; + std::string key{it, key_end}; + + if (std::find(it, key_end, '#') != key_end) + { + throw_parse_exception("Bare key " + key + " cannot contain #"); + } + + if (std::find_if(it, key_end, + [](char c) { return c == ' ' || c == '\t'; }) + != key_end) + { + throw_parse_exception("Bare key " + key + + " cannot contain whitespace"); + } + + if (std::find_if(it, key_end, + [](char c) { return c == '[' || c == ']'; }) + != key_end) + { + throw_parse_exception("Bare key " + key + + " cannot contain '[' or ']'"); + } + + it = end; + return key; + } + + std::string parse_quoted_key(std::string::iterator& it, + const std::string::iterator& end) + { + return string_literal(it, end, '"'); + } + + enum class parse_type + { + STRING = 1, + LOCAL_TIME, + LOCAL_DATE, + LOCAL_DATETIME, + OFFSET_DATETIME, + INT, + FLOAT, + BOOL, + ARRAY, + INLINE_TABLE + }; + + std::shared_ptr<base> parse_value(std::string::iterator& it, + std::string::iterator& end) + { + parse_type type = determine_value_type(it, end); + switch (type) + { + case parse_type::STRING: + return parse_string(it, end); + case parse_type::LOCAL_TIME: + return parse_time(it, end); + case parse_type::LOCAL_DATE: + case parse_type::LOCAL_DATETIME: + case parse_type::OFFSET_DATETIME: + return parse_date(it, end); + case parse_type::INT: + case parse_type::FLOAT: + return parse_number(it, end); + case parse_type::BOOL: + return parse_bool(it, end); + case parse_type::ARRAY: + return parse_array(it, end); + case parse_type::INLINE_TABLE: + return parse_inline_table(it, end); + default: + throw_parse_exception("Failed to parse value"); + } + } + + parse_type determine_value_type(const std::string::iterator& it, + const std::string::iterator& end) + { + if(it == end) + { + throw_parse_exception("Failed to parse value type"); + } + if (*it == '"' || *it == '\'') + { + return parse_type::STRING; + } + else if (is_time(it, end)) + { + return parse_type::LOCAL_TIME; + } + else if (auto dtype = date_type(it, end)) + { + return *dtype; + } + else if (is_number(*it) || *it == '-' || *it == '+') + { + return determine_number_type(it, end); + } + else if (*it == 't' || *it == 'f') + { + return parse_type::BOOL; + } + else if (*it == '[') + { + return parse_type::ARRAY; + } + else if (*it == '{') + { + return parse_type::INLINE_TABLE; + } + throw_parse_exception("Failed to parse value type"); + } + + parse_type determine_number_type(const std::string::iterator& it, + const std::string::iterator& end) + { + // determine if we are an integer or a float + auto check_it = it; + if (*check_it == '-' || *check_it == '+') + ++check_it; + while (check_it != end && is_number(*check_it)) + ++check_it; + if (check_it != end && *check_it == '.') + { + ++check_it; + while (check_it != end && is_number(*check_it)) + ++check_it; + return parse_type::FLOAT; + } + else + { + return parse_type::INT; + } + } + + std::shared_ptr<value<std::string>> parse_string(std::string::iterator& it, + std::string::iterator& end) + { + auto delim = *it; + assert(delim == '"' || delim == '\''); + + // end is non-const here because we have to be able to potentially + // parse multiple lines in a string, not just one + auto check_it = it; + ++check_it; + if (check_it != end && *check_it == delim) + { + ++check_it; + if (check_it != end && *check_it == delim) + { + it = ++check_it; + return parse_multiline_string(it, end, delim); + } + } + return make_value<std::string>(string_literal(it, end, delim)); + } + + std::shared_ptr<value<std::string>> + parse_multiline_string(std::string::iterator& it, + std::string::iterator& end, char delim) + { + std::stringstream ss; + + auto is_ws = [](char c) { return c == ' ' || c == '\t'; }; + + bool consuming = false; + std::shared_ptr<value<std::string>> ret; + + auto handle_line + = [&](std::string::iterator& local_it, + std::string::iterator& local_end) { + if (consuming) + { + local_it = std::find_if_not(local_it, local_end, is_ws); + + // whole line is whitespace + if (local_it == local_end) + return; + } + + consuming = false; + + while (local_it != local_end) + { + // handle escaped characters + if (delim == '"' && *local_it == '\\') + { + auto check = local_it; + // check if this is an actual escape sequence or a + // whitespace escaping backslash + ++check; + consume_whitespace(check, local_end); + if (check == local_end) + { + consuming = true; + break; + } + + ss << parse_escape_code(local_it, local_end); + continue; + } + + // if we can end the string + if (std::distance(local_it, local_end) >= 3) + { + auto check = local_it; + // check for """ + if (*check++ == delim && *check++ == delim + && *check++ == delim) + { + local_it = check; + ret = make_value<std::string>(ss.str()); + break; + } + } + + ss << *local_it++; + } + }; + + // handle the remainder of the current line + handle_line(it, end); + if (ret) + return ret; + + // start eating lines + while (detail::getline(input_, line_)) + { + ++line_number_; + + it = line_.begin(); + end = line_.end(); + + handle_line(it, end); + + if (ret) + return ret; + + if (!consuming) + ss << std::endl; + } + + throw_parse_exception("Unterminated multi-line basic string"); + } + + std::string string_literal(std::string::iterator& it, + const std::string::iterator& end, char delim) + { + ++it; + std::string val; + while (it != end) + { + // handle escaped characters + if (delim == '"' && *it == '\\') + { + val += parse_escape_code(it, end); + } + else if (*it == delim) + { + ++it; + consume_whitespace(it, end); + return val; + } + else + { + val += *it++; + } + } + throw_parse_exception("Unterminated string literal"); + } + + std::string parse_escape_code(std::string::iterator& it, + const std::string::iterator& end) + { + ++it; + if (it == end) + throw_parse_exception("Invalid escape sequence"); + char value; + if (*it == 'b') + { + value = '\b'; + } + else if (*it == 't') + { + value = '\t'; + } + else if (*it == 'n') + { + value = '\n'; + } + else if (*it == 'f') + { + value = '\f'; + } + else if (*it == 'r') + { + value = '\r'; + } + else if (*it == '"') + { + value = '"'; + } + else if (*it == '\\') + { + value = '\\'; + } + else if (*it == 'u' || *it == 'U') + { + return parse_unicode(it, end); + } + else + { + throw_parse_exception("Invalid escape sequence"); + } + ++it; + return std::string(1, value); + } + + std::string parse_unicode(std::string::iterator& it, + const std::string::iterator& end) + { + bool large = *it++ == 'U'; + auto codepoint = parse_hex(it, end, large ? 0x10000000 : 0x1000); + + if ((codepoint > 0xd7ff && codepoint < 0xe000) || codepoint > 0x10ffff) + { + throw_parse_exception( + "Unicode escape sequence is not a Unicode scalar value"); + } + + std::string result; + // See Table 3-6 of the Unicode standard + if (codepoint <= 0x7f) + { + // 1-byte codepoints: 00000000 0xxxxxxx + // repr: 0xxxxxxx + result += static_cast<char>(codepoint & 0x7f); + } + else if (codepoint <= 0x7ff) + { + // 2-byte codepoints: 00000yyy yyxxxxxx + // repr: 110yyyyy 10xxxxxx + // + // 0x1f = 00011111 + // 0xc0 = 11000000 + // + result += static_cast<char>(0xc0 | ((codepoint >> 6) & 0x1f)); + // + // 0x80 = 10000000 + // 0x3f = 00111111 + // + result += static_cast<char>(0x80 | (codepoint & 0x3f)); + } + else if (codepoint <= 0xffff) + { + // 3-byte codepoints: zzzzyyyy yyxxxxxx + // repr: 1110zzzz 10yyyyyy 10xxxxxx + // + // 0xe0 = 11100000 + // 0x0f = 00001111 + // + result += static_cast<char>(0xe0 | ((codepoint >> 12) & 0x0f)); + result += static_cast<char>(0x80 | ((codepoint >> 6) & 0x1f)); + result += static_cast<char>(0x80 | (codepoint & 0x3f)); + } + else + { + // 4-byte codepoints: 000uuuuu zzzzyyyy yyxxxxxx + // repr: 11110uuu 10uuzzzz 10yyyyyy 10xxxxxx + // + // 0xf0 = 11110000 + // 0x07 = 00000111 + // + result += static_cast<char>(0xf0 | ((codepoint >> 18) & 0x07)); + result += static_cast<char>(0x80 | ((codepoint >> 12) & 0x3f)); + result += static_cast<char>(0x80 | ((codepoint >> 6) & 0x3f)); + result += static_cast<char>(0x80 | (codepoint & 0x3f)); + } + return result; + } + + uint32_t parse_hex(std::string::iterator& it, + const std::string::iterator& end, uint32_t place) + { + uint32_t value = 0; + while (place > 0) + { + if (it == end) + throw_parse_exception("Unexpected end of unicode sequence"); + + if (!is_hex(*it)) + throw_parse_exception("Invalid unicode escape sequence"); + + value += place * hex_to_digit(*it++); + place /= 16; + } + return value; + } + + bool is_hex(char c) + { + return is_number(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); + } + + uint32_t hex_to_digit(char c) + { + if (is_number(c)) + return static_cast<uint32_t>(c - '0'); + return 10 + static_cast<uint32_t>( + c - ((c >= 'a' && c <= 'f') ? 'a' : 'A')); + } + + std::shared_ptr<base> parse_number(std::string::iterator& it, + const std::string::iterator& end) + { + auto check_it = it; + auto check_end = find_end_of_number(it, end); + + auto eat_sign = [&]() { + if (check_it != end && (*check_it == '-' || *check_it == '+')) + ++check_it; + }; + + eat_sign(); + + auto eat_numbers = [&]() { + auto beg = check_it; + while (check_it != end && is_number(*check_it)) + { + ++check_it; + if (check_it != end && *check_it == '_') + { + ++check_it; + if (check_it == end || !is_number(*check_it)) + throw_parse_exception("Malformed number"); + } + } + + if (check_it == beg) + throw_parse_exception("Malformed number"); + }; + + auto check_no_leading_zero = [&]() { + if (check_it != end && *check_it == '0' && check_it + 1 != check_end + && check_it[1] != '.') + { + throw_parse_exception("Numbers may not have leading zeros"); + } + }; + + check_no_leading_zero(); + eat_numbers(); + + if (check_it != end + && (*check_it == '.' || *check_it == 'e' || *check_it == 'E')) + { + bool is_exp = *check_it == 'e' || *check_it == 'E'; + + ++check_it; + if (check_it == end) + throw_parse_exception("Floats must have trailing digits"); + + auto eat_exp = [&]() { + eat_sign(); + check_no_leading_zero(); + eat_numbers(); + }; + + if (is_exp) + eat_exp(); + else + eat_numbers(); + + if (!is_exp && check_it != end + && (*check_it == 'e' || *check_it == 'E')) + { + ++check_it; + eat_exp(); + } + + return parse_float(it, check_it); + } + else + { + return parse_int(it, check_it); + } + } + + std::shared_ptr<value<int64_t>> parse_int(std::string::iterator& it, + const std::string::iterator& end) + { + std::string v{it, end}; + v.erase(std::remove(v.begin(), v.end(), '_'), v.end()); + it = end; + try + { + return make_value<int64_t>(std::stoll(v)); + } + catch (const std::invalid_argument& ex) + { + throw_parse_exception("Malformed number (invalid argument: " + + std::string{ex.what()} + ")"); + } + catch (const std::out_of_range& ex) + { + throw_parse_exception("Malformed number (out of range: " + + std::string{ex.what()} + ")"); + } + } + + std::shared_ptr<value<double>> parse_float(std::string::iterator& it, + const std::string::iterator& end) + { + std::string v{it, end}; + v.erase(std::remove(v.begin(), v.end(), '_'), v.end()); + it = end; + char decimal_point = std::localeconv()->decimal_point[0]; + std::replace(v.begin(), v.end(), '.', decimal_point); + try + { + return make_value<double>(std::stod(v)); + } + catch (const std::invalid_argument& ex) + { + throw_parse_exception("Malformed number (invalid argument: " + + std::string{ex.what()} + ")"); + } + catch (const std::out_of_range& ex) + { + throw_parse_exception("Malformed number (out of range: " + + std::string{ex.what()} + ")"); + } + } + + std::shared_ptr<value<bool>> parse_bool(std::string::iterator& it, + const std::string::iterator& end) + { + auto eat = make_consumer(it, end, [this]() { + throw_parse_exception("Attempted to parse invalid boolean value"); + }); + + if (*it == 't') + { + eat("true"); + return make_value<bool>(true); + } + else if (*it == 'f') + { + eat("false"); + return make_value<bool>(false); + } + + eat.error(); + return nullptr; + } + + std::string::iterator find_end_of_number(std::string::iterator it, + std::string::iterator end) + { + return std::find_if(it, end, [](char c) { + return !is_number(c) && c != '_' && c != '.' && c != 'e' && c != 'E' + && c != '-' && c != '+'; + }); + } + + std::string::iterator find_end_of_date(std::string::iterator it, + std::string::iterator end) + { + return std::find_if(it, end, [](char c) { + return !is_number(c) && c != 'T' && c != 'Z' && c != ':' && c != '-' + && c != '+' && c != '.'; + }); + } + + std::string::iterator find_end_of_time(std::string::iterator it, + std::string::iterator end) + { + return std::find_if(it, end, [](char c) { + return !is_number(c) && c != ':' && c != '.'; + }); + } + + local_time read_time(std::string::iterator& it, + const std::string::iterator& end) + { + auto time_end = find_end_of_time(it, end); + + auto eat = make_consumer( + it, time_end, [&]() { throw_parse_exception("Malformed time"); }); + + local_time ltime; + + ltime.hour = eat.eat_digits(2); + eat(':'); + ltime.minute = eat.eat_digits(2); + eat(':'); + ltime.second = eat.eat_digits(2); + + int power = 100000; + if (it != time_end && *it == '.') + { + ++it; + while (it != time_end && is_number(*it)) + { + ltime.microsecond += power * (*it++ - '0'); + power /= 10; + } + } + + if (it != time_end) + throw_parse_exception("Malformed time"); + + return ltime; + } + + std::shared_ptr<value<local_time>> + parse_time(std::string::iterator& it, const std::string::iterator& end) + { + return make_value(read_time(it, end)); + } + + std::shared_ptr<base> parse_date(std::string::iterator& it, + const std::string::iterator& end) + { + auto date_end = find_end_of_date(it, end); + + auto eat = make_consumer( + it, date_end, [&]() { throw_parse_exception("Malformed date"); }); + + local_date ldate; + ldate.year = eat.eat_digits(4); + eat('-'); + ldate.month = eat.eat_digits(2); + eat('-'); + ldate.day = eat.eat_digits(2); + + if (it == date_end) + return make_value(ldate); + + eat('T'); + + local_datetime ldt; + static_cast<local_date&>(ldt) = ldate; + static_cast<local_time&>(ldt) = read_time(it, date_end); + + if (it == date_end) + return make_value(ldt); + + offset_datetime dt; + static_cast<local_datetime&>(dt) = ldt; + + int hoff = 0; + int moff = 0; + if (*it == '+' || *it == '-') + { + auto plus = *it == '+'; + ++it; + + hoff = eat.eat_digits(2); + dt.hour_offset = (plus) ? hoff : -hoff; + eat(':'); + moff = eat.eat_digits(2); + dt.minute_offset = (plus) ? moff : -moff; + } + else if (*it == 'Z') + { + ++it; + } + + if (it != date_end) + throw_parse_exception("Malformed date"); + + return make_value(dt); + } + + std::shared_ptr<base> parse_array(std::string::iterator& it, + std::string::iterator& end) + { + // this gets ugly because of the "homogeneity" restriction: + // arrays can either be of only one type, or contain arrays + // (each of those arrays could be of different types, though) + // + // because of the latter portion, we don't really have a choice + // but to represent them as arrays of base values... + ++it; + + // ugh---have to read the first value to determine array type... + skip_whitespace_and_comments(it, end); + + // edge case---empty array + if (*it == ']') + { + ++it; + return make_array(); + } + + auto val_end = std::find_if( + it, end, [](char c) { return c == ',' || c == ']' || c == '#'; }); + parse_type type = determine_value_type(it, val_end); + switch (type) + { + case parse_type::STRING: + return parse_value_array<std::string>(it, end); + case parse_type::LOCAL_TIME: + return parse_value_array<local_time>(it, end); + case parse_type::LOCAL_DATE: + return parse_value_array<local_date>(it, end); + case parse_type::LOCAL_DATETIME: + return parse_value_array<local_datetime>(it, end); + case parse_type::OFFSET_DATETIME: + return parse_value_array<offset_datetime>(it, end); + case parse_type::INT: + return parse_value_array<int64_t>(it, end); + case parse_type::FLOAT: + return parse_value_array<double>(it, end); + case parse_type::BOOL: + return parse_value_array<bool>(it, end); + case parse_type::ARRAY: + return parse_object_array<array>(&parser::parse_array, '[', it, + end); + case parse_type::INLINE_TABLE: + return parse_object_array<table_array>( + &parser::parse_inline_table, '{', it, end); + default: + throw_parse_exception("Unable to parse array"); + } + } + + template <class Value> + std::shared_ptr<array> parse_value_array(std::string::iterator& it, + std::string::iterator& end) + { + auto arr = make_array(); + while (it != end && *it != ']') + { + auto value = parse_value(it, end); + if (auto v = value->as<Value>()) + arr->get().push_back(value); + else + throw_parse_exception("Arrays must be homogeneous"); + skip_whitespace_and_comments(it, end); + if (*it != ',') + break; + ++it; + skip_whitespace_and_comments(it, end); + } + if (it != end) + ++it; + return arr; + } + + template <class Object, class Function> + std::shared_ptr<Object> parse_object_array(Function&& fun, char delim, + std::string::iterator& it, + std::string::iterator& end) + { + auto arr = make_element<Object>(); + + while (it != end && *it != ']') + { + if (*it != delim) + throw_parse_exception("Unexpected character in array"); + + arr->get().push_back(((*this).*fun)(it, end)); + skip_whitespace_and_comments(it, end); + + if (*it != ',') + break; + + ++it; + skip_whitespace_and_comments(it, end); + } + + if (it == end || *it != ']') + throw_parse_exception("Unterminated array"); + + ++it; + return arr; + } + + std::shared_ptr<table> parse_inline_table(std::string::iterator& it, + std::string::iterator& end) + { + auto tbl = make_table(); + do + { + ++it; + if (it == end) + throw_parse_exception("Unterminated inline table"); + + consume_whitespace(it, end); + parse_key_value(it, end, tbl.get()); + consume_whitespace(it, end); + } while (*it == ','); + + if (it == end || *it != '}') + throw_parse_exception("Unterminated inline table"); + + ++it; + consume_whitespace(it, end); + + return tbl; + } + + void skip_whitespace_and_comments(std::string::iterator& start, + std::string::iterator& end) + { + consume_whitespace(start, end); + while (start == end || *start == '#') + { + if (!detail::getline(input_, line_)) + throw_parse_exception("Unclosed array"); + line_number_++; + start = line_.begin(); + end = line_.end(); + consume_whitespace(start, end); + } + } + + void consume_whitespace(std::string::iterator& it, + const std::string::iterator& end) + { + while (it != end && (*it == ' ' || *it == '\t')) + ++it; + } + + void consume_backwards_whitespace(std::string::iterator& back, + const std::string::iterator& front) + { + while (back != front && (*back == ' ' || *back == '\t')) + --back; + } + + void eol_or_comment(const std::string::iterator& it, + const std::string::iterator& end) + { + if (it != end && *it != '#') + throw_parse_exception("Unidentified trailing character '" + + std::string{*it} + + "'---did you forget a '#'?"); + } + + bool is_time(const std::string::iterator& it, + const std::string::iterator& end) + { + auto time_end = find_end_of_time(it, end); + auto len = std::distance(it, time_end); + + if (len < 8) + return false; + + if (it[2] != ':' || it[5] != ':') + return false; + + if (len > 8) + return it[8] == '.' && len > 9; + + return true; + } + + option<parse_type> date_type(const std::string::iterator& it, + const std::string::iterator& end) + { + auto date_end = find_end_of_date(it, end); + auto len = std::distance(it, date_end); + + if (len < 10) + return {}; + + if (it[4] != '-' || it[7] != '-') + return {}; + + if (len >= 19 && it[10] == 'T' && is_time(it + 11, date_end)) + { + // datetime type + auto time_end = find_end_of_time(it + 11, date_end); + if (time_end == date_end) + return {parse_type::LOCAL_DATETIME}; + else + return {parse_type::OFFSET_DATETIME}; + } + else if (len == 10) + { + // just a regular date + return {parse_type::LOCAL_DATE}; + } + + return {}; + } + + std::istream& input_; + std::string line_; + std::size_t line_number_ = 0; +}; + +/** + * Utility function to parse a file as a TOML file. Returns the root table. + * Throws a parse_exception if the file cannot be opened. + */ +inline std::shared_ptr<table> parse_file(const std::string& filename) +{ +#if defined(BOOST_NOWIDE_FSTREAM_INCLUDED_HPP) + boost::nowide::ifstream file{filename.c_str()}; +#elif defined(NOWIDE_FSTREAM_INCLUDED_HPP) + nowide::ifstream file{filename.c_str()}; +#else + std::ifstream file{filename}; +#endif + if (!file.is_open()) + throw parse_exception{filename + " could not be opened for parsing"}; + parser p{file}; + return p.parse(); +} + +template <class... Ts> +struct value_accept; + +template <> +struct value_accept<> +{ + template <class Visitor, class... Args> + static void accept(const base&, Visitor&&, Args&&...) + { + // nothing + } +}; + +template <class T, class... Ts> +struct value_accept<T, Ts...> +{ + template <class Visitor, class... Args> + static void accept(const base& b, Visitor&& visitor, Args&&... args) + { + if (auto v = b.as<T>()) + { + visitor.visit(*v, std::forward<Args>(args)...); + } + else + { + value_accept<Ts...>::accept(b, std::forward<Visitor>(visitor), + std::forward<Args>(args)...); + } + } +}; + +/** + * base implementation of accept() that calls visitor.visit() on the concrete + * class. + */ +template <class Visitor, class... Args> +void base::accept(Visitor&& visitor, Args&&... args) const +{ + if (is_value()) + { + using value_acceptor + = value_accept<std::string, int64_t, double, bool, local_date, + local_time, local_datetime, offset_datetime>; + value_acceptor::accept(*this, std::forward<Visitor>(visitor), + std::forward<Args>(args)...); + } + else if (is_table()) + { + visitor.visit(static_cast<const table&>(*this), + std::forward<Args>(args)...); + } + else if (is_array()) + { + visitor.visit(static_cast<const array&>(*this), + std::forward<Args>(args)...); + } + else if (is_table_array()) + { + visitor.visit(static_cast<const table_array&>(*this), + std::forward<Args>(args)...); + } +} + +/** + * Writer that can be passed to accept() functions of cpptoml objects and + * will output valid TOML to a stream. + */ +class toml_writer +{ + public: + /** + * Construct a toml_writer that will write to the given stream + */ + toml_writer(std::ostream& s, const std::string& indent_space = "\t") + : stream_(s), indent_(indent_space), has_naked_endline_(false) + { + // nothing + } + + public: + /** + * Output a base value of the TOML tree. + */ + template <class T> + void visit(const value<T>& v, bool = false) + { + write(v); + } + + /** + * Output a table element of the TOML tree + */ + void visit(const table& t, bool in_array = false) + { + write_table_header(in_array); + std::vector<std::string> values; + std::vector<std::string> tables; + + for (const auto& i : t) + { + if (i.second->is_table() || i.second->is_table_array()) + { + tables.push_back(i.first); + } + else + { + values.push_back(i.first); + } + } + + for (unsigned int i = 0; i < values.size(); ++i) + { + path_.push_back(values[i]); + + if (i > 0) + endline(); + + write_table_item_header(*t.get(values[i])); + t.get(values[i])->accept(*this, false); + path_.pop_back(); + } + + for (unsigned int i = 0; i < tables.size(); ++i) + { + path_.push_back(tables[i]); + + if (values.size() > 0 || i > 0) + endline(); + + write_table_item_header(*t.get(tables[i])); + t.get(tables[i])->accept(*this, false); + path_.pop_back(); + } + + endline(); + } + + /** + * Output an array element of the TOML tree + */ + void visit(const array& a, bool = false) + { + write("["); + + for (unsigned int i = 0; i < a.get().size(); ++i) + { + if (i > 0) + write(", "); + + if (a.get()[i]->is_array()) + { + a.get()[i]->as_array()->accept(*this, true); + } + else + { + a.get()[i]->accept(*this, true); + } + } + + write("]"); + } + + /** + * Output a table_array element of the TOML tree + */ + void visit(const table_array& t, bool = false) + { + for (unsigned int j = 0; j < t.get().size(); ++j) + { + if (j > 0) + endline(); + + t.get()[j]->accept(*this, true); + } + + endline(); + } + + /** + * Escape a string for output. + */ + static std::string escape_string(const std::string& str) + { + std::string res; + for (auto it = str.begin(); it != str.end(); ++it) + { + if (*it == '\b') + { + res += "\\b"; + } + else if (*it == '\t') + { + res += "\\t"; + } + else if (*it == '\n') + { + res += "\\n"; + } + else if (*it == '\f') + { + res += "\\f"; + } + else if (*it == '\r') + { + res += "\\r"; + } + else if (*it == '"') + { + res += "\\\""; + } + else if (*it == '\\') + { + res += "\\\\"; + } + else if ((const uint32_t)*it <= 0x001f) + { + res += "\\u"; + std::stringstream ss; + ss << std::hex << static_cast<uint32_t>(*it); + res += ss.str(); + } + else + { + res += *it; + } + } + return res; + } + + protected: + /** + * Write out a string. + */ + void write(const value<std::string>& v) + { + write("\""); + write(escape_string(v.get())); + write("\""); + } + + /** + * Write out a double. + */ + void write(const value<double>& v) + { + std::ios::fmtflags flags{stream_.flags()}; + + stream_ << std::showpoint; + write(v.get()); + + stream_.flags(flags); + } + + /** + * Write out an integer, local_date, local_time, local_datetime, or + * offset_datetime. + */ + template <class T> + typename std::enable_if<is_one_of<T, int64_t, local_date, local_time, + local_datetime, + offset_datetime>::value>::type + write(const value<T>& v) + { + write(v.get()); + } + + /** + * Write out a boolean. + */ + void write(const value<bool>& v) + { + write((v.get() ? "true" : "false")); + } + + /** + * Write out the header of a table. + */ + void write_table_header(bool in_array = false) + { + if (!path_.empty()) + { + indent(); + + write("["); + + if (in_array) + { + write("["); + } + + for (unsigned int i = 0; i < path_.size(); ++i) + { + if (i > 0) + { + write("."); + } + + if (path_[i].find_first_not_of("ABCDEFGHIJKLMNOPQRSTUVWXYZabcde" + "fghijklmnopqrstuvwxyz0123456789" + "_-") + == std::string::npos) + { + write(path_[i]); + } + else + { + write("\""); + write(escape_string(path_[i])); + write("\""); + } + } + + if (in_array) + { + write("]"); + } + + write("]"); + endline(); + } + } + + /** + * Write out the identifier for an item in a table. + */ + void write_table_item_header(const base& b) + { + if (!b.is_table() && !b.is_table_array()) + { + indent(); + + if (path_.back().find_first_not_of("ABCDEFGHIJKLMNOPQRSTUVWXYZabcde" + "fghijklmnopqrstuvwxyz0123456789" + "_-") + == std::string::npos) + { + write(path_.back()); + } + else + { + write("\""); + write(escape_string(path_.back())); + write("\""); + } + + write(" = "); + } + } + + private: + /** + * Indent the proper number of tabs given the size of + * the path. + */ + void indent() + { + for (std::size_t i = 1; i < path_.size(); ++i) + write(indent_); + } + + /** + * Write a value out to the stream. + */ + template <class T> + void write(const T& v) + { + stream_ << v; + has_naked_endline_ = false; + } + + /** + * Write an endline out to the stream + */ + void endline() + { + if (!has_naked_endline_) + { + stream_ << "\n"; + has_naked_endline_ = true; + } + } + + private: + std::ostream& stream_; + const std::string indent_; + std::vector<std::string> path_; + bool has_naked_endline_; +}; + +inline std::ostream& operator<<(std::ostream& stream, const base& b) +{ + toml_writer writer{stream}; + b.accept(writer); + return stream; +} + +template <class T> +std::ostream& operator<<(std::ostream& stream, const value<T>& v) +{ + toml_writer writer{stream}; + v.accept(writer); + return stream; +} + +inline std::ostream& operator<<(std::ostream& stream, const table& t) +{ + toml_writer writer{stream}; + t.accept(writer); + return stream; +} + +inline std::ostream& operator<<(std::ostream& stream, const table_array& t) +{ + toml_writer writer{stream}; + t.accept(writer); + return stream; +} + +inline std::ostream& operator<<(std::ostream& stream, const array& a) +{ + toml_writer writer{stream}; + a.accept(writer); + return stream; +} +} +#endif diff --git a/src/libexpr/attr-set.cc b/src/libexpr/attr-set.cc index b284daa3c2f7..0785897d2513 100644 --- a/src/libexpr/attr-set.cc +++ b/src/libexpr/attr-set.cc @@ -1,5 +1,5 @@ #include "attr-set.hh" -#include "eval.hh" +#include "eval-inline.hh" #include <algorithm> @@ -7,30 +7,18 @@ namespace nix { -/* Note: Various places expect the allocated memory to be zeroed. */ -static void * allocBytes(size_t n) -{ - void * p; -#if HAVE_BOEHMGC - p = GC_malloc(n); -#else - p = calloc(n, 1); -#endif - if (!p) throw std::bad_alloc(); - return p; -} - - /* Allocate a new array of attributes for an attribute set with a specific capacity. The space is implicitly reserved after the Bindings structure. */ -Bindings * EvalState::allocBindings(Bindings::size_t capacity) +Bindings * EvalState::allocBindings(size_t capacity) { - return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings(capacity); + if (capacity > std::numeric_limits<Bindings::size_t>::max()) + throw Error("attribute set of size %d is too big", capacity); + return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings((Bindings::size_t) capacity); } -void EvalState::mkAttrs(Value & v, unsigned int capacity) +void EvalState::mkAttrs(Value & v, size_t capacity) { if (capacity == 0) { v = vEmptySet; diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index 8cc50e561354..c27116e3b448 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -78,4 +78,18 @@ inline void EvalState::forceList(Value & v, const Pos & pos) throwTypeError("value is %1% while a list was expected, at %2%", v, pos); } +/* Note: Various places expect the allocated memory to be zeroed. */ +inline void * allocBytes(size_t n) +{ + void * p; +#if HAVE_BOEHMGC + p = GC_MALLOC(n); +#else + p = calloc(n, 1); +#endif + if (!p) throw std::bad_alloc(); + return p; +} + + } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 37b977736e28..d8e10d9f20e1 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -6,27 +6,26 @@ #include "globals.hh" #include "eval-inline.hh" #include "download.hh" +#include "json.hh" #include <algorithm> #include <cstring> #include <unistd.h> #include <sys/time.h> #include <sys/resource.h> +#include <iostream> +#include <fstream> + +#include <sys/time.h> +#include <sys/resource.h> #if HAVE_BOEHMGC #include <gc/gc.h> #include <gc/gc_cpp.h> -#define NEW new (UseGC) - -#else - -#define NEW new - #endif - namespace nix { @@ -34,7 +33,7 @@ static char * dupString(const char * s) { char * t; #if HAVE_BOEHMGC - t = GC_strdup(s); + t = GC_STRDUP(s); #else t = strdup(s); #endif @@ -43,20 +42,6 @@ static char * dupString(const char * s) } -/* Note: Various places expect the allocated memory to be zeroed. */ -static void * allocBytes(size_t n) -{ - void * p; -#if HAVE_BOEHMGC - p = GC_malloc(n); -#else - p = calloc(n, 1); -#endif - if (!p) throw std::bad_alloc(); - return p; -} - - static void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v) { checkInterrupt(); @@ -145,6 +130,16 @@ std::ostream & operator << (std::ostream & str, const Value & v) } +const Value *getPrimOp(const Value &v) { + const Value * primOp = &v; + while (primOp->type == tPrimOpApp) { + primOp = primOp->primOpApp.left; + } + assert(primOp->type == tPrimOp); + return primOp; +} + + string showType(const Value & v) { switch (v.type) { @@ -159,8 +154,10 @@ string showType(const Value & v) case tApp: return "a function application"; case tLambda: return "a function"; case tBlackhole: return "a black hole"; - case tPrimOp: return "a built-in function"; - case tPrimOpApp: return "a partially applied built-in function"; + case tPrimOp: + return fmt("the built-in function '%s'", string(v.primOp->name)); + case tPrimOpApp: + return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name)); case tExternal: return v.external->showType(); case tFloat: return "a float"; } @@ -199,8 +196,15 @@ void initGC() #if HAVE_BOEHMGC /* Initialise the Boehm garbage collector. */ + + /* Don't look for interior pointers. This reduces the odds of + misdetection a bit. */ GC_set_all_interior_pointers(0); + /* We don't have any roots in data segments, so don't scan from + there. */ + GC_set_no_dls(1); + GC_INIT(); GC_set_oom_fn(oomHandler); @@ -307,20 +311,32 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store) assert(gcInitialised); + static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes"); + /* Initialise the Nix expression search path. */ - if (!settings.pureEval) { + if (!evalSettings.pureEval) { Strings paths = parseNixPath(getEnv("NIX_PATH", "")); for (auto & i : _searchPath) addToSearchPath(i); for (auto & i : paths) addToSearchPath(i); } addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs", true)); - if (settings.restrictEval || settings.pureEval) { + if (evalSettings.restrictEval || evalSettings.pureEval) { allowedPaths = PathSet(); + for (auto & i : searchPath) { auto r = resolveSearchPathElem(i); if (!r.first) continue; - allowedPaths->insert(r.second); + + auto path = r.second; + + if (store->isInStore(r.second)) { + PathSet closure; + store->computeFSClosure(store->toStorePath(r.second), closure); + for (auto & path : closure) + allowedPaths->insert(path); + } else + allowedPaths->insert(r.second); } } @@ -334,7 +350,6 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store) EvalState::~EvalState() { - fileEvalCache.clear(); } @@ -342,25 +357,37 @@ Path EvalState::checkSourcePath(const Path & path_) { if (!allowedPaths) return path_; + auto i = resolvedPaths.find(path_); + if (i != resolvedPaths.end()) + return i->second; + bool found = false; + /* First canonicalize the path without symlinks, so we make sure an + * attacker can't append ../../... to a path that would be in allowedPaths + * and thus leak symlink targets. + */ + Path abspath = canonPath(path_); + for (auto & i : *allowedPaths) { - if (isDirOrInDir(path_, i)) { + if (isDirOrInDir(abspath, i)) { found = true; break; } } if (!found) - throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", path_); + throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", abspath); /* Resolve symlinks. */ - debug(format("checking access to '%s'") % path_); - Path path = canonPath(path_, true); + debug(format("checking access to '%s'") % abspath); + Path path = canonPath(abspath, true); for (auto & i : *allowedPaths) { - if (isDirOrInDir(path, i)) + if (isDirOrInDir(path, i)) { + resolvedPaths[path_] = path; return path; + } } throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", path); @@ -369,13 +396,13 @@ Path EvalState::checkSourcePath(const Path & path_) void EvalState::checkURI(const std::string & uri) { - if (!settings.restrictEval) return; + if (!evalSettings.restrictEval) return; /* 'uri' should be equal to a prefix, or in a subdirectory of a prefix. Thus, the prefix https://github.co does not permit access to https://github.com. Note: this allows 'http://' and 'https://' as prefixes for any http/https URI. */ - for (auto & prefix : settings.allowedUris.get()) + for (auto & prefix : evalSettings.allowedUris.get()) if (uri == prefix || (uri.size() > prefix.size() && prefix.size() > 0 @@ -422,7 +449,7 @@ Value * EvalState::addConstant(const string & name, Value & v) Value * EvalState::addPrimOp(const string & name, - unsigned int arity, PrimOpFun primOp) + size_t arity, PrimOpFun primOp) { if (arity == 0) { Value v; @@ -433,7 +460,7 @@ Value * EvalState::addPrimOp(const string & name, string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; Symbol sym = symbols.create(name2); v->type = tPrimOp; - v->primOp = NEW PrimOp(primOp, arity, sym); + v->primOp = new PrimOp(primOp, arity, sym); staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl; baseEnv.values[baseEnvDispl++] = v; baseEnv.values[0]->attrs->push_back(Attr(sym, v)); @@ -528,7 +555,7 @@ Value & mkString(Value & v, const string & s, const PathSet & context) { mkString(v, s.c_str()); if (!context.empty()) { - unsigned int n = 0; + size_t n = 0; v.string.context = (const char * *) allocBytes((context.size() + 1) * sizeof(char *)); for (auto & i : context) @@ -547,17 +574,17 @@ void mkPath(Value & v, const char * s) inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) { - for (unsigned int l = var.level; l; --l, env = env->up) ; + for (size_t l = var.level; l; --l, env = env->up) ; if (!var.fromWith) return env->values[var.displ]; while (1) { - if (!env->haveWithAttrs) { + if (env->type == Env::HasWithExpr) { if (noEval) return 0; Value * v = allocValue(); evalAttrs(*env->up, (Expr *) env->values[0], *v); env->values[0] = v; - env->haveWithAttrs = true; + env->type = Env::HasWithAttrs; } Bindings::iterator j = env->values[0]->attrs->find(var.name); if (j != env->values[0]->attrs->end()) { @@ -566,26 +593,37 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) } if (!env->prevWith) throwUndefinedVarError("undefined variable '%1%' at %2%", var.name, var.pos); - for (unsigned int l = env->prevWith; l; --l, env = env->up) ; + for (size_t l = env->prevWith; l; --l, env = env->up) ; } } +std::atomic<uint64_t> nrValuesFreed{0}; + +void finalizeValue(void * obj, void * data) +{ + nrValuesFreed++; +} + Value * EvalState::allocValue() { nrValues++; - return (Value *) allocBytes(sizeof(Value)); + auto v = (Value *) allocBytes(sizeof(Value)); + //GC_register_finalizer_no_order(v, finalizeValue, nullptr, nullptr, nullptr); + return v; } -Env & EvalState::allocEnv(unsigned int size) +Env & EvalState::allocEnv(size_t size) { - assert(size <= std::numeric_limits<decltype(Env::size)>::max()); + if (size > std::numeric_limits<decltype(Env::size)>::max()) + throw Error("environment size %d is too big", size); nrEnvs++; nrValuesInEnvs += size; Env * env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *)); - env->size = size; + env->size = (decltype(Env::size)) size; + env->type = Env::Plain; /* We assume that env->values has been cleared by the allocator; maybeThunk() and lookupVar fromWith expect this. */ @@ -593,7 +631,7 @@ Env & EvalState::allocEnv(unsigned int size) } -void EvalState::mkList(Value & v, unsigned int size) +void EvalState::mkList(Value & v, size_t size) { clearValue(v); if (size == 1) @@ -628,7 +666,7 @@ void EvalState::mkThunk_(Value & v, Expr * expr) void EvalState::mkPos(Value & v, Pos * pos) { - if (pos) { + if (pos && pos->file.set()) { mkAttrs(v, 3); mkString(*allocAttr(v, sFile), pos->file); mkInt(*allocAttr(v, sLine), pos->line); @@ -705,7 +743,17 @@ void EvalState::evalFile(const Path & path_, Value & v) } printTalkative("evaluating file '%1%'", path2); - Expr * e = parseExprFromFile(checkSourcePath(path2)); + Expr * e = nullptr; + + auto j = fileParseCache.find(path2); + if (j != fileParseCache.end()) + e = j->second; + + if (!e) + e = parseExprFromFile(checkSourcePath(path2)); + + fileParseCache[path2] = e; + try { eval(e, v); } catch (Error & e) { @@ -721,6 +769,7 @@ void EvalState::evalFile(const Path & path_, Value & v) void EvalState::resetFileCache() { fileEvalCache.clear(); + fileParseCache.clear(); } @@ -805,7 +854,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) /* The recursive attributes are evaluated in the new environment, while the inherited attributes are evaluated in the original environment. */ - unsigned int displ = 0; + size_t displ = 0; for (auto & i : attrs) { Value * vAttr; if (hasOverrides && !i.second.inherited) { @@ -879,7 +928,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v) /* The recursive attributes are evaluated in the new environment, while the inherited attributes are evaluated in the original environment. */ - unsigned int displ = 0; + size_t displ = 0; for (auto & i : attrs->attrs) env2.values[displ++] = i.second.e->maybeThunk(state, i.second.inherited ? env : env2); @@ -890,7 +939,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v) void ExprList::eval(EvalState & state, Env & env, Value & v) { state.mkList(v, elems.size()); - for (unsigned int n = 0; n < elems.size(); ++n) + for (size_t n = 0; n < elems.size(); ++n) v.listElems()[n] = elems[n]->maybeThunk(state, env); } @@ -1012,22 +1061,22 @@ void ExprApp::eval(EvalState & state, Env & env, Value & v) void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) { /* Figure out the number of arguments still needed. */ - unsigned int argsDone = 0; + size_t argsDone = 0; Value * primOp = &fun; while (primOp->type == tPrimOpApp) { argsDone++; primOp = primOp->primOpApp.left; } assert(primOp->type == tPrimOp); - unsigned int arity = primOp->primOp->arity; - unsigned int argsLeft = arity - argsDone; + auto arity = primOp->primOp->arity; + auto argsLeft = arity - argsDone; if (argsLeft == 1) { /* We have all the arguments, so call the primop. */ /* Put all the arguments in an array. */ Value * vArgs[arity]; - unsigned int n = arity - 1; + auto n = arity - 1; vArgs[n--] = &arg; for (Value * arg = &fun; arg->type == tPrimOpApp; arg = arg->primOpApp.left) vArgs[n--] = arg->primOpApp.right; @@ -1048,6 +1097,8 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & pos) { + forceValue(fun, pos); + if (fun.type == tPrimOp || fun.type == tPrimOpApp) { callPrimOp(fun, arg, v, pos); return; @@ -1063,10 +1114,8 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po auto & fun2 = *allocValue(); fun2 = fun; /* !!! Should we use the attr pos here? */ - forceValue(*found->value, pos); Value v2; callFunction(*found->value, fun2, v2, pos); - forceValue(v2, pos); return callFunction(v2, arg, v, pos); } } @@ -1076,13 +1125,13 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po ExprLambda & lambda(*fun.lambda.fun); - unsigned int size = + auto size = (lambda.arg.empty() ? 0 : 1) + (lambda.matchAttrs ? lambda.formals->formals.size() : 0); Env & env2(allocEnv(size)); env2.up = fun.lambda.env; - unsigned int displ = 0; + size_t displ = 0; if (!lambda.matchAttrs) env2.values[displ++] = &arg; @@ -1096,7 +1145,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po /* For each formal argument, get the actual argument. If there is no matching actual argument but the formal argument has a default, use the default. */ - unsigned int attrsUsed = 0; + size_t attrsUsed = 0; for (auto & i : lambda.formals->formals) { Bindings::iterator j = arg.attrs->find(i.name); if (j == arg.attrs->end()) { @@ -1153,7 +1202,6 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) if (fun.type == tAttrs) { auto found = fun.attrs->find(sFunctor); if (found != fun.attrs->end()) { - forceValue(*found->value); Value * v = allocValue(); callFunction(*found->value, fun, *v, noPos); forceValue(*v); @@ -1188,7 +1236,7 @@ void ExprWith::eval(EvalState & state, Env & env, Value & v) Env & env2(state.allocEnv(1)); env2.up = &env; env2.prevWith = prevWith; - env2.haveWithAttrs = false; + env2.type = Env::HasWithExpr; env2.values[0] = (Value *) attrs; body->eval(state, env2, v); @@ -1294,15 +1342,15 @@ void ExprOpConcatLists::eval(EvalState & state, Env & env, Value & v) } -void EvalState::concatLists(Value & v, unsigned int nrLists, Value * * lists, const Pos & pos) +void EvalState::concatLists(Value & v, size_t nrLists, Value * * lists, const Pos & pos) { nrListConcats++; Value * nonEmpty = 0; - unsigned int len = 0; - for (unsigned int n = 0; n < nrLists; ++n) { + size_t len = 0; + for (size_t n = 0; n < nrLists; ++n) { forceList(*lists[n], pos); - unsigned int l = lists[n]->listSize(); + auto l = lists[n]->listSize(); len += l; if (l) nonEmpty = lists[n]; } @@ -1314,8 +1362,8 @@ void EvalState::concatLists(Value & v, unsigned int nrLists, Value * * lists, co mkList(v, len); auto out = v.listElems(); - for (unsigned int n = 0, pos = 0; n < nrLists; ++n) { - unsigned int l = lists[n]->listSize(); + for (size_t n = 0, pos = 0; n < nrLists; ++n) { + auto l = lists[n]->listSize(); if (l) memcpy(out + pos, lists[n]->listElems(), l * sizeof(Value *)); pos += l; @@ -1410,7 +1458,7 @@ void EvalState::forceValueDeep(Value & v) } else if (v.isList()) { - for (unsigned int n = 0; n < v.listSize(); ++n) + for (size_t n = 0; n < v.listSize(); ++n) recurse(*v.listElems()[n]); } }; @@ -1537,7 +1585,6 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, if (v.type == tAttrs) { auto i = v.attrs->find(sToString); if (i != v.attrs->end()) { - forceValue(*i->value, pos); Value v1; callFunction(*i->value, v, v1, pos); return coerceToString(pos, v1, context, coerceMore, copyToStore); @@ -1562,7 +1609,7 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, if (v.isList()) { string result; - for (unsigned int n = 0; n < v.listSize(); ++n) { + for (size_t n = 0; n < v.listSize(); ++n) { result += coerceToString(pos, *v.listElems()[n], context, coerceMore, copyToStore); if (n < v.listSize() - 1 @@ -1649,7 +1696,7 @@ bool EvalState::eqValues(Value & v1, Value & v2) case tList2: case tListN: if (v1.listSize() != v2.listSize()) return false; - for (unsigned int n = 0; n < v1.listSize(); ++n) + for (size_t n = 0; n < v1.listSize(); ++n) if (!eqValues(*v1.listElems()[n], *v2.listElems()[n])) return false; return true; @@ -1691,12 +1738,9 @@ bool EvalState::eqValues(Value & v1, Value & v2) } } - void EvalState::printStats() { bool showStats = getEnv("NIX_SHOW_STATS", "0") != "0"; - Verbosity v = showStats ? lvlInfo : lvlDebug; - printMsg(v, "evaluation statistics:"); struct rusage buf; getrusage(RUSAGE_SELF, &buf); @@ -1707,62 +1751,107 @@ void EvalState::printStats() uint64_t bValues = nrValues * sizeof(Value); uint64_t bAttrsets = nrAttrsets * sizeof(Bindings) + nrAttrsInAttrsets * sizeof(Attr); - printMsg(v, format(" time elapsed: %1%") % cpuTime); - printMsg(v, format(" size of a value: %1%") % sizeof(Value)); - printMsg(v, format(" size of an attr: %1%") % sizeof(Attr)); - printMsg(v, format(" environments allocated count: %1%") % nrEnvs); - printMsg(v, format(" environments allocated bytes: %1%") % bEnvs); - printMsg(v, format(" list elements count: %1%") % nrListElems); - printMsg(v, format(" list elements bytes: %1%") % bLists); - printMsg(v, format(" list concatenations: %1%") % nrListConcats); - printMsg(v, format(" values allocated count: %1%") % nrValues); - printMsg(v, format(" values allocated bytes: %1%") % bValues); - printMsg(v, format(" sets allocated: %1% (%2% bytes)") % nrAttrsets % bAttrsets); - printMsg(v, format(" right-biased unions: %1%") % nrOpUpdates); - printMsg(v, format(" values copied in right-biased unions: %1%") % nrOpUpdateValuesCopied); - printMsg(v, format(" symbols in symbol table: %1%") % symbols.size()); - printMsg(v, format(" size of symbol table: %1%") % symbols.totalSize()); - printMsg(v, format(" number of thunks: %1%") % nrThunks); - printMsg(v, format(" number of thunks avoided: %1%") % nrAvoided); - printMsg(v, format(" number of attr lookups: %1%") % nrLookups); - printMsg(v, format(" number of primop calls: %1%") % nrPrimOpCalls); - printMsg(v, format(" number of function calls: %1%") % nrFunctionCalls); - printMsg(v, format(" total allocations: %1% bytes") % (bEnvs + bLists + bValues + bAttrsets)); - #if HAVE_BOEHMGC GC_word heapSize, totalBytes; GC_get_heap_usage_safe(&heapSize, 0, 0, 0, &totalBytes); - printMsg(v, format(" current Boehm heap size: %1% bytes") % heapSize); - printMsg(v, format(" total Boehm heap allocations: %1% bytes") % totalBytes); +#endif + if (showStats) { + auto outPath = getEnv("NIX_SHOW_STATS_PATH","-"); + std::fstream fs; + if (outPath != "-") + fs.open(outPath, std::fstream::out); + JSONObject topObj(outPath == "-" ? std::cerr : fs, true); + topObj.attr("cpuTime",cpuTime); + { + auto envs = topObj.object("envs"); + envs.attr("number", nrEnvs); + envs.attr("elements", nrValuesInEnvs); + envs.attr("bytes", bEnvs); + } + { + auto lists = topObj.object("list"); + lists.attr("elements", nrListElems); + lists.attr("bytes", bLists); + lists.attr("concats", nrListConcats); + } + { + auto values = topObj.object("values"); + values.attr("number", nrValues); + values.attr("bytes", bValues); + } + { + auto syms = topObj.object("symbols"); + syms.attr("number", symbols.size()); + syms.attr("bytes", symbols.totalSize()); + } + { + auto sets = topObj.object("sets"); + sets.attr("number", nrAttrsets); + sets.attr("bytes", bAttrsets); + sets.attr("elements", nrAttrsInAttrsets); + } + { + auto sizes = topObj.object("sizes"); + sizes.attr("Env", sizeof(Env)); + sizes.attr("Value", sizeof(Value)); + sizes.attr("Bindings", sizeof(Bindings)); + sizes.attr("Attr", sizeof(Attr)); + } + topObj.attr("nrOpUpdates", nrOpUpdates); + topObj.attr("nrOpUpdateValuesCopied", nrOpUpdateValuesCopied); + topObj.attr("nrThunks", nrThunks); + topObj.attr("nrAvoided", nrAvoided); + topObj.attr("nrLookups", nrLookups); + topObj.attr("nrPrimOpCalls", nrPrimOpCalls); + topObj.attr("nrFunctionCalls", nrFunctionCalls); +#if HAVE_BOEHMGC + { + auto gc = topObj.object("gc"); + gc.attr("heapSize", heapSize); + gc.attr("totalBytes", totalBytes); + } #endif - if (countCalls) { - v = lvlInfo; - - printMsg(v, format("calls to %1% primops:") % primOpCalls.size()); - typedef std::multimap<unsigned int, Symbol> PrimOpCalls_; - PrimOpCalls_ primOpCalls_; - for (auto & i : primOpCalls) - primOpCalls_.insert(std::pair<unsigned int, Symbol>(i.second, i.first)); - for (auto i = primOpCalls_.rbegin(); i != primOpCalls_.rend(); ++i) - printMsg(v, format("%1$10d %2%") % i->first % i->second); - - printMsg(v, format("calls to %1% functions:") % functionCalls.size()); - typedef std::multimap<unsigned int, ExprLambda *> FunctionCalls_; - FunctionCalls_ functionCalls_; - for (auto & i : functionCalls) - functionCalls_.insert(std::pair<unsigned int, ExprLambda *>(i.second, i.first)); - for (auto i = functionCalls_.rbegin(); i != functionCalls_.rend(); ++i) - printMsg(v, format("%1$10d %2%") % i->first % i->second->showNamePos()); - - printMsg(v, format("evaluations of %1% attributes:") % attrSelects.size()); - typedef std::multimap<unsigned int, Pos> AttrSelects_; - AttrSelects_ attrSelects_; - for (auto & i : attrSelects) - attrSelects_.insert(std::pair<unsigned int, Pos>(i.second, i.first)); - for (auto i = attrSelects_.rbegin(); i != attrSelects_.rend(); ++i) - printMsg(v, format("%1$10d %2%") % i->first % i->second); + if (countCalls) { + { + auto obj = topObj.object("primops"); + for (auto & i : primOpCalls) + obj.attr(i.first, i.second); + } + { + auto list = topObj.list("functions"); + for (auto & i : functionCalls) { + auto obj = list.object(); + if (i.first->name.set()) + obj.attr("name", (const string &) i.first->name); + else + obj.attr("name", nullptr); + if (i.first->pos) { + obj.attr("file", (const string &) i.first->pos.file); + obj.attr("line", i.first->pos.line); + obj.attr("column", i.first->pos.column); + } + obj.attr("count", i.second); + } + } + { + auto list = topObj.list("attributes"); + for (auto & i : attrSelects) { + auto obj = list.object(); + if (i.first) { + obj.attr("file", (const string &) i.first.file); + obj.attr("line", i.first.line); + obj.attr("column", i.first.column); + } + obj.attr("count", i.second); + } + } + } + if (getEnv("NIX_SHOW_SYMBOLS", "0") != "0") { + auto list = topObj.list("symbols"); + symbols.dump([&](const std::string & s) { list.elem(s); }); + } } } @@ -1810,7 +1899,7 @@ size_t valueSize(Value & v) if (seen.find(v.listElems()) == seen.end()) { seen.insert(v.listElems()); sz += v.listSize() * sizeof(Value *); - for (unsigned int n = 0; n < v.listSize(); ++n) + for (size_t n = 0; n < v.listSize(); ++n) sz += doValue(*v.listElems()[n]); } break; @@ -1846,9 +1935,10 @@ size_t valueSize(Value & v) size_t sz = sizeof(Env) + sizeof(Value *) * env.size; - for (unsigned int i = 0; i < env.size; ++i) - if (env.values[i]) - sz += doValue(*env.values[i]); + if (env.type != Env::HasWithExpr) + for (size_t i = 0; i < env.size; ++i) + if (env.values[i]) + sz += doValue(*env.values[i]); if (env.up) sz += doEnv(*env.up); @@ -1877,4 +1967,9 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) { } +EvalSettings evalSettings; + +static GlobalConfig::Register r1(&evalSettings); + + } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 9d8799b7906b..a314e01e0a71 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -5,8 +5,10 @@ #include "nixexpr.hh" #include "symbol-table.hh" #include "hash.hh" +#include "config.hh" #include <map> +#include <unordered_map> namespace nix { @@ -34,8 +36,8 @@ struct Env { Env * up; unsigned short size; // used by ‘valueSize’ - unsigned short prevWith:15; // nr of levels up to next `with' environment - unsigned short haveWithAttrs:1; + unsigned short prevWith:14; // nr of levels up to next `with' environment + enum { Plain = 0, HasWithExpr, HasWithAttrs } type:2; Value * values[0]; }; @@ -79,7 +81,7 @@ public: /* The allowed filesystem paths in restricted or pure evaluation mode. */ - std::experimental::optional<PathSet> allowedPaths; + std::optional<PathSet> allowedPaths; Value vEmptySet; @@ -88,6 +90,14 @@ public: private: SrcToStore srcToStore; + /* A cache from path names to parse trees. */ +#if HAVE_BOEHMGC + typedef std::map<Path, Expr *, std::less<Path>, traceable_allocator<std::pair<const Path, Expr *> > > FileParseCache; +#else + typedef std::map<Path, Expr *> FileParseCache; +#endif + FileParseCache fileParseCache; + /* A cache from path names to values. */ #if HAVE_BOEHMGC typedef std::map<Path, Value, std::less<Path>, traceable_allocator<std::pair<const Path, Value> > > FileEvalCache; @@ -100,6 +110,9 @@ private: std::map<std::string, std::pair<bool, std::string>> searchPathResolved; + /* Cache used by checkSourcePath(). */ + std::unordered_map<Path, Path> resolvedPaths; + public: EvalState(const Strings & _searchPath, ref<Store> store); @@ -214,7 +227,7 @@ private: Value * addConstant(const string & name, Value & v); Value * addPrimOp(const string & name, - unsigned int arity, PrimOpFun primOp); + size_t arity, PrimOpFun primOp); public: @@ -248,18 +261,18 @@ public: /* Allocation primitives. */ Value * allocValue(); - Env & allocEnv(unsigned int size); + Env & allocEnv(size_t size); Value * allocAttr(Value & vAttrs, const Symbol & name); - Bindings * allocBindings(Bindings::size_t capacity); + Bindings * allocBindings(size_t capacity); - void mkList(Value & v, unsigned int length); - void mkAttrs(Value & v, unsigned int capacity); + void mkList(Value & v, size_t length); + void mkAttrs(Value & v, size_t capacity); void mkThunk_(Value & v, Expr * expr); void mkPos(Value & v, Pos * pos); - void concatLists(Value & v, unsigned int nrLists, Value * * lists, const Pos & pos); + void concatLists(Value & v, size_t nrLists, Value * * lists, const Pos & pos); /* Print statistics. */ void printStats(); @@ -282,15 +295,15 @@ private: bool countCalls; - typedef std::map<Symbol, unsigned int> PrimOpCalls; + typedef std::map<Symbol, size_t> PrimOpCalls; PrimOpCalls primOpCalls; - typedef std::map<ExprLambda *, unsigned int> FunctionCalls; + typedef std::map<ExprLambda *, size_t> FunctionCalls; FunctionCalls functionCalls; void incrFunctionCall(ExprLambda * fun); - typedef std::map<Pos, unsigned int> AttrSelects; + typedef std::map<Pos, size_t> AttrSelects; AttrSelects attrSelects; friend struct ExprOpUpdate; @@ -303,6 +316,9 @@ private: /* Return a string representing the type of the value `v'. */ string showType(const Value & v); +/* Decode a context string ‘!<name>!<path>’ into a pair <path, + name>. */ +std::pair<string, string> decodeContext(const string & s); /* If `path' refers to a directory, then append "/default.nix". */ Path resolveExprPath(Path path); @@ -316,4 +332,25 @@ struct InvalidPathError : EvalError #endif }; +struct EvalSettings : Config +{ + Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", + "Whether builtin functions that allow executing native code should be enabled."}; + + Setting<bool> restrictEval{this, false, "restrict-eval", + "Whether to restrict file system access to paths in $NIX_PATH, " + "and network access to the URI prefixes listed in 'allowed-uris'."}; + + Setting<bool> pureEval{this, false, "pure-eval", + "Whether to restrict file system and network access to files specified by cryptographic hash."}; + + Setting<bool> enableImportFromDerivation{this, true, "allow-import-from-derivation", + "Whether the evaluator allows importing the result of a derivation."}; + + Setting<Strings> allowedUris{this, {}, "allowed-uris", + "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."}; +}; + +extern EvalSettings evalSettings; + } diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index d38ed2df3b18..21a4d7917fce 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -295,7 +295,7 @@ static bool getDerivation(EvalState & state, Value & v, } -std::experimental::optional<DrvInfo> getDerivation(EvalState & state, Value & v, +std::optional<DrvInfo> getDerivation(EvalState & state, Value & v, bool ignoreAssertionFailures) { Done done; diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh index 4d9128e3f448..d7860fc6a4bc 100644 --- a/src/libexpr/get-drvs.hh +++ b/src/libexpr/get-drvs.hh @@ -44,7 +44,7 @@ public: string queryDrvPath() const; string queryOutPath() const; string queryOutputName() const; - /** Return the list of outputs. The "outputs to install" are determined by `mesa.outputsToInstall`. */ + /** Return the list of outputs. The "outputs to install" are determined by `meta.outputsToInstall`. */ Outputs queryOutputs(bool onlyOutputsToInstall = false); StringSet queryMetaNames(); @@ -78,7 +78,7 @@ typedef list<DrvInfo> DrvInfos; /* If value `v' denotes a derivation, return a DrvInfo object describing it. Otherwise return nothing. */ -std::experimental::optional<DrvInfo> getDerivation(EvalState & state, +std::optional<DrvInfo> getDerivation(EvalState & state, Value & v, bool ignoreAssertionFailures); void getDerivations(EvalState & state, Value & v, const string & pathPrefix, diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 8b1404595548..3f6017957782 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -110,7 +110,7 @@ static void parseJSON(EvalState & state, const char * & s, Value & v) if (number_type == tFloat) mkFloat(v, stod(tmp_number)); else - mkInt(v, stoi(tmp_number)); + mkInt(v, stol(tmp_number)); } catch (std::invalid_argument e) { throw JSONParseError("invalid JSON number"); } catch (std::out_of_range e) { diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 1e9c29afa133..c34e5c383923 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -6,12 +6,14 @@ %option nounput noyy_top_state +%s DEFAULT %x STRING %x IND_STRING -%x INSIDE_DOLLAR_CURLY %{ +#include <boost/lexical_cast.hpp> + #include "nixexpr.hh" #include "parser-tab.hh" @@ -97,8 +99,6 @@ URI [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~ %% -<INITIAL,INSIDE_DOLLAR_CURLY>{ - if { return IF; } then { return THEN; } @@ -124,9 +124,11 @@ or { return OR_KW; } {ID} { yylval->id = strdup(yytext); return ID; } {INT} { errno = 0; - yylval->n = strtol(yytext, 0, 10); - if (errno != 0) + try { + yylval->n = boost::lexical_cast<int64_t>(yytext); + } catch (const boost::bad_lexical_cast &) { throw ParseError(format("invalid integer '%1%'") % yytext); + } return INT; } {FLOAT} { errno = 0; @@ -136,17 +138,19 @@ or { return OR_KW; } return FLOAT; } -\$\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; } -} +\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; } -\} { return '}'; } -<INSIDE_DOLLAR_CURLY>\} { POP_STATE(); return '}'; } -\{ { return '{'; } -<INSIDE_DOLLAR_CURLY>\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return '{'; } +\} { /* State INITIAL only exists at the bottom of the stack and is + used as a marker. DEFAULT replaces it everywhere else. + Popping when in INITIAL state causes an empty stack exception, + so don't */ + if (YYSTATE != INITIAL) + POP_STATE(); + return '}'; + } +\{ { PUSH_STATE(DEFAULT); return '{'; } -<INITIAL,INSIDE_DOLLAR_CURLY>\" { - PUSH_STATE(STRING); return '"'; - } +\" { PUSH_STATE(STRING); return '"'; } <STRING>([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})*\$/\" | <STRING>([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})+ { /* It is impossible to match strings ending with '$' with one @@ -155,7 +159,7 @@ or { return OR_KW; } yylval->e = unescapeStr(data->symbols, yytext, yyleng); return STR; } -<STRING>\$\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; } +<STRING>\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; } <STRING>\" { POP_STATE(); return '"'; } <STRING>\$|\\|\$\\ { /* This can only occur when we reach EOF, otherwise the above @@ -165,7 +169,7 @@ or { return OR_KW; } return STR; } -<INITIAL,INSIDE_DOLLAR_CURLY>\'\'(\ *\n)? { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; } +\'\'(\ *\n)? { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; } <IND_STRING>([^\$\']|\$[^\{\']|\'[^\'\$])+ { yylval->e = new ExprIndStr(yytext); return IND_STR; @@ -183,14 +187,13 @@ or { return OR_KW; } yylval->e = unescapeStr(data->symbols, yytext + 2, yyleng - 2); return IND_STR; } -<IND_STRING>\$\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; } +<IND_STRING>\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; } <IND_STRING>\'\' { POP_STATE(); return IND_STRING_CLOSE; } <IND_STRING>\' { yylval->e = new ExprIndStr("'"); return IND_STR; } -<INITIAL,INSIDE_DOLLAR_CURLY>{ {PATH} { if (yytext[yyleng-1] == '/') throw ParseError("path '%s' has a trailing slash", yytext); @@ -209,11 +212,11 @@ or { return OR_KW; } \#[^\r\n]* /* single-line comments */ \/\*([^*]|\*+[^*/])*\*+\/ /* long comments */ -{ANY} return yytext[0]; - -} - -<<EOF>> { data->atEnd = true; return 0; } +{ANY} { + /* Don't return a negative number, as this will cause + Bison to stop parsing without an error. */ + return (unsigned char) yytext[0]; + } %% diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index daa3258f0d3c..ccd5293e4e5e 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -6,7 +6,7 @@ libexpr_DIR := $(d) libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc -libexpr_LIBS = libutil libstore libformat +libexpr_LIBS = libutil libstore libexpr_LDFLAGS = ifneq ($(OS), FreeBSD) diff --git a/src/libexpr/nix-expr.pc.in b/src/libexpr/nix-expr.pc.in index 79f3e2f4506e..80f7a492b1a1 100644 --- a/src/libexpr/nix-expr.pc.in +++ b/src/libexpr/nix-expr.pc.in @@ -7,4 +7,4 @@ Description: Nix Package Manager Version: @PACKAGE_VERSION@ Requires: nix-store bdw-gc Libs: -L${libdir} -lnixexpr -Cflags: -I${includedir}/nix -std=c++14 +Cflags: -I${includedir}/nix -std=c++17 diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 8c8f39640681..665a42987dc1 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -11,7 +11,6 @@ namespace nix { MakeError(EvalError, Error) MakeError(ParseError, Error) -MakeError(IncompleteParseError, ParseError) MakeError(AssertionError, EvalError) MakeError(ThrownError, AssertionError) MakeError(Abort, EvalError) @@ -255,7 +254,7 @@ struct ExprWith : Expr { Pos pos; Expr * attrs, * body; - unsigned int prevWith; + size_t prevWith; ExprWith(const Pos & pos, Expr * attrs, Expr * body) : pos(pos), attrs(attrs), body(body) { }; COMMON_METHODS }; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index df4fdf032479..97e1a65849e2 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -1,7 +1,7 @@ %glr-parser %pure-parser %locations -%error-verbose +%define parse.error verbose %defines /* %no-lines */ %parse-param { void * scanner } @@ -31,12 +31,10 @@ namespace nix { Path basePath; Symbol path; string error; - bool atEnd; Symbol sLetBody; ParseData(EvalState & state) : state(state) , symbols(state.symbols) - , atEnd(false) , sLetBody(symbols.create("<let-body>")) { }; }; @@ -156,8 +154,8 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex whitespace-only final lines are not taken into account. (So the " " in "\n ''" is ignored, but the " " in "\n foo''" is.) */ bool atStartOfLine = true; /* = seen only whitespace in the current line */ - unsigned int minIndent = 1000000; - unsigned int curIndent = 0; + size_t minIndent = 1000000; + size_t curIndent = 0; for (auto & i : es) { ExprIndStr * e = dynamic_cast<ExprIndStr *>(i); if (!e) { @@ -168,7 +166,7 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex } continue; } - for (unsigned int j = 0; j < e->s.size(); ++j) { + for (size_t j = 0; j < e->s.size(); ++j) { if (atStartOfLine) { if (e->s[j] == ' ') curIndent++; @@ -190,8 +188,8 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex /* Strip spaces from each line. */ vector<Expr *> * es2 = new vector<Expr *>; atStartOfLine = true; - unsigned int curDropped = 0; - unsigned int n = es.size(); + size_t curDropped = 0; + size_t n = es.size(); for (vector<Expr *>::iterator i = es.begin(); i != es.end(); ++i, --n) { ExprIndStr * e = dynamic_cast<ExprIndStr *>(*i); if (!e) { @@ -202,7 +200,7 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex } string s2; - for (unsigned int j = 0; j < e->s.size(); ++j) { + for (size_t j = 0; j < e->s.size(); ++j) { if (atStartOfLine) { if (e->s[j] == ' ') { if (curDropped++ >= minIndent) @@ -295,11 +293,11 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err %token IND_STRING_OPEN IND_STRING_CLOSE %token ELLIPSIS -%nonassoc IMPL +%right IMPL %left OR %left AND %nonassoc EQ NEQ -%left '<' '>' LEQ GEQ +%nonassoc '<' '>' LEQ GEQ %right UPDATE %left NOT %left '+' '-' @@ -561,12 +559,7 @@ Expr * EvalState::parse(const char * text, int res = yyparse(scanner, &data); yylex_destroy(scanner); - if (res) { - if (data.atEnd) - throw IncompleteParseError(data.error); - else - throw ParseError(data.error); - } + if (res) throw ParseError(data.error); data.result->bindVars(staticEnv); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 9e7651da725f..06f577f36fce 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -73,7 +73,7 @@ void EvalState::realiseContext(const PathSet & context) if (drvs.empty()) return; - if (!settings.enableImportFromDerivation) + if (!evalSettings.enableImportFromDerivation) throw EvalError(format("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false") % *(drvs.begin())); /* For performance, prefetch all substitute info. */ @@ -271,7 +271,18 @@ static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Valu static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0]); - mkBool(v, args[0]->type == tLambda); + bool res; + switch (args[0]->type) { + case tLambda: + case tPrimOp: + case tPrimOpApp: + res = true; + break; + default: + res = false; + break; + } + mkBool(v, res); } @@ -304,6 +315,12 @@ static void prim_isBool(EvalState & state, const Pos & pos, Value * * args, Valu mkBool(v, args[0]->type == tBool); } +/* Determine whether the argument is a path. */ +static void prim_isPath(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tPath); +} struct CompareValues { @@ -453,7 +470,7 @@ static void prim_tryEval(EvalState & state, const Pos & pos, Value * * args, Val static void prim_getEnv(EvalState & state, const Pos & pos, Value * * args, Value & v) { string name = state.forceStringNoCtx(*args[0], pos); - mkString(v, settings.restrictEval || settings.pureEval ? "" : getEnv(name)); + mkString(v, evalSettings.restrictEval || evalSettings.pureEval ? "" : getEnv(name)); } @@ -544,7 +561,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * PathSet context; - std::experimental::optional<std::string> outputHash; + std::optional<std::string> outputHash; std::string outputHashAlgo; bool outputHashRecursive = false; @@ -676,21 +693,12 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * } } - /* See prim_unsafeDiscardOutputDependency. */ - else if (path.at(0) == '~') - drv.inputSrcs.insert(string(path, 1)); - /* Handle derivation outputs of the form ‘!<name>!<path>’. */ else if (path.at(0) == '!') { std::pair<string, string> ctx = decodeContext(path); drv.inputDrvs[ctx.first].insert(ctx.second); } - /* Handle derivation contexts returned by - ‘builtins.storePath’. */ - else if (isDerivation(path)) - drv.inputDrvs[path] = state.store->queryDerivationOutputNames(path); - /* Otherwise it's a source file. */ else drv.inputSrcs.insert(path); @@ -713,16 +721,14 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (outputs.size() != 1 || *(outputs.begin()) != "out") throw Error(format("multiple outputs are not supported in fixed-output derivations, at %1%") % posDrvName); - HashType ht = parseHashType(outputHashAlgo); - if (ht == htUnknown) - throw EvalError(format("unknown hash algorithm '%1%', at %2%") % outputHashAlgo % posDrvName); + HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo); Hash h(*outputHash, ht); - outputHash = h.to_string(Base16, false); - if (outputHashRecursive) outputHashAlgo = "r:" + outputHashAlgo; Path outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName); if (!jsonObject) drv.env["out"] = outPath; - drv.outputs["out"] = DerivationOutput(outPath, outputHashAlgo, *outputHash); + drv.outputs["out"] = DerivationOutput(outPath, + (outputHashRecursive ? "r:" : "") + printHashType(h.type), + h.to_string(Base16, false)); } else { @@ -855,7 +861,7 @@ static void prim_baseNameOf(EvalState & state, const Pos & pos, Value * * args, static void prim_dirOf(EvalState & state, const Pos & pos, Value * * args, Value & v) { PathSet context; - Path dir = dirOf(state.coerceToPath(pos, *args[0], context)); + Path dir = dirOf(state.coerceToString(pos, *args[0], context, false, false)); if (args[0]->type == tPath) mkPath(v, dir.c_str()); else mkString(v, dir, context); } @@ -917,6 +923,20 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va mkPath(v, state.checkSourcePath(state.findFile(searchPath, path, pos)).c_str()); } +/* Return the cryptographic hash of a file in base-16. */ +static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + string type = state.forceStringNoCtx(*args[0], pos); + HashType ht = parseHashType(type); + if (ht == htUnknown) + throw Error(format("unknown hash type '%1%', at %2%") % type % pos); + + PathSet context; // discarded + Path p = state.coerceToPath(pos, *args[1], context); + + mkString(v, hashFile(ht, state.checkSourcePath(p)).to_string(Base16, false), context); +} + /* Read a directory (without . or ..) */ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -995,13 +1015,8 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu PathSet refs; for (auto path : context) { - if (path.at(0) == '=') path = string(path, 1); - if (isDerivation(path)) { - /* See prim_unsafeDiscardOutputDependency. */ - if (path.at(0) != '~') - throw EvalError(format("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%") % name % pos); - path = string(path, 1); - } + if (path.at(0) != '/') + throw EvalError(format("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%") % name % pos); refs.insert(path); } @@ -1020,7 +1035,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_, Value * filterFun, bool recursive, const Hash & expectedHash, Value & v) { - const auto path = settings.pureEval && expectedHash ? + const auto path = evalSettings.pureEval && expectedHash ? path_ : state.checkSourcePath(path_); PathFilter filter = filterFun ? ([&](const Path & path) { @@ -1345,6 +1360,24 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args } +/* Apply a function to every element of an attribute set. */ +static void prim_mapAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceAttrs(*args[1], pos); + + state.mkAttrs(v, args[1]->attrs->size()); + + for (auto & i : *args[1]->attrs) { + Value * vName = state.allocValue(); + Value * vFun2 = state.allocValue(); + mkString(*vName, i.name); + mkApp(*vFun2, *args[0], *vName); + mkApp(*state.allocAttr(v, i.name), *vFun2, *i.value); + } +} + + + /************************************************************* * Lists *************************************************************/ @@ -1399,7 +1432,6 @@ static void prim_tail(EvalState & state, const Pos & pos, Value * * args, Value /* Apply a function to every element of a list. */ static void prim_map(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceFunction(*args[0], pos); state.forceList(*args[1], pos); state.mkList(v, args[1]->listSize()); @@ -1478,19 +1510,20 @@ static void prim_foldlStrict(EvalState & state, const Pos & pos, Value * * args, state.forceFunction(*args[0], pos); state.forceList(*args[2], pos); - Value * vCur = args[1]; + if (args[2]->listSize()) { + Value * vCur = args[1]; - if (args[2]->listSize()) for (unsigned int n = 0; n < args[2]->listSize(); ++n) { Value vTmp; state.callFunction(*args[0], *vCur, vTmp, pos); vCur = n == args[2]->listSize() - 1 ? &v : state.allocValue(); state.callFunction(vTmp, *args[2]->listElems()[n], *vCur, pos); } - else - v = *vCur; - - state.forceValue(v); + state.forceValue(v); + } else { + state.forceValue(*args[1]); + v = *args[1]; + } } @@ -1527,7 +1560,6 @@ static void prim_all(EvalState & state, const Pos & pos, Value * * args, Value & static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceFunction(*args[0], pos); auto len = state.forceInt(*args[1], pos); if (len < 0) @@ -1616,6 +1648,35 @@ static void prim_partition(EvalState & state, const Pos & pos, Value * * args, V } +/* concatMap = f: list: concatLists (map f list); */ +/* C++-version is to avoid allocating `mkApp', call `f' eagerly */ +static void prim_concatMap(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceFunction(*args[0], pos); + state.forceList(*args[1], pos); + auto nrLists = args[1]->listSize(); + + Value lists[nrLists]; + size_t len = 0; + + for (unsigned int n = 0; n < nrLists; ++n) { + Value * vElem = args[1]->listElems()[n]; + state.callFunction(*args[0], *vElem, lists[n], pos); + state.forceList(lists[n], pos); + len += lists[n].listSize(); + } + + state.mkList(v, len); + auto out = v.listElems(); + for (unsigned int n = 0, pos = 0; n < nrLists; ++n) { + auto l = lists[n].listSize(); + if (l) + memcpy(out + pos, lists[n].listElems(), l * sizeof(Value *)); + pos += l; + } +} + + /************************************************************* * Integer arithmetic *************************************************************/ @@ -1623,6 +1684,8 @@ static void prim_partition(EvalState & state, const Pos & pos, Value * * args, V static void prim_add(EvalState & state, const Pos & pos, Value * * args, Value & v) { + state.forceValue(*args[0], pos); + state.forceValue(*args[1], pos); if (args[0]->type == tFloat || args[1]->type == tFloat) mkFloat(v, state.forceFloat(*args[0], pos) + state.forceFloat(*args[1], pos)); else @@ -1632,6 +1695,8 @@ static void prim_add(EvalState & state, const Pos & pos, Value * * args, Value & static void prim_sub(EvalState & state, const Pos & pos, Value * * args, Value & v) { + state.forceValue(*args[0], pos); + state.forceValue(*args[1], pos); if (args[0]->type == tFloat || args[1]->type == tFloat) mkFloat(v, state.forceFloat(*args[0], pos) - state.forceFloat(*args[1], pos)); else @@ -1641,6 +1706,8 @@ static void prim_sub(EvalState & state, const Pos & pos, Value * * args, Value & static void prim_mul(EvalState & state, const Pos & pos, Value * * args, Value & v) { + state.forceValue(*args[0], pos); + state.forceValue(*args[1], pos); if (args[0]->type == tFloat || args[1]->type == tFloat) mkFloat(v, state.forceFloat(*args[0], pos) * state.forceFloat(*args[1], pos)); else @@ -1650,6 +1717,9 @@ static void prim_mul(EvalState & state, const Pos & pos, Value * * args, Value & static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & v) { + state.forceValue(*args[0], pos); + state.forceValue(*args[1], pos); + NixFloat f2 = state.forceFloat(*args[1], pos); if (f2 == 0) throw EvalError(format("division by zero, at %1%") % pos); @@ -1665,6 +1735,20 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & } } +static void prim_bitAnd(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + mkInt(v, state.forceInt(*args[0], pos) & state.forceInt(*args[1], pos)); +} + +static void prim_bitOr(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + mkInt(v, state.forceInt(*args[0], pos) | state.forceInt(*args[1], pos)); +} + +static void prim_bitXor(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + mkInt(v, state.forceInt(*args[0], pos) ^ state.forceInt(*args[1], pos)); +} static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -1716,41 +1800,6 @@ static void prim_stringLength(EvalState & state, const Pos & pos, Value * * args } -static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - string s = state.coerceToString(pos, *args[0], context); - mkString(v, s, PathSet()); -} - - -static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - state.forceString(*args[0], context, pos); - mkBool(v, !context.empty()); -} - - -/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a - builder without causing the derivation to be built (for instance, - in the derivation that builds NARs in nix-push, when doing - source-only deployment). This primop marks the string context so - that builtins.derivation adds the path to drv.inputSrcs rather than - drv.inputDrvs. */ -static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - string s = state.coerceToString(pos, *args[0], context); - - PathSet context2; - for (auto & p : context) - context2.insert(p.at(0) == '=' ? "~" + string(p, 1) : p); - - mkString(v, s, context2); -} - - /* Return the cryptographic hash of a string in base-16. */ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -2031,7 +2080,7 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, state.checkURI(url); - if (settings.pureEval && !expectedHash) + if (evalSettings.pureEval && !expectedHash) throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); Path res = getDownloader()->downloadCached(state.store, url, unpack, name, expectedHash); @@ -2099,12 +2148,12 @@ void EvalState::createBaseEnv() addConstant(name, v); }; - if (!settings.pureEval) { + if (!evalSettings.pureEval) { mkInt(v, time(0)); addConstant("__currentTime", v); } - if (!settings.pureEval) { + if (!evalSettings.pureEval) { mkString(v, settings.thisSystem); addConstant("__currentSystem", v); } @@ -2129,7 +2178,7 @@ void EvalState::createBaseEnv() mkApp(v, *vScopedImport, *v2); forceValue(v); addConstant("import", v); - if (settings.enableNativeCode) { + if (evalSettings.enableNativeCode) { addPrimOp("__importNative", 2, prim_importNative); addPrimOp("__exec", 1, prim_exec); } @@ -2140,6 +2189,7 @@ void EvalState::createBaseEnv() addPrimOp("__isInt", 1, prim_isInt); addPrimOp("__isFloat", 1, prim_isFloat); addPrimOp("__isBool", 1, prim_isBool); + addPrimOp("__isPath", 1, prim_isPath); addPrimOp("__genericClosure", 1, prim_genericClosure); addPrimOp("abort", 1, prim_abort); addPrimOp("__addErrorContext", 2, prim_addErrorContext); @@ -2156,7 +2206,7 @@ void EvalState::createBaseEnv() // Paths addPrimOp("__toPath", 1, prim_toPath); - if (settings.pureEval) + if (evalSettings.pureEval) addPurityError("__storePath"); else addPrimOp("__storePath", 1, prim_storePath); @@ -2166,6 +2216,7 @@ void EvalState::createBaseEnv() addPrimOp("__readFile", 1, prim_readFile); addPrimOp("__readDir", 1, prim_readDir); addPrimOp("__findFile", 2, prim_findFile); + addPrimOp("__hashFile", 2, prim_hashFile); // Creating files addPrimOp("__toXML", 1, prim_toXML); @@ -2187,6 +2238,7 @@ void EvalState::createBaseEnv() addPrimOp("__intersectAttrs", 2, prim_intersectAttrs); addPrimOp("__catAttrs", 2, prim_catAttrs); addPrimOp("__functionArgs", 1, prim_functionArgs); + addPrimOp("__mapAttrs", 2, prim_mapAttrs); // Lists addPrimOp("__isList", 1, prim_isList); @@ -2204,21 +2256,22 @@ void EvalState::createBaseEnv() addPrimOp("__genList", 2, prim_genList); addPrimOp("__sort", 2, prim_sort); addPrimOp("__partition", 2, prim_partition); + addPrimOp("__concatMap", 2, prim_concatMap); // Integer arithmetic addPrimOp("__add", 2, prim_add); addPrimOp("__sub", 2, prim_sub); addPrimOp("__mul", 2, prim_mul); addPrimOp("__div", 2, prim_div); + addPrimOp("__bitAnd", 2, prim_bitAnd); + addPrimOp("__bitOr", 2, prim_bitOr); + addPrimOp("__bitXor", 2, prim_bitXor); addPrimOp("__lessThan", 2, prim_lessThan); // String manipulation addPrimOp("toString", 1, prim_toString); addPrimOp("__substring", 3, prim_substring); addPrimOp("__stringLength", 1, prim_stringLength); - addPrimOp("__hasContext", 1, prim_hasContext); - addPrimOp("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext); - addPrimOp("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency); addPrimOp("__hashString", 2, prim_hashString); addPrimOp("__match", 2, prim_match); addPrimOp("__split", 2, prim_split); diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc new file mode 100644 index 000000000000..2d79739ea047 --- /dev/null +++ b/src/libexpr/primops/context.cc @@ -0,0 +1,187 @@ +#include "primops.hh" +#include "eval-inline.hh" +#include "derivations.hh" + +namespace nix { + +static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + string s = state.coerceToString(pos, *args[0], context); + mkString(v, s, PathSet()); +} + +static RegisterPrimOp r1("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext); + + +static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + state.forceString(*args[0], context, pos); + mkBool(v, !context.empty()); +} + +static RegisterPrimOp r2("__hasContext", 1, prim_hasContext); + + +/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a + builder without causing the derivation to be built (for instance, + in the derivation that builds NARs in nix-push, when doing + source-only deployment). This primop marks the string context so + that builtins.derivation adds the path to drv.inputSrcs rather than + drv.inputDrvs. */ +static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + string s = state.coerceToString(pos, *args[0], context); + + PathSet context2; + for (auto & p : context) + context2.insert(p.at(0) == '=' ? string(p, 1) : p); + + mkString(v, s, context2); +} + +static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency); + + +/* Extract the context of a string as a structured Nix value. + + The context is represented as an attribute set whose keys are the + paths in the context set and whose values are attribute sets with + the following keys: + path: True if the relevant path is in the context as a plain store + path (i.e. the kind of context you get when interpolating + a Nix path (e.g. ./.) into a string). False if missing. + allOutputs: True if the relevant path is a derivation and it is + in the context as a drv file with all of its outputs + (i.e. the kind of context you get when referencing + .drvPath of some derivation). False if missing. + outputs: If a non-empty list, the relevant path is a derivation + and the provided outputs are referenced in the context + (i.e. the kind of context you get when referencing + .outPath of some derivation). Empty list if missing. + Note that for a given path any combination of the above attributes + may be present. +*/ +static void prim_getContext(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + struct ContextInfo { + bool path = false; + bool allOutputs = false; + Strings outputs; + }; + PathSet context; + state.forceString(*args[0], context, pos); + auto contextInfos = std::map<Path, ContextInfo>(); + for (const auto & p : context) { + Path drv; + string output; + const Path * path = &p; + if (p.at(0) == '=') { + drv = string(p, 1); + path = &drv; + } else if (p.at(0) == '!') { + std::pair<string, string> ctx = decodeContext(p); + drv = ctx.first; + output = ctx.second; + path = &drv; + } + auto isPath = drv.empty(); + auto isAllOutputs = (!drv.empty()) && output.empty(); + + auto iter = contextInfos.find(*path); + if (iter == contextInfos.end()) { + contextInfos.emplace(*path, ContextInfo{isPath, isAllOutputs, output.empty() ? Strings{} : Strings{std::move(output)}}); + } else { + if (isPath) + iter->second.path = true; + else if (isAllOutputs) + iter->second.allOutputs = true; + else + iter->second.outputs.emplace_back(std::move(output)); + } + } + + state.mkAttrs(v, contextInfos.size()); + + auto sPath = state.symbols.create("path"); + auto sAllOutputs = state.symbols.create("allOutputs"); + for (const auto & info : contextInfos) { + auto & infoVal = *state.allocAttr(v, state.symbols.create(info.first)); + state.mkAttrs(infoVal, 3); + if (info.second.path) + mkBool(*state.allocAttr(infoVal, sPath), true); + if (info.second.allOutputs) + mkBool(*state.allocAttr(infoVal, sAllOutputs), true); + if (!info.second.outputs.empty()) { + auto & outputsVal = *state.allocAttr(infoVal, state.sOutputs); + state.mkList(outputsVal, info.second.outputs.size()); + size_t i = 0; + for (const auto & output : info.second.outputs) { + mkString(*(outputsVal.listElems()[i++] = state.allocValue()), output); + } + } + infoVal.attrs->sort(); + } + v.attrs->sort(); +} + +static RegisterPrimOp r4("__getContext", 1, prim_getContext); + + +/* Append the given context to a given string. + + See the commentary above unsafeGetContext for details of the + context representation. +*/ +static void prim_appendContext(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + auto orig = state.forceString(*args[0], context, pos); + + state.forceAttrs(*args[1], pos); + + auto sPath = state.symbols.create("path"); + auto sAllOutputs = state.symbols.create("allOutputs"); + for (auto & i : *args[1]->attrs) { + if (!state.store->isStorePath(i.name)) + throw EvalError("Context key '%s' is not a store path, at %s", i.name, i.pos); + if (!settings.readOnlyMode) + state.store->ensurePath(i.name); + state.forceAttrs(*i.value, *i.pos); + auto iter = i.value->attrs->find(sPath); + if (iter != i.value->attrs->end()) { + if (state.forceBool(*iter->value, *iter->pos)) + context.insert(i.name); + } + + iter = i.value->attrs->find(sAllOutputs); + if (iter != i.value->attrs->end()) { + if (state.forceBool(*iter->value, *iter->pos)) { + if (!isDerivation(i.name)) { + throw EvalError("Tried to add all-outputs context of %s, which is not a derivation, to a string, at %s", i.name, i.pos); + } + context.insert("=" + string(i.name)); + } + } + + iter = i.value->attrs->find(state.sOutputs); + if (iter != i.value->attrs->end()) { + state.forceList(*iter->value, *iter->pos); + if (iter->value->listSize() && !isDerivation(i.name)) { + throw EvalError("Tried to add derivation output context of %s, which is not a derivation, to a string, at %s", i.name, i.pos); + } + for (unsigned int n = 0; n < iter->value->listSize(); ++n) { + auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos); + context.insert("!" + name + "!" + string(i.name)); + } + } + } + + mkString(v, orig, context); +} + +static RegisterPrimOp r5("__appendContext", 2, prim_appendContext); + +} diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 8bb74dad639e..aaf02c856d4f 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -3,6 +3,7 @@ #include "download.hh" #include "store-api.hh" #include "pathlocks.hh" +#include "hash.hh" #include <sys/time.h> @@ -25,10 +26,10 @@ struct GitInfo std::regex revRegex("^[0-9a-fA-F]{40}$"); GitInfo exportGit(ref<Store> store, const std::string & uri, - std::experimental::optional<std::string> ref, std::string rev, + std::optional<std::string> ref, std::string rev, const std::string & name) { - if (settings.pureEval && rev == "") + if (evalSettings.pureEval && rev == "") throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) { @@ -84,15 +85,16 @@ GitInfo exportGit(ref<Store> store, const std::string & uri, if (rev != "" && !std::regex_match(rev, revRegex)) throw Error("invalid Git revision '%s'", rev); - Path cacheDir = getCacheDir() + "/nix/git"; + deletePath(getCacheDir() + "/nix/git"); + + Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false); if (!pathExists(cacheDir)) { + createDirs(dirOf(cacheDir)); runProgram("git", true, { "init", "--bare", cacheDir }); } - std::string localRef = hashString(htSHA256, fmt("%s-%s", uri, *ref)).to_string(Base32, false); - - Path localRefFile = cacheDir + "/refs/heads/" + localRef; + Path localRefFile = cacheDir + "/refs/heads/" + *ref; bool doFetch; time_t now = time(0); @@ -122,7 +124,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri, // FIXME: git stderr messes up our progress indicator, so // we're using --quiet for now. Should process its stderr. - runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, *ref + ":" + localRef }); + runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) }); struct timeval times[2]; times[0].tv_sec = now; @@ -188,7 +190,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri, static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v) { std::string url; - std::experimental::optional<std::string> ref; + std::optional<std::string> ref; std::string rev; std::string name = "source"; PathSet context; @@ -219,8 +221,6 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va } else url = state.coerceToString(pos, *args[0], context, false, false); - if (!isUri(url)) url = absPath(url); - // FIXME: git externals probably can be used to bypass the URI // whitelist. Ah well. state.checkURI(url); diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index a75c5fc2ddff..66f49f374321 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -27,7 +27,7 @@ std::regex commitHashRegex("^[0-9a-fA-F]{40}$"); HgInfo exportMercurial(ref<Store> store, const std::string & uri, std::string rev, const std::string & name) { - if (settings.pureEval && rev == "") + if (evalSettings.pureEval && rev == "") throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision"); if (rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.hg")) { @@ -93,7 +93,22 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri, Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", uri)); if (pathExists(cacheDir)) { - runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri }); + try { + runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri }); + } + catch (ExecError & e){ + string transJournal = cacheDir + "/.hg/store/journal"; + /* hg throws "abandoned transaction" error only if this file exists */ + if (pathExists(transJournal)) + { + runProgram("hg", true, { "recover", "-R", cacheDir }); + runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri }); + } + else + { + throw ExecError(e.status, fmt("program hg '%1%' ", statusToString(e.status))); + } + } } else { createDirs(dirOf(cacheDir)); runProgram("hg", true, { "clone", "--noupdate", "--", uri, cacheDir }); @@ -184,8 +199,6 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar } else url = state.coerceToString(pos, *args[0], context, false, false); - if (!isUri(url)) url = absPath(url); - // FIXME: git externals probably can be used to bypass the URI // whitelist. Ah well. state.checkURI(url); diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc new file mode 100644 index 000000000000..4128de05d0cf --- /dev/null +++ b/src/libexpr/primops/fromTOML.cc @@ -0,0 +1,77 @@ +#include "primops.hh" +#include "eval-inline.hh" + +#include "cpptoml/cpptoml.h" + +namespace nix { + +static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + using namespace cpptoml; + + auto toml = state.forceStringNoCtx(*args[0], pos); + + std::istringstream tomlStream(toml); + + std::function<void(Value &, std::shared_ptr<base>)> visit; + + visit = [&](Value & v, std::shared_ptr<base> t) { + + if (auto t2 = t->as_table()) { + + size_t size = 0; + for (auto & i : *t2) { (void) i; size++; } + + state.mkAttrs(v, size); + + for (auto & i : *t2) { + auto & v2 = *state.allocAttr(v, state.symbols.create(i.first)); + + if (auto i2 = i.second->as_table_array()) { + size_t size2 = i2->get().size(); + state.mkList(v2, size2); + for (size_t j = 0; j < size2; ++j) + visit(*(v2.listElems()[j] = state.allocValue()), i2->get()[j]); + } + else + visit(v2, i.second); + } + + v.attrs->sort(); + } + + else if (auto t2 = t->as_array()) { + size_t size = t2->get().size(); + + state.mkList(v, size); + + for (size_t i = 0; i < size; ++i) + visit(*(v.listElems()[i] = state.allocValue()), t2->get()[i]); + } + + else if (t->is_value()) { + if (auto val = t->as<int64_t>()) + mkInt(v, val->get()); + else if (auto val = t->as<NixFloat>()) + mkFloat(v, val->get()); + else if (auto val = t->as<bool>()) + mkBool(v, val->get()); + else if (auto val = t->as<std::string>()) + mkString(v, val->get()); + else + throw EvalError("unsupported value type in TOML"); + } + + else abort(); + }; + + try { + visit(v, parser(tomlStream).parse()); + } catch (std::runtime_error & e) { + throw EvalError("while parsing a TOML string at %s: %s", pos, e.what()); + } +} + +static RegisterPrimOp r("fromTOML", 1, prim_fromTOML); + +} diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh index c2ee49dd32fb..91faea122ce1 100644 --- a/src/libexpr/symbol-table.hh +++ b/src/libexpr/symbol-table.hh @@ -69,12 +69,19 @@ public: return Symbol(&*res.first); } - unsigned int size() const + size_t size() const { return symbols.size(); } size_t totalSize() const; + + template<typename T> + void dump(T callback) + { + for (auto & s : symbols) + callback(s); + } }; } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 9df516f062ef..e1ec87d3b84c 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -43,8 +43,8 @@ class XMLWriter; class JSONPlaceholder; -typedef long NixInt; -typedef float NixFloat; +typedef int64_t NixInt; +typedef double NixFloat; /* External values must descend from ExternalValueBase, so that * type-agnostic nix functions (e.g. showType) can be implemented @@ -128,7 +128,7 @@ struct Value const char * path; Bindings * attrs; struct { - unsigned int size; + size_t size; Value * * elems; } bigList; Value * smallList[2]; @@ -166,7 +166,7 @@ struct Value return type == tList1 || type == tList2 ? smallList : bigList.elems; } - unsigned int listSize() const + size_t listSize() const { return type == tList1 ? 1 : type == tList2 ? 2 : bigList.size; } diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index bcc05c2cdad6..4c35a4199590 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -29,14 +29,14 @@ MixCommonArgs::MixCommonArgs(const string & programName) .arity(2) .handler([](std::vector<std::string> ss) { try { - settings.set(ss[0], ss[1]); + globalConfig.set(ss[0], ss[1]); } catch (UsageError & e) { warn(e.what()); } }); std::string cat = "config"; - settings.convertToArgs(*this, cat); + globalConfig.convertToArgs(*this, cat); // Backward compatibility hack: nix-env already had a --system flag. if (programName == "nix-env") longFlags.erase("system"); diff --git a/src/libmain/local.mk b/src/libmain/local.mk index f1fd3eb72424..0c80f5a0a037 100644 --- a/src/libmain/local.mk +++ b/src/libmain/local.mk @@ -8,7 +8,7 @@ libmain_SOURCES := $(wildcard $(d)/*.cc) libmain_LDFLAGS = $(OPENSSL_LIBS) -libmain_LIBS = libstore libutil libformat +libmain_LIBS = libstore libutil libmain_ALLOW_UNDEFINED = 1 diff --git a/src/libmain/nix-main.pc.in b/src/libmain/nix-main.pc.in index 38bc85c484eb..37b03dcd42c0 100644 --- a/src/libmain/nix-main.pc.in +++ b/src/libmain/nix-main.pc.in @@ -6,4 +6,4 @@ Name: Nix Description: Nix Package Manager Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lnixmain -Cflags: -I${includedir}/nix -std=c++14 +Cflags: -I${includedir}/nix -std=c++17 diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 91a4eaf922a6..4ed34e54dc55 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -109,7 +109,7 @@ void initNix() opensslLocks = std::vector<std::mutex>(CRYPTO_num_locks()); CRYPTO_set_locking_callback(opensslLockCallback); - settings.loadConfFile(); + loadConfFile(); startSignalHandlerThread(); diff --git a/src/libmain/stack.cc b/src/libmain/stack.cc index cc0eea68fca3..e6224de7d28f 100644 --- a/src/libmain/stack.cc +++ b/src/libmain/stack.cc @@ -30,7 +30,7 @@ static void sigsegvHandler(int signo, siginfo_t * info, void * ctx) if (diff < 0) diff = -diff; if (diff < 4096) { char msg[] = "error: stack overflow (possible infinite recursion)\n"; - [[gnu::unused]] int res = write(2, msg, strlen(msg)); + [[gnu::unused]] auto res = write(2, msg, strlen(msg)); _exit(1); // maybe abort instead? } } @@ -63,7 +63,7 @@ void detectStackOverflow() act.sa_sigaction = sigsegvHandler; act.sa_flags = SA_SIGINFO | SA_ONSTACK; if (sigaction(SIGSEGV, &act, 0)) - throw SysError("resetting SIGCHLD"); + throw SysError("resetting SIGSEGV"); #endif } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 2e9a13e564ca..4527ee6ba660 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -54,17 +54,38 @@ void BinaryCacheStore::init() } } -std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path) +void BinaryCacheStore::getFile(const std::string & path, + Callback<std::shared_ptr<std::string>> callback) +{ + try { + callback(getFile(path)); + } catch (...) { callback.rethrow(); } +} + +void BinaryCacheStore::getFile(const std::string & path, Sink & sink) { std::promise<std::shared_ptr<std::string>> promise; getFile(path, - [&](std::shared_ptr<std::string> result) { - promise.set_value(result); - }, - [&](std::exception_ptr exc) { - promise.set_exception(exc); - }); - return promise.get_future().get(); + {[&](std::future<std::shared_ptr<std::string>> result) { + try { + promise.set_value(result.get()); + } catch (...) { + promise.set_exception(std::current_exception()); + } + }}); + auto data = promise.get_future().get(); + sink((unsigned char *) data->data(), data->size()); +} + +std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path) +{ + StringSink sink; + try { + getFile(path, sink); + } catch (NoSuchBinaryCacheFile &) { + return nullptr; + } + return sink.s; } Path BinaryCacheStore::narInfoFileFor(const Path & storePath) @@ -196,30 +217,30 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink) { auto info = queryPathInfo(storePath).cast<const NarInfo>(); - auto nar = getFile(info->url); - - if (!nar) throw Error(format("file '%s' missing from binary cache") % info->url); - - stats.narRead++; - stats.narReadCompressedBytes += nar->size(); - uint64_t narSize = 0; - StringSource source(*nar); - LambdaSink wrapperSink([&](const unsigned char * data, size_t len) { sink(data, len); narSize += len; }); - decompress(info->compression, source, wrapperSink); + auto decompressor = makeDecompressionSink(info->compression, wrapperSink); + try { + getFile(info->url, *decompressor); + } catch (NoSuchBinaryCacheFile & e) { + throw SubstituteGone(e.what()); + } + + decompressor->finish(); + + stats.narRead++; + //stats.narReadCompressedBytes += nar->size(); // FIXME stats.narReadBytes += narSize; } void BinaryCacheStore::queryPathInfoUncached(const Path & storePath, - std::function<void(std::shared_ptr<ValidPathInfo>)> success, - std::function<void(std::exception_ptr exc)> failure) + Callback<std::shared_ptr<ValidPathInfo>> callback) { auto uri = getUri(); auto act = std::make_shared<Activity>(*logger, lvlTalkative, actQueryPathInfo, @@ -229,17 +250,22 @@ void BinaryCacheStore::queryPathInfoUncached(const Path & storePath, auto narInfoFile = narInfoFileFor(storePath); getFile(narInfoFile, - [=](std::shared_ptr<std::string> data) { - if (!data) return success(0); + {[=](std::future<std::shared_ptr<std::string>> fut) { + try { + auto data = fut.get(); - stats.narInfoRead++; + if (!data) return callback(nullptr); - callSuccess(success, failure, (std::shared_ptr<ValidPathInfo>) - std::make_shared<NarInfo>(*this, *data, narInfoFile)); + stats.narInfoRead++; - (void) act; // force Activity into this lambda to ensure it stays alive - }, - failure); + callback((std::shared_ptr<ValidPathInfo>) + std::make_shared<NarInfo>(*this, *data, narInfoFile)); + + (void) act; // force Activity into this lambda to ensure it stays alive + } catch (...) { + callback.rethrow(); + } + }}); } Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath, diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index e20b968442b7..953f3b90af46 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -38,11 +38,16 @@ public: const std::string & data, const std::string & mimeType) = 0; - /* Return the contents of the specified file, or null if it - doesn't exist. */ + /* Note: subclasses must implement at least one of the two + following getFile() methods. */ + + /* Dump the contents of the specified file to a sink. */ + virtual void getFile(const std::string & path, Sink & sink); + + /* Fetch the specified file and call the specified callback with + the result. A subclass may implement this asynchronously. */ virtual void getFile(const std::string & path, - std::function<void(std::shared_ptr<std::string>)> success, - std::function<void(std::exception_ptr exc)> failure) = 0; + Callback<std::shared_ptr<std::string>> callback); std::shared_ptr<std::string> getFile(const std::string & path); @@ -67,25 +72,11 @@ public: bool isValidPathUncached(const Path & path) override; - PathSet queryAllValidPaths() override - { unsupported(); } - void queryPathInfoUncached(const Path & path, - std::function<void(std::shared_ptr<ValidPathInfo>)> success, - std::function<void(std::exception_ptr exc)> failure) override; - - void queryReferrers(const Path & path, - PathSet & referrers) override - { unsupported(); } - - PathSet queryDerivationOutputs(const Path & path) override - { unsupported(); } - - StringSet queryDerivationOutputNames(const Path & path) override - { unsupported(); } + Callback<std::shared_ptr<ValidPathInfo>> callback) override; Path queryPathFromHashPart(const string & hashPart) override - { unsupported(); } + { unsupported("queryPathFromHashPart"); } bool wantMassQuery() override { return wantMassQuery_; } @@ -104,22 +95,10 @@ public: BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, BuildMode buildMode) override - { unsupported(); } + { unsupported("buildDerivation"); } void ensurePath(const Path & path) override - { unsupported(); } - - void addTempRoot(const Path & path) override - { unsupported(); } - - void addIndirectRoot(const Path & path) override - { unsupported(); } - - Roots findRoots() override - { unsupported(); } - - void collectGarbage(const GCOptions & options, GCResults & results) override - { unsupported(); } + { unsupported("ensurePath"); } ref<FSAccessor> getFSAccessor() override; @@ -131,4 +110,6 @@ public: }; +MakeError(NoSuchBinaryCacheFile, Error); + } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 122a754e1e1d..0bd7388097c6 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -11,6 +11,8 @@ #include "compression.hh" #include "json.hh" #include "nar-info.hh" +#include "parsed-derivations.hh" +#include "machines.hh" #include <algorithm> #include <iostream> @@ -20,6 +22,7 @@ #include <future> #include <chrono> #include <regex> +#include <queue> #include <limits.h> #include <sys/time.h> @@ -29,7 +32,9 @@ #include <sys/utsname.h> #include <sys/select.h> #include <sys/resource.h> +#include <sys/socket.h> #include <fcntl.h> +#include <netdb.h> #include <unistd.h> #include <errno.h> #include <cstring> @@ -456,6 +461,28 @@ static void commonChildInit(Pipe & logPipe) close(fdDevNull); } +void handleDiffHook(uid_t uid, uid_t gid, Path tryA, Path tryB, Path drvPath, Path tmpDir) +{ + auto diffHook = settings.diffHook; + if (diffHook != "" && settings.runDiffHook) { + try { + RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir}); + diffHookOptions.searchPath = true; + diffHookOptions.uid = uid; + diffHookOptions.gid = gid; + diffHookOptions.chdir = "/"; + + auto diffRes = runProgram(diffHookOptions); + if (!statusOk(diffRes.first)) + throw ExecError(diffRes.first, fmt("diff-hook program '%1%' %2%", diffHook, statusToString(diffRes.first))); + + if (diffRes.second != "") + printError(chomp(diffRes.second)); + } catch (Error & error) { + printError("diff hook execution failed: %s", error.what()); + } + } +} ////////////////////////////////////////////////////////////////////// @@ -672,8 +699,10 @@ HookInstance::HookInstance() toHook.readSide = -1; sink = FdSink(toHook.writeSide.get()); - for (auto & setting : settings.getSettings()) - sink << 1 << setting.first << setting.second; + std::map<std::string, Config::SettingInfo> settings; + globalConfig.getSettings(settings); + for (auto & setting : settings) + sink << 1 << setting.first << setting.second.value; sink << 0; } @@ -731,11 +760,13 @@ private: /* Whether to retry substituting the outputs after building the inputs. */ - bool retrySubstitution = false; + bool retrySubstitution; /* The derivation stored at drvPath. */ std::unique_ptr<BasicDerivation> drv; + std::unique_ptr<ParsedDerivation> parsedDrv; + /* The remainder is state held during the build. */ /* Locks on the output paths. */ @@ -842,15 +873,15 @@ private: BuildResult result; /* The current round, if we're building multiple times. */ - unsigned int curRound = 1; + size_t curRound = 1; - unsigned int nrRounds; + size_t nrRounds; /* Path registration info from the previous round, if we're building multiple times. Since this contains the hash, it allows us to compare whether two rounds produced the same result. */ - ValidPathInfos prevInfos; + std::map<Path, ValidPathInfo> prevInfos; const uid_t sandboxUid = 1000; const gid_t sandboxGid = 100; @@ -873,6 +904,9 @@ public: Worker & worker, BuildMode buildMode = bmNormal); ~DerivationGoal(); + /* Whether we need to perform hash rewriting if there are valid output paths. */ + bool needsHashRewrite(); + void timedOut() override; string key() override @@ -931,6 +965,11 @@ private: as valid. */ void registerOutputs(); + /* Check that an output meets the requirements specified by the + 'outputChecks' attribute (or the legacy + '{allowed,disallowed}{References,Requisites}' attributes). */ + void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs); + /* Open a log file and a pipe to it. */ Path openLogFile(); @@ -964,6 +1003,8 @@ private: } void done(BuildResult::Status status, const string & msg = ""); + + PathSet exportReferences(PathSet storePaths); }; @@ -1018,6 +1059,17 @@ DerivationGoal::~DerivationGoal() } +inline bool DerivationGoal::needsHashRewrite() +{ +#if __linux__ + return !useChroot; +#else + /* Darwin requires hash rewriting even when sandboxing is enabled. */ + return true; +#endif +} + + void DerivationGoal::killChild() { if (pid != -1) { @@ -1119,6 +1171,8 @@ void DerivationGoal::haveDerivation() { trace("have derivation"); + retrySubstitution = false; + for (auto & i : drv->outputs) worker.store.addTempRoot(i.second.path); @@ -1131,6 +1185,8 @@ void DerivationGoal::haveDerivation() return; } + parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv); + /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build them. */ @@ -1157,7 +1213,7 @@ void DerivationGoal::outputsSubstituted() /* If the substitutes form an incomplete closure, then we should build the dependencies of this derivation, but after that, we can still use the substitutes for this derivation itself. */ - if (nrIncompleteClosure > 0 && !retrySubstitution) retrySubstitution = true; + if (nrIncompleteClosure > 0) retrySubstitution = true; nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; @@ -1167,7 +1223,7 @@ void DerivationGoal::outputsSubstituted() return; } - unsigned int nrInvalid = checkPathValidity(false, buildMode == bmRepair).size(); + auto nrInvalid = checkPathValidity(false, buildMode == bmRepair).size(); if (buildMode == bmNormal && nrInvalid == 0) { done(BuildResult::Substituted); return; @@ -1387,7 +1443,7 @@ void DerivationGoal::tryToBuild() /* Don't do a remote build if the derivation has the attribute `preferLocalBuild' set. Also, check and repair modes are only supported for local builds. */ - bool buildLocally = buildMode != bmNormal || drv->willBuildLocally(); + bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(); auto started = [&]() { auto msg = fmt( @@ -1633,19 +1689,13 @@ HookReply DerivationGoal::tryBuildHook() try { - /* Tell the hook about system features (beyond the system type) - required from the build machine. (The hook could parse the - drv file itself, but this is easier.) */ - Strings features = tokenizeString<Strings>(get(drv->env, "requiredSystemFeatures")); - for (auto & i : features) checkStoreName(i); /* !!! abuse */ - /* Send the request to the hook. */ worker.hook->sink << "try" << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0) << drv->platform << drvPath - << features; + << parsedDrv->getRequiredSystemFeatures(); worker.hook->sink.flush(); /* Read the first line of input, which should be a word indicating @@ -1730,22 +1780,23 @@ int childEntry(void * arg) } -PathSet exportReferences(Store & store, PathSet storePaths) +PathSet DerivationGoal::exportReferences(PathSet storePaths) { PathSet paths; for (auto storePath : storePaths) { /* Check that the store path is valid. */ - if (!store.isInStore(storePath)) + if (!worker.store.isInStore(storePath)) throw BuildError(format("'exportReferencesGraph' contains a non-store path '%1%'") % storePath); - storePath = store.toStorePath(storePath); - if (!store.isValidPath(storePath)) - throw BuildError(format("'exportReferencesGraph' contains an invalid path '%1%'") - % storePath); - store.computeFSClosure(storePath, paths); + storePath = worker.store.toStorePath(storePath); + + if (!inputPaths.count(storePath)) + throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", storePath); + + worker.store.computeFSClosure(storePath, paths); } /* If there are derivations in the graph, then include their @@ -1756,9 +1807,9 @@ PathSet exportReferences(Store & store, PathSet storePaths) for (auto & j : paths2) { if (isDerivation(j)) { - Derivation drv = store.derivationFromPath(j); + Derivation drv = worker.store.derivationFromPath(j); for (auto & k : drv.outputs) - store.computeFSClosure(k.second.path, paths); + worker.store.computeFSClosure(k.second.path, paths); } } @@ -1770,35 +1821,40 @@ static std::once_flag dns_resolve_flag; static void preloadNSS() { /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already - been loaded in the parent. So we force a download of an invalid URL to force the NSS machinery to + been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to load its lookup libraries in the parent before any child gets a chance to. */ std::call_once(dns_resolve_flag, []() { - DownloadRequest request("http://this.pre-initializes.the.dns.resolvers.invalid"); - request.tries = 1; // We only need to do it once, and this also suppresses an annoying warning - try { getDownloader()->download(request); } catch (...) {} + struct addrinfo *res = NULL; + + if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) != 0) { + if (res) freeaddrinfo(res); + } }); } void DerivationGoal::startBuilder() { /* Right platform? */ - if (!drv->canBuildLocally()) { - throw Error( - format("a '%1%' is required to build '%3%', but I am a '%2%'") - % drv->platform % settings.thisSystem % drvPath); - } + if (!parsedDrv->canBuildLocally()) + throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}", + drv->platform, + concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), + drvPath, + settings.thisSystem, + concatStringsSep(", ", settings.systemFeatures)); if (drv->isBuiltin()) preloadNSS(); #if __APPLE__ - additionalSandboxProfile = get(drv->env, "__sandboxProfile"); + additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or(""); #endif /* Are we doing a chroot build? */ { + auto noChroot = parsedDrv->getBoolAttr("__noChroot"); if (settings.sandboxMode == smEnabled) { - if (get(drv->env, "__noChroot") == "1") + if (noChroot) throw Error(format("derivation '%1%' has '__noChroot' set, " "but that's not allowed when 'sandbox' is 'true'") % drvPath); #if __APPLE__ @@ -1811,7 +1867,7 @@ void DerivationGoal::startBuilder() else if (settings.sandboxMode == smDisabled) useChroot = false; else if (settings.sandboxMode == smRelaxed) - useChroot = !fixedOutput && get(drv->env, "__noChroot") != "1"; + useChroot = !fixedOutput && !noChroot; } if (worker.store.storeDir != worker.store.realStoreDir) { @@ -1862,7 +1918,7 @@ void DerivationGoal::startBuilder() writeStructuredAttrs(); /* Handle exportReferencesGraph(), if set. */ - if (!drv->env.count("__json")) { + if (!parsedDrv->getStructuredAttrs()) { /* The `exportReferencesGraph' feature allows the references graph to be passed to a builder. This attribute should be a list of pairs [name1 path1 name2 path2 ...]. The references graph of @@ -1882,7 +1938,7 @@ void DerivationGoal::startBuilder() /* Write closure info to <fileName>. */ writeFile(tmpDir + "/" + fileName, worker.store.makeValidityRegistration( - exportReferences(worker.store, {storePath}), false, false)); + exportReferences({storePath}), false, false)); } } @@ -1927,7 +1983,7 @@ void DerivationGoal::startBuilder() PathSet allowedPaths = settings.allowedImpureHostPrefixes; /* This works like the above, except on a per-derivation level */ - Strings impurePaths = tokenizeString<Strings>(get(drv->env, "__impureHostDeps")); + auto impurePaths = parsedDrv->getStringsAttr("__impureHostDeps").value_or(Strings()); for (auto & i : impurePaths) { bool found = false; @@ -1996,7 +2052,7 @@ void DerivationGoal::startBuilder() /* Create /etc/hosts with localhost entry. */ if (!fixedOutput) - writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n"); + writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n"); /* Make the closure of the inputs available in the chroot, rather than the whole Nix store. This prevents any access @@ -2053,7 +2109,7 @@ void DerivationGoal::startBuilder() #endif } - else { + if (needsHashRewrite()) { if (pathExists(homeDir)) throw Error(format("directory '%1%' exists; please remove it") % homeDir); @@ -2295,7 +2351,7 @@ void DerivationGoal::initEnv() passAsFile is ignored in structure mode because it's not needed (attributes are not passed through the environment, so there is no size constraint). */ - if (!drv->env.count("__json")) { + if (!parsedDrv->getStructuredAttrs()) { StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile")); int fileNr = 0; @@ -2342,8 +2398,8 @@ void DerivationGoal::initEnv() fixed-output derivations is by definition pure (since we already know the cryptographic hash of the output). */ if (fixedOutput) { - Strings varNames = tokenizeString<Strings>(get(drv->env, "impureEnvVars")); - for (auto & i : varNames) env[i] = getEnv(i); + for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) + env[i] = getEnv(i); } /* Currently structured log messages piggyback on stderr, but we @@ -2358,111 +2414,103 @@ static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); void DerivationGoal::writeStructuredAttrs() { - auto jsonAttr = drv->env.find("__json"); - if (jsonAttr == drv->env.end()) return; - - try { - - auto jsonStr = rewriteStrings(jsonAttr->second, inputRewrites); + auto & structuredAttrs = parsedDrv->getStructuredAttrs(); + if (!structuredAttrs) return; - auto json = nlohmann::json::parse(jsonStr); + auto json = *structuredAttrs; - /* Add an "outputs" object containing the output paths. */ - nlohmann::json outputs; - for (auto & i : drv->outputs) - outputs[i.first] = rewriteStrings(i.second.path, inputRewrites); - json["outputs"] = outputs; - - /* Handle exportReferencesGraph. */ - auto e = json.find("exportReferencesGraph"); - if (e != json.end() && e->is_object()) { - for (auto i = e->begin(); i != e->end(); ++i) { - std::ostringstream str; - { - JSONPlaceholder jsonRoot(str, true); - PathSet storePaths; - for (auto & p : *i) - storePaths.insert(p.get<std::string>()); - worker.store.pathInfoToJSON(jsonRoot, - exportReferences(worker.store, storePaths), false, true); - } - json[i.key()] = nlohmann::json::parse(str.str()); // urgh + /* Add an "outputs" object containing the output paths. */ + nlohmann::json outputs; + for (auto & i : drv->outputs) + outputs[i.first] = rewriteStrings(i.second.path, inputRewrites); + json["outputs"] = outputs; + + /* Handle exportReferencesGraph. */ + auto e = json.find("exportReferencesGraph"); + if (e != json.end() && e->is_object()) { + for (auto i = e->begin(); i != e->end(); ++i) { + std::ostringstream str; + { + JSONPlaceholder jsonRoot(str, true); + PathSet storePaths; + for (auto & p : *i) + storePaths.insert(p.get<std::string>()); + worker.store.pathInfoToJSON(jsonRoot, + exportReferences(storePaths), false, true); } + json[i.key()] = nlohmann::json::parse(str.str()); // urgh } + } - writeFile(tmpDir + "/.attrs.json", json.dump()); + writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); - /* As a convenience to bash scripts, write a shell file that - maps all attributes that are representable in bash - - namely, strings, integers, nulls, Booleans, and arrays and - objects consisting entirely of those values. (So nested - arrays or objects are not supported.) */ + /* As a convenience to bash scripts, write a shell file that + maps all attributes that are representable in bash - + namely, strings, integers, nulls, Booleans, and arrays and + objects consisting entirely of those values. (So nested + arrays or objects are not supported.) */ - auto handleSimpleType = [](const nlohmann::json & value) -> std::experimental::optional<std::string> { - if (value.is_string()) - return shellEscape(value); + auto handleSimpleType = [](const nlohmann::json & value) -> std::optional<std::string> { + if (value.is_string()) + return shellEscape(value); - if (value.is_number()) { - auto f = value.get<float>(); - if (std::ceil(f) == f) - return std::to_string(value.get<int>()); - } + if (value.is_number()) { + auto f = value.get<float>(); + if (std::ceil(f) == f) + return std::to_string(value.get<int>()); + } - if (value.is_null()) - return std::string("''"); + if (value.is_null()) + return std::string("''"); - if (value.is_boolean()) - return value.get<bool>() ? std::string("1") : std::string(""); + if (value.is_boolean()) + return value.get<bool>() ? std::string("1") : std::string(""); - return {}; - }; + return {}; + }; - std::string jsonSh; + std::string jsonSh; - for (auto i = json.begin(); i != json.end(); ++i) { + for (auto i = json.begin(); i != json.end(); ++i) { - if (!std::regex_match(i.key(), shVarName)) continue; + if (!std::regex_match(i.key(), shVarName)) continue; - auto & value = i.value(); + auto & value = i.value(); - auto s = handleSimpleType(value); - if (s) - jsonSh += fmt("declare %s=%s\n", i.key(), *s); + auto s = handleSimpleType(value); + if (s) + jsonSh += fmt("declare %s=%s\n", i.key(), *s); - else if (value.is_array()) { - std::string s2; - bool good = true; + else if (value.is_array()) { + std::string s2; + bool good = true; - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); - if (!s3) { good = false; break; } - s2 += *s3; s2 += ' '; - } - - if (good) - jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2); + for (auto i = value.begin(); i != value.end(); ++i) { + auto s3 = handleSimpleType(i.value()); + if (!s3) { good = false; break; } + s2 += *s3; s2 += ' '; } - else if (value.is_object()) { - std::string s2; - bool good = true; + if (good) + jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2); + } - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); - if (!s3) { good = false; break; } - s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3); - } + else if (value.is_object()) { + std::string s2; + bool good = true; - if (good) - jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2); + for (auto i = value.begin(); i != value.end(); ++i) { + auto s3 = handleSimpleType(i.value()); + if (!s3) { good = false; break; } + s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3); } - } - writeFile(tmpDir + "/.attrs.sh", jsonSh); - - } catch (std::exception & e) { - throw Error("cannot process __json attribute of '%s': %s", drvPath, e.what()); + if (good) + jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2); + } } + + writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); } @@ -2488,14 +2536,18 @@ void setupSeccomp() seccomp_release(ctx); }); - if (settings.thisSystem == "x86_64-linux" && + if (nativeSystem == "x86_64-linux" && seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0) throw SysError("unable to add 32-bit seccomp architecture"); - if (settings.thisSystem == "x86_64-linux" && + if (nativeSystem == "x86_64-linux" && seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0) throw SysError("unable to add X32 seccomp architecture"); + if (nativeSystem == "aarch64-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0) + printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes"); + /* Prevent builders from creating setuid/setgid binaries. */ for (int perm : { S_ISUID, S_ISGID }) { if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1, @@ -2613,7 +2665,7 @@ void DerivationGoal::runChild() createDirs(chrootRootDir + "/dev/shm"); createDirs(chrootRootDir + "/dev/pts"); ss.push_back("/dev/full"); - if (pathExists("/dev/kvm")) + if (settings.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) ss.push_back("/dev/kvm"); ss.push_back("/dev/null"); ss.push_back("/dev/random"); @@ -2696,8 +2748,8 @@ void DerivationGoal::runChild() } else { if (errno != EINVAL) throw SysError("mounting /dev/pts"); - doBind("/dev/pts", "/dev/pts"); - doBind("/dev/ptmx", "/dev/ptmx"); + doBind("/dev/pts", chrootRootDir + "/dev/pts"); + doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx"); } } @@ -2857,6 +2909,10 @@ void DerivationGoal::runChild() for (auto & i : missingPaths) { sandboxProfile += (format("\t(subpath \"%1%\")\n") % i.c_str()).str(); } + /* Also add redirected outputs to the chroot */ + for (auto & i : redirectedOutputs) { + sandboxProfile += (format("\t(subpath \"%1%\")\n") % i.second.c_str()).str(); + } sandboxProfile += ")\n"; /* Our inputs (transitive dependencies and any impurities computed above) @@ -2902,7 +2958,7 @@ void DerivationGoal::runChild() writeFile(sandboxFile, sandboxProfile); - bool allowLocalNetworking = get(drv->env, "__darwinAllowLocalNetworking") == "1"; + bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking"); /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */ @@ -2974,10 +3030,9 @@ void DerivationGoal::runChild() /* Parse a list of reference specifiers. Each element must either be a store path, or the symbolic name of the output of the derivation (such as `out'). */ -PathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv, string attr) +PathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv, const Strings & paths) { PathSet result; - Paths paths = tokenizeString<Paths>(attr); for (auto & i : paths) { if (store.isStorePath(i)) result.insert(i); @@ -3002,7 +3057,7 @@ void DerivationGoal::registerOutputs() if (allValid) return; } - ValidPathInfos infos; + std::map<std::string, ValidPathInfo> infos; /* Set of inodes seen during calls to canonicalisePathMetaData() for this build's outputs. This needs to be shared between @@ -3010,8 +3065,7 @@ void DerivationGoal::registerOutputs() InodesSeen inodesSeen; Path checkSuffix = ".check"; - bool runDiffHook = settings.runDiffHook; - bool keepPreviousRound = settings.keepFailed || runDiffHook; + bool keepPreviousRound = settings.keepFailed || settings.runDiffHook; std::exception_ptr delayedException; @@ -3036,7 +3090,9 @@ void DerivationGoal::registerOutputs() throw SysError(format("moving build output '%1%' from the sandbox to the Nix store") % path); } if (buildMode != bmCheck) actualPath = worker.store.toRealPath(path); - } else { + } + + if (needsHashRewrite()) { Path redirected = redirectedOutputs[path]; if (buildMode == bmRepair && redirectedBadOutputs.find(path) != redirectedBadOutputs.end() @@ -3106,15 +3162,15 @@ void DerivationGoal::registerOutputs() the derivation to its content-addressed location. */ Hash h2 = recursive ? hashPath(h.type, actualPath).first : hashFile(h.type, actualPath); - Path dest = worker.store.makeFixedOutputPath(recursive, h2, drv->env["name"]); + Path dest = worker.store.makeFixedOutputPath(recursive, h2, storePathToName(path)); if (h != h2) { /* Throw an error after registering the path as valid. */ delayedException = std::make_exception_ptr( - BuildError("fixed-output derivation produced path '%s' with %s hash '%s' instead of the expected hash '%s'", - dest, printHashType(h.type), printHash16or32(h2), printHash16or32(h))); + BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", + dest, h.to_string(), h2.to_string())); Path actualDest = worker.store.toRealPath(dest); @@ -3154,11 +3210,17 @@ void DerivationGoal::registerOutputs() if (!worker.store.isValidPath(path)) continue; auto info = *worker.store.queryPathInfo(path); if (hash.first != info.narHash) { - if (settings.keepFailed) { + if (settings.runDiffHook || settings.keepFailed) { Path dst = worker.store.toRealPath(path + checkSuffix); deletePath(dst); if (rename(actualPath.c_str(), dst.c_str())) throw SysError(format("renaming '%1%' to '%2%'") % actualPath % dst); + + handleDiffHook( + buildUser ? buildUser->getUID() : getuid(), + buildUser ? buildUser->getGID() : getgid(), + path, dst, drvPath, tmpDir); + throw Error(format("derivation '%1%' may not be deterministic: output '%2%' differs from '%3%'") % drvPath % path % dst); } else @@ -3187,48 +3249,6 @@ void DerivationGoal::registerOutputs() debug(format("referenced input: '%1%'") % i); } - /* Enforce `allowedReferences' and friends. */ - auto checkRefs = [&](const string & attrName, bool allowed, bool recursive) { - if (drv->env.find(attrName) == drv->env.end()) return; - - PathSet spec = parseReferenceSpecifiers(worker.store, *drv, get(drv->env, attrName)); - - PathSet used; - if (recursive) { - /* Our requisites are the union of the closures of our references. */ - for (auto & i : references) - /* Don't call computeFSClosure on ourselves. */ - if (path != i) - worker.store.computeFSClosure(i, used); - } else - used = references; - - PathSet badPaths; - - for (auto & i : used) - if (allowed) { - if (spec.find(i) == spec.end()) - badPaths.insert(i); - } else { - if (spec.find(i) != spec.end()) - badPaths.insert(i); - } - - if (!badPaths.empty()) { - string badPathsStr; - for (auto & i : badPaths) { - badPathsStr += "\n\t"; - badPathsStr += i; - } - throw BuildError(format("output '%1%' is not allowed to refer to the following paths:%2%") % actualPath % badPathsStr); - } - }; - - checkRefs("allowedReferences", true, false); - checkRefs("allowedRequisites", true, true); - checkRefs("disallowedReferences", false, false); - checkRefs("disallowedRequisites", false, true); - if (curRound == nrRounds) { worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences() worker.markContentsGood(path); @@ -3242,11 +3262,16 @@ void DerivationGoal::registerOutputs() info.ultimate = true; worker.store.signPathInfo(info); - infos.push_back(info); + if (!info.references.empty()) info.ca.clear(); + + infos[i.first] = info; } if (buildMode == bmCheck) return; + /* Apply output checks. */ + checkOutputs(infos); + /* Compare the result with the previous round, and report which path is different, if any.*/ if (curRound > 1 && prevInfos != infos) { @@ -3254,22 +3279,16 @@ void DerivationGoal::registerOutputs() for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j) if (!(*i == *j)) { result.isNonDeterministic = true; - Path prev = i->path + checkSuffix; + Path prev = i->second.path + checkSuffix; bool prevExists = keepPreviousRound && pathExists(prev); auto msg = prevExists - ? fmt("output '%1%' of '%2%' differs from '%3%' from previous round", i->path, drvPath, prev) - : fmt("output '%1%' of '%2%' differs from previous round", i->path, drvPath); - - auto diffHook = settings.diffHook; - if (prevExists && diffHook != "" && runDiffHook) { - try { - auto diff = runProgram(diffHook, true, {prev, i->path}); - if (diff != "") - printError(chomp(diff)); - } catch (Error & error) { - printError("diff hook execution failed: %s", error.what()); - } - } + ? fmt("output '%1%' of '%2%' differs from '%3%' from previous round", i->second.path, drvPath, prev) + : fmt("output '%1%' of '%2%' differs from previous round", i->second.path, drvPath); + + handleDiffHook( + buildUser ? buildUser->getUID() : getuid(), + buildUser ? buildUser->getGID() : getgid(), + prev, i->second.path, drvPath, tmpDir); if (settings.enforceDeterminism) throw NotDeterministic(msg); @@ -3308,7 +3327,11 @@ void DerivationGoal::registerOutputs() /* Register each output path as valid, and register the sets of paths referenced by each of them. If there are cycles in the outputs, this will fail. */ - worker.store.registerValidPaths(infos); + { + ValidPathInfos infos2; + for (auto & i : infos) infos2.push_back(i.second); + worker.store.registerValidPaths(infos2); + } /* In case of a fixed-output derivation hash mismatch, throw an exception now that we have registered the output as valid. */ @@ -3317,6 +3340,158 @@ void DerivationGoal::registerOutputs() } +void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs) +{ + std::map<Path, const ValidPathInfo &> outputsByPath; + for (auto & output : outputs) + outputsByPath.emplace(output.second.path, output.second); + + for (auto & output : outputs) { + auto & outputName = output.first; + auto & info = output.second; + + struct Checks + { + bool ignoreSelfRefs = false; + std::optional<uint64_t> maxSize, maxClosureSize; + std::optional<Strings> allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites; + }; + + /* Compute the closure and closure size of some output. This + is slightly tricky because some of its references (namely + other outputs) may not be valid yet. */ + auto getClosure = [&](const Path & path) + { + uint64_t closureSize = 0; + PathSet pathsDone; + std::queue<Path> pathsLeft; + pathsLeft.push(path); + + while (!pathsLeft.empty()) { + auto path = pathsLeft.front(); + pathsLeft.pop(); + if (!pathsDone.insert(path).second) continue; + + auto i = outputsByPath.find(path); + if (i != outputsByPath.end()) { + closureSize += i->second.narSize; + for (auto & ref : i->second.references) + pathsLeft.push(ref); + } else { + auto info = worker.store.queryPathInfo(path); + closureSize += info->narSize; + for (auto & ref : info->references) + pathsLeft.push(ref); + } + } + + return std::make_pair(pathsDone, closureSize); + }; + + auto applyChecks = [&](const Checks & checks) + { + if (checks.maxSize && info.narSize > *checks.maxSize) + throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes", + info.path, info.narSize, *checks.maxSize); + + if (checks.maxClosureSize) { + uint64_t closureSize = getClosure(info.path).second; + if (closureSize > *checks.maxClosureSize) + throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes", + info.path, closureSize, *checks.maxClosureSize); + } + + auto checkRefs = [&](const std::optional<Strings> & value, bool allowed, bool recursive) + { + if (!value) return; + + PathSet spec = parseReferenceSpecifiers(worker.store, *drv, *value); + + PathSet used = recursive ? getClosure(info.path).first : info.references; + + if (recursive && checks.ignoreSelfRefs) + used.erase(info.path); + + PathSet badPaths; + + for (auto & i : used) + if (allowed) { + if (!spec.count(i)) + badPaths.insert(i); + } else { + if (spec.count(i)) + badPaths.insert(i); + } + + if (!badPaths.empty()) { + string badPathsStr; + for (auto & i : badPaths) { + badPathsStr += "\n "; + badPathsStr += i; + } + throw BuildError("output '%s' is not allowed to refer to the following paths:%s", info.path, badPathsStr); + } + }; + + checkRefs(checks.allowedReferences, true, false); + checkRefs(checks.allowedRequisites, true, true); + checkRefs(checks.disallowedReferences, false, false); + checkRefs(checks.disallowedRequisites, false, true); + }; + + if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { + auto outputChecks = structuredAttrs->find("outputChecks"); + if (outputChecks != structuredAttrs->end()) { + auto output = outputChecks->find(outputName); + + if (output != outputChecks->end()) { + Checks checks; + + auto maxSize = output->find("maxSize"); + if (maxSize != output->end()) + checks.maxSize = maxSize->get<uint64_t>(); + + auto maxClosureSize = output->find("maxClosureSize"); + if (maxClosureSize != output->end()) + checks.maxClosureSize = maxClosureSize->get<uint64_t>(); + + auto get = [&](const std::string & name) -> std::optional<Strings> { + auto i = output->find(name); + if (i != output->end()) { + Strings res; + for (auto j = i->begin(); j != i->end(); ++j) { + if (!j->is_string()) + throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath); + res.push_back(j->get<std::string>()); + } + checks.disallowedRequisites = res; + return res; + } + return {}; + }; + + checks.allowedReferences = get("allowedReferences"); + checks.allowedRequisites = get("allowedRequisites"); + checks.disallowedReferences = get("disallowedReferences"); + checks.disallowedRequisites = get("disallowedRequisites"); + + applyChecks(checks); + } + } + } else { + // legacy non-structured-attributes case + Checks checks; + checks.ignoreSelfRefs = true; + checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences"); + checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites"); + checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences"); + checks.disallowedRequisites = parsedDrv->getStringsAttr("disallowedRequisites"); + applyChecks(checks); + } + } +} + + Path DerivationGoal::openLogFile() { logSize = 0; @@ -3513,8 +3688,8 @@ private: /* The current substituter. */ std::shared_ptr<Store> sub; - /* Whether any substituter can realise this path. */ - bool hasSubstitute; + /* Whether a substituter failed. */ + bool substituterFailed = false; /* Path info returned by the substituter's query info operation. */ std::shared_ptr<const ValidPathInfo> info; @@ -3578,7 +3753,6 @@ public: SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker, RepairFlag repair) : Goal(worker) - , hasSubstitute(false) , repair(repair) { this->storePath = storePath; @@ -3642,9 +3816,9 @@ void SubstitutionGoal::tryNext() /* Hack: don't indicate failure if there were no substituters. In that case the calling derivation should just do a build. */ - amDone(hasSubstitute ? ecFailed : ecNoSubstituters); + amDone(substituterFailed ? ecFailed : ecNoSubstituters); - if (hasSubstitute) { + if (substituterFailed) { worker.failedSubstitutions++; worker.updateProgress(); } @@ -3666,6 +3840,19 @@ void SubstitutionGoal::tryNext() } catch (InvalidPath &) { tryNext(); return; + } catch (SubstituterDisabled &) { + if (settings.tryFallback) { + tryNext(); + return; + } + throw; + } catch (Error & e) { + if (settings.tryFallback) { + printError(e.what()); + tryNext(); + return; + } + throw; } /* Update the total expected download size. */ @@ -3680,8 +3867,6 @@ void SubstitutionGoal::tryNext() worker.updateProgress(); - hasSubstitute = true; - /* Bail out early if this substituter lacks a valid signature. LocalStore::addToStore() also checks for this, but only after we've downloaded the path. */ @@ -3796,8 +3981,19 @@ void SubstitutionGoal::finished() state = &SubstitutionGoal::init; worker.waitForAWhile(shared_from_this()); return; - } catch (Error & e) { - printError(e.msg()); + } catch (std::exception & e) { + printError(e.what()); + + /* Cause the parent build to fail unless --fallback is given, + or the substitute has disappeared. The latter case behaves + the same as the substitute never having existed in the + first place. */ + try { + throw; + } catch (SubstituteGone &) { + } catch (...) { + substituterFailed = true; + } /* Try the next substitute. */ state = &SubstitutionGoal::tryNext; @@ -4257,6 +4453,11 @@ static void primeCache(Store & store, const PathSet & paths) PathSet willBuild, willSubstitute, unknown; unsigned long long downloadSize, narSize; store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); + + if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty()) + throw Error( + "%d derivations need to be built, but neither local builds ('--max-jobs') " + "nor remote builds ('--builders') are enabled", willBuild.size()); } diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 4ca4a838e3c4..92aec63a0379 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -22,52 +22,56 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) return i->second; }; - auto fetch = [&](const string & url) { - /* No need to do TLS verification, because we check the hash of - the result anyway. */ - DownloadRequest request(url); - request.verifyTLS = false; - request.decompress = false; + Path storePath = getAttr("out"); + auto mainUrl = getAttr("url"); + bool unpack = get(drv.env, "unpack", "") == "1"; - /* Note: have to use a fresh downloader here because we're in - a forked process. */ - auto data = makeDownloader()->download(request); - assert(data.data); + /* Note: have to use a fresh downloader here because we're in + a forked process. */ + auto downloader = makeDownloader(); - return data.data; - }; + auto fetch = [&](const std::string & url) { + + auto source = sinkToSource([&](Sink & sink) { + + /* No need to do TLS verification, because we check the hash of + the result anyway. */ + DownloadRequest request(url); + request.verifyTLS = false; + request.decompress = false; + + auto decompressor = makeDecompressionSink( + unpack && hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink); + downloader->download(std::move(request), *decompressor); + decompressor->finish(); + }); + + if (unpack) + restorePath(storePath, *source); + else + writeFile(storePath, *source); - std::shared_ptr<std::string> data; + auto executable = drv.env.find("executable"); + if (executable != drv.env.end() && executable->second == "1") { + if (chmod(storePath.c_str(), 0755) == -1) + throw SysError(format("making '%1%' executable") % storePath); + } + }; + /* Try the hashed mirrors first. */ if (getAttr("outputHashMode") == "flat") for (auto hashedMirror : settings.hashedMirrors.get()) try { if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; auto ht = parseHashType(getAttr("outputHashAlgo")); - data = fetch(hashedMirror + printHashType(ht) + "/" + Hash(getAttr("outputHash"), ht).to_string(Base16, false)); - break; + fetch(hashedMirror + printHashType(ht) + "/" + Hash(getAttr("outputHash"), ht).to_string(Base16, false)); + return; } catch (Error & e) { debug(e.what()); } - if (!data) data = fetch(getAttr("url")); - - Path storePath = getAttr("out"); - - auto unpack = drv.env.find("unpack"); - if (unpack != drv.env.end() && unpack->second == "1") { - if (string(*data, 0, 6) == string("\xfd" "7zXZ\0", 6)) - data = decompress("xz", *data); - StringSource source(*data); - restorePath(storePath, source); - } else - writeFile(storePath, *data); - - auto executable = drv.env.find("executable"); - if (executable != drv.env.end() && executable->second == "1") { - if (chmod(storePath.c_str(), 0755) == -1) - throw SysError(format("making '%1%' executable") % storePath); - } + /* Otherwise try the specified URL. */ + fetch(mainUrl); } } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index a0a0d78b7d30..3961126fff9c 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -36,12 +36,6 @@ Path BasicDerivation::findOutput(const string & id) const } -bool BasicDerivation::willBuildLocally() const -{ - return get(env, "preferLocalBuild") == "1" && canBuildLocally(); -} - - bool BasicDerivation::substitutesAllowed() const { return get(env, "allowSubstitutes", "1") == "1"; @@ -54,22 +48,6 @@ bool BasicDerivation::isBuiltin() const } -bool BasicDerivation::canBuildLocally() const -{ - return platform == settings.thisSystem - || isBuiltin() -#if __linux__ - || (platform == "i686-linux" && settings.thisSystem == "x86_64-linux") - || (platform == "armv6l-linux" && settings.thisSystem == "armv7l-linux") - || (platform == "armv5tel-linux" && (settings.thisSystem == "armv7l-linux" || settings.thisSystem == "armv6l-linux")) -#elif __FreeBSD__ - || (platform == "i686-linux" && settings.thisSystem == "x86_64-freebsd") - || (platform == "i686-linux" && settings.thisSystem == "i686-freebsd") -#endif - ; -} - - Path writeDerivation(ref<Store> store, const Derivation & drv, const string & name, RepairFlag repair) { @@ -350,7 +328,7 @@ Hash hashDerivationModulo(Store & store, Derivation drv) Hash h = drvHashes[i.first]; if (!h) { assert(store.isValidPath(i.first)); - Derivation drv2 = readDerivation(i.first); + Derivation drv2 = readDerivation(store.toRealPath(i.first)); h = hashDerivationModulo(store, drv2); drvHashes[i.first] = h; } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 7b97730d3bf2..9753e796db5f 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -56,14 +56,10 @@ struct BasicDerivation the given derivation. */ Path findOutput(const string & id) const; - bool willBuildLocally() const; - bool substitutesAllowed() const; bool isBuiltin() const; - bool canBuildLocally() const; - /* Return true iff this is a fixed-output derivation. */ bool isFixedOutput() const; diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 4d7f5690192d..22382ab1d6e8 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -7,6 +7,7 @@ #include "s3.hh" #include "compression.hh" #include "pathlocks.hh" +#include "finally.hh" #ifdef ENABLE_S3 #include <aws/core/client/ClientConfiguration.h> @@ -29,12 +30,25 @@ using namespace std::string_literals; namespace nix { -double getTime() +struct DownloadSettings : Config { - struct timeval tv; - gettimeofday(&tv, 0); - return tv.tv_sec + (tv.tv_usec / 1000000.0); -} + Setting<bool> enableHttp2{this, true, "http2", + "Whether to enable HTTP/2 support."}; + + Setting<std::string> userAgentSuffix{this, "", "user-agent-suffix", + "String appended to the user agent in HTTP requests."}; + + Setting<size_t> httpConnections{this, 25, "http-connections", + "Number of parallel HTTP connections.", + {"binary-caches-parallel-connections"}}; + + Setting<unsigned long> connectTimeout{this, 0, "connect-timeout", + "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."}; +}; + +static DownloadSettings downloadSettings; + +static GlobalConfig::Register r1(&downloadSettings); std::string resolveUri(const std::string & uri) { @@ -44,16 +58,6 @@ std::string resolveUri(const std::string & uri) return uri; } -ref<std::string> decodeContent(const std::string & encoding, ref<std::string> data) -{ - if (encoding == "") - return data; - else if (encoding == "br") - return decompress(encoding, *data); - else - throw Error("unsupported Content-Encoding '%s'", encoding); -} - struct CurlDownloader : public Downloader { CURLM * curlm = 0; @@ -61,8 +65,6 @@ struct CurlDownloader : public Downloader std::random_device rd; std::mt19937 mt19937; - bool enableHttp2; - struct DownloadItem : public std::enable_shared_from_this<DownloadItem> { CurlDownloader & downloader; @@ -70,8 +72,7 @@ struct CurlDownloader : public Downloader DownloadResult result; Activity act; bool done = false; // whether either the success or failure function has been called - std::function<void(const DownloadResult &)> success; - std::function<void(std::exception_ptr exc)> failure; + Callback<DownloadResult> callback; CURL * req = 0; bool active = false; // whether the handle has been added to the multi object std::string status; @@ -86,10 +87,21 @@ struct CurlDownloader : public Downloader std::string encoding; - DownloadItem(CurlDownloader & downloader, const DownloadRequest & request) + DownloadItem(CurlDownloader & downloader, + const DownloadRequest & request, + Callback<DownloadResult> callback) : downloader(downloader) , request(request) - , act(*logger, lvlTalkative, actDownload, fmt("downloading '%s'", request.uri), {request.uri}, request.parentAct) + , act(*logger, lvlTalkative, actDownload, + fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri), + {request.uri}, request.parentAct) + , callback(callback) + , finalSink([this](const unsigned char * data, size_t len) { + if (this->request.dataCallback) + this->request.dataCallback((char *) data, len); + else + this->result.data->append((char *) data, len); + }) { if (!request.expectedETag.empty()) requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str()); @@ -113,19 +125,40 @@ struct CurlDownloader : public Downloader } } - template<class T> - void fail(const T & e) + void failEx(std::exception_ptr ex) { assert(!done); done = true; - callFailure(failure, std::make_exception_ptr(e)); + callback.rethrow(ex); } + template<class T> + void fail(const T & e) + { + failEx(std::make_exception_ptr(e)); + } + + LambdaSink finalSink; + std::shared_ptr<CompressionSink> decompressionSink; + + std::exception_ptr writeException; + size_t writeCallback(void * contents, size_t size, size_t nmemb) { - size_t realSize = size * nmemb; - result.data->append((char *) contents, realSize); - return realSize; + try { + size_t realSize = size * nmemb; + result.bodySize += realSize; + + if (!decompressionSink) + decompressionSink = makeDecompressionSink(encoding, finalSink); + + (*decompressionSink)((unsigned char *) contents, realSize); + + return realSize; + } catch (...) { + writeException = std::current_exception(); + return 0; + } } static size_t writeCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) @@ -143,6 +176,7 @@ struct CurlDownloader : public Downloader auto ss = tokenizeString<vector<string>>(line, " "); status = ss.size() >= 2 ? ss[1] : ""; result.data = std::make_shared<std::string>(); + result.bodySize = 0; encoding = ""; } else { auto i = line.find(':'); @@ -173,7 +207,11 @@ struct CurlDownloader : public Downloader int progressCallback(double dltotal, double dlnow) { - act.progress(dlnow, dltotal); + try { + act.progress(dlnow, dltotal); + } catch (nix::Interrupted &) { + assert(_isInterrupted); + } return _isInterrupted; } @@ -190,7 +228,7 @@ struct CurlDownloader : public Downloader } size_t readOffset = 0; - int readCallback(char *buffer, size_t size, size_t nitems) + size_t readCallback(char *buffer, size_t size, size_t nitems) { if (readOffset == request.data->length()) return 0; @@ -201,7 +239,7 @@ struct CurlDownloader : public Downloader return count; } - static int readCallbackWrapper(char *buffer, size_t size, size_t nitems, void * userp) + static size_t readCallbackWrapper(char *buffer, size_t size, size_t nitems, void * userp) { return ((DownloadItem *) userp)->readCallback(buffer, size, nitems); } @@ -221,15 +259,16 @@ struct CurlDownloader : public Downloader curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str()); curl_easy_setopt(req, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(req, CURLOPT_MAXREDIRS, 10); curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1); curl_easy_setopt(req, CURLOPT_USERAGENT, ("curl/" LIBCURL_VERSION " Nix/" + nixVersion + - (settings.userAgentSuffix != "" ? " " + settings.userAgentSuffix.get() : "")).c_str()); + (downloadSettings.userAgentSuffix != "" ? " " + downloadSettings.userAgentSuffix.get() : "")).c_str()); #if LIBCURL_VERSION_NUM >= 0x072b00 curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1); #endif #if LIBCURL_VERSION_NUM >= 0x072f00 - if (downloader.enableHttp2) + if (downloadSettings.enableHttp2) curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS); #endif curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, DownloadItem::writeCallbackWrapper); @@ -261,7 +300,7 @@ struct CurlDownloader : public Downloader curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0); } - curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, settings.connectTimeout.get()); + curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, downloadSettings.connectTimeout.get()); curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L); curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, lowSpeedTimeout); @@ -272,6 +311,7 @@ struct CurlDownloader : public Downloader curl_easy_setopt(req, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); result.data = std::make_shared<std::string>(); + result.bodySize = 0; } void finish(CURLcode code) @@ -284,34 +324,40 @@ struct CurlDownloader : public Downloader if (effectiveUrlCStr) result.effectiveUrl = effectiveUrlCStr; - debug(format("finished download of '%s'; curl status = %d, HTTP status = %d, body = %d bytes") - % request.uri % code % httpStatus % (result.data ? result.data->size() : 0)); + debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes", + request.verb(), request.uri, code, httpStatus, result.bodySize); + + if (decompressionSink) + decompressionSink->finish(); if (code == CURLE_WRITE_ERROR && result.etag == request.expectedETag) { code = CURLE_OK; httpStatus = 304; } - if (code == CURLE_OK && + if (writeException) + failEx(writeException); + + else if (code == CURLE_OK && (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */)) { result.cached = httpStatus == 304; done = true; try { - if (request.decompress) - result.data = decodeContent(encoding, ref<std::string>(result.data)); - callSuccess(success, failure, const_cast<const DownloadResult &>(result)); - act.progress(result.data->size(), result.data->size()); + act.progress(result.bodySize, result.bodySize); + callback(std::move(result)); } catch (...) { done = true; - callFailure(failure, std::current_exception()); + callback.rethrow(); } - } else { + } + + else { // We treat most errors as transient, but won't retry when hopeless Error err = Transient; - if (httpStatus == 404 || code == CURLE_FILE_COULDNT_READ_FILE) { + if (httpStatus == 404 || httpStatus == 410 || code == CURLE_FILE_COULDNT_READ_FILE) { // The file is definitely not there err = NotFound; } else if (httpStatus == 401 || httpStatus == 403 || httpStatus == 407) { @@ -341,6 +387,8 @@ struct CurlDownloader : public Downloader case CURLE_INTERFACE_FAILED: case CURLE_UNKNOWN_OPTION: case CURLE_SSL_CACERT_BADFILE: + case CURLE_TOO_MANY_REDIRECTS: + case CURLE_WRITE_ERROR: err = Misc; break; default: // Shut up warnings @@ -352,10 +400,16 @@ struct CurlDownloader : public Downloader auto exc = code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted - ? DownloadError(Interrupted, format("download of '%s' was interrupted") % request.uri) + ? DownloadError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) : httpStatus != 0 - ? DownloadError(err, format("unable to download '%s': HTTP error %d (curl error: %s)") % request.uri % httpStatus % curl_easy_strerror(code)) - : DownloadError(err, format("unable to download '%s': %s (%d)") % request.uri % curl_easy_strerror(code) % code); + ? DownloadError(err, + fmt("unable to %s '%s': HTTP error %d", + request.verb(), request.uri, httpStatus) + + (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) + ) + : DownloadError(err, + fmt("unable to %s '%s': %s (%d)", + request.verb(), request.uri, curl_easy_strerror(code), code)); /* If this is a transient error, then maybe retry the download after a while. */ @@ -404,11 +458,9 @@ struct CurlDownloader : public Downloader #endif #if LIBCURL_VERSION_NUM >= 0x071e00 // Max connections requires >= 7.30.0 curl_multi_setopt(curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS, - settings.binaryCachesParallelConnections.get()); + downloadSettings.httpConnections.get()); #endif - enableHttp2 = settings.enableHttp2; - wakeupPipe.create(); fcntl(wakeupPipe.readSide.get(), F_SETFL, O_NONBLOCK); @@ -476,10 +528,11 @@ struct CurlDownloader : public Downloader extraFDs[0].fd = wakeupPipe.readSide.get(); extraFDs[0].events = CURL_WAIT_POLLIN; extraFDs[0].revents = 0; + long maxSleepTimeMs = items.empty() ? 10000 : 100; auto sleepTimeMs = nextWakeup != std::chrono::steady_clock::time_point() ? std::max(0, (int) std::chrono::duration_cast<std::chrono::milliseconds>(nextWakeup - std::chrono::steady_clock::now()).count()) - : 10000; + : maxSleepTimeMs; vomit("download thread waiting for %d ms", sleepTimeMs); mc = curl_multi_wait(curlm, extraFDs, 1, sleepTimeMs, &numfds); if (mc != CURLM_OK) @@ -518,7 +571,7 @@ struct CurlDownloader : public Downloader } for (auto & item : incoming) { - debug(format("starting download of %s") % item->request.uri); + debug("starting %s of %s", item->request.verb(), item->request.uri); item->init(); curl_multi_add_handle(curlm, item->req); item->active = true; @@ -547,6 +600,11 @@ struct CurlDownloader : public Downloader void enqueueItem(std::shared_ptr<DownloadItem> item) { + if (item->request.data + && !hasPrefix(item->request.uri, "http://") + && !hasPrefix(item->request.uri, "https://")) + throw nix::Error("uploading to '%s' is not supported", item->request.uri); + { auto state(state_.lock()); if (state->quit) @@ -556,48 +614,61 @@ struct CurlDownloader : public Downloader writeFull(wakeupPipe.writeSide.get(), " "); } +#ifdef ENABLE_S3 + std::tuple<std::string, std::string, Store::Params> parseS3Uri(std::string uri) + { + auto [path, params] = splitUriAndParams(uri); + + auto slash = path.find('/', 5); // 5 is the length of "s3://" prefix + if (slash == std::string::npos) + throw nix::Error("bad S3 URI '%s'", path); + + std::string bucketName(path, 5, slash - 5); + std::string key(path, slash + 1); + + return {bucketName, key, params}; + } +#endif + void enqueueDownload(const DownloadRequest & request, - std::function<void(const DownloadResult &)> success, - std::function<void(std::exception_ptr exc)> failure) override + Callback<DownloadResult> callback) override { /* Ugly hack to support s3:// URIs. */ if (hasPrefix(request.uri, "s3://")) { // FIXME: do this on a worker thread - sync2async<DownloadResult>(success, failure, [&]() -> DownloadResult { + try { #ifdef ENABLE_S3 - S3Helper s3Helper("", Aws::Region::US_EAST_1); // FIXME: make configurable - auto slash = request.uri.find('/', 5); - if (slash == std::string::npos) - throw nix::Error("bad S3 URI '%s'", request.uri); - std::string bucketName(request.uri, 5, slash - 5); - std::string key(request.uri, slash + 1); + auto [bucketName, key, params] = parseS3Uri(request.uri); + + std::string profile = get(params, "profile", ""); + std::string region = get(params, "region", Aws::Region::US_EAST_1); + std::string scheme = get(params, "scheme", ""); + std::string endpoint = get(params, "endpoint", ""); + + S3Helper s3Helper(profile, region, scheme, endpoint); + // FIXME: implement ETag auto s3Res = s3Helper.getObject(bucketName, key); DownloadResult res; if (!s3Res.data) throw DownloadError(NotFound, fmt("S3 object '%s' does not exist", request.uri)); res.data = s3Res.data; - return res; + callback(std::move(res)); #else throw nix::Error("cannot download '%s' because Nix is not built with S3 support", request.uri); #endif - }); + } catch (...) { callback.rethrow(); } return; } - auto item = std::make_shared<DownloadItem>(*this, request); - item->success = success; - item->failure = failure; - enqueueItem(item); + enqueueItem(std::make_shared<DownloadItem>(*this, request, callback)); } }; ref<Downloader> getDownloader() { - static std::shared_ptr<Downloader> downloader; - static std::once_flag downloaderCreated; - std::call_once(downloaderCreated, [&]() { downloader = makeDownloader(); }); - return ref<Downloader>(downloader); + static ref<Downloader> downloader = makeDownloader(); + return downloader; } ref<Downloader> makeDownloader() @@ -609,8 +680,13 @@ std::future<DownloadResult> Downloader::enqueueDownload(const DownloadRequest & { auto promise = std::make_shared<std::promise<DownloadResult>>(); enqueueDownload(request, - [promise](const DownloadResult & result) { promise->set_value(result); }, - [promise](std::exception_ptr exc) { promise->set_exception(exc); }); + {[promise](std::future<DownloadResult> fut) { + try { + promise->set_value(fut.get()); + } catch (...) { + promise->set_exception(std::current_exception()); + } + }}); return promise->get_future(); } @@ -619,7 +695,102 @@ DownloadResult Downloader::download(const DownloadRequest & request) return enqueueDownload(request).get(); } -Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpack, string name, const Hash & expectedHash, string * effectiveUrl) +void Downloader::download(DownloadRequest && request, Sink & sink) +{ + /* Note: we can't call 'sink' via request.dataCallback, because + that would cause the sink to execute on the downloader + thread. If 'sink' is a coroutine, this will fail. Also, if the + sink is expensive (e.g. one that does decompression and writing + to the Nix store), it would stall the download thread too much. + Therefore we use a buffer to communicate data between the + download thread and the calling thread. */ + + struct State { + bool quit = false; + std::exception_ptr exc; + std::string data; + std::condition_variable avail, request; + }; + + auto _state = std::make_shared<Sync<State>>(); + + /* In case of an exception, wake up the download thread. FIXME: + abort the download request. */ + Finally finally([&]() { + auto state(_state->lock()); + state->quit = true; + state->request.notify_one(); + }); + + request.dataCallback = [_state](char * buf, size_t len) { + + auto state(_state->lock()); + + if (state->quit) return; + + /* If the buffer is full, then go to sleep until the calling + thread wakes us up (i.e. when it has removed data from the + buffer). We don't wait forever to prevent stalling the + download thread. (Hopefully sleeping will throttle the + sender.) */ + if (state->data.size() > 1024 * 1024) { + debug("download buffer is full; going to sleep"); + state.wait_for(state->request, std::chrono::seconds(10)); + } + + /* Append data to the buffer and wake up the calling + thread. */ + state->data.append(buf, len); + state->avail.notify_one(); + }; + + enqueueDownload(request, + {[_state](std::future<DownloadResult> fut) { + auto state(_state->lock()); + state->quit = true; + try { + fut.get(); + } catch (...) { + state->exc = std::current_exception(); + } + state->avail.notify_one(); + state->request.notify_one(); + }}); + + while (true) { + checkInterrupt(); + + std::string chunk; + + /* Grab data if available, otherwise wait for the download + thread to wake us up. */ + { + auto state(_state->lock()); + + while (state->data.empty()) { + + if (state->quit) { + if (state->exc) std::rethrow_exception(state->exc); + return; + } + + state.wait(state->avail); + } + + chunk = std::move(state->data); + + state->request.notify_one(); + } + + /* Flush the data to the sink and wake up the download thread + if it's blocked on a full buffer. We don't hold the state + lock while doing this to prevent blocking the download + thread if sink() takes a long time. */ + sink((unsigned char *) chunk.data(), chunk.size()); + } +} + +Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpack, string name, const Hash & expectedHash, string * effectiveUrl, int ttl) { auto url = resolveUri(url_); @@ -649,7 +820,6 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa string expectedETag; - int ttl = settings.tarballTtl; bool skip = false; if (pathExists(fileLink) && pathExists(dataFile)) { @@ -730,7 +900,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa Hash gotHash = unpack ? hashPath(expectedHash.type, store->toRealPath(storePath)).first : hashFile(expectedHash.type, store->toRealPath(storePath)); - throw nix::Error("hash mismatch in file downloaded from '%s': expected hash '%s', got '%s'", + throw nix::Error("hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", url, expectedHash.to_string(), gotHash.to_string()); } diff --git a/src/libstore/download.hh b/src/libstore/download.hh index 0b8d29b21dfe..f0228f7d053a 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -2,6 +2,7 @@ #include "types.hh" #include "hash.hh" +#include "globals.hh" #include <string> #include <future> @@ -20,9 +21,15 @@ struct DownloadRequest bool decompress = true; std::shared_ptr<std::string> data; std::string mimeType; + std::function<void(char *, size_t)> dataCallback; DownloadRequest(const std::string & uri) : uri(uri), parentAct(getCurActivity()) { } + + std::string verb() + { + return data ? "upload" : "download"; + } }; struct DownloadResult @@ -31,6 +38,7 @@ struct DownloadResult std::string etag; std::string effectiveUrl; std::shared_ptr<std::string> data; + uint64_t bodySize = 0; }; class Store; @@ -41,20 +49,23 @@ struct Downloader the download. The future may throw a DownloadError exception. */ virtual void enqueueDownload(const DownloadRequest & request, - std::function<void(const DownloadResult &)> success, - std::function<void(std::exception_ptr exc)> failure) = 0; + Callback<DownloadResult> callback) = 0; std::future<DownloadResult> enqueueDownload(const DownloadRequest & request); /* Synchronously download a file. */ DownloadResult download(const DownloadRequest & request); + /* Download a file, writing its data to a sink. The sink will be + invoked on the thread of the caller. */ + void download(DownloadRequest && request, Sink & sink); + /* Check if the specified file is already in ~/.cache/nix/tarballs and is more recent than ‘tarball-ttl’ seconds. Otherwise, use the recorded ETag to verify if the server has a more recent version, and if so, download it to the Nix store. */ Path downloadCached(ref<Store> store, const string & uri, bool unpack, string name = "", - const Hash & expectedHash = Hash(), string * effectiveUri = nullptr); + const Hash & expectedHash = Hash(), string * effectiveUri = nullptr, int ttl = settings.tarballTtl); enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; }; @@ -77,7 +88,4 @@ public: bool isUri(const string & s); -/* Decode data according to the Content-Encoding header. */ -ref<std::string> decodeContent(const std::string & encoding, ref<std::string> data); - } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index ba49749d830a..26e2b0dca7ca 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -7,6 +7,7 @@ #include <queue> #include <algorithm> #include <regex> +#include <random> #include <sys/types.h> #include <sys/stat.h> @@ -128,8 +129,8 @@ Path LocalFSStore::addPermRoot(const Path & _storePath, check if the root is in a directory in or linked from the gcroots directory. */ if (settings.checkRootReachability) { - Roots roots = findRoots(); - if (roots.find(gcRoot) == roots.end()) + Roots roots = findRoots(false); + if (roots[storePath].count(gcRoot) == 0) printError( format( "warning: '%1%' is not in a directory where the garbage collector looks for roots; " @@ -196,10 +197,11 @@ void LocalStore::addTempRoot(const Path & path) } -std::set<std::pair<pid_t, Path>> LocalStore::readTempRoots(FDs & fds) -{ - std::set<std::pair<pid_t, Path>> tempRoots; +static std::string censored = "{censored}"; + +void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) +{ /* Read the `temproots' directory for per-process temporary root files. */ for (auto & i : readDirectory(tempRootsDir)) { @@ -249,14 +251,12 @@ std::set<std::pair<pid_t, Path>> LocalStore::readTempRoots(FDs & fds) Path root(contents, pos, end - pos); debug("got temporary root '%s'", root); assertStorePath(root); - tempRoots.emplace(pid, root); + tempRoots[root].emplace(censor ? censored : fmt("{temp:%d}", pid)); pos = end + 1; } fds.push_back(fd); /* keep open */ } - - return tempRoots; } @@ -265,7 +265,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) auto foundRoot = [&](const Path & path, const Path & target) { Path storePath = toStorePath(target); if (isStorePath(storePath) && isValidPath(storePath)) - roots[path] = storePath; + roots[storePath].emplace(path); else printInfo(format("skipping invalid root from '%1%' to '%2%'") % path % storePath); }; @@ -305,7 +305,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) else if (type == DT_REG) { Path storePath = storeDir + "/" + baseNameOf(path); if (isStorePath(storePath) && isValidPath(storePath)) - roots[path] = storePath; + roots[storePath].emplace(path); } } @@ -320,44 +320,31 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) } -Roots LocalStore::findRootsNoTemp() +void LocalStore::findRootsNoTemp(Roots & roots, bool censor) { - Roots roots; - /* Process direct roots in {gcroots,profiles}. */ findRoots(stateDir + "/" + gcRootsDir, DT_UNKNOWN, roots); findRoots(stateDir + "/profiles", DT_UNKNOWN, roots); - /* Add additional roots returned by the program specified by the - NIX_ROOT_FINDER environment variable. This is typically used - to add running programs to the set of roots (to prevent them - from being garbage collected). */ - size_t n = 0; - for (auto & root : findRuntimeRoots()) - roots[fmt("{memory:%d}", n++)] = root; - - return roots; + /* Add additional roots returned by different platforms-specific + heuristics. This is typically used to add running programs to + the set of roots (to prevent them from being garbage collected). */ + findRuntimeRoots(roots, censor); } -Roots LocalStore::findRoots() +Roots LocalStore::findRoots(bool censor) { - Roots roots = findRootsNoTemp(); + Roots roots; + findRootsNoTemp(roots, censor); FDs fds; - pid_t prev = -1; - size_t n = 0; - for (auto & root : readTempRoots(fds)) { - if (prev != root.first) n = 0; - prev = root.first; - roots[fmt("{temp:%d:%d}", root.first, n++)] = root.second; - } + findTempRoots(fds, roots, censor); return roots; } - -static void readProcLink(const string & file, StringSet & paths) +static void readProcLink(const string & file, Roots & roots) { /* 64 is the starting buffer size gnu readlink uses... */ auto bufsiz = ssize_t{64}; @@ -365,7 +352,7 @@ try_again: char buf[bufsiz]; auto res = readlink(file.c_str(), buf, bufsiz); if (res == -1) { - if (errno == ENOENT || errno == EACCES) + if (errno == ENOENT || errno == EACCES || errno == ESRCH) return; throw SysError("reading symlink"); } @@ -376,8 +363,8 @@ try_again: goto try_again; } if (res > 0 && buf[0] == '/') - paths.emplace(static_cast<char *>(buf), res); - return; + roots[std::string(static_cast<char *>(buf), res)] + .emplace(file); } static string quoteRegexChars(const string & raw) @@ -386,20 +373,20 @@ static string quoteRegexChars(const string & raw) return std::regex_replace(raw, specialRegex, R"(\$&)"); } -static void readFileRoots(const char * path, StringSet & paths) +static void readFileRoots(const char * path, Roots & roots) { try { - paths.emplace(readFile(path)); + roots[readFile(path)].emplace(path); } catch (SysError & e) { if (e.errNo != ENOENT && e.errNo != EACCES) throw; } } -PathSet LocalStore::findRuntimeRoots() +void LocalStore::findRuntimeRoots(Roots & roots, bool censor) { - PathSet roots; - StringSet paths; + Roots unchecked; + auto procDir = AutoCloseDir{opendir("/proc")}; if (procDir) { struct dirent * ent; @@ -409,10 +396,10 @@ PathSet LocalStore::findRuntimeRoots() while (errno = 0, ent = readdir(procDir.get())) { checkInterrupt(); if (std::regex_match(ent->d_name, digitsRegex)) { - readProcLink((format("/proc/%1%/exe") % ent->d_name).str(), paths); - readProcLink((format("/proc/%1%/cwd") % ent->d_name).str(), paths); + readProcLink(fmt("/proc/%s/exe" ,ent->d_name), unchecked); + readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked); - auto fdStr = (format("/proc/%1%/fd") % ent->d_name).str(); + auto fdStr = fmt("/proc/%s/fd", ent->d_name); auto fdDir = AutoCloseDir(opendir(fdStr.c_str())); if (!fdDir) { if (errno == ENOENT || errno == EACCES) @@ -421,29 +408,32 @@ PathSet LocalStore::findRuntimeRoots() } struct dirent * fd_ent; while (errno = 0, fd_ent = readdir(fdDir.get())) { - if (fd_ent->d_name[0] != '.') { - readProcLink((format("%1%/%2%") % fdStr % fd_ent->d_name).str(), paths); - } + if (fd_ent->d_name[0] != '.') + readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked); } - if (errno) + if (errno) { + if (errno == ESRCH) + continue; throw SysError(format("iterating /proc/%1%/fd") % ent->d_name); - fdDir.reset(); - - auto mapLines = - tokenizeString<std::vector<string>>(readFile((format("/proc/%1%/maps") % ent->d_name).str(), true), "\n"); - for (const auto& line : mapLines) { - auto match = std::smatch{}; - if (std::regex_match(line, match, mapRegex)) - paths.emplace(match[1]); } + fdDir.reset(); try { - auto envString = readFile((format("/proc/%1%/environ") % ent->d_name).str(), true); + auto mapFile = fmt("/proc/%s/maps", ent->d_name); + auto mapLines = tokenizeString<std::vector<string>>(readFile(mapFile, true), "\n"); + for (const auto & line : mapLines) { + auto match = std::smatch{}; + if (std::regex_match(line, match, mapRegex)) + unchecked[match[1]].emplace(mapFile); + } + + auto envFile = fmt("/proc/%s/environ", ent->d_name); + auto envString = readFile(envFile, true); auto env_end = std::sregex_iterator{}; for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i) - paths.emplace(i->str()); + unchecked[i->str()].emplace(envFile); } catch (SysError & e) { - if (errno == ENOENT || errno == EACCES) + if (errno == ENOENT || errno == EACCES || errno == ESRCH) continue; throw; } @@ -461,7 +451,7 @@ PathSet LocalStore::findRuntimeRoots() for (const auto & line : lsofLines) { std::smatch match; if (std::regex_match(line, match, lsofRegex)) - paths.emplace(match[1]); + unchecked[match[1]].emplace("{lsof}"); } } catch (ExecError & e) { /* lsof not installed, lsof failed */ @@ -469,21 +459,23 @@ PathSet LocalStore::findRuntimeRoots() #endif #if defined(__linux__) - readFileRoots("/proc/sys/kernel/modprobe", paths); - readFileRoots("/proc/sys/kernel/fbsplash", paths); - readFileRoots("/proc/sys/kernel/poweroff_cmd", paths); + readFileRoots("/proc/sys/kernel/modprobe", unchecked); + readFileRoots("/proc/sys/kernel/fbsplash", unchecked); + readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked); #endif - for (auto & i : paths) - if (isInStore(i)) { - Path path = toStorePath(i); - if (roots.find(path) == roots.end() && isStorePath(path) && isValidPath(path)) { + for (auto & [target, links] : unchecked) { + if (isInStore(target)) { + Path path = toStorePath(target); + if (isStorePath(path) && isValidPath(path)) { debug(format("got additional root '%1%'") % path); - roots.insert(path); + if (censor) + roots[path].insert(censored); + else + roots[path].insert(links.begin(), links.end()); } } - - return roots; + } } @@ -750,16 +742,20 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* Find the roots. Since we've grabbed the GC lock, the set of permanent roots cannot increase now. */ printError(format("finding garbage collector roots...")); - Roots rootMap = options.ignoreLiveness ? Roots() : findRootsNoTemp(); + Roots rootMap; + if (!options.ignoreLiveness) + findRootsNoTemp(rootMap, true); - for (auto & i : rootMap) state.roots.insert(i.second); + for (auto & i : rootMap) state.roots.insert(i.first); /* Read the temporary roots. This acquires read locks on all per-process temporary root files. So after this point no paths can be added to the set of temporary roots. */ FDs fds; - for (auto & root : readTempRoots(fds)) - state.tempRoots.insert(root.second); + Roots tempRoots; + findTempRoots(fds, tempRoots, true); + for (auto & root : tempRoots) + state.tempRoots.insert(root.first); state.roots.insert(state.tempRoots.begin(), state.tempRoots.end()); /* After this point the set of roots or temporary roots cannot @@ -829,7 +825,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) alphabetically first (e.g. /nix/store/000...). This matters when using --max-freed etc. */ vector<Path> entries_(entries.begin(), entries.end()); - random_shuffle(entries_.begin(), entries_.end()); + std::mt19937 gen(1); + std::shuffle(entries_.begin(), entries_.end(), gen); for (auto & i : entries_) tryToDelete(state, i); diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 544566e0b573..1c2c08715a14 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -28,9 +28,10 @@ namespace nix { Settings settings; +static GlobalConfig::Register r1(&settings); + Settings::Settings() - : Config({}) - , nixPrefix(NIX_PREFIX) + : nixPrefix(NIX_PREFIX) , nixStore(canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR)))) , nixDataDir(canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR))) , nixLogDir(canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR))) @@ -69,25 +70,39 @@ Settings::Settings() allowedImpureHostPrefixes = tokenizeString<StringSet>(DEFAULT_ALLOWED_IMPURE_PREFIXES); } -void Settings::loadConfFile() +void loadConfFile() { - applyConfigFile(nixConfDir + "/nix.conf"); + globalConfig.applyConfigFile(settings.nixConfDir + "/nix.conf"); /* We only want to send overrides to the daemon, i.e. stuff from ~/.nix/nix.conf or the command line. */ - resetOverriden(); + globalConfig.resetOverriden(); - applyConfigFile(getConfigDir() + "/nix/nix.conf"); + auto dirs = getConfigDirs(); + // Iterate over them in reverse so that the ones appearing first in the path take priority + for (auto dir = dirs.rbegin(); dir != dirs.rend(); dir++) { + globalConfig.applyConfigFile(*dir + "/nix/nix.conf"); + } } -void Settings::set(const string & name, const string & value) +unsigned int Settings::getDefaultCores() { - Config::set(name, value); + return std::max(1U, std::thread::hardware_concurrency()); } -unsigned int Settings::getDefaultCores() +StringSet Settings::getDefaultSystemFeatures() { - return std::max(1U, std::thread::hardware_concurrency()); + /* For backwards compatibility, accept some "features" that are + used in Nixpkgs to route builds to certain machines but don't + actually require anything special on the machines. */ + StringSet features{"nixos-test", "benchmark", "big-parallel"}; + + #if __linux__ + if (access("/dev/kvm", R_OK | W_OK) == 0) + features.insert("kvm"); + #endif + + return features; } const string nixVersion = PACKAGE_VERSION; @@ -162,23 +177,11 @@ void initPlugins() throw Error("could not dynamically open plugin file '%s': %s", file, dlerror()); } } - /* We handle settings registrations here, since plugins can add settings */ - if (RegisterSetting::settingRegistrations) { - for (auto & registration : *RegisterSetting::settingRegistrations) - settings.addSetting(registration); - delete RegisterSetting::settingRegistrations; - } - settings.handleUnknownSettings(); -} -RegisterSetting::SettingRegistrations * RegisterSetting::settingRegistrations; - -RegisterSetting::RegisterSetting(AbstractSetting * s) -{ - if (!settingRegistrations) - settingRegistrations = new SettingRegistrations; - settingRegistrations->emplace_back(s); + /* Since plugins can add settings, try to re-apply previously + unknown settings. */ + globalConfig.reapplyUnknownSettings(); + globalConfig.warnUnknownSettings(); } - } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 7430bbedbe44..53efc6a90fb6 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -13,26 +13,6 @@ namespace nix { typedef enum { smEnabled, smRelaxed, smDisabled } SandboxMode; -extern bool useCaseHack; // FIXME - -struct CaseHackSetting : public BaseSetting<bool> -{ - CaseHackSetting(Config * options, - const std::string & name, - const std::string & description, - const std::set<std::string> & aliases = {}) - : BaseSetting<bool>(useCaseHack, name, description, aliases) - { - options->addSetting(this); - } - - void set(const std::string & str) override - { - BaseSetting<bool>::set(str); - nix::useCaseHack = value; - } -}; - struct MaxBuildJobsSetting : public BaseSetting<unsigned int> { MaxBuildJobsSetting(Config * options, @@ -52,14 +32,12 @@ class Settings : public Config { unsigned int getDefaultCores(); + StringSet getDefaultSystemFeatures(); + public: Settings(); - void loadConfFile(); - - void set(const string & name, const string & value); - Path nixPrefix; /* The directory where we store sources and derived files. */ @@ -104,9 +82,9 @@ public: /* Whether to show build log output in real time. */ bool verboseBuild = true; - /* If verboseBuild is false, the number of lines of the tail of - the log to show if a build fails. */ - size_t logLines = 10; + Setting<size_t> logLines{this, 10, "log-lines", + "If verbose-build is false, the number of lines of the tail of " + "the log to show if a build fails."}; MaxBuildJobsSetting maxBuildJobs{this, 1, "max-jobs", "Maximum number of parallel build jobs. \"auto\" means use number of cores.", @@ -217,10 +195,13 @@ public: Setting<bool> showTrace{this, false, "show-trace", "Whether to show a stack trace on evaluation errors."}; - Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", - "Whether builtin functions that allow executing native code should be enabled."}; - - Setting<SandboxMode> sandboxMode{this, smDisabled, "sandbox", + Setting<SandboxMode> sandboxMode{this, + #if __linux__ + smEnabled + #else + smDisabled + #endif + , "sandbox", "Whether to enable sandboxed builds. Can be \"true\", \"false\" or \"relaxed\".", {"build-use-chroot", "build-use-sandbox"}}; @@ -232,13 +213,6 @@ public: "Additional paths to make available inside the build sandbox.", {"build-extra-chroot-dirs", "build-extra-sandbox-paths"}}; - Setting<bool> restrictEval{this, false, "restrict-eval", - "Whether to restrict file system access to paths in $NIX_PATH, " - "and network access to the URI prefixes listed in 'allowed-uris'."}; - - Setting<bool> pureEval{this, false, "pure-eval", - "Whether to restrict file system and network access to files specified by cryptographic hash."}; - Setting<size_t> buildRepeat{this, 0, "repeat", "The number of times to repeat a build in order to verify determinism.", {"build-repeat"}}; @@ -280,13 +254,6 @@ public: Setting<Strings> secretKeyFiles{this, {}, "secret-key-files", "Secret keys with which to sign local builds."}; - Setting<size_t> binaryCachesParallelConnections{this, 25, "http-connections", - "Number of parallel HTTP connections.", - {"binary-caches-parallel-connections"}}; - - Setting<bool> enableHttp2{this, true, "http2", - "Whether to enable HTTP/2 support."}; - Setting<unsigned int> tarballTtl{this, 60 * 60, "tarball-ttl", "How soon to expire files fetched by builtins.fetchTarball and builtins.fetchurl."}; @@ -295,6 +262,17 @@ public: "Nix store has a valid signature (that is, one signed using a key " "listed in 'trusted-public-keys'."}; + Setting<StringSet> extraPlatforms{this, + std::string{SYSTEM} == "x86_64-linux" ? StringSet{"i686-linux"} : StringSet{}, + "extra-platforms", + "Additional platforms that can be built on the local system. " + "These may be supported natively (e.g. armv7 on some aarch64 CPUs " + "or using hacks like qemu-user."}; + + Setting<StringSet> systemFeatures{this, getDefaultSystemFeatures(), + "system-features", + "Optional features that this system implements (like \"kvm\")."}; + Setting<Strings> substituters{this, nixStore == "/nix/store" ? Strings{"https://cache.nixos.org/"} : Strings(), "substituters", @@ -343,18 +321,6 @@ public: /* Path to the SSL CA file used */ Path caFile; - Setting<bool> enableImportFromDerivation{this, true, "allow-import-from-derivation", - "Whether the evaluator allows importing the result of a derivation."}; - - CaseHackSetting useCaseHack{this, "use-case-hack", - "Whether to enable a Darwin-specific hack for dealing with file name collisions."}; - - Setting<unsigned long> connectTimeout{this, 0, "connect-timeout", - "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."}; - - Setting<std::string> userAgentSuffix{this, "", "user-agent-suffix", - "String appended to the user agent in HTTP requests."}; - #if __linux__ Setting<bool> filterSyscalls{this, true, "filter-syscalls", "Whether to prevent certain dangerous system calls, such as " @@ -376,9 +342,6 @@ public: Setting<uint64_t> maxFree{this, std::numeric_limits<uint64_t>::max(), "max-free", "Stop deleting garbage when free disk space is above the specified amount."}; - Setting<Strings> allowedUris{this, {}, "allowed-uris", - "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."}; - Setting<Paths> pluginFiles{this, {}, "plugin-files", "Plugins to dynamically load at nix initialization time."}; }; @@ -391,15 +354,8 @@ extern Settings settings; anything else */ void initPlugins(); +void loadConfFile(); extern const string nixVersion; -struct RegisterSetting -{ - typedef std::vector<AbstractSetting *> SettingRegistrations; - static SettingRegistrations * settingRegistrations; - RegisterSetting(AbstractSetting * s); -}; - - } diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index b9e9cd5daba5..8da0e2f9d82a 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -13,6 +13,14 @@ private: Path cacheUri; + struct State + { + bool enabled = true; + std::chrono::steady_clock::time_point disabledUntil; + }; + + Sync<State> _state; + public: HttpBinaryCacheStore( @@ -46,8 +54,33 @@ public: protected: + void maybeDisable() + { + auto state(_state.lock()); + if (state->enabled && settings.tryFallback) { + int t = 60; + printError("disabling binary cache '%s' for %s seconds", getUri(), t); + state->enabled = false; + state->disabledUntil = std::chrono::steady_clock::now() + std::chrono::seconds(t); + } + } + + void checkEnabled() + { + auto state(_state.lock()); + if (state->enabled) return; + if (std::chrono::steady_clock::now() > state->disabledUntil) { + state->enabled = true; + debug("re-enabling binary cache '%s'", getUri()); + return; + } + throw SubstituterDisabled("substituter '%s' is disabled", getUri()); + } + bool fileExists(const std::string & path) override { + checkEnabled(); + try { DownloadRequest request(cacheUri + "/" + path); request.head = true; @@ -59,6 +92,7 @@ protected: bucket is unlistable, so treat 403 as 404. */ if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) return false; + maybeDisable(); throw; } } @@ -73,32 +107,51 @@ protected: try { getDownloader()->download(req); } catch (DownloadError & e) { - throw UploadToHTTP(format("uploading to HTTP binary cache at %1% not supported: %2%") % cacheUri % e.msg()); + throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg()); } } - void getFile(const std::string & path, - std::function<void(std::shared_ptr<std::string>)> success, - std::function<void(std::exception_ptr exc)> failure) override + DownloadRequest makeRequest(const std::string & path) { DownloadRequest request(cacheUri + "/" + path); request.tries = 8; + return request; + } + + void getFile(const std::string & path, Sink & sink) override + { + checkEnabled(); + auto request(makeRequest(path)); + try { + getDownloader()->download(std::move(request), sink); + } catch (DownloadError & e) { + if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) + throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri()); + maybeDisable(); + throw; + } + } + + void getFile(const std::string & path, + Callback<std::shared_ptr<std::string>> callback) override + { + checkEnabled(); + + auto request(makeRequest(path)); getDownloader()->enqueueDownload(request, - [success](const DownloadResult & result) { - success(result.data); - }, - [success, failure](std::exception_ptr exc) { + {[callback, this](std::future<DownloadResult> result) { try { - std::rethrow_exception(exc); + callback(result.get().data); } catch (DownloadError & e) { if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) - return success(0); - failure(exc); + return callback(std::shared_ptr<std::string>()); + maybeDisable(); + callback.rethrow(); } catch (...) { - failure(exc); + callback.rethrow(); } - }); + }}); } }; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 5dee25308f7f..7c9bc2b68ba8 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -17,6 +17,7 @@ struct LegacySSHStore : public Store const Setting<Path> sshKey{this, "", "ssh-key", "path to an SSH private key"}; const Setting<bool> compress{this, false, "compress", "whether to compress the connection"}; const Setting<Path> remoteProgram{this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"}; + const Setting<std::string> remoteStore{this, "", "remote-store", "URI of the store on the remote system"}; // Hack for getting remote build log output. const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; @@ -27,6 +28,7 @@ struct LegacySSHStore : public Store FdSink to; FdSource from; int remoteVersion; + bool good = true; }; std::string host; @@ -41,7 +43,7 @@ struct LegacySSHStore : public Store , connections(make_ref<Pool<Connection>>( std::max(1, (int) maxConnections), [this]() { return openConnection(); }, - [](const ref<Connection> & r) { return true; } + [](const ref<Connection> & r) { return r->good; } )) , master( host, @@ -56,7 +58,9 @@ struct LegacySSHStore : public Store ref<Connection> openConnection() { auto conn = make_ref<Connection>(); - conn->sshConn = master.startCommand(fmt("%s --serve --write", remoteProgram)); + conn->sshConn = master.startCommand( + fmt("%s --serve --write", remoteProgram) + + (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get()))); conn->to = FdSink(conn->sshConn->in.get()); conn->from = FdSource(conn->sshConn->out.get()); @@ -84,10 +88,9 @@ struct LegacySSHStore : public Store } void queryPathInfoUncached(const Path & path, - std::function<void(std::shared_ptr<ValidPathInfo>)> success, - std::function<void(std::exception_ptr exc)> failure) override + Callback<std::shared_ptr<ValidPathInfo>> callback) override { - sync2async<std::shared_ptr<ValidPathInfo>>(success, failure, [&]() -> std::shared_ptr<ValidPathInfo> { + try { auto conn(connections->get()); debug("querying remote host '%s' for info on '%s'", host, path); @@ -97,7 +100,7 @@ struct LegacySSHStore : public Store auto info = std::make_shared<ValidPathInfo>(); conn->from >> info->path; - if (info->path.empty()) return nullptr; + if (info->path.empty()) return callback(nullptr); assert(path == info->path); PathSet references; @@ -116,8 +119,8 @@ struct LegacySSHStore : public Store auto s = readString(conn->from); assert(s == ""); - return info; - }); + callback(std::move(info)); + } catch (...) { callback.rethrow(); } } void addToStore(const ValidPathInfo & info, Source & source, @@ -128,18 +131,48 @@ struct LegacySSHStore : public Store auto conn(connections->get()); - conn->to - << cmdImportPaths - << 1; - copyNAR(source, conn->to); - conn->to - << exportMagic - << info.path - << info.references - << info.deriver - << 0 - << 0; - conn->to.flush(); + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 5) { + + conn->to + << cmdAddToStoreNar + << info.path + << info.deriver + << info.narHash.to_string(Base16, false) + << info.references + << info.registrationTime + << info.narSize + << info.ultimate + << info.sigs + << info.ca; + try { + copyNAR(source, conn->to); + } catch (...) { + conn->good = false; + throw; + } + conn->to.flush(); + + } else { + + conn->to + << cmdImportPaths + << 1; + try { + copyNAR(source, conn->to); + } catch (...) { + conn->good = false; + throw; + } + conn->to + << exportMagic + << info.path + << info.references + << info.deriver + << 0 + << 0; + conn->to.flush(); + + } if (readInt(conn->from) != 1) throw Error("failed to add path '%s' to remote host '%s', info.path, host"); @@ -154,28 +187,17 @@ struct LegacySSHStore : public Store copyNAR(conn->from, sink); } - PathSet queryAllValidPaths() override { unsupported(); } - - void queryReferrers(const Path & path, PathSet & referrers) override - { unsupported(); } - - PathSet queryDerivationOutputs(const Path & path) override - { unsupported(); } - - StringSet queryDerivationOutputNames(const Path & path) override - { unsupported(); } - Path queryPathFromHashPart(const string & hashPart) override - { unsupported(); } + { unsupported("queryPathFromHashPart"); } Path addToStore(const string & name, const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) override - { unsupported(); } + { unsupported("addToStore"); } Path addTextToStore(const string & name, const string & s, const PathSet & references, RepairFlag repair) override - { unsupported(); } + { unsupported("addTextToStore"); } BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, BuildMode buildMode) override @@ -209,25 +231,7 @@ struct LegacySSHStore : public Store } void ensurePath(const Path & path) override - { unsupported(); } - - void addTempRoot(const Path & path) override - { unsupported(); } - - void addIndirectRoot(const Path & path) override - { unsupported(); } - - Roots findRoots() override - { unsupported(); } - - void collectGarbage(const GCOptions & options, GCResults & results) override - { unsupported(); } - - ref<FSAccessor> getFSAccessor() override - { unsupported(); } - - void addSignatures(const Path & storePath, const StringSet & sigs) override - { unsupported(); } + { unsupported("ensurePath"); } void computeFSClosure(const PathSet & paths, PathSet & out, bool flipDirection = false, @@ -270,6 +274,12 @@ struct LegacySSHStore : public Store { auto conn(connections->get()); } + + unsigned int getProtocol() override + { + auto conn(connections->get()); + return conn->remoteVersion; + } }; static RegisterStoreImplementation regStore([]( diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 2577e90aef23..b7001795be4d 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -34,18 +34,14 @@ protected: const std::string & data, const std::string & mimeType) override; - void getFile(const std::string & path, - std::function<void(std::shared_ptr<std::string>)> success, - std::function<void(std::exception_ptr exc)> failure) override + void getFile(const std::string & path, Sink & sink) override { - sync2async<std::shared_ptr<std::string>>(success, failure, [&]() { - try { - return std::make_shared<std::string>(readFile(binaryCacheDir + "/" + path)); - } catch (SysError & e) { - if (e.errNo == ENOENT) return std::shared_ptr<std::string>(); - throw; - } - }); + try { + readFile(binaryCacheDir + "/" + path, sink); + } catch (SysError & e) { + if (e.errNo == ENOENT) + throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache", path); + } } PathSet queryAllValidPaths() override diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b63584f28a30..485fdd691932 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -450,7 +450,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0); if (eaSize < 0) { - if (errno != ENOTSUP) + if (errno != ENOTSUP && errno != ENODATA) throw SysError("querying extended attributes of '%s'", path); } else if (eaSize > 0) { std::vector<char> eaBuf(eaSize); @@ -581,7 +581,8 @@ void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation & uint64_t LocalStore::addValidPath(State & state, const ValidPathInfo & info, bool checkOutputs) { - assert(info.ca == "" || info.isContentAddressed(*this)); + if (info.ca != "" && !info.isContentAddressed(*this)) + throw Error("cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't", info.path); state.stmtRegisterValidPath.use() (info.path) @@ -628,17 +629,15 @@ uint64_t LocalStore::addValidPath(State & state, void LocalStore::queryPathInfoUncached(const Path & path, - std::function<void(std::shared_ptr<ValidPathInfo>)> success, - std::function<void(std::exception_ptr exc)> failure) + Callback<std::shared_ptr<ValidPathInfo>> callback) { - sync2async<std::shared_ptr<ValidPathInfo>>(success, failure, [&]() { - + try { auto info = std::make_shared<ValidPathInfo>(); info->path = path; assertStorePath(path); - return retrySQLite<std::shared_ptr<ValidPathInfo>>([&]() { + callback(retrySQLite<std::shared_ptr<ValidPathInfo>>([&]() { auto state(_state.lock()); /* Get the path info. */ @@ -678,8 +677,9 @@ void LocalStore::queryPathInfoUncached(const Path & path, info->references.insert(useQueryReferences.getStr(0)); return info; - }); - }); + })); + + } catch (...) { callback.rethrow(); } } @@ -880,6 +880,12 @@ void LocalStore::querySubstitutablePathInfos(const PathSet & paths, narInfo ? narInfo->fileSize : 0, info->narSize}; } catch (InvalidPath) { + } catch (SubstituterDisabled) { + } catch (Error & e) { + if (settings.tryFallback) + printError(e.what()); + else + throw; } } } @@ -975,7 +981,8 @@ const PublicKeys & LocalStore::getPublicKeys() void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) { - assert(info.narHash); + if (!info.narHash) + throw Error("cannot add path '%s' because it lacks a hash", info.path); if (requireSigs && checkSigs && !info.checkSignatures(*this, getPublicKeys())) throw Error("cannot add path '%s' because it lacks a valid signature", info.path); @@ -1013,11 +1020,11 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, auto hashResult = hashSink.finish(); if (hashResult.first != info.narHash) - throw Error("hash mismatch importing path '%s'; expected hash '%s', got '%s'", + throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s", info.path, info.narHash.to_string(), hashResult.first.to_string()); if (hashResult.second != info.narSize) - throw Error("size mismatch importing path '%s'; expected %s, got %s", + throw Error("size mismatch importing path '%s';\n wanted: %s\n got: %s", info.path, info.narSize, hashResult.second); autoGC(); @@ -1331,6 +1338,12 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store, } +unsigned int LocalStore::getProtocol() +{ + return PROTOCOL_VERSION; +} + + #if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && defined(FS_IMMUTABLE_FL) static void makeMutable(const Path & path) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 1209a06356f7..6b655647b031 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -127,8 +127,7 @@ public: PathSet queryAllValidPaths() override; void queryPathInfoUncached(const Path & path, - std::function<void(std::shared_ptr<ValidPathInfo>)> success, - std::function<void(std::exception_ptr exc)> failure) override; + Callback<std::shared_ptr<ValidPathInfo>> callback) override; void queryReferrers(const Path & path, PathSet & referrers) override; @@ -181,11 +180,11 @@ private: typedef std::shared_ptr<AutoCloseFD> FDPtr; typedef list<FDPtr> FDs; - std::set<std::pair<pid_t, Path>> readTempRoots(FDs & fds); + void findTempRoots(FDs & fds, Roots & roots, bool censor); public: - Roots findRoots() override; + Roots findRoots(bool censor) override; void collectGarbage(const GCOptions & options, GCResults & results) override; @@ -210,6 +209,8 @@ public: void registerValidPaths(const ValidPathInfos & infos); + unsigned int getProtocol() override; + void vacuumDB(); /* Repair the contents of the given path by redownloading it using @@ -266,9 +267,9 @@ private: void findRoots(const Path & path, unsigned char type, Roots & roots); - Roots findRootsNoTemp(); + void findRootsNoTemp(Roots & roots, bool censor); - PathSet findRuntimeRoots(); + void findRuntimeRoots(Roots & roots, bool censor); void removeUnusedLinks(const GCState & state); diff --git a/src/libstore/local.mk b/src/libstore/local.mk index a7279aa3939f..89fc918c30fd 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -6,7 +6,7 @@ libstore_DIR := $(d) libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc) -libstore_LIBS = libutil libformat +libstore_LIBS = libutil libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread ifneq ($(OS), FreeBSD) @@ -18,7 +18,7 @@ libstore_FILES = sandbox-defaults.sb sandbox-minimal.sb sandbox-network.sb $(foreach file,$(libstore_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/sandbox))) ifeq ($(ENABLE_S3), 1) - libstore_LDFLAGS += -laws-cpp-sdk-s3 -laws-cpp-sdk-core + libstore_LDFLAGS += -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core endif ifeq ($(OS), SunOS) diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index edd03d147832..f848582dafd4 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -89,10 +89,11 @@ void parseMachines(const std::string & s, Machines & machines) Machines getMachines() { - Machines machines; - - parseMachines(settings.builders, machines); - + static auto machines = [&]() { + Machines machines; + parseMachines(settings.builders, machines); + return machines; + }(); return machines; } diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index a82aa4e9cfa5..adcce026fa1d 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -33,9 +33,11 @@ void Store::computeFSClosure(const PathSet & startPaths, state->pending++; } - queryPathInfo(path, - [&, path](ref<ValidPathInfo> info) { - // FIXME: calls to isValidPath() should be async + queryPathInfo(path, {[&, path](std::future<ref<ValidPathInfo>> fut) { + // FIXME: calls to isValidPath() should be async + + try { + auto info = fut.get(); if (flipDirection) { @@ -75,14 +77,13 @@ void Store::computeFSClosure(const PathSet & startPaths, if (!--state->pending) done.notify_one(); } - }, - - [&, path](std::exception_ptr exc) { + } catch (...) { auto state(state_.lock()); - if (!state->exc) state->exc = exc; + if (!state->exc) state->exc = std::current_exception(); assert(state->pending); if (!--state->pending) done.notify_one(); - }); + }; + }}); }; for (auto & startPath : startPaths) diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 35403e5df56f..32ad7f2b27ff 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -31,6 +31,7 @@ create table if not exists NARs ( refs text, deriver text, sigs text, + ca text, timestamp integer not null, present integer not null, primary key (cache, hashPart), @@ -72,7 +73,7 @@ public: { auto state(_state.lock()); - Path dbPath = getCacheDir() + "/nix/binary-cache-v5.sqlite"; + Path dbPath = getCacheDir() + "/nix/binary-cache-v6.sqlite"; createDirs(dirOf(dbPath)); state->db = SQLite(dbPath); @@ -94,13 +95,13 @@ public: state->insertNAR.create(state->db, "insert or replace into NARs(cache, hashPart, namePart, url, compression, fileHash, fileSize, narHash, " - "narSize, refs, deriver, sigs, timestamp, present) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)"); + "narSize, refs, deriver, sigs, ca, timestamp, present) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)"); state->insertMissingNAR.create(state->db, "insert or replace into NARs(cache, hashPart, timestamp, present) values (?, ?, ?, 0)"); state->queryNAR.create(state->db, - "select * from NARs where cache = ? and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 and timestamp > ?))"); + "select present, namePart, url, compression, fileHash, fileSize, narHash, narSize, refs, deriver, sigs, ca from NARs where cache = ? and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 and timestamp > ?))"); /* Periodically purge expired entries from the database. */ retrySQLite<void>([&]() { @@ -189,27 +190,28 @@ public: if (!queryNAR.next()) return {oUnknown, 0}; - if (!queryNAR.getInt(13)) + if (!queryNAR.getInt(0)) return {oInvalid, 0}; auto narInfo = make_ref<NarInfo>(); - auto namePart = queryNAR.getStr(2); + auto namePart = queryNAR.getStr(1); narInfo->path = cache.storeDir + "/" + hashPart + (namePart.empty() ? "" : "-" + namePart); - narInfo->url = queryNAR.getStr(3); - narInfo->compression = queryNAR.getStr(4); - if (!queryNAR.isNull(5)) - narInfo->fileHash = Hash(queryNAR.getStr(5)); - narInfo->fileSize = queryNAR.getInt(6); - narInfo->narHash = Hash(queryNAR.getStr(7)); - narInfo->narSize = queryNAR.getInt(8); - for (auto & r : tokenizeString<Strings>(queryNAR.getStr(9), " ")) + narInfo->url = queryNAR.getStr(2); + narInfo->compression = queryNAR.getStr(3); + if (!queryNAR.isNull(4)) + narInfo->fileHash = Hash(queryNAR.getStr(4)); + narInfo->fileSize = queryNAR.getInt(5); + narInfo->narHash = Hash(queryNAR.getStr(6)); + narInfo->narSize = queryNAR.getInt(7); + for (auto & r : tokenizeString<Strings>(queryNAR.getStr(8), " ")) narInfo->references.insert(cache.storeDir + "/" + r); - if (!queryNAR.isNull(10)) - narInfo->deriver = cache.storeDir + "/" + queryNAR.getStr(10); - for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(11), " ")) + if (!queryNAR.isNull(9)) + narInfo->deriver = cache.storeDir + "/" + queryNAR.getStr(9); + for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " ")) narInfo->sigs.insert(sig); + narInfo->ca = queryNAR.getStr(11); return {oValid, narInfo}; }); @@ -243,6 +245,7 @@ public: (concatStringsSep(" ", info->shortRefs())) (info->deriver != "" ? baseNameOf(info->deriver) : "", info->deriver != "") (concatStringsSep(" ", info->sigs)) + (info->ca) (time(0)).exec(); } else { diff --git a/src/libstore/nix-store.pc.in b/src/libstore/nix-store.pc.in index 5cf22faadcbe..6d67b1e03808 100644 --- a/src/libstore/nix-store.pc.in +++ b/src/libstore/nix-store.pc.in @@ -6,4 +6,4 @@ Name: Nix Description: Nix Package Manager Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lnixstore -lnixutil -Cflags: -I${includedir}/nix -std=c++14 +Cflags: -I${includedir}/nix -std=c++17 diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 7840167d7772..991512f21795 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -104,8 +104,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, *.app/Contents/Resources/\*.lproj seem to be the only paths affected. See https://github.com/NixOS/nix/issues/1443 for more discussion. */ - if (std::regex_search(path, std::regex("\\.app/Contents/PkgInfo$")) || - std::regex_search(path, std::regex("\\.app/Contents/Resources/.+\\.lproj$"))) + if (std::regex_search(path, std::regex("\\.app/Contents/.+$"))) { debug(format("'%1%' is not allowed to be linked in macOS") % path); return; diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc new file mode 100644 index 000000000000..17fde00a0167 --- /dev/null +++ b/src/libstore/parsed-derivations.cc @@ -0,0 +1,111 @@ +#include "parsed-derivations.hh" + +namespace nix { + +ParsedDerivation::ParsedDerivation(const Path & drvPath, BasicDerivation & drv) + : drvPath(drvPath), drv(drv) +{ + /* Parse the __json attribute, if any. */ + auto jsonAttr = drv.env.find("__json"); + if (jsonAttr != drv.env.end()) { + try { + structuredAttrs = nlohmann::json::parse(jsonAttr->second); + } catch (std::exception & e) { + throw Error("cannot process __json attribute of '%s': %s", drvPath, e.what()); + } + } +} + +std::optional<std::string> ParsedDerivation::getStringAttr(const std::string & name) const +{ + if (structuredAttrs) { + auto i = structuredAttrs->find(name); + if (i == structuredAttrs->end()) + return {}; + else { + if (!i->is_string()) + throw Error("attribute '%s' of derivation '%s' must be a string", name, drvPath); + return i->get<std::string>(); + } + } else { + auto i = drv.env.find(name); + if (i == drv.env.end()) + return {}; + else + return i->second; + } +} + +bool ParsedDerivation::getBoolAttr(const std::string & name, bool def) const +{ + if (structuredAttrs) { + auto i = structuredAttrs->find(name); + if (i == structuredAttrs->end()) + return def; + else { + if (!i->is_boolean()) + throw Error("attribute '%s' of derivation '%s' must be a Boolean", name, drvPath); + return i->get<bool>(); + } + } else { + auto i = drv.env.find(name); + if (i == drv.env.end()) + return def; + else + return i->second == "1"; + } +} + +std::optional<Strings> ParsedDerivation::getStringsAttr(const std::string & name) const +{ + if (structuredAttrs) { + auto i = structuredAttrs->find(name); + if (i == structuredAttrs->end()) + return {}; + else { + if (!i->is_array()) + throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath); + Strings res; + for (auto j = i->begin(); j != i->end(); ++j) { + if (!j->is_string()) + throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath); + res.push_back(j->get<std::string>()); + } + return res; + } + } else { + auto i = drv.env.find(name); + if (i == drv.env.end()) + return {}; + else + return tokenizeString<Strings>(i->second); + } +} + +StringSet ParsedDerivation::getRequiredSystemFeatures() const +{ + StringSet res; + for (auto & i : getStringsAttr("requiredSystemFeatures").value_or(Strings())) + res.insert(i); + return res; +} + +bool ParsedDerivation::canBuildLocally() const +{ + if (drv.platform != settings.thisSystem.get() + && !settings.extraPlatforms.get().count(drv.platform) + && !drv.isBuiltin()) + return false; + + for (auto & feature : getRequiredSystemFeatures()) + if (!settings.systemFeatures.get().count(feature)) return false; + + return true; +} + +bool ParsedDerivation::willBuildLocally() const +{ + return getBoolAttr("preferLocalBuild") && canBuildLocally(); +} + +} diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh new file mode 100644 index 000000000000..ed07dc652e8d --- /dev/null +++ b/src/libstore/parsed-derivations.hh @@ -0,0 +1,35 @@ +#include "derivations.hh" + +#include <nlohmann/json.hpp> + +namespace nix { + +class ParsedDerivation +{ + Path drvPath; + BasicDerivation & drv; + std::optional<nlohmann::json> structuredAttrs; + +public: + + ParsedDerivation(const Path & drvPath, BasicDerivation & drv); + + const std::optional<nlohmann::json> & getStructuredAttrs() const + { + return structuredAttrs; + } + + std::optional<std::string> getStringAttr(const std::string & name) const; + + bool getBoolAttr(const std::string & name, bool def = false) const; + + std::optional<Strings> getStringsAttr(const std::string & name) const; + + StringSet getRequiredSystemFeatures() const; + + bool canBuildLocally() const; + + bool willBuildLocally() const; +}; + +} diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 4a607b584506..4c6af567ae6f 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -157,6 +157,29 @@ void deleteGenerations(const Path & profile, const std::set<unsigned int> & gens } } +void deleteGenerationsGreaterThan(const Path & profile, int max, bool dryRun) +{ + PathLocks lock; + lockProfile(lock, profile); + + int curGen; + bool fromCurGen = false; + Generations gens = findGenerations(profile, curGen); + for (auto i = gens.rbegin(); i != gens.rend(); ++i) { + if (i->number == curGen) { + fromCurGen = true; + max--; + continue; + } + if (fromCurGen) { + if (max) { + max--; + continue; + } + deleteGeneration2(profile, i->number, dryRun); + } + } +} void deleteOldGenerations(const Path & profile, bool dryRun) { diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh index 1d4e6d3037db..5fa1533de311 100644 --- a/src/libstore/profiles.hh +++ b/src/libstore/profiles.hh @@ -39,6 +39,8 @@ void deleteGeneration(const Path & profile, unsigned int gen); void deleteGenerations(const Path & profile, const std::set<unsigned int> & gensToDelete, bool dryRun); +void deleteGenerationsGreaterThan(const Path & profile, const int max, bool dryRun); + void deleteOldGenerations(const Path & profile, bool dryRun); void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun); diff --git a/src/libstore/references.cc b/src/libstore/references.cc index ba9f18b9ca5e..5b7eb1f846af 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -13,7 +13,7 @@ namespace nix { static unsigned int refLength = 32; /* characters */ -static void search(const unsigned char * s, unsigned int len, +static void search(const unsigned char * s, size_t len, StringSet & hashes, StringSet & seen) { static bool initialised = false; @@ -25,7 +25,7 @@ static void search(const unsigned char * s, unsigned int len, initialised = true; } - for (unsigned int i = 0; i + refLength <= len; ) { + for (size_t i = 0; i + refLength <= len; ) { int j; bool match = true; for (j = refLength - 1; j >= 0; --j) @@ -73,7 +73,7 @@ void RefScanSink::operator () (const unsigned char * data, size_t len) search(data, len, hashes, seen); - unsigned int tailLen = len <= refLength ? len : refLength; + size_t tailLen = len <= refLength ? len : refLength; tail = string(tail, tail.size() < refLength - tailLen ? 0 : tail.size() - (refLength - tailLen)) + string((const char *) data + len - tailLen, tailLen); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 080cef93d214..15faf78a526d 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -7,6 +7,7 @@ #include "globals.hh" #include "derivations.hh" #include "pool.hh" +#include "finally.hh" #include <sys/types.h> #include <sys/stat.h> @@ -160,7 +161,8 @@ void RemoteStore::initConnection(Connection & conn) if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 11) conn.to << false; - conn.processStderr(); + auto ex = conn.processStderr(); + if (ex) std::rethrow_exception(ex); } catch (Error & e) { throw Error("cannot open connection to remote store '%s': %s", getUri(), e.what()); @@ -187,28 +189,75 @@ void RemoteStore::setOptions(Connection & conn) << settings.useSubstitutes; if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) { - auto overrides = settings.getSettings(true); + std::map<std::string, Config::SettingInfo> overrides; + globalConfig.getSettings(overrides, true); conn.to << overrides.size(); for (auto & i : overrides) - conn.to << i.first << i.second; + conn.to << i.first << i.second.value; } - conn.processStderr(); + auto ex = conn.processStderr(); + if (ex) std::rethrow_exception(ex); +} + + +/* A wrapper around Pool<RemoteStore::Connection>::Handle that marks + the connection as bad (causing it to be closed) if a non-daemon + exception is thrown before the handle is closed. Such an exception + causes a deviation from the expected protocol and therefore a + desynchronization between the client and daemon. */ +struct ConnectionHandle +{ + Pool<RemoteStore::Connection>::Handle handle; + bool daemonException = false; + + ConnectionHandle(Pool<RemoteStore::Connection>::Handle && handle) + : handle(std::move(handle)) + { } + + ConnectionHandle(ConnectionHandle && h) + : handle(std::move(h.handle)) + { } + + ~ConnectionHandle() + { + if (!daemonException && std::uncaught_exception()) { + handle.markBad(); + debug("closing daemon connection because of an exception"); + } + } + + RemoteStore::Connection * operator -> () { return &*handle; } + + void processStderr(Sink * sink = 0, Source * source = 0) + { + auto ex = handle->processStderr(sink, source); + if (ex) { + daemonException = true; + std::rethrow_exception(ex); + } + } +}; + + +ConnectionHandle RemoteStore::getConnection() +{ + return ConnectionHandle(connections->get()); } bool RemoteStore::isValidPathUncached(const Path & path) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopIsValidPath << path; - conn->processStderr(); + conn.processStderr(); return readInt(conn->from); } PathSet RemoteStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubstitute) { - auto conn(connections->get()); + auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { PathSet res; for (auto & i : paths) @@ -216,7 +265,7 @@ PathSet RemoteStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybe return res; } else { conn->to << wopQueryValidPaths << paths; - conn->processStderr(); + conn.processStderr(); return readStorePaths<PathSet>(*this, conn->from); } } @@ -224,27 +273,27 @@ PathSet RemoteStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybe PathSet RemoteStore::queryAllValidPaths() { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopQueryAllValidPaths; - conn->processStderr(); + conn.processStderr(); return readStorePaths<PathSet>(*this, conn->from); } PathSet RemoteStore::querySubstitutablePaths(const PathSet & paths) { - auto conn(connections->get()); + auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { PathSet res; for (auto & i : paths) { conn->to << wopHasSubstitutes << i; - conn->processStderr(); + conn.processStderr(); if (readInt(conn->from)) res.insert(i); } return res; } else { conn->to << wopQuerySubstitutablePaths << paths; - conn->processStderr(); + conn.processStderr(); return readStorePaths<PathSet>(*this, conn->from); } } @@ -255,14 +304,14 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths, { if (paths.empty()) return; - auto conn(connections->get()); + auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { for (auto & i : paths) { SubstitutablePathInfo info; conn->to << wopQuerySubstitutablePathInfo << i; - conn->processStderr(); + conn.processStderr(); unsigned int reply = readInt(conn->from); if (reply == 0) continue; info.deriver = readString(conn->from); @@ -276,7 +325,7 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths, } else { conn->to << wopQuerySubstitutablePathInfos << paths; - conn->processStderr(); + conn.processStderr(); size_t count = readNum<size_t>(conn->from); for (size_t n = 0; n < count; n++) { Path path = readStorePath(*this, conn->from); @@ -293,47 +342,49 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths, void RemoteStore::queryPathInfoUncached(const Path & path, - std::function<void(std::shared_ptr<ValidPathInfo>)> success, - std::function<void(std::exception_ptr exc)> failure) -{ - sync2async<std::shared_ptr<ValidPathInfo>>(success, failure, [&]() { - auto conn(connections->get()); - conn->to << wopQueryPathInfo << path; - try { - conn->processStderr(); - } catch (Error & e) { - // Ugly backwards compatibility hack. - if (e.msg().find("is not valid") != std::string::npos) - throw InvalidPath(e.what()); - throw; - } - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) { - bool valid; conn->from >> valid; - if (!valid) throw InvalidPath(format("path '%s' is not valid") % path); - } - auto info = std::make_shared<ValidPathInfo>(); - info->path = path; - info->deriver = readString(conn->from); - if (info->deriver != "") assertStorePath(info->deriver); - info->narHash = Hash(readString(conn->from), htSHA256); - info->references = readStorePaths<PathSet>(*this, conn->from); - conn->from >> info->registrationTime >> info->narSize; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { - conn->from >> info->ultimate; - info->sigs = readStrings<StringSet>(conn->from); - conn->from >> info->ca; + Callback<std::shared_ptr<ValidPathInfo>> callback) +{ + try { + std::shared_ptr<ValidPathInfo> info; + { + auto conn(getConnection()); + conn->to << wopQueryPathInfo << path; + try { + conn.processStderr(); + } catch (Error & e) { + // Ugly backwards compatibility hack. + if (e.msg().find("is not valid") != std::string::npos) + throw InvalidPath(e.what()); + throw; + } + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) { + bool valid; conn->from >> valid; + if (!valid) throw InvalidPath(format("path '%s' is not valid") % path); + } + info = std::make_shared<ValidPathInfo>(); + info->path = path; + info->deriver = readString(conn->from); + if (info->deriver != "") assertStorePath(info->deriver); + info->narHash = Hash(readString(conn->from), htSHA256); + info->references = readStorePaths<PathSet>(*this, conn->from); + conn->from >> info->registrationTime >> info->narSize; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { + conn->from >> info->ultimate; + info->sigs = readStrings<StringSet>(conn->from); + conn->from >> info->ca; + } } - return info; - }); + callback(std::move(info)); + } catch (...) { callback.rethrow(); } } void RemoteStore::queryReferrers(const Path & path, PathSet & referrers) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopQueryReferrers << path; - conn->processStderr(); + conn.processStderr(); PathSet referrers2 = readStorePaths<PathSet>(*this, conn->from); referrers.insert(referrers2.begin(), referrers2.end()); } @@ -341,36 +392,36 @@ void RemoteStore::queryReferrers(const Path & path, PathSet RemoteStore::queryValidDerivers(const Path & path) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopQueryValidDerivers << path; - conn->processStderr(); + conn.processStderr(); return readStorePaths<PathSet>(*this, conn->from); } PathSet RemoteStore::queryDerivationOutputs(const Path & path) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopQueryDerivationOutputs << path; - conn->processStderr(); + conn.processStderr(); return readStorePaths<PathSet>(*this, conn->from); } PathSet RemoteStore::queryDerivationOutputNames(const Path & path) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopQueryDerivationOutputNames << path; - conn->processStderr(); + conn.processStderr(); return readStrings<PathSet>(conn->from); } Path RemoteStore::queryPathFromHashPart(const string & hashPart) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopQueryPathFromHashPart << hashPart; - conn->processStderr(); + conn.processStderr(); Path path = readString(conn->from); if (!path.empty()) assertStorePath(path); return path; @@ -380,7 +431,7 @@ Path RemoteStore::queryPathFromHashPart(const string & hashPart) void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) { - auto conn(connections->get()); + auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) { conn->to << wopImportPaths; @@ -399,7 +450,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, ; }); - conn->processStderr(0, source2.get()); + conn.processStderr(0, source2.get()); auto importedPaths = readStorePaths<PathSet>(*this, conn->from); assert(importedPaths.size() <= 1); @@ -411,8 +462,9 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << info.references << info.registrationTime << info.narSize << info.ultimate << info.sigs << info.ca << repair << !checkSigs; - copyNAR(source, conn->to); - conn->processStderr(); + bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21; + if (!tunnel) copyNAR(source, conn->to); + conn.processStderr(0, tunnel ? &source : nullptr); } } @@ -422,7 +474,7 @@ Path RemoteStore::addToStore(const string & name, const Path & _srcPath, { if (repair) throw Error("repairing is not supported when building through the Nix daemon"); - auto conn(connections->get()); + auto conn(getConnection()); Path srcPath(absPath(_srcPath)); @@ -435,16 +487,18 @@ Path RemoteStore::addToStore(const string & name, const Path & _srcPath, conn->to.written = 0; conn->to.warn = true; connections->incCapacity(); - dumpPath(srcPath, conn->to, filter); - connections->decCapacity(); + { + Finally cleanup([&]() { connections->decCapacity(); }); + dumpPath(srcPath, conn->to, filter); + } conn->to.warn = false; - conn->processStderr(); + conn.processStderr(); } catch (SysError & e) { /* Daemon closed while we were sending the path. Probably OOM or I/O error. */ if (e.errNo == EPIPE) try { - conn->processStderr(); + conn.processStderr(); } catch (EndOfFile & e) { } throw; } @@ -458,17 +512,17 @@ Path RemoteStore::addTextToStore(const string & name, const string & s, { if (repair) throw Error("repairing is not supported when building through the Nix daemon"); - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopAddTextToStore << name << s << references; - conn->processStderr(); + conn.processStderr(); return readStorePath(*this, conn->from); } void RemoteStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopBuildPaths; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13) { conn->to << drvPaths; @@ -487,7 +541,7 @@ void RemoteStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) drvPaths2.insert(string(i, 0, i.find('!'))); conn->to << drvPaths2; } - conn->processStderr(); + conn.processStderr(); readInt(conn->from); } @@ -495,9 +549,9 @@ void RemoteStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) BuildResult RemoteStore::buildDerivation(const Path & drvPath, const BasicDerivation & drv, BuildMode buildMode) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopBuildDerivation << drvPath << drv << buildMode; - conn->processStderr(); + conn.processStderr(); BuildResult res; unsigned int status; conn->from >> status >> res.errorMsg; @@ -508,51 +562,51 @@ BuildResult RemoteStore::buildDerivation(const Path & drvPath, const BasicDeriva void RemoteStore::ensurePath(const Path & path) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopEnsurePath << path; - conn->processStderr(); + conn.processStderr(); readInt(conn->from); } void RemoteStore::addTempRoot(const Path & path) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopAddTempRoot << path; - conn->processStderr(); + conn.processStderr(); readInt(conn->from); } void RemoteStore::addIndirectRoot(const Path & path) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopAddIndirectRoot << path; - conn->processStderr(); + conn.processStderr(); readInt(conn->from); } void RemoteStore::syncWithGC() { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopSyncWithGC; - conn->processStderr(); + conn.processStderr(); readInt(conn->from); } -Roots RemoteStore::findRoots() +Roots RemoteStore::findRoots(bool censor) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopFindRoots; - conn->processStderr(); + conn.processStderr(); size_t count = readNum<size_t>(conn->from); Roots result; while (count--) { Path link = readString(conn->from); Path target = readStorePath(*this, conn->from); - result[link] = target; + result[target].emplace(link); } return result; } @@ -560,7 +614,7 @@ Roots RemoteStore::findRoots() void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopCollectGarbage << options.action << options.pathsToDelete << options.ignoreLiveness @@ -568,7 +622,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) /* removed options */ << 0 << 0 << 0; - conn->processStderr(); + conn.processStderr(); results.paths = readStrings<PathSet>(conn->from); results.bytesFreed = readLongLong(conn->from); @@ -583,27 +637,27 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) void RemoteStore::optimiseStore() { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopOptimiseStore; - conn->processStderr(); + conn.processStderr(); readInt(conn->from); } bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopVerifyStore << checkContents << repair; - conn->processStderr(); + conn.processStderr(); return readInt(conn->from); } void RemoteStore::addSignatures(const Path & storePath, const StringSet & sigs) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopAddSignatures << storePath << sigs; - conn->processStderr(); + conn.processStderr(); readInt(conn->from); } @@ -613,13 +667,13 @@ void RemoteStore::queryMissing(const PathSet & targets, unsigned long long & downloadSize, unsigned long long & narSize) { { - auto conn(connections->get()); + auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 19) // Don't hold the connection handle in the fallback case // to prevent a deadlock. goto fallback; conn->to << wopQueryMissing << targets; - conn->processStderr(); + conn.processStderr(); willBuild = readStorePaths<PathSet>(*this, conn->from); willSubstitute = readStorePaths<PathSet>(*this, conn->from); unknown = readStorePaths<PathSet>(*this, conn->from); @@ -635,7 +689,14 @@ void RemoteStore::queryMissing(const PathSet & targets, void RemoteStore::connect() { + auto conn(getConnection()); +} + + +unsigned int RemoteStore::getProtocol() +{ auto conn(connections->get()); + return conn->daemonVersion; } @@ -672,7 +733,7 @@ static Logger::Fields readFields(Source & from) } -void RemoteStore::Connection::processStderr(Sink * sink, Source * source) +std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source) { to.flush(); @@ -697,7 +758,7 @@ void RemoteStore::Connection::processStderr(Sink * sink, Source * source) else if (msg == STDERR_ERROR) { string error = readString(from); unsigned int status = readInt(from); - throw Error(status, error); + return std::make_exception_ptr(Error(status, error)); } else if (msg == STDERR_NEXT) @@ -731,6 +792,8 @@ void RemoteStore::Connection::processStderr(Sink * sink, Source * source) else throw Error("got unknown message type %x from Nix daemon", msg); } + + return nullptr; } static std::string uriScheme = "unix://"; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 95fa59a2069d..80f18ab715d9 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -14,6 +14,7 @@ class Pid; struct FdSink; struct FdSource; template<typename T> class Pool; +struct ConnectionHandle; /* FIXME: RemoteStore is a misnomer - should be something like @@ -40,8 +41,7 @@ public: PathSet queryAllValidPaths() override; void queryPathInfoUncached(const Path & path, - std::function<void(std::shared_ptr<ValidPathInfo>)> success, - std::function<void(std::exception_ptr exc)> failure) override; + Callback<std::shared_ptr<ValidPathInfo>> callback) override; void queryReferrers(const Path & path, PathSet & referrers) override; @@ -82,7 +82,7 @@ public: void syncWithGC() override; - Roots findRoots() override; + Roots findRoots(bool censor) override; void collectGarbage(const GCOptions & options, GCResults & results) override; @@ -98,12 +98,15 @@ public: void connect() override; + unsigned int getProtocol() override; + void flushBadConnections(); protected: struct Connection { + AutoCloseFD fd; FdSink to; FdSource from; unsigned int daemonVersion; @@ -111,7 +114,7 @@ protected: virtual ~Connection(); - void processStderr(Sink * sink = 0, Source * source = 0); + std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0); }; ref<Connection> openConnectionWrapper(); @@ -124,6 +127,10 @@ protected: virtual void setOptions(Connection & conn); + ConnectionHandle getConnection(); + + friend struct ConnectionHandle; + private: std::atomic_bool failed{false}; @@ -141,13 +148,8 @@ public: private: - struct Connection : RemoteStore::Connection - { - AutoCloseFD fd; - }; - ref<RemoteStore::Connection> openConnection() override; - std::experimental::optional<std::string> path; + std::optional<std::string> path; }; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 23af452094cf..cd547a964850 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -17,13 +17,15 @@ #include <aws/core/client/DefaultRetryStrategy.h> #include <aws/core/utils/logging/FormattedLogSystem.h> #include <aws/core/utils/logging/LogMacros.h> +#include <aws/core/utils/threading/Executor.h> #include <aws/s3/S3Client.h> -#include <aws/s3/model/CreateBucketRequest.h> -#include <aws/s3/model/GetBucketLocationRequest.h> #include <aws/s3/model/GetObjectRequest.h> #include <aws/s3/model/HeadObjectRequest.h> #include <aws/s3/model/ListObjectsRequest.h> #include <aws/s3/model/PutObjectRequest.h> +#include <aws/transfer/TransferManager.h> + +using namespace Aws::Transfer; namespace nix { @@ -80,8 +82,8 @@ static void initAWS() }); } -S3Helper::S3Helper(const std::string & profile, const std::string & region) - : config(makeConfig(region)) +S3Helper::S3Helper(const string & profile, const string & region, const string & scheme, const string & endpoint) + : config(makeConfig(region, scheme, endpoint)) , client(make_ref<Aws::S3::S3Client>( profile == "" ? std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>( @@ -95,7 +97,7 @@ S3Helper::S3Helper(const std::string & profile, const std::string & region) #else Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, #endif - false)) + endpoint.empty())) { } @@ -112,12 +114,19 @@ class RetryStrategy : public Aws::Client::DefaultRetryStrategy } }; -ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region) +ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region, const string & scheme, const string & endpoint) { initAWS(); auto res = make_ref<Aws::Client::ClientConfiguration>(); res->region = region; + if (!scheme.empty()) { + res->scheme = Aws::Http::SchemeMapper::FromString(scheme.c_str()); + } + if (!endpoint.empty()) { + res->endpointOverride = endpoint; + } res->requestTimeoutMs = 600 * 1000; + res->connectTimeoutMs = 5 * 1000; res->retryStrategy = std::make_shared<RetryStrategy>(); res->caFile = settings.caFile; return res; @@ -146,10 +155,8 @@ S3Helper::DownloadResult S3Helper::getObject( auto result = checkAws(fmt("AWS error fetching '%s'", key), client->GetObject(request)); - res.data = decodeContent( - result.GetContentEncoding(), - make_ref<std::string>( - dynamic_cast<std::stringstream &>(result.GetBody()).str())); + res.data = decompress(result.GetContentEncoding(), + dynamic_cast<std::stringstream &>(result.GetBody()).str()); } catch (S3Error & e) { if (e.err != Aws::S3::S3Errors::NO_SUCH_KEY) throw; @@ -166,9 +173,15 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore { const Setting<std::string> profile{this, "", "profile", "The name of the AWS configuration profile to use."}; const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}}; + const Setting<std::string> scheme{this, "", "scheme", "The scheme to use for S3 requests, https by default."}; + const Setting<std::string> endpoint{this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."}; const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"}; const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"}; const Setting<std::string> logCompression{this, "", "log-compression", "compression method for log/* files"}; + const Setting<bool> multipartUpload{ + this, false, "multipart-upload", "whether to use multi-part uploads"}; + const Setting<uint64_t> bufferSize{ + this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"}; std::string bucketName; @@ -180,7 +193,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore const Params & params, const std::string & bucketName) : S3BinaryCacheStore(params) , bucketName(bucketName) - , s3Helper(profile, region) + , s3Helper(profile, region, scheme, endpoint) { diskCache = getNarInfoDiskCache(); } @@ -194,32 +207,6 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore { if (!diskCache->cacheExists(getUri(), wantMassQuery_, priority)) { - /* Create the bucket if it doesn't already exists. */ - // FIXME: HeadBucket would be more appropriate, but doesn't return - // an easily parsed 404 message. - auto res = s3Helper.client->GetBucketLocation( - Aws::S3::Model::GetBucketLocationRequest().WithBucket(bucketName)); - - if (!res.IsSuccess()) { - if (res.GetError().GetErrorType() != Aws::S3::S3Errors::NO_SUCH_BUCKET) - throw Error(format("AWS error checking bucket '%s': %s") % bucketName % res.GetError().GetMessage()); - - printInfo("creating S3 bucket '%s'...", bucketName); - - // Stupid S3 bucket locations. - auto bucketConfig = Aws::S3::Model::CreateBucketConfiguration(); - if (s3Helper.config->region != "us-east-1") - bucketConfig.SetLocationConstraint( - Aws::S3::Model::BucketLocationConstraintMapper::GetBucketLocationConstraintForName( - s3Helper.config->region)); - - checkAws(format("AWS error creating bucket '%s'") % bucketName, - s3Helper.client->CreateBucket( - Aws::S3::Model::CreateBucketRequest() - .WithBucket(bucketName) - .WithCreateBucketConfiguration(bucketConfig))); - } - BinaryCacheStore::init(); diskCache->createCache(getUri(), storeDir, wantMassQuery_, priority); @@ -267,40 +254,100 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore return true; } + std::shared_ptr<TransferManager> transferManager; + std::once_flag transferManagerCreated; + void uploadFile(const std::string & path, const std::string & data, const std::string & mimeType, const std::string & contentEncoding) { - auto request = - Aws::S3::Model::PutObjectRequest() - .WithBucket(bucketName) - .WithKey(path); + auto stream = std::make_shared<istringstream_nocopy>(data); - request.SetContentType(mimeType); + auto maxThreads = std::thread::hardware_concurrency(); + + static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor> + executor = std::make_shared<Aws::Utils::Threading::PooledThreadExecutor>(maxThreads); + + std::call_once(transferManagerCreated, [&]() + { + if (multipartUpload) { + TransferManagerConfiguration transferConfig(executor.get()); + + transferConfig.s3Client = s3Helper.client; + transferConfig.bufferSize = bufferSize; + + transferConfig.uploadProgressCallback = + [](const TransferManager *transferManager, + const std::shared_ptr<const TransferHandle> + &transferHandle) + { + //FIXME: find a way to properly abort the multipart upload. + //checkInterrupt(); + debug("upload progress ('%s'): '%d' of '%d' bytes", + transferHandle->GetKey(), + transferHandle->GetBytesTransferred(), + transferHandle->GetBytesTotalSize()); + }; + + transferManager = TransferManager::Create(transferConfig); + } + }); - if (contentEncoding != "") - request.SetContentEncoding(contentEncoding); + auto now1 = std::chrono::steady_clock::now(); - auto stream = std::make_shared<istringstream_nocopy>(data); + if (transferManager) { - request.SetBody(stream); + if (contentEncoding != "") + throw Error("setting a content encoding is not supported with S3 multi-part uploads"); - stats.put++; - stats.putBytes += data.size(); + std::shared_ptr<TransferHandle> transferHandle = + transferManager->UploadFile( + stream, bucketName, path, mimeType, + Aws::Map<Aws::String, Aws::String>(), + nullptr /*, contentEncoding */); - auto now1 = std::chrono::steady_clock::now(); + transferHandle->WaitUntilFinished(); + + if (transferHandle->GetStatus() == TransferStatus::FAILED) + throw Error("AWS error: failed to upload 's3://%s/%s': %s", + bucketName, path, transferHandle->GetLastError().GetMessage()); + + if (transferHandle->GetStatus() != TransferStatus::COMPLETED) + throw Error("AWS error: transfer status of 's3://%s/%s' in unexpected state", + bucketName, path); + + } else { + + auto request = + Aws::S3::Model::PutObjectRequest() + .WithBucket(bucketName) + .WithKey(path); + + request.SetContentType(mimeType); + + if (contentEncoding != "") + request.SetContentEncoding(contentEncoding); - auto result = checkAws(format("AWS error uploading '%s'") % path, - s3Helper.client->PutObject(request)); + auto stream = std::make_shared<istringstream_nocopy>(data); + + request.SetBody(stream); + + auto result = checkAws(fmt("AWS error uploading '%s'", path), + s3Helper.client->PutObject(request)); + } auto now2 = std::chrono::steady_clock::now(); - auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count(); + auto duration = + std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1) + .count(); - printInfo(format("uploaded 's3://%1%/%2%' (%3% bytes) in %4% ms") - % bucketName % path % data.size() % duration); + printInfo(format("uploaded 's3://%1%/%2%' (%3% bytes) in %4% ms") % + bucketName % path % data.size() % duration); stats.putTimeMs += duration; + stats.putBytes += data.size(); + stats.put++; } void upsertFile(const std::string & path, const std::string & data, @@ -316,24 +363,23 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore uploadFile(path, data, mimeType, ""); } - void getFile(const std::string & path, - std::function<void(std::shared_ptr<std::string>)> success, - std::function<void(std::exception_ptr exc)> failure) override + void getFile(const std::string & path, Sink & sink) override { - sync2async<std::shared_ptr<std::string>>(success, failure, [&]() { - stats.get++; + stats.get++; - auto res = s3Helper.getObject(bucketName, path); + // FIXME: stream output to sink. + auto res = s3Helper.getObject(bucketName, path); - stats.getBytes += res.data ? res.data->size() : 0; - stats.getTimeMs += res.durationMs; + stats.getBytes += res.data ? res.data->size() : 0; + stats.getTimeMs += res.durationMs; - if (res.data) - printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms", - bucketName, path, res.data->size(), res.durationMs); + if (res.data) { + printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms", + bucketName, path, res.data->size(), res.durationMs); - return res.data; - }); + sink((unsigned char *) res.data->data(), res.data->size()); + } else + throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri()); } PathSet queryAllValidPaths() override diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh index 4f996400343c..ef5f23d0f253 100644 --- a/src/libstore/s3.hh +++ b/src/libstore/s3.hh @@ -14,9 +14,9 @@ struct S3Helper ref<Aws::Client::ClientConfiguration> config; ref<Aws::S3::S3Client> client; - S3Helper(const std::string & profile, const std::string & region); + S3Helper(const std::string & profile, const std::string & region, const std::string & scheme, const std::string & endpoint); - ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region); + ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region, const std::string & scheme, const std::string & endpoint); struct DownloadResult { diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index f67d1e2580a5..9fae6d5349f1 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -5,7 +5,7 @@ namespace nix { #define SERVE_MAGIC_1 0x390c9deb #define SERVE_MAGIC_2 0x5452eecb -#define SERVE_PROTOCOL_VERSION 0x204 +#define SERVE_PROTOCOL_VERSION 0x205 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -18,6 +18,7 @@ typedef enum { cmdBuildPaths = 6, cmdQueryClosure = 7, cmdBuildDerivation = 8, + cmdAddToStoreNar = 9, } ServeCommand; } diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 42d40e71d8be..a061d64f36d8 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -10,6 +10,7 @@ namespace nix { [[noreturn]] void throwSQLiteError(sqlite3 * db, const FormatOrString & fs) { int err = sqlite3_errcode(db); + int exterr = sqlite3_extended_errcode(db); auto path = sqlite3_db_filename(db, nullptr); if (!path) path = "(in-memory)"; @@ -21,7 +22,7 @@ namespace nix { : fmt("SQLite database '%s' is busy", path)); } else - throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(err), path); + throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(exterr), path); } SQLite::SQLite(const Path & path) diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 033c580936ad..5e0e44935cca 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -4,8 +4,9 @@ namespace nix { SSHMaster::SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD) : host(host) + , fakeSSH(host == "localhost") , keyFile(keyFile) - , useMaster(useMaster) + , useMaster(useMaster && !fakeSSH) , compress(compress) , logFD(logFD) { @@ -45,12 +46,19 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string if (logFD != -1 && dup2(logFD, STDERR_FILENO) == -1) throw SysError("duping over stderr"); - Strings args = { "ssh", host.c_str(), "-x", "-a" }; - addCommonSSHOpts(args); - if (socketPath != "") - args.insert(args.end(), {"-S", socketPath}); - if (verbosity >= lvlChatty) - args.push_back("-v"); + Strings args; + + if (fakeSSH) { + args = { "bash", "-c" }; + } else { + args = { "ssh", host.c_str(), "-x", "-a" }; + addCommonSSHOpts(args); + if (socketPath != "") + args.insert(args.end(), {"-S", socketPath}); + if (verbosity >= lvlChatty) + args.push_back("-v"); + } + args.push_back(command); execvp(args.begin()->c_str(), stringsToCharPtrs(args).data()); diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh index 1268e6d00054..4f0f0bd29f9f 100644 --- a/src/libstore/ssh.hh +++ b/src/libstore/ssh.hh @@ -10,6 +10,7 @@ class SSHMaster private: const std::string host; + bool fakeSSH; const std::string keyFile; const bool useMaster; const bool compress; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 1a0d12ca78c2..c13ff11564ec 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -253,6 +253,8 @@ std::string Store::getUri() bool Store::isValidPath(const Path & storePath) { + assertStorePath(storePath); + auto hashPart = storePathToHash(storePath); { @@ -303,21 +305,23 @@ ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath) std::promise<ref<ValidPathInfo>> promise; queryPathInfo(storePath, - [&](ref<ValidPathInfo> info) { - promise.set_value(info); - }, - [&](std::exception_ptr exc) { - promise.set_exception(exc); - }); + {[&](std::future<ref<ValidPathInfo>> result) { + try { + promise.set_value(result.get()); + } catch (...) { + promise.set_exception(std::current_exception()); + } + }}); return promise.get_future().get(); } void Store::queryPathInfo(const Path & storePath, - std::function<void(ref<ValidPathInfo>)> success, - std::function<void(std::exception_ptr exc)> failure) + Callback<ref<ValidPathInfo>> callback) { + assertStorePath(storePath); + auto hashPart = storePathToHash(storePath); try { @@ -328,7 +332,7 @@ void Store::queryPathInfo(const Path & storePath, stats.narInfoReadAverted++; if (!*res) throw InvalidPath(format("path '%s' is not valid") % storePath); - return success(ref<ValidPathInfo>(*res)); + return callback(ref<ValidPathInfo>(*res)); } } @@ -344,35 +348,36 @@ void Store::queryPathInfo(const Path & storePath, (res.second->path != storePath && storePathToName(storePath) != "")) throw InvalidPath(format("path '%s' is not valid") % storePath); } - return success(ref<ValidPathInfo>(res.second)); + return callback(ref<ValidPathInfo>(res.second)); } } - } catch (std::exception & e) { - return callFailure(failure); - } + } catch (...) { return callback.rethrow(); } queryPathInfoUncached(storePath, - [this, storePath, hashPart, success, failure](std::shared_ptr<ValidPathInfo> info) { + {[this, storePath, hashPart, callback](std::future<std::shared_ptr<ValidPathInfo>> fut) { - if (diskCache) - diskCache->upsertNarInfo(getUri(), hashPart, info); + try { + auto info = fut.get(); - { - auto state_(state.lock()); - state_->pathInfoCache.upsert(hashPart, info); - } + if (diskCache) + diskCache->upsertNarInfo(getUri(), hashPart, info); - if (!info - || (info->path != storePath && storePathToName(storePath) != "")) - { - stats.narInfoMissing++; - return failure(std::make_exception_ptr(InvalidPath(format("path '%s' is not valid") % storePath))); - } + { + auto state_(state.lock()); + state_->pathInfoCache.upsert(hashPart, info); + } - callSuccess(success, failure, ref<ValidPathInfo>(info)); + if (!info + || (info->path != storePath && storePathToName(storePath) != "")) + { + stats.narInfoMissing++; + throw InvalidPath("path '%s' is not valid", storePath); + } - }, failure); + callback(ref<ValidPathInfo>(info)); + } catch (...) { callback.rethrow(); } + }}); } @@ -392,26 +397,19 @@ PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubsti auto doQuery = [&](const Path & path ) { checkInterrupt(); - queryPathInfo(path, - [path, &state_, &wakeup](ref<ValidPathInfo> info) { - auto state(state_.lock()); + queryPathInfo(path, {[path, &state_, &wakeup](std::future<ref<ValidPathInfo>> fut) { + auto state(state_.lock()); + try { + auto info = fut.get(); state->valid.insert(path); - assert(state->left); - if (!--state->left) - wakeup.notify_one(); - }, - [path, &state_, &wakeup](std::exception_ptr exc) { - auto state(state_.lock()); - try { - std::rethrow_exception(exc); - } catch (InvalidPath &) { - } catch (...) { - state->exc = exc; - } - assert(state->left); - if (!--state->left) - wakeup.notify_one(); - }); + } catch (InvalidPath &) { + } catch (...) { + state->exc = std::current_exception(); + } + assert(state->left); + if (!--state->left) + wakeup.notify_one(); + }}); }; for (auto & path : paths) @@ -564,10 +562,10 @@ void Store::buildPaths(const PathSet & paths, BuildMode buildMode) { for (auto & path : paths) if (isDerivation(path)) - unsupported(); + unsupported("buildPaths"); if (queryValidPaths(paths).size() != paths.size()) - unsupported(); + unsupported("buildPaths"); } @@ -590,15 +588,19 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, uint64_t total = 0; - // FIXME -#if 0 if (!info->narHash) { + StringSink sink; + srcStore->narFromPath({storePath}, sink); auto info2 = make_ref<ValidPathInfo>(*info); info2->narHash = hashString(htSHA256, *sink.s); if (!info->narSize) info2->narSize = sink.s->size(); + if (info->ultimate) info2->ultimate = false; info = info2; + + StringSource source(*sink.s); + dstStore->addToStore(*info, source, repair, checkSigs); + return; } -#endif if (info->ultimate) { auto info2 = make_ref<ValidPathInfo>(*info); @@ -613,6 +615,8 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, act.progress(total, info->narSize); }); srcStore->narFromPath({storePath}, wrapperSink); + }, [&]() { + throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri()); }); dstStore->addToStore(*info, *source, repair, checkSigs); @@ -633,11 +637,12 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); std::atomic<size_t> nrDone{0}; + std::atomic<size_t> nrFailed{0}; std::atomic<uint64_t> bytesExpected{0}; std::atomic<uint64_t> nrRunning{0}; auto showProgress = [&]() { - act.progress(nrDone, missing.size(), nrRunning); + act.progress(nrDone, missing.size(), nrRunning, nrFailed); }; ThreadPool pool; @@ -666,7 +671,16 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa if (!dstStore->isValidPath(storePath)) { MaintainCount<decltype(nrRunning)> mc(nrRunning); showProgress(); - copyStorePath(srcStore, dstStore, storePath, repair, checkSigs); + try { + copyStorePath(srcStore, dstStore, storePath, repair, checkSigs); + } catch (Error &e) { + nrFailed++; + if (!settings.keepGoing) + throw e; + logger->log(lvlError, format("could not copy %s: %s") % storePath % e.what()); + showProgress(); + return; + } } nrDone++; @@ -828,26 +842,50 @@ namespace nix { RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0; - -ref<Store> openStore(const std::string & uri_, - const Store::Params & extraParams) +/* Split URI into protocol+hierarchy part and its parameter set. */ +std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_) { auto uri(uri_); - Store::Params params(extraParams); + Store::Params params; auto q = uri.find('?'); if (q != std::string::npos) { for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) { auto e = s.find('='); - if (e != std::string::npos) - params[s.substr(0, e)] = s.substr(e + 1); + if (e != std::string::npos) { + auto value = s.substr(e + 1); + std::string decoded; + for (size_t i = 0; i < value.size(); ) { + if (value[i] == '%') { + if (i + 2 >= value.size()) + throw Error("invalid URI parameter '%s'", value); + try { + decoded += std::stoul(std::string(value, i + 1, 2), 0, 16); + i += 3; + } catch (...) { + throw Error("invalid URI parameter '%s'", value); + } + } else + decoded += value[i++]; + } + params[s.substr(0, e)] = decoded; + } } uri = uri_.substr(0, q); } + return {uri, params}; +} + +ref<Store> openStore(const std::string & uri_, + const Store::Params & extraParams) +{ + auto [uri, uriParams] = splitUriAndParams(uri_); + auto params = extraParams; + params.insert(uriParams.begin(), uriParams.end()); for (auto fun : *RegisterStoreImplementation::implementations) { auto store = fun(uri, params); if (store) { - store->handleUnknownSettings(); + store->warnUnknownSettings(); return ref<Store>(store); } } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index ea259f07e8ab..7a1b31d0ff59 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -11,6 +11,8 @@ #include <atomic> #include <limits> #include <map> +#include <unordered_map> +#include <unordered_set> #include <memory> #include <string> @@ -22,6 +24,8 @@ MakeError(SubstError, Error) MakeError(BuildError, Error) /* denotes a permanent build failure */ MakeError(InvalidPath, Error) MakeError(Unsupported, Error) +MakeError(SubstituteGone, Error) +MakeError(SubstituterDisabled, Error) struct BasicDerivation; @@ -45,7 +49,7 @@ const size_t storePathHashLen = 32; // i.e. 160 bits const uint32_t exportMagic = 0x4558494e; -typedef std::map<Path, Path> Roots; +typedef std::unordered_map<Path, std::unordered_set<std::string>> Roots; struct GCOptions @@ -347,7 +351,8 @@ public: (i.e. you'll get /nix/store/<hash> rather than /nix/store/<hash>-<name>). Use queryPathInfo() to obtain the full store path. */ - virtual PathSet queryAllValidPaths() = 0; + virtual PathSet queryAllValidPaths() + { unsupported("queryAllValidPaths"); } /* Query information about a valid path. It is permitted to omit the name part of the store path. */ @@ -355,21 +360,19 @@ public: /* Asynchronous version of queryPathInfo(). */ void queryPathInfo(const Path & path, - std::function<void(ref<ValidPathInfo>)> success, - std::function<void(std::exception_ptr exc)> failure); + Callback<ref<ValidPathInfo>> callback); protected: virtual void queryPathInfoUncached(const Path & path, - std::function<void(std::shared_ptr<ValidPathInfo>)> success, - std::function<void(std::exception_ptr exc)> failure) = 0; + Callback<std::shared_ptr<ValidPathInfo>> callback) = 0; public: /* Queries the set of incoming FS references for a store path. The result is not cleared. */ - virtual void queryReferrers(const Path & path, - PathSet & referrers) = 0; + virtual void queryReferrers(const Path & path, PathSet & referrers) + { unsupported("queryReferrers"); } /* Return all currently valid derivations that have `path' as an output. (Note that the result of `queryDeriver()' is the @@ -378,10 +381,12 @@ public: virtual PathSet queryValidDerivers(const Path & path) { return {}; }; /* Query the outputs of the derivation denoted by `path'. */ - virtual PathSet queryDerivationOutputs(const Path & path) = 0; + virtual PathSet queryDerivationOutputs(const Path & path) + { unsupported("queryDerivationOutputs"); } /* Query the output names of the derivation denoted by `path'. */ - virtual StringSet queryDerivationOutputNames(const Path & path) = 0; + virtual StringSet queryDerivationOutputNames(const Path & path) + { unsupported("queryDerivationOutputNames"); } /* Query the full store path given the hash part of a valid store path, or "" if the path doesn't exist. */ @@ -447,14 +452,16 @@ public: /* Add a store path as a temporary root of the garbage collector. The root disappears as soon as we exit. */ - virtual void addTempRoot(const Path & path) = 0; + virtual void addTempRoot(const Path & path) + { unsupported("addTempRoot"); } /* Add an indirect root, which is merely a symlink to `path' from /nix/var/nix/gcroots/auto/<hash of `path'>. `path' is supposed to be a symlink to a store path. The garbage collector will automatically remove the indirect root when it finds that `path' has disappeared. */ - virtual void addIndirectRoot(const Path & path) = 0; + virtual void addIndirectRoot(const Path & path) + { unsupported("addIndirectRoot"); } /* Acquire the global GC lock, then immediately release it. This function must be called after registering a new permanent root, @@ -478,11 +485,15 @@ public: /* Find the roots of the garbage collector. Each root is a pair (link, storepath) where `link' is the path of the symlink - outside of the Nix store that point to `storePath'. */ - virtual Roots findRoots() = 0; + outside of the Nix store that point to `storePath'. If + 'censor' is true, privacy-sensitive information about roots + found in /proc is censored. */ + virtual Roots findRoots(bool censor) + { unsupported("findRoots"); } /* Perform a garbage collection. */ - virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0; + virtual void collectGarbage(const GCOptions & options, GCResults & results) + { unsupported("collectGarbage"); } /* Return a string representing information about the path that can be loaded into the database using `nix-store --load-db' or @@ -513,11 +524,13 @@ public: virtual bool verifyStore(bool checkContents, RepairFlag repair = NoRepair) { return false; }; /* Return an object to access files in the Nix store. */ - virtual ref<FSAccessor> getFSAccessor() = 0; + virtual ref<FSAccessor> getFSAccessor() + { unsupported("getFSAccessor"); } /* Add signatures to the specified store path. The signatures are not verified. */ - virtual void addSignatures(const Path & storePath, const StringSet & sigs) = 0; + virtual void addSignatures(const Path & storePath, const StringSet & sigs) + { unsupported("addSignatures"); } /* Utility functions. */ @@ -599,6 +612,12 @@ public: a notion of connection. Otherwise this is a no-op. */ virtual void connect() { }; + /* Get the protocol version of this store or it's connection. */ + virtual unsigned int getProtocol() + { + return 0; + }; + /* Get the priority of the store, used to order substituters. In particular, binary caches can specify a priority field in their "nix-cache-info" file. Lower value means higher priority. */ @@ -614,9 +633,9 @@ protected: Stats stats; /* Unsupported methods. */ - [[noreturn]] void unsupported() + [[noreturn]] void unsupported(const std::string & op) { - throw Unsupported("requested operation is not supported by store '%s'", getUri()); + throw Unsupported("operation '%s' is not supported by store '%s'", op, getUri()); } }; @@ -783,4 +802,8 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, for paths created by makeFixedOutputPath() / addToStore(). */ std::string makeFixedOutputCA(bool recursive, const Hash & hash); + +/* Split URI into protocol+hierarchy part and its parameter set. */ +std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri); + } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 996e1d25355f..5ebdfaf134d6 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -6,7 +6,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x114 +#define PROTOCOL_VERSION 0x115 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 154e2d20430c..3aa120270970 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -13,17 +13,25 @@ #include "archive.hh" #include "util.hh" - +#include "config.hh" namespace nix { +struct ArchiveSettings : Config +{ + Setting<bool> useCaseHack{this, + #if __APPLE__ + true, + #else + false, + #endif + "use-case-hack", + "Whether to enable a Darwin-specific hack for dealing with file name collisions."}; +}; -bool useCaseHack = -#if __APPLE__ - true; -#else - false; -#endif +static ArchiveSettings archiveSettings; + +static GlobalConfig::Register r1(&archiveSettings); const std::string narVersionMagic1 = "nix-archive-1"; @@ -78,7 +86,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter) the case hack applied by restorePath(). */ std::map<string, string> unhacked; for (auto & i : readDirectory(path)) - if (useCaseHack) { + if (archiveSettings.useCaseHack) { string name(i.name); size_t pos = i.name.find(caseHackSuffix); if (pos != string::npos) { @@ -243,7 +251,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path) if (name <= prevName) throw Error("NAR directory is not sorted"); prevName = name; - if (useCaseHack) { + if (archiveSettings.useCaseHack) { auto i = names.find(name); if (i != names.end()) { debug(format("case collision between '%1%' and '%2%'") % i->first % name); @@ -275,7 +283,7 @@ void parseDump(ParseSink & sink, Source & source) { string version; try { - version = readString(source); + version = readString(source, narVersionMagic1.size()); } catch (SerialisationError & e) { /* This generally means the integer at the start couldn't be decoded. Ignore and throw the exception below. */ @@ -323,7 +331,7 @@ struct RestoreSink : ParseSink filesystem doesn't support preallocation (e.g. on OpenSolaris). Since preallocation is just an optimisation, ignore it. */ - if (errno && errno != EINVAL) + if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) throw SysError(format("preallocating file of %1% bytes") % len); } #endif diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 7a0e688e4201..25be426c1a4d 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -78,10 +78,6 @@ void restorePath(const Path & path, Source & source); void copyNAR(Source & source, Sink & sink); -// FIXME: global variables are bad m'kay. -extern bool useCaseHack; - - extern const std::string narVersionMagic1; diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index 81cb5e98c763..0dd84e32034a 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -8,246 +8,265 @@ #include <cstdio> #include <cstring> -#if HAVE_BROTLI #include <brotli/decode.h> #include <brotli/encode.h> -#endif // HAVE_BROTLI #include <iostream> namespace nix { -static const size_t bufSize = 32 * 1024; - -static void decompressNone(Source & source, Sink & sink) +// Don't feed brotli too much at once. +struct ChunkedCompressionSink : CompressionSink { - std::vector<unsigned char> buf(bufSize); - while (true) { - size_t n; - try { - n = source.read(buf.data(), buf.size()); - } catch (EndOfFile &) { - break; + uint8_t outbuf[32 * 1024]; + + void write(const unsigned char * data, size_t len) override + { + const size_t CHUNK_SIZE = sizeof(outbuf) << 2; + while (len) { + size_t n = std::min(CHUNK_SIZE, len); + writeInternal(data, n); + data += n; + len -= n; } - sink(buf.data(), n); } -} -static void decompressXZ(Source & source, Sink & sink) + virtual void writeInternal(const unsigned char * data, size_t len) = 0; +}; + +struct NoneSink : CompressionSink { - lzma_stream strm(LZMA_STREAM_INIT); - - lzma_ret ret = lzma_stream_decoder( - &strm, UINT64_MAX, LZMA_CONCATENATED); - if (ret != LZMA_OK) - throw CompressionError("unable to initialise lzma decoder"); - - Finally free([&]() { lzma_end(&strm); }); - - lzma_action action = LZMA_RUN; - std::vector<uint8_t> inbuf(bufSize), outbuf(bufSize); - strm.next_in = nullptr; - strm.avail_in = 0; - strm.next_out = outbuf.data(); - strm.avail_out = outbuf.size(); - bool eof = false; - - while (true) { - checkInterrupt(); - - if (strm.avail_in == 0 && !eof) { - strm.next_in = inbuf.data(); - try { - strm.avail_in = source.read((unsigned char *) strm.next_in, inbuf.size()); - } catch (EndOfFile &) { - eof = true; - } - } + Sink & nextSink; + NoneSink(Sink & nextSink) : nextSink(nextSink) { } + void finish() override { flush(); } + void write(const unsigned char * data, size_t len) override { nextSink(data, len); } +}; - if (strm.avail_in == 0) - action = LZMA_FINISH; +struct XzDecompressionSink : CompressionSink +{ + Sink & nextSink; + uint8_t outbuf[BUFSIZ]; + lzma_stream strm = LZMA_STREAM_INIT; + bool finished = false; - lzma_ret ret = lzma_code(&strm, action); + XzDecompressionSink(Sink & nextSink) : nextSink(nextSink) + { + lzma_ret ret = lzma_stream_decoder( + &strm, UINT64_MAX, LZMA_CONCATENATED); + if (ret != LZMA_OK) + throw CompressionError("unable to initialise lzma decoder"); - if (strm.avail_out < outbuf.size()) { - sink((unsigned char *) outbuf.data(), outbuf.size() - strm.avail_out); - strm.next_out = outbuf.data(); - strm.avail_out = outbuf.size(); - } + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } - if (ret == LZMA_STREAM_END) return; + ~XzDecompressionSink() + { + lzma_end(&strm); + } - if (ret != LZMA_OK) - throw CompressionError("error %d while decompressing xz file", ret); + void finish() override + { + CompressionSink::flush(); + write(nullptr, 0); } -} -static void decompressBzip2(Source & source, Sink & sink) -{ - bz_stream strm; - memset(&strm, 0, sizeof(strm)); - - int ret = BZ2_bzDecompressInit(&strm, 0, 0); - if (ret != BZ_OK) - throw CompressionError("unable to initialise bzip2 decoder"); - - Finally free([&]() { BZ2_bzDecompressEnd(&strm); }); - - std::vector<char> inbuf(bufSize), outbuf(bufSize); - strm.next_in = nullptr; - strm.avail_in = 0; - strm.next_out = outbuf.data(); - strm.avail_out = outbuf.size(); - bool eof = false; - - while (true) { - checkInterrupt(); - - if (strm.avail_in == 0 && !eof) { - strm.next_in = inbuf.data(); - try { - strm.avail_in = source.read((unsigned char *) strm.next_in, inbuf.size()); - } catch (EndOfFile &) { - eof = true; - } - } + void write(const unsigned char * data, size_t len) override + { + strm.next_in = data; + strm.avail_in = len; + + while (!finished && (!data || strm.avail_in)) { + checkInterrupt(); - int ret = BZ2_bzDecompress(&strm); + lzma_ret ret = lzma_code(&strm, data ? LZMA_RUN : LZMA_FINISH); + if (ret != LZMA_OK && ret != LZMA_STREAM_END) + throw CompressionError("error %d while decompressing xz file", ret); - if (strm.avail_in == 0 && strm.avail_out == outbuf.size() && eof) - throw CompressionError("bzip2 data ends prematurely"); + finished = ret == LZMA_STREAM_END; - if (strm.avail_out < outbuf.size()) { - sink((unsigned char *) outbuf.data(), outbuf.size() - strm.avail_out); - strm.next_out = outbuf.data(); - strm.avail_out = outbuf.size(); + if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { + nextSink(outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } } + } +}; - if (ret == BZ_STREAM_END) return; +struct BzipDecompressionSink : ChunkedCompressionSink +{ + Sink & nextSink; + bz_stream strm; + bool finished = false; + BzipDecompressionSink(Sink & nextSink) : nextSink(nextSink) + { + memset(&strm, 0, sizeof(strm)); + int ret = BZ2_bzDecompressInit(&strm, 0, 0); if (ret != BZ_OK) - throw CompressionError("error while decompressing bzip2 file"); + throw CompressionError("unable to initialise bzip2 decoder"); + + strm.next_out = (char *) outbuf; + strm.avail_out = sizeof(outbuf); } -} -static void decompressBrotli(Source & source, Sink & sink) -{ -#if !HAVE_BROTLI - RunOptions options(BROTLI, {"-d"}); - options.standardIn = &source; - options.standardOut = &sink; - runProgram2(options); -#else - auto *s = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr); - if (!s) - throw CompressionError("unable to initialize brotli decoder"); - - Finally free([s]() { BrotliDecoderDestroyInstance(s); }); - - std::vector<uint8_t> inbuf(bufSize), outbuf(bufSize); - const uint8_t * next_in = nullptr; - size_t avail_in = 0; - bool eof = false; - - while (true) { - checkInterrupt(); - - if (avail_in == 0 && !eof) { - next_in = inbuf.data(); - try { - avail_in = source.read((unsigned char *) next_in, inbuf.size()); - } catch (EndOfFile &) { - eof = true; + ~BzipDecompressionSink() + { + BZ2_bzDecompressEnd(&strm); + } + + void finish() override + { + flush(); + write(nullptr, 0); + } + + void writeInternal(const unsigned char * data, size_t len) override + { + assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max()); + + strm.next_in = (char *) data; + strm.avail_in = len; + + while (strm.avail_in) { + checkInterrupt(); + + int ret = BZ2_bzDecompress(&strm); + if (ret != BZ_OK && ret != BZ_STREAM_END) + throw CompressionError("error while decompressing bzip2 file"); + + finished = ret == BZ_STREAM_END; + + if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { + nextSink(outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = (char *) outbuf; + strm.avail_out = sizeof(outbuf); } } + } +}; - uint8_t * next_out = outbuf.data(); - size_t avail_out = outbuf.size(); - - auto ret = BrotliDecoderDecompressStream(s, - &avail_in, &next_in, - &avail_out, &next_out, - nullptr); - - switch (ret) { - case BROTLI_DECODER_RESULT_ERROR: - throw CompressionError("error while decompressing brotli file"); - case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT: - if (eof) - throw CompressionError("incomplete or corrupt brotli file"); - break; - case BROTLI_DECODER_RESULT_SUCCESS: - if (avail_in != 0) - throw CompressionError("unexpected input after brotli decompression"); - break; - case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT: - // I'm not sure if this can happen, but abort if this happens with empty buffer - if (avail_out == outbuf.size()) - throw CompressionError("brotli decompression requires larger buffer"); - break; - } +struct BrotliDecompressionSink : ChunkedCompressionSink +{ + Sink & nextSink; + BrotliDecoderState * state; + bool finished = false; - // Always ensure we have full buffer for next invocation - if (avail_out < outbuf.size()) - sink((unsigned char *) outbuf.data(), outbuf.size() - avail_out); + BrotliDecompressionSink(Sink & nextSink) : nextSink(nextSink) + { + state = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr); + if (!state) + throw CompressionError("unable to initialize brotli decoder"); + } - if (ret == BROTLI_DECODER_RESULT_SUCCESS) return; + ~BrotliDecompressionSink() + { + BrotliDecoderDestroyInstance(state); } -#endif // HAVE_BROTLI -} + + void finish() override + { + flush(); + writeInternal(nullptr, 0); + } + + void writeInternal(const unsigned char * data, size_t len) override + { + const uint8_t * next_in = data; + size_t avail_in = len; + uint8_t * next_out = outbuf; + size_t avail_out = sizeof(outbuf); + + while (!finished && (!data || avail_in)) { + checkInterrupt(); + + if (!BrotliDecoderDecompressStream(state, + &avail_in, &next_in, + &avail_out, &next_out, + nullptr)) + throw CompressionError("error while decompressing brotli file"); + + if (avail_out < sizeof(outbuf) || avail_in == 0) { + nextSink(outbuf, sizeof(outbuf) - avail_out); + next_out = outbuf; + avail_out = sizeof(outbuf); + } + + finished = BrotliDecoderIsFinished(state); + } + } +}; ref<std::string> decompress(const std::string & method, const std::string & in) { - StringSource source(in); - StringSink sink; - decompress(method, source, sink); - return sink.s; + StringSink ssink; + auto sink = makeDecompressionSink(method, ssink); + (*sink)(in); + sink->finish(); + return ssink.s; } -void decompress(const std::string & method, Source & source, Sink & sink) +ref<CompressionSink> makeDecompressionSink(const std::string & method, Sink & nextSink) { - if (method == "none") - return decompressNone(source, sink); + if (method == "none" || method == "") + return make_ref<NoneSink>(nextSink); else if (method == "xz") - return decompressXZ(source, sink); + return make_ref<XzDecompressionSink>(nextSink); else if (method == "bzip2") - return decompressBzip2(source, sink); + return make_ref<BzipDecompressionSink>(nextSink); else if (method == "br") - return decompressBrotli(source, sink); + return make_ref<BrotliDecompressionSink>(nextSink); else throw UnknownCompressionMethod("unknown compression method '%s'", method); } -struct NoneSink : CompressionSink -{ - Sink & nextSink; - NoneSink(Sink & nextSink) : nextSink(nextSink) { } - void finish() override { flush(); } - void write(const unsigned char * data, size_t len) override { nextSink(data, len); } -}; - -struct XzSink : CompressionSink +struct XzCompressionSink : CompressionSink { Sink & nextSink; uint8_t outbuf[BUFSIZ]; lzma_stream strm = LZMA_STREAM_INIT; bool finished = false; - template <typename F> - XzSink(Sink & nextSink, F&& initEncoder) : nextSink(nextSink) { - lzma_ret ret = initEncoder(); + XzCompressionSink(Sink & nextSink, bool parallel) : nextSink(nextSink) + { + lzma_ret ret; + bool done = false; + + if (parallel) { +#ifdef HAVE_LZMA_MT + lzma_mt mt_options = {}; + mt_options.flags = 0; + mt_options.timeout = 300; // Using the same setting as the xz cmd line + mt_options.preset = LZMA_PRESET_DEFAULT; + mt_options.filters = NULL; + mt_options.check = LZMA_CHECK_CRC64; + mt_options.threads = lzma_cputhreads(); + mt_options.block_size = 0; + if (mt_options.threads == 0) + mt_options.threads = 1; + // FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the + // number of threads. + ret = lzma_stream_encoder_mt(&strm, &mt_options); + done = true; +#else + printMsg(lvlError, "warning: parallel XZ compression requested but not supported, falling back to single-threaded compression"); +#endif + } + + if (!done) + ret = lzma_easy_encoder(&strm, 6, LZMA_CHECK_CRC64); + if (ret != LZMA_OK) throw CompressionError("unable to initialise lzma encoder"); + // FIXME: apply the x86 BCJ filter? strm.next_out = outbuf; strm.avail_out = sizeof(outbuf); } - XzSink(Sink & nextSink) : XzSink(nextSink, [this]() { - return lzma_easy_encoder(&strm, 6, LZMA_CHECK_CRC64); - }) {} - ~XzSink() + ~XzCompressionSink() { lzma_end(&strm); } @@ -255,43 +274,25 @@ struct XzSink : CompressionSink void finish() override { CompressionSink::flush(); - - assert(!finished); - finished = true; - - while (true) { - checkInterrupt(); - - lzma_ret ret = lzma_code(&strm, LZMA_FINISH); - if (ret != LZMA_OK && ret != LZMA_STREAM_END) - throw CompressionError("error while flushing xz file"); - - if (strm.avail_out == 0 || ret == LZMA_STREAM_END) { - nextSink(outbuf, sizeof(outbuf) - strm.avail_out); - strm.next_out = outbuf; - strm.avail_out = sizeof(outbuf); - } - - if (ret == LZMA_STREAM_END) break; - } + write(nullptr, 0); } void write(const unsigned char * data, size_t len) override { - assert(!finished); - strm.next_in = data; strm.avail_in = len; - while (strm.avail_in) { + while (!finished && (!data || strm.avail_in)) { checkInterrupt(); - lzma_ret ret = lzma_code(&strm, LZMA_RUN); - if (ret != LZMA_OK) - throw CompressionError("error while compressing xz file"); + lzma_ret ret = lzma_code(&strm, data ? LZMA_RUN : LZMA_FINISH); + if (ret != LZMA_OK && ret != LZMA_STREAM_END) + throw CompressionError("error %d while compressing xz file", ret); + + finished = ret == LZMA_STREAM_END; - if (strm.avail_out == 0) { - nextSink(outbuf, sizeof(outbuf)); + if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { + nextSink(outbuf, sizeof(outbuf) - strm.avail_out); strm.next_out = outbuf; strm.avail_out = sizeof(outbuf); } @@ -299,46 +300,24 @@ struct XzSink : CompressionSink } }; -#ifdef HAVE_LZMA_MT -struct ParallelXzSink : public XzSink -{ - ParallelXzSink(Sink &nextSink) : XzSink(nextSink, [this]() { - lzma_mt mt_options = {}; - mt_options.flags = 0; - mt_options.timeout = 300; // Using the same setting as the xz cmd line - mt_options.preset = LZMA_PRESET_DEFAULT; - mt_options.filters = NULL; - mt_options.check = LZMA_CHECK_CRC64; - mt_options.threads = lzma_cputhreads(); - mt_options.block_size = 0; - if (mt_options.threads == 0) - mt_options.threads = 1; - // FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the - // number of threads. - return lzma_stream_encoder_mt(&strm, &mt_options); - }) {} -}; -#endif - -struct BzipSink : CompressionSink +struct BzipCompressionSink : ChunkedCompressionSink { Sink & nextSink; - char outbuf[BUFSIZ]; bz_stream strm; bool finished = false; - BzipSink(Sink & nextSink) : nextSink(nextSink) + BzipCompressionSink(Sink & nextSink) : nextSink(nextSink) { memset(&strm, 0, sizeof(strm)); int ret = BZ2_bzCompressInit(&strm, 9, 0, 30); if (ret != BZ_OK) throw CompressionError("unable to initialise bzip2 encoder"); - strm.next_out = outbuf; + strm.next_out = (char *) outbuf; strm.avail_out = sizeof(outbuf); } - ~BzipSink() + ~BzipCompressionSink() { BZ2_bzCompressEnd(&strm); } @@ -346,101 +325,49 @@ struct BzipSink : CompressionSink void finish() override { flush(); - - assert(!finished); - finished = true; - - while (true) { - checkInterrupt(); - - int ret = BZ2_bzCompress(&strm, BZ_FINISH); - if (ret != BZ_FINISH_OK && ret != BZ_STREAM_END) - throw CompressionError("error while flushing bzip2 file"); - - if (strm.avail_out == 0 || ret == BZ_STREAM_END) { - nextSink((unsigned char *) outbuf, sizeof(outbuf) - strm.avail_out); - strm.next_out = outbuf; - strm.avail_out = sizeof(outbuf); - } - - if (ret == BZ_STREAM_END) break; - } + writeInternal(nullptr, 0); } - void write(const unsigned char * data, size_t len) override + void writeInternal(const unsigned char * data, size_t len) override { - assert(!finished); + assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max()); strm.next_in = (char *) data; strm.avail_in = len; - while (strm.avail_in) { + while (!finished && (!data || strm.avail_in)) { checkInterrupt(); - int ret = BZ2_bzCompress(&strm, BZ_RUN); - if (ret != BZ_OK) - CompressionError("error while compressing bzip2 file"); + int ret = BZ2_bzCompress(&strm, data ? BZ_RUN : BZ_FINISH); + if (ret != BZ_RUN_OK && ret != BZ_FINISH_OK && ret != BZ_STREAM_END) + throw CompressionError("error %d while compressing bzip2 file", ret); - if (strm.avail_out == 0) { - nextSink((unsigned char *) outbuf, sizeof(outbuf)); - strm.next_out = outbuf; + finished = ret == BZ_STREAM_END; + + if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { + nextSink(outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = (char *) outbuf; strm.avail_out = sizeof(outbuf); } } } }; -struct LambdaCompressionSink : CompressionSink -{ - Sink & nextSink; - std::string data; - using CompressFnTy = std::function<std::string(const std::string&)>; - CompressFnTy compressFn; - LambdaCompressionSink(Sink& nextSink, CompressFnTy compressFn) - : nextSink(nextSink) - , compressFn(std::move(compressFn)) - { - }; - - void finish() override - { - flush(); - nextSink(compressFn(data)); - } - - void write(const unsigned char * data, size_t len) override - { - checkInterrupt(); - this->data.append((const char *) data, len); - } -}; - -struct BrotliCmdSink : LambdaCompressionSink -{ - BrotliCmdSink(Sink& nextSink) - : LambdaCompressionSink(nextSink, [](const std::string& data) { - return runProgram(BROTLI, true, {}, data); - }) - { - } -}; - -#if HAVE_BROTLI -struct BrotliSink : CompressionSink +struct BrotliCompressionSink : ChunkedCompressionSink { Sink & nextSink; uint8_t outbuf[BUFSIZ]; BrotliEncoderState *state; bool finished = false; - BrotliSink(Sink & nextSink) : nextSink(nextSink) + BrotliCompressionSink(Sink & nextSink) : nextSink(nextSink) { state = BrotliEncoderCreateInstance(nullptr, nullptr, nullptr); if (!state) throw CompressionError("unable to initialise brotli encoder"); } - ~BrotliSink() + ~BrotliCompressionSink() { BrotliEncoderDestroyInstance(state); } @@ -448,96 +375,47 @@ struct BrotliSink : CompressionSink void finish() override { flush(); - assert(!finished); - - const uint8_t *next_in = nullptr; - size_t avail_in = 0; - uint8_t *next_out = outbuf; - size_t avail_out = sizeof(outbuf); - while (!finished) { - checkInterrupt(); - - if (!BrotliEncoderCompressStream(state, - BROTLI_OPERATION_FINISH, - &avail_in, &next_in, - &avail_out, &next_out, - nullptr)) - throw CompressionError("error while finishing brotli file"); - - finished = BrotliEncoderIsFinished(state); - if (avail_out == 0 || finished) { - nextSink(outbuf, sizeof(outbuf) - avail_out); - next_out = outbuf; - avail_out = sizeof(outbuf); - } - } + writeInternal(nullptr, 0); } - void write(const unsigned char * data, size_t len) override - { - assert(!finished); - - // Don't feed brotli too much at once - const size_t CHUNK_SIZE = sizeof(outbuf) << 2; - while (len) { - size_t n = std::min(CHUNK_SIZE, len); - writeInternal(data, n); - data += n; - len -= n; - } - } - private: - void writeInternal(const unsigned char * data, size_t len) + void writeInternal(const unsigned char * data, size_t len) override { - assert(!finished); - - const uint8_t *next_in = data; + const uint8_t * next_in = data; size_t avail_in = len; - uint8_t *next_out = outbuf; + uint8_t * next_out = outbuf; size_t avail_out = sizeof(outbuf); - while (avail_in > 0) { + while (!finished && (!data || avail_in)) { checkInterrupt(); if (!BrotliEncoderCompressStream(state, - BROTLI_OPERATION_PROCESS, - &avail_in, &next_in, - &avail_out, &next_out, - nullptr)) - throw CompressionError("error while compressing brotli file"); + data ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH, + &avail_in, &next_in, + &avail_out, &next_out, + nullptr)) + throw CompressionError("error while compressing brotli compression"); if (avail_out < sizeof(outbuf) || avail_in == 0) { nextSink(outbuf, sizeof(outbuf) - avail_out); next_out = outbuf; avail_out = sizeof(outbuf); } + + finished = BrotliEncoderIsFinished(state); } } }; -#endif // HAVE_BROTLI ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel) { - if (parallel) { -#ifdef HAVE_LZMA_MT - if (method == "xz") - return make_ref<ParallelXzSink>(nextSink); -#endif - printMsg(lvlError, format("Warning: parallel compression requested but not supported for method '%1%', falling back to single-threaded compression") % method); - } - if (method == "none") return make_ref<NoneSink>(nextSink); else if (method == "xz") - return make_ref<XzSink>(nextSink); + return make_ref<XzCompressionSink>(nextSink, parallel); else if (method == "bzip2") - return make_ref<BzipSink>(nextSink); + return make_ref<BzipCompressionSink>(nextSink); else if (method == "br") -#if HAVE_BROTLI - return make_ref<BrotliSink>(nextSink); -#else - return make_ref<BrotliCmdSink>(nextSink); -#endif + return make_ref<BrotliCompressionSink>(nextSink); else throw UnknownCompressionMethod(format("unknown compression method '%s'") % method); } diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh index f7a3e3fbd32e..dd666a4e19fd 100644 --- a/src/libutil/compression.hh +++ b/src/libutil/compression.hh @@ -8,17 +8,17 @@ namespace nix { -ref<std::string> decompress(const std::string & method, const std::string & in); - -void decompress(const std::string & method, Source & source, Sink & sink); - -ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel = false); - struct CompressionSink : BufferedSink { virtual void finish() = 0; }; +ref<std::string> decompress(const std::string & method, const std::string & in); + +ref<CompressionSink> makeDecompressionSink(const std::string & method, Sink & nextSink); + +ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel = false); + ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false); MakeError(UnknownCompressionMethod, Error); diff --git a/src/libutil/config.cc b/src/libutil/config.cc index ce6858f0d65a..9023cb1bb6de 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -4,15 +4,13 @@ namespace nix { -void Config::set(const std::string & name, const std::string & value) +bool Config::set(const std::string & name, const std::string & value) { auto i = _settings.find(name); - if (i == _settings.end()) { - extras.emplace(name, value); - } else { - i->second.setting->set(value); - i->second.setting->overriden = true; - } + if (i == _settings.end()) return false; + i->second.setting->set(value); + i->second.setting->overriden = true; + return true; } void Config::addSetting(AbstractSetting * setting) @@ -23,46 +21,51 @@ void Config::addSetting(AbstractSetting * setting) bool set = false; - auto i = extras.find(setting->name); - if (i != extras.end()) { + auto i = unknownSettings.find(setting->name); + if (i != unknownSettings.end()) { setting->set(i->second); setting->overriden = true; - extras.erase(i); + unknownSettings.erase(i); set = true; } for (auto & alias : setting->aliases) { - auto i = extras.find(alias); - if (i != extras.end()) { + auto i = unknownSettings.find(alias); + if (i != unknownSettings.end()) { if (set) warn("setting '%s' is set, but it's an alias of '%s' which is also set", alias, setting->name); else { setting->set(i->second); setting->overriden = true; - extras.erase(i); + unknownSettings.erase(i); set = true; } } } } -void Config::handleUnknownSettings() +void AbstractConfig::warnUnknownSettings() { - for (auto & s : extras) + for (auto & s : unknownSettings) warn("unknown setting '%s'", s.first); } -StringMap Config::getSettings(bool overridenOnly) +void AbstractConfig::reapplyUnknownSettings() +{ + auto unknownSettings2 = std::move(unknownSettings); + for (auto & s : unknownSettings2) + set(s.first, s.second); +} + +void Config::getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly) { - StringMap res; for (auto & opt : _settings) if (!opt.second.isAlias && (!overridenOnly || opt.second.setting->overriden)) - res.emplace(opt.first, opt.second.setting->to_string()); - return res; + res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description}); } -void Config::applyConfigFile(const Path & path) +void AbstractConfig::applyConfigFile(const Path & path) { try { string contents = readFile(path); @@ -287,4 +290,49 @@ void PathSetting::set(const std::string & str) value = canonPath(str); } +bool GlobalConfig::set(const std::string & name, const std::string & value) +{ + for (auto & config : *configRegistrations) + if (config->set(name, value)) return true; + + unknownSettings.emplace(name, value); + + return false; +} + +void GlobalConfig::getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly) +{ + for (auto & config : *configRegistrations) + config->getSettings(res, overridenOnly); +} + +void GlobalConfig::resetOverriden() +{ + for (auto & config : *configRegistrations) + config->resetOverriden(); +} + +void GlobalConfig::toJSON(JSONObject & out) +{ + for (auto & config : *configRegistrations) + config->toJSON(out); +} + +void GlobalConfig::convertToArgs(Args & args, const std::string & category) +{ + for (auto & config : *configRegistrations) + config->convertToArgs(args, category); +} + +GlobalConfig globalConfig; + +GlobalConfig::ConfigRegistrations * GlobalConfig::configRegistrations; + +GlobalConfig::Register::Register(Config * config) +{ + if (!configRegistrations) + configRegistrations = new ConfigRegistrations; + configRegistrations->emplace_back(config); +} + } diff --git a/src/libutil/config.hh b/src/libutil/config.hh index d2e7faf17434..d86c65ff033a 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -12,6 +12,40 @@ class AbstractSetting; class JSONPlaceholder; class JSONObject; +class AbstractConfig +{ +protected: + StringMap unknownSettings; + + AbstractConfig(const StringMap & initials = {}) + : unknownSettings(initials) + { } + +public: + + virtual bool set(const std::string & name, const std::string & value) = 0; + + struct SettingInfo + { + std::string value; + std::string description; + }; + + virtual void getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly = false) = 0; + + void applyConfigFile(const Path & path); + + virtual void resetOverriden() = 0; + + virtual void toJSON(JSONObject & out) = 0; + + virtual void convertToArgs(Args & args, const std::string & category) = 0; + + void warnUnknownSettings(); + + void reapplyUnknownSettings(); +}; + /* A class to simplify providing configuration settings. The typical use is to inherit Config and add Setting<T> members: @@ -27,7 +61,7 @@ class JSONObject; }; */ -class Config +class Config : public AbstractConfig { friend class AbstractSetting; @@ -48,31 +82,23 @@ private: Settings _settings; - StringMap extras; - public: - Config(const StringMap & initials) - : extras(initials) + Config(const StringMap & initials = {}) + : AbstractConfig(initials) { } - void set(const std::string & name, const std::string & value); + bool set(const std::string & name, const std::string & value) override; void addSetting(AbstractSetting * setting); - void handleUnknownSettings(); - - StringMap getSettings(bool overridenOnly = false); + void getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly = false) override; - const Settings & _getSettings() { return _settings; } - - void applyConfigFile(const Path & path); + void resetOverriden() override; - void resetOverriden(); + void toJSON(JSONObject & out) override; - void toJSON(JSONObject & out); - - void convertToArgs(Args & args, const std::string & category); + void convertToArgs(Args & args, const std::string & category) override; }; class AbstractSetting @@ -209,4 +235,27 @@ public: void operator =(const Path & v) { this->assign(v); } }; +struct GlobalConfig : public AbstractConfig +{ + typedef std::vector<Config*> ConfigRegistrations; + static ConfigRegistrations * configRegistrations; + + bool set(const std::string & name, const std::string & value) override; + + void getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly = false) override; + + void resetOverriden() override; + + void toJSON(JSONObject & out) override; + + void convertToArgs(Args & args, const std::string & category) override; + + struct Register + { + Register(Config * config); + }; +}; + +extern GlobalConfig globalConfig; + } diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index a01d651e1ef5..1c14ebb187cc 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -82,7 +82,7 @@ static string printHash32(const Hash & hash) string s; s.reserve(len); - for (int n = len - 1; n >= 0; n--) { + for (int n = (int) len - 1; n >= 0; n--) { unsigned int b = n * 5; unsigned int i = b / 8; unsigned int j = b % 8; @@ -105,9 +105,9 @@ string printHash16or32(const Hash & hash) std::string Hash::to_string(Base base, bool includeType) const { std::string s; - if (includeType) { + if (base == SRI || includeType) { s += printHashType(type); - s += ':'; + s += base == SRI ? '-' : ':'; } switch (base) { case Base16: @@ -117,6 +117,7 @@ std::string Hash::to_string(Base base, bool includeType) const s += printHash32(*this); break; case Base64: + case SRI: s += base64Encode(std::string((const char *) hash, hashSize)); break; } @@ -127,28 +128,33 @@ std::string Hash::to_string(Base base, bool includeType) const Hash::Hash(const std::string & s, HashType type) : type(type) { - auto colon = s.find(':'); - size_t pos = 0; - - if (colon == string::npos) { - if (type == htUnknown) + bool isSRI = false; + + auto sep = s.find(':'); + if (sep == string::npos) { + sep = s.find('-'); + if (sep != string::npos) { + isSRI = true; + } else if (type == htUnknown) throw BadHash("hash '%s' does not include a type", s); - } else { - string hts = string(s, 0, colon); + } + + if (sep != string::npos) { + string hts = string(s, 0, sep); this->type = parseHashType(hts); if (this->type == htUnknown) throw BadHash("unknown hash type '%s'", hts); if (type != htUnknown && type != this->type) throw BadHash("hash '%s' should have type '%s'", s, printHashType(type)); - pos = colon + 1; + pos = sep + 1; } init(); size_t size = s.size() - pos; - if (size == base16Len()) { + if (!isSRI && size == base16Len()) { auto parseHexDigit = [&](char c) { if (c >= '0' && c <= '9') return c - '0'; @@ -164,7 +170,7 @@ Hash::Hash(const std::string & s, HashType type) } } - else if (size == base32Len()) { + else if (!isSRI && size == base32Len()) { for (unsigned int n = 0; n < size; ++n) { char c = s[pos + size - n - 1]; @@ -187,10 +193,10 @@ Hash::Hash(const std::string & s, HashType type) } } - else if (size == base64Len()) { + else if (isSRI || size == base64Len()) { auto d = base64Decode(std::string(s, pos)); if (d.size() != hashSize) - throw BadHash("invalid base-64 hash '%s'", s); + throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", s); assert(hashSize); memcpy(hash, d.data(), hashSize); } diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index fd7a61df8e46..2dbc3b630814 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -20,7 +20,7 @@ const int sha512HashSize = 64; extern const string base32Chars; -enum Base : int { Base64, Base32, Base16 }; +enum Base : int { Base64, Base32, Base16, SRI }; struct Hash @@ -38,8 +38,9 @@ struct Hash Hash(HashType type) : type(type) { init(); }; /* Initialize the hash from a string representation, in the format - "[<type>:]<base16|base32|base64>". If the 'type' argument is - htUnknown, then the hash type must be specified in the + "[<type>:]<base16|base32|base64>" or "<type>-<base64>" (a + Subresource Integrity hash expression). If the 'type' argument + is htUnknown, then the hash type must be specified in the string. */ Hash(const std::string & s, HashType type = htUnknown); diff --git a/src/libutil/json.cc b/src/libutil/json.cc index 813b257016e4..0a6fb65f0605 100644 --- a/src/libutil/json.cc +++ b/src/libutil/json.cc @@ -31,6 +31,7 @@ template<> void toJSON<unsigned long>(std::ostream & str, const unsigned long & template<> void toJSON<long long>(std::ostream & str, const long long & n) { str << n; } template<> void toJSON<unsigned long long>(std::ostream & str, const unsigned long long & n) { str << n; } template<> void toJSON<float>(std::ostream & str, const float & n) { str << n; } +template<> void toJSON<double>(std::ostream & str, const double & n) { str << n; } template<> void toJSON<std::string>(std::ostream & str, const std::string & s) { diff --git a/src/libutil/local.mk b/src/libutil/local.mk index 824f48fbfc9f..3ccc23fd5c1b 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -7,7 +7,3 @@ libutil_DIR := $(d) libutil_SOURCES := $(wildcard $(d)/*.cc) libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) -lboost_context - -libutil_LIBS = libformat - -libutil_CXXFLAGS = -DBROTLI=\"$(brotli)\" diff --git a/src/libutil/lru-cache.hh b/src/libutil/lru-cache.hh index 9b8290e634c9..8b83f842c324 100644 --- a/src/libutil/lru-cache.hh +++ b/src/libutil/lru-cache.hh @@ -2,7 +2,7 @@ #include <map> #include <list> -#include <experimental/optional> +#include <optional> namespace nix { @@ -64,7 +64,7 @@ public: /* Look up an item in the cache. If it exists, it becomes the most recently used item. */ - std::experimental::optional<Value> get(const Key & key) + std::optional<Value> get(const Key & key) { auto i = data.find(key); if (i == data.end()) return {}; diff --git a/src/libutil/pool.hh b/src/libutil/pool.hh index 0b142b0597c7..d49067bb95dc 100644 --- a/src/libutil/pool.hh +++ b/src/libutil/pool.hh @@ -97,6 +97,7 @@ public: private: Pool & pool; std::shared_ptr<R> r; + bool bad = false; friend Pool; @@ -112,7 +113,8 @@ public: if (!r) return; { auto state_(pool.state.lock()); - state_->idle.push_back(ref<R>(r)); + if (!bad) + state_->idle.push_back(ref<R>(r)); assert(state_->inUse); state_->inUse--; } @@ -121,6 +123,8 @@ public: R * operator -> () { return &*r; } R & operator * () { return *r; } + + void markBad() { bad = true; } }; Handle get() diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 33ae1ea389d7..8201549fd7d0 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -133,7 +133,7 @@ size_t FdSource::readUnbuffered(unsigned char * data, size_t len) ssize_t n; do { checkInterrupt(); - n = ::read(fd, (char *) data, bufSize); + n = ::read(fd, (char *) data, len); } while (n == -1 && errno == EINTR); if (n == -1) { _good = false; throw SysError("reading from file"); } if (n == 0) { _good = false; throw EndOfFile("unexpected end-of-file"); } @@ -157,21 +157,25 @@ size_t StringSource::read(unsigned char * data, size_t len) } -std::unique_ptr<Source> sinkToSource(std::function<void(Sink &)> fun) +#if BOOST_VERSION >= 106300 && BOOST_VERSION < 106600 +#error Coroutines are broken in this version of Boost! +#endif + +std::unique_ptr<Source> sinkToSource( + std::function<void(Sink &)> fun, + std::function<void()> eof) { struct SinkToSource : Source { typedef boost::coroutines2::coroutine<std::string> coro_t; - coro_t::pull_type coro; + std::function<void(Sink &)> fun; + std::function<void()> eof; + std::optional<coro_t::pull_type> coro; + bool started = false; - SinkToSource(std::function<void(Sink &)> fun) - : coro([&](coro_t::push_type & yield) { - LambdaSink sink([&](const unsigned char * data, size_t len) { - if (len) yield(std::string((const char *) data, len)); - }); - fun(sink); - }) + SinkToSource(std::function<void(Sink &)> fun, std::function<void()> eof) + : fun(fun), eof(eof) { } @@ -181,11 +185,18 @@ std::unique_ptr<Source> sinkToSource(std::function<void(Sink &)> fun) size_t read(unsigned char * data, size_t len) override { if (!coro) - throw EndOfFile("coroutine has finished"); + coro = coro_t::pull_type([&](coro_t::push_type & yield) { + LambdaSink sink([&](const unsigned char * data, size_t len) { + if (len) yield(std::string((const char *) data, len)); + }); + fun(sink); + }); + + if (!*coro) { eof(); abort(); } if (pos == cur.size()) { - if (!cur.empty()) coro(); - cur = coro.get(); + if (!cur.empty()) (*coro)(); + cur = coro->get(); pos = 0; } @@ -197,7 +208,7 @@ std::unique_ptr<Source> sinkToSource(std::function<void(Sink &)> fun) } }; - return std::make_unique<SinkToSource>(fun); + return std::make_unique<SinkToSource>(fun, eof); } @@ -261,16 +272,17 @@ void readPadding(size_t len, Source & source) size_t readString(unsigned char * buf, size_t max, Source & source) { auto len = readNum<size_t>(source); - if (len > max) throw Error("string is too long"); + if (len > max) throw SerialisationError("string is too long"); source(buf, len); readPadding(len, source); return len; } -string readString(Source & source) +string readString(Source & source, size_t max) { auto len = readNum<size_t>(source); + if (len > max) throw SerialisationError("string is too long"); std::string res(len, 0); source((unsigned char*) res.data(), len); readPadding(len, source); diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index d0b4552e3399..969e4dff383d 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -77,10 +77,12 @@ struct BufferedSource : Source size_t read(unsigned char * data, size_t len) override; - /* Underlying read call, to be overridden. */ - virtual size_t readUnbuffered(unsigned char * data, size_t len) = 0; bool hasData(); + +protected: + /* Underlying read call, to be overridden. */ + virtual size_t readUnbuffered(unsigned char * data, size_t len) = 0; }; @@ -134,8 +136,9 @@ struct FdSource : BufferedSource return *this; } - size_t readUnbuffered(unsigned char * data, size_t len) override; bool good() override; +protected: + size_t readUnbuffered(unsigned char * data, size_t len) override; private: bool _good = true; }; @@ -211,7 +214,11 @@ struct LambdaSource : Source /* Convert a function that feeds data into a Sink into a Source. The Source executes the function as a coroutine. */ -std::unique_ptr<Source> sinkToSource(std::function<void(Sink &)> fun); +std::unique_ptr<Source> sinkToSource( + std::function<void(Sink &)> fun, + std::function<void()> eof = []() { + throw EndOfFile("coroutine has finished"); + }); void writePadding(size_t len, Sink & sink); @@ -227,7 +234,7 @@ inline Sink & operator << (Sink & sink, uint64_t n) buf[4] = (n >> 32) & 0xff; buf[5] = (n >> 40) & 0xff; buf[6] = (n >> 48) & 0xff; - buf[7] = (n >> 56) & 0xff; + buf[7] = (unsigned char) (n >> 56) & 0xff; sink(buf, sizeof(buf)); return sink; } @@ -259,7 +266,7 @@ T readNum(Source & source) if (n > std::numeric_limits<T>::max()) throw SerialisationError("serialised integer %d is too large for type '%s'", n, typeid(T).name()); - return n; + return (T) n; } @@ -277,7 +284,7 @@ inline uint64_t readLongLong(Source & source) void readPadding(size_t len, Source & source); size_t readString(unsigned char * buf, size_t max, Source & source); -string readString(Source & source); +string readString(Source & source, size_t max = std::numeric_limits<size_t>::max()); template<class T> T readStrings(Source & source); Source & operator >> (Source & in, string & s); diff --git a/src/libutil/sync.hh b/src/libutil/sync.hh index 3b2710f6fd30..e1d591d77a84 100644 --- a/src/libutil/sync.hh +++ b/src/libutil/sync.hh @@ -57,11 +57,11 @@ public: } template<class Rep, class Period> - void wait_for(std::condition_variable & cv, + std::cv_status wait_for(std::condition_variable & cv, const std::chrono::duration<Rep, Period> & duration) { assert(s); - cv.wait_for(lk, duration); + return cv.wait_for(lk, duration); } template<class Rep, class Period, class Predicate> diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 15962236ec65..17aee2d5c3d0 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -16,6 +16,7 @@ #include <future> #include <fcntl.h> +#include <grp.h> #include <limits.h> #include <pwd.h> #include <sys/ioctl.h> @@ -38,6 +39,9 @@ extern char * * environ; namespace nix { +const std::string nativeSystem = SYSTEM; + + BaseError & BaseError::addPrefix(const FormatOrString & fs) { prefix_ = fs.s + prefix_; @@ -167,7 +171,7 @@ Path dirOf(const Path & path) { Path::size_type pos = path.rfind('/'); if (pos == string::npos) - throw Error(format("invalid file name '%1%'") % path); + return "."; return pos == 0 ? "/" : Path(path, 0, pos); } @@ -202,7 +206,7 @@ bool isInDir(const Path & path, const Path & dir) bool isDirOrInDir(const Path & path, const Path & dir) { - return path == dir or isInDir(path, dir); + return path == dir || isInDir(path, dir); } @@ -311,6 +315,14 @@ string readFile(const Path & path, bool drain) } +void readFile(const Path & path, Sink & sink) +{ + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) throw SysError("opening file '%s'", path); + drainFD(fd.get(), sink); +} + + void writeFile(const Path & path, const string & s, mode_t mode) { AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); @@ -320,6 +332,23 @@ void writeFile(const Path & path, const string & s, mode_t mode) } +void writeFile(const Path & path, Source & source, mode_t mode) +{ + AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); + if (!fd) + throw SysError(format("opening file '%1%'") % path); + + std::vector<unsigned char> buf(64 * 1024); + + while (true) { + try { + auto n = source.read(buf.data(), buf.size()); + writeFull(fd.get(), (unsigned char *) buf.data(), n); + } catch (EndOfFile &) { break; } + } +} + + string readLine(int fd) { string s; @@ -443,7 +472,7 @@ static Lazy<Path> getHome2([]() { std::vector<char> buf(16384); struct passwd pwbuf; struct passwd * pw; - if (getpwuid_r(getuid(), &pwbuf, buf.data(), buf.size(), &pw) != 0 + if (getpwuid_r(geteuid(), &pwbuf, buf.data(), buf.size(), &pw) != 0 || !pw || !pw->pw_dir || !pw->pw_dir[0]) throw Error("cannot determine user's home directory"); homeDir = pw->pw_dir; @@ -471,6 +500,15 @@ Path getConfigDir() return configDir; } +std::vector<Path> getConfigDirs() +{ + Path configHome = getConfigDir(); + string configDirs = getEnv("XDG_CONFIG_DIRS"); + std::vector<Path> result = tokenizeString<std::vector<string>>(configDirs, ":"); + result.insert(result.begin(), configHome); + return result; +} + Path getDataDir() { @@ -593,7 +631,7 @@ void drainFD(int fd, Sink & sink, bool block) throw SysError("making file descriptor non-blocking"); } - std::vector<unsigned char> buf(4096); + std::vector<unsigned char> buf(64 * 1024); while (1) { checkInterrupt(); ssize_t rd = read(fd, buf.data(), buf.size()); @@ -931,7 +969,7 @@ std::vector<char *> stringsToCharPtrs(const Strings & ss) string runProgram(Path program, bool searchPath, const Strings & args, - const std::experimental::optional<std::string> & input) + const std::optional<std::string> & input) { RunOptions opts(program, args); opts.searchPath = searchPath; @@ -988,6 +1026,16 @@ void runProgram2(const RunOptions & options) if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1) throw SysError("dupping stdin"); + if (options.chdir && chdir((*options.chdir).c_str()) == -1) + throw SysError("chdir failed"); + if (options.gid && setgid(*options.gid) == -1) + throw SysError("setgid failed"); + /* Drop all other groups if we're setgid. */ + if (options.gid && setgroups(0, 0) == -1) + throw SysError("setgroups failed"); + if (options.uid && setuid(*options.uid) == -1) + throw SysError("setuid failed"); + Strings args_(options.args); args_.push_front(options.program); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 743d238611fc..fce3cab8def5 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -14,7 +14,8 @@ #include <cstdio> #include <map> #include <sstream> -#include <experimental/optional> +#include <optional> +#include <future> #ifndef HAVE_STRUCT_DIRENT_D_TYPE #define DT_UNKNOWN 0 @@ -29,6 +30,10 @@ struct Sink; struct Source; +/* The system for which Nix is compiled. */ +extern const std::string nativeSystem; + + /* Return an environment variable. */ string getEnv(const string & key, const string & def = ""); @@ -97,10 +102,13 @@ unsigned char getFileType(const Path & path); /* Read the contents of a file into a string. */ string readFile(int fd); string readFile(const Path & path, bool drain = false); +void readFile(const Path & path, Sink & sink); /* Write a string to a file. */ void writeFile(const Path & path, const string & s, mode_t mode = 0666); +void writeFile(const Path & path, Source & source, mode_t mode = 0666); + /* Read a line from a file descriptor. */ string readLine(int fd); @@ -127,6 +135,9 @@ Path getCacheDir(); /* Return $XDG_CONFIG_HOME or $HOME/.config. */ Path getConfigDir(); +/* Return the directories to search for user configuration files */ +std::vector<Path> getConfigDirs(); + /* Return $XDG_DATA_HOME or $HOME/.local/share. */ Path getDataDir(); @@ -252,14 +263,17 @@ pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = P shell backtick operator). */ string runProgram(Path program, bool searchPath = false, const Strings & args = Strings(), - const std::experimental::optional<std::string> & input = {}); + const std::optional<std::string> & input = {}); struct RunOptions { + std::optional<uid_t> uid; + std::optional<uid_t> gid; + std::optional<Path> chdir; Path program; bool searchPath = true; Strings args; - std::experimental::optional<std::string> input; + std::optional<std::string> input; Source * standardIn = nullptr; Sink * standardOut = nullptr; bool _killStderr = false; @@ -394,6 +408,7 @@ void ignoreException(); /* Some ANSI escape sequences. */ #define ANSI_NORMAL "\e[0m" #define ANSI_BOLD "\e[1m" +#define ANSI_FAINT "\e[2m" #define ANSI_RED "\e[31;1m" #define ANSI_GREEN "\e[32;1m" #define ANSI_BLUE "\e[34;1m" @@ -424,44 +439,30 @@ string get(const T & map, const string & key, const string & def = "") } -/* Call ‘failure’ with the current exception as argument. If ‘failure’ - throws an exception, abort the program. */ -void callFailure(const std::function<void(std::exception_ptr exc)> & failure, - std::exception_ptr exc = std::current_exception()); +/* A callback is a wrapper around a lambda that accepts a valid of + type T or an exception. (We abuse std::future<T> to pass the value or + exception.) */ +template<typename T> +struct Callback +{ + std::function<void(std::future<T>)> fun; + Callback(std::function<void(std::future<T>)> fun) : fun(fun) { } -/* Evaluate the function ‘f’. If it returns a value, call ‘success’ - with that value as its argument. If it or ‘success’ throws an - exception, call ‘failure’. If ‘failure’ throws an exception, abort - the program. */ -template<class T> -void sync2async( - const std::function<void(T)> & success, - const std::function<void(std::exception_ptr exc)> & failure, - const std::function<T()> & f) -{ - try { - success(f()); - } catch (...) { - callFailure(failure); + void operator()(T && t) const + { + std::promise<T> promise; + promise.set_value(std::move(t)); + fun(promise.get_future()); } -} - -/* Call the function ‘success’. If it throws an exception, call - ‘failure’. If that throws an exception, abort the program. */ -template<class T> -void callSuccess( - const std::function<void(T)> & success, - const std::function<void(std::exception_ptr exc)> & failure, - T && arg) -{ - try { - success(arg); - } catch (...) { - callFailure(failure); + void rethrow(const std::exception_ptr & exc = std::current_exception()) const + { + std::promise<T> promise; + promise.set_exception(exc); + fun(promise.get_future()); } -} +}; /* Start a thread that handles various signals. Also block those signals diff --git a/src/libutil/xml-writer.cc b/src/libutil/xml-writer.cc index 98bd058d18be..e5cc2e9fc719 100644 --- a/src/libutil/xml-writer.cc +++ b/src/libutil/xml-writer.cc @@ -28,7 +28,7 @@ void XMLWriter::close() } -void XMLWriter::indent_(unsigned int depth) +void XMLWriter::indent_(size_t depth) { if (!indent) return; output << string(depth * 2, ' '); @@ -75,7 +75,7 @@ void XMLWriter::writeAttrs(const XMLAttrs & attrs) { for (auto & i : attrs) { output << " " << i.first << "=\""; - for (unsigned int j = 0; j < i.second.size(); ++j) { + for (size_t j = 0; j < i.second.size(); ++j) { char c = i.second[j]; if (c == '"') output << """; else if (c == '<') output << "<"; diff --git a/src/libutil/xml-writer.hh b/src/libutil/xml-writer.hh index 3cefe3712c08..b98b445265a2 100644 --- a/src/libutil/xml-writer.hh +++ b/src/libutil/xml-writer.hh @@ -44,7 +44,7 @@ public: private: void writeAttrs(const XMLAttrs & attrs); - void indent_(unsigned int depth); + void indent_(size_t depth); }; diff --git a/src/linenoise/ConvertUTF.cpp b/src/linenoise/ConvertUTF.cpp deleted file mode 100644 index f7e5915d5e8f..000000000000 --- a/src/linenoise/ConvertUTF.cpp +++ /dev/null @@ -1,542 +0,0 @@ -/* - * Copyright 2001-2004 Unicode, Inc. - * - * Disclaimer - * - * This source code is provided as is by Unicode, Inc. No claims are - * made as to fitness for any particular purpose. No warranties of any - * kind are expressed or implied. The recipient agrees to determine - * applicability of information provided. If this file has been - * purchased on magnetic or optical media from Unicode, Inc., the - * sole remedy for any claim will be exchange of defective media - * within 90 days of receipt. - * - * Limitations on Rights to Redistribute This Code - * - * Unicode, Inc. hereby grants the right to freely use the information - * supplied in this file in the creation of products supporting the - * Unicode Standard, and to make copies of this file in any form - * for internal or external distribution as long as this notice - * remains attached. - */ - -/* --------------------------------------------------------------------- - - Conversions between UTF32, UTF-16, and UTF-8. Source code file. - Author: Mark E. Davis, 1994. - Rev History: Rick McGowan, fixes & updates May 2001. - Sept 2001: fixed const & error conditions per - mods suggested by S. Parent & A. Lillich. - June 2002: Tim Dodd added detection and handling of incomplete - source sequences, enhanced error detection, added casts - to eliminate compiler warnings. - July 2003: slight mods to back out aggressive FFFE detection. - Jan 2004: updated switches in from-UTF8 conversions. - Oct 2004: updated to use UNI_MAX_LEGAL_UTF32 in UTF-32 conversions. - - See the header file "ConvertUTF.h" for complete documentation. - ------------------------------------------------------------------------- */ - -#include "ConvertUTF.h" -#ifdef CVTUTF_DEBUG -#include <stdio.h> -#endif - -namespace linenoise_ng { - -static const int halfShift = 10; /* used for shifting by 10 bits */ - -static const UTF32 halfBase = 0x0010000UL; -static const UTF32 halfMask = 0x3FFUL; - -#define UNI_SUR_HIGH_START (UTF32)0xD800 -#define UNI_SUR_HIGH_END (UTF32)0xDBFF -#define UNI_SUR_LOW_START (UTF32)0xDC00 -#define UNI_SUR_LOW_END (UTF32)0xDFFF -#define false 0 -#define true 1 - -/* --------------------------------------------------------------------- */ - -ConversionResult ConvertUTF32toUTF16 ( - const UTF32** sourceStart, const UTF32* sourceEnd, - char16_t** targetStart, char16_t* targetEnd, ConversionFlags flags) { - ConversionResult result = conversionOK; - const UTF32* source = *sourceStart; - char16_t* target = *targetStart; - while (source < sourceEnd) { - UTF32 ch; - if (target >= targetEnd) { - result = targetExhausted; break; - } - ch = *source++; - if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */ - /* UTF-16 surrogate values are illegal in UTF-32; 0xffff or 0xfffe are both reserved values */ - if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { - if (flags == strictConversion) { - --source; /* return to the illegal value itself */ - result = sourceIllegal; - break; - } else { - *target++ = UNI_REPLACEMENT_CHAR; - } - } else { - *target++ = (UTF16)ch; /* normal case */ - } - } else if (ch > UNI_MAX_LEGAL_UTF32) { - if (flags == strictConversion) { - result = sourceIllegal; - } else { - *target++ = UNI_REPLACEMENT_CHAR; - } - } else { - /* target is a character in range 0xFFFF - 0x10FFFF. */ - if (target + 1 >= targetEnd) { - --source; /* Back up source pointer! */ - result = targetExhausted; break; - } - ch -= halfBase; - *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START); - *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START); - } - } - *sourceStart = source; - *targetStart = target; - return result; -} - -/* --------------------------------------------------------------------- */ - -ConversionResult ConvertUTF16toUTF32 ( - const UTF16** sourceStart, const UTF16* sourceEnd, - UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) { - ConversionResult result = conversionOK; - const UTF16* source = *sourceStart; - UTF32* target = *targetStart; - UTF32 ch, ch2; - while (source < sourceEnd) { - const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */ - ch = *source++; - /* If we have a surrogate pair, convert to UTF32 first. */ - if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) { - /* If the 16 bits following the high surrogate are in the source buffer... */ - if (source < sourceEnd) { - ch2 = *source; - /* If it's a low surrogate, convert to UTF32. */ - if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) { - ch = ((ch - UNI_SUR_HIGH_START) << halfShift) - + (ch2 - UNI_SUR_LOW_START) + halfBase; - ++source; - } else if (flags == strictConversion) { /* it's an unpaired high surrogate */ - --source; /* return to the illegal value itself */ - result = sourceIllegal; - break; - } - } else { /* We don't have the 16 bits following the high surrogate. */ - --source; /* return to the high surrogate */ - result = sourceExhausted; - break; - } - } else if (flags == strictConversion) { - /* UTF-16 surrogate values are illegal in UTF-32 */ - if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) { - --source; /* return to the illegal value itself */ - result = sourceIllegal; - break; - } - } - if (target >= targetEnd) { - source = oldSource; /* Back up source pointer! */ - result = targetExhausted; break; - } - *target++ = ch; - } - *sourceStart = source; - *targetStart = target; -#ifdef CVTUTF_DEBUG -if (result == sourceIllegal) { - fprintf(stderr, "ConvertUTF16toUTF32 illegal seq 0x%04x,%04x\n", ch, ch2); - fflush(stderr); -} -#endif - return result; -} - -/* --------------------------------------------------------------------- */ - -/* - * Index into the table below with the first byte of a UTF-8 sequence to - * get the number of trailing bytes that are supposed to follow it. - * Note that *legal* UTF-8 values can't have 4 or 5-bytes. The table is - * left as-is for anyone who may want to do such conversion, which was - * allowed in earlier algorithms. - */ -static const char trailingBytesForUTF8[256] = { - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, - 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5 -}; - -/* - * Magic values subtracted from a buffer value during UTF8 conversion. - * This table contains as many values as there might be trailing bytes - * in a UTF-8 sequence. - */ -static const UTF32 offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL, - 0x03C82080UL, 0xFA082080UL, 0x82082080UL }; - -/* - * Once the bits are split out into bytes of UTF-8, this is a mask OR-ed - * into the first byte, depending on how many bytes follow. There are - * as many entries in this table as there are UTF-8 sequence types. - * (I.e., one byte sequence, two byte... etc.). Remember that sequencs - * for *legal* UTF-8 will be 4 or fewer bytes total. - */ -static const UTF8 firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC }; - -/* --------------------------------------------------------------------- */ - -/* The interface converts a whole buffer to avoid function-call overhead. - * Constants have been gathered. Loops & conditionals have been removed as - * much as possible for efficiency, in favor of drop-through switches. - * (See "Note A" at the bottom of the file for equivalent code.) - * If your compiler supports it, the "isLegalUTF8" call can be turned - * into an inline function. - */ - -/* --------------------------------------------------------------------- */ - -ConversionResult ConvertUTF16toUTF8 ( - const UTF16** sourceStart, const UTF16* sourceEnd, - UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) { - ConversionResult result = conversionOK; - const UTF16* source = *sourceStart; - UTF8* target = *targetStart; - while (source < sourceEnd) { - UTF32 ch; - unsigned short bytesToWrite = 0; - const UTF32 byteMask = 0xBF; - const UTF32 byteMark = 0x80; - const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */ - ch = *source++; - /* If we have a surrogate pair, convert to UTF32 first. */ - if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) { - /* If the 16 bits following the high surrogate are in the source buffer... */ - if (source < sourceEnd) { - UTF32 ch2 = *source; - /* If it's a low surrogate, convert to UTF32. */ - if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) { - ch = ((ch - UNI_SUR_HIGH_START) << halfShift) - + (ch2 - UNI_SUR_LOW_START) + halfBase; - ++source; - } else if (flags == strictConversion) { /* it's an unpaired high surrogate */ - --source; /* return to the illegal value itself */ - result = sourceIllegal; - break; - } - } else { /* We don't have the 16 bits following the high surrogate. */ - --source; /* return to the high surrogate */ - result = sourceExhausted; - break; - } - } else if (flags == strictConversion) { - /* UTF-16 surrogate values are illegal in UTF-32 */ - if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) { - --source; /* return to the illegal value itself */ - result = sourceIllegal; - break; - } - } - /* Figure out how many bytes the result will require */ - if (ch < (UTF32)0x80) { bytesToWrite = 1; - } else if (ch < (UTF32)0x800) { bytesToWrite = 2; - } else if (ch < (UTF32)0x10000) { bytesToWrite = 3; - } else if (ch < (UTF32)0x110000) { bytesToWrite = 4; - } else { bytesToWrite = 3; - ch = UNI_REPLACEMENT_CHAR; - } - - target += bytesToWrite; - if (target > targetEnd) { - source = oldSource; /* Back up source pointer! */ - target -= bytesToWrite; result = targetExhausted; break; - } - switch (bytesToWrite) { /* note: everything falls through. */ - case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; - case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; - case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; - case 1: *--target = (UTF8)(ch | firstByteMark[bytesToWrite]); - } - target += bytesToWrite; - } - *sourceStart = source; - *targetStart = target; - return result; -} - -/* --------------------------------------------------------------------- */ - -/* - * Utility routine to tell whether a sequence of bytes is legal UTF-8. - * This must be called with the length pre-determined by the first byte. - * If not calling this from ConvertUTF8to*, then the length can be set by: - * length = trailingBytesForUTF8[*source]+1; - * and the sequence is illegal right away if there aren't that many bytes - * available. - * If presented with a length > 4, this returns false. The Unicode - * definition of UTF-8 goes up to 4-byte sequences. - */ - -static Boolean isLegalUTF8(const UTF8 *source, int length) { - UTF8 a; - const UTF8 *srcptr = source+length; - switch (length) { - default: return false; - /* Everything else falls through when "true"... */ - case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false; - case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false; - case 2: if ((a = (*--srcptr)) > 0xBF) return false; - - switch (*source) { - /* no fall-through in this inner switch */ - case 0xE0: if (a < 0xA0) return false; break; - case 0xED: if (a > 0x9F) return false; break; - case 0xF0: if (a < 0x90) return false; break; - case 0xF4: if (a > 0x8F) return false; break; - default: if (a < 0x80) return false; - } - - case 1: if (*source >= 0x80 && *source < 0xC2) return false; - } - if (*source > 0xF4) return false; - return true; -} - -/* --------------------------------------------------------------------- */ - -/* - * Exported function to return whether a UTF-8 sequence is legal or not. - * This is not used here; it's just exported. - */ -Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd) { - int length = trailingBytesForUTF8[*source]+1; - if (source+length > sourceEnd) { - return false; - } - return isLegalUTF8(source, length); -} - -/* --------------------------------------------------------------------- */ - -ConversionResult ConvertUTF8toUTF16 ( - const UTF8** sourceStart, const UTF8* sourceEnd, - UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) { - ConversionResult result = conversionOK; - const UTF8* source = *sourceStart; - UTF16* target = *targetStart; - while (source < sourceEnd) { - UTF32 ch = 0; - unsigned short extraBytesToRead = trailingBytesForUTF8[*source]; - if (source + extraBytesToRead >= sourceEnd) { - result = sourceExhausted; break; - } - /* Do this check whether lenient or strict */ - if (! isLegalUTF8(source, extraBytesToRead+1)) { - result = sourceIllegal; - break; - } - /* - * The cases all fall through. See "Note A" below. - */ - switch (extraBytesToRead) { - case 5: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */ - case 4: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */ - case 3: ch += *source++; ch <<= 6; - case 2: ch += *source++; ch <<= 6; - case 1: ch += *source++; ch <<= 6; - case 0: ch += *source++; - } - ch -= offsetsFromUTF8[extraBytesToRead]; - - if (target >= targetEnd) { - source -= (extraBytesToRead+1); /* Back up source pointer! */ - result = targetExhausted; break; - } - if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */ - /* UTF-16 surrogate values are illegal in UTF-32 */ - if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { - if (flags == strictConversion) { - source -= (extraBytesToRead+1); /* return to the illegal value itself */ - result = sourceIllegal; - break; - } else { - *target++ = UNI_REPLACEMENT_CHAR; - } - } else { - *target++ = (UTF16)ch; /* normal case */ - } - } else if (ch > UNI_MAX_UTF16) { - if (flags == strictConversion) { - result = sourceIllegal; - source -= (extraBytesToRead+1); /* return to the start */ - break; /* Bail out; shouldn't continue */ - } else { - *target++ = UNI_REPLACEMENT_CHAR; - } - } else { - /* target is a character in range 0xFFFF - 0x10FFFF. */ - if (target + 1 >= targetEnd) { - source -= (extraBytesToRead+1); /* Back up source pointer! */ - result = targetExhausted; break; - } - ch -= halfBase; - *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START); - *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START); - } - } - *sourceStart = source; - *targetStart = target; - return result; -} - -/* --------------------------------------------------------------------- */ - -ConversionResult ConvertUTF32toUTF8 ( - const UTF32** sourceStart, const UTF32* sourceEnd, - UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) { - ConversionResult result = conversionOK; - const UTF32* source = *sourceStart; - UTF8* target = *targetStart; - while (source < sourceEnd) { - UTF32 ch; - unsigned short bytesToWrite = 0; - const UTF32 byteMask = 0xBF; - const UTF32 byteMark = 0x80; - ch = *source++; - if (flags == strictConversion ) { - /* UTF-16 surrogate values are illegal in UTF-32 */ - if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { - --source; /* return to the illegal value itself */ - result = sourceIllegal; - break; - } - } - /* - * Figure out how many bytes the result will require. Turn any - * illegally large UTF32 things (> Plane 17) into replacement chars. - */ - if (ch < (UTF32)0x80) { bytesToWrite = 1; - } else if (ch < (UTF32)0x800) { bytesToWrite = 2; - } else if (ch < (UTF32)0x10000) { bytesToWrite = 3; - } else if (ch <= UNI_MAX_LEGAL_UTF32) { bytesToWrite = 4; - } else { bytesToWrite = 3; - ch = UNI_REPLACEMENT_CHAR; - result = sourceIllegal; - } - - target += bytesToWrite; - if (target > targetEnd) { - --source; /* Back up source pointer! */ - target -= bytesToWrite; result = targetExhausted; break; - } - switch (bytesToWrite) { /* note: everything falls through. */ - case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; - case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; - case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; - case 1: *--target = (UTF8) (ch | firstByteMark[bytesToWrite]); - } - target += bytesToWrite; - } - *sourceStart = source; - *targetStart = target; - return result; -} - -/* --------------------------------------------------------------------- */ - -ConversionResult ConvertUTF8toUTF32 ( - const UTF8** sourceStart, const UTF8* sourceEnd, - UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) { - ConversionResult result = conversionOK; - const UTF8* source = *sourceStart; - UTF32* target = *targetStart; - while (source < sourceEnd) { - UTF32 ch = 0; - unsigned short extraBytesToRead = trailingBytesForUTF8[*source]; - if (source + extraBytesToRead >= sourceEnd) { - result = sourceExhausted; break; - } - /* Do this check whether lenient or strict */ - if (! isLegalUTF8(source, extraBytesToRead+1)) { - result = sourceIllegal; - break; - } - /* - * The cases all fall through. See "Note A" below. - */ - switch (extraBytesToRead) { - case 5: ch += *source++; ch <<= 6; - case 4: ch += *source++; ch <<= 6; - case 3: ch += *source++; ch <<= 6; - case 2: ch += *source++; ch <<= 6; - case 1: ch += *source++; ch <<= 6; - case 0: ch += *source++; - } - ch -= offsetsFromUTF8[extraBytesToRead]; - - if (target >= targetEnd) { - source -= (extraBytesToRead+1); /* Back up the source pointer! */ - result = targetExhausted; break; - } - if (ch <= UNI_MAX_LEGAL_UTF32) { - /* - * UTF-16 surrogate values are illegal in UTF-32, and anything - * over Plane 17 (> 0x10FFFF) is illegal. - */ - if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { - if (flags == strictConversion) { - source -= (extraBytesToRead+1); /* return to the illegal value itself */ - result = sourceIllegal; - break; - } else { - *target++ = UNI_REPLACEMENT_CHAR; - } - } else { - *target++ = ch; - } - } else { /* i.e., ch > UNI_MAX_LEGAL_UTF32 */ - result = sourceIllegal; - *target++ = UNI_REPLACEMENT_CHAR; - } - } - *sourceStart = source; - *targetStart = target; - return result; -} - -} - -/* --------------------------------------------------------------------- - - Note A. - The fall-through switches in UTF-8 reading code save a - temp variable, some decrements & conditionals. The switches - are equivalent to the following loop: - { - int tmpBytesToRead = extraBytesToRead+1; - do { - ch += *source++; - --tmpBytesToRead; - if (tmpBytesToRead) ch <<= 6; - } while (tmpBytesToRead > 0); - } - In UTF-8 writing code, the switches on "bytesToWrite" are - similarly unrolled loops. - - --------------------------------------------------------------------- */ diff --git a/src/linenoise/ConvertUTF.h b/src/linenoise/ConvertUTF.h deleted file mode 100755 index 8a296235dcd9..000000000000 --- a/src/linenoise/ConvertUTF.h +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Copyright 2001-2004 Unicode, Inc. - * - * Disclaimer - * - * This source code is provided as is by Unicode, Inc. No claims are - * made as to fitness for any particular purpose. No warranties of any - * kind are expressed or implied. The recipient agrees to determine - * applicability of information provided. If this file has been - * purchased on magnetic or optical media from Unicode, Inc., the - * sole remedy for any claim will be exchange of defective media - * within 90 days of receipt. - * - * Limitations on Rights to Redistribute This Code - * - * Unicode, Inc. hereby grants the right to freely use the information - * supplied in this file in the creation of products supporting the - * Unicode Standard, and to make copies of this file in any form - * for internal or external distribution as long as this notice - * remains attached. - */ - -/* --------------------------------------------------------------------- - - Conversions between UTF32, UTF-16, and UTF-8. Header file. - - Several funtions are included here, forming a complete set of - conversions between the three formats. UTF-7 is not included - here, but is handled in a separate source file. - - Each of these routines takes pointers to input buffers and output - buffers. The input buffers are const. - - Each routine converts the text between *sourceStart and sourceEnd, - putting the result into the buffer between *targetStart and - targetEnd. Note: the end pointers are *after* the last item: e.g. - *(sourceEnd - 1) is the last item. - - The return result indicates whether the conversion was successful, - and if not, whether the problem was in the source or target buffers. - (Only the first encountered problem is indicated.) - - After the conversion, *sourceStart and *targetStart are both - updated to point to the end of last text successfully converted in - the respective buffers. - - Input parameters: - sourceStart - pointer to a pointer to the source buffer. - The contents of this are modified on return so that - it points at the next thing to be converted. - targetStart - similarly, pointer to pointer to the target buffer. - sourceEnd, targetEnd - respectively pointers to the ends of the - two buffers, for overflow checking only. - - These conversion functions take a ConversionFlags argument. When this - flag is set to strict, both irregular sequences and isolated surrogates - will cause an error. When the flag is set to lenient, both irregular - sequences and isolated surrogates are converted. - - Whether the flag is strict or lenient, all illegal sequences will cause - an error return. This includes sequences such as: <F4 90 80 80>, <C0 80>, - or <A0> in UTF-8, and values above 0x10FFFF in UTF-32. Conformant code - must check for illegal sequences. - - When the flag is set to lenient, characters over 0x10FFFF are converted - to the replacement character; otherwise (when the flag is set to strict) - they constitute an error. - - Output parameters: - The value "sourceIllegal" is returned from some routines if the input - sequence is malformed. When "sourceIllegal" is returned, the source - value will point to the illegal value that caused the problem. E.g., - in UTF-8 when a sequence is malformed, it points to the start of the - malformed sequence. - - Author: Mark E. Davis, 1994. - Rev History: Rick McGowan, fixes & updates May 2001. - Fixes & updates, Sept 2001. - ------------------------------------------------------------------------- */ - -/* --------------------------------------------------------------------- - The following 4 definitions are compiler-specific. - The C standard does not guarantee that wchar_t has at least - 16 bits, so wchar_t is no less portable than unsigned short! - All should be unsigned values to avoid sign extension during - bit mask & shift operations. ------------------------------------------------------------------------- */ - -#if 0 -typedef unsigned long UTF32; /* at least 32 bits */ -typedef unsigned short UTF16; /* at least 16 bits */ -typedef unsigned char UTF8; /* typically 8 bits */ -#endif - -#include <stdint.h> -#include <string> - -namespace linenoise_ng { - -typedef uint32_t UTF32; -typedef uint16_t UTF16; -typedef uint8_t UTF8; -typedef unsigned char Boolean; /* 0 or 1 */ - -/* Some fundamental constants */ -#define UNI_REPLACEMENT_CHAR (UTF32)0x0000FFFD -#define UNI_MAX_BMP (UTF32)0x0000FFFF -#define UNI_MAX_UTF16 (UTF32)0x0010FFFF -#define UNI_MAX_UTF32 (UTF32)0x7FFFFFFF -#define UNI_MAX_LEGAL_UTF32 (UTF32)0x0010FFFF - -typedef enum { - conversionOK, /* conversion successful */ - sourceExhausted, /* partial character in source, but hit end */ - targetExhausted, /* insuff. room in target for conversion */ - sourceIllegal /* source sequence is illegal/malformed */ -} ConversionResult; - -typedef enum { - strictConversion = 0, - lenientConversion -} ConversionFlags; - -// /* This is for C++ and does no harm in C */ -// #ifdef __cplusplus -// extern "C" { -// #endif - -ConversionResult ConvertUTF8toUTF16 ( - const UTF8** sourceStart, const UTF8* sourceEnd, - UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags); - -ConversionResult ConvertUTF16toUTF8 ( - const UTF16** sourceStart, const UTF16* sourceEnd, - UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags); - -ConversionResult ConvertUTF8toUTF32 ( - const UTF8** sourceStart, const UTF8* sourceEnd, - UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags); - -ConversionResult ConvertUTF32toUTF8 ( - const UTF32** sourceStart, const UTF32* sourceEnd, - UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags); - -ConversionResult ConvertUTF16toUTF32 ( - const UTF16** sourceStart, const UTF16* sourceEnd, - UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags); - -ConversionResult ConvertUTF32toUTF16 ( - const UTF32** sourceStart, const UTF32* sourceEnd, - char16_t** targetStart, char16_t* targetEnd, ConversionFlags flags); - -Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd); - -// #ifdef __cplusplus -// } -// #endif - -} - -/* --------------------------------------------------------------------- */ diff --git a/src/linenoise/LICENSE b/src/linenoise/LICENSE deleted file mode 100644 index b7c58c445860..000000000000 --- a/src/linenoise/LICENSE +++ /dev/null @@ -1,66 +0,0 @@ -linenoise.cpp -============= - -Copyright (c) 2010, Salvatore Sanfilippo <antirez at gmail dot com> -Copyright (c) 2010, Pieter Noordhuis <pcnoordhuis at gmail dot com> - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of Redis nor the names of its contributors may be used - to endorse or promote products derived from this software without - specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. - - -wcwidth.cpp -=========== - -Markus Kuhn -- 2007-05-26 (Unicode 5.0) - -Permission to use, copy, modify, and distribute this software -for any purpose and without fee is hereby granted. The author -disclaims all warranties with regard to this software. - - - -ConvertUTF.cpp -============== - -Copyright 2001-2004 Unicode, Inc. - -Disclaimer - -This source code is provided as is by Unicode, Inc. No claims are -made as to fitness for any particular purpose. No warranties of any -kind are expressed or implied. The recipient agrees to determine -applicability of information provided. If this file has been -purchased on magnetic or optical media from Unicode, Inc., the -sole remedy for any claim will be exchange of defective media -within 90 days of receipt. - -Limitations on Rights to Redistribute This Code - -Unicode, Inc. hereby grants the right to freely use the information -supplied in this file in the creation of products supporting the -Unicode Standard, and to make copies of this file in any form -for internal or external distribution as long as this notice -remains attached. diff --git a/src/linenoise/linenoise.cpp b/src/linenoise/linenoise.cpp deleted file mode 100644 index c57505d2fa97..000000000000 --- a/src/linenoise/linenoise.cpp +++ /dev/null @@ -1,3450 +0,0 @@ -/* linenoise.c -- guerrilla line editing library against the idea that a - * line editing lib needs to be 20,000 lines of C code. - * - * Copyright (c) 2010, Salvatore Sanfilippo <antirez at gmail dot com> - * Copyright (c) 2010, Pieter Noordhuis <pcnoordhuis at gmail dot com> - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Redis nor the names of its contributors may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * - * line editing lib needs to be 20,000 lines of C code. - * - * You can find the latest source code at: - * - * http://github.com/antirez/linenoise - * - * Does a number of crazy assumptions that happen to be true in 99.9999% of - * the 2010 UNIX computers around. - * - * References: - * - http://invisible-island.net/xterm/ctlseqs/ctlseqs.html - * - http://www.3waylabs.com/nw/WWW/products/wizcon/vt220.html - * - * Todo list: - * - Switch to gets() if $TERM is something we can't support. - * - Filter bogus Ctrl+<char> combinations. - * - Win32 support - * - * Bloat: - * - Completion? - * - History search like Ctrl+r in readline? - * - * List of escape sequences used by this program, we do everything just - * with three sequences. In order to be so cheap we may have some - * flickering effect with some slow terminal, but the lesser sequences - * the more compatible. - * - * CHA (Cursor Horizontal Absolute) - * Sequence: ESC [ n G - * Effect: moves cursor to column n (1 based) - * - * EL (Erase Line) - * Sequence: ESC [ n K - * Effect: if n is 0 or missing, clear from cursor to end of line - * Effect: if n is 1, clear from beginning of line to cursor - * Effect: if n is 2, clear entire line - * - * CUF (Cursor Forward) - * Sequence: ESC [ n C - * Effect: moves cursor forward of n chars - * - * The following are used to clear the screen: ESC [ H ESC [ 2 J - * This is actually composed of two sequences: - * - * cursorhome - * Sequence: ESC [ H - * Effect: moves the cursor to upper left corner - * - * ED2 (Clear entire screen) - * Sequence: ESC [ 2 J - * Effect: clear the whole screen - * - */ - -#ifdef _WIN32 - -#include <conio.h> -#include <windows.h> -#include <io.h> - -#if defined(_MSC_VER) && _MSC_VER < 1900 -#define snprintf _snprintf // Microsoft headers use underscores in some names -#endif - -#if !defined GNUC -#define strcasecmp _stricmp -#endif - -#define strdup _strdup -#define isatty _isatty -#define write _write -#define STDIN_FILENO 0 - -#else /* _WIN32 */ - -#include <signal.h> -#include <termios.h> -#include <unistd.h> -#include <stdlib.h> -#include <string.h> -#include <sys/types.h> -#include <sys/ioctl.h> -#include <cctype> -#include <wctype.h> - -#endif /* _WIN32 */ - -#include <stdio.h> -#include <errno.h> -#include <fcntl.h> - -#include "linenoise.h" -#include "ConvertUTF.h" - -#include <string> -#include <vector> -#include <memory> - -using std::string; -using std::vector; -using std::unique_ptr; -using namespace linenoise_ng; - -typedef unsigned char char8_t; - -static ConversionResult copyString8to32(char32_t* dst, size_t dstSize, - size_t& dstCount, const char* src) { - const UTF8* sourceStart = reinterpret_cast<const UTF8*>(src); - const UTF8* sourceEnd = sourceStart + strlen(src); - UTF32* targetStart = reinterpret_cast<UTF32*>(dst); - UTF32* targetEnd = targetStart + dstSize; - - ConversionResult res = ConvertUTF8toUTF32( - &sourceStart, sourceEnd, &targetStart, targetEnd, lenientConversion); - - if (res == conversionOK) { - dstCount = targetStart - reinterpret_cast<UTF32*>(dst); - - if (dstCount < dstSize) { - *targetStart = 0; - } - } - - return res; -} - -static ConversionResult copyString8to32(char32_t* dst, size_t dstSize, - size_t& dstCount, const char8_t* src) { - return copyString8to32(dst, dstSize, dstCount, - reinterpret_cast<const char*>(src)); -} - -static size_t strlen32(const char32_t* str) { - const char32_t* ptr = str; - - while (*ptr) { - ++ptr; - } - - return ptr - str; -} - -static size_t strlen8(const char8_t* str) { - return strlen(reinterpret_cast<const char*>(str)); -} - -static char8_t* strdup8(const char* src) { - return reinterpret_cast<char8_t*>(strdup(src)); -} - -#ifdef _WIN32 -static const int FOREGROUND_WHITE = - FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE; -static const int BACKGROUND_WHITE = - BACKGROUND_RED | BACKGROUND_GREEN | BACKGROUND_BLUE; -static const int INTENSITY = FOREGROUND_INTENSITY | BACKGROUND_INTENSITY; - -class WinAttributes { - public: - WinAttributes() { - CONSOLE_SCREEN_BUFFER_INFO info; - GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &info); - _defaultAttribute = info.wAttributes & INTENSITY; - _defaultColor = info.wAttributes & FOREGROUND_WHITE; - _defaultBackground = info.wAttributes & BACKGROUND_WHITE; - - _consoleAttribute = _defaultAttribute; - _consoleColor = _defaultColor | _defaultBackground; - } - - public: - int _defaultAttribute; - int _defaultColor; - int _defaultBackground; - - int _consoleAttribute; - int _consoleColor; -}; - -static WinAttributes WIN_ATTR; - -static void copyString32to16(char16_t* dst, size_t dstSize, size_t* dstCount, - const char32_t* src, size_t srcSize) { - const UTF32* sourceStart = reinterpret_cast<const UTF32*>(src); - const UTF32* sourceEnd = sourceStart + srcSize; - char16_t* targetStart = reinterpret_cast<char16_t*>(dst); - char16_t* targetEnd = targetStart + dstSize; - - ConversionResult res = ConvertUTF32toUTF16( - &sourceStart, sourceEnd, &targetStart, targetEnd, lenientConversion); - - if (res == conversionOK) { - *dstCount = targetStart - reinterpret_cast<char16_t*>(dst); - - if (*dstCount < dstSize) { - *targetStart = 0; - } - } -} -#endif - -static void copyString32to8(char* dst, size_t dstSize, size_t* dstCount, - const char32_t* src, size_t srcSize) { - const UTF32* sourceStart = reinterpret_cast<const UTF32*>(src); - const UTF32* sourceEnd = sourceStart + srcSize; - UTF8* targetStart = reinterpret_cast<UTF8*>(dst); - UTF8* targetEnd = targetStart + dstSize; - - ConversionResult res = ConvertUTF32toUTF8( - &sourceStart, sourceEnd, &targetStart, targetEnd, lenientConversion); - - if (res == conversionOK) { - *dstCount = targetStart - reinterpret_cast<UTF8*>(dst); - - if (*dstCount < dstSize) { - *targetStart = 0; - } - } -} - -static void copyString32to8(char* dst, size_t dstLen, const char32_t* src) { - size_t dstCount = 0; - copyString32to8(dst, dstLen, &dstCount, src, strlen32(src)); -} - -static void copyString32(char32_t* dst, const char32_t* src, size_t len) { - while (0 < len && *src) { - *dst++ = *src++; - --len; - } - - *dst = 0; -} - -static int strncmp32(const char32_t* left, const char32_t* right, size_t len) { - while (0 < len && *left) { - if (*left != *right) { - return *left - *right; - } - - ++left; - ++right; - --len; - } - - return 0; -} - -#ifdef _WIN32 -#include <iostream> - -static size_t OutputWin(char16_t* text16, char32_t* text32, size_t len32) { - size_t count16 = 0; - - copyString32to16(text16, len32, &count16, text32, len32); - WriteConsoleW(GetStdHandle(STD_OUTPUT_HANDLE), text16, - static_cast<DWORD>(count16), nullptr, nullptr); - - return count16; -} - -static char32_t* HandleEsc(char32_t* p, char32_t* end) { - if (*p == '[') { - int code = 0; - - for (++p; p < end; ++p) { - char32_t c = *p; - - if ('0' <= c && c <= '9') { - code = code * 10 + (c - '0'); - } else if (c == 'm' || c == ';') { - switch (code) { - case 0: - WIN_ATTR._consoleAttribute = WIN_ATTR._defaultAttribute; - WIN_ATTR._consoleColor = - WIN_ATTR._defaultColor | WIN_ATTR._defaultBackground; - break; - - case 1: // BOLD - case 5: // BLINK - WIN_ATTR._consoleAttribute = - (WIN_ATTR._defaultAttribute ^ FOREGROUND_INTENSITY) & INTENSITY; - break; - - case 30: - WIN_ATTR._consoleColor = BACKGROUND_WHITE; - break; - - case 31: - WIN_ATTR._consoleColor = - FOREGROUND_RED | WIN_ATTR._defaultBackground; - break; - - case 32: - WIN_ATTR._consoleColor = - FOREGROUND_GREEN | WIN_ATTR._defaultBackground; - break; - - case 33: - WIN_ATTR._consoleColor = - FOREGROUND_RED | FOREGROUND_GREEN | WIN_ATTR._defaultBackground; - break; - - case 34: - WIN_ATTR._consoleColor = - FOREGROUND_BLUE | WIN_ATTR._defaultBackground; - break; - - case 35: - WIN_ATTR._consoleColor = - FOREGROUND_BLUE | FOREGROUND_RED | WIN_ATTR._defaultBackground; - break; - - case 36: - WIN_ATTR._consoleColor = FOREGROUND_BLUE | FOREGROUND_GREEN | - WIN_ATTR._defaultBackground; - break; - - case 37: - WIN_ATTR._consoleColor = FOREGROUND_GREEN | FOREGROUND_RED | - FOREGROUND_BLUE | - WIN_ATTR._defaultBackground; - break; - } - - code = 0; - } - - if (*p == 'm') { - ++p; - break; - } - } - } else { - ++p; - } - - auto handle = GetStdHandle(STD_OUTPUT_HANDLE); - SetConsoleTextAttribute(handle, - WIN_ATTR._consoleAttribute | WIN_ATTR._consoleColor); - - return p; -} - -static size_t WinWrite32(char16_t* text16, char32_t* text32, size_t len32) { - char32_t* p = text32; - char32_t* q = p; - char32_t* e = text32 + len32; - size_t count16 = 0; - - while (p < e) { - if (*p == 27) { - if (q < p) { - count16 += OutputWin(text16, q, p - q); - } - - q = p = HandleEsc(p + 1, e); - } else { - ++p; - } - } - - if (q < p) { - count16 += OutputWin(text16, q, p - q); - } - - return count16; -} -#endif - -static int write32(int fd, char32_t* text32, int len32) { -#ifdef _WIN32 - if (isatty(fd)) { - size_t len16 = 2 * len32 + 1; - unique_ptr<char16_t[]> text16(new char16_t[len16]); - size_t count16 = WinWrite32(text16.get(), text32, len32); - - return static_cast<int>(count16); - } else { - size_t len8 = 4 * len32 + 1; - unique_ptr<char[]> text8(new char[len8]); - size_t count8 = 0; - - copyString32to8(text8.get(), len8, &count8, text32, len32); - - return write(fd, text8.get(), static_cast<unsigned int>(count8)); - } -#else - size_t len8 = 4 * len32 + 1; - unique_ptr<char[]> text8(new char[len8]); - size_t count8 = 0; - - copyString32to8(text8.get(), len8, &count8, text32, len32); - - return write(fd, text8.get(), count8); -#endif -} - -class Utf32String { - public: - Utf32String() : _length(0), _data(nullptr) { - // note: parens intentional, _data must be properly initialized - _data = new char32_t[1](); - } - - explicit Utf32String(const char* src) : _length(0), _data(nullptr) { - size_t len = strlen(src); - // note: parens intentional, _data must be properly initialized - _data = new char32_t[len + 1](); - copyString8to32(_data, len + 1, _length, src); - } - - explicit Utf32String(const char8_t* src) : _length(0), _data(nullptr) { - size_t len = strlen(reinterpret_cast<const char*>(src)); - // note: parens intentional, _data must be properly initialized - _data = new char32_t[len + 1](); - copyString8to32(_data, len + 1, _length, src); - } - - explicit Utf32String(const char32_t* src) : _length(0), _data(nullptr) { - for (_length = 0; src[_length] != 0; ++_length) { - } - - // note: parens intentional, _data must be properly initialized - _data = new char32_t[_length + 1](); - memcpy(_data, src, _length * sizeof(char32_t)); - } - - explicit Utf32String(const char32_t* src, int len) : _length(len), _data(nullptr) { - // note: parens intentional, _data must be properly initialized - _data = new char32_t[len + 1](); - memcpy(_data, src, len * sizeof(char32_t)); - } - - explicit Utf32String(int len) : _length(0), _data(nullptr) { - // note: parens intentional, _data must be properly initialized - _data = new char32_t[len](); - } - - explicit Utf32String(const Utf32String& that) : _length(that._length), _data(nullptr) { - // note: parens intentional, _data must be properly initialized - _data = new char32_t[_length + 1](); - memcpy(_data, that._data, sizeof(char32_t) * _length); - } - - Utf32String& operator=(const Utf32String& that) { - if (this != &that) { - delete[] _data; - _data = new char32_t[that._length](); - _length = that._length; - memcpy(_data, that._data, sizeof(char32_t) * _length); - } - - return *this; - } - - ~Utf32String() { delete[] _data; } - - public: - char32_t* get() const { return _data; } - - size_t length() const { return _length; } - - size_t chars() const { return _length; } - - void initFromBuffer() { - for (_length = 0; _data[_length] != 0; ++_length) { - } - } - - const char32_t& operator[](size_t pos) const { return _data[pos]; } - - char32_t& operator[](size_t pos) { return _data[pos]; } - - private: - size_t _length; - char32_t* _data; -}; - -class Utf8String { - Utf8String(const Utf8String&) = delete; - Utf8String& operator=(const Utf8String&) = delete; - - public: - explicit Utf8String(const Utf32String& src) { - size_t len = src.length() * 4 + 1; - _data = new char[len]; - copyString32to8(_data, len, src.get()); - } - - ~Utf8String() { delete[] _data; } - - public: - char* get() const { return _data; } - - private: - char* _data; -}; - -struct linenoiseCompletions { - vector<Utf32String> completionStrings; -}; - -#define LINENOISE_DEFAULT_HISTORY_MAX_LEN 100 -#define LINENOISE_MAX_LINE 4096 - -// make control-characters more readable -#define ctrlChar(upperCaseASCII) (upperCaseASCII - 0x40) - -/** - * Recompute widths of all characters in a char32_t buffer - * @param text input buffer of Unicode characters - * @param widths output buffer of character widths - * @param charCount number of characters in buffer - */ -namespace linenoise_ng { -int mk_wcwidth(char32_t ucs); -} - -static void recomputeCharacterWidths(const char32_t* text, char* widths, - int charCount) { - for (int i = 0; i < charCount; ++i) { - widths[i] = mk_wcwidth(text[i]); - } -} - -/** - * Calculate a new screen position given a starting position, screen width and - * character count - * @param x initial x position (zero-based) - * @param y initial y position (zero-based) - * @param screenColumns screen column count - * @param charCount character positions to advance - * @param xOut returned x position (zero-based) - * @param yOut returned y position (zero-based) - */ -static void calculateScreenPosition(int x, int y, int screenColumns, - int charCount, int& xOut, int& yOut) { - xOut = x; - yOut = y; - int charsRemaining = charCount; - while (charsRemaining > 0) { - int charsThisRow = (x + charsRemaining < screenColumns) ? charsRemaining - : screenColumns - x; - xOut = x + charsThisRow; - yOut = y; - charsRemaining -= charsThisRow; - x = 0; - ++y; - } - if (xOut == screenColumns) { // we have to special-case line wrap - xOut = 0; - ++yOut; - } -} - -/** - * Calculate a column width using mk_wcswidth() - * @param buf32 text to calculate - * @param len length of text to calculate - */ -namespace linenoise_ng { -int mk_wcswidth(const char32_t* pwcs, size_t n); -} - -static int calculateColumnPosition(char32_t* buf32, int len) { - int width = mk_wcswidth(reinterpret_cast<const char32_t*>(buf32), len); - if (width == -1) - return len; - else - return width; -} - -static bool isControlChar(char32_t testChar) { - return (testChar < ' ') || // C0 controls - (testChar >= 0x7F && testChar <= 0x9F); // DEL and C1 controls -} - -struct PromptBase { // a convenience struct for grouping prompt info - Utf32String promptText; // our copy of the prompt text, edited - char* promptCharWidths; // character widths from mk_wcwidth() - int promptChars; // chars in promptText - int promptBytes; // bytes in promptText - int promptExtraLines; // extra lines (beyond 1) occupied by prompt - int promptIndentation; // column offset to end of prompt - int promptLastLinePosition; // index into promptText where last line begins - int promptPreviousInputLen; // promptChars of previous input line, for - // clearing - int promptCursorRowOffset; // where the cursor is relative to the start of - // the prompt - int promptScreenColumns; // width of screen in columns - int promptPreviousLen; // help erasing - int promptErrorCode; // error code (invalid UTF-8) or zero - - PromptBase() : promptPreviousInputLen(0) {} - - bool write() { - if (write32(1, promptText.get(), promptBytes) == -1) return false; - - return true; - } -}; - -struct PromptInfo : public PromptBase { - PromptInfo(const char* textPtr, int columns) { - promptExtraLines = 0; - promptLastLinePosition = 0; - promptPreviousLen = 0; - promptScreenColumns = columns; - Utf32String tempUnicode(textPtr); - - // strip control characters from the prompt -- we do allow newline - char32_t* pIn = tempUnicode.get(); - char32_t* pOut = pIn; - - int len = 0; - int x = 0; - - bool const strip = (isatty(1) == 0); - - while (*pIn) { - char32_t c = *pIn; - if ('\n' == c || !isControlChar(c)) { - *pOut = c; - ++pOut; - ++pIn; - ++len; - if ('\n' == c || ++x >= promptScreenColumns) { - x = 0; - ++promptExtraLines; - promptLastLinePosition = len; - } - } else if (c == '\x1b') { - if (strip) { - // jump over control chars - ++pIn; - if (*pIn == '[') { - ++pIn; - while (*pIn && ((*pIn == ';') || ((*pIn >= '0' && *pIn <= '9')))) { - ++pIn; - } - if (*pIn == 'm') { - ++pIn; - } - } - } else { - // copy control chars - *pOut = *pIn; - ++pOut; - ++pIn; - if (*pIn == '[') { - *pOut = *pIn; - ++pOut; - ++pIn; - while (*pIn && ((*pIn == ';') || ((*pIn >= '0' && *pIn <= '9')))) { - *pOut = *pIn; - ++pOut; - ++pIn; - } - if (*pIn == 'm') { - *pOut = *pIn; - ++pOut; - ++pIn; - } - } - } - } else { - ++pIn; - } - } - *pOut = 0; - promptChars = len; - promptBytes = static_cast<int>(pOut - tempUnicode.get()); - promptText = tempUnicode; - - promptIndentation = len - promptLastLinePosition; - promptCursorRowOffset = promptExtraLines; - } -}; - -// Used with DynamicPrompt (history search) -// -static const Utf32String forwardSearchBasePrompt("(i-search)`"); -static const Utf32String reverseSearchBasePrompt("(reverse-i-search)`"); -static const Utf32String endSearchBasePrompt("': "); -static Utf32String - previousSearchText; // remembered across invocations of linenoise() - -// changing prompt for "(reverse-i-search)`text':" etc. -// -struct DynamicPrompt : public PromptBase { - Utf32String searchText; // text we are searching for - char* searchCharWidths; // character widths from mk_wcwidth() - int searchTextLen; // chars in searchText - int direction; // current search direction, 1=forward, -1=reverse - - DynamicPrompt(PromptBase& pi, int initialDirection) - : searchTextLen(0), direction(initialDirection) { - promptScreenColumns = pi.promptScreenColumns; - promptCursorRowOffset = 0; - Utf32String emptyString(1); - searchText = emptyString; - const Utf32String* basePrompt = - (direction > 0) ? &forwardSearchBasePrompt : &reverseSearchBasePrompt; - size_t promptStartLength = basePrompt->length(); - promptChars = - static_cast<int>(promptStartLength + endSearchBasePrompt.length()); - promptBytes = promptChars; - promptLastLinePosition = promptChars; // TODO fix this, we are asssuming - // that the history prompt won't wrap - // (!) - promptPreviousLen = promptChars; - Utf32String tempUnicode(promptChars + 1); - memcpy(tempUnicode.get(), basePrompt->get(), - sizeof(char32_t) * promptStartLength); - memcpy(&tempUnicode[promptStartLength], endSearchBasePrompt.get(), - sizeof(char32_t) * (endSearchBasePrompt.length() + 1)); - tempUnicode.initFromBuffer(); - promptText = tempUnicode; - calculateScreenPosition(0, 0, pi.promptScreenColumns, promptChars, - promptIndentation, promptExtraLines); - } - - void updateSearchPrompt(void) { - const Utf32String* basePrompt = - (direction > 0) ? &forwardSearchBasePrompt : &reverseSearchBasePrompt; - size_t promptStartLength = basePrompt->length(); - promptChars = static_cast<int>(promptStartLength + searchTextLen + - endSearchBasePrompt.length()); - promptBytes = promptChars; - Utf32String tempUnicode(promptChars + 1); - memcpy(tempUnicode.get(), basePrompt->get(), - sizeof(char32_t) * promptStartLength); - memcpy(&tempUnicode[promptStartLength], searchText.get(), - sizeof(char32_t) * searchTextLen); - size_t endIndex = promptStartLength + searchTextLen; - memcpy(&tempUnicode[endIndex], endSearchBasePrompt.get(), - sizeof(char32_t) * (endSearchBasePrompt.length() + 1)); - tempUnicode.initFromBuffer(); - promptText = tempUnicode; - } - - void updateSearchText(const char32_t* textPtr) { - Utf32String tempUnicode(textPtr); - searchTextLen = static_cast<int>(tempUnicode.chars()); - searchText = tempUnicode; - updateSearchPrompt(); - } -}; - -class KillRing { - static const int capacity = 10; - int size; - int index; - char indexToSlot[10]; - vector<Utf32String> theRing; - - public: - enum action { actionOther, actionKill, actionYank }; - action lastAction; - size_t lastYankSize; - - KillRing() : size(0), index(0), lastAction(actionOther) { - theRing.reserve(capacity); - } - - void kill(const char32_t* text, int textLen, bool forward) { - if (textLen == 0) { - return; - } - Utf32String killedText(text, textLen); - if (lastAction == actionKill && size > 0) { - int slot = indexToSlot[0]; - int currentLen = static_cast<int>(theRing[slot].length()); - int resultLen = currentLen + textLen; - Utf32String temp(resultLen + 1); - if (forward) { - memcpy(temp.get(), theRing[slot].get(), currentLen * sizeof(char32_t)); - memcpy(&temp[currentLen], killedText.get(), textLen * sizeof(char32_t)); - } else { - memcpy(temp.get(), killedText.get(), textLen * sizeof(char32_t)); - memcpy(&temp[textLen], theRing[slot].get(), - currentLen * sizeof(char32_t)); - } - temp[resultLen] = 0; - temp.initFromBuffer(); - theRing[slot] = temp; - } else { - if (size < capacity) { - if (size > 0) { - memmove(&indexToSlot[1], &indexToSlot[0], size); - } - indexToSlot[0] = size; - size++; - theRing.push_back(killedText); - } else { - int slot = indexToSlot[capacity - 1]; - theRing[slot] = killedText; - memmove(&indexToSlot[1], &indexToSlot[0], capacity - 1); - indexToSlot[0] = slot; - } - index = 0; - } - } - - Utf32String* yank() { return (size > 0) ? &theRing[indexToSlot[index]] : 0; } - - Utf32String* yankPop() { - if (size == 0) { - return 0; - } - ++index; - if (index == size) { - index = 0; - } - return &theRing[indexToSlot[index]]; - } -}; - -class InputBuffer { - char32_t* buf32; // input buffer - char* charWidths; // character widths from mk_wcwidth() - int buflen; // buffer size in characters - int len; // length of text in input buffer - int pos; // character position in buffer ( 0 <= pos <= len ) - - void clearScreen(PromptBase& pi); - int incrementalHistorySearch(PromptBase& pi, int startChar); - int completeLine(PromptBase& pi); - void refreshLine(PromptBase& pi); - - public: - InputBuffer(char32_t* buffer, char* widthArray, int bufferLen) - : buf32(buffer), - charWidths(widthArray), - buflen(bufferLen - 1), - len(0), - pos(0) { - buf32[0] = 0; - } - void preloadBuffer(const char* preloadText) { - size_t ucharCount = 0; - copyString8to32(buf32, buflen + 1, ucharCount, preloadText); - recomputeCharacterWidths(buf32, charWidths, static_cast<int>(ucharCount)); - len = static_cast<int>(ucharCount); - pos = static_cast<int>(ucharCount); - } - int getInputLine(PromptBase& pi); - int length(void) const { return len; } -}; - -// Special codes for keyboard input: -// -// Between Windows and the various Linux "terminal" programs, there is some -// pretty diverse behavior in the "scan codes" and escape sequences we are -// presented with. So ... we'll translate them all into our own pidgin -// pseudocode, trying to stay out of the way of UTF-8 and international -// characters. Here's the general plan. -// -// "User input keystrokes" (key chords, whatever) will be encoded as a single -// value. -// The low 21 bits are reserved for Unicode characters. Popular function-type -// keys -// get their own codes in the range 0x10200000 to (if needed) 0x1FE00000, -// currently -// just arrow keys, Home, End and Delete. Keypresses with Ctrl get ORed with -// 0x20000000, with Alt get ORed with 0x40000000. So, Ctrl+Alt+Home is encoded -// as 0x20000000 + 0x40000000 + 0x10A00000 == 0x70A00000. To keep things -// complicated, -// the Alt key is equivalent to prefixing the keystroke with ESC, so ESC -// followed by -// D is treated the same as Alt + D ... we'll just use Emacs terminology and -// call -// this "Meta". So, we will encode both ESC followed by D and Alt held down -// while D -// is pressed the same, as Meta-D, encoded as 0x40000064. -// -// Here are the definitions of our component constants: -// -// Maximum unsigned 32-bit value = 0xFFFFFFFF; // For reference, max 32-bit -// value -// Highest allocated Unicode char = 0x001FFFFF; // For reference, max -// Unicode value -static const int META = 0x40000000; // Meta key combination -static const int CTRL = 0x20000000; // Ctrl key combination -// static const int SPECIAL_KEY = 0x10000000; // Common bit for all special -// keys -static const int UP_ARROW_KEY = 0x10200000; // Special keys -static const int DOWN_ARROW_KEY = 0x10400000; -static const int RIGHT_ARROW_KEY = 0x10600000; -static const int LEFT_ARROW_KEY = 0x10800000; -static const int HOME_KEY = 0x10A00000; -static const int END_KEY = 0x10C00000; -static const int DELETE_KEY = 0x10E00000; -static const int PAGE_UP_KEY = 0x11000000; -static const int PAGE_DOWN_KEY = 0x11200000; - -static const char* unsupported_term[] = {"dumb", "cons25", "emacs", NULL}; -static linenoiseCompletionCallback* completionCallback = NULL; - -#ifdef _WIN32 -static HANDLE console_in, console_out; -static DWORD oldMode; -static WORD oldDisplayAttribute; -#else -static struct termios orig_termios; /* in order to restore at exit */ -#endif - -static KillRing killRing; - -static int rawmode = 0; /* for atexit() function to check if restore is needed*/ -static int atexit_registered = 0; /* register atexit just 1 time */ -static int historyMaxLen = LINENOISE_DEFAULT_HISTORY_MAX_LEN; -static int historyLen = 0; -static int historyIndex = 0; -static char8_t** history = NULL; - -// used to emulate Windows command prompt on down-arrow after a recall -// we use -2 as our "not set" value because we add 1 to the previous index on -// down-arrow, -// and zero is a valid index (so -1 is a valid "previous index") -static int historyPreviousIndex = -2; -static bool historyRecallMostRecent = false; - -static void linenoiseAtExit(void); - -static bool isUnsupportedTerm(void) { - char* term = getenv("TERM"); - if (term == NULL) return false; - for (int j = 0; unsupported_term[j]; ++j) - if (!strcasecmp(term, unsupported_term[j])) { - return true; - } - return false; -} - -static void beep() { - fprintf(stderr, "\x7"); // ctrl-G == bell/beep - fflush(stderr); -} - -void linenoiseHistoryFree(void) { - if (history) { - for (int j = 0; j < historyLen; ++j) free(history[j]); - historyLen = 0; - free(history); - history = 0; - } -} - -static int enableRawMode(void) { -#ifdef _WIN32 - if (!console_in) { - console_in = GetStdHandle(STD_INPUT_HANDLE); - console_out = GetStdHandle(STD_OUTPUT_HANDLE); - - GetConsoleMode(console_in, &oldMode); - SetConsoleMode(console_in, oldMode & - ~(ENABLE_LINE_INPUT | ENABLE_ECHO_INPUT | - ENABLE_PROCESSED_INPUT)); - } - return 0; -#else - struct termios raw; - - if (!isatty(STDIN_FILENO)) goto fatal; - if (!atexit_registered) { - atexit(linenoiseAtExit); - atexit_registered = 1; - } - if (tcgetattr(0, &orig_termios) == -1) goto fatal; - - raw = orig_termios; /* modify the original mode */ - /* input modes: no break, no CR to NL, no parity check, no strip char, - * no start/stop output control. */ - raw.c_iflag &= ~(BRKINT | ICRNL | INPCK | ISTRIP | IXON); - /* output modes - disable post processing */ - // this is wrong, we don't want raw output, it turns newlines into straight - // linefeeds - // raw.c_oflag &= ~(OPOST); - /* control modes - set 8 bit chars */ - raw.c_cflag |= (CS8); - /* local modes - echoing off, canonical off, no extended functions, - * no signal chars (^Z,^C) */ - raw.c_lflag &= ~(ECHO | ICANON | IEXTEN | ISIG); - /* control chars - set return condition: min number of bytes and timer. - * We want read to return every single byte, without timeout. */ - raw.c_cc[VMIN] = 1; - raw.c_cc[VTIME] = 0; /* 1 byte, no timer */ - - /* put terminal in raw mode after flushing */ - if (tcsetattr(0, TCSADRAIN, &raw) < 0) goto fatal; - rawmode = 1; - return 0; - -fatal: - errno = ENOTTY; - return -1; -#endif -} - -static void disableRawMode(void) { -#ifdef _WIN32 - SetConsoleMode(console_in, oldMode); - console_in = 0; - console_out = 0; -#else - if (rawmode && tcsetattr(0, TCSADRAIN, &orig_termios) != -1) rawmode = 0; -#endif -} - -// At exit we'll try to fix the terminal to the initial conditions -static void linenoiseAtExit(void) { disableRawMode(); } - -static int getScreenColumns(void) { - int cols; -#ifdef _WIN32 - CONSOLE_SCREEN_BUFFER_INFO inf; - GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &inf); - cols = inf.dwSize.X; -#else - struct winsize ws; - cols = (ioctl(1, TIOCGWINSZ, &ws) == -1) ? 80 : ws.ws_col; -#endif - // cols is 0 in certain circumstances like inside debugger, which creates - // further issues - return (cols > 0) ? cols : 80; -} - -static int getScreenRows(void) { - int rows; -#ifdef _WIN32 - CONSOLE_SCREEN_BUFFER_INFO inf; - GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &inf); - rows = 1 + inf.srWindow.Bottom - inf.srWindow.Top; -#else - struct winsize ws; - rows = (ioctl(1, TIOCGWINSZ, &ws) == -1) ? 24 : ws.ws_row; -#endif - return (rows > 0) ? rows : 24; -} - -static void setDisplayAttribute(bool enhancedDisplay, bool error) { -#ifdef _WIN32 - if (enhancedDisplay) { - CONSOLE_SCREEN_BUFFER_INFO inf; - GetConsoleScreenBufferInfo(console_out, &inf); - oldDisplayAttribute = inf.wAttributes; - BYTE oldLowByte = oldDisplayAttribute & 0xFF; - BYTE newLowByte; - switch (oldLowByte) { - case 0x07: - // newLowByte = FOREGROUND_BLUE | FOREGROUND_INTENSITY; // too dim - // newLowByte = FOREGROUND_BLUE; // even dimmer - newLowByte = FOREGROUND_BLUE | - FOREGROUND_GREEN; // most similar to xterm appearance - break; - case 0x70: - newLowByte = BACKGROUND_BLUE | BACKGROUND_INTENSITY; - break; - default: - newLowByte = oldLowByte ^ 0xFF; // default to inverse video - break; - } - inf.wAttributes = (inf.wAttributes & 0xFF00) | newLowByte; - SetConsoleTextAttribute(console_out, inf.wAttributes); - } else { - SetConsoleTextAttribute(console_out, oldDisplayAttribute); - } -#else - if (enhancedDisplay) { - char const* p = (error ? "\x1b[1;31m" : "\x1b[1;34m"); - if (write(1, p, 7) == -1) - return; /* bright blue (visible with both B&W bg) */ - } else { - if (write(1, "\x1b[0m", 4) == -1) return; /* reset */ - } -#endif -} - -/** - * Display the dynamic incremental search prompt and the current user input - * line. - * @param pi PromptBase struct holding information about the prompt and our - * screen position - * @param buf32 input buffer to be displayed - * @param len count of characters in the buffer - * @param pos current cursor position within the buffer (0 <= pos <= len) - */ -static void dynamicRefresh(PromptBase& pi, char32_t* buf32, int len, int pos) { - // calculate the position of the end of the prompt - int xEndOfPrompt, yEndOfPrompt; - calculateScreenPosition(0, 0, pi.promptScreenColumns, pi.promptChars, - xEndOfPrompt, yEndOfPrompt); - pi.promptIndentation = xEndOfPrompt; - - // calculate the position of the end of the input line - int xEndOfInput, yEndOfInput; - calculateScreenPosition(xEndOfPrompt, yEndOfPrompt, pi.promptScreenColumns, - calculateColumnPosition(buf32, len), xEndOfInput, - yEndOfInput); - - // calculate the desired position of the cursor - int xCursorPos, yCursorPos; - calculateScreenPosition(xEndOfPrompt, yEndOfPrompt, pi.promptScreenColumns, - calculateColumnPosition(buf32, pos), xCursorPos, - yCursorPos); - -#ifdef _WIN32 - // position at the start of the prompt, clear to end of previous input - CONSOLE_SCREEN_BUFFER_INFO inf; - GetConsoleScreenBufferInfo(console_out, &inf); - inf.dwCursorPosition.X = 0; - inf.dwCursorPosition.Y -= pi.promptCursorRowOffset /*- pi.promptExtraLines*/; - SetConsoleCursorPosition(console_out, inf.dwCursorPosition); - DWORD count; - FillConsoleOutputCharacterA(console_out, ' ', - pi.promptPreviousLen + pi.promptPreviousInputLen, - inf.dwCursorPosition, &count); - pi.promptPreviousLen = pi.promptIndentation; - pi.promptPreviousInputLen = len; - - // display the prompt - if (!pi.write()) return; - - // display the input line - if (write32(1, buf32, len) == -1) return; - - // position the cursor - GetConsoleScreenBufferInfo(console_out, &inf); - inf.dwCursorPosition.X = xCursorPos; // 0-based on Win32 - inf.dwCursorPosition.Y -= yEndOfInput - yCursorPos; - SetConsoleCursorPosition(console_out, inf.dwCursorPosition); -#else // _WIN32 - char seq[64]; - int cursorRowMovement = pi.promptCursorRowOffset - pi.promptExtraLines; - if (cursorRowMovement > 0) { // move the cursor up as required - snprintf(seq, sizeof seq, "\x1b[%dA", cursorRowMovement); - if (write(1, seq, strlen(seq)) == -1) return; - } - // position at the start of the prompt, clear to end of screen - snprintf(seq, sizeof seq, "\x1b[1G\x1b[J"); // 1-based on VT100 - if (write(1, seq, strlen(seq)) == -1) return; - - // display the prompt - if (!pi.write()) return; - - // display the input line - if (write32(1, buf32, len) == -1) return; - - // we have to generate our own newline on line wrap - if (xEndOfInput == 0 && yEndOfInput > 0) - if (write(1, "\n", 1) == -1) return; - - // position the cursor - cursorRowMovement = yEndOfInput - yCursorPos; - if (cursorRowMovement > 0) { // move the cursor up as required - snprintf(seq, sizeof seq, "\x1b[%dA", cursorRowMovement); - if (write(1, seq, strlen(seq)) == -1) return; - } - // position the cursor within the line - snprintf(seq, sizeof seq, "\x1b[%dG", xCursorPos + 1); // 1-based on VT100 - if (write(1, seq, strlen(seq)) == -1) return; -#endif - - pi.promptCursorRowOffset = - pi.promptExtraLines + yCursorPos; // remember row for next pass -} - -/** - * Refresh the user's input line: the prompt is already onscreen and is not - * redrawn here - * @param pi PromptBase struct holding information about the prompt and our - * screen position - */ -void InputBuffer::refreshLine(PromptBase& pi) { - // check for a matching brace/bracket/paren, remember its position if found - int highlight = -1; - bool indicateError = false; - if (pos < len) { - /* this scans for a brace matching buf32[pos] to highlight */ - unsigned char part1, part2; - int scanDirection = 0; - if (strchr("}])", buf32[pos])) { - scanDirection = -1; /* backwards */ - if (buf32[pos] == '}') { - part1 = '}'; part2 = '{'; - } else if (buf32[pos] == ']') { - part1 = ']'; part2 = '['; - } else { - part1 = ')'; part2 = '('; - } - } - else if (strchr("{[(", buf32[pos])) { - scanDirection = 1; /* forwards */ - if (buf32[pos] == '{') { - //part1 = '{'; part2 = '}'; - part1 = '}'; part2 = '{'; - } else if (buf32[pos] == '[') { - //part1 = '['; part2 = ']'; - part1 = ']'; part2 = '['; - } else { - //part1 = '('; part2 = ')'; - part1 = ')'; part2 = '('; - } - } - - if (scanDirection) { - int unmatched = scanDirection; - int unmatchedOther = 0; - for (int i = pos + scanDirection; i >= 0 && i < len; i += scanDirection) { - /* TODO: the right thing when inside a string */ - if (strchr("}])", buf32[i])) { - if (buf32[i] == part1) { - --unmatched; - } else { - --unmatchedOther; - } - } else if (strchr("{[(", buf32[i])) { - if (buf32[i] == part2) { - ++unmatched; - } else { - ++unmatchedOther; - } - } -/* - if (strchr("}])", buf32[i])) - --unmatched; - else if (strchr("{[(", buf32[i])) - ++unmatched; -*/ - if (unmatched == 0) { - highlight = i; - indicateError = (unmatchedOther != 0); - break; - } - } - } - } - - // calculate the position of the end of the input line - int xEndOfInput, yEndOfInput; - calculateScreenPosition(pi.promptIndentation, 0, pi.promptScreenColumns, - calculateColumnPosition(buf32, len), xEndOfInput, - yEndOfInput); - - // calculate the desired position of the cursor - int xCursorPos, yCursorPos; - calculateScreenPosition(pi.promptIndentation, 0, pi.promptScreenColumns, - calculateColumnPosition(buf32, pos), xCursorPos, - yCursorPos); - -#ifdef _WIN32 - // position at the end of the prompt, clear to end of previous input - CONSOLE_SCREEN_BUFFER_INFO inf; - GetConsoleScreenBufferInfo(console_out, &inf); - inf.dwCursorPosition.X = pi.promptIndentation; // 0-based on Win32 - inf.dwCursorPosition.Y -= pi.promptCursorRowOffset - pi.promptExtraLines; - SetConsoleCursorPosition(console_out, inf.dwCursorPosition); - DWORD count; - if (len < pi.promptPreviousInputLen) - FillConsoleOutputCharacterA(console_out, ' ', pi.promptPreviousInputLen, - inf.dwCursorPosition, &count); - pi.promptPreviousInputLen = len; - - // display the input line - if (highlight == -1) { - if (write32(1, buf32, len) == -1) return; - } else { - if (write32(1, buf32, highlight) == -1) return; - setDisplayAttribute(true, indicateError); /* bright blue (visible with both B&W bg) */ - if (write32(1, &buf32[highlight], 1) == -1) return; - setDisplayAttribute(false, indicateError); - if (write32(1, buf32 + highlight + 1, len - highlight - 1) == -1) return; - } - - // position the cursor - GetConsoleScreenBufferInfo(console_out, &inf); - inf.dwCursorPosition.X = xCursorPos; // 0-based on Win32 - inf.dwCursorPosition.Y -= yEndOfInput - yCursorPos; - SetConsoleCursorPosition(console_out, inf.dwCursorPosition); -#else // _WIN32 - char seq[64]; - int cursorRowMovement = pi.promptCursorRowOffset - pi.promptExtraLines; - if (cursorRowMovement > 0) { // move the cursor up as required - snprintf(seq, sizeof seq, "\x1b[%dA", cursorRowMovement); - if (write(1, seq, strlen(seq)) == -1) return; - } - // position at the end of the prompt, clear to end of screen - snprintf(seq, sizeof seq, "\x1b[%dG\x1b[J", - pi.promptIndentation + 1); // 1-based on VT100 - if (write(1, seq, strlen(seq)) == -1) return; - - if (highlight == -1) { // write unhighlighted text - if (write32(1, buf32, len) == -1) return; - } else { // highlight the matching brace/bracket/parenthesis - if (write32(1, buf32, highlight) == -1) return; - setDisplayAttribute(true, indicateError); - if (write32(1, &buf32[highlight], 1) == -1) return; - setDisplayAttribute(false, indicateError); - if (write32(1, buf32 + highlight + 1, len - highlight - 1) == -1) return; - } - - // we have to generate our own newline on line wrap - if (xEndOfInput == 0 && yEndOfInput > 0) - if (write(1, "\n", 1) == -1) return; - - // position the cursor - cursorRowMovement = yEndOfInput - yCursorPos; - if (cursorRowMovement > 0) { // move the cursor up as required - snprintf(seq, sizeof seq, "\x1b[%dA", cursorRowMovement); - if (write(1, seq, strlen(seq)) == -1) return; - } - // position the cursor within the line - snprintf(seq, sizeof seq, "\x1b[%dG", xCursorPos + 1); // 1-based on VT100 - if (write(1, seq, strlen(seq)) == -1) return; -#endif - - pi.promptCursorRowOffset = - pi.promptExtraLines + yCursorPos; // remember row for next pass -} - -#ifndef _WIN32 - -/** - * Read a UTF-8 sequence from the non-Windows keyboard and return the Unicode - * (char32_t) character it - * encodes - * - * @return char32_t Unicode character - */ -static char32_t readUnicodeCharacter(void) { - static char8_t utf8String[5]; - static size_t utf8Count = 0; - while (true) { - char8_t c; - - /* Continue reading if interrupted by signal. */ - ssize_t nread; - do { - nread = read(0, &c, 1); - } while ((nread == -1) && (errno == EINTR)); - - if (nread <= 0) return 0; - if (c <= 0x7F) { // short circuit ASCII - utf8Count = 0; - return c; - } else if (utf8Count < sizeof(utf8String) - 1) { - utf8String[utf8Count++] = c; - utf8String[utf8Count] = 0; - char32_t unicodeChar[2]; - size_t ucharCount; - ConversionResult res = - copyString8to32(unicodeChar, 2, ucharCount, utf8String); - if (res == conversionOK && ucharCount) { - utf8Count = 0; - return unicodeChar[0]; - } - } else { - utf8Count = - 0; // this shouldn't happen: got four bytes but no UTF-8 character - } - } -} - -namespace EscapeSequenceProcessing { // move these out of global namespace - -// This chunk of code does parsing of the escape sequences sent by various Linux -// terminals. -// -// It handles arrow keys, Home, End and Delete keys by interpreting the -// sequences sent by -// gnome terminal, xterm, rxvt, konsole, aterm and yakuake including the Alt and -// Ctrl key -// combinations that are understood by linenoise. -// -// The parsing uses tables, a bunch of intermediate dispatch routines and a -// doDispatch -// loop that reads the tables and sends control to "deeper" routines to continue -// the -// parsing. The starting call to doDispatch( c, initialDispatch ) will -// eventually return -// either a character (with optional CTRL and META bits set), or -1 if parsing -// fails, or -// zero if an attempt to read from the keyboard fails. -// -// This is rather sloppy escape sequence processing, since we're not paying -// attention to what the -// actual TERM is set to and are processing all key sequences for all terminals, -// but it works with -// the most common keystrokes on the most common terminals. It's intricate, but -// the nested 'if' -// statements required to do it directly would be worse. This way has the -// advantage of allowing -// changes and extensions without having to touch a lot of code. - -// This is a typedef for the routine called by doDispatch(). It takes the -// current character -// as input, does any required processing including reading more characters and -// calling other -// dispatch routines, then eventually returns the final (possibly extended or -// special) character. -// -typedef char32_t (*CharacterDispatchRoutine)(char32_t); - -// This structure is used by doDispatch() to hold a list of characters to test -// for and -// a list of routines to call if the character matches. The dispatch routine -// list is one -// longer than the character list; the final entry is used if no character -// matches. -// -struct CharacterDispatch { - unsigned int len; // length of the chars list - const char* chars; // chars to test - CharacterDispatchRoutine* dispatch; // array of routines to call -}; - -// This dispatch routine is given a dispatch table and then farms work out to -// routines -// listed in the table based on the character it is called with. The dispatch -// routines can -// read more input characters to decide what should eventually be returned. -// Eventually, -// a called routine returns either a character or -1 to indicate parsing -// failure. -// -static char32_t doDispatch(char32_t c, CharacterDispatch& dispatchTable) { - for (unsigned int i = 0; i < dispatchTable.len; ++i) { - if (static_cast<unsigned char>(dispatchTable.chars[i]) == c) { - return dispatchTable.dispatch[i](c); - } - } - return dispatchTable.dispatch[dispatchTable.len](c); -} - -static char32_t thisKeyMetaCtrl = - 0; // holds pre-set Meta and/or Ctrl modifiers - -// Final dispatch routines -- return something -// -static char32_t normalKeyRoutine(char32_t c) { return thisKeyMetaCtrl | c; } -static char32_t upArrowKeyRoutine(char32_t) { - return thisKeyMetaCtrl | UP_ARROW_KEY; -} -static char32_t downArrowKeyRoutine(char32_t) { - return thisKeyMetaCtrl | DOWN_ARROW_KEY; -} -static char32_t rightArrowKeyRoutine(char32_t) { - return thisKeyMetaCtrl | RIGHT_ARROW_KEY; -} -static char32_t leftArrowKeyRoutine(char32_t) { - return thisKeyMetaCtrl | LEFT_ARROW_KEY; -} -static char32_t homeKeyRoutine(char32_t) { return thisKeyMetaCtrl | HOME_KEY; } -static char32_t endKeyRoutine(char32_t) { return thisKeyMetaCtrl | END_KEY; } -static char32_t pageUpKeyRoutine(char32_t) { - return thisKeyMetaCtrl | PAGE_UP_KEY; -} -static char32_t pageDownKeyRoutine(char32_t) { - return thisKeyMetaCtrl | PAGE_DOWN_KEY; -} -static char32_t deleteCharRoutine(char32_t) { - return thisKeyMetaCtrl | ctrlChar('H'); -} // key labeled Backspace -static char32_t deleteKeyRoutine(char32_t) { - return thisKeyMetaCtrl | DELETE_KEY; -} // key labeled Delete -static char32_t ctrlUpArrowKeyRoutine(char32_t) { - return thisKeyMetaCtrl | CTRL | UP_ARROW_KEY; -} -static char32_t ctrlDownArrowKeyRoutine(char32_t) { - return thisKeyMetaCtrl | CTRL | DOWN_ARROW_KEY; -} -static char32_t ctrlRightArrowKeyRoutine(char32_t) { - return thisKeyMetaCtrl | CTRL | RIGHT_ARROW_KEY; -} -static char32_t ctrlLeftArrowKeyRoutine(char32_t) { - return thisKeyMetaCtrl | CTRL | LEFT_ARROW_KEY; -} -static char32_t escFailureRoutine(char32_t) { - beep(); - return -1; -} - -// Handle ESC [ 1 ; 3 (or 5) <more stuff> escape sequences -// -static CharacterDispatchRoutine escLeftBracket1Semicolon3or5Routines[] = { - upArrowKeyRoutine, downArrowKeyRoutine, rightArrowKeyRoutine, - leftArrowKeyRoutine, escFailureRoutine}; -static CharacterDispatch escLeftBracket1Semicolon3or5Dispatch = { - 4, "ABCD", escLeftBracket1Semicolon3or5Routines}; - -// Handle ESC [ 1 ; <more stuff> escape sequences -// -static char32_t escLeftBracket1Semicolon3Routine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - thisKeyMetaCtrl |= META; - return doDispatch(c, escLeftBracket1Semicolon3or5Dispatch); -} -static char32_t escLeftBracket1Semicolon5Routine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - thisKeyMetaCtrl |= CTRL; - return doDispatch(c, escLeftBracket1Semicolon3or5Dispatch); -} -static CharacterDispatchRoutine escLeftBracket1SemicolonRoutines[] = { - escLeftBracket1Semicolon3Routine, escLeftBracket1Semicolon5Routine, - escFailureRoutine}; -static CharacterDispatch escLeftBracket1SemicolonDispatch = { - 2, "35", escLeftBracket1SemicolonRoutines}; - -// Handle ESC [ 1 <more stuff> escape sequences -// -static char32_t escLeftBracket1SemicolonRoutine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - return doDispatch(c, escLeftBracket1SemicolonDispatch); -} -static CharacterDispatchRoutine escLeftBracket1Routines[] = { - homeKeyRoutine, escLeftBracket1SemicolonRoutine, escFailureRoutine}; -static CharacterDispatch escLeftBracket1Dispatch = {2, "~;", - escLeftBracket1Routines}; - -// Handle ESC [ 3 <more stuff> escape sequences -// -static CharacterDispatchRoutine escLeftBracket3Routines[] = {deleteKeyRoutine, - escFailureRoutine}; -static CharacterDispatch escLeftBracket3Dispatch = {1, "~", - escLeftBracket3Routines}; - -// Handle ESC [ 4 <more stuff> escape sequences -// -static CharacterDispatchRoutine escLeftBracket4Routines[] = {endKeyRoutine, - escFailureRoutine}; -static CharacterDispatch escLeftBracket4Dispatch = {1, "~", - escLeftBracket4Routines}; - -// Handle ESC [ 5 <more stuff> escape sequences -// -static CharacterDispatchRoutine escLeftBracket5Routines[] = {pageUpKeyRoutine, - escFailureRoutine}; -static CharacterDispatch escLeftBracket5Dispatch = {1, "~", - escLeftBracket5Routines}; - -// Handle ESC [ 6 <more stuff> escape sequences -// -static CharacterDispatchRoutine escLeftBracket6Routines[] = {pageDownKeyRoutine, - escFailureRoutine}; -static CharacterDispatch escLeftBracket6Dispatch = {1, "~", - escLeftBracket6Routines}; - -// Handle ESC [ 7 <more stuff> escape sequences -// -static CharacterDispatchRoutine escLeftBracket7Routines[] = {homeKeyRoutine, - escFailureRoutine}; -static CharacterDispatch escLeftBracket7Dispatch = {1, "~", - escLeftBracket7Routines}; - -// Handle ESC [ 8 <more stuff> escape sequences -// -static CharacterDispatchRoutine escLeftBracket8Routines[] = {endKeyRoutine, - escFailureRoutine}; -static CharacterDispatch escLeftBracket8Dispatch = {1, "~", - escLeftBracket8Routines}; - -// Handle ESC [ <digit> escape sequences -// -static char32_t escLeftBracket0Routine(char32_t c) { - return escFailureRoutine(c); -} -static char32_t escLeftBracket1Routine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - return doDispatch(c, escLeftBracket1Dispatch); -} -static char32_t escLeftBracket2Routine(char32_t c) { - return escFailureRoutine(c); // Insert key, unused -} -static char32_t escLeftBracket3Routine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - return doDispatch(c, escLeftBracket3Dispatch); -} -static char32_t escLeftBracket4Routine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - return doDispatch(c, escLeftBracket4Dispatch); -} -static char32_t escLeftBracket5Routine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - return doDispatch(c, escLeftBracket5Dispatch); -} -static char32_t escLeftBracket6Routine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - return doDispatch(c, escLeftBracket6Dispatch); -} -static char32_t escLeftBracket7Routine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - return doDispatch(c, escLeftBracket7Dispatch); -} -static char32_t escLeftBracket8Routine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - return doDispatch(c, escLeftBracket8Dispatch); -} -static char32_t escLeftBracket9Routine(char32_t c) { - return escFailureRoutine(c); -} - -// Handle ESC [ <more stuff> escape sequences -// -static CharacterDispatchRoutine escLeftBracketRoutines[] = { - upArrowKeyRoutine, downArrowKeyRoutine, rightArrowKeyRoutine, - leftArrowKeyRoutine, homeKeyRoutine, endKeyRoutine, - escLeftBracket0Routine, escLeftBracket1Routine, escLeftBracket2Routine, - escLeftBracket3Routine, escLeftBracket4Routine, escLeftBracket5Routine, - escLeftBracket6Routine, escLeftBracket7Routine, escLeftBracket8Routine, - escLeftBracket9Routine, escFailureRoutine}; -static CharacterDispatch escLeftBracketDispatch = {16, "ABCDHF0123456789", - escLeftBracketRoutines}; - -// Handle ESC O <char> escape sequences -// -static CharacterDispatchRoutine escORoutines[] = { - upArrowKeyRoutine, downArrowKeyRoutine, rightArrowKeyRoutine, - leftArrowKeyRoutine, homeKeyRoutine, endKeyRoutine, - ctrlUpArrowKeyRoutine, ctrlDownArrowKeyRoutine, ctrlRightArrowKeyRoutine, - ctrlLeftArrowKeyRoutine, escFailureRoutine}; -static CharacterDispatch escODispatch = {10, "ABCDHFabcd", escORoutines}; - -// Initial ESC dispatch -- could be a Meta prefix or the start of an escape -// sequence -// -static char32_t escLeftBracketRoutine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - return doDispatch(c, escLeftBracketDispatch); -} -static char32_t escORoutine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - return doDispatch(c, escODispatch); -} -static char32_t setMetaRoutine(char32_t c); // need forward reference -static CharacterDispatchRoutine escRoutines[] = {escLeftBracketRoutine, - escORoutine, setMetaRoutine}; -static CharacterDispatch escDispatch = {2, "[O", escRoutines}; - -// Initial dispatch -- we are not in the middle of anything yet -// -static char32_t escRoutine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - return doDispatch(c, escDispatch); -} -static CharacterDispatchRoutine initialRoutines[] = { - escRoutine, deleteCharRoutine, normalKeyRoutine}; -static CharacterDispatch initialDispatch = {2, "\x1B\x7F", initialRoutines}; - -// Special handling for the ESC key because it does double duty -// -static char32_t setMetaRoutine(char32_t c) { - thisKeyMetaCtrl = META; - if (c == 0x1B) { // another ESC, stay in ESC processing mode - c = readUnicodeCharacter(); - if (c == 0) return 0; - return doDispatch(c, escDispatch); - } - return doDispatch(c, initialDispatch); -} - -} // namespace EscapeSequenceProcessing // move these out of global namespace - -#endif // #ifndef _WIN32 - -// linenoiseReadChar -- read a keystroke or keychord from the keyboard, and -// translate it -// into an encoded "keystroke". When convenient, extended keys are translated -// into their -// simpler Emacs keystrokes, so an unmodified "left arrow" becomes Ctrl-B. -// -// A return value of zero means "no input available", and a return value of -1 -// means "invalid key". -// -static char32_t linenoiseReadChar(void) { -#ifdef _WIN32 - - INPUT_RECORD rec; - DWORD count; - int modifierKeys = 0; - bool escSeen = false; - while (true) { - ReadConsoleInputW(console_in, &rec, 1, &count); -#if 0 // helper for debugging keystrokes, display info in the debug "Output" - // window in the debugger - { - if ( rec.EventType == KEY_EVENT ) { - //if ( rec.Event.KeyEvent.uChar.UnicodeChar ) { - char buf[1024]; - sprintf( - buf, - "Unicode character 0x%04X, repeat count %d, virtual keycode 0x%04X, " - "virtual scancode 0x%04X, key %s%s%s%s%s\n", - rec.Event.KeyEvent.uChar.UnicodeChar, - rec.Event.KeyEvent.wRepeatCount, - rec.Event.KeyEvent.wVirtualKeyCode, - rec.Event.KeyEvent.wVirtualScanCode, - rec.Event.KeyEvent.bKeyDown ? "down" : "up", - (rec.Event.KeyEvent.dwControlKeyState & LEFT_CTRL_PRESSED) ? - " L-Ctrl" : "", - (rec.Event.KeyEvent.dwControlKeyState & RIGHT_CTRL_PRESSED) ? - " R-Ctrl" : "", - (rec.Event.KeyEvent.dwControlKeyState & LEFT_ALT_PRESSED) ? - " L-Alt" : "", - (rec.Event.KeyEvent.dwControlKeyState & RIGHT_ALT_PRESSED) ? - " R-Alt" : "" - ); - OutputDebugStringA( buf ); - //} - } - } -#endif - if (rec.EventType != KEY_EVENT) { - continue; - } - // Windows provides for entry of characters that are not on your keyboard by - // sending the - // Unicode characters as a "key up" with virtual keycode 0x12 (VK_MENU == - // Alt key) ... - // accept these characters, otherwise only process characters on "key down" - if (!rec.Event.KeyEvent.bKeyDown && - rec.Event.KeyEvent.wVirtualKeyCode != VK_MENU) { - continue; - } - modifierKeys = 0; - // AltGr is encoded as ( LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED ), so don't - // treat this - // combination as either CTRL or META we just turn off those two bits, so it - // is still - // possible to combine CTRL and/or META with an AltGr key by using - // right-Ctrl and/or - // left-Alt - if ((rec.Event.KeyEvent.dwControlKeyState & - (LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED)) == - (LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED)) { - rec.Event.KeyEvent.dwControlKeyState &= - ~(LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED); - } - if (rec.Event.KeyEvent.dwControlKeyState & - (RIGHT_CTRL_PRESSED | LEFT_CTRL_PRESSED)) { - modifierKeys |= CTRL; - } - if (rec.Event.KeyEvent.dwControlKeyState & - (RIGHT_ALT_PRESSED | LEFT_ALT_PRESSED)) { - modifierKeys |= META; - } - if (escSeen) { - modifierKeys |= META; - } - if (rec.Event.KeyEvent.uChar.UnicodeChar == 0) { - switch (rec.Event.KeyEvent.wVirtualKeyCode) { - case VK_LEFT: - return modifierKeys | LEFT_ARROW_KEY; - case VK_RIGHT: - return modifierKeys | RIGHT_ARROW_KEY; - case VK_UP: - return modifierKeys | UP_ARROW_KEY; - case VK_DOWN: - return modifierKeys | DOWN_ARROW_KEY; - case VK_DELETE: - return modifierKeys | DELETE_KEY; - case VK_HOME: - return modifierKeys | HOME_KEY; - case VK_END: - return modifierKeys | END_KEY; - case VK_PRIOR: - return modifierKeys | PAGE_UP_KEY; - case VK_NEXT: - return modifierKeys | PAGE_DOWN_KEY; - default: - continue; // in raw mode, ReadConsoleInput shows shift, ctrl ... - } // ... ignore them - } else if (rec.Event.KeyEvent.uChar.UnicodeChar == - ctrlChar('[')) { // ESC, set flag for later - escSeen = true; - continue; - } else { - // we got a real character, return it - return modifierKeys | rec.Event.KeyEvent.uChar.UnicodeChar; - } - } - -#else - char32_t c; - c = readUnicodeCharacter(); - if (c == 0) return 0; - -// If _DEBUG_LINUX_KEYBOARD is set, then ctrl-^ puts us into a keyboard -// debugging mode -// where we print out decimal and decoded values for whatever the "terminal" -// program -// gives us on different keystrokes. Hit ctrl-C to exit this mode. -// -#define _DEBUG_LINUX_KEYBOARD -#if defined(_DEBUG_LINUX_KEYBOARD) - if (c == ctrlChar('^')) { // ctrl-^, special debug mode, prints all keys hit, - // ctrl-C to get out - printf( - "\nEntering keyboard debugging mode (on ctrl-^), press ctrl-C to exit " - "this mode\n"); - while (true) { - unsigned char keys[10]; - int ret = read(0, keys, 10); - - if (ret <= 0) { - printf("\nret: %d\n", ret); - } - for (int i = 0; i < ret; ++i) { - char32_t key = static_cast<char32_t>(keys[i]); - char* friendlyTextPtr; - char friendlyTextBuf[10]; - const char* prefixText = (key < 0x80) ? "" : "0x80+"; - char32_t keyCopy = (key < 0x80) ? key : key - 0x80; - if (keyCopy >= '!' && keyCopy <= '~') { // printable - friendlyTextBuf[0] = '\''; - friendlyTextBuf[1] = keyCopy; - friendlyTextBuf[2] = '\''; - friendlyTextBuf[3] = 0; - friendlyTextPtr = friendlyTextBuf; - } else if (keyCopy == ' ') { - friendlyTextPtr = const_cast<char*>("space"); - } else if (keyCopy == 27) { - friendlyTextPtr = const_cast<char*>("ESC"); - } else if (keyCopy == 0) { - friendlyTextPtr = const_cast<char*>("NUL"); - } else if (keyCopy == 127) { - friendlyTextPtr = const_cast<char*>("DEL"); - } else { - friendlyTextBuf[0] = '^'; - friendlyTextBuf[1] = keyCopy + 0x40; - friendlyTextBuf[2] = 0; - friendlyTextPtr = friendlyTextBuf; - } - printf("%d x%02X (%s%s) ", key, key, prefixText, friendlyTextPtr); - } - printf("\x1b[1G\n"); // go to first column of new line - - // drop out of this loop on ctrl-C - if (keys[0] == ctrlChar('C')) { - printf("Leaving keyboard debugging mode (on ctrl-C)\n"); - fflush(stdout); - return -2; - } - } - } -#endif // _DEBUG_LINUX_KEYBOARD - - EscapeSequenceProcessing::thisKeyMetaCtrl = - 0; // no modifiers yet at initialDispatch - return EscapeSequenceProcessing::doDispatch( - c, EscapeSequenceProcessing::initialDispatch); -#endif // #_WIN32 -} - -/** - * Free memory used in a recent command completion session - * - * @param lc pointer to a linenoiseCompletions struct - */ -static void freeCompletions(linenoiseCompletions* lc) { - lc->completionStrings.clear(); -} - -/** - * convert {CTRL + 'A'}, {CTRL + 'a'} and {CTRL + ctrlChar( 'A' )} into - * ctrlChar( 'A' ) - * leave META alone - * - * @param c character to clean up - * @return cleaned-up character - */ -static int cleanupCtrl(int c) { - if (c & CTRL) { - int d = c & 0x1FF; - if (d >= 'a' && d <= 'z') { - c = (c + ('a' - ctrlChar('A'))) & ~CTRL; - } - if (d >= 'A' && d <= 'Z') { - c = (c + ('A' - ctrlChar('A'))) & ~CTRL; - } - if (d >= ctrlChar('A') && d <= ctrlChar('Z')) { - c = c & ~CTRL; - } - } - return c; -} - -// break characters that may precede items to be completed -static const char breakChars[] = " =+-/\\*?\"'`&<>;|@{([])}"; - -// maximum number of completions to display without asking -static const size_t completionCountCutoff = 100; - -/** - * Handle command completion, using a completionCallback() routine to provide - * possible substitutions - * This routine handles the mechanics of updating the user's input buffer with - * possible replacement - * of text as the user selects a proposed completion string, or cancels the - * completion attempt. - * @param pi PromptBase struct holding information about the prompt and our - * screen position - */ -int InputBuffer::completeLine(PromptBase& pi) { - linenoiseCompletions lc; - char32_t c = 0; - - // completionCallback() expects a parsable entity, so find the previous break - // character and - // extract a copy to parse. we also handle the case where tab is hit while - // not at end-of-line. - int startIndex = pos; - while (--startIndex >= 0) { - if (strchr(breakChars, buf32[startIndex])) { - break; - } - } - ++startIndex; - int itemLength = pos - startIndex; - Utf32String unicodeCopy(&buf32[startIndex], itemLength); - Utf8String parseItem(unicodeCopy); - - // get a list of completions - completionCallback(parseItem.get(), &lc); - - // if no completions, we are done - if (lc.completionStrings.size() == 0) { - beep(); - freeCompletions(&lc); - return 0; - } - - // at least one completion - int longestCommonPrefix = 0; - int displayLength = 0; - if (lc.completionStrings.size() == 1) { - longestCommonPrefix = static_cast<int>(lc.completionStrings[0].length()); - } else { - bool keepGoing = true; - while (keepGoing) { - for (size_t j = 0; j < lc.completionStrings.size() - 1; ++j) { - char32_t c1 = lc.completionStrings[j][longestCommonPrefix]; - char32_t c2 = lc.completionStrings[j + 1][longestCommonPrefix]; - if ((0 == c1) || (0 == c2) || (c1 != c2)) { - keepGoing = false; - break; - } - } - if (keepGoing) { - ++longestCommonPrefix; - } - } - } - if (lc.completionStrings.size() != 1) { // beep if ambiguous - beep(); - } - - // if we can extend the item, extend it and return to main loop - if (longestCommonPrefix > itemLength) { - displayLength = len + longestCommonPrefix - itemLength; - if (displayLength > buflen) { - longestCommonPrefix -= displayLength - buflen; // don't overflow buffer - displayLength = buflen; // truncate the insertion - beep(); // and make a noise - } - Utf32String displayText(displayLength + 1); - memcpy(displayText.get(), buf32, sizeof(char32_t) * startIndex); - memcpy(&displayText[startIndex], &lc.completionStrings[0][0], - sizeof(char32_t) * longestCommonPrefix); - int tailIndex = startIndex + longestCommonPrefix; - memcpy(&displayText[tailIndex], &buf32[pos], - sizeof(char32_t) * (displayLength - tailIndex + 1)); - copyString32(buf32, displayText.get(), displayLength); - pos = startIndex + longestCommonPrefix; - len = displayLength; - refreshLine(pi); - return 0; - } - - // we can't complete any further, wait for second tab - do { - c = linenoiseReadChar(); - c = cleanupCtrl(c); - } while (c == static_cast<char32_t>(-1)); - - // if any character other than tab, pass it to the main loop - if (c != ctrlChar('I')) { - freeCompletions(&lc); - return c; - } - - // we got a second tab, maybe show list of possible completions - bool showCompletions = true; - bool onNewLine = false; - if (lc.completionStrings.size() > completionCountCutoff) { - int savePos = - pos; // move cursor to EOL to avoid overwriting the command line - pos = len; - refreshLine(pi); - pos = savePos; - printf("\nDisplay all %u possibilities? (y or n)", - static_cast<unsigned int>(lc.completionStrings.size())); - fflush(stdout); - onNewLine = true; - while (c != 'y' && c != 'Y' && c != 'n' && c != 'N' && c != ctrlChar('C')) { - do { - c = linenoiseReadChar(); - c = cleanupCtrl(c); - } while (c == static_cast<char32_t>(-1)); - } - switch (c) { - case 'n': - case 'N': - showCompletions = false; - freeCompletions(&lc); - break; - case ctrlChar('C'): - showCompletions = false; - freeCompletions(&lc); - if (write(1, "^C", 2) == -1) return -1; // Display the ^C we got - c = 0; - break; - } - } - - // if showing the list, do it the way readline does it - bool stopList = false; - if (showCompletions) { - int longestCompletion = 0; - for (size_t j = 0; j < lc.completionStrings.size(); ++j) { - itemLength = static_cast<int>(lc.completionStrings[j].length()); - if (itemLength > longestCompletion) { - longestCompletion = itemLength; - } - } - longestCompletion += 2; - int columnCount = pi.promptScreenColumns / longestCompletion; - if (columnCount < 1) { - columnCount = 1; - } - if (!onNewLine) { // skip this if we showed "Display all %d possibilities?" - int savePos = - pos; // move cursor to EOL to avoid overwriting the command line - pos = len; - refreshLine(pi); - pos = savePos; - } - size_t pauseRow = getScreenRows() - 1; - size_t rowCount = - (lc.completionStrings.size() + columnCount - 1) / columnCount; - for (size_t row = 0; row < rowCount; ++row) { - if (row == pauseRow) { - printf("\n--More--"); - fflush(stdout); - c = 0; - bool doBeep = false; - while (c != ' ' && c != '\r' && c != '\n' && c != 'y' && c != 'Y' && - c != 'n' && c != 'N' && c != 'q' && c != 'Q' && - c != ctrlChar('C')) { - if (doBeep) { - beep(); - } - doBeep = true; - do { - c = linenoiseReadChar(); - c = cleanupCtrl(c); - } while (c == static_cast<char32_t>(-1)); - } - switch (c) { - case ' ': - case 'y': - case 'Y': - printf("\r \r"); - pauseRow += getScreenRows() - 1; - break; - case '\r': - case '\n': - printf("\r \r"); - ++pauseRow; - break; - case 'n': - case 'N': - case 'q': - case 'Q': - printf("\r \r"); - stopList = true; - break; - case ctrlChar('C'): - if (write(1, "^C", 2) == -1) return -1; // Display the ^C we got - stopList = true; - break; - } - } else { - printf("\n"); - } - if (stopList) { - break; - } - for (int column = 0; column < columnCount; ++column) { - size_t index = (column * rowCount) + row; - if (index < lc.completionStrings.size()) { - itemLength = static_cast<int>(lc.completionStrings[index].length()); - fflush(stdout); - if (write32(1, lc.completionStrings[index].get(), itemLength) == -1) - return -1; - if (((column + 1) * rowCount) + row < lc.completionStrings.size()) { - for (int k = itemLength; k < longestCompletion; ++k) { - printf(" "); - } - } - } - } - } - fflush(stdout); - freeCompletions(&lc); - } - - // display the prompt on a new line, then redisplay the input buffer - if (!stopList || c == ctrlChar('C')) { - if (write(1, "\n", 1) == -1) return 0; - } - if (!pi.write()) return 0; -#ifndef _WIN32 - // we have to generate our own newline on line wrap on Linux - if (pi.promptIndentation == 0 && pi.promptExtraLines > 0) - if (write(1, "\n", 1) == -1) return 0; -#endif - pi.promptCursorRowOffset = pi.promptExtraLines; - refreshLine(pi); - return 0; -} - -/** - * Clear the screen ONLY (no redisplay of anything) - */ -void linenoiseClearScreen(void) { -#ifdef _WIN32 - COORD coord = {0, 0}; - CONSOLE_SCREEN_BUFFER_INFO inf; - HANDLE screenHandle = GetStdHandle(STD_OUTPUT_HANDLE); - GetConsoleScreenBufferInfo(screenHandle, &inf); - SetConsoleCursorPosition(screenHandle, coord); - DWORD count; - FillConsoleOutputCharacterA(screenHandle, ' ', inf.dwSize.X * inf.dwSize.Y, - coord, &count); -#else - if (write(1, "\x1b[H\x1b[2J", 7) <= 0) return; -#endif -} - -void InputBuffer::clearScreen(PromptBase& pi) { - linenoiseClearScreen(); - if (!pi.write()) return; -#ifndef _WIN32 - // we have to generate our own newline on line wrap on Linux - if (pi.promptIndentation == 0 && pi.promptExtraLines > 0) - if (write(1, "\n", 1) == -1) return; -#endif - pi.promptCursorRowOffset = pi.promptExtraLines; - refreshLine(pi); -} - -/** - * Incremental history search -- take over the prompt and keyboard as the user - * types a search - * string, deletes characters from it, changes direction, and either accepts the - * found line (for - * execution orediting) or cancels. - * @param pi PromptBase struct holding information about the (old, - * static) prompt and our - * screen position - * @param startChar the character that began the search, used to set the initial - * direction - */ -int InputBuffer::incrementalHistorySearch(PromptBase& pi, int startChar) { - size_t bufferSize; - size_t ucharCount = 0; - - // if not already recalling, add the current line to the history list so we - // don't have to - // special case it - if (historyIndex == historyLen - 1) { - free(history[historyLen - 1]); - bufferSize = sizeof(char32_t) * len + 1; - unique_ptr<char[]> tempBuffer(new char[bufferSize]); - copyString32to8(tempBuffer.get(), bufferSize, buf32); - history[historyLen - 1] = strdup8(tempBuffer.get()); - } - int historyLineLength = len; - int historyLinePosition = pos; - char32_t emptyBuffer[1]; - char emptyWidths[1]; - InputBuffer empty(emptyBuffer, emptyWidths, 1); - empty.refreshLine(pi); // erase the old input first - DynamicPrompt dp(pi, (startChar == ctrlChar('R')) ? -1 : 1); - - dp.promptPreviousLen = pi.promptPreviousLen; - dp.promptPreviousInputLen = pi.promptPreviousInputLen; - dynamicRefresh(dp, buf32, historyLineLength, - historyLinePosition); // draw user's text with our prompt - - // loop until we get an exit character - int c = 0; - bool keepLooping = true; - bool useSearchedLine = true; - bool searchAgain = false; - char32_t* activeHistoryLine = 0; - while (keepLooping) { - c = linenoiseReadChar(); - c = cleanupCtrl(c); // convert CTRL + <char> into normal ctrl - - switch (c) { - // these characters keep the selected text but do not execute it - case ctrlChar('A'): // ctrl-A, move cursor to start of line - case HOME_KEY: - case ctrlChar('B'): // ctrl-B, move cursor left by one character - case LEFT_ARROW_KEY: - case META + 'b': // meta-B, move cursor left by one word - case META + 'B': - case CTRL + LEFT_ARROW_KEY: - case META + LEFT_ARROW_KEY: // Emacs allows Meta, bash & readline don't - case ctrlChar('D'): - case META + 'd': // meta-D, kill word to right of cursor - case META + 'D': - case ctrlChar('E'): // ctrl-E, move cursor to end of line - case END_KEY: - case ctrlChar('F'): // ctrl-F, move cursor right by one character - case RIGHT_ARROW_KEY: - case META + 'f': // meta-F, move cursor right by one word - case META + 'F': - case CTRL + RIGHT_ARROW_KEY: - case META + RIGHT_ARROW_KEY: // Emacs allows Meta, bash & readline don't - case META + ctrlChar('H'): - case ctrlChar('J'): - case ctrlChar('K'): // ctrl-K, kill from cursor to end of line - case ctrlChar('M'): - case ctrlChar('N'): // ctrl-N, recall next line in history - case ctrlChar('P'): // ctrl-P, recall previous line in history - case DOWN_ARROW_KEY: - case UP_ARROW_KEY: - case ctrlChar('T'): // ctrl-T, transpose characters - case ctrlChar( - 'U'): // ctrl-U, kill all characters to the left of the cursor - case ctrlChar('W'): - case META + 'y': // meta-Y, "yank-pop", rotate popped text - case META + 'Y': - case 127: - case DELETE_KEY: - case META + '<': // start of history - case PAGE_UP_KEY: - case META + '>': // end of history - case PAGE_DOWN_KEY: - keepLooping = false; - break; - - // these characters revert the input line to its previous state - case ctrlChar('C'): // ctrl-C, abort this line - case ctrlChar('G'): - case ctrlChar('L'): // ctrl-L, clear screen and redisplay line - keepLooping = false; - useSearchedLine = false; - if (c != ctrlChar('L')) { - c = -1; // ctrl-C and ctrl-G just abort the search and do nothing - // else - } - break; - - // these characters stay in search mode and update the display - case ctrlChar('S'): - case ctrlChar('R'): - if (dp.searchTextLen == - 0) { // if no current search text, recall previous text - if (previousSearchText.length()) { - dp.updateSearchText(previousSearchText.get()); - } - } - if ((dp.direction == 1 && c == ctrlChar('R')) || - (dp.direction == -1 && c == ctrlChar('S'))) { - dp.direction = 0 - dp.direction; // reverse direction - dp.updateSearchPrompt(); // change the prompt - } else { - searchAgain = true; // same direction, search again - } - break; - -// job control is its own thing -#ifndef _WIN32 - case ctrlChar('Z'): // ctrl-Z, job control - disableRawMode(); // Returning to Linux (whatever) shell, leave raw - // mode - raise(SIGSTOP); // Break out in mid-line - enableRawMode(); // Back from Linux shell, re-enter raw mode - { - bufferSize = historyLineLength + 1; - unique_ptr<char32_t[]> tempUnicode(new char32_t[bufferSize]); - copyString8to32(tempUnicode.get(), bufferSize, ucharCount, - history[historyIndex]); - dynamicRefresh(dp, tempUnicode.get(), historyLineLength, - historyLinePosition); - } - continue; - break; -#endif - - // these characters update the search string, and hence the selected input - // line - case ctrlChar('H'): // backspace/ctrl-H, delete char to left of cursor - if (dp.searchTextLen > 0) { - unique_ptr<char32_t[]> tempUnicode(new char32_t[dp.searchTextLen]); - --dp.searchTextLen; - dp.searchText[dp.searchTextLen] = 0; - copyString32(tempUnicode.get(), dp.searchText.get(), - dp.searchTextLen); - dp.updateSearchText(tempUnicode.get()); - } else { - beep(); - } - break; - - case ctrlChar('Y'): // ctrl-Y, yank killed text - break; - - default: - if (!isControlChar(c) && c <= 0x0010FFFF) { // not an action character - unique_ptr<char32_t[]> tempUnicode( - new char32_t[dp.searchTextLen + 2]); - copyString32(tempUnicode.get(), dp.searchText.get(), - dp.searchTextLen); - tempUnicode[dp.searchTextLen] = c; - tempUnicode[dp.searchTextLen + 1] = 0; - dp.updateSearchText(tempUnicode.get()); - } else { - beep(); - } - } // switch - - // if we are staying in search mode, search now - if (keepLooping) { - bufferSize = historyLineLength + 1; - if (activeHistoryLine) { - delete[] activeHistoryLine; - activeHistoryLine = nullptr; - } - activeHistoryLine = new char32_t[bufferSize]; - copyString8to32(activeHistoryLine, bufferSize, ucharCount, - history[historyIndex]); - if (dp.searchTextLen > 0) { - bool found = false; - int historySearchIndex = historyIndex; - int lineLength = static_cast<int>(ucharCount); - int lineSearchPos = historyLinePosition; - if (searchAgain) { - lineSearchPos += dp.direction; - } - searchAgain = false; - while (true) { - while ((dp.direction > 0) ? (lineSearchPos < lineLength) - : (lineSearchPos >= 0)) { - if (strncmp32(dp.searchText.get(), - &activeHistoryLine[lineSearchPos], - dp.searchTextLen) == 0) { - found = true; - break; - } - lineSearchPos += dp.direction; - } - if (found) { - historyIndex = historySearchIndex; - historyLineLength = lineLength; - historyLinePosition = lineSearchPos; - break; - } else if ((dp.direction > 0) ? (historySearchIndex < historyLen - 1) - : (historySearchIndex > 0)) { - historySearchIndex += dp.direction; - bufferSize = strlen8(history[historySearchIndex]) + 1; - delete[] activeHistoryLine; - activeHistoryLine = nullptr; - activeHistoryLine = new char32_t[bufferSize]; - copyString8to32(activeHistoryLine, bufferSize, ucharCount, - history[historySearchIndex]); - lineLength = static_cast<int>(ucharCount); - lineSearchPos = - (dp.direction > 0) ? 0 : (lineLength - dp.searchTextLen); - } else { - beep(); - break; - } - }; // while - } - if (activeHistoryLine) { - delete[] activeHistoryLine; - activeHistoryLine = nullptr; - } - bufferSize = historyLineLength + 1; - activeHistoryLine = new char32_t[bufferSize]; - copyString8to32(activeHistoryLine, bufferSize, ucharCount, - history[historyIndex]); - dynamicRefresh(dp, activeHistoryLine, historyLineLength, - historyLinePosition); // draw user's text with our prompt - } - } // while - - // leaving history search, restore previous prompt, maybe make searched line - // current - PromptBase pb; - pb.promptChars = pi.promptIndentation; - pb.promptBytes = pi.promptBytes; - Utf32String tempUnicode(pb.promptBytes + 1); - - copyString32(tempUnicode.get(), &pi.promptText[pi.promptLastLinePosition], - pb.promptBytes - pi.promptLastLinePosition); - tempUnicode.initFromBuffer(); - pb.promptText = tempUnicode; - pb.promptExtraLines = 0; - pb.promptIndentation = pi.promptIndentation; - pb.promptLastLinePosition = 0; - pb.promptPreviousInputLen = historyLineLength; - pb.promptCursorRowOffset = dp.promptCursorRowOffset; - pb.promptScreenColumns = pi.promptScreenColumns; - pb.promptPreviousLen = dp.promptChars; - if (useSearchedLine && activeHistoryLine) { - historyRecallMostRecent = true; - copyString32(buf32, activeHistoryLine, buflen + 1); - len = historyLineLength; - pos = historyLinePosition; - } - if (activeHistoryLine) { - delete[] activeHistoryLine; - activeHistoryLine = nullptr; - } - dynamicRefresh(pb, buf32, len, - pos); // redraw the original prompt with current input - pi.promptPreviousInputLen = len; - pi.promptCursorRowOffset = pi.promptExtraLines + pb.promptCursorRowOffset; - previousSearchText = - dp.searchText; // save search text for possible reuse on ctrl-R ctrl-R - return c; // pass a character or -1 back to main loop -} - -static bool isCharacterAlphanumeric(char32_t testChar) { -#ifdef _WIN32 - return (iswalnum((wint_t)testChar) != 0 ? true : false); -#else - return (iswalnum(testChar) != 0 ? true : false); -#endif -} - -#ifndef _WIN32 -static bool gotResize = false; -#endif -static int keyType = 0; - -int InputBuffer::getInputLine(PromptBase& pi) { - keyType = 0; - - // The latest history entry is always our current buffer - if (len > 0) { - size_t bufferSize = sizeof(char32_t) * len + 1; - unique_ptr<char[]> tempBuffer(new char[bufferSize]); - copyString32to8(tempBuffer.get(), bufferSize, buf32); - linenoiseHistoryAdd(tempBuffer.get()); - } else { - linenoiseHistoryAdd(""); - } - historyIndex = historyLen - 1; - historyRecallMostRecent = false; - - // display the prompt - if (!pi.write()) return -1; - -#ifndef _WIN32 - // we have to generate our own newline on line wrap on Linux - if (pi.promptIndentation == 0 && pi.promptExtraLines > 0) - if (write(1, "\n", 1) == -1) return -1; -#endif - - // the cursor starts out at the end of the prompt - pi.promptCursorRowOffset = pi.promptExtraLines; - - // kill and yank start in "other" mode - killRing.lastAction = KillRing::actionOther; - - // when history search returns control to us, we execute its terminating - // keystroke - int terminatingKeystroke = -1; - - // if there is already text in the buffer, display it first - if (len > 0) { - refreshLine(pi); - } - - // loop collecting characters, respond to line editing characters - while (true) { - int c; - if (terminatingKeystroke == -1) { - c = linenoiseReadChar(); // get a new keystroke - - keyType = 0; - if (c != 0) { - // set flag that we got some input - if (c == ctrlChar('C')) { - keyType = 1; - } else if (c == ctrlChar('D')) { - keyType = 2; - } - } - -#ifndef _WIN32 - if (c == 0 && gotResize) { - // caught a window resize event - // now redraw the prompt and line - gotResize = false; - pi.promptScreenColumns = getScreenColumns(); - dynamicRefresh(pi, buf32, len, - pos); // redraw the original prompt with current input - continue; - } -#endif - } else { - c = terminatingKeystroke; // use the terminating keystroke from search - terminatingKeystroke = -1; // clear it once we've used it - } - - c = cleanupCtrl(c); // convert CTRL + <char> into normal ctrl - - if (c == 0) { - return len; - } - - if (c == -1) { - refreshLine(pi); - continue; - } - - if (c == -2) { - if (!pi.write()) return -1; - refreshLine(pi); - continue; - } - - // ctrl-I/tab, command completion, needs to be before switch statement - if (c == ctrlChar('I') && completionCallback) { - killRing.lastAction = KillRing::actionOther; - historyRecallMostRecent = false; - - // completeLine does the actual completion and replacement - c = completeLine(pi); - - if (c < 0) // return on error - return len; - - if (c == 0) // read next character when 0 - continue; - - // deliberate fall-through here, so we use the terminating character - } - - switch (c) { - case ctrlChar('A'): // ctrl-A, move cursor to start of line - case HOME_KEY: - killRing.lastAction = KillRing::actionOther; - pos = 0; - refreshLine(pi); - break; - - case ctrlChar('B'): // ctrl-B, move cursor left by one character - case LEFT_ARROW_KEY: - killRing.lastAction = KillRing::actionOther; - if (pos > 0) { - --pos; - refreshLine(pi); - } - break; - - case META + 'b': // meta-B, move cursor left by one word - case META + 'B': - case CTRL + LEFT_ARROW_KEY: - case META + LEFT_ARROW_KEY: // Emacs allows Meta, bash & readline don't - killRing.lastAction = KillRing::actionOther; - if (pos > 0) { - while (pos > 0 && !isCharacterAlphanumeric(buf32[pos - 1])) { - --pos; - } - while (pos > 0 && isCharacterAlphanumeric(buf32[pos - 1])) { - --pos; - } - refreshLine(pi); - } - break; - - case ctrlChar('C'): // ctrl-C, abort this line - killRing.lastAction = KillRing::actionOther; - historyRecallMostRecent = false; - errno = EAGAIN; - --historyLen; - free(history[historyLen]); - // we need one last refresh with the cursor at the end of the line - // so we don't display the next prompt over the previous input line - pos = len; // pass len as pos for EOL - refreshLine(pi); - if (write(1, "^C", 2) == -1) return -1; // Display the ^C we got - return -1; - - case META + 'c': // meta-C, give word initial Cap - case META + 'C': - killRing.lastAction = KillRing::actionOther; - historyRecallMostRecent = false; - if (pos < len) { - while (pos < len && !isCharacterAlphanumeric(buf32[pos])) { - ++pos; - } - if (pos < len && isCharacterAlphanumeric(buf32[pos])) { - if (buf32[pos] >= 'a' && buf32[pos] <= 'z') { - buf32[pos] += 'A' - 'a'; - } - ++pos; - } - while (pos < len && isCharacterAlphanumeric(buf32[pos])) { - if (buf32[pos] >= 'A' && buf32[pos] <= 'Z') { - buf32[pos] += 'a' - 'A'; - } - ++pos; - } - refreshLine(pi); - } - break; - - // ctrl-D, delete the character under the cursor - // on an empty line, exit the shell - case ctrlChar('D'): - killRing.lastAction = KillRing::actionOther; - if (len > 0 && pos < len) { - historyRecallMostRecent = false; - memmove(buf32 + pos, buf32 + pos + 1, sizeof(char32_t) * (len - pos)); - --len; - refreshLine(pi); - } else if (len == 0) { - --historyLen; - free(history[historyLen]); - return -1; - } - break; - - case META + 'd': // meta-D, kill word to right of cursor - case META + 'D': - if (pos < len) { - historyRecallMostRecent = false; - int endingPos = pos; - while (endingPos < len && - !isCharacterAlphanumeric(buf32[endingPos])) { - ++endingPos; - } - while (endingPos < len && isCharacterAlphanumeric(buf32[endingPos])) { - ++endingPos; - } - killRing.kill(&buf32[pos], endingPos - pos, true); - memmove(buf32 + pos, buf32 + endingPos, - sizeof(char32_t) * (len - endingPos + 1)); - len -= endingPos - pos; - refreshLine(pi); - } - killRing.lastAction = KillRing::actionKill; - break; - - case ctrlChar('E'): // ctrl-E, move cursor to end of line - case END_KEY: - killRing.lastAction = KillRing::actionOther; - pos = len; - refreshLine(pi); - break; - - case ctrlChar('F'): // ctrl-F, move cursor right by one character - case RIGHT_ARROW_KEY: - killRing.lastAction = KillRing::actionOther; - if (pos < len) { - ++pos; - refreshLine(pi); - } - break; - - case META + 'f': // meta-F, move cursor right by one word - case META + 'F': - case CTRL + RIGHT_ARROW_KEY: - case META + RIGHT_ARROW_KEY: // Emacs allows Meta, bash & readline don't - killRing.lastAction = KillRing::actionOther; - if (pos < len) { - while (pos < len && !isCharacterAlphanumeric(buf32[pos])) { - ++pos; - } - while (pos < len && isCharacterAlphanumeric(buf32[pos])) { - ++pos; - } - refreshLine(pi); - } - break; - - case ctrlChar('H'): // backspace/ctrl-H, delete char to left of cursor - killRing.lastAction = KillRing::actionOther; - if (pos > 0) { - historyRecallMostRecent = false; - memmove(buf32 + pos - 1, buf32 + pos, - sizeof(char32_t) * (1 + len - pos)); - --pos; - --len; - refreshLine(pi); - } - break; - - // meta-Backspace, kill word to left of cursor - case META + ctrlChar('H'): - if (pos > 0) { - historyRecallMostRecent = false; - int startingPos = pos; - while (pos > 0 && !isCharacterAlphanumeric(buf32[pos - 1])) { - --pos; - } - while (pos > 0 && isCharacterAlphanumeric(buf32[pos - 1])) { - --pos; - } - killRing.kill(&buf32[pos], startingPos - pos, false); - memmove(buf32 + pos, buf32 + startingPos, - sizeof(char32_t) * (len - startingPos + 1)); - len -= startingPos - pos; - refreshLine(pi); - } - killRing.lastAction = KillRing::actionKill; - break; - - case ctrlChar('J'): // ctrl-J/linefeed/newline, accept line - case ctrlChar('M'): // ctrl-M/return/enter - killRing.lastAction = KillRing::actionOther; - // we need one last refresh with the cursor at the end of the line - // so we don't display the next prompt over the previous input line - pos = len; // pass len as pos for EOL - refreshLine(pi); - historyPreviousIndex = historyRecallMostRecent ? historyIndex : -2; - --historyLen; - free(history[historyLen]); - return len; - - case ctrlChar('K'): // ctrl-K, kill from cursor to end of line - killRing.kill(&buf32[pos], len - pos, true); - buf32[pos] = '\0'; - len = pos; - refreshLine(pi); - killRing.lastAction = KillRing::actionKill; - historyRecallMostRecent = false; - break; - - case ctrlChar('L'): // ctrl-L, clear screen and redisplay line - clearScreen(pi); - break; - - case META + 'l': // meta-L, lowercase word - case META + 'L': - killRing.lastAction = KillRing::actionOther; - if (pos < len) { - historyRecallMostRecent = false; - while (pos < len && !isCharacterAlphanumeric(buf32[pos])) { - ++pos; - } - while (pos < len && isCharacterAlphanumeric(buf32[pos])) { - if (buf32[pos] >= 'A' && buf32[pos] <= 'Z') { - buf32[pos] += 'a' - 'A'; - } - ++pos; - } - refreshLine(pi); - } - break; - - case ctrlChar('N'): // ctrl-N, recall next line in history - case ctrlChar('P'): // ctrl-P, recall previous line in history - case DOWN_ARROW_KEY: - case UP_ARROW_KEY: - killRing.lastAction = KillRing::actionOther; - // if not already recalling, add the current line to the history list so - // we don't - // have to special case it - if (historyIndex == historyLen - 1) { - free(history[historyLen - 1]); - size_t tempBufferSize = sizeof(char32_t) * len + 1; - unique_ptr<char[]> tempBuffer(new char[tempBufferSize]); - copyString32to8(tempBuffer.get(), tempBufferSize, buf32); - history[historyLen - 1] = strdup8(tempBuffer.get()); - } - if (historyLen > 1) { - if (c == UP_ARROW_KEY) { - c = ctrlChar('P'); - } - if (historyPreviousIndex != -2 && c != ctrlChar('P')) { - historyIndex = - 1 + historyPreviousIndex; // emulate Windows down-arrow - } else { - historyIndex += (c == ctrlChar('P')) ? -1 : 1; - } - historyPreviousIndex = -2; - if (historyIndex < 0) { - historyIndex = 0; - break; - } else if (historyIndex >= historyLen) { - historyIndex = historyLen - 1; - break; - } - historyRecallMostRecent = true; - size_t ucharCount = 0; - copyString8to32(buf32, buflen, ucharCount, history[historyIndex]); - len = pos = static_cast<int>(ucharCount); - refreshLine(pi); - } - break; - - case ctrlChar('R'): // ctrl-R, reverse history search - case ctrlChar('S'): // ctrl-S, forward history search - terminatingKeystroke = incrementalHistorySearch(pi, c); - break; - - case ctrlChar('T'): // ctrl-T, transpose characters - killRing.lastAction = KillRing::actionOther; - if (pos > 0 && len > 1) { - historyRecallMostRecent = false; - size_t leftCharPos = (pos == len) ? pos - 2 : pos - 1; - char32_t aux = buf32[leftCharPos]; - buf32[leftCharPos] = buf32[leftCharPos + 1]; - buf32[leftCharPos + 1] = aux; - if (pos != len) ++pos; - refreshLine(pi); - } - break; - - case ctrlChar( - 'U'): // ctrl-U, kill all characters to the left of the cursor - if (pos > 0) { - historyRecallMostRecent = false; - killRing.kill(&buf32[0], pos, false); - len -= pos; - memmove(buf32, buf32 + pos, sizeof(char32_t) * (len + 1)); - pos = 0; - refreshLine(pi); - } - killRing.lastAction = KillRing::actionKill; - break; - - case META + 'u': // meta-U, uppercase word - case META + 'U': - killRing.lastAction = KillRing::actionOther; - if (pos < len) { - historyRecallMostRecent = false; - while (pos < len && !isCharacterAlphanumeric(buf32[pos])) { - ++pos; - } - while (pos < len && isCharacterAlphanumeric(buf32[pos])) { - if (buf32[pos] >= 'a' && buf32[pos] <= 'z') { - buf32[pos] += 'A' - 'a'; - } - ++pos; - } - refreshLine(pi); - } - break; - - // ctrl-W, kill to whitespace (not word) to left of cursor - case ctrlChar('W'): - if (pos > 0) { - historyRecallMostRecent = false; - int startingPos = pos; - while (pos > 0 && buf32[pos - 1] == ' ') { - --pos; - } - while (pos > 0 && buf32[pos - 1] != ' ') { - --pos; - } - killRing.kill(&buf32[pos], startingPos - pos, false); - memmove(buf32 + pos, buf32 + startingPos, - sizeof(char32_t) * (len - startingPos + 1)); - len -= startingPos - pos; - refreshLine(pi); - } - killRing.lastAction = KillRing::actionKill; - break; - - case ctrlChar('Y'): // ctrl-Y, yank killed text - historyRecallMostRecent = false; - { - Utf32String* restoredText = killRing.yank(); - if (restoredText) { - bool truncated = false; - size_t ucharCount = restoredText->length(); - if (ucharCount > static_cast<size_t>(buflen - len)) { - ucharCount = buflen - len; - truncated = true; - } - memmove(buf32 + pos + ucharCount, buf32 + pos, - sizeof(char32_t) * (len - pos + 1)); - memmove(buf32 + pos, restoredText->get(), - sizeof(char32_t) * ucharCount); - pos += static_cast<int>(ucharCount); - len += static_cast<int>(ucharCount); - refreshLine(pi); - killRing.lastAction = KillRing::actionYank; - killRing.lastYankSize = ucharCount; - if (truncated) { - beep(); - } - } else { - beep(); - } - } - break; - - case META + 'y': // meta-Y, "yank-pop", rotate popped text - case META + 'Y': - if (killRing.lastAction == KillRing::actionYank) { - historyRecallMostRecent = false; - Utf32String* restoredText = killRing.yankPop(); - if (restoredText) { - bool truncated = false; - size_t ucharCount = restoredText->length(); - if (ucharCount > - static_cast<size_t>(killRing.lastYankSize + buflen - len)) { - ucharCount = killRing.lastYankSize + buflen - len; - truncated = true; - } - if (ucharCount > killRing.lastYankSize) { - memmove(buf32 + pos + ucharCount - killRing.lastYankSize, - buf32 + pos, sizeof(char32_t) * (len - pos + 1)); - memmove(buf32 + pos - killRing.lastYankSize, restoredText->get(), - sizeof(char32_t) * ucharCount); - } else { - memmove(buf32 + pos - killRing.lastYankSize, restoredText->get(), - sizeof(char32_t) * ucharCount); - memmove(buf32 + pos + ucharCount - killRing.lastYankSize, - buf32 + pos, sizeof(char32_t) * (len - pos + 1)); - } - pos += static_cast<int>(ucharCount - killRing.lastYankSize); - len += static_cast<int>(ucharCount - killRing.lastYankSize); - killRing.lastYankSize = ucharCount; - refreshLine(pi); - if (truncated) { - beep(); - } - break; - } - } - beep(); - break; - -#ifndef _WIN32 - case ctrlChar('Z'): // ctrl-Z, job control - disableRawMode(); // Returning to Linux (whatever) shell, leave raw - // mode - raise(SIGSTOP); // Break out in mid-line - enableRawMode(); // Back from Linux shell, re-enter raw mode - if (!pi.write()) break; // Redraw prompt - refreshLine(pi); // Refresh the line - break; -#endif - - // DEL, delete the character under the cursor - case 127: - case DELETE_KEY: - killRing.lastAction = KillRing::actionOther; - if (len > 0 && pos < len) { - historyRecallMostRecent = false; - memmove(buf32 + pos, buf32 + pos + 1, sizeof(char32_t) * (len - pos)); - --len; - refreshLine(pi); - } - break; - - case META + '<': // meta-<, beginning of history - case PAGE_UP_KEY: // Page Up, beginning of history - case META + '>': // meta->, end of history - case PAGE_DOWN_KEY: // Page Down, end of history - killRing.lastAction = KillRing::actionOther; - // if not already recalling, add the current line to the history list so - // we don't - // have to special case it - if (historyIndex == historyLen - 1) { - free(history[historyLen - 1]); - size_t tempBufferSize = sizeof(char32_t) * len + 1; - unique_ptr<char[]> tempBuffer(new char[tempBufferSize]); - copyString32to8(tempBuffer.get(), tempBufferSize, buf32); - history[historyLen - 1] = strdup8(tempBuffer.get()); - } - if (historyLen > 1) { - historyIndex = - (c == META + '<' || c == PAGE_UP_KEY) ? 0 : historyLen - 1; - historyPreviousIndex = -2; - historyRecallMostRecent = true; - size_t ucharCount = 0; - copyString8to32(buf32, buflen, ucharCount, history[historyIndex]); - len = pos = static_cast<int>(ucharCount); - refreshLine(pi); - } - break; - - // not one of our special characters, maybe insert it in the buffer - default: - killRing.lastAction = KillRing::actionOther; - historyRecallMostRecent = false; - if (c & (META | CTRL)) { // beep on unknown Ctrl and/or Meta keys - beep(); - break; - } - if (len < buflen) { - if (isControlChar(c)) { // don't insert control characters - beep(); - break; - } - if (len == pos) { // at end of buffer - buf32[pos] = c; - ++pos; - ++len; - buf32[len] = '\0'; - int inputLen = calculateColumnPosition(buf32, len); - if (pi.promptIndentation + inputLen < pi.promptScreenColumns) { - if (inputLen > pi.promptPreviousInputLen) - pi.promptPreviousInputLen = inputLen; - /* Avoid a full update of the line in the - * trivial case. */ - if (write32(1, reinterpret_cast<char32_t*>(&c), 1) == -1) - return -1; - } else { - refreshLine(pi); - } - } else { // not at end of buffer, have to move characters to our - // right - memmove(buf32 + pos + 1, buf32 + pos, - sizeof(char32_t) * (len - pos)); - buf32[pos] = c; - ++len; - ++pos; - buf32[len] = '\0'; - refreshLine(pi); - } - } else { - beep(); // buffer is full, beep on new characters - } - break; - } - } - return len; -} - -static string preloadedBufferContents; // used with linenoisePreloadBuffer -static string preloadErrorMessage; - -/** - * linenoisePreloadBuffer provides text to be inserted into the command buffer - * - * the provided text will be processed to be usable and will be used to preload - * the input buffer on the next call to linenoise() - * - * @param preloadText text to begin with on the next call to linenoise() - */ -void linenoisePreloadBuffer(const char* preloadText) { - if (!preloadText) { - return; - } - int bufferSize = static_cast<int>(strlen(preloadText) + 1); - unique_ptr<char[]> tempBuffer(new char[bufferSize]); - strncpy(&tempBuffer[0], preloadText, bufferSize); - - // remove characters that won't display correctly - char* pIn = &tempBuffer[0]; - char* pOut = pIn; - bool controlsStripped = false; - bool whitespaceSeen = false; - while (*pIn) { - unsigned char c = - *pIn++; // we need unsigned so chars 0x80 and above are allowed - if ('\r' == c) { // silently skip CR - continue; - } - if ('\n' == c || '\t' == c) { // note newline or tab - whitespaceSeen = true; - continue; - } - if (isControlChar( - c)) { // remove other control characters, flag for message - controlsStripped = true; - *pOut++ = ' '; - continue; - } - if (whitespaceSeen) { // convert whitespace to a single space - *pOut++ = ' '; - whitespaceSeen = false; - } - *pOut++ = c; - } - *pOut = 0; - int processedLength = static_cast<int>(pOut - tempBuffer.get()); - bool lineTruncated = false; - if (processedLength > (LINENOISE_MAX_LINE - 1)) { - lineTruncated = true; - tempBuffer[LINENOISE_MAX_LINE - 1] = 0; - } - preloadedBufferContents = tempBuffer.get(); - if (controlsStripped) { - preloadErrorMessage += - " [Edited line: control characters were converted to spaces]\n"; - } - if (lineTruncated) { - preloadErrorMessage += " [Edited line: the line length was reduced from "; - char buf[128]; - snprintf(buf, sizeof(buf), "%d to %d]\n", processedLength, - (LINENOISE_MAX_LINE - 1)); - preloadErrorMessage += buf; - } -} - -/** - * linenoise is a readline replacement. - * - * call it with a prompt to display and it will return a line of input from the - * user - * - * @param prompt text of prompt to display to the user - * @return the returned string belongs to the caller on return and must be - * freed to prevent - * memory leaks - */ -char* linenoise(const char* prompt) { -#ifndef _WIN32 - gotResize = false; -#endif - if (isatty(STDIN_FILENO)) { // input is from a terminal - char32_t buf32[LINENOISE_MAX_LINE]; - char charWidths[LINENOISE_MAX_LINE]; - if (!preloadErrorMessage.empty()) { - printf("%s", preloadErrorMessage.c_str()); - fflush(stdout); - preloadErrorMessage.clear(); - } - PromptInfo pi(prompt, getScreenColumns()); - if (isUnsupportedTerm()) { - if (!pi.write()) return 0; - fflush(stdout); - if (preloadedBufferContents.empty()) { - unique_ptr<char[]> buf8(new char[LINENOISE_MAX_LINE]); - if (fgets(buf8.get(), LINENOISE_MAX_LINE, stdin) == NULL) { - return NULL; - } - size_t len = strlen(buf8.get()); - while (len && (buf8[len - 1] == '\n' || buf8[len - 1] == '\r')) { - --len; - buf8[len] = '\0'; - } - return strdup(buf8.get()); // caller must free buffer - } else { - char* buf8 = strdup(preloadedBufferContents.c_str()); - preloadedBufferContents.clear(); - return buf8; // caller must free buffer - } - } else { - if (enableRawMode() == -1) { - return NULL; - } - InputBuffer ib(buf32, charWidths, LINENOISE_MAX_LINE); - if (!preloadedBufferContents.empty()) { - ib.preloadBuffer(preloadedBufferContents.c_str()); - preloadedBufferContents.clear(); - } - int count = ib.getInputLine(pi); - disableRawMode(); - printf("\n"); - if (count == -1) { - return NULL; - } - size_t bufferSize = sizeof(char32_t) * ib.length() + 1; - unique_ptr<char[]> buf8(new char[bufferSize]); - copyString32to8(buf8.get(), bufferSize, buf32); - return strdup(buf8.get()); // caller must free buffer - } - } else { // input not from a terminal, we should work with piped input, i.e. - // redirected stdin - unique_ptr<char[]> buf8(new char[LINENOISE_MAX_LINE]); - if (fgets(buf8.get(), LINENOISE_MAX_LINE, stdin) == NULL) { - return NULL; - } - - // if fgets() gave us the newline, remove it - int count = static_cast<int>(strlen(buf8.get())); - if (count > 0 && buf8[count - 1] == '\n') { - --count; - buf8[count] = '\0'; - } - return strdup(buf8.get()); // caller must free buffer - } -} - -/* Register a callback function to be called for tab-completion. */ -void linenoiseSetCompletionCallback(linenoiseCompletionCallback* fn) { - completionCallback = fn; -} - -void linenoiseAddCompletion(linenoiseCompletions* lc, const char* str) { - lc->completionStrings.push_back(Utf32String(str)); -} - -int linenoiseHistoryAdd(const char* line) { - if (historyMaxLen == 0) { - return 0; - } - if (history == NULL) { - history = - reinterpret_cast<char8_t**>(malloc(sizeof(char8_t*) * historyMaxLen)); - if (history == NULL) { - return 0; - } - memset(history, 0, (sizeof(char*) * historyMaxLen)); - } - char8_t* linecopy = strdup8(line); - if (!linecopy) { - return 0; - } - - // convert newlines in multi-line code to spaces before storing - char8_t* p = linecopy; - while (*p) { - if (*p == '\n') { - *p = ' '; - } - ++p; - } - - // prevent duplicate history entries - if (historyLen > 0 && history[historyLen - 1] != nullptr && - strcmp(reinterpret_cast<char const*>(history[historyLen - 1]), - reinterpret_cast<char const*>(linecopy)) == 0) { - free(linecopy); - return 0; - } - - if (historyLen == historyMaxLen) { - free(history[0]); - memmove(history, history + 1, sizeof(char*) * (historyMaxLen - 1)); - --historyLen; - if (--historyPreviousIndex < -1) { - historyPreviousIndex = -2; - } - } - - history[historyLen] = linecopy; - ++historyLen; - return 1; -} - -int linenoiseHistorySetMaxLen(int len) { - if (len < 1) { - return 0; - } - if (history) { - int tocopy = historyLen; - char8_t** newHistory = - reinterpret_cast<char8_t**>(malloc(sizeof(char8_t*) * len)); - if (newHistory == NULL) { - return 0; - } - if (len < tocopy) { - tocopy = len; - } - memcpy(newHistory, history + historyMaxLen - tocopy, - sizeof(char8_t*) * tocopy); - free(history); - history = newHistory; - } - historyMaxLen = len; - if (historyLen > historyMaxLen) { - historyLen = historyMaxLen; - } - return 1; -} - -/* Fetch a line of the history by (zero-based) index. If the requested - * line does not exist, NULL is returned. The return value is a heap-allocated - * copy of the line, and the caller is responsible for de-allocating it. */ -char* linenoiseHistoryLine(int index) { - if (index < 0 || index >= historyLen) return NULL; - - return strdup(reinterpret_cast<char const*>(history[index])); -} - -/* Save the history in the specified file. On success 0 is returned - * otherwise -1 is returned. */ -int linenoiseHistorySave(const char* filename) { -#if _WIN32 - FILE* fp = fopen(filename, "wt"); -#else - int fd = open(filename, O_CREAT | O_TRUNC | O_WRONLY, S_IRUSR | S_IWUSR); - - if (fd < 0) { - return -1; - } - - FILE* fp = fdopen(fd, "wt"); -#endif - - if (fp == NULL) { - return -1; - } - - for (int j = 0; j < historyLen; ++j) { - if (history[j][0] != '\0') { - fprintf(fp, "%s\n", history[j]); - } - } - - fclose(fp); - - return 0; -} - -/* Load the history from the specified file. If the file does not exist - * zero is returned and no operation is performed. - * - * If the file exists and the operation succeeded 0 is returned, otherwise - * on error -1 is returned. */ -int linenoiseHistoryLoad(const char* filename) { - FILE* fp = fopen(filename, "rt"); - if (fp == NULL) { - return -1; - } - - char buf[LINENOISE_MAX_LINE]; - while (fgets(buf, LINENOISE_MAX_LINE, fp) != NULL) { - char* p = strchr(buf, '\r'); - if (!p) { - p = strchr(buf, '\n'); - } - if (p) { - *p = '\0'; - } - if (p != buf) { - linenoiseHistoryAdd(buf); - } - } - fclose(fp); - return 0; -} - -/* Set if to use or not the multi line mode. */ -/* note that this is a stub only, as linenoise-ng always multi-line */ -void linenoiseSetMultiLine(int) {} - -/* This special mode is used by linenoise in order to print scan codes - * on screen for debugging / development purposes. It is implemented - * by the linenoise_example program using the --keycodes option. */ -void linenoisePrintKeyCodes(void) { - char quit[4]; - - printf( - "Linenoise key codes debugging mode.\n" - "Press keys to see scan codes. Type 'quit' at any time to exit.\n"); - if (enableRawMode() == -1) return; - memset(quit, ' ', 4); - while (1) { - char c; - int nread; - -#if _WIN32 - nread = _read(STDIN_FILENO, &c, 1); -#else - nread = read(STDIN_FILENO, &c, 1); -#endif - if (nread <= 0) continue; - memmove(quit, quit + 1, sizeof(quit) - 1); /* shift string to left. */ - quit[sizeof(quit) - 1] = c; /* Insert current char on the right. */ - if (memcmp(quit, "quit", sizeof(quit)) == 0) break; - - printf("'%c' %02x (%d) (type quit to exit)\n", isprint(c) ? c : '?', (int)c, - (int)c); - printf("\r"); /* Go left edge manually, we are in raw mode. */ - fflush(stdout); - } - disableRawMode(); -} - -#ifndef _WIN32 -static void WindowSizeChanged(int) { - // do nothing here but setting this flag - gotResize = true; -} -#endif - -int linenoiseInstallWindowChangeHandler(void) { -#ifndef _WIN32 - struct sigaction sa; - sigemptyset(&sa.sa_mask); - sa.sa_flags = 0; - sa.sa_handler = &WindowSizeChanged; - - if (sigaction(SIGWINCH, &sa, nullptr) == -1) { - return errno; - } -#endif - return 0; -} - -int linenoiseKeyType(void) { - return keyType; -} diff --git a/src/linenoise/linenoise.h b/src/linenoise/linenoise.h deleted file mode 100644 index 3a8eb9f7ee63..000000000000 --- a/src/linenoise/linenoise.h +++ /dev/null @@ -1,73 +0,0 @@ -/* linenoise.h -- guerrilla line editing library against the idea that a - * line editing lib needs to be 20,000 lines of C code. - * - * See linenoise.c for more information. - * - * Copyright (c) 2010, Salvatore Sanfilippo <antirez at gmail dot com> - * Copyright (c) 2010, Pieter Noordhuis <pcnoordhuis at gmail dot com> - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Redis nor the names of its contributors may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef __LINENOISE_H -#define __LINENOISE_H - -#define LINENOISE_VERSION "1.0.0" -#define LINENOISE_VERSION_MAJOR 1 -#define LINENOISE_VERSION_MINOR 1 - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct linenoiseCompletions linenoiseCompletions; - -typedef void(linenoiseCompletionCallback)(const char*, linenoiseCompletions*); -void linenoiseSetCompletionCallback(linenoiseCompletionCallback* fn); -void linenoiseAddCompletion(linenoiseCompletions* lc, const char* str); - -char* linenoise(const char* prompt); -void linenoisePreloadBuffer(const char* preloadText); -int linenoiseHistoryAdd(const char* line); -int linenoiseHistorySetMaxLen(int len); -char* linenoiseHistoryLine(int index); -int linenoiseHistorySave(const char* filename); -int linenoiseHistoryLoad(const char* filename); -void linenoiseHistoryFree(void); -void linenoiseClearScreen(void); -void linenoiseSetMultiLine(int ml); -void linenoisePrintKeyCodes(void); -/* the following are extensions to the original linenoise API */ -int linenoiseInstallWindowChangeHandler(void); -/* returns type of key pressed: 1 = CTRL-C, 2 = CTRL-D, 0 = other */ -int linenoiseKeyType(void); - -#ifdef __cplusplus -} -#endif - -#endif /* __LINENOISE_H */ diff --git a/src/linenoise/wcwidth.cpp b/src/linenoise/wcwidth.cpp deleted file mode 100644 index deec0ba6b57f..000000000000 --- a/src/linenoise/wcwidth.cpp +++ /dev/null @@ -1,315 +0,0 @@ -/* - * This is an implementation of wcwidth() and wcswidth() (defined in - * IEEE Std 1002.1-2001) for Unicode. - * - * http://www.opengroup.org/onlinepubs/007904975/functions/wcwidth.html - * http://www.opengroup.org/onlinepubs/007904975/functions/wcswidth.html - * - * In fixed-width output devices, Latin characters all occupy a single - * "cell" position of equal width, whereas ideographic CJK characters - * occupy two such cells. Interoperability between terminal-line - * applications and (teletype-style) character terminals using the - * UTF-8 encoding requires agreement on which character should advance - * the cursor by how many cell positions. No established formal - * standards exist at present on which Unicode character shall occupy - * how many cell positions on character terminals. These routines are - * a first attempt of defining such behavior based on simple rules - * applied to data provided by the Unicode Consortium. - * - * For some graphical characters, the Unicode standard explicitly - * defines a character-cell width via the definition of the East Asian - * FullWidth (F), Wide (W), Half-width (H), and Narrow (Na) classes. - * In all these cases, there is no ambiguity about which width a - * terminal shall use. For characters in the East Asian Ambiguous (A) - * class, the width choice depends purely on a preference of backward - * compatibility with either historic CJK or Western practice. - * Choosing single-width for these characters is easy to justify as - * the appropriate long-term solution, as the CJK practice of - * displaying these characters as double-width comes from historic - * implementation simplicity (8-bit encoded characters were displayed - * single-width and 16-bit ones double-width, even for Greek, - * Cyrillic, etc.) and not any typographic considerations. - * - * Much less clear is the choice of width for the Not East Asian - * (Neutral) class. Existing practice does not dictate a width for any - * of these characters. It would nevertheless make sense - * typographically to allocate two character cells to characters such - * as for instance EM SPACE or VOLUME INTEGRAL, which cannot be - * represented adequately with a single-width glyph. The following - * routines at present merely assign a single-cell width to all - * neutral characters, in the interest of simplicity. This is not - * entirely satisfactory and should be reconsidered before - * establishing a formal standard in this area. At the moment, the - * decision which Not East Asian (Neutral) characters should be - * represented by double-width glyphs cannot yet be answered by - * applying a simple rule from the Unicode database content. Setting - * up a proper standard for the behavior of UTF-8 character terminals - * will require a careful analysis not only of each Unicode character, - * but also of each presentation form, something the author of these - * routines has avoided to do so far. - * - * http://www.unicode.org/unicode/reports/tr11/ - * - * Markus Kuhn -- 2007-05-26 (Unicode 5.0) - * - * Permission to use, copy, modify, and distribute this software - * for any purpose and without fee is hereby granted. The author - * disclaims all warranties with regard to this software. - * - * Latest version: http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c - */ - -#include <wchar.h> -#include <string> -#include <memory> - -namespace linenoise_ng { - -struct interval { - char32_t first; - char32_t last; -}; - -/* auxiliary function for binary search in interval table */ -static int bisearch(char32_t ucs, const struct interval *table, int max) { - int min = 0; - int mid; - - if (ucs < table[0].first || ucs > table[max].last) - return 0; - while (max >= min) { - mid = (min + max) / 2; - if (ucs > table[mid].last) - min = mid + 1; - else if (ucs < table[mid].first) - max = mid - 1; - else - return 1; - } - - return 0; -} - - -/* The following two functions define the column width of an ISO 10646 - * character as follows: - * - * - The null character (U+0000) has a column width of 0. - * - * - Other C0/C1 control characters and DEL will lead to a return - * value of -1. - * - * - Non-spacing and enclosing combining characters (general - * category code Mn or Me in the Unicode database) have a - * column width of 0. - * - * - SOFT HYPHEN (U+00AD) has a column width of 1. - * - * - Other format characters (general category code Cf in the Unicode - * database) and ZERO WIDTH SPACE (U+200B) have a column width of 0. - * - * - Hangul Jamo medial vowels and final consonants (U+1160-U+11FF) - * have a column width of 0. - * - * - Spacing characters in the East Asian Wide (W) or East Asian - * Full-width (F) category as defined in Unicode Technical - * Report #11 have a column width of 2. - * - * - All remaining characters (including all printable - * ISO 8859-1 and WGL4 characters, Unicode control characters, - * etc.) have a column width of 1. - * - * This implementation assumes that wchar_t characters are encoded - * in ISO 10646. - */ - -int mk_wcwidth(char32_t ucs) -{ - /* sorted list of non-overlapping intervals of non-spacing characters */ - /* generated by "uniset +cat=Me +cat=Mn +cat=Cf -00AD +1160-11FF +200B c" */ - static const struct interval combining[] = { - { 0x0300, 0x036F }, { 0x0483, 0x0486 }, { 0x0488, 0x0489 }, - { 0x0591, 0x05BD }, { 0x05BF, 0x05BF }, { 0x05C1, 0x05C2 }, - { 0x05C4, 0x05C5 }, { 0x05C7, 0x05C7 }, { 0x0600, 0x0603 }, - { 0x0610, 0x0615 }, { 0x064B, 0x065E }, { 0x0670, 0x0670 }, - { 0x06D6, 0x06E4 }, { 0x06E7, 0x06E8 }, { 0x06EA, 0x06ED }, - { 0x070F, 0x070F }, { 0x0711, 0x0711 }, { 0x0730, 0x074A }, - { 0x07A6, 0x07B0 }, { 0x07EB, 0x07F3 }, { 0x0901, 0x0902 }, - { 0x093C, 0x093C }, { 0x0941, 0x0948 }, { 0x094D, 0x094D }, - { 0x0951, 0x0954 }, { 0x0962, 0x0963 }, { 0x0981, 0x0981 }, - { 0x09BC, 0x09BC }, { 0x09C1, 0x09C4 }, { 0x09CD, 0x09CD }, - { 0x09E2, 0x09E3 }, { 0x0A01, 0x0A02 }, { 0x0A3C, 0x0A3C }, - { 0x0A41, 0x0A42 }, { 0x0A47, 0x0A48 }, { 0x0A4B, 0x0A4D }, - { 0x0A70, 0x0A71 }, { 0x0A81, 0x0A82 }, { 0x0ABC, 0x0ABC }, - { 0x0AC1, 0x0AC5 }, { 0x0AC7, 0x0AC8 }, { 0x0ACD, 0x0ACD }, - { 0x0AE2, 0x0AE3 }, { 0x0B01, 0x0B01 }, { 0x0B3C, 0x0B3C }, - { 0x0B3F, 0x0B3F }, { 0x0B41, 0x0B43 }, { 0x0B4D, 0x0B4D }, - { 0x0B56, 0x0B56 }, { 0x0B82, 0x0B82 }, { 0x0BC0, 0x0BC0 }, - { 0x0BCD, 0x0BCD }, { 0x0C3E, 0x0C40 }, { 0x0C46, 0x0C48 }, - { 0x0C4A, 0x0C4D }, { 0x0C55, 0x0C56 }, { 0x0CBC, 0x0CBC }, - { 0x0CBF, 0x0CBF }, { 0x0CC6, 0x0CC6 }, { 0x0CCC, 0x0CCD }, - { 0x0CE2, 0x0CE3 }, { 0x0D41, 0x0D43 }, { 0x0D4D, 0x0D4D }, - { 0x0DCA, 0x0DCA }, { 0x0DD2, 0x0DD4 }, { 0x0DD6, 0x0DD6 }, - { 0x0E31, 0x0E31 }, { 0x0E34, 0x0E3A }, { 0x0E47, 0x0E4E }, - { 0x0EB1, 0x0EB1 }, { 0x0EB4, 0x0EB9 }, { 0x0EBB, 0x0EBC }, - { 0x0EC8, 0x0ECD }, { 0x0F18, 0x0F19 }, { 0x0F35, 0x0F35 }, - { 0x0F37, 0x0F37 }, { 0x0F39, 0x0F39 }, { 0x0F71, 0x0F7E }, - { 0x0F80, 0x0F84 }, { 0x0F86, 0x0F87 }, { 0x0F90, 0x0F97 }, - { 0x0F99, 0x0FBC }, { 0x0FC6, 0x0FC6 }, { 0x102D, 0x1030 }, - { 0x1032, 0x1032 }, { 0x1036, 0x1037 }, { 0x1039, 0x1039 }, - { 0x1058, 0x1059 }, { 0x1160, 0x11FF }, { 0x135F, 0x135F }, - { 0x1712, 0x1714 }, { 0x1732, 0x1734 }, { 0x1752, 0x1753 }, - { 0x1772, 0x1773 }, { 0x17B4, 0x17B5 }, { 0x17B7, 0x17BD }, - { 0x17C6, 0x17C6 }, { 0x17C9, 0x17D3 }, { 0x17DD, 0x17DD }, - { 0x180B, 0x180D }, { 0x18A9, 0x18A9 }, { 0x1920, 0x1922 }, - { 0x1927, 0x1928 }, { 0x1932, 0x1932 }, { 0x1939, 0x193B }, - { 0x1A17, 0x1A18 }, { 0x1B00, 0x1B03 }, { 0x1B34, 0x1B34 }, - { 0x1B36, 0x1B3A }, { 0x1B3C, 0x1B3C }, { 0x1B42, 0x1B42 }, - { 0x1B6B, 0x1B73 }, { 0x1DC0, 0x1DCA }, { 0x1DFE, 0x1DFF }, - { 0x200B, 0x200F }, { 0x202A, 0x202E }, { 0x2060, 0x2063 }, - { 0x206A, 0x206F }, { 0x20D0, 0x20EF }, { 0x302A, 0x302F }, - { 0x3099, 0x309A }, { 0xA806, 0xA806 }, { 0xA80B, 0xA80B }, - { 0xA825, 0xA826 }, { 0xFB1E, 0xFB1E }, { 0xFE00, 0xFE0F }, - { 0xFE20, 0xFE23 }, { 0xFEFF, 0xFEFF }, { 0xFFF9, 0xFFFB }, - { 0x10A01, 0x10A03 }, { 0x10A05, 0x10A06 }, { 0x10A0C, 0x10A0F }, - { 0x10A38, 0x10A3A }, { 0x10A3F, 0x10A3F }, { 0x1D167, 0x1D169 }, - { 0x1D173, 0x1D182 }, { 0x1D185, 0x1D18B }, { 0x1D1AA, 0x1D1AD }, - { 0x1D242, 0x1D244 }, { 0xE0001, 0xE0001 }, { 0xE0020, 0xE007F }, - { 0xE0100, 0xE01EF } - }; - - /* test for 8-bit control characters */ - if (ucs == 0) - return 0; - if (ucs < 32 || (ucs >= 0x7f && ucs < 0xa0)) - return -1; - - /* binary search in table of non-spacing characters */ - if (bisearch(ucs, combining, - sizeof(combining) / sizeof(struct interval) - 1)) - return 0; - - /* if we arrive here, ucs is not a combining or C0/C1 control character */ - - return 1 + - (ucs >= 0x1100 && - (ucs <= 0x115f || /* Hangul Jamo init. consonants */ - ucs == 0x2329 || ucs == 0x232a || - (ucs >= 0x2e80 && ucs <= 0xa4cf && - ucs != 0x303f) || /* CJK ... Yi */ - (ucs >= 0xac00 && ucs <= 0xd7a3) || /* Hangul Syllables */ - (ucs >= 0xf900 && ucs <= 0xfaff) || /* CJK Compatibility Ideographs */ - (ucs >= 0xfe10 && ucs <= 0xfe19) || /* Vertical forms */ - (ucs >= 0xfe30 && ucs <= 0xfe6f) || /* CJK Compatibility Forms */ - (ucs >= 0xff00 && ucs <= 0xff60) || /* Fullwidth Forms */ - (ucs >= 0xffe0 && ucs <= 0xffe6) || - (ucs >= 0x20000 && ucs <= 0x2fffd) || - (ucs >= 0x30000 && ucs <= 0x3fffd))); -} - - -int mk_wcswidth(const char32_t* pwcs, size_t n) -{ - int w, width = 0; - - for (;*pwcs && n-- > 0; pwcs++) - if ((w = mk_wcwidth(*pwcs)) < 0) - return -1; - else - width += w; - - return width; -} - - -/* - * The following functions are the same as mk_wcwidth() and - * mk_wcswidth(), except that spacing characters in the East Asian - * Ambiguous (A) category as defined in Unicode Technical Report #11 - * have a column width of 2. This variant might be useful for users of - * CJK legacy encodings who want to migrate to UCS without changing - * the traditional terminal character-width behaviour. It is not - * otherwise recommended for general use. - */ -int mk_wcwidth_cjk(wchar_t ucs) -{ - /* sorted list of non-overlapping intervals of East Asian Ambiguous - * characters, generated by "uniset +WIDTH-A -cat=Me -cat=Mn -cat=Cf c" */ - static const struct interval ambiguous[] = { - { 0x00A1, 0x00A1 }, { 0x00A4, 0x00A4 }, { 0x00A7, 0x00A8 }, - { 0x00AA, 0x00AA }, { 0x00AE, 0x00AE }, { 0x00B0, 0x00B4 }, - { 0x00B6, 0x00BA }, { 0x00BC, 0x00BF }, { 0x00C6, 0x00C6 }, - { 0x00D0, 0x00D0 }, { 0x00D7, 0x00D8 }, { 0x00DE, 0x00E1 }, - { 0x00E6, 0x00E6 }, { 0x00E8, 0x00EA }, { 0x00EC, 0x00ED }, - { 0x00F0, 0x00F0 }, { 0x00F2, 0x00F3 }, { 0x00F7, 0x00FA }, - { 0x00FC, 0x00FC }, { 0x00FE, 0x00FE }, { 0x0101, 0x0101 }, - { 0x0111, 0x0111 }, { 0x0113, 0x0113 }, { 0x011B, 0x011B }, - { 0x0126, 0x0127 }, { 0x012B, 0x012B }, { 0x0131, 0x0133 }, - { 0x0138, 0x0138 }, { 0x013F, 0x0142 }, { 0x0144, 0x0144 }, - { 0x0148, 0x014B }, { 0x014D, 0x014D }, { 0x0152, 0x0153 }, - { 0x0166, 0x0167 }, { 0x016B, 0x016B }, { 0x01CE, 0x01CE }, - { 0x01D0, 0x01D0 }, { 0x01D2, 0x01D2 }, { 0x01D4, 0x01D4 }, - { 0x01D6, 0x01D6 }, { 0x01D8, 0x01D8 }, { 0x01DA, 0x01DA }, - { 0x01DC, 0x01DC }, { 0x0251, 0x0251 }, { 0x0261, 0x0261 }, - { 0x02C4, 0x02C4 }, { 0x02C7, 0x02C7 }, { 0x02C9, 0x02CB }, - { 0x02CD, 0x02CD }, { 0x02D0, 0x02D0 }, { 0x02D8, 0x02DB }, - { 0x02DD, 0x02DD }, { 0x02DF, 0x02DF }, { 0x0391, 0x03A1 }, - { 0x03A3, 0x03A9 }, { 0x03B1, 0x03C1 }, { 0x03C3, 0x03C9 }, - { 0x0401, 0x0401 }, { 0x0410, 0x044F }, { 0x0451, 0x0451 }, - { 0x2010, 0x2010 }, { 0x2013, 0x2016 }, { 0x2018, 0x2019 }, - { 0x201C, 0x201D }, { 0x2020, 0x2022 }, { 0x2024, 0x2027 }, - { 0x2030, 0x2030 }, { 0x2032, 0x2033 }, { 0x2035, 0x2035 }, - { 0x203B, 0x203B }, { 0x203E, 0x203E }, { 0x2074, 0x2074 }, - { 0x207F, 0x207F }, { 0x2081, 0x2084 }, { 0x20AC, 0x20AC }, - { 0x2103, 0x2103 }, { 0x2105, 0x2105 }, { 0x2109, 0x2109 }, - { 0x2113, 0x2113 }, { 0x2116, 0x2116 }, { 0x2121, 0x2122 }, - { 0x2126, 0x2126 }, { 0x212B, 0x212B }, { 0x2153, 0x2154 }, - { 0x215B, 0x215E }, { 0x2160, 0x216B }, { 0x2170, 0x2179 }, - { 0x2190, 0x2199 }, { 0x21B8, 0x21B9 }, { 0x21D2, 0x21D2 }, - { 0x21D4, 0x21D4 }, { 0x21E7, 0x21E7 }, { 0x2200, 0x2200 }, - { 0x2202, 0x2203 }, { 0x2207, 0x2208 }, { 0x220B, 0x220B }, - { 0x220F, 0x220F }, { 0x2211, 0x2211 }, { 0x2215, 0x2215 }, - { 0x221A, 0x221A }, { 0x221D, 0x2220 }, { 0x2223, 0x2223 }, - { 0x2225, 0x2225 }, { 0x2227, 0x222C }, { 0x222E, 0x222E }, - { 0x2234, 0x2237 }, { 0x223C, 0x223D }, { 0x2248, 0x2248 }, - { 0x224C, 0x224C }, { 0x2252, 0x2252 }, { 0x2260, 0x2261 }, - { 0x2264, 0x2267 }, { 0x226A, 0x226B }, { 0x226E, 0x226F }, - { 0x2282, 0x2283 }, { 0x2286, 0x2287 }, { 0x2295, 0x2295 }, - { 0x2299, 0x2299 }, { 0x22A5, 0x22A5 }, { 0x22BF, 0x22BF }, - { 0x2312, 0x2312 }, { 0x2460, 0x24E9 }, { 0x24EB, 0x254B }, - { 0x2550, 0x2573 }, { 0x2580, 0x258F }, { 0x2592, 0x2595 }, - { 0x25A0, 0x25A1 }, { 0x25A3, 0x25A9 }, { 0x25B2, 0x25B3 }, - { 0x25B6, 0x25B7 }, { 0x25BC, 0x25BD }, { 0x25C0, 0x25C1 }, - { 0x25C6, 0x25C8 }, { 0x25CB, 0x25CB }, { 0x25CE, 0x25D1 }, - { 0x25E2, 0x25E5 }, { 0x25EF, 0x25EF }, { 0x2605, 0x2606 }, - { 0x2609, 0x2609 }, { 0x260E, 0x260F }, { 0x2614, 0x2615 }, - { 0x261C, 0x261C }, { 0x261E, 0x261E }, { 0x2640, 0x2640 }, - { 0x2642, 0x2642 }, { 0x2660, 0x2661 }, { 0x2663, 0x2665 }, - { 0x2667, 0x266A }, { 0x266C, 0x266D }, { 0x266F, 0x266F }, - { 0x273D, 0x273D }, { 0x2776, 0x277F }, { 0xE000, 0xF8FF }, - { 0xFFFD, 0xFFFD }, { 0xF0000, 0xFFFFD }, { 0x100000, 0x10FFFD } - }; - - /* binary search in table of non-spacing characters */ - if (bisearch(ucs, ambiguous, - sizeof(ambiguous) / sizeof(struct interval) - 1)) - return 2; - - return mk_wcwidth(ucs); -} - - -int mk_wcswidth_cjk(const wchar_t *pwcs, size_t n) -{ - int w, width = 0; - - for (;*pwcs && n-- > 0; pwcs++) - if ((w = mk_wcwidth_cjk(*pwcs)) < 0) - return -1; - else - width += w; - - return width; -} - -} diff --git a/src/nix-build/local.mk b/src/nix-build/local.mk deleted file mode 100644 index a2d1c91dfd9d..000000000000 --- a/src/nix-build/local.mk +++ /dev/null @@ -1,9 +0,0 @@ -programs += nix-build - -nix-build_DIR := $(d) - -nix-build_SOURCES := $(d)/nix-build.cc - -nix-build_LIBS = libmain libexpr libstore libutil libformat - -$(eval $(call install-symlink, nix-build, $(bindir)/nix-shell)) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index a63b3e07ae77..c6a4d416648f 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -16,6 +16,7 @@ #include "get-drvs.hh" #include "common-eval-args.hh" #include "attr-path.hh" +#include "legacy.hh" using namespace nix; using namespace std::string_literals; @@ -66,11 +67,8 @@ std::vector<string> shellwords(const string & s) return res; } -void mainWrapped(int argc, char * * argv) +static void _main(int argc, char * * argv) { - initNix(); - initGC(); - auto dryRun = false; auto runEnv = std::regex_search(argv[0], std::regex("nix-shell$")); auto pure = false; @@ -85,7 +83,6 @@ void mainWrapped(int argc, char * * argv) BuildMode buildMode = bmNormal; bool readStdin = false; - auto shell = getEnv("SHELL", "/bin/sh"); std::string envCommand; // interactive shell Strings envExclude; @@ -99,6 +96,9 @@ void mainWrapped(int argc, char * * argv) std::string outLink = "./result"; + // List of environment variables kept for --pure + std::set<string> keepVars{"HOME", "USER", "LOGNAME", "DISPLAY", "PATH", "TERM", "IN_NIX_SHELL", "TZ", "PAGER", "NIX_BUILD_SHELL", "SHLVL"}; + Strings args; for (int i = 1; i < argc; ++i) args.push_back(argv[i]); @@ -218,6 +218,9 @@ void mainWrapped(int argc, char * * argv) } } + else if (*arg == "--keep") + keepVars.insert(getArg(*arg, arg, end)); + else if (*arg == "-") readStdin = true; @@ -239,10 +242,10 @@ void mainWrapped(int argc, char * * argv) auto store = openStore(); - EvalState state(myArgs.searchPath, store); - state.repair = repair; + auto state = std::make_unique<EvalState>(myArgs.searchPath, store); + state->repair = repair; - Bindings & autoArgs = *myArgs.getAutoArgs(state); + Bindings & autoArgs = *myArgs.getAutoArgs(*state); if (packages) { std::ostringstream joined; @@ -268,22 +271,24 @@ void mainWrapped(int argc, char * * argv) std::vector<Expr *> exprs; if (readStdin) - exprs = {state.parseStdin()}; + exprs = {state->parseStdin()}; else for (auto i : left) { - auto absolute = i; - try { - absolute = canonPath(absPath(i), true); - } catch (Error e) {}; if (fromArgs) - exprs.push_back(state.parseExprFromString(i, absPath("."))); - else if (store->isStorePath(absolute) && std::regex_match(absolute, std::regex(".*\\.drv(!.*)?"))) - drvs.push_back(DrvInfo(state, store, absolute)); + exprs.push_back(state->parseExprFromString(i, absPath("."))); + else { + auto absolute = i; + try { + absolute = canonPath(absPath(i), true); + } catch (Error e) {}; + if (store->isStorePath(absolute) && std::regex_match(absolute, std::regex(".*\\.drv(!.*)?"))) + drvs.push_back(DrvInfo(*state, store, absolute)); else /* If we're in a #! script, interpret filenames relative to the script. */ - exprs.push_back(state.parseExprFromFile(resolveExprPath(state.checkSourcePath(lookupFileArg(state, + exprs.push_back(state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state, inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i))))); + } } /* Evaluate them into derivations. */ @@ -291,15 +296,17 @@ void mainWrapped(int argc, char * * argv) for (auto e : exprs) { Value vRoot; - state.eval(e, vRoot); + state->eval(e, vRoot); for (auto & i : attrPaths) { - Value & v(*findAlongAttrPath(state, i, autoArgs, vRoot)); - state.forceValue(v); - getDerivations(state, v, "", autoArgs, drvs, false); + Value & v(*findAlongAttrPath(*state, i, autoArgs, vRoot)); + state->forceValue(v); + getDerivations(*state, v, "", autoArgs, drvs, false); } } + state->printStats(); + auto buildPaths = [&](const PathSet & paths) { /* Note: we do this even when !printMissing to efficiently fetch binary cache data. */ @@ -332,12 +339,12 @@ void mainWrapped(int argc, char * * argv) if (shell == "") { try { - auto expr = state.parseExprFromString("(import <nixpkgs> {}).bashInteractive", absPath(".")); + auto expr = state->parseExprFromString("(import <nixpkgs> {}).bashInteractive", absPath(".")); Value v; - state.eval(expr, v); + state->eval(expr, v); - auto drv = getDerivation(state, v, false); + auto drv = getDerivation(*state, v, false); if (!drv) throw Error("the 'bashInteractive' attribute in <nixpkgs> did not evaluate to a derivation"); @@ -354,7 +361,7 @@ void mainWrapped(int argc, char * * argv) // Build or fetch all dependencies of the derivation. for (const auto & input : drv.inputDrvs) if (std::all_of(envExclude.cbegin(), envExclude.cend(), [&](const string & exclude) { return !std::regex_search(input.first, std::regex(exclude)); })) - pathsToBuild.insert(input.first); + pathsToBuild.insert(makeDrvPathWithOutputs(input.first, input.second)); for (const auto & src : drv.inputSrcs) pathsToBuild.insert(src); @@ -368,7 +375,6 @@ void mainWrapped(int argc, char * * argv) auto tmp = getEnv("TMPDIR", getEnv("XDG_RUNTIME_DIR", "/tmp")); if (pure) { - std::set<string> keepVars{"HOME", "USER", "LOGNAME", "DISPLAY", "PATH", "TERM", "IN_NIX_SHELL", "TZ", "PAGER", "NIX_BUILD_SHELL", "SHLVL"}; decltype(env) newEnv; for (auto & i : env) if (keepVars.count(i.first)) @@ -411,17 +417,20 @@ void mainWrapped(int argc, char * * argv) "dontAddDisableDepTrack=1; " "[ -e $stdenv/setup ] && source $stdenv/setup; " "%3%" + "PATH=\"%4%:$PATH\"; " + "SHELL=%5%; " "set +e; " R"s([ -n "$PS1" ] && PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s" "if [ \"$(type -t runHook)\" = function ]; then runHook shellHook; fi; " "unset NIX_ENFORCE_PURITY; " - "unset NIX_INDENT_MAKE; " "shopt -u nullglob; " - "unset TZ; %4%" - "%5%", + "unset TZ; %6%" + "%7%", (Path) tmpDir, (pure ? "" : "p=$PATH; "), (pure ? "" : "PATH=$PATH:$p; unset p; "), + dirOf(shell), + shell, (getenv("TZ") ? (string("export TZ='") + getenv("TZ") + "'; ") : ""), envCommand)); @@ -495,9 +504,5 @@ void mainWrapped(int argc, char * * argv) } } -int main(int argc, char * * argv) -{ - return handleExceptions(argv[0], [&]() { - return mainWrapped(argc, argv); - }); -} +static RegisterLegacyCommand s1("nix-build", _main); +static RegisterLegacyCommand s2("nix-shell", _main); diff --git a/src/nix-channel/local.mk b/src/nix-channel/local.mk deleted file mode 100644 index 49fc105c6f79..000000000000 --- a/src/nix-channel/local.mk +++ /dev/null @@ -1,7 +0,0 @@ -programs += nix-channel - -nix-channel_DIR := $(d) - -nix-channel_LIBS = libmain libutil libformat libstore - -nix-channel_SOURCES := $(d)/nix-channel.cc diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index d1b47ede803f..8b66cc7e314e 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -1,9 +1,11 @@ #include "shared.hh" #include "globals.hh" #include "download.hh" +#include "store-api.hh" +#include "legacy.hh" + #include <fcntl.h> #include <regex> -#include "store-api.hh" #include <pwd.h> using namespace nix; @@ -86,7 +88,7 @@ static void update(const StringSet & channelNames) // definition from a consistent location if the redirect changes mid-download. std::string effectiveUrl; auto dl = getDownloader(); - auto filename = dl->downloadCached(store, url, false, "", Hash(), &effectiveUrl); + auto filename = dl->downloadCached(store, url, false, "", Hash(), &effectiveUrl, 0); url = chomp(std::move(effectiveUrl)); // If the URL contains a version number, append it to the name @@ -157,11 +159,9 @@ static void update(const StringSet & channelNames) replaceSymlink(profile, channelLink); } -int main(int argc, char ** argv) +static int _main(int argc, char ** argv) { - return handleExceptions(argv[0], [&]() { - initNix(); - + { // Figure out the name of the `.nix-channels' file to use auto home = getHome(); channelsList = home + "/.nix-channels"; @@ -169,7 +169,7 @@ int main(int argc, char ** argv) // Figure out the name of the channels profile. ; - auto pw = getpwuid(getuid()); + auto pw = getpwuid(geteuid()); std::string name = pw ? pw->pw_name : getEnv("USER", ""); if (name.empty()) throw Error("cannot figure out user name"); @@ -255,5 +255,9 @@ int main(int argc, char ** argv) runProgram(settings.nixBinDir + "/nix-env", false, envArgs); break; } - }); + + return 0; + } } + +static RegisterLegacyCommand s1("nix-channel", _main); diff --git a/src/nix-collect-garbage/local.mk b/src/nix-collect-garbage/local.mk deleted file mode 100644 index 02d14cf62199..000000000000 --- a/src/nix-collect-garbage/local.mk +++ /dev/null @@ -1,7 +0,0 @@ -programs += nix-collect-garbage - -nix-collect-garbage_DIR := $(d) - -nix-collect-garbage_SOURCES := $(d)/nix-collect-garbage.cc - -nix-collect-garbage_LIBS = libmain libstore libutil libformat diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index 37fe22f48134..d4060ac937fc 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -2,6 +2,7 @@ #include "profiles.hh" #include "shared.hh" #include "globals.hh" +#include "legacy.hh" #include <iostream> #include <cerrno> @@ -48,12 +49,10 @@ void removeOldGenerations(std::string dir) } } -int main(int argc, char * * argv) +static int _main(int argc, char * * argv) { - bool removeOld = false; - - return handleExceptions(argv[0], [&]() { - initNix(); + { + bool removeOld = false; GCOptions options; @@ -90,5 +89,9 @@ int main(int argc, char * * argv) PrintFreed freed(true, results); store->collectGarbage(options, results); } - }); + + return 0; + } } + +static RegisterLegacyCommand s1("nix-collect-garbage", _main); diff --git a/src/nix-copy-closure/local.mk b/src/nix-copy-closure/local.mk deleted file mode 100644 index 42bb34dd8201..000000000000 --- a/src/nix-copy-closure/local.mk +++ /dev/null @@ -1,7 +0,0 @@ -programs += nix-copy-closure - -nix-copy-closure_DIR := $(d) - -nix-copy-closure_LIBS = libmain libutil libformat libstore - -nix-copy-closure_SOURCES := $(d)/nix-copy-closure.cc diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc index dfb1b8fc5dc4..fdcde8b076b5 100755 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -1,13 +1,12 @@ #include "shared.hh" #include "store-api.hh" +#include "legacy.hh" using namespace nix; -int main(int argc, char ** argv) +static int _main(int argc, char ** argv) { - return handleExceptions(argv[0], [&]() { - initNix(); - + { auto gzip = false; auto toMode = true; auto includeOutputs = false; @@ -61,5 +60,9 @@ int main(int argc, char ** argv) from->computeFSClosure(storePaths2, closure, false, includeOutputs); copyPaths(from, to, closure, NoRepair, NoCheckSigs, useSubstitutes); - }); + + return 0; + } } + +static RegisterLegacyCommand s1("nix-copy-closure", _main); diff --git a/src/nix-daemon/local.mk b/src/nix-daemon/local.mk deleted file mode 100644 index 5a4474465b3c..000000000000 --- a/src/nix-daemon/local.mk +++ /dev/null @@ -1,13 +0,0 @@ -programs += nix-daemon - -nix-daemon_DIR := $(d) - -nix-daemon_SOURCES := $(d)/nix-daemon.cc - -nix-daemon_LIBS = libmain libstore libutil libformat - -nix-daemon_LDFLAGS = -pthread - -ifeq ($(OS), SunOS) - nix-daemon_LDFLAGS += -lsocket -endif diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 35603af7082a..8d63b8f362ec 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -9,6 +9,7 @@ #include "monitor-fd.hh" #include "derivations.hh" #include "finally.hh" +#include "legacy.hh" #include <algorithm> @@ -120,8 +121,6 @@ struct TunnelLogger : public Logger want to send out stderr to the client. */ void startWork() { - std::vector<std::string> pendingMsgs; - auto state(state_.lock()); state->canSendStderr = true; @@ -197,7 +196,8 @@ struct TunnelSource : BufferedSource { Source & from; TunnelSource(Source & from) : from(from) { } - size_t readUnbuffered(unsigned char * data, size_t len) +protected: + size_t readUnbuffered(unsigned char * data, size_t len) override { to << STDERR_READ << len; to.flush(); @@ -234,7 +234,7 @@ struct RetrieveRegularNARSink : ParseSink }; -static void performOp(TunnelLogger * logger, ref<LocalStore> store, +static void performOp(TunnelLogger * logger, ref<Store> store, bool trusted, unsigned int clientVersion, Source & from, Sink & to, unsigned int op) { @@ -363,7 +363,11 @@ static void performOp(TunnelLogger * logger, ref<LocalStore> store, logger->startWork(); if (!savedRegular.regular) throw Error("regular file expected"); - Path path = store->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo); + + auto store2 = store.dynamic_pointer_cast<LocalStore>(); + if (!store2) throw Error("operation is only supported by LocalStore"); + + Path path = store2->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo); logger->stopWork(); to << path; @@ -471,11 +475,19 @@ static void performOp(TunnelLogger * logger, ref<LocalStore> store, case wopFindRoots: { logger->startWork(); - Roots roots = store->findRoots(); + Roots roots = store->findRoots(!trusted); logger->stopWork(); - to << roots.size(); + + size_t size = 0; for (auto & i : roots) - to << i.first << i.second; + size += i.second.size(); + + to << size; + + for (auto & [target, links] : roots) + for (auto & link : links) + to << link << target; + break; } @@ -554,7 +566,8 @@ static void performOp(TunnelLogger * logger, ref<LocalStore> store, ; else if (trusted || name == settings.buildTimeout.name - || name == settings.connectTimeout.name) + || name == "connect-timeout" + || (name == "builders" && value == "")) settings.set(name, value); else if (setSubstituters(settings.substituters)) ; @@ -691,12 +704,23 @@ static void performOp(TunnelLogger * logger, ref<LocalStore> store, if (!trusted) info.ultimate = false; - TeeSink tee(from); - parseDump(tee, tee.source); + std::string saved; + std::unique_ptr<Source> source; + if (GET_PROTOCOL_MINOR(clientVersion) >= 21) + source = std::make_unique<TunnelSource>(from); + else { + TeeSink tee(from); + parseDump(tee, tee.source); + saved = std::move(*tee.source.data); + source = std::make_unique<StringSource>(saved); + } logger->startWork(); - store.cast<Store>()->addToStore(info, tee.source.data, (RepairFlag) repair, + + // FIXME: race if addToStore doesn't read source? + store->addToStore(info, *source, (RepairFlag) repair, dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr); + logger->stopWork(); break; } @@ -767,7 +791,7 @@ static void processConnection(bool trusted) Store::Params params; // FIXME: get params from somewhere // Disable caching since the client already does that. params["path-info-cache-size"] = "0"; - auto store = make_ref<LocalStore>(params); + auto store = openStore(settings.storeUri, params); tunnelLogger->stopWork(); to.flush(); @@ -1043,11 +1067,9 @@ static void daemonLoop(char * * argv) } -int main(int argc, char * * argv) +static int _main(int argc, char * * argv) { - return handleExceptions(argv[0], [&]() { - initNix(); - + { auto stdio = false; parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { @@ -1107,7 +1129,7 @@ int main(int argc, char * * argv) if (res == -1) throw SysError("splicing data from stdin to daemon socket"); else if (res == 0) - return; + return 0; } } } else { @@ -1116,5 +1138,9 @@ int main(int argc, char * * argv) } else { daemonLoop(argv); } - }); + + return 0; + } } + +static RegisterLegacyCommand s1("nix-daemon", _main); diff --git a/src/nix-env/local.mk b/src/nix-env/local.mk deleted file mode 100644 index e80719cd76f7..000000000000 --- a/src/nix-env/local.mk +++ /dev/null @@ -1,7 +0,0 @@ -programs += nix-env - -nix-env_DIR := $(d) - -nix-env_SOURCES := $(wildcard $(d)/*.cc) - -nix-env_LIBS = libexpr libmain libstore libutil libformat diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 97e66cbd937e..56ed75daee44 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -13,6 +13,7 @@ #include "json.hh" #include "value-to-json.hh" #include "xml-writer.hh" +#include "legacy.hh" #include <cerrno> #include <ctime> @@ -150,10 +151,8 @@ static void loadSourceExpr(EvalState & state, const Path & path, Value & v) if (stat(path.c_str(), &st) == -1) throw SysError(format("getting information about '%1%'") % path); - if (isNixExpr(path, st)) { + if (isNixExpr(path, st)) state.evalFile(path, v); - return; - } /* The path is a directory. Put the Nix expressions in the directory in a set, with the file name of each expression as @@ -161,13 +160,15 @@ static void loadSourceExpr(EvalState & state, const Path & path, Value & v) set flat, not nested, to make it easier for a user to have a ~/.nix-defexpr directory that includes some system-wide directory). */ - if (S_ISDIR(st.st_mode)) { + else if (S_ISDIR(st.st_mode)) { state.mkAttrs(v, 1024); state.mkList(*state.allocAttr(v, state.symbols.create("_combineChannels")), 0); StringSet attrs; getAllExprs(state, path, attrs, v); v.attrs->sort(); } + + else throw Error("path '%s' is not a directory or a Nix expression", path); } @@ -198,13 +199,13 @@ static Path getDefNixExprPath() } -static int getPriority(EvalState & state, DrvInfo & drv) +static long getPriority(EvalState & state, DrvInfo & drv) { return drv.queryMetaInt("priority", 0); } -static int comparePriorities(EvalState & state, DrvInfo & drv1, DrvInfo & drv2) +static long comparePriorities(EvalState & state, DrvInfo & drv1, DrvInfo & drv2) { return getPriority(state, drv2) - getPriority(state, drv1); } @@ -270,7 +271,7 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems, for (auto & j : matches) { DrvName drvName(j.first.queryName()); - int d = 1; + long d = 1; Newest::iterator k = newest.find(drvName.name); @@ -578,7 +579,7 @@ static void upgradeDerivations(Globals & globals, (upgradeType == utEq && d == 0) || upgradeType == utAlways) { - int d2 = -1; + long d2 = -1; if (bestElem != availElems.end()) { d2 = comparePriorities(*globals.state, *bestElem, *j); if (d2 == 0) d2 = compareVersions(bestVersion, newName.version); @@ -784,22 +785,22 @@ typedef list<Strings> Table; void printTable(Table & table) { - unsigned int nrColumns = table.size() > 0 ? table.front().size() : 0; + auto nrColumns = table.size() > 0 ? table.front().size() : 0; - vector<unsigned int> widths; + vector<size_t> widths; widths.resize(nrColumns); for (auto & i : table) { assert(i.size() == nrColumns); Strings::iterator j; - unsigned int column; + size_t column; for (j = i.begin(), column = 0; j != i.end(); ++j, ++column) if (j->size() > widths[column]) widths[column] = j->size(); } for (auto & i : table) { Strings::iterator j; - unsigned int column; + size_t column; for (j = i.begin(), column = 0; j != i.end(); ++j, ++column) { string s = *j; replace(s.begin(), s.end(), '\n', ' '); @@ -1284,6 +1285,14 @@ static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opAr deleteOldGenerations(globals.profile, globals.dryRun); } else if (opArgs.size() == 1 && opArgs.front().find('d') != string::npos) { deleteGenerationsOlderThan(globals.profile, opArgs.front(), globals.dryRun); + } else if (opArgs.size() == 1 && opArgs.front().find('+') != string::npos) { + if(opArgs.front().size() < 2) + throw Error(format("invalid number of generations ‘%1%’") % opArgs.front()); + string str_max = string(opArgs.front(), 1, opArgs.front().size()); + int max; + if (!string2Int(str_max, max) || max == 0) + throw Error(format("invalid number of generations to keep ‘%1%’") % opArgs.front()); + deleteGenerationsGreaterThan(globals.profile, max, globals.dryRun); } else { std::set<unsigned int> gens; for (auto & i : opArgs) { @@ -1303,12 +1312,9 @@ static void opVersion(Globals & globals, Strings opFlags, Strings opArgs) } -int main(int argc, char * * argv) +static int _main(int argc, char * * argv) { - return handleExceptions(argv[0], [&]() { - initNix(); - initGC(); - + { Strings opFlags, opArgs; Operation op = 0; RepairFlag repair = NoRepair; @@ -1420,5 +1426,9 @@ int main(int argc, char * * argv) op(globals, opFlags, opArgs); globals.state->printStats(); - }); + + return 0; + } } + +static RegisterLegacyCommand s1("nix-env", _main); diff --git a/src/nix-instantiate/local.mk b/src/nix-instantiate/local.mk deleted file mode 100644 index 7d1bc5ec9dfb..000000000000 --- a/src/nix-instantiate/local.mk +++ /dev/null @@ -1,7 +0,0 @@ -programs += nix-instantiate - -nix-instantiate_DIR := $(d) - -nix-instantiate_SOURCES := $(d)/nix-instantiate.cc - -nix-instantiate_LIBS = libexpr libmain libstore libutil libformat diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 5049460c7544..a736caa8f056 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -9,6 +9,7 @@ #include "util.hh" #include "store-api.hh" #include "common-eval-args.hh" +#include "legacy.hh" #include <map> #include <iostream> @@ -83,12 +84,9 @@ void processExpr(EvalState & state, const Strings & attrPaths, } -int main(int argc, char * * argv) +static int _main(int argc, char * * argv) { - return handleExceptions(argv[0], [&]() { - initNix(); - initGC(); - + { Strings files; bool readStdin = false; bool fromArgs = false; @@ -158,37 +156,41 @@ int main(int argc, char * * argv) auto store = openStore(); - EvalState state(myArgs.searchPath, store); - state.repair = repair; + auto state = std::make_unique<EvalState>(myArgs.searchPath, store); + state->repair = repair; - Bindings & autoArgs = *myArgs.getAutoArgs(state); + Bindings & autoArgs = *myArgs.getAutoArgs(*state); if (attrPaths.empty()) attrPaths = {""}; if (findFile) { for (auto & i : files) { - Path p = state.findFile(i); + Path p = state->findFile(i); if (p == "") throw Error(format("unable to find '%1%'") % i); std::cout << p << std::endl; } - return; + return 0; } if (readStdin) { - Expr * e = state.parseStdin(); - processExpr(state, attrPaths, parseOnly, strict, autoArgs, + Expr * e = state->parseStdin(); + processExpr(*state, attrPaths, parseOnly, strict, autoArgs, evalOnly, outputKind, xmlOutputSourceLocation, e); } else if (files.empty() && !fromArgs) files.push_back("./default.nix"); for (auto & i : files) { Expr * e = fromArgs - ? state.parseExprFromString(i, absPath(".")) - : state.parseExprFromFile(resolveExprPath(state.checkSourcePath(lookupFileArg(state, i)))); - processExpr(state, attrPaths, parseOnly, strict, autoArgs, + ? state->parseExprFromString(i, absPath(".")) + : state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state, i)))); + processExpr(*state, attrPaths, parseOnly, strict, autoArgs, evalOnly, outputKind, xmlOutputSourceLocation, e); } - state.printStats(); - }); + state->printStats(); + + return 0; + } } + +static RegisterLegacyCommand s1("nix-instantiate", _main); diff --git a/src/nix-prefetch-url/local.mk b/src/nix-prefetch-url/local.mk deleted file mode 100644 index 3e7735406af0..000000000000 --- a/src/nix-prefetch-url/local.mk +++ /dev/null @@ -1,7 +0,0 @@ -programs += nix-prefetch-url - -nix-prefetch-url_DIR := $(d) - -nix-prefetch-url_SOURCES := $(d)/nix-prefetch-url.cc - -nix-prefetch-url_LIBS = libmain libexpr libstore libutil libformat diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index fa7ee254500c..f54706a8a011 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -6,9 +6,16 @@ #include "eval-inline.hh" #include "common-eval-args.hh" #include "attr-path.hh" +#include "legacy.hh" +#include "finally.hh" +#include "progress-bar.hh" #include <iostream> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> + using namespace nix; @@ -40,12 +47,9 @@ string resolveMirrorUri(EvalState & state, string uri) } -int main(int argc, char * * argv) +static int _main(int argc, char * * argv) { - return handleExceptions(argv[0], [&]() { - initNix(); - initGC(); - + { HashType ht = htSHA256; std::vector<string> args; bool printPath = getEnv("PRINT_PATH") != ""; @@ -94,10 +98,15 @@ int main(int argc, char * * argv) if (args.size() > 2) throw UsageError("too many arguments"); + Finally f([]() { stopProgressBar(); }); + + if (isatty(STDERR_FILENO)) + startProgressBar(); + auto store = openStore(); - EvalState state(myArgs.searchPath, store); + auto state = std::make_unique<EvalState>(myArgs.searchPath, store); - Bindings & autoArgs = *myArgs.getAutoArgs(state); + Bindings & autoArgs = *myArgs.getAutoArgs(*state); /* If -A is given, get the URI from the specified Nix expression. */ @@ -107,33 +116,33 @@ int main(int argc, char * * argv) throw UsageError("you must specify a URI"); uri = args[0]; } else { - Path path = resolveExprPath(lookupFileArg(state, args.empty() ? "." : args[0])); + Path path = resolveExprPath(lookupFileArg(*state, args.empty() ? "." : args[0])); Value vRoot; - state.evalFile(path, vRoot); - Value & v(*findAlongAttrPath(state, attrPath, autoArgs, vRoot)); - state.forceAttrs(v); + state->evalFile(path, vRoot); + Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot)); + state->forceAttrs(v); /* Extract the URI. */ - auto attr = v.attrs->find(state.symbols.create("urls")); + auto attr = v.attrs->find(state->symbols.create("urls")); if (attr == v.attrs->end()) throw Error("attribute set does not contain a 'urls' attribute"); - state.forceList(*attr->value); + state->forceList(*attr->value); if (attr->value->listSize() < 1) throw Error("'urls' list is empty"); - uri = state.forceString(*attr->value->listElems()[0]); + uri = state->forceString(*attr->value->listElems()[0]); /* Extract the hash mode. */ - attr = v.attrs->find(state.symbols.create("outputHashMode")); + attr = v.attrs->find(state->symbols.create("outputHashMode")); if (attr == v.attrs->end()) printInfo("warning: this does not look like a fetchurl call"); else - unpack = state.forceString(*attr->value) == "recursive"; + unpack = state->forceString(*attr->value) == "recursive"; /* Extract the name. */ if (name.empty()) { - attr = v.attrs->find(state.symbols.create("name")); + attr = v.attrs->find(state->symbols.create("name")); if (attr != v.attrs->end()) - name = state.forceString(*attr->value); + name = state->forceString(*attr->value); } } @@ -158,16 +167,22 @@ int main(int argc, char * * argv) if (storePath.empty()) { - auto actualUri = resolveMirrorUri(state, uri); - - /* Download the file. */ - DownloadRequest req(actualUri); - req.decompress = false; - auto result = getDownloader()->download(req); + auto actualUri = resolveMirrorUri(*state, uri); AutoDelete tmpDir(createTempDir(), true); Path tmpFile = (Path) tmpDir + "/tmp"; - writeFile(tmpFile, *result.data); + + /* Download the file. */ + { + AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0600); + if (!fd) throw SysError("creating temporary file '%s'", tmpFile); + + FdSink sink(fd.get()); + + DownloadRequest req(actualUri); + req.decompress = false; + getDownloader()->download(std::move(req), sink); + } /* Optionally unpack the file. */ if (unpack) { @@ -191,7 +206,7 @@ int main(int argc, char * * argv) /* FIXME: inefficient; addToStore() will also hash this. */ - hash = unpack ? hashPath(ht, tmpFile).first : hashString(ht, *result.data); + hash = unpack ? hashPath(ht, tmpFile).first : hashFile(ht, tmpFile); if (expectedHash != Hash(ht) && expectedHash != hash) throw Error(format("hash mismatch for '%1%'") % uri); @@ -205,11 +220,17 @@ int main(int argc, char * * argv) assert(storePath == store->makeFixedOutputPath(unpack, hash, name)); } + stopProgressBar(); + if (!printPath) printInfo(format("path is '%1%'") % storePath); std::cout << printHash16or32(hash) << std::endl; if (printPath) std::cout << storePath << std::endl; - }); + + return 0; + } } + +static RegisterLegacyCommand s1("nix-prefetch-url", _main); diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc index 51dedcf0a092..abdfa5e58f93 100644 --- a/src/nix-store/dotgraph.cc +++ b/src/nix-store/dotgraph.cc @@ -47,8 +47,7 @@ static string makeNode(const string & id, const string & label, static string symbolicName(const string & path) { string p = baseNameOf(path); - int dash = p.find('-'); - return string(p, dash + 1); + return string(p, p.find('-') + 1); } diff --git a/src/nix-store/graphml.cc b/src/nix-store/graphml.cc new file mode 100644 index 000000000000..670fbe227a4c --- /dev/null +++ b/src/nix-store/graphml.cc @@ -0,0 +1,90 @@ +#include "graphml.hh" +#include "util.hh" +#include "store-api.hh" +#include "derivations.hh" + +#include <iostream> + + +using std::cout; + +namespace nix { + + +static inline const string & xmlQuote(const string & s) +{ + // Luckily, store paths shouldn't contain any character that needs to be + // quoted. + return s; +} + + +static string symbolicName(const string & path) +{ + string p = baseNameOf(path); + return string(p, p.find('-') + 1); +} + + +static string makeEdge(const string & src, const string & dst) +{ + return fmt(" <edge source=\"%1%\" target=\"%2%\"/>\n", + xmlQuote(src), xmlQuote(dst)); +} + + +static string makeNode(const ValidPathInfo & info) +{ + return fmt( + " <node id=\"%1%\">\n" + " <data key=\"narSize\">%2%</data>\n" + " <data key=\"name\">%3%</data>\n" + " <data key=\"type\">%4%</data>\n" + " </node>\n", + info.path, + info.narSize, + symbolicName(info.path), + (isDerivation(info.path) ? "derivation" : "output-path")); +} + + +void printGraphML(ref<Store> store, const PathSet & roots) +{ + PathSet workList(roots); + PathSet doneSet; + std::pair<PathSet::iterator,bool> ret; + + cout << "<?xml version='1.0' encoding='utf-8'?>\n" + << "<graphml xmlns='http://graphml.graphdrawing.org/xmlns'\n" + << " xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'\n" + << " xsi:schemaLocation='http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd'>\n" + << "<key id='narSize' for='node' attr.name='narSize' attr.type='int'/>" + << "<key id='name' for='node' attr.name='name' attr.type='string'/>" + << "<key id='type' for='node' attr.name='type' attr.type='string'/>" + << "<graph id='G' edgedefault='directed'>\n"; + + while (!workList.empty()) { + Path path = *(workList.begin()); + workList.erase(path); + + ret = doneSet.insert(path); + if (ret.second == false) continue; + + ValidPathInfo info = *(store->queryPathInfo(path)); + cout << makeNode(info); + + for (auto & p : store->queryPathInfo(path)->references) { + if (p != path) { + workList.insert(p); + cout << makeEdge(path, p); + } + } + + } + + cout << "</graph>\n"; + cout << "</graphml>\n"; +} + + +} diff --git a/src/nix-store/xmlgraph.hh b/src/nix-store/graphml.hh index a6e7d4e2805a..b78df1e49a67 100644 --- a/src/nix-store/xmlgraph.hh +++ b/src/nix-store/graphml.hh @@ -6,6 +6,6 @@ namespace nix { class Store; -void printXmlGraph(ref<Store> store, const PathSet & roots); +void printGraphML(ref<Store> store, const PathSet & roots); } diff --git a/src/nix-store/local.mk b/src/nix-store/local.mk deleted file mode 100644 index ade0b233adf3..000000000000 --- a/src/nix-store/local.mk +++ /dev/null @@ -1,9 +0,0 @@ -programs += nix-store - -nix-store_DIR := $(d) - -nix-store_SOURCES := $(wildcard $(d)/*.cc) - -nix-store_LIBS = libmain libstore libutil libformat - -nix-store_LDFLAGS = -lbz2 -pthread $(SODIUM_LIBS) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index e1e27ceef94d..f324056bb3a1 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -8,7 +8,8 @@ #include "shared.hh" #include "util.hh" #include "worker-protocol.hh" -#include "xmlgraph.hh" +#include "graphml.hh" +#include "legacy.hh" #include <iostream> #include <algorithm> @@ -273,7 +274,7 @@ static void opQuery(Strings opFlags, Strings opArgs) enum QueryType { qDefault, qOutputs, qRequisites, qReferences, qReferrers , qReferrersClosure, qDeriver, qBinding, qHash, qSize - , qTree, qGraph, qXml, qResolve, qRoots }; + , qTree, qGraph, qGraphML, qResolve, qRoots }; QueryType query = qDefault; bool useOutput = false; bool includeOutputs = false; @@ -299,7 +300,7 @@ static void opQuery(Strings opFlags, Strings opArgs) else if (i == "--size") query = qSize; else if (i == "--tree") query = qTree; else if (i == "--graph") query = qGraph; - else if (i == "--xml") query = qXml; + else if (i == "--graphml") query = qGraphML; else if (i == "--resolve") query = qResolve; else if (i == "--roots") query = qRoots; else if (i == "--use-output" || i == "-u") useOutput = true; @@ -403,13 +404,13 @@ static void opQuery(Strings opFlags, Strings opArgs) break; } - case qXml: { + case qGraphML: { PathSet roots; for (auto & i : opArgs) { PathSet paths = maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise); roots.insert(paths.begin(), paths.end()); } - printXmlGraph(ref<Store>(store), roots); + printGraphML(ref<Store>(store), roots); break; } @@ -426,10 +427,11 @@ static void opQuery(Strings opFlags, Strings opArgs) maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise), referrers, true, settings.gcKeepOutputs, settings.gcKeepDerivations); } - Roots roots = store->findRoots(); - for (auto & i : roots) - if (referrers.find(i.second) != referrers.end()) - cout << format("%1%\n") % i.first; + Roots roots = store->findRoots(false); + for (auto & [target, links] : roots) + if (referrers.find(target) != referrers.end()) + for (auto & link : links) + cout << format("%1% -> %2%\n") % link % target; break; } @@ -484,11 +486,16 @@ static void opReadLog(Strings opFlags, Strings opArgs) static void opDumpDB(Strings opFlags, Strings opArgs) { if (!opFlags.empty()) throw UsageError("unknown flag"); - if (!opArgs.empty()) - throw UsageError("no arguments expected"); - PathSet validPaths = store->queryAllValidPaths(); - for (auto & i : validPaths) - cout << store->makeValidityRegistration({i}, true, true); + if (!opArgs.empty()) { + for (auto & i : opArgs) + i = store->followLinksToStorePath(i); + for (auto & i : opArgs) + cout << store->makeValidityRegistration({i}, true, true); + } else { + PathSet validPaths = store->queryAllValidPaths(); + for (auto & i : validPaths) + cout << store->makeValidityRegistration({i}, true, true); + } } @@ -584,9 +591,14 @@ static void opGC(Strings opFlags, Strings opArgs) if (!opArgs.empty()) throw UsageError("no arguments expected"); if (printRoots) { - Roots roots = store->findRoots(); - for (auto & i : roots) - cout << i.first << " -> " << i.second << std::endl; + Roots roots = store->findRoots(false); + std::set<std::pair<Path, Path>> roots2; + // Transpose and sort the roots. + for (auto & [target, links] : roots) + for (auto & link : links) + roots2.emplace(link, target); + for (auto & [link, target] : roots2) + std::cout << link << " -> " << target << "\n"; } else { @@ -860,7 +872,7 @@ static void opServe(Strings opFlags, Strings opArgs) } case cmdDumpStorePath: - dumpPath(readStorePath(*store, in), out); + store->narFromPath(readStorePath(*store, in), out); break; case cmdImportPaths: { @@ -924,6 +936,28 @@ static void opServe(Strings opFlags, Strings opArgs) break; } + case cmdAddToStoreNar: { + if (!writeAllowed) throw Error("importing paths is not allowed"); + + ValidPathInfo info; + info.path = readStorePath(*store, in); + in >> info.deriver; + if (!info.deriver.empty()) + store->assertStorePath(info.deriver); + info.narHash = Hash(readString(in), htSHA256); + info.references = readStorePaths<PathSet>(*store, in); + in >> info.registrationTime >> info.narSize >> info.ultimate; + info.sigs = readStrings<StringSet>(in); + in >> info.ca; + + // FIXME: race if addToStore doesn't read source? + store->addToStore(info, in, NoRepair, NoCheckSigs); + + out << 1; // indicate success + + break; + } + default: throw Error(format("unknown serve command %1%") % cmd); } @@ -971,11 +1005,9 @@ static void opVersion(Strings opFlags, Strings opArgs) /* Scan the arguments; find the operation, set global flags, put all other flags in a list, and put all other arguments in another list. */ -int main(int argc, char * * argv) +static int _main(int argc, char * * argv) { - return handleExceptions(argv[0], [&]() { - initNix(); - + { Strings opFlags, opArgs; Operation op = 0; @@ -1062,5 +1094,9 @@ int main(int argc, char * * argv) store = openStore(); op(opFlags, opArgs); - }); + + return 0; + } } + +static RegisterLegacyCommand s1("nix-store", _main); diff --git a/src/nix-store/xmlgraph.cc b/src/nix-store/xmlgraph.cc deleted file mode 100644 index 0f7be7f7a02d..000000000000 --- a/src/nix-store/xmlgraph.cc +++ /dev/null @@ -1,66 +0,0 @@ -#include "xmlgraph.hh" -#include "util.hh" -#include "store-api.hh" - -#include <iostream> - - -using std::cout; - -namespace nix { - - -static inline const string & xmlQuote(const string & s) -{ - // Luckily, store paths shouldn't contain any character that needs to be - // quoted. - return s; -} - - -static string makeEdge(const string & src, const string & dst) -{ - format f = format(" <edge src=\"%1%\" dst=\"%2%\"/>\n") - % xmlQuote(src) % xmlQuote(dst); - return f.str(); -} - - -static string makeNode(const string & id) -{ - format f = format(" <node name=\"%1%\"/>\n") % xmlQuote(id); - return f.str(); -} - - -void printXmlGraph(ref<Store> store, const PathSet & roots) -{ - PathSet workList(roots); - PathSet doneSet; - - cout << "<?xml version='1.0' encoding='utf-8'?>\n" - << "<nix>\n"; - - while (!workList.empty()) { - Path path = *(workList.begin()); - workList.erase(path); - - if (doneSet.find(path) != doneSet.end()) continue; - doneSet.insert(path); - - cout << makeNode(path); - - for (auto & p : store->queryPathInfo(path)->references) { - if (p != path) { - workList.insert(p); - cout << makeEdge(p, path); - } - } - - } - - cout << "</nix>\n"; -} - - -} diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index d0003790d3b9..e86b96e3f3f2 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -8,7 +8,7 @@ using namespace nix; struct CmdAddToStore : MixDryRun, StoreCommand { Path path; - std::experimental::optional<std::string> namePart; + std::optional<std::string> namePart; CmdAddToStore() { diff --git a/src/nix/copy.cc b/src/nix/copy.cc index e4e6c3e303ed..96bd453d87b4 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -69,8 +69,12 @@ struct CmdCopy : StorePathsCommand }, #ifdef ENABLE_S3 Example{ - "To populate the current folder build output to a S3 binary cache:", - "nix copy --to s3://my-bucket?region=eu-west-1" + "To copy Hello to an S3 binary cache:", + "nix copy --to s3://my-bucket?region=eu-west-1 nixpkgs.hello" + }, + Example{ + "To copy Hello to an S3-compatible binary cache:", + "nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com nixpkgs.hello" }, #endif }; diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc new file mode 100644 index 000000000000..7b5444619470 --- /dev/null +++ b/src/nix/doctor.cc @@ -0,0 +1,124 @@ +#include "command.hh" +#include "serve-protocol.hh" +#include "shared.hh" +#include "store-api.hh" +#include "worker-protocol.hh" + +using namespace nix; + +std::string formatProtocol(unsigned int proto) +{ + if (proto) { + auto major = GET_PROTOCOL_MAJOR(proto) >> 8; + auto minor = GET_PROTOCOL_MINOR(proto); + return (format("%1%.%2%") % major % minor).str(); + } + return "unknown"; +} + +struct CmdDoctor : StoreCommand +{ + bool success = true; + + std::string name() override + { + return "doctor"; + } + + std::string description() override + { + return "check your system for potential problems"; + } + + void run(ref<Store> store) override + { + std::cout << "Store uri: " << store->getUri() << std::endl; + std::cout << std::endl; + + auto type = getStoreType(); + + if (type < tOther) { + success &= checkNixInPath(); + success &= checkProfileRoots(store); + } + success &= checkStoreProtocol(store->getProtocol()); + + if (!success) + throw Exit(2); + } + + bool checkNixInPath() + { + PathSet dirs; + + for (auto & dir : tokenizeString<Strings>(getEnv("PATH"), ":")) + if (pathExists(dir + "/nix-env")) + dirs.insert(dirOf(canonPath(dir + "/nix-env", true))); + + if (dirs.size() != 1) { + std::cout << "Warning: multiple versions of nix found in PATH." << std::endl; + std::cout << std::endl; + for (auto & dir : dirs) + std::cout << " " << dir << std::endl; + std::cout << std::endl; + return false; + } + + return true; + } + + bool checkProfileRoots(ref<Store> store) + { + PathSet dirs; + + for (auto & dir : tokenizeString<Strings>(getEnv("PATH"), ":")) { + Path profileDir = dirOf(dir); + try { + Path userEnv = canonPath(profileDir, true); + + if (store->isStorePath(userEnv) && hasSuffix(userEnv, "user-environment")) { + while (profileDir.find("/profiles/") == std::string::npos && isLink(profileDir)) + profileDir = absPath(readLink(profileDir), dirOf(profileDir)); + + if (profileDir.find("/profiles/") == std::string::npos) + dirs.insert(dir); + } + } catch (SysError &) {} + } + + if (!dirs.empty()) { + std::cout << "Warning: found profiles outside of " << settings.nixStateDir << "/profiles." << std::endl; + std::cout << "The generation this profile points to might not have a gcroot and could be" << std::endl; + std::cout << "garbage collected, resulting in broken symlinks." << std::endl; + std::cout << std::endl; + for (auto & dir : dirs) + std::cout << " " << dir << std::endl; + std::cout << std::endl; + return false; + } + + return true; + } + + bool checkStoreProtocol(unsigned int storeProto) + { + unsigned int clientProto = GET_PROTOCOL_MAJOR(SERVE_PROTOCOL_VERSION) == GET_PROTOCOL_MAJOR(storeProto) + ? SERVE_PROTOCOL_VERSION + : PROTOCOL_VERSION; + + if (clientProto != storeProto) { + std::cout << "Warning: protocol version of this client does not match the store." << std::endl; + std::cout << "While this is not necessarily a problem it's recommended to keep the client in" << std::endl; + std::cout << "sync with the daemon." << std::endl; + std::cout << std::endl; + std::cout << "Client protocol: " << formatProtocol(clientProto) << std::endl; + std::cout << "Store protocol: " << formatProtocol(storeProto) << std::endl; + std::cout << std::endl; + return false; + } + + return true; + } +}; + +static RegisterCommand r1(make_ref<CmdDoctor>()); diff --git a/src/nix/edit.cc b/src/nix/edit.cc index 7eaa86e2f914..c9671f76d0fa 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -61,7 +61,7 @@ struct CmdEdit : InstallableCommand auto editor = getEnv("EDITOR", "cat"); - Strings args{editor}; + auto args = tokenizeString<Strings>(editor); if (editor.find("emacs") != std::string::npos || editor.find("nano") != std::string::npos || @@ -72,7 +72,7 @@ struct CmdEdit : InstallableCommand stopProgressBar(); - execvp(editor.c_str(), stringsToCharPtrs(args).data()); + execvp(args.front().c_str(), stringsToCharPtrs(args).data()); throw SysError("cannot run editor '%s'", editor); } diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 64062fb97955..af4105e28904 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -9,13 +9,14 @@ struct CmdHash : Command { enum Mode { mFile, mPath }; Mode mode; - Base base = Base16; + Base base = SRI; bool truncate = false; - HashType ht = htSHA512; + HashType ht = htSHA256; std::vector<std::string> paths; CmdHash(Mode mode) : mode(mode) { + mkFlag(0, "sri", "print hash in SRI format", &base, SRI); mkFlag(0, "base64", "print hash in base-64", &base, Base64); mkFlag(0, "base32", "print hash in base-32 (Nix-specific)", &base, Base32); mkFlag(0, "base16", "print hash in base-16", &base, Base16); @@ -43,7 +44,7 @@ struct CmdHash : Command Hash h = mode == mFile ? hashFile(ht, path) : hashPath(ht, path).first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); std::cout << format("%1%\n") % - h.to_string(base, false); + h.to_string(base, base == SRI); } } }; @@ -54,7 +55,7 @@ static RegisterCommand r2(make_ref<CmdHash>(CmdHash::mPath)); struct CmdToBase : Command { Base base; - HashType ht = htSHA512; + HashType ht = htUnknown; std::vector<std::string> args; CmdToBase(Base base) : base(base) @@ -70,26 +71,30 @@ struct CmdToBase : Command return base == Base16 ? "to-base16" : base == Base32 ? "to-base32" : - "to-base64"; + base == Base64 ? "to-base64" : + "to-sri"; } std::string description() override { - return fmt("convert a hash to base-%d representation", - base == Base16 ? 16 : - base == Base32 ? 32 : 64); + return fmt("convert a hash to %s representation", + base == Base16 ? "base-16" : + base == Base32 ? "base-32" : + base == Base64 ? "base-64" : + "SRI"); } void run() override { for (auto s : args) - std::cout << fmt("%s\n", Hash(s, ht).to_string(base, false)); + std::cout << fmt("%s\n", Hash(s, ht).to_string(base, base == SRI)); } }; static RegisterCommand r3(make_ref<CmdToBase>(Base16)); static RegisterCommand r4(make_ref<CmdToBase>(Base32)); static RegisterCommand r5(make_ref<CmdToBase>(Base64)); +static RegisterCommand r6(make_ref<CmdToBase>(SRI)); /* Legacy nix-hash command. */ static int compatNixHash(int argc, char * * argv) diff --git a/src/nix/installables.cc b/src/nix/installables.cc index a3fdd8a2808d..0c1ad3ab3db0 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -86,22 +86,6 @@ Buildable Installable::toBuildable() return std::move(buildables[0]); } -struct InstallableStoreDrv : Installable -{ - Path drvPath; - - InstallableStoreDrv(const Path & drvPath) : drvPath(drvPath) { } - - std::string what() override { return drvPath; } - - Buildables toBuildables() override - { - Buildable b = {drvPath}; - // FIXME: add outputs? - return {b}; - } -}; - struct InstallableStorePath : Installable { Path storePath; @@ -112,7 +96,7 @@ struct InstallableStorePath : Installable Buildables toBuildables() override { - return {{"", {{"out", storePath}}}}; + return {{isDerivation(storePath) ? storePath : "", {{"out", storePath}}}}; } }; @@ -226,12 +210,8 @@ static std::vector<std::shared_ptr<Installable>> parseInstallables( auto path = store->toStorePath(store->followLinksToStore(s)); - if (store->isStorePath(path)) { - if (isDerivation(path)) - result.push_back(std::make_shared<InstallableStoreDrv>(path)); - else - result.push_back(std::make_shared<InstallableStorePath>(path)); - } + if (store->isStorePath(path)) + result.push_back(std::make_shared<InstallableStorePath>(path)); } else if (s == "" || std::regex_match(s, attrPathRegex)) diff --git a/src/nix/local.mk b/src/nix/local.mk index f76da194467c..ca4604d566c3 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -2,10 +2,24 @@ programs += nix nix_DIR := $(d) -nix_SOURCES := $(wildcard $(d)/*.cc) $(wildcard src/linenoise/*.cpp) +nix_SOURCES := \ + $(wildcard $(d)/*.cc) \ + $(wildcard src/build-remote/*.cc) \ + $(wildcard src/nix-build/*.cc) \ + $(wildcard src/nix-channel/*.cc) \ + $(wildcard src/nix-collect-garbage/*.cc) \ + $(wildcard src/nix-copy-closure/*.cc) \ + $(wildcard src/nix-daemon/*.cc) \ + $(wildcard src/nix-env/*.cc) \ + $(wildcard src/nix-instantiate/*.cc) \ + $(wildcard src/nix-prefetch-url/*.cc) \ + $(wildcard src/nix-store/*.cc) \ -nix_LIBS = libexpr libmain libstore libutil libformat +nix_LIBS = libexpr libmain libstore libutil -nix_LDFLAGS = -pthread +nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) -$(eval $(call install-symlink, nix, $(bindir)/nix-hash)) +$(foreach name, \ + nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \ + $(eval $(call install-symlink, nix, $(bindir)/$(name)))) +$(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote)) diff --git a/src/nix/ls.cc b/src/nix/ls.cc index e99622faf472..d089be42fb20 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -148,7 +148,7 @@ struct CmdLsNar : Command, MixLs void run() override { - list(makeNarAccessor(make_ref<std::string>(readFile(narPath)))); + list(makeNarAccessor(make_ref<std::string>(readFile(narPath, true)))); } }; diff --git a/src/nix/main.cc b/src/nix/main.cc index bb107ec7d3f6..4f87ad72b65c 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -20,11 +20,12 @@ std::string programPath; struct NixArgs : virtual MultiCommand, virtual MixCommonArgs { + bool printBuildLogs = false; + NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix") { mkFlag() .longName("help") - .shortName('h') .description("show usage information") .handler([&]() { showHelpAndExit(); }); @@ -34,14 +35,20 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs .handler([&]() { std::cout << "The following configuration options are available:\n\n"; Table2 tbl; - for (const auto & s : settings._getSettings()) - if (!s.second.isAlias) - tbl.emplace_back(s.first, s.second.setting->description); + std::map<std::string, Config::SettingInfo> settings; + globalConfig.getSettings(settings); + for (const auto & s : settings) + tbl.emplace_back(s.first, s.second.description); printTable(std::cout, tbl); throw Exit(); }); mkFlag() + .longName("print-build-logs") + .description("print full build logs on stderr") + .set(&printBuildLogs, true); + + mkFlag() .longName("version") .description("show version information") .handler([&]() { printVersion(programName); }); @@ -67,9 +74,6 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs void mainWrapped(int argc, char * * argv) { - verbosity = lvlError; - settings.verboseBuild = false; - /* The chroot helper needs to be run before any threads have been started. */ if (argc > 0 && argv[0] == chrootHelperName) { @@ -88,6 +92,9 @@ void mainWrapped(int argc, char * * argv) if (legacy) return legacy(argc, argv); } + verbosity = lvlError; + settings.verboseBuild = false; + NixArgs args; args.parseCmdline(argvToStrings(argc, argv)); @@ -98,8 +105,7 @@ void mainWrapped(int argc, char * * argv) Finally f([]() { stopProgressBar(); }); - if (isatty(STDERR_FILENO)) - startProgressBar(); + startProgressBar(args.printBuildLogs); args.command->prepare(); args.command->run(); diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 47caa401d3c9..dea5f0557b81 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -4,8 +4,8 @@ #include "json.hh" #include "common-args.hh" -#include <iomanip> #include <algorithm> +#include <array> using namespace nix; @@ -13,12 +13,14 @@ struct CmdPathInfo : StorePathsCommand, MixJSON { bool showSize = false; bool showClosureSize = false; + bool humanReadable = false; bool showSigs = false; CmdPathInfo() { mkFlag('s', "size", "print size of the NAR dump of each path", &showSize); mkFlag('S', "closure-size", "print sum size of the NAR dumps of the closure of each path", &showClosureSize); + mkFlag('h', "human-readable", "with -s and -S, print sizes like 1K 234M 5.67G etc.", &humanReadable); mkFlag(0, "sigs", "show signatures", &showSigs); } @@ -40,6 +42,10 @@ struct CmdPathInfo : StorePathsCommand, MixJSON "nix path-info -rS /run/current-system | sort -nk2" }, Example{ + "To show a package's closure size and all its dependencies with human readable sizes:", + "nix path-info -rsSh nixpkgs.rust" + }, + Example{ "To check the existence of a path in a binary cache:", "nix path-info -r /nix/store/7qvk5c91...-geeqie-1.1 --store https://cache.nixos.org/" }, @@ -58,6 +64,25 @@ struct CmdPathInfo : StorePathsCommand, MixJSON }; } + void printSize(unsigned long long value) + { + if (!humanReadable) { + std::cout << fmt("\t%11d", value); + return; + } + + static const std::array<char, 9> idents{{ + ' ', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y' + }}; + size_t power = 0; + double res = value; + while (res > 1024 && power < idents.size()) { + ++power; + res /= 1024; + } + std::cout << fmt("\t%6.1f%c", res, idents.at(power)); + } + void run(ref<Store> store, Paths storePaths) override { size_t pathLen = 0; @@ -78,13 +103,16 @@ struct CmdPathInfo : StorePathsCommand, MixJSON auto info = store->queryPathInfo(storePath); storePath = info->path; // FIXME: screws up padding - std::cout << storePath << std::string(std::max(0, (int) pathLen - (int) storePath.size()), ' '); + std::cout << storePath; + + if (showSize || showClosureSize || showSigs) + std::cout << std::string(std::max(0, (int) pathLen - (int) storePath.size()), ' '); if (showSize) - std::cout << '\t' << std::setw(11) << info->narSize; + printSize(info->narSize); if (showClosureSize) - std::cout << '\t' << std::setw(11) << store->getClosureSize(storePath).first; + printSize(store->getClosureSize(storePath).first); if (showSigs) { std::cout << '\t'; diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index 40b905ba3243..e7104540816b 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -2,6 +2,7 @@ #include "util.hh" #include "sync.hh" #include "store-api.hh" +#include "names.hh" #include <atomic> #include <map> @@ -38,6 +39,7 @@ private: std::map<ActivityType, uint64_t> expectedByType; bool visible = true; ActivityId parent; + std::optional<std::string> name; }; struct ActivitiesByType @@ -68,10 +70,16 @@ private: std::condition_variable quitCV, updateCV; + bool printBuildLogs; + bool isTTY; + public: - ProgressBar() + ProgressBar(bool printBuildLogs, bool isTTY) + : printBuildLogs(printBuildLogs) + , isTTY(isTTY) { + state_.lock()->active = isTTY; updateThread = std::thread([&]() { auto state(state_.lock()); while (state->active) { @@ -109,8 +117,14 @@ public: void log(State & state, Verbosity lvl, const std::string & s) { - writeToStderr("\r\e[K" + s + ANSI_NORMAL "\n"); - draw(state); + if (state.active) { + writeToStderr("\r\e[K" + s + ANSI_NORMAL "\n"); + draw(state); + } else { + auto s2 = s + ANSI_NORMAL "\n"; + if (!isTTY) s2 = filterANSIEscapes(s2, true); + writeToStderr(s2); + } } void startActivity(ActivityId act, Verbosity lvl, ActivityType type, @@ -141,6 +155,7 @@ public: auto nrRounds = getI(fields, 3); if (nrRounds != 1) i->s += fmt(" (round %d/%d)", curRound, nrRounds); + i->name = DrvName(name).name; } if (type == actSubstitute) { @@ -217,11 +232,15 @@ public: auto i = state->its.find(act); assert(i != state->its.end()); ActInfo info = *i->second; - state->activities.erase(i->second); - info.lastLine = lastLine; - state->activities.emplace_back(info); - i->second = std::prev(state->activities.end()); - update(); + if (printBuildLogs) { + log(*state, lvlInfo, ANSI_FAINT + info.name.value_or("unnamed") + "> " + ANSI_NORMAL + lastLine); + } else { + state->activities.erase(i->second); + info.lastLine = lastLine; + state->activities.emplace_back(info); + i->second = std::prev(state->activities.end()); + update(); + } } } @@ -333,11 +352,18 @@ public: if (running || done || expected || failed) { if (running) - s = fmt(ANSI_BLUE + numberFmt + ANSI_NORMAL "/" ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt, - running / unit, done / unit, expected / unit); + if (expected != 0) + s = fmt(ANSI_BLUE + numberFmt + ANSI_NORMAL "/" ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt, + running / unit, done / unit, expected / unit); + else + s = fmt(ANSI_BLUE + numberFmt + ANSI_NORMAL "/" ANSI_GREEN + numberFmt + ANSI_NORMAL, + running / unit, done / unit); else if (expected != done) - s = fmt(ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt, - done / unit, expected / unit); + if (expected != 0) + s = fmt(ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt, + done / unit, expected / unit); + else + s = fmt(ANSI_GREEN + numberFmt + ANSI_NORMAL, done / unit); else s = fmt(done ? ANSI_GREEN + numberFmt + ANSI_NORMAL : numberFmt, done / unit); s = fmt(itemFmt, s); @@ -395,9 +421,9 @@ public: } }; -void startProgressBar() +void startProgressBar(bool printBuildLogs) { - logger = new ProgressBar(); + logger = new ProgressBar(printBuildLogs, isatty(STDERR_FILENO)); } void stopProgressBar() diff --git a/src/nix/progress-bar.hh b/src/nix/progress-bar.hh index af8eda5a84fd..4d61175c24e4 100644 --- a/src/nix/progress-bar.hh +++ b/src/nix/progress-bar.hh @@ -4,7 +4,7 @@ namespace nix { -void startProgressBar(); +void startProgressBar(bool printBuildLogs = false); void stopProgressBar(); diff --git a/src/nix/repl.cc b/src/nix/repl.cc index f84774a53367..d8f812149069 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -1,8 +1,17 @@ #include <iostream> #include <cstdlib> +#include <cstring> +#include <climits> #include <setjmp.h> +#ifdef READLINE +#include <readline/history.h> +#include <readline/readline.h> +#else +#include <editline.h> +#endif + #include "shared.hh" #include "eval.hh" #include "eval-inline.hh" @@ -15,8 +24,6 @@ #include "command.hh" #include "finally.hh" -#include "src/linenoise/linenoise.h" - namespace nix { #define ESC_RED "\033[31m" @@ -31,6 +38,7 @@ struct NixRepl { string curDir; EvalState state; + Bindings * autoArgs; Strings loadedFiles; @@ -117,19 +125,81 @@ NixRepl::NixRepl(const Strings & searchPath, nix::ref<Store> store) NixRepl::~NixRepl() { - linenoiseHistorySave(historyFile.c_str()); + write_history(historyFile.c_str()); } - static NixRepl * curRepl; // ugly -static void completionCallback(const char * s, linenoiseCompletions *lc) -{ - /* Otherwise, return all symbols that start with the prefix. */ - for (auto & c : curRepl->completePrefix(s)) - linenoiseAddCompletion(lc, c.c_str()); +static char * completionCallback(char * s, int *match) { + auto possible = curRepl->completePrefix(s); + if (possible.size() == 1) { + *match = 1; + auto *res = strdup(possible.begin()->c_str() + strlen(s)); + if (!res) throw Error("allocation failure"); + return res; + } else if (possible.size() > 1) { + auto checkAllHaveSameAt = [&](size_t pos) { + auto &first = *possible.begin(); + for (auto &p : possible) { + if (p.size() <= pos || p[pos] != first[pos]) + return false; + } + return true; + }; + size_t start = strlen(s); + size_t len = 0; + while (checkAllHaveSameAt(start + len)) ++len; + if (len > 0) { + *match = 1; + auto *res = strdup(std::string(*possible.begin(), start, len).c_str()); + if (!res) throw Error("allocation failure"); + return res; + } + } + + *match = 0; + return nullptr; +} + +static int listPossibleCallback(char *s, char ***avp) { + auto possible = curRepl->completePrefix(s); + + if (possible.size() > (INT_MAX / sizeof(char*))) + throw Error("too many completions"); + + int ac = 0; + char **vp = nullptr; + + auto check = [&](auto *p) { + if (!p) { + if (vp) { + while (--ac >= 0) + free(vp[ac]); + free(vp); + } + throw Error("allocation failure"); + } + return p; + }; + + vp = check((char **)malloc(possible.size() * sizeof(char*))); + + for (auto & p : possible) + vp[ac++] = check(strdup(p.c_str())); + + *avp = vp; + + return ac; } +namespace { + // Used to communicate to NixRepl::getLine whether a signal occurred in ::readline. + volatile sig_atomic_t g_signal_received = 0; + + void sigintHandler(int signo) { + g_signal_received = signo; + } +} void NixRepl::mainLoop(const std::vector<std::string> & files) { @@ -142,12 +212,18 @@ void NixRepl::mainLoop(const std::vector<std::string> & files) reloadFiles(); if (!loadedFiles.empty()) std::cout << std::endl; + // Allow nix-repl specific settings in .inputrc + rl_readline_name = "nix-repl"; createDirs(dirOf(historyFile)); - linenoiseHistorySetMaxLen(1000); - linenoiseHistoryLoad(historyFile.c_str()); - +#ifndef READLINE + el_hist_size = 1000; +#endif + read_history(historyFile.c_str()); curRepl = this; - linenoiseSetCompletionCallback(completionCallback); +#ifndef READLINE + rl_set_complete_func(completionCallback); + rl_set_list_possib_func(listPossibleCallback); +#endif std::string input; @@ -175,7 +251,6 @@ void NixRepl::mainLoop(const std::vector<std::string> & files) // We handled the current input fully, so we should clear it // and read brand new input. - linenoiseHistoryAdd(input.c_str()); input.clear(); std::cout << std::endl; } @@ -184,19 +259,42 @@ void NixRepl::mainLoop(const std::vector<std::string> & files) bool NixRepl::getLine(string & input, const std::string &prompt) { - char * s = linenoise(prompt.c_str()); + struct sigaction act, old; + sigset_t savedSignalMask, set; + + auto setupSignals = [&]() { + act.sa_handler = sigintHandler; + sigfillset(&act.sa_mask); + act.sa_flags = 0; + if (sigaction(SIGINT, &act, &old)) + throw SysError("installing handler for SIGINT"); + + sigemptyset(&set); + sigaddset(&set, SIGINT); + if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask)) + throw SysError("unblocking SIGINT"); + }; + auto restoreSignals = [&]() { + if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) + throw SysError("restoring signals"); + + if (sigaction(SIGINT, &old, 0)) + throw SysError("restoring handler for SIGINT"); + }; + + setupSignals(); + char * s = readline(prompt.c_str()); Finally doFree([&]() { free(s); }); - if (!s) { - switch (auto type = linenoiseKeyType()) { - case 1: // ctrl-C - input = ""; - return true; - case 2: // ctrl-D - return false; - default: - throw Error(format("Unexpected linenoise keytype: %1%") % type); - } + restoreSignals(); + + if (g_signal_received) { + g_signal_received = 0; + input.clear(); + return true; } + + if (!s) + return false; input += s; input += '\n'; return true; @@ -385,7 +483,7 @@ bool NixRepl::processLine(string line) /* We could do the build in this process using buildPaths(), but doing it in a child makes it easier to recover from problems / SIGINT. */ - if (runProgram(settings.nixBinDir + "/nix-store", Strings{"-r", drvPath}) == 0) { + if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPath}) == 0) { Derivation drv = readDerivation(drvPath); std::cout << std::endl << "this derivation produced the following outputs:" << std::endl; for (auto & i : drv.outputs) @@ -441,8 +539,7 @@ void NixRepl::loadFile(const Path & path) loadedFiles.push_back(path); Value v, v2; state.evalFile(lookupFileArg(state, path), v); - Bindings & bindings(*state.allocBindings(0)); - state.autoCallFunction(bindings, v, v2); + state.autoCallFunction(*autoArgs, v, v2); addAttrsToScope(v2); } @@ -584,30 +681,13 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m for (auto & i : *v.attrs) sorted[i.name] = i.value; - /* If this is a derivation, then don't show the - self-references ("all", "out", etc.). */ - StringSet hidden; - if (isDrv) { - hidden.insert("all"); - Bindings::iterator i = v.attrs->find(state.sOutputs); - if (i == v.attrs->end()) - hidden.insert("out"); - else { - state.forceList(*i->value); - for (unsigned int j = 0; j < i->value->listSize(); ++j) - hidden.insert(state.forceStringNoCtx(*i->value->listElems()[j])); - } - } - for (auto & i : sorted) { if (isVarName(i.first)) str << i.first; else printStringValue(str, i.first.c_str()); str << " = "; - if (hidden.find(i.first) != hidden.end()) - str << "«...»"; - else if (seen.find(i.second) != seen.end()) + if (seen.find(i.second) != seen.end()) str << "«repeated»"; else try { @@ -693,8 +773,9 @@ struct CmdRepl : StoreCommand, MixEvalArgs void run(ref<Store> store) override { - NixRepl repl(searchPath, openStore()); - repl.mainLoop(files); + auto repl = std::make_unique<NixRepl>(searchPath, openStore()); + repl->autoArgs = getAutoArgs(repl->state); + repl->mainLoop(files); } }; diff --git a/src/nix/run.cc b/src/nix/run.cc index d04e106e037b..35b763345872 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -7,11 +7,14 @@ #include "finally.hh" #include "fs-accessor.hh" #include "progress-bar.hh" +#include "affinity.hh" #if __linux__ #include <sys/mount.h> #endif +#include <queue> + using namespace nix; std::string chrootHelperName = "__run_in_chroot"; @@ -121,10 +124,27 @@ struct CmdRun : InstallablesCommand unsetenv(var.c_str()); } + std::unordered_set<Path> done; + std::queue<Path> todo; + for (auto & path : outPaths) todo.push(path); + auto unixPath = tokenizeString<Strings>(getEnv("PATH"), ":"); - for (auto & path : outPaths) - if (accessor->stat(path + "/bin").type != FSAccessor::tMissing) + + while (!todo.empty()) { + Path path = todo.front(); + todo.pop(); + if (!done.insert(path).second) continue; + + if (true) unixPath.push_front(path + "/bin"); + + auto propPath = path + "/nix-support/propagated-user-env-packages"; + if (accessor->stat(propPath).type == FSAccessor::tRegular) { + for (auto & p : tokenizeString<Paths>(readFile(propPath))) + todo.push(p); + } + } + setenv("PATH", concatStringsSep(":", unixPath).c_str(), 1); std::string cmd = *command.begin(); @@ -135,6 +155,8 @@ struct CmdRun : InstallablesCommand restoreSignals(); + restoreAffinity(); + /* If this is a diverted store (i.e. its "logical" location (typically /nix/store) differs from its "physical" location (e.g. /home/eelco/nix/store), then run the command in a diff --git a/src/nix/search.cc b/src/nix/search.cc index 5ccf1b7cf529..e086de2260a6 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -7,32 +7,38 @@ #include "common-args.hh" #include "json.hh" #include "json-to-value.hh" +#include "shared.hh" #include <regex> #include <fstream> using namespace nix; -std::string hilite(const std::string & s, const std::smatch & m) +std::string wrap(std::string prefix, std::string s) +{ + return prefix + s + ANSI_NORMAL; +} + +std::string hilite(const std::string & s, const std::smatch & m, std::string postfix) { return m.empty() ? s : std::string(m.prefix()) - + ANSI_RED + std::string(m.str()) + ANSI_NORMAL + + ANSI_RED + std::string(m.str()) + postfix + std::string(m.suffix()); } struct CmdSearch : SourceExprCommand, MixJSON { - std::string re; + std::vector<std::string> res; bool writeCache = true; bool useCache = true; CmdSearch() { - expectArg("regex", &re, true); + expectArgs("regex", &res); mkFlag() .longName("update-cache") @@ -68,9 +74,17 @@ struct CmdSearch : SourceExprCommand, MixJSON "nix search blender" }, Example{ - "To search for Firefox and Chromium:", + "To search for Firefox or Chromium:", "nix search 'firefox|chromium'" }, + Example{ + "To search for git and frontend or gui:", + "nix search git 'frontend|gui'" + }, + Example{ + "To display the description of the found packages:", + "nix search git --verbose" + } }; } @@ -81,9 +95,16 @@ struct CmdSearch : SourceExprCommand, MixJSON // Empty search string should match all packages // Use "^" here instead of ".*" due to differences in resulting highlighting // (see #1893 -- libc++ claims empty search string is not in POSIX grammar) - if (re.empty()) re = "^"; + if (res.empty()) { + res.push_back("^"); + } - std::regex regex(re, std::regex::extended | std::regex::icase); + std::vector<std::regex> regexes; + regexes.reserve(res.size()); + + for (auto &re : res) { + regexes.push_back(std::regex(re, std::regex::extended | std::regex::icase)); + } auto state = getEvalState(); @@ -102,6 +123,7 @@ struct CmdSearch : SourceExprCommand, MixJSON debug("at attribute '%s'", attrPath); try { + uint found = 0; state->forceValue(*v); @@ -115,25 +137,33 @@ struct CmdSearch : SourceExprCommand, MixJSON if (state->isDerivation(*v)) { DrvInfo drv(*state, attrPath, v->attrs); + std::string description; + std::smatch attrPathMatch; + std::smatch descriptionMatch; + std::smatch nameMatch; + std::string name; DrvName parsed(drv.queryName()); - std::smatch attrPathMatch; - std::regex_search(attrPath, attrPathMatch, regex); + for (auto ®ex : regexes) { + std::regex_search(attrPath, attrPathMatch, regex); - auto name = parsed.name; - std::smatch nameMatch; - std::regex_search(name, nameMatch, regex); + name = parsed.name; + std::regex_search(name, nameMatch, regex); - std::string description = drv.queryMetaString("description"); - std::replace(description.begin(), description.end(), '\n', ' '); - std::smatch descriptionMatch; - std::regex_search(description, descriptionMatch, regex); + description = drv.queryMetaString("description"); + std::replace(description.begin(), description.end(), '\n', ' '); + std::regex_search(description, descriptionMatch, regex); - if (!attrPathMatch.empty() - || !nameMatch.empty() - || !descriptionMatch.empty()) - { + if (!attrPathMatch.empty() + || !nameMatch.empty() + || !descriptionMatch.empty()) + { + found++; + } + } + + if (found == res.size()) { if (json) { auto jsonElem = jsonOut->object(attrPath); @@ -143,15 +173,13 @@ struct CmdSearch : SourceExprCommand, MixJSON jsonElem.attr("description", description); } else { + auto name = hilite(parsed.name, nameMatch, "\e[0;2m") + + std::string(parsed.fullName, parsed.name.length()); results[attrPath] = fmt( - "Attribute name: %s\n" - "Package name: %s\n" - "Version: %s\n" - "Description: %s\n", - hilite(attrPath, attrPathMatch), - hilite(name, nameMatch), - parsed.version, - hilite(description, descriptionMatch)); + "* %s (%s)\n %s\n", + wrap("\e[0;1m", hilite(attrPath, attrPathMatch, "\e[0;1m")), + wrap("\e[0;2m", hilite(name, nameMatch, "\e[0;2m")), + hilite(description, descriptionMatch, ANSI_NORMAL)); } } @@ -243,6 +271,10 @@ struct CmdSearch : SourceExprCommand, MixJSON throw SysError("cannot rename '%s' to '%s'", tmpFile, jsonCacheFileName); } + if (results.size() == 0) + throw Error("no results for the given search term(s)!"); + + RunPager pager; for (auto el : results) std::cout << el.second << "\n"; } diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc index c64b12c8dd62..86638b50d2c6 100644 --- a/src/nix/show-config.cc +++ b/src/nix/show-config.cc @@ -27,10 +27,12 @@ struct CmdShowConfig : Command, MixJSON if (json) { // FIXME: use appropriate JSON types (bool, ints, etc). JSONObject jsonObj(std::cout); - settings.toJSON(jsonObj); + globalConfig.toJSON(jsonObj); } else { - for (auto & s : settings.getSettings()) - std::cout << s.first + " = " + s.second + "\n"; + std::map<std::string, Config::SettingInfo> settings; + globalConfig.getSettings(settings); + for (auto & s : settings) + std::cout << s.first + " = " + s.second.value + "\n"; } } }; diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 758bbbc688bc..35c44a70cf52 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -1,14 +1,18 @@ #include "command.hh" +#include "common-args.hh" #include "store-api.hh" #include "download.hh" #include "eval.hh" #include "attr-path.hh" +#include "names.hh" +#include "progress-bar.hh" using namespace nix; -struct CmdUpgradeNix : StoreCommand +struct CmdUpgradeNix : MixDryRun, StoreCommand { Path profileDir; + std::string storePathsUrl = "https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix"; CmdUpgradeNix() { @@ -18,6 +22,12 @@ struct CmdUpgradeNix : StoreCommand .labels({"profile-dir"}) .description("the Nix profile to upgrade") .dest(&profileDir); + + mkFlag() + .longName("nix-store-paths-url") + .labels({"url"}) + .description("URL of the file that contains the store paths of the latest Nix release") + .dest(&storePathsUrl); } std::string name() override @@ -46,7 +56,7 @@ struct CmdUpgradeNix : StoreCommand void run(ref<Store> store) override { - settings.pureEval = true; + evalSettings.pureEval = true; if (profileDir == "") profileDir = getProfileDir(store); @@ -59,6 +69,14 @@ struct CmdUpgradeNix : StoreCommand storePath = getLatestNix(store); } + auto version = DrvName(storePathToName(storePath)).version; + + if (dryRun) { + stopProgressBar(); + printError("would upgrade to version %s", version); + return; + } + { Activity act(*logger, lvlInfo, actUnknown, fmt("downloading '%s'...", storePath)); store->ensurePath(storePath); @@ -72,11 +90,15 @@ struct CmdUpgradeNix : StoreCommand throw Error("could not verify that '%s' works", program); } + stopProgressBar(); + { Activity act(*logger, lvlInfo, actUnknown, fmt("installing '%s' into profile '%s'...", storePath, profileDir)); runProgram(settings.nixBinDir + "/nix-env", false, {"--profile", profileDir, "-i", storePath, "--no-sandbox"}); } + + printError(ANSI_GREEN "upgrade to version %s done" ANSI_NORMAL, version); } /* Return the profile in which Nix is installed. */ @@ -98,11 +120,18 @@ struct CmdUpgradeNix : StoreCommand if (hasPrefix(where, "/run/current-system")) throw Error("Nix on NixOS must be upgraded via 'nixos-rebuild'"); - Path profileDir; - Path userEnv; + Path profileDir = dirOf(where); + + // Resolve profile to /nix/var/nix/profiles/<name> link. + while (canonPath(profileDir).find("/profiles/") == std::string::npos && isLink(profileDir)) + profileDir = readLink(profileDir); + + printInfo("found profile '%s'", profileDir); + + Path userEnv = canonPath(profileDir, true); if (baseNameOf(where) != "bin" || - !hasSuffix(userEnv = canonPath(profileDir = dirOf(where), true), "user-environment")) + !hasSuffix(userEnv, "user-environment")) throw Error("directory '%s' does not appear to be part of a Nix profile", where); if (!store->isValidPath(userEnv)) @@ -115,16 +144,16 @@ struct CmdUpgradeNix : StoreCommand Path getLatestNix(ref<Store> store) { // FIXME: use nixos.org? - auto req = DownloadRequest("https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix"); + auto req = DownloadRequest(storePathsUrl); auto res = getDownloader()->download(req); - EvalState state(Strings(), store); - auto v = state.allocValue(); - state.eval(state.parseExprFromString(*res.data, "/no-such-path"), *v); - Bindings & bindings(*state.allocBindings(0)); - auto v2 = findAlongAttrPath(state, settings.thisSystem, bindings, *v); + auto state = std::make_unique<EvalState>(Strings(), store); + auto v = state->allocValue(); + state->eval(state->parseExprFromString(*res.data, "/no-such-path"), *v); + Bindings & bindings(*state->allocBindings(0)); + auto v2 = findAlongAttrPath(*state, settings.thisSystem, bindings, *v); - return state.forceString(*v2); + return state->forceString(*v2); } }; diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 6540208a8a2c..7ef571561a0e 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -120,7 +120,7 @@ struct CmdVerify : StorePathsCommand for (auto sig : sigs) { if (sigsSeen.count(sig)) continue; sigsSeen.insert(sig); - if (info->checkSignature(publicKeys, sig)) + if (validSigs < ValidPathInfo::maxSigs && info->checkSignature(publicKeys, sig)) validSigs++; } }; diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 17e0595ae887..325a2be0a793 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -2,6 +2,7 @@ #include "store-api.hh" #include "progress-bar.hh" #include "fs-accessor.hh" +#include "shared.hh" #include <queue> @@ -237,6 +238,7 @@ struct CmdWhyDepends : SourceExprCommand visitPath(node.path); + RunPager pager; for (auto & ref : refs) { auto hash = storePathToHash(ref.second->path); diff --git a/src/nlohmann/json.hpp b/src/nlohmann/json.hpp index 5b0b0ea5b301..c9af0bed36d6 100644 --- a/src/nlohmann/json.hpp +++ b/src/nlohmann/json.hpp @@ -1,11 +1,12 @@ /* __ _____ _____ _____ __| | __| | | | JSON for Modern C++ -| | |__ | | | | | | version 3.0.1 +| | |__ | | | | | | version 3.5.0 |_____|_____|_____|_|___| https://github.com/nlohmann/json Licensed under the MIT License <http://opensource.org/licenses/MIT>. -Copyright (c) 2013-2017 Niels Lohmann <http://nlohmann.me>. +SPDX-License-Identifier: MIT +Copyright (c) 2013-2018 Niels Lohmann <http://nlohmann.me>. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -29,42 +30,104 @@ SOFTWARE. #ifndef NLOHMANN_JSON_HPP #define NLOHMANN_JSON_HPP -#include <algorithm> // all_of, copy, fill, find, for_each, generate_n, none_of, remove, reverse, transform -#include <array> // array +#define NLOHMANN_JSON_VERSION_MAJOR 3 +#define NLOHMANN_JSON_VERSION_MINOR 5 +#define NLOHMANN_JSON_VERSION_PATCH 0 + +#include <algorithm> // all_of, find, for_each #include <cassert> // assert #include <ciso646> // and, not, or -#include <clocale> // lconv, localeconv -#include <cmath> // isfinite, labs, ldexp, signbit #include <cstddef> // nullptr_t, ptrdiff_t, size_t -#include <cstdint> // int64_t, uint64_t -#include <cstdlib> // abort, strtod, strtof, strtold, strtoul, strtoll, strtoull -#include <cstring> // memcpy, strlen -#include <forward_list> // forward_list -#include <functional> // function, hash, less +#include <functional> // hash, less #include <initializer_list> // initializer_list -#include <iomanip> // hex -#include <iosfwd> // istream, ostream -#include <iterator> // advance, begin, back_inserter, bidirectional_iterator_tag, distance, end, inserter, iterator, iterator_traits, next, random_access_iterator_tag, reverse_iterator -#include <limits> // numeric_limits -#include <locale> // locale -#include <map> // map -#include <memory> // addressof, allocator, allocator_traits, unique_ptr +#include <iosfwd> // istream, ostream +#include <iterator> // random_access_iterator_tag #include <numeric> // accumulate -#include <sstream> // stringstream -#include <string> // getline, stoi, string, to_string -#include <type_traits> // add_pointer, conditional, decay, enable_if, false_type, integral_constant, is_arithmetic, is_base_of, is_const, is_constructible, is_convertible, is_default_constructible, is_enum, is_floating_point, is_integral, is_nothrow_move_assignable, is_nothrow_move_constructible, is_pointer, is_reference, is_same, is_scalar, is_signed, remove_const, remove_cv, remove_pointer, remove_reference, true_type, underlying_type -#include <utility> // declval, forward, make_pair, move, pair, swap -#include <valarray> // valarray +#include <string> // string, stoi, to_string +#include <utility> // declval, forward, move, pair, swap + +// #include <nlohmann/json_fwd.hpp> +#ifndef NLOHMANN_JSON_FWD_HPP +#define NLOHMANN_JSON_FWD_HPP + +#include <cstdint> // int64_t, uint64_t +#include <map> // map +#include <memory> // allocator +#include <string> // string #include <vector> // vector +/*! +@brief namespace for Niels Lohmann +@see https://github.com/nlohmann +@since version 1.0.0 +*/ +namespace nlohmann +{ +/*! +@brief default JSONSerializer template argument + +This serializer ignores the template arguments and uses ADL +([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl)) +for serialization. +*/ +template<typename T = void, typename SFINAE = void> +struct adl_serializer; + +template<template<typename U, typename V, typename... Args> class ObjectType = + std::map, + template<typename U, typename... Args> class ArrayType = std::vector, + class StringType = std::string, class BooleanType = bool, + class NumberIntegerType = std::int64_t, + class NumberUnsignedType = std::uint64_t, + class NumberFloatType = double, + template<typename U> class AllocatorType = std::allocator, + template<typename T, typename SFINAE = void> class JSONSerializer = + adl_serializer> +class basic_json; + +/*! +@brief JSON Pointer + +A JSON pointer defines a string syntax for identifying a specific value +within a JSON document. It can be used with functions `at` and +`operator[]`. Furthermore, JSON pointers are the base for JSON patches. + +@sa [RFC 6901](https://tools.ietf.org/html/rfc6901) + +@since version 2.0.0 +*/ +template<typename BasicJsonType> +class json_pointer; + +/*! +@brief default JSON class + +This type is the default specialization of the @ref basic_json class which +uses the standard template types. + +@since version 1.0.0 +*/ +using json = basic_json<>; +} // namespace nlohmann + +#endif + +// #include <nlohmann/detail/macro_scope.hpp> + + +// This file contains all internal macro definitions +// You MUST include macro_unscope.hpp at the end of json.hpp to undef all of them + // exclude unsupported compilers -#if defined(__clang__) - #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400 - #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" - #endif -#elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER)) - #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40900 - #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" +#if !defined(JSON_SKIP_UNSUPPORTED_COMPILER_CHECK) + #if defined(__clang__) + #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400 + #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" + #endif + #elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER)) + #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40800 + #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" + #endif #endif #endif @@ -90,14 +153,36 @@ SOFTWARE. #endif // allow to disable exceptions -#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && not defined(JSON_NOEXCEPTION) +#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION) #define JSON_THROW(exception) throw exception #define JSON_TRY try #define JSON_CATCH(exception) catch(exception) + #define JSON_INTERNAL_CATCH(exception) catch(exception) #else #define JSON_THROW(exception) std::abort() #define JSON_TRY if(true) #define JSON_CATCH(exception) if(false) + #define JSON_INTERNAL_CATCH(exception) if(false) +#endif + +// override exception macros +#if defined(JSON_THROW_USER) + #undef JSON_THROW + #define JSON_THROW JSON_THROW_USER +#endif +#if defined(JSON_TRY_USER) + #undef JSON_TRY + #define JSON_TRY JSON_TRY_USER +#endif +#if defined(JSON_CATCH_USER) + #undef JSON_CATCH + #define JSON_CATCH JSON_CATCH_USER + #undef JSON_INTERNAL_CATCH + #define JSON_INTERNAL_CATCH JSON_CATCH_USER +#endif +#if defined(JSON_INTERNAL_CATCH_USER) + #undef JSON_INTERNAL_CATCH + #define JSON_INTERNAL_CATCH JSON_INTERNAL_CATCH_USER #endif // manual branch prediction @@ -118,25 +203,35 @@ SOFTWARE. #endif /*! -@brief namespace for Niels Lohmann -@see https://github.com/nlohmann -@since version 1.0.0 +@brief macro to briefly define a mapping between an enum and JSON +@def NLOHMANN_JSON_SERIALIZE_ENUM +@since version 3.4.0 */ -namespace nlohmann -{ -template<typename = void, typename = void> -struct adl_serializer; - -// forward declaration of basic_json (required to split the class) -template<template<typename, typename, typename...> class ObjectType = std::map, - template<typename, typename...> class ArrayType = std::vector, - class StringType = std::string, class BooleanType = bool, - class NumberIntegerType = std::int64_t, - class NumberUnsignedType = std::uint64_t, - class NumberFloatType = double, - template<typename> class AllocatorType = std::allocator, - template<typename, typename = void> class JSONSerializer = adl_serializer> -class basic_json; +#define NLOHMANN_JSON_SERIALIZE_ENUM(ENUM_TYPE, ...) \ + template<typename BasicJsonType> \ + inline void to_json(BasicJsonType& j, const ENUM_TYPE& e) \ + { \ + static_assert(std::is_enum<ENUM_TYPE>::value, #ENUM_TYPE " must be an enum!"); \ + static const std::pair<ENUM_TYPE, BasicJsonType> m[] = __VA_ARGS__; \ + auto it = std::find_if(std::begin(m), std::end(m), \ + [e](const std::pair<ENUM_TYPE, BasicJsonType>& ej_pair) -> bool \ + { \ + return ej_pair.first == e; \ + }); \ + j = ((it != std::end(m)) ? it : std::begin(m))->second; \ + } \ + template<typename BasicJsonType> \ + inline void from_json(const BasicJsonType& j, ENUM_TYPE& e) \ + { \ + static_assert(std::is_enum<ENUM_TYPE>::value, #ENUM_TYPE " must be an enum!"); \ + static const std::pair<ENUM_TYPE, BasicJsonType> m[] = __VA_ARGS__; \ + auto it = std::find_if(std::begin(m), std::end(m), \ + [j](const std::pair<ENUM_TYPE, BasicJsonType>& ej_pair) -> bool \ + { \ + return ej_pair.second == j; \ + }); \ + e = ((it != std::end(m)) ? it : std::begin(m))->first; \ + } // Ugly macros to avoid uglier copy-paste when specializing basic_json. They // may be removed in the future once the class is split. @@ -154,17 +249,590 @@ class basic_json; NumberIntegerType, NumberUnsignedType, NumberFloatType, \ AllocatorType, JSONSerializer> +// #include <nlohmann/detail/meta/cpp_future.hpp> + + +#include <ciso646> // not +#include <cstddef> // size_t +#include <type_traits> // conditional, enable_if, false_type, integral_constant, is_constructible, is_integral, is_same, remove_cv, remove_reference, true_type + +namespace nlohmann +{ +namespace detail +{ +// alias templates to reduce boilerplate +template<bool B, typename T = void> +using enable_if_t = typename std::enable_if<B, T>::type; + +template<typename T> +using uncvref_t = typename std::remove_cv<typename std::remove_reference<T>::type>::type; + +// implementation of C++14 index_sequence and affiliates +// source: https://stackoverflow.com/a/32223343 +template<std::size_t... Ints> +struct index_sequence +{ + using type = index_sequence; + using value_type = std::size_t; + static constexpr std::size_t size() noexcept + { + return sizeof...(Ints); + } +}; + +template<class Sequence1, class Sequence2> +struct merge_and_renumber; + +template<std::size_t... I1, std::size_t... I2> +struct merge_and_renumber<index_sequence<I1...>, index_sequence<I2...>> + : index_sequence < I1..., (sizeof...(I1) + I2)... > {}; + +template<std::size_t N> +struct make_index_sequence + : merge_and_renumber < typename make_index_sequence < N / 2 >::type, + typename make_index_sequence < N - N / 2 >::type > {}; + +template<> struct make_index_sequence<0> : index_sequence<> {}; +template<> struct make_index_sequence<1> : index_sequence<0> {}; + +template<typename... Ts> +using index_sequence_for = make_index_sequence<sizeof...(Ts)>; + +// dispatch utility (taken from ranges-v3) +template<unsigned N> struct priority_tag : priority_tag < N - 1 > {}; +template<> struct priority_tag<0> {}; + +// taken from ranges-v3 +template<typename T> +struct static_const +{ + static constexpr T value{}; +}; + +template<typename T> +constexpr T static_const<T>::value; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/meta/type_traits.hpp> + + +#include <ciso646> // not +#include <limits> // numeric_limits +#include <type_traits> // false_type, is_constructible, is_integral, is_same, true_type +#include <utility> // declval + +// #include <nlohmann/json_fwd.hpp> + +// #include <nlohmann/detail/iterators/iterator_traits.hpp> + + +#include <iterator> // random_access_iterator_tag + +// #include <nlohmann/detail/meta/void_t.hpp> + + +namespace nlohmann +{ +namespace detail +{ +template <typename ...Ts> struct make_void +{ + using type = void; +}; +template <typename ...Ts> using void_t = typename make_void<Ts...>::type; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/meta/cpp_future.hpp> + +namespace nlohmann +{ +namespace detail +{ +template <typename It, typename = void> +struct iterator_types {}; + +template <typename It> +struct iterator_types < + It, + void_t<typename It::difference_type, typename It::value_type, typename It::pointer, + typename It::reference, typename It::iterator_category >> +{ + using difference_type = typename It::difference_type; + using value_type = typename It::value_type; + using pointer = typename It::pointer; + using reference = typename It::reference; + using iterator_category = typename It::iterator_category; +}; + +// This is required as some compilers implement std::iterator_traits in a way that +// doesn't work with SFINAE. See https://github.com/nlohmann/json/issues/1341. +template <typename T, typename = void> +struct iterator_traits +{ +}; + +template <typename T> +struct iterator_traits < T, enable_if_t < !std::is_pointer<T>::value >> + : iterator_types<T> +{ +}; + +template <typename T> +struct iterator_traits<T*, enable_if_t<std::is_object<T>::value>> +{ + using iterator_category = std::random_access_iterator_tag; + using value_type = T; + using difference_type = ptrdiff_t; + using pointer = T*; + using reference = T&; +}; +} +} + +// #include <nlohmann/detail/meta/cpp_future.hpp> + +// #include <nlohmann/detail/meta/detected.hpp> + + +#include <type_traits> + +// #include <nlohmann/detail/meta/void_t.hpp> + + +// http://en.cppreference.com/w/cpp/experimental/is_detected +namespace nlohmann +{ +namespace detail +{ +struct nonesuch +{ + nonesuch() = delete; + ~nonesuch() = delete; + nonesuch(nonesuch const&) = delete; + void operator=(nonesuch const&) = delete; +}; + +template <class Default, + class AlwaysVoid, + template <class...> class Op, + class... Args> +struct detector +{ + using value_t = std::false_type; + using type = Default; +}; + +template <class Default, template <class...> class Op, class... Args> +struct detector<Default, void_t<Op<Args...>>, Op, Args...> +{ + using value_t = std::true_type; + using type = Op<Args...>; +}; + +template <template <class...> class Op, class... Args> +using is_detected = typename detector<nonesuch, void, Op, Args...>::value_t; + +template <template <class...> class Op, class... Args> +using detected_t = typename detector<nonesuch, void, Op, Args...>::type; + +template <class Default, template <class...> class Op, class... Args> +using detected_or = detector<Default, void, Op, Args...>; + +template <class Default, template <class...> class Op, class... Args> +using detected_or_t = typename detected_or<Default, Op, Args...>::type; + +template <class Expected, template <class...> class Op, class... Args> +using is_detected_exact = std::is_same<Expected, detected_t<Op, Args...>>; + +template <class To, template <class...> class Op, class... Args> +using is_detected_convertible = + std::is_convertible<detected_t<Op, Args...>, To>; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/macro_scope.hpp> + + +namespace nlohmann +{ /*! -@brief unnamed namespace with internal helper functions +@brief detail namespace with internal helper functions -This namespace collects some functions that could not be defined inside the -@ref basic_json class. +This namespace collects functions that should not be exposed, +implementations of some @ref basic_json methods, and meta-programming helpers. @since version 2.1.0 */ namespace detail { +///////////// +// helpers // +///////////// + +// Note to maintainers: +// +// Every trait in this file expects a non CV-qualified type. +// The only exceptions are in the 'aliases for detected' section +// (i.e. those of the form: decltype(T::member_function(std::declval<T>()))) +// +// In this case, T has to be properly CV-qualified to constraint the function arguments +// (e.g. to_json(BasicJsonType&, const T&)) + +template<typename> struct is_basic_json : std::false_type {}; + +NLOHMANN_BASIC_JSON_TPL_DECLARATION +struct is_basic_json<NLOHMANN_BASIC_JSON_TPL> : std::true_type {}; + +////////////////////////// +// aliases for detected // +////////////////////////// + +template <typename T> +using mapped_type_t = typename T::mapped_type; + +template <typename T> +using key_type_t = typename T::key_type; + +template <typename T> +using value_type_t = typename T::value_type; + +template <typename T> +using difference_type_t = typename T::difference_type; + +template <typename T> +using pointer_t = typename T::pointer; + +template <typename T> +using reference_t = typename T::reference; + +template <typename T> +using iterator_category_t = typename T::iterator_category; + +template <typename T> +using iterator_t = typename T::iterator; + +template <typename T, typename... Args> +using to_json_function = decltype(T::to_json(std::declval<Args>()...)); + +template <typename T, typename... Args> +using from_json_function = decltype(T::from_json(std::declval<Args>()...)); + +template <typename T, typename U> +using get_template_function = decltype(std::declval<T>().template get<U>()); + +// trait checking if JSONSerializer<T>::from_json(json const&, udt&) exists +template <typename BasicJsonType, typename T, typename = void> +struct has_from_json : std::false_type {}; + +template <typename BasicJsonType, typename T> +struct has_from_json<BasicJsonType, T, + enable_if_t<not is_basic_json<T>::value>> +{ + using serializer = typename BasicJsonType::template json_serializer<T, void>; + + static constexpr bool value = + is_detected_exact<void, from_json_function, serializer, + const BasicJsonType&, T&>::value; +}; + +// This trait checks if JSONSerializer<T>::from_json(json const&) exists +// this overload is used for non-default-constructible user-defined-types +template <typename BasicJsonType, typename T, typename = void> +struct has_non_default_from_json : std::false_type {}; + +template<typename BasicJsonType, typename T> +struct has_non_default_from_json<BasicJsonType, T, enable_if_t<not is_basic_json<T>::value>> +{ + using serializer = typename BasicJsonType::template json_serializer<T, void>; + + static constexpr bool value = + is_detected_exact<T, from_json_function, serializer, + const BasicJsonType&>::value; +}; + +// This trait checks if BasicJsonType::json_serializer<T>::to_json exists +// Do not evaluate the trait when T is a basic_json type, to avoid template instantiation infinite recursion. +template <typename BasicJsonType, typename T, typename = void> +struct has_to_json : std::false_type {}; + +template <typename BasicJsonType, typename T> +struct has_to_json<BasicJsonType, T, enable_if_t<not is_basic_json<T>::value>> +{ + using serializer = typename BasicJsonType::template json_serializer<T, void>; + + static constexpr bool value = + is_detected_exact<void, to_json_function, serializer, BasicJsonType&, + T>::value; +}; + + +/////////////////// +// is_ functions // +/////////////////// + +template <typename T, typename = void> +struct is_iterator_traits : std::false_type {}; + +template <typename T> +struct is_iterator_traits<iterator_traits<T>> +{ + private: + using traits = iterator_traits<T>; + + public: + static constexpr auto value = + is_detected<value_type_t, traits>::value && + is_detected<difference_type_t, traits>::value && + is_detected<pointer_t, traits>::value && + is_detected<iterator_category_t, traits>::value && + is_detected<reference_t, traits>::value; +}; + +// source: https://stackoverflow.com/a/37193089/4116453 + +template <typename T, typename = void> +struct is_complete_type : std::false_type {}; + +template <typename T> +struct is_complete_type<T, decltype(void(sizeof(T)))> : std::true_type {}; + +template <typename BasicJsonType, typename CompatibleObjectType, + typename = void> +struct is_compatible_object_type_impl : std::false_type {}; + +template <typename BasicJsonType, typename CompatibleObjectType> +struct is_compatible_object_type_impl < + BasicJsonType, CompatibleObjectType, + enable_if_t<is_detected<mapped_type_t, CompatibleObjectType>::value and + is_detected<key_type_t, CompatibleObjectType>::value >> +{ + + using object_t = typename BasicJsonType::object_t; + + // macOS's is_constructible does not play well with nonesuch... + static constexpr bool value = + std::is_constructible<typename object_t::key_type, + typename CompatibleObjectType::key_type>::value and + std::is_constructible<typename object_t::mapped_type, + typename CompatibleObjectType::mapped_type>::value; +}; + +template <typename BasicJsonType, typename CompatibleObjectType> +struct is_compatible_object_type + : is_compatible_object_type_impl<BasicJsonType, CompatibleObjectType> {}; + +template <typename BasicJsonType, typename ConstructibleObjectType, + typename = void> +struct is_constructible_object_type_impl : std::false_type {}; + +template <typename BasicJsonType, typename ConstructibleObjectType> +struct is_constructible_object_type_impl < + BasicJsonType, ConstructibleObjectType, + enable_if_t<is_detected<mapped_type_t, ConstructibleObjectType>::value and + is_detected<key_type_t, ConstructibleObjectType>::value >> +{ + using object_t = typename BasicJsonType::object_t; + + static constexpr bool value = + (std::is_constructible<typename ConstructibleObjectType::key_type, typename object_t::key_type>::value and + std::is_same<typename object_t::mapped_type, typename ConstructibleObjectType::mapped_type>::value) or + (has_from_json<BasicJsonType, typename ConstructibleObjectType::mapped_type>::value or + has_non_default_from_json<BasicJsonType, typename ConstructibleObjectType::mapped_type >::value); +}; + +template <typename BasicJsonType, typename ConstructibleObjectType> +struct is_constructible_object_type + : is_constructible_object_type_impl<BasicJsonType, + ConstructibleObjectType> {}; + +template <typename BasicJsonType, typename CompatibleStringType, + typename = void> +struct is_compatible_string_type_impl : std::false_type {}; + +template <typename BasicJsonType, typename CompatibleStringType> +struct is_compatible_string_type_impl < + BasicJsonType, CompatibleStringType, + enable_if_t<is_detected_exact<typename BasicJsonType::string_t::value_type, + value_type_t, CompatibleStringType>::value >> +{ + static constexpr auto value = + std::is_constructible<typename BasicJsonType::string_t, CompatibleStringType>::value; +}; + +template <typename BasicJsonType, typename ConstructibleStringType> +struct is_compatible_string_type + : is_compatible_string_type_impl<BasicJsonType, ConstructibleStringType> {}; + +template <typename BasicJsonType, typename ConstructibleStringType, + typename = void> +struct is_constructible_string_type_impl : std::false_type {}; + +template <typename BasicJsonType, typename ConstructibleStringType> +struct is_constructible_string_type_impl < + BasicJsonType, ConstructibleStringType, + enable_if_t<is_detected_exact<typename BasicJsonType::string_t::value_type, + value_type_t, ConstructibleStringType>::value >> +{ + static constexpr auto value = + std::is_constructible<ConstructibleStringType, + typename BasicJsonType::string_t>::value; +}; + +template <typename BasicJsonType, typename ConstructibleStringType> +struct is_constructible_string_type + : is_constructible_string_type_impl<BasicJsonType, ConstructibleStringType> {}; + +template <typename BasicJsonType, typename CompatibleArrayType, typename = void> +struct is_compatible_array_type_impl : std::false_type {}; + +template <typename BasicJsonType, typename CompatibleArrayType> +struct is_compatible_array_type_impl < + BasicJsonType, CompatibleArrayType, + enable_if_t<is_detected<value_type_t, CompatibleArrayType>::value and + is_detected<iterator_t, CompatibleArrayType>::value and +// This is needed because json_reverse_iterator has a ::iterator type... +// Therefore it is detected as a CompatibleArrayType. +// The real fix would be to have an Iterable concept. + not is_iterator_traits< + iterator_traits<CompatibleArrayType>>::value >> +{ + static constexpr bool value = + std::is_constructible<BasicJsonType, + typename CompatibleArrayType::value_type>::value; +}; + +template <typename BasicJsonType, typename CompatibleArrayType> +struct is_compatible_array_type + : is_compatible_array_type_impl<BasicJsonType, CompatibleArrayType> {}; + +template <typename BasicJsonType, typename ConstructibleArrayType, typename = void> +struct is_constructible_array_type_impl : std::false_type {}; + +template <typename BasicJsonType, typename ConstructibleArrayType> +struct is_constructible_array_type_impl < + BasicJsonType, ConstructibleArrayType, + enable_if_t<std::is_same<ConstructibleArrayType, + typename BasicJsonType::value_type>::value >> + : std::true_type {}; + +template <typename BasicJsonType, typename ConstructibleArrayType> +struct is_constructible_array_type_impl < + BasicJsonType, ConstructibleArrayType, + enable_if_t<not std::is_same<ConstructibleArrayType, + typename BasicJsonType::value_type>::value and + is_detected<value_type_t, ConstructibleArrayType>::value and + is_detected<iterator_t, ConstructibleArrayType>::value and + is_complete_type< + detected_t<value_type_t, ConstructibleArrayType>>::value >> +{ + static constexpr bool value = + // This is needed because json_reverse_iterator has a ::iterator type, + // furthermore, std::back_insert_iterator (and other iterators) have a base class `iterator`... + // Therefore it is detected as a ConstructibleArrayType. + // The real fix would be to have an Iterable concept. + not is_iterator_traits < + iterator_traits<ConstructibleArrayType >>::value and + + (std::is_same<typename ConstructibleArrayType::value_type, typename BasicJsonType::array_t::value_type>::value or + has_from_json<BasicJsonType, + typename ConstructibleArrayType::value_type>::value or + has_non_default_from_json < + BasicJsonType, typename ConstructibleArrayType::value_type >::value); +}; + +template <typename BasicJsonType, typename ConstructibleArrayType> +struct is_constructible_array_type + : is_constructible_array_type_impl<BasicJsonType, ConstructibleArrayType> {}; + +template <typename RealIntegerType, typename CompatibleNumberIntegerType, + typename = void> +struct is_compatible_integer_type_impl : std::false_type {}; + +template <typename RealIntegerType, typename CompatibleNumberIntegerType> +struct is_compatible_integer_type_impl < + RealIntegerType, CompatibleNumberIntegerType, + enable_if_t<std::is_integral<RealIntegerType>::value and + std::is_integral<CompatibleNumberIntegerType>::value and + not std::is_same<bool, CompatibleNumberIntegerType>::value >> +{ + // is there an assert somewhere on overflows? + using RealLimits = std::numeric_limits<RealIntegerType>; + using CompatibleLimits = std::numeric_limits<CompatibleNumberIntegerType>; + + static constexpr auto value = + std::is_constructible<RealIntegerType, + CompatibleNumberIntegerType>::value and + CompatibleLimits::is_integer and + RealLimits::is_signed == CompatibleLimits::is_signed; +}; + +template <typename RealIntegerType, typename CompatibleNumberIntegerType> +struct is_compatible_integer_type + : is_compatible_integer_type_impl<RealIntegerType, + CompatibleNumberIntegerType> {}; + +template <typename BasicJsonType, typename CompatibleType, typename = void> +struct is_compatible_type_impl: std::false_type {}; + +template <typename BasicJsonType, typename CompatibleType> +struct is_compatible_type_impl < + BasicJsonType, CompatibleType, + enable_if_t<is_complete_type<CompatibleType>::value >> +{ + static constexpr bool value = + has_to_json<BasicJsonType, CompatibleType>::value; +}; + +template <typename BasicJsonType, typename CompatibleType> +struct is_compatible_type + : is_compatible_type_impl<BasicJsonType, CompatibleType> {}; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/exceptions.hpp> + + +#include <exception> // exception +#include <stdexcept> // runtime_error +#include <string> // to_string + +// #include <nlohmann/detail/input/position_t.hpp> + + +#include <cstddef> // size_t + +namespace nlohmann +{ +namespace detail +{ +/// struct to capture the start position of the current token +struct position_t +{ + /// the total number of characters read + std::size_t chars_read_total = 0; + /// the number of characters read in the current line + std::size_t chars_read_current_line = 0; + /// the number of lines read + std::size_t lines_read = 0; + + /// conversion to size_t to preserve SAX interface + constexpr operator size_t() const + { + return chars_read_total; + } +}; + +} +} + + +namespace nlohmann +{ +namespace detail +{ //////////////// // exceptions // //////////////// @@ -248,6 +916,7 @@ json.exception.parse_error.109 | parse error: array index 'one' is not a number json.exception.parse_error.110 | parse error at 1: cannot read 2 bytes from vector | When parsing CBOR or MessagePack, the byte vector ends before the complete value has been read. json.exception.parse_error.112 | parse error at 1: error reading CBOR; last byte: 0xF8 | Not all types of CBOR or MessagePack are supported. This exception occurs if an unsupported byte was read. json.exception.parse_error.113 | parse error at 2: expected a CBOR string; last byte: 0x98 | While parsing a map key, a value that is not a string has been read. +json.exception.parse_error.114 | parse error: Unsupported BSON record type 0x0F | The parsing of the corresponding BSON record type is not implemented (yet). @note For an input with n bytes, 1 is the index of the first character and n+1 is the index of the terminating null byte or the end of file. This also @@ -271,15 +940,23 @@ class parse_error : public exception /*! @brief create a parse error exception @param[in] id_ the id of the exception - @param[in] byte_ the byte index where the error occurred (or 0 if the - position cannot be determined) + @param[in] position the position where the error occurred (or with + chars_read_total=0 if the position cannot be + determined) @param[in] what_arg the explanatory string @return parse_error object */ + static parse_error create(int id_, const position_t& pos, const std::string& what_arg) + { + std::string w = exception::name("parse_error", id_) + "parse error" + + position_string(pos) + ": " + what_arg; + return parse_error(id_, pos.chars_read_total, w.c_str()); + } + static parse_error create(int id_, std::size_t byte_, const std::string& what_arg) { std::string w = exception::name("parse_error", id_) + "parse error" + - (byte_ != 0 ? (" at " + std::to_string(byte_)) : "") + + (byte_ != 0 ? (" at byte " + std::to_string(byte_)) : "") + ": " + what_arg; return parse_error(id_, byte_, w.c_str()); } @@ -298,6 +975,12 @@ class parse_error : public exception private: parse_error(int id_, std::size_t byte_, const char* what_arg) : exception(id_, what_arg), byte(byte_) {} + + static std::string position_string(const position_t& pos) + { + return " at line " + std::to_string(pos.lines_read + 1) + + ", column " + std::to_string(pos.chars_read_current_line); + } }; /*! @@ -377,6 +1060,7 @@ json.exception.type_error.313 | invalid value to unflatten | The @ref unflatten json.exception.type_error.314 | only objects can be unflattened | The @ref unflatten function only works for an object whose keys are JSON Pointers. json.exception.type_error.315 | values in object must be primitive | The @ref unflatten function only works for an object whose keys are JSON Pointers and whose values are primitive. json.exception.type_error.316 | invalid UTF-8 byte at index 10: 0x7E | The @ref dump function only works with UTF-8 encoded strings; that is, if you assign a `std::string` to a JSON value, make sure it is UTF-8 encoded. | +json.exception.type_error.317 | JSON value cannot be serialized to requested format | The dynamic type of the object cannot be represented in the requested serialization format (e.g. a raw `true` or `null` JSON object cannot be serialized to BSON) | @liveexample{The following code shows how a `type_error` exception can be caught.,type_error} @@ -419,6 +1103,9 @@ json.exception.out_of_range.403 | key 'foo' not found | The provided key was not json.exception.out_of_range.404 | unresolved reference token 'foo' | A reference token in a JSON Pointer could not be resolved. json.exception.out_of_range.405 | JSON pointer has no parent | The JSON Patch operations 'remove' and 'add' can not be applied to the root element of the JSON value. json.exception.out_of_range.406 | number overflow parsing '10E1000' | A parsed number could not be stored as without changing it to NaN or INF. +json.exception.out_of_range.407 | number overflow serializing '9223372036854775808' | UBJSON and BSON only support integer numbers up to 9223372036854775807. | +json.exception.out_of_range.408 | excessive array size: 8658170730974374167 | The size (following `#`) of an UBJSON array or object exceeds the maximal capacity. | +json.exception.out_of_range.409 | BSON key cannot contain code point U+0000 (at byte 2) | Key identifiers to be serialized to BSON cannot contain code point U+0000, since the key is stored as zero-terminated c-string | @liveexample{The following code shows how an `out_of_range` exception can be caught.,out_of_range} @@ -481,9 +1168,21 @@ class other_error : public exception private: other_error(int id_, const char* what_arg) : exception(id_, what_arg) {} }; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/value_t.hpp> +#include <array> // array +#include <ciso646> // and +#include <cstddef> // size_t +#include <cstdint> // uint8_t +namespace nlohmann +{ +namespace detail +{ /////////////////////////// // JSON type enumeration // /////////////////////////// @@ -512,7 +1211,7 @@ value with the default value for a given type @since version 1.0.0 */ -enum class value_t : uint8_t +enum class value_t : std::uint8_t { null, ///< null value object, ///< object (unordered set of name/value pairs) @@ -537,7 +1236,7 @@ Returns an ordering that is similar to Python: */ inline bool operator<(const value_t lhs, const value_t rhs) noexcept { - static constexpr std::array<uint8_t, 8> order = {{ + static constexpr std::array<std::uint8_t, 8> order = {{ 0 /* null */, 3 /* object */, 4 /* array */, 5 /* string */, 1 /* boolean */, 2 /* integer */, 2 /* unsigned */, 2 /* float */ } @@ -547,80 +1246,573 @@ inline bool operator<(const value_t lhs, const value_t rhs) noexcept const auto r_index = static_cast<std::size_t>(rhs); return l_index < order.size() and r_index < order.size() and order[l_index] < order[r_index]; } +} // namespace detail +} // namespace nlohmann +// #include <nlohmann/detail/conversions/from_json.hpp> -///////////// -// helpers // -///////////// -template<typename> struct is_basic_json : std::false_type {}; +#include <algorithm> // transform +#include <array> // array +#include <ciso646> // and, not +#include <forward_list> // forward_list +#include <iterator> // inserter, front_inserter, end +#include <map> // map +#include <string> // string +#include <tuple> // tuple, make_tuple +#include <type_traits> // is_arithmetic, is_same, is_enum, underlying_type, is_convertible +#include <unordered_map> // unordered_map +#include <utility> // pair, declval +#include <valarray> // valarray -NLOHMANN_BASIC_JSON_TPL_DECLARATION -struct is_basic_json<NLOHMANN_BASIC_JSON_TPL> : std::true_type {}; +// #include <nlohmann/detail/exceptions.hpp> -// alias templates to reduce boilerplate -template<bool B, typename T = void> -using enable_if_t = typename std::enable_if<B, T>::type; +// #include <nlohmann/detail/macro_scope.hpp> -template<typename T> -using uncvref_t = typename std::remove_cv<typename std::remove_reference<T>::type>::type; +// #include <nlohmann/detail/meta/cpp_future.hpp> -// implementation of C++14 index_sequence and affiliates -// source: https://stackoverflow.com/a/32223343 -template<std::size_t... Ints> -struct index_sequence +// #include <nlohmann/detail/meta/type_traits.hpp> + +// #include <nlohmann/detail/value_t.hpp> + + +namespace nlohmann { - using type = index_sequence; - using value_type = std::size_t; - static constexpr std::size_t size() noexcept +namespace detail +{ +template<typename BasicJsonType> +void from_json(const BasicJsonType& j, typename std::nullptr_t& n) +{ + if (JSON_UNLIKELY(not j.is_null())) { - return sizeof...(Ints); + JSON_THROW(type_error::create(302, "type must be null, but is " + std::string(j.type_name()))); + } + n = nullptr; +} + +// overloads for basic_json template parameters +template<typename BasicJsonType, typename ArithmeticType, + enable_if_t<std::is_arithmetic<ArithmeticType>::value and + not std::is_same<ArithmeticType, typename BasicJsonType::boolean_t>::value, + int> = 0> +void get_arithmetic_value(const BasicJsonType& j, ArithmeticType& val) +{ + switch (static_cast<value_t>(j)) + { + case value_t::number_unsigned: + { + val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_unsigned_t*>()); + break; + } + case value_t::number_integer: + { + val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_integer_t*>()); + break; + } + case value_t::number_float: + { + val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_float_t*>()); + break; + } + + default: + JSON_THROW(type_error::create(302, "type must be number, but is " + std::string(j.type_name()))); + } +} + +template<typename BasicJsonType> +void from_json(const BasicJsonType& j, typename BasicJsonType::boolean_t& b) +{ + if (JSON_UNLIKELY(not j.is_boolean())) + { + JSON_THROW(type_error::create(302, "type must be boolean, but is " + std::string(j.type_name()))); + } + b = *j.template get_ptr<const typename BasicJsonType::boolean_t*>(); +} + +template<typename BasicJsonType> +void from_json(const BasicJsonType& j, typename BasicJsonType::string_t& s) +{ + if (JSON_UNLIKELY(not j.is_string())) + { + JSON_THROW(type_error::create(302, "type must be string, but is " + std::string(j.type_name()))); + } + s = *j.template get_ptr<const typename BasicJsonType::string_t*>(); +} + +template < + typename BasicJsonType, typename ConstructibleStringType, + enable_if_t < + is_constructible_string_type<BasicJsonType, ConstructibleStringType>::value and + not std::is_same<typename BasicJsonType::string_t, + ConstructibleStringType>::value, + int > = 0 > +void from_json(const BasicJsonType& j, ConstructibleStringType& s) +{ + if (JSON_UNLIKELY(not j.is_string())) + { + JSON_THROW(type_error::create(302, "type must be string, but is " + std::string(j.type_name()))); + } + + s = *j.template get_ptr<const typename BasicJsonType::string_t*>(); +} + +template<typename BasicJsonType> +void from_json(const BasicJsonType& j, typename BasicJsonType::number_float_t& val) +{ + get_arithmetic_value(j, val); +} + +template<typename BasicJsonType> +void from_json(const BasicJsonType& j, typename BasicJsonType::number_unsigned_t& val) +{ + get_arithmetic_value(j, val); +} + +template<typename BasicJsonType> +void from_json(const BasicJsonType& j, typename BasicJsonType::number_integer_t& val) +{ + get_arithmetic_value(j, val); +} + +template<typename BasicJsonType, typename EnumType, + enable_if_t<std::is_enum<EnumType>::value, int> = 0> +void from_json(const BasicJsonType& j, EnumType& e) +{ + typename std::underlying_type<EnumType>::type val; + get_arithmetic_value(j, val); + e = static_cast<EnumType>(val); +} + +// forward_list doesn't have an insert method +template<typename BasicJsonType, typename T, typename Allocator, + enable_if_t<std::is_convertible<BasicJsonType, T>::value, int> = 0> +void from_json(const BasicJsonType& j, std::forward_list<T, Allocator>& l) +{ + if (JSON_UNLIKELY(not j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()))); + } + std::transform(j.rbegin(), j.rend(), + std::front_inserter(l), [](const BasicJsonType & i) + { + return i.template get<T>(); + }); +} + +// valarray doesn't have an insert method +template<typename BasicJsonType, typename T, + enable_if_t<std::is_convertible<BasicJsonType, T>::value, int> = 0> +void from_json(const BasicJsonType& j, std::valarray<T>& l) +{ + if (JSON_UNLIKELY(not j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()))); + } + l.resize(j.size()); + std::copy(j.m_value.array->begin(), j.m_value.array->end(), std::begin(l)); +} + +template<typename BasicJsonType> +void from_json_array_impl(const BasicJsonType& j, typename BasicJsonType::array_t& arr, priority_tag<3> /*unused*/) +{ + arr = *j.template get_ptr<const typename BasicJsonType::array_t*>(); +} + +template <typename BasicJsonType, typename T, std::size_t N> +auto from_json_array_impl(const BasicJsonType& j, std::array<T, N>& arr, + priority_tag<2> /*unused*/) +-> decltype(j.template get<T>(), void()) +{ + for (std::size_t i = 0; i < N; ++i) + { + arr[i] = j.at(i).template get<T>(); + } +} + +template<typename BasicJsonType, typename ConstructibleArrayType> +auto from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr, priority_tag<1> /*unused*/) +-> decltype( + arr.reserve(std::declval<typename ConstructibleArrayType::size_type>()), + j.template get<typename ConstructibleArrayType::value_type>(), + void()) +{ + using std::end; + + arr.reserve(j.size()); + std::transform(j.begin(), j.end(), + std::inserter(arr, end(arr)), [](const BasicJsonType & i) + { + // get<BasicJsonType>() returns *this, this won't call a from_json + // method when value_type is BasicJsonType + return i.template get<typename ConstructibleArrayType::value_type>(); + }); +} + +template <typename BasicJsonType, typename ConstructibleArrayType> +void from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr, + priority_tag<0> /*unused*/) +{ + using std::end; + + std::transform( + j.begin(), j.end(), std::inserter(arr, end(arr)), + [](const BasicJsonType & i) + { + // get<BasicJsonType>() returns *this, this won't call a from_json + // method when value_type is BasicJsonType + return i.template get<typename ConstructibleArrayType::value_type>(); + }); +} + +template <typename BasicJsonType, typename ConstructibleArrayType, + enable_if_t < + is_constructible_array_type<BasicJsonType, ConstructibleArrayType>::value and + not is_constructible_object_type<BasicJsonType, ConstructibleArrayType>::value and + not is_constructible_string_type<BasicJsonType, ConstructibleArrayType>::value and + not is_basic_json<ConstructibleArrayType>::value, + int > = 0 > + +auto from_json(const BasicJsonType& j, ConstructibleArrayType& arr) +-> decltype(from_json_array_impl(j, arr, priority_tag<3> {}), +j.template get<typename ConstructibleArrayType::value_type>(), +void()) +{ + if (JSON_UNLIKELY(not j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + + std::string(j.type_name()))); + } + + from_json_array_impl(j, arr, priority_tag<3> {}); +} + +template<typename BasicJsonType, typename ConstructibleObjectType, + enable_if_t<is_constructible_object_type<BasicJsonType, ConstructibleObjectType>::value, int> = 0> +void from_json(const BasicJsonType& j, ConstructibleObjectType& obj) +{ + if (JSON_UNLIKELY(not j.is_object())) + { + JSON_THROW(type_error::create(302, "type must be object, but is " + std::string(j.type_name()))); + } + + auto inner_object = j.template get_ptr<const typename BasicJsonType::object_t*>(); + using value_type = typename ConstructibleObjectType::value_type; + std::transform( + inner_object->begin(), inner_object->end(), + std::inserter(obj, obj.begin()), + [](typename BasicJsonType::object_t::value_type const & p) + { + return value_type(p.first, p.second.template get<typename ConstructibleObjectType::mapped_type>()); + }); +} + +// overload for arithmetic types, not chosen for basic_json template arguments +// (BooleanType, etc..); note: Is it really necessary to provide explicit +// overloads for boolean_t etc. in case of a custom BooleanType which is not +// an arithmetic type? +template<typename BasicJsonType, typename ArithmeticType, + enable_if_t < + std::is_arithmetic<ArithmeticType>::value and + not std::is_same<ArithmeticType, typename BasicJsonType::number_unsigned_t>::value and + not std::is_same<ArithmeticType, typename BasicJsonType::number_integer_t>::value and + not std::is_same<ArithmeticType, typename BasicJsonType::number_float_t>::value and + not std::is_same<ArithmeticType, typename BasicJsonType::boolean_t>::value, + int> = 0> +void from_json(const BasicJsonType& j, ArithmeticType& val) +{ + switch (static_cast<value_t>(j)) + { + case value_t::number_unsigned: + { + val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_unsigned_t*>()); + break; + } + case value_t::number_integer: + { + val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_integer_t*>()); + break; + } + case value_t::number_float: + { + val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_float_t*>()); + break; + } + case value_t::boolean: + { + val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::boolean_t*>()); + break; + } + + default: + JSON_THROW(type_error::create(302, "type must be number, but is " + std::string(j.type_name()))); + } +} + +template<typename BasicJsonType, typename A1, typename A2> +void from_json(const BasicJsonType& j, std::pair<A1, A2>& p) +{ + p = {j.at(0).template get<A1>(), j.at(1).template get<A2>()}; +} + +template<typename BasicJsonType, typename Tuple, std::size_t... Idx> +void from_json_tuple_impl(const BasicJsonType& j, Tuple& t, index_sequence<Idx...> /*unused*/) +{ + t = std::make_tuple(j.at(Idx).template get<typename std::tuple_element<Idx, Tuple>::type>()...); +} + +template<typename BasicJsonType, typename... Args> +void from_json(const BasicJsonType& j, std::tuple<Args...>& t) +{ + from_json_tuple_impl(j, t, index_sequence_for<Args...> {}); +} + +template <typename BasicJsonType, typename Key, typename Value, typename Compare, typename Allocator, + typename = enable_if_t<not std::is_constructible< + typename BasicJsonType::string_t, Key>::value>> +void from_json(const BasicJsonType& j, std::map<Key, Value, Compare, Allocator>& m) +{ + if (JSON_UNLIKELY(not j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()))); + } + for (const auto& p : j) + { + if (JSON_UNLIKELY(not p.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(p.type_name()))); + } + m.emplace(p.at(0).template get<Key>(), p.at(1).template get<Value>()); + } +} + +template <typename BasicJsonType, typename Key, typename Value, typename Hash, typename KeyEqual, typename Allocator, + typename = enable_if_t<not std::is_constructible< + typename BasicJsonType::string_t, Key>::value>> +void from_json(const BasicJsonType& j, std::unordered_map<Key, Value, Hash, KeyEqual, Allocator>& m) +{ + if (JSON_UNLIKELY(not j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()))); + } + for (const auto& p : j) + { + if (JSON_UNLIKELY(not p.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(p.type_name()))); + } + m.emplace(p.at(0).template get<Key>(), p.at(1).template get<Value>()); + } +} + +struct from_json_fn +{ + template<typename BasicJsonType, typename T> + auto operator()(const BasicJsonType& j, T& val) const + noexcept(noexcept(from_json(j, val))) + -> decltype(from_json(j, val), void()) + { + return from_json(j, val); } }; +} // namespace detail -template<class Sequence1, class Sequence2> -struct merge_and_renumber; +/// namespace to hold default `from_json` function +/// to see why this is required: +/// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2015/n4381.html +namespace +{ +constexpr const auto& from_json = detail::static_const<detail::from_json_fn>::value; +} // namespace +} // namespace nlohmann -template<std::size_t... I1, std::size_t... I2> -struct merge_and_renumber<index_sequence<I1...>, index_sequence<I2...>> - : index_sequence < I1..., (sizeof...(I1) + I2)... > {}; +// #include <nlohmann/detail/conversions/to_json.hpp> -template<std::size_t N> -struct make_index_sequence - : merge_and_renumber < typename make_index_sequence < N / 2 >::type, - typename make_index_sequence < N - N / 2 >::type > {}; -template<> struct make_index_sequence<0> : index_sequence<> {}; -template<> struct make_index_sequence<1> : index_sequence<0> {}; +#include <ciso646> // or, and, not +#include <iterator> // begin, end +#include <tuple> // tuple, get +#include <type_traits> // is_same, is_constructible, is_floating_point, is_enum, underlying_type +#include <utility> // move, forward, declval, pair +#include <valarray> // valarray +#include <vector> // vector -template<typename... Ts> -using index_sequence_for = make_index_sequence<sizeof...(Ts)>; +// #include <nlohmann/detail/meta/cpp_future.hpp> -/* -Implementation of two C++17 constructs: conjunction, negation. This is needed -to avoid evaluating all the traits in a condition +// #include <nlohmann/detail/meta/type_traits.hpp> -For example: not std::is_same<void, T>::value and has_value_type<T>::value -will not compile when T = void (on MSVC at least). Whereas -conjunction<negation<std::is_same<void, T>>, has_value_type<T>>::value will -stop evaluating if negation<...>::value == false +// #include <nlohmann/detail/value_t.hpp> -Please note that those constructs must be used with caution, since symbols can -become very long quickly (which can slow down compilation and cause MSVC -internal compiler errors). Only use it when you have to (see example ahead). -*/ -template<class...> struct conjunction : std::true_type {}; -template<class B1> struct conjunction<B1> : B1 {}; -template<class B1, class... Bn> -struct conjunction<B1, Bn...> : std::conditional<bool(B1::value), conjunction<Bn...>, B1>::type {}; +// #include <nlohmann/detail/iterators/iteration_proxy.hpp> -template<class B> struct negation : std::integral_constant<bool, not B::value> {}; -// dispatch utility (taken from ranges-v3) -template<unsigned N> struct priority_tag : priority_tag < N - 1 > {}; -template<> struct priority_tag<0> {}; +#include <cstddef> // size_t +#include <string> // string, to_string +#include <iterator> // input_iterator_tag +#include <tuple> // tuple_size, get, tuple_element + +// #include <nlohmann/detail/value_t.hpp> + +// #include <nlohmann/detail/meta/type_traits.hpp> + + +namespace nlohmann +{ +namespace detail +{ +template <typename IteratorType> class iteration_proxy_value +{ + public: + using difference_type = std::ptrdiff_t; + using value_type = iteration_proxy_value; + using pointer = value_type * ; + using reference = value_type & ; + using iterator_category = std::input_iterator_tag; + + private: + /// the iterator + IteratorType anchor; + /// an index for arrays (used to create key names) + std::size_t array_index = 0; + /// last stringified array index + mutable std::size_t array_index_last = 0; + /// a string representation of the array index + mutable std::string array_index_str = "0"; + /// an empty string (to return a reference for primitive values) + const std::string empty_str = ""; + + public: + explicit iteration_proxy_value(IteratorType it) noexcept : anchor(it) {} + + /// dereference operator (needed for range-based for) + iteration_proxy_value& operator*() + { + return *this; + } + + /// increment operator (needed for range-based for) + iteration_proxy_value& operator++() + { + ++anchor; + ++array_index; + + return *this; + } + + /// equality operator (needed for InputIterator) + bool operator==(const iteration_proxy_value& o) const noexcept + { + return anchor == o.anchor; + } + + /// inequality operator (needed for range-based for) + bool operator!=(const iteration_proxy_value& o) const noexcept + { + return anchor != o.anchor; + } + + /// return key of the iterator + const std::string& key() const + { + assert(anchor.m_object != nullptr); + + switch (anchor.m_object->type()) + { + // use integer array index as key + case value_t::array: + { + if (array_index != array_index_last) + { + array_index_str = std::to_string(array_index); + array_index_last = array_index; + } + return array_index_str; + } + + // use key from the object + case value_t::object: + return anchor.key(); + + // use an empty key for all primitive types + default: + return empty_str; + } + } + + /// return value of the iterator + typename IteratorType::reference value() const + { + return anchor.value(); + } +}; + +/// proxy class for the items() function +template<typename IteratorType> class iteration_proxy +{ + private: + /// the container to iterate + typename IteratorType::reference container; + + public: + /// construct iteration proxy from a container + explicit iteration_proxy(typename IteratorType::reference cont) noexcept + : container(cont) {} + + /// return iterator begin (needed for range-based for) + iteration_proxy_value<IteratorType> begin() noexcept + { + return iteration_proxy_value<IteratorType>(container.begin()); + } + + /// return iterator end (needed for range-based for) + iteration_proxy_value<IteratorType> end() noexcept + { + return iteration_proxy_value<IteratorType>(container.end()); + } +}; +// Structured Bindings Support +// For further reference see https://blog.tartanllama.xyz/structured-bindings/ +// And see https://github.com/nlohmann/json/pull/1391 +template <std::size_t N, typename IteratorType, enable_if_t<N == 0, int> = 0> +auto get(const nlohmann::detail::iteration_proxy_value<IteratorType>& i) -> decltype(i.key()) +{ + return i.key(); +} +// Structured Bindings Support +// For further reference see https://blog.tartanllama.xyz/structured-bindings/ +// And see https://github.com/nlohmann/json/pull/1391 +template <std::size_t N, typename IteratorType, enable_if_t<N == 1, int> = 0> +auto get(const nlohmann::detail::iteration_proxy_value<IteratorType>& i) -> decltype(i.value()) +{ + return i.value(); +} +} // namespace detail +} // namespace nlohmann +// The Addition to the STD Namespace is required to add +// Structured Bindings Support to the iteration_proxy_value class +// For further reference see https://blog.tartanllama.xyz/structured-bindings/ +// And see https://github.com/nlohmann/json/pull/1391 +namespace std +{ +template <typename IteratorType> +class tuple_size<::nlohmann::detail::iteration_proxy_value<IteratorType>> + : public std::integral_constant<std::size_t, 2> {}; +template <std::size_t N, typename IteratorType> +class tuple_element<N, ::nlohmann::detail::iteration_proxy_value<IteratorType >> +{ + public: + using type = decltype( + get<N>(std::declval < + ::nlohmann::detail::iteration_proxy_value<IteratorType >> ())); +}; +} + +namespace nlohmann +{ +namespace detail +{ ////////////////// // constructors // ////////////////// @@ -657,6 +1849,16 @@ struct external_constructor<value_t::string> j.m_value = std::move(s); j.assert_invariant(); } + + template<typename BasicJsonType, typename CompatibleStringType, + enable_if_t<not std::is_same<CompatibleStringType, typename BasicJsonType::string_t>::value, + int> = 0> + static void construct(BasicJsonType& j, const CompatibleStringType& str) + { + j.m_type = value_t::string; + j.m_value.string = j.template create<typename BasicJsonType::string_t>(str); + j.assert_invariant(); + } }; template<> @@ -783,159 +1985,6 @@ struct external_constructor<value_t::object> } }; - -//////////////////////// -// has_/is_ functions // -//////////////////////// - -/*! -@brief Helper to determine whether there's a key_type for T. - -This helper is used to tell associative containers apart from other containers -such as sequence containers. For instance, `std::map` passes the test as it -contains a `mapped_type`, whereas `std::vector` fails the test. - -@sa http://stackoverflow.com/a/7728728/266378 -@since version 1.0.0, overworked in version 2.0.6 -*/ -#define NLOHMANN_JSON_HAS_HELPER(type) \ - template<typename T> struct has_##type { \ - private: \ - template<typename U, typename = typename U::type> \ - static int detect(U &&); \ - static void detect(...); \ - public: \ - static constexpr bool value = \ - std::is_integral<decltype(detect(std::declval<T>()))>::value; \ - } - -NLOHMANN_JSON_HAS_HELPER(mapped_type); -NLOHMANN_JSON_HAS_HELPER(key_type); -NLOHMANN_JSON_HAS_HELPER(value_type); -NLOHMANN_JSON_HAS_HELPER(iterator); - -#undef NLOHMANN_JSON_HAS_HELPER - - -template<bool B, class RealType, class CompatibleObjectType> -struct is_compatible_object_type_impl : std::false_type {}; - -template<class RealType, class CompatibleObjectType> -struct is_compatible_object_type_impl<true, RealType, CompatibleObjectType> -{ - static constexpr auto value = - std::is_constructible<typename RealType::key_type, typename CompatibleObjectType::key_type>::value and - std::is_constructible<typename RealType::mapped_type, typename CompatibleObjectType::mapped_type>::value; -}; - -template<class BasicJsonType, class CompatibleObjectType> -struct is_compatible_object_type -{ - static auto constexpr value = is_compatible_object_type_impl < - conjunction<negation<std::is_same<void, CompatibleObjectType>>, - has_mapped_type<CompatibleObjectType>, - has_key_type<CompatibleObjectType>>::value, - typename BasicJsonType::object_t, CompatibleObjectType >::value; -}; - -template<typename BasicJsonType, typename T> -struct is_basic_json_nested_type -{ - static auto constexpr value = std::is_same<T, typename BasicJsonType::iterator>::value or - std::is_same<T, typename BasicJsonType::const_iterator>::value or - std::is_same<T, typename BasicJsonType::reverse_iterator>::value or - std::is_same<T, typename BasicJsonType::const_reverse_iterator>::value; -}; - -template<class BasicJsonType, class CompatibleArrayType> -struct is_compatible_array_type -{ - static auto constexpr value = - conjunction<negation<std::is_same<void, CompatibleArrayType>>, - negation<is_compatible_object_type< - BasicJsonType, CompatibleArrayType>>, - negation<std::is_constructible<typename BasicJsonType::string_t, - CompatibleArrayType>>, - negation<is_basic_json_nested_type<BasicJsonType, CompatibleArrayType>>, - has_value_type<CompatibleArrayType>, - has_iterator<CompatibleArrayType>>::value; -}; - -template<bool, typename, typename> -struct is_compatible_integer_type_impl : std::false_type {}; - -template<typename RealIntegerType, typename CompatibleNumberIntegerType> -struct is_compatible_integer_type_impl<true, RealIntegerType, CompatibleNumberIntegerType> -{ - // is there an assert somewhere on overflows? - using RealLimits = std::numeric_limits<RealIntegerType>; - using CompatibleLimits = std::numeric_limits<CompatibleNumberIntegerType>; - - static constexpr auto value = - std::is_constructible<RealIntegerType, CompatibleNumberIntegerType>::value and - CompatibleLimits::is_integer and - RealLimits::is_signed == CompatibleLimits::is_signed; -}; - -template<typename RealIntegerType, typename CompatibleNumberIntegerType> -struct is_compatible_integer_type -{ - static constexpr auto value = - is_compatible_integer_type_impl < - std::is_integral<CompatibleNumberIntegerType>::value and - not std::is_same<bool, CompatibleNumberIntegerType>::value, - RealIntegerType, CompatibleNumberIntegerType >::value; -}; - - -// trait checking if JSONSerializer<T>::from_json(json const&, udt&) exists -template<typename BasicJsonType, typename T> -struct has_from_json -{ - private: - // also check the return type of from_json - template<typename U, typename = enable_if_t<std::is_same<void, decltype(uncvref_t<U>::from_json( - std::declval<BasicJsonType>(), std::declval<T&>()))>::value>> - static int detect(U&&); - static void detect(...); - - public: - static constexpr bool value = std::is_integral<decltype( - detect(std::declval<typename BasicJsonType::template json_serializer<T, void>>()))>::value; -}; - -// This trait checks if JSONSerializer<T>::from_json(json const&) exists -// this overload is used for non-default-constructible user-defined-types -template<typename BasicJsonType, typename T> -struct has_non_default_from_json -{ - private: - template<typename U, typename = - enable_if_t<std::is_same<T, decltype(uncvref_t<U>::from_json(std::declval<BasicJsonType>()))>::value>> - static int detect(U&&); - static void detect(...); - - public: - static constexpr bool value = std::is_integral<decltype(detect( - std::declval<typename BasicJsonType::template json_serializer<T, void>>()))>::value; -}; - -// This trait checks if BasicJsonType::json_serializer<T>::to_json exists -template<typename BasicJsonType, typename T> -struct has_to_json -{ - private: - template<typename U, typename = decltype(uncvref_t<U>::to_json( - std::declval<BasicJsonType&>(), std::declval<T>()))> - static int detect(U&&); - static void detect(...); - - public: - static constexpr bool value = std::is_integral<decltype(detect( - std::declval<typename BasicJsonType::template json_serializer<T, void>>()))>::value; -}; - - ///////////// // to_json // ///////////// @@ -995,10 +2044,14 @@ void to_json(BasicJsonType& j, const std::vector<bool>& e) external_constructor<value_t::array>::construct(j, e); } -template<typename BasicJsonType, typename CompatibleArrayType, - enable_if_t<is_compatible_array_type<BasicJsonType, CompatibleArrayType>::value or - std::is_same<typename BasicJsonType::array_t, CompatibleArrayType>::value, - int> = 0> +template <typename BasicJsonType, typename CompatibleArrayType, + enable_if_t<is_compatible_array_type<BasicJsonType, + CompatibleArrayType>::value and + not is_compatible_object_type< + BasicJsonType, CompatibleArrayType>::value and + not is_compatible_string_type<BasicJsonType, CompatibleArrayType>::value and + not is_basic_json<CompatibleArrayType>::value, + int> = 0> void to_json(BasicJsonType& j, const CompatibleArrayType& arr) { external_constructor<value_t::array>::construct(j, arr); @@ -1006,7 +2059,7 @@ void to_json(BasicJsonType& j, const CompatibleArrayType& arr) template<typename BasicJsonType, typename T, enable_if_t<std::is_convertible<T, BasicJsonType>::value, int> = 0> -void to_json(BasicJsonType& j, std::valarray<T> arr) +void to_json(BasicJsonType& j, const std::valarray<T>& arr) { external_constructor<value_t::array>::construct(j, std::move(arr)); } @@ -1018,7 +2071,7 @@ void to_json(BasicJsonType& j, typename BasicJsonType::array_t&& arr) } template<typename BasicJsonType, typename CompatibleObjectType, - enable_if_t<is_compatible_object_type<BasicJsonType, CompatibleObjectType>::value, int> = 0> + enable_if_t<is_compatible_object_type<BasicJsonType, CompatibleObjectType>::value and not is_basic_json<CompatibleObjectType>::value, int> = 0> void to_json(BasicJsonType& j, const CompatibleObjectType& obj) { external_constructor<value_t::object>::construct(j, obj); @@ -1030,9 +2083,12 @@ void to_json(BasicJsonType& j, typename BasicJsonType::object_t&& obj) external_constructor<value_t::object>::construct(j, std::move(obj)); } -template<typename BasicJsonType, typename T, std::size_t N, - enable_if_t<not std::is_constructible<typename BasicJsonType::string_t, T (&)[N]>::value, int> = 0> -void to_json(BasicJsonType& j, T (&arr)[N]) +template < + typename BasicJsonType, typename T, std::size_t N, + enable_if_t<not std::is_constructible<typename BasicJsonType::string_t, + const T(&)[N]>::value, + int> = 0 > +void to_json(BasicJsonType& j, const T(&arr)[N]) { external_constructor<value_t::array>::construct(j, arr); } @@ -1040,351 +2096,71 @@ void to_json(BasicJsonType& j, T (&arr)[N]) template<typename BasicJsonType, typename... Args> void to_json(BasicJsonType& j, const std::pair<Args...>& p) { - j = {p.first, p.second}; -} - -template<typename BasicJsonType, typename Tuple, std::size_t... Idx> -void to_json_tuple_impl(BasicJsonType& j, const Tuple& t, index_sequence<Idx...>) -{ - j = {std::get<Idx>(t)...}; -} - -template<typename BasicJsonType, typename... Args> -void to_json(BasicJsonType& j, const std::tuple<Args...>& t) -{ - to_json_tuple_impl(j, t, index_sequence_for<Args...> {}); -} - -/////////////// -// from_json // -/////////////// - -// overloads for basic_json template parameters -template<typename BasicJsonType, typename ArithmeticType, - enable_if_t<std::is_arithmetic<ArithmeticType>::value and - not std::is_same<ArithmeticType, typename BasicJsonType::boolean_t>::value, - int> = 0> -void get_arithmetic_value(const BasicJsonType& j, ArithmeticType& val) -{ - switch (static_cast<value_t>(j)) - { - case value_t::number_unsigned: - { - val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_unsigned_t*>()); - break; - } - case value_t::number_integer: - { - val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_integer_t*>()); - break; - } - case value_t::number_float: - { - val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_float_t*>()); - break; - } - - default: - JSON_THROW(type_error::create(302, "type must be number, but is " + std::string(j.type_name()))); - } -} - -template<typename BasicJsonType> -void from_json(const BasicJsonType& j, typename BasicJsonType::boolean_t& b) -{ - if (JSON_UNLIKELY(not j.is_boolean())) - { - JSON_THROW(type_error::create(302, "type must be boolean, but is " + std::string(j.type_name()))); - } - b = *j.template get_ptr<const typename BasicJsonType::boolean_t*>(); -} - -template<typename BasicJsonType> -void from_json(const BasicJsonType& j, typename BasicJsonType::string_t& s) -{ - if (JSON_UNLIKELY(not j.is_string())) - { - JSON_THROW(type_error::create(302, "type must be string, but is " + std::string(j.type_name()))); - } - s = *j.template get_ptr<const typename BasicJsonType::string_t*>(); -} - -template<typename BasicJsonType> -void from_json(const BasicJsonType& j, typename BasicJsonType::number_float_t& val) -{ - get_arithmetic_value(j, val); -} - -template<typename BasicJsonType> -void from_json(const BasicJsonType& j, typename BasicJsonType::number_unsigned_t& val) -{ - get_arithmetic_value(j, val); -} - -template<typename BasicJsonType> -void from_json(const BasicJsonType& j, typename BasicJsonType::number_integer_t& val) -{ - get_arithmetic_value(j, val); -} - -template<typename BasicJsonType, typename EnumType, - enable_if_t<std::is_enum<EnumType>::value, int> = 0> -void from_json(const BasicJsonType& j, EnumType& e) -{ - typename std::underlying_type<EnumType>::type val; - get_arithmetic_value(j, val); - e = static_cast<EnumType>(val); -} - -template<typename BasicJsonType> -void from_json(const BasicJsonType& j, typename BasicJsonType::array_t& arr) -{ - if (JSON_UNLIKELY(not j.is_array())) - { - JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()))); - } - arr = *j.template get_ptr<const typename BasicJsonType::array_t*>(); -} - -// forward_list doesn't have an insert method -template<typename BasicJsonType, typename T, typename Allocator, - enable_if_t<std::is_convertible<BasicJsonType, T>::value, int> = 0> -void from_json(const BasicJsonType& j, std::forward_list<T, Allocator>& l) -{ - if (JSON_UNLIKELY(not j.is_array())) - { - JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()))); - } - std::transform(j.rbegin(), j.rend(), - std::front_inserter(l), [](const BasicJsonType & i) - { - return i.template get<T>(); - }); -} - -// valarray doesn't have an insert method -template<typename BasicJsonType, typename T, - enable_if_t<std::is_convertible<BasicJsonType, T>::value, int> = 0> -void from_json(const BasicJsonType& j, std::valarray<T>& l) -{ - if (JSON_UNLIKELY(not j.is_array())) - { - JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()))); - } - l.resize(j.size()); - std::copy(j.m_value.array->begin(), j.m_value.array->end(), std::begin(l)); -} - -template<typename BasicJsonType, typename CompatibleArrayType> -void from_json_array_impl(const BasicJsonType& j, CompatibleArrayType& arr, priority_tag<0> /*unused*/) -{ - using std::end; - - std::transform(j.begin(), j.end(), - std::inserter(arr, end(arr)), [](const BasicJsonType & i) - { - // get<BasicJsonType>() returns *this, this won't call a from_json - // method when value_type is BasicJsonType - return i.template get<typename CompatibleArrayType::value_type>(); - }); -} - -template<typename BasicJsonType, typename CompatibleArrayType> -auto from_json_array_impl(const BasicJsonType& j, CompatibleArrayType& arr, priority_tag<1> /*unused*/) --> decltype( - arr.reserve(std::declval<typename CompatibleArrayType::size_type>()), - void()) -{ - using std::end; - - arr.reserve(j.size()); - std::transform(j.begin(), j.end(), - std::inserter(arr, end(arr)), [](const BasicJsonType & i) - { - // get<BasicJsonType>() returns *this, this won't call a from_json - // method when value_type is BasicJsonType - return i.template get<typename CompatibleArrayType::value_type>(); - }); -} - -template<typename BasicJsonType, typename T, std::size_t N> -void from_json_array_impl(const BasicJsonType& j, std::array<T, N>& arr, priority_tag<2> /*unused*/) -{ - for (std::size_t i = 0; i < N; ++i) - { - arr[i] = j.at(i).template get<T>(); - } -} - -template<typename BasicJsonType, typename CompatibleArrayType, - enable_if_t<is_compatible_array_type<BasicJsonType, CompatibleArrayType>::value and - std::is_convertible<BasicJsonType, typename CompatibleArrayType::value_type>::value and - not std::is_same<typename BasicJsonType::array_t, CompatibleArrayType>::value, int> = 0> -void from_json(const BasicJsonType& j, CompatibleArrayType& arr) -{ - if (JSON_UNLIKELY(not j.is_array())) - { - JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()))); - } - - from_json_array_impl(j, arr, priority_tag<2> {}); -} - -template<typename BasicJsonType, typename CompatibleObjectType, - enable_if_t<is_compatible_object_type<BasicJsonType, CompatibleObjectType>::value, int> = 0> -void from_json(const BasicJsonType& j, CompatibleObjectType& obj) -{ - if (JSON_UNLIKELY(not j.is_object())) - { - JSON_THROW(type_error::create(302, "type must be object, but is " + std::string(j.type_name()))); - } - - auto inner_object = j.template get_ptr<const typename BasicJsonType::object_t*>(); - using value_type = typename CompatibleObjectType::value_type; - std::transform( - inner_object->begin(), inner_object->end(), - std::inserter(obj, obj.begin()), - [](typename BasicJsonType::object_t::value_type const & p) - { - return value_type(p.first, p.second.template get<typename CompatibleObjectType::mapped_type>()); - }); -} - -// overload for arithmetic types, not chosen for basic_json template arguments -// (BooleanType, etc..); note: Is it really necessary to provide explicit -// overloads for boolean_t etc. in case of a custom BooleanType which is not -// an arithmetic type? -template<typename BasicJsonType, typename ArithmeticType, - enable_if_t < - std::is_arithmetic<ArithmeticType>::value and - not std::is_same<ArithmeticType, typename BasicJsonType::number_unsigned_t>::value and - not std::is_same<ArithmeticType, typename BasicJsonType::number_integer_t>::value and - not std::is_same<ArithmeticType, typename BasicJsonType::number_float_t>::value and - not std::is_same<ArithmeticType, typename BasicJsonType::boolean_t>::value, - int> = 0> -void from_json(const BasicJsonType& j, ArithmeticType& val) -{ - switch (static_cast<value_t>(j)) - { - case value_t::number_unsigned: - { - val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_unsigned_t*>()); - break; - } - case value_t::number_integer: - { - val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_integer_t*>()); - break; - } - case value_t::number_float: - { - val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_float_t*>()); - break; - } - case value_t::boolean: - { - val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::boolean_t*>()); - break; - } - - default: - JSON_THROW(type_error::create(302, "type must be number, but is " + std::string(j.type_name()))); - } + j = { p.first, p.second }; } -template<typename BasicJsonType, typename A1, typename A2> -void from_json(const BasicJsonType& j, std::pair<A1, A2>& p) +// for https://github.com/nlohmann/json/pull/1134 +template < typename BasicJsonType, typename T, + enable_if_t<std::is_same<T, iteration_proxy_value<typename BasicJsonType::iterator>>::value, int> = 0> +void to_json(BasicJsonType& j, const T& b) { - p = {j.at(0).template get<A1>(), j.at(1).template get<A2>()}; + j = { {b.key(), b.value()} }; } template<typename BasicJsonType, typename Tuple, std::size_t... Idx> -void from_json_tuple_impl(const BasicJsonType& j, Tuple& t, index_sequence<Idx...>) +void to_json_tuple_impl(BasicJsonType& j, const Tuple& t, index_sequence<Idx...> /*unused*/) { - t = std::make_tuple(j.at(Idx).template get<typename std::tuple_element<Idx, Tuple>::type>()...); + j = { std::get<Idx>(t)... }; } template<typename BasicJsonType, typename... Args> -void from_json(const BasicJsonType& j, std::tuple<Args...>& t) +void to_json(BasicJsonType& j, const std::tuple<Args...>& t) { - from_json_tuple_impl(j, t, index_sequence_for<Args...> {}); + to_json_tuple_impl(j, t, index_sequence_for<Args...> {}); } struct to_json_fn { - private: template<typename BasicJsonType, typename T> - auto call(BasicJsonType& j, T&& val, priority_tag<1> /*unused*/) const noexcept(noexcept(to_json(j, std::forward<T>(val)))) + auto operator()(BasicJsonType& j, T&& val) const noexcept(noexcept(to_json(j, std::forward<T>(val)))) -> decltype(to_json(j, std::forward<T>(val)), void()) { return to_json(j, std::forward<T>(val)); } +}; +} // namespace detail - template<typename BasicJsonType, typename T> - void call(BasicJsonType& /*unused*/, T&& /*unused*/, priority_tag<0> /*unused*/) const noexcept - { - static_assert(sizeof(BasicJsonType) == 0, - "could not find to_json() method in T's namespace"); +/// namespace to hold default `to_json` function +namespace +{ +constexpr const auto& to_json = detail::static_const<detail::to_json_fn>::value; +} // namespace +} // namespace nlohmann -#ifdef _MSC_VER - // MSVC does not show a stacktrace for the above assert - using decayed = uncvref_t<T>; - static_assert(sizeof(typename decayed::force_msvc_stacktrace) == 0, - "forcing MSVC stacktrace to show which T we're talking about."); -#endif - } +// #include <nlohmann/detail/input/input_adapters.hpp> - public: - template<typename BasicJsonType, typename T> - void operator()(BasicJsonType& j, T&& val) const - noexcept(noexcept(std::declval<to_json_fn>().call(j, std::forward<T>(val), priority_tag<1> {}))) - { - return call(j, std::forward<T>(val), priority_tag<1> {}); - } -}; -struct from_json_fn -{ - private: - template<typename BasicJsonType, typename T> - auto call(const BasicJsonType& j, T& val, priority_tag<1> /*unused*/) const - noexcept(noexcept(from_json(j, val))) - -> decltype(from_json(j, val), void()) - { - return from_json(j, val); - } +#include <cassert> // assert +#include <cstddef> // size_t +#include <cstring> // strlen +#include <istream> // istream +#include <iterator> // begin, end, iterator_traits, random_access_iterator_tag, distance, next +#include <memory> // shared_ptr, make_shared, addressof +#include <numeric> // accumulate +#include <string> // string, char_traits +#include <type_traits> // enable_if, is_base_of, is_pointer, is_integral, remove_pointer +#include <utility> // pair, declval +#include <cstdio> //FILE * - template<typename BasicJsonType, typename T> - void call(const BasicJsonType& /*unused*/, T& /*unused*/, priority_tag<0> /*unused*/) const noexcept - { - static_assert(sizeof(BasicJsonType) == 0, - "could not find from_json() method in T's namespace"); -#ifdef _MSC_VER - // MSVC does not show a stacktrace for the above assert - using decayed = uncvref_t<T>; - static_assert(sizeof(typename decayed::force_msvc_stacktrace) == 0, - "forcing MSVC stacktrace to show which T we're talking about."); -#endif - } +// #include <nlohmann/detail/macro_scope.hpp> - public: - template<typename BasicJsonType, typename T> - void operator()(const BasicJsonType& j, T& val) const - noexcept(noexcept(std::declval<from_json_fn>().call(j, val, priority_tag<1> {}))) - { - return call(j, val, priority_tag<1> {}); - } -}; -// taken from ranges-v3 -template<typename T> -struct static_const +namespace nlohmann { - static constexpr T value{}; -}; - -template<typename T> -constexpr T static_const<T>::value; +namespace detail +{ +/// the supported input formats +enum class input_format_t { json, cbor, msgpack, ubjson, bson }; //////////////////// // input adapters // @@ -1394,19 +2170,17 @@ constexpr T static_const<T>::value; @brief abstract input adapter interface Produces a stream of std::char_traits<char>::int_type characters from a -std::istream, a buffer, or some other input type. Accepts the return of exactly -one non-EOF character for future input. The int_type characters returned -consist of all valid char values as positive values (typically unsigned char), -plus an EOF value outside that range, specified by the value of the function -std::char_traits<char>::eof(). This value is typically -1, but could be any -arbitrary value which is not a valid char value. +std::istream, a buffer, or some other input type. Accepts the return of +exactly one non-EOF character for future input. The int_type characters +returned consist of all valid char values as positive values (typically +unsigned char), plus an EOF value outside that range, specified by the value +of the function std::char_traits<char>::eof(). This value is typically -1, but +could be any arbitrary value which is not a valid char value. */ struct input_adapter_protocol { /// get a character [0,255] or std::char_traits<char>::eof(). virtual std::char_traits<char>::int_type get_character() = 0; - /// restore the last non-eof() character to input - virtual void unget_character() = 0; virtual ~input_adapter_protocol() = default; }; @@ -1414,6 +2188,27 @@ struct input_adapter_protocol using input_adapter_t = std::shared_ptr<input_adapter_protocol>; /*! +Input adapter for stdio file access. This adapter read only 1 byte and do not use any + buffer. This adapter is a very low level adapter. +*/ +class file_input_adapter : public input_adapter_protocol +{ + public: + explicit file_input_adapter(std::FILE* f) noexcept + : m_file(f) + {} + + std::char_traits<char>::int_type get_character() noexcept override + { + return std::fgetc(m_file); + } + private: + /// the file pointer to read from + std::FILE* m_file; +}; + + +/*! Input adapter for a (caching) istream. Ignores a UFT Byte Order Mark at beginning of input. Does not support changing the underlying std::streambuf in mid-input. Maintains underlying std::istream and std::streambuf to support @@ -1428,56 +2223,32 @@ class input_stream_adapter : public input_adapter_protocol ~input_stream_adapter() override { // clear stream flags; we use underlying streambuf I/O, do not - // maintain ifstream flags - is.clear(); + // maintain ifstream flags, except eof + is.clear(is.rdstate() & std::ios::eofbit); } explicit input_stream_adapter(std::istream& i) : is(i), sb(*i.rdbuf()) - { - // skip byte order mark - std::char_traits<char>::int_type c; - if ((c = get_character()) == 0xEF) - { - if ((c = get_character()) == 0xBB) - { - if ((c = get_character()) == 0xBF) - { - return; // Ignore BOM - } - else if (c != std::char_traits<char>::eof()) - { - is.unget(); - } - is.putback('\xBB'); - } - else if (c != std::char_traits<char>::eof()) - { - is.unget(); - } - is.putback('\xEF'); - } - else if (c != std::char_traits<char>::eof()) - { - is.unget(); // no byte order mark; process as usual - } - } + {} // delete because of pointer members input_stream_adapter(const input_stream_adapter&) = delete; input_stream_adapter& operator=(input_stream_adapter&) = delete; + input_stream_adapter(input_stream_adapter&&) = delete; + input_stream_adapter& operator=(input_stream_adapter&&) = delete; // std::istream/std::streambuf use std::char_traits<char>::to_int_type, to // ensure that std::char_traits<char>::eof() and the character 0xFF do not // end up as the same value, eg. 0xFFFFFFFF. std::char_traits<char>::int_type get_character() override { - return sb.sbumpc(); - } - - void unget_character() override - { - sb.sungetc(); // is.unget() avoided for performance + auto res = sb.sbumpc(); + // set eof manually, as we don't use the istream interface. + if (res == EOF) + { + is.clear(is.rdstate() | std::ios::eofbit); + } + return res; } private: @@ -1490,19 +2261,16 @@ class input_stream_adapter : public input_adapter_protocol class input_buffer_adapter : public input_adapter_protocol { public: - input_buffer_adapter(const char* b, const std::size_t l) - : cursor(b), limit(b + l), start(b) - { - // skip byte order mark - if (l >= 3 and b[0] == '\xEF' and b[1] == '\xBB' and b[2] == '\xBF') - { - cursor += 3; - } - } + input_buffer_adapter(const char* b, const std::size_t l) noexcept + : cursor(b), limit(b + l) + {} // delete because of pointer members input_buffer_adapter(const input_buffer_adapter&) = delete; input_buffer_adapter& operator=(input_buffer_adapter&) = delete; + input_buffer_adapter(input_buffer_adapter&&) = delete; + input_buffer_adapter& operator=(input_buffer_adapter&&) = delete; + ~input_buffer_adapter() override = default; std::char_traits<char>::int_type get_character() noexcept override { @@ -1514,28 +2282,182 @@ class input_buffer_adapter : public input_adapter_protocol return std::char_traits<char>::eof(); } - void unget_character() noexcept override + private: + /// pointer to the current character + const char* cursor; + /// pointer past the last character + const char* const limit; +}; + +template<typename WideStringType, size_t T> +struct wide_string_input_helper +{ + // UTF-32 + static void fill_buffer(const WideStringType& str, size_t& current_wchar, std::array<std::char_traits<char>::int_type, 4>& utf8_bytes, size_t& utf8_bytes_index, size_t& utf8_bytes_filled) { - if (JSON_LIKELY(cursor > start)) + utf8_bytes_index = 0; + + if (current_wchar == str.size()) { - --cursor; + utf8_bytes[0] = std::char_traits<char>::eof(); + utf8_bytes_filled = 1; + } + else + { + // get the current character + const auto wc = static_cast<int>(str[current_wchar++]); + + // UTF-32 to UTF-8 encoding + if (wc < 0x80) + { + utf8_bytes[0] = wc; + utf8_bytes_filled = 1; + } + else if (wc <= 0x7FF) + { + utf8_bytes[0] = 0xC0 | ((wc >> 6) & 0x1F); + utf8_bytes[1] = 0x80 | (wc & 0x3F); + utf8_bytes_filled = 2; + } + else if (wc <= 0xFFFF) + { + utf8_bytes[0] = 0xE0 | ((wc >> 12) & 0x0F); + utf8_bytes[1] = 0x80 | ((wc >> 6) & 0x3F); + utf8_bytes[2] = 0x80 | (wc & 0x3F); + utf8_bytes_filled = 3; + } + else if (wc <= 0x10FFFF) + { + utf8_bytes[0] = 0xF0 | ((wc >> 18) & 0x07); + utf8_bytes[1] = 0x80 | ((wc >> 12) & 0x3F); + utf8_bytes[2] = 0x80 | ((wc >> 6) & 0x3F); + utf8_bytes[3] = 0x80 | (wc & 0x3F); + utf8_bytes_filled = 4; + } + else + { + // unknown character + utf8_bytes[0] = wc; + utf8_bytes_filled = 1; + } } } +}; + +template<typename WideStringType> +struct wide_string_input_helper<WideStringType, 2> +{ + // UTF-16 + static void fill_buffer(const WideStringType& str, size_t& current_wchar, std::array<std::char_traits<char>::int_type, 4>& utf8_bytes, size_t& utf8_bytes_index, size_t& utf8_bytes_filled) + { + utf8_bytes_index = 0; + + if (current_wchar == str.size()) + { + utf8_bytes[0] = std::char_traits<char>::eof(); + utf8_bytes_filled = 1; + } + else + { + // get the current character + const auto wc = static_cast<int>(str[current_wchar++]); + + // UTF-16 to UTF-8 encoding + if (wc < 0x80) + { + utf8_bytes[0] = wc; + utf8_bytes_filled = 1; + } + else if (wc <= 0x7FF) + { + utf8_bytes[0] = 0xC0 | ((wc >> 6)); + utf8_bytes[1] = 0x80 | (wc & 0x3F); + utf8_bytes_filled = 2; + } + else if (0xD800 > wc or wc >= 0xE000) + { + utf8_bytes[0] = 0xE0 | ((wc >> 12)); + utf8_bytes[1] = 0x80 | ((wc >> 6) & 0x3F); + utf8_bytes[2] = 0x80 | (wc & 0x3F); + utf8_bytes_filled = 3; + } + else + { + if (current_wchar < str.size()) + { + const auto wc2 = static_cast<int>(str[current_wchar++]); + const int charcode = 0x10000 + (((wc & 0x3FF) << 10) | (wc2 & 0x3FF)); + utf8_bytes[0] = 0xf0 | (charcode >> 18); + utf8_bytes[1] = 0x80 | ((charcode >> 12) & 0x3F); + utf8_bytes[2] = 0x80 | ((charcode >> 6) & 0x3F); + utf8_bytes[3] = 0x80 | (charcode & 0x3F); + utf8_bytes_filled = 4; + } + else + { + // unknown character + ++current_wchar; + utf8_bytes[0] = wc; + utf8_bytes_filled = 1; + } + } + } + } +}; + +template<typename WideStringType> +class wide_string_input_adapter : public input_adapter_protocol +{ + public: + explicit wide_string_input_adapter(const WideStringType& w) noexcept + : str(w) + {} + + std::char_traits<char>::int_type get_character() noexcept override + { + // check if buffer needs to be filled + if (utf8_bytes_index == utf8_bytes_filled) + { + fill_buffer<sizeof(typename WideStringType::value_type)>(); + + assert(utf8_bytes_filled > 0); + assert(utf8_bytes_index == 0); + } + + // use buffer + assert(utf8_bytes_filled > 0); + assert(utf8_bytes_index < utf8_bytes_filled); + return utf8_bytes[utf8_bytes_index++]; + } private: - /// pointer to the current character - const char* cursor; - /// pointer past the last character - const char* limit; - /// pointer to the first character - const char* start; + template<size_t T> + void fill_buffer() + { + wide_string_input_helper<WideStringType, T>::fill_buffer(str, current_wchar, utf8_bytes, utf8_bytes_index, utf8_bytes_filled); + } + + /// the wstring to process + const WideStringType& str; + + /// index of the current wchar in str + std::size_t current_wchar = 0; + + /// a buffer for UTF-8 bytes + std::array<std::char_traits<char>::int_type, 4> utf8_bytes = {{0, 0, 0, 0}}; + + /// index to the utf8_codes array for the next valid byte + std::size_t utf8_bytes_index = 0; + /// number of valid bytes in the utf8_codes array + std::size_t utf8_bytes_filled = 0; }; class input_adapter { public: // native support - + input_adapter(std::FILE* file) + : ia(std::make_shared<file_input_adapter>(file)) {} /// input adapter for input stream input_adapter(std::istream& i) : ia(std::make_shared<input_stream_adapter>(i)) {} @@ -1544,6 +2466,15 @@ class input_adapter input_adapter(std::istream&& i) : ia(std::make_shared<input_stream_adapter>(i)) {} + input_adapter(const std::wstring& ws) + : ia(std::make_shared<wide_string_input_adapter<std::wstring>>(ws)) {} + + input_adapter(const std::u16string& ws) + : ia(std::make_shared<wide_string_input_adapter<std::u16string>>(ws)) {} + + input_adapter(const std::u32string& ws) + : ia(std::make_shared<wide_string_input_adapter<std::u32string>>(ws)) {} + /// input adapter for buffer template<typename CharT, typename std::enable_if< @@ -1570,23 +2501,26 @@ class input_adapter /// input adapter for iterator range with contiguous storage template<class IteratorType, typename std::enable_if< - std::is_same<typename std::iterator_traits<IteratorType>::iterator_category, std::random_access_iterator_tag>::value, + std::is_same<typename iterator_traits<IteratorType>::iterator_category, std::random_access_iterator_tag>::value, int>::type = 0> input_adapter(IteratorType first, IteratorType last) { +#ifndef NDEBUG // assertion to check that the iterator range is indeed contiguous, // see http://stackoverflow.com/a/35008842/266378 for more discussion - assert(std::accumulate( - first, last, std::pair<bool, int>(true, 0), - [&first](std::pair<bool, int> res, decltype(*first) val) + const auto is_contiguous = std::accumulate( + first, last, std::pair<bool, int>(true, 0), + [&first](std::pair<bool, int> res, decltype(*first) val) { res.first &= (val == *(std::next(std::addressof(*first), res.second++))); return res; - }).first); + }).first; + assert(is_contiguous); +#endif // assertion to check that each element is 1 byte long static_assert( - sizeof(typename std::iterator_traits<IteratorType>::value_type) == 1, + sizeof(typename iterator_traits<IteratorType>::value_type) == 1, "each element in the iterator range must have the size of 1 byte"); const auto len = static_cast<size_t>(std::distance(first, last)); @@ -1610,7 +2544,7 @@ class input_adapter /// input adapter for contiguous container template<class ContiguousContainer, typename std::enable_if<not std::is_pointer<ContiguousContainer>::value and - std::is_base_of<std::random_access_iterator_tag, typename std::iterator_traits<decltype(std::begin(std::declval<ContiguousContainer const>()))>::iterator_category>::value, + std::is_base_of<std::random_access_iterator_tag, typename iterator_traits<decltype(std::begin(std::declval<ContiguousContainer const>()))>::iterator_category>::value, int>::type = 0> input_adapter(const ContiguousContainer& c) : input_adapter(std::begin(c), std::end(c)) {} @@ -1624,10 +2558,34 @@ class input_adapter /// the actual adapter input_adapter_t ia = nullptr; }; +} // namespace detail +} // namespace nlohmann -////////////////////// -// lexer and parser // -////////////////////// +// #include <nlohmann/detail/input/lexer.hpp> + + +#include <clocale> // localeconv +#include <cstddef> // size_t +#include <cstdlib> // strtof, strtod, strtold, strtoll, strtoull +#include <cstdio> // snprintf +#include <initializer_list> // initializer_list +#include <string> // char_traits, string +#include <vector> // vector + +// #include <nlohmann/detail/macro_scope.hpp> + +// #include <nlohmann/detail/input/input_adapters.hpp> + +// #include <nlohmann/detail/input/position_t.hpp> + + +namespace nlohmann +{ +namespace detail +{ +/////////// +// lexer // +/////////// /*! @brief lexical analysis @@ -1640,6 +2598,7 @@ class lexer using number_integer_t = typename BasicJsonType::number_integer_t; using number_unsigned_t = typename BasicJsonType::number_unsigned_t; using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; public: /// token types for the parser @@ -1701,17 +2660,22 @@ class lexer return "end of input"; case token_type::literal_or_value: return "'[', '{', or a literal"; + // LCOV_EXCL_START default: // catch non-enum values - return "unknown token"; // LCOV_EXCL_LINE + return "unknown token"; + // LCOV_EXCL_STOP } } - explicit lexer(detail::input_adapter_t adapter) + explicit lexer(detail::input_adapter_t&& adapter) : ia(std::move(adapter)), decimal_point_char(get_decimal_point()) {} // delete because of pointer members lexer(const lexer&) = delete; + lexer(lexer&&) = delete; lexer& operator=(lexer&) = delete; + lexer& operator=(lexer&&) = delete; + ~lexer() = default; private: ///////////////////// @@ -1819,9 +2783,10 @@ class lexer @brief scan a string literal This function scans a string according to Sect. 7 of RFC 7159. While - scanning, bytes are escaped and copied into buffer yytext. Then the function - returns successfully, yytext is *not* null-terminated (as it may contain \0 - bytes), and yytext.size() is the number of bytes in the string. + scanning, bytes are escaped and copied into buffer token_buffer. Then the + function returns successfully, token_buffer is *not* null-terminated (as it + may contain \0 bytes), and token_buffer.size() is the number of bytes in the + string. @return token_type::value_string if string could be successfully scanned, token_type::parse_error otherwise @@ -1831,7 +2796,7 @@ class lexer */ token_type scan_string() { - // reset yytext (ignore opening quote) + // reset token_buffer (ignore opening quote) reset(); // we entered the function by reading an open quote @@ -1999,39 +2964,194 @@ class lexer // invalid control characters case 0x00: + { + error_message = "invalid string: control character U+0000 (NUL) must be escaped to \\u0000"; + return token_type::parse_error; + } + case 0x01: + { + error_message = "invalid string: control character U+0001 (SOH) must be escaped to \\u0001"; + return token_type::parse_error; + } + case 0x02: + { + error_message = "invalid string: control character U+0002 (STX) must be escaped to \\u0002"; + return token_type::parse_error; + } + case 0x03: + { + error_message = "invalid string: control character U+0003 (ETX) must be escaped to \\u0003"; + return token_type::parse_error; + } + case 0x04: + { + error_message = "invalid string: control character U+0004 (EOT) must be escaped to \\u0004"; + return token_type::parse_error; + } + case 0x05: + { + error_message = "invalid string: control character U+0005 (ENQ) must be escaped to \\u0005"; + return token_type::parse_error; + } + case 0x06: + { + error_message = "invalid string: control character U+0006 (ACK) must be escaped to \\u0006"; + return token_type::parse_error; + } + case 0x07: + { + error_message = "invalid string: control character U+0007 (BEL) must be escaped to \\u0007"; + return token_type::parse_error; + } + case 0x08: + { + error_message = "invalid string: control character U+0008 (BS) must be escaped to \\u0008 or \\b"; + return token_type::parse_error; + } + case 0x09: + { + error_message = "invalid string: control character U+0009 (HT) must be escaped to \\u0009 or \\t"; + return token_type::parse_error; + } + case 0x0A: + { + error_message = "invalid string: control character U+000A (LF) must be escaped to \\u000A or \\n"; + return token_type::parse_error; + } + case 0x0B: + { + error_message = "invalid string: control character U+000B (VT) must be escaped to \\u000B"; + return token_type::parse_error; + } + case 0x0C: + { + error_message = "invalid string: control character U+000C (FF) must be escaped to \\u000C or \\f"; + return token_type::parse_error; + } + case 0x0D: + { + error_message = "invalid string: control character U+000D (CR) must be escaped to \\u000D or \\r"; + return token_type::parse_error; + } + case 0x0E: + { + error_message = "invalid string: control character U+000E (SO) must be escaped to \\u000E"; + return token_type::parse_error; + } + case 0x0F: + { + error_message = "invalid string: control character U+000F (SI) must be escaped to \\u000F"; + return token_type::parse_error; + } + case 0x10: + { + error_message = "invalid string: control character U+0010 (DLE) must be escaped to \\u0010"; + return token_type::parse_error; + } + case 0x11: + { + error_message = "invalid string: control character U+0011 (DC1) must be escaped to \\u0011"; + return token_type::parse_error; + } + case 0x12: + { + error_message = "invalid string: control character U+0012 (DC2) must be escaped to \\u0012"; + return token_type::parse_error; + } + case 0x13: + { + error_message = "invalid string: control character U+0013 (DC3) must be escaped to \\u0013"; + return token_type::parse_error; + } + case 0x14: + { + error_message = "invalid string: control character U+0014 (DC4) must be escaped to \\u0014"; + return token_type::parse_error; + } + case 0x15: + { + error_message = "invalid string: control character U+0015 (NAK) must be escaped to \\u0015"; + return token_type::parse_error; + } + case 0x16: + { + error_message = "invalid string: control character U+0016 (SYN) must be escaped to \\u0016"; + return token_type::parse_error; + } + case 0x17: + { + error_message = "invalid string: control character U+0017 (ETB) must be escaped to \\u0017"; + return token_type::parse_error; + } + case 0x18: + { + error_message = "invalid string: control character U+0018 (CAN) must be escaped to \\u0018"; + return token_type::parse_error; + } + case 0x19: + { + error_message = "invalid string: control character U+0019 (EM) must be escaped to \\u0019"; + return token_type::parse_error; + } + case 0x1A: + { + error_message = "invalid string: control character U+001A (SUB) must be escaped to \\u001A"; + return token_type::parse_error; + } + case 0x1B: + { + error_message = "invalid string: control character U+001B (ESC) must be escaped to \\u001B"; + return token_type::parse_error; + } + case 0x1C: + { + error_message = "invalid string: control character U+001C (FS) must be escaped to \\u001C"; + return token_type::parse_error; + } + case 0x1D: + { + error_message = "invalid string: control character U+001D (GS) must be escaped to \\u001D"; + return token_type::parse_error; + } + case 0x1E: + { + error_message = "invalid string: control character U+001E (RS) must be escaped to \\u001E"; + return token_type::parse_error; + } + case 0x1F: { - error_message = "invalid string: control character must be escaped"; + error_message = "invalid string: control character U+001F (US) must be escaped to \\u001F"; return token_type::parse_error; } @@ -2303,7 +3423,7 @@ class lexer contains cycles, but any cycle can be left when EOF is read. Therefore, the function is guaranteed to terminate. - During scanning, the read bytes are stored in yytext. This string is + During scanning, the read bytes are stored in token_buffer. This string is then converted to a signed integer, an unsigned integer, or a floating-point number. @@ -2315,9 +3435,9 @@ class lexer locale's decimal point is used instead of `.` to work with the locale-dependent converters. */ - token_type scan_number() + token_type scan_number() // lgtm [cpp/use-of-goto] { - // reset yytext to store the number's bytes + // reset token_buffer to store the number's bytes reset(); // the type of the parsed number; initially set to unsigned; will be @@ -2353,11 +3473,13 @@ class lexer goto scan_number_any1; } + // LCOV_EXCL_START default: { // all other characters are rejected outside scan_number() - assert(false); // LCOV_EXCL_LINE + assert(false); } + // LCOV_EXCL_STOP } scan_number_minus: @@ -2601,10 +3723,10 @@ scan_number_done: // try to parse integers first and fall back to floats if (number_type == token_type::value_unsigned) { - const auto x = std::strtoull(yytext.data(), &endptr, 10); + const auto x = std::strtoull(token_buffer.data(), &endptr, 10); // we checked the number format before - assert(endptr == yytext.data() + yytext.size()); + assert(endptr == token_buffer.data() + token_buffer.size()); if (errno == 0) { @@ -2617,10 +3739,10 @@ scan_number_done: } else if (number_type == token_type::value_integer) { - const auto x = std::strtoll(yytext.data(), &endptr, 10); + const auto x = std::strtoll(token_buffer.data(), &endptr, 10); // we checked the number format before - assert(endptr == yytext.data() + yytext.size()); + assert(endptr == token_buffer.data() + token_buffer.size()); if (errno == 0) { @@ -2634,10 +3756,10 @@ scan_number_done: // this code is reached if we parse a floating-point number or if an // integer conversion above failed - strtof(value_float, yytext.data(), &endptr); + strtof(value_float, token_buffer.data(), &endptr); // we checked the number format before - assert(endptr == yytext.data() + yytext.size()); + assert(endptr == token_buffer.data() + token_buffer.size()); return token_type::value_float; } @@ -2666,10 +3788,10 @@ scan_number_done: // input management ///////////////////// - /// reset yytext; current character is beginning of token + /// reset token_buffer; current character is beginning of token void reset() noexcept { - yytext.clear(); + token_buffer.clear(); token_string.clear(); token_string.push_back(std::char_traits<char>::to_char_type(current)); } @@ -2686,31 +3808,71 @@ scan_number_done: */ std::char_traits<char>::int_type get() { - ++chars_read; - current = ia->get_character(); + ++position.chars_read_total; + ++position.chars_read_current_line; + + if (next_unget) + { + // just reset the next_unget variable and work with current + next_unget = false; + } + else + { + current = ia->get_character(); + } + if (JSON_LIKELY(current != std::char_traits<char>::eof())) { token_string.push_back(std::char_traits<char>::to_char_type(current)); } + + if (current == '\n') + { + ++position.lines_read; + ++position.chars_read_current_line = 0; + } + return current; } - /// unget current character (return it again on next get) + /*! + @brief unget current character (read it again on next get) + + We implement unget by setting variable next_unget to true. The input is not + changed - we just simulate ungetting by modifying chars_read_total, + chars_read_current_line, and token_string. The next call to get() will + behave as if the unget character is read again. + */ void unget() { - --chars_read; + next_unget = true; + + --position.chars_read_total; + + // in case we "unget" a newline, we have to also decrement the lines_read + if (position.chars_read_current_line == 0) + { + if (position.lines_read > 0) + { + --position.lines_read; + } + } + else + { + --position.chars_read_current_line; + } + if (JSON_LIKELY(current != std::char_traits<char>::eof())) { - ia->unget_character(); assert(token_string.size() != 0); token_string.pop_back(); } } - /// add a character to yytext + /// add a character to token_buffer void add(int c) { - yytext.push_back(std::char_traits<char>::to_char_type(c)); + token_buffer.push_back(std::char_traits<char>::to_char_type(c)); } public: @@ -2737,9 +3899,9 @@ scan_number_done: } /// return current string value (implicitly resets the token; useful only once) - std::string move_string() + string_t& get_string() { - return std::move(yytext); + return token_buffer; } ///////////////////// @@ -2747,9 +3909,9 @@ scan_number_done: ///////////////////// /// return position of last read token - constexpr std::size_t get_position() const noexcept + constexpr position_t get_position() const noexcept { - return chars_read; + return position; } /// return the last read token (for errors only). Will never contain EOF @@ -2764,10 +3926,9 @@ scan_number_done: if ('\x00' <= c and c <= '\x1F') { // escape control characters - std::stringstream ss; - ss << "<U+" << std::setw(4) << std::uppercase << std::setfill('0') - << std::hex << static_cast<int>(c) << ">"; - result += ss.str(); + char cs[9]; + (std::snprintf)(cs, 9, "<U+%.4X>", static_cast<unsigned char>(c)); + result += cs; } else { @@ -2789,8 +3950,33 @@ scan_number_done: // actual scanner ///////////////////// + /*! + @brief skip the UTF-8 byte order mark + @return true iff there is no BOM or the correct BOM has been skipped + */ + bool skip_bom() + { + if (get() == 0xEF) + { + // check if we completely parse the BOM + return get() == 0xBB and get() == 0xBF; + } + + // the first character is not the beginning of the BOM; unget it to + // process is later + unget(); + return true; + } + token_type scan() { + // initially, skip the BOM + if (position.chars_read_total == 0 and not skip_bom()) + { + error_message = "invalid BOM; must be 0xEF 0xBB 0xBF if given"; + return token_type::parse_error; + } + // read next character and ignore whitespace do { @@ -2860,14 +4046,17 @@ scan_number_done: /// the current character std::char_traits<char>::int_type current = std::char_traits<char>::eof(); - /// the number of characters read - std::size_t chars_read = 0; + /// whether the next get() call should just return current + bool next_unget = false; + + /// the start position of the current token + position_t position; /// raw input token string (for error messages) std::vector<char> token_string {}; /// buffer for variable-length tokens (numbers, strings) - std::string yytext {}; + string_t token_buffer {}; /// a description of occurred lexer errors const char* error_message = ""; @@ -2880,6 +4069,887 @@ scan_number_done: /// the decimal point const char decimal_point_char = '.'; }; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/input/parser.hpp> + + +#include <cassert> // assert +#include <cmath> // isfinite +#include <cstdint> // uint8_t +#include <functional> // function +#include <string> // string +#include <utility> // move + +// #include <nlohmann/detail/exceptions.hpp> + +// #include <nlohmann/detail/macro_scope.hpp> + +// #include <nlohmann/detail/meta/is_sax.hpp> + + +#include <cstdint> // size_t +#include <utility> // declval + +// #include <nlohmann/detail/meta/detected.hpp> + +// #include <nlohmann/detail/meta/type_traits.hpp> + + +namespace nlohmann +{ +namespace detail +{ +template <typename T> +using null_function_t = decltype(std::declval<T&>().null()); + +template <typename T> +using boolean_function_t = + decltype(std::declval<T&>().boolean(std::declval<bool>())); + +template <typename T, typename Integer> +using number_integer_function_t = + decltype(std::declval<T&>().number_integer(std::declval<Integer>())); + +template <typename T, typename Unsigned> +using number_unsigned_function_t = + decltype(std::declval<T&>().number_unsigned(std::declval<Unsigned>())); + +template <typename T, typename Float, typename String> +using number_float_function_t = decltype(std::declval<T&>().number_float( + std::declval<Float>(), std::declval<const String&>())); + +template <typename T, typename String> +using string_function_t = + decltype(std::declval<T&>().string(std::declval<String&>())); + +template <typename T> +using start_object_function_t = + decltype(std::declval<T&>().start_object(std::declval<std::size_t>())); + +template <typename T, typename String> +using key_function_t = + decltype(std::declval<T&>().key(std::declval<String&>())); + +template <typename T> +using end_object_function_t = decltype(std::declval<T&>().end_object()); + +template <typename T> +using start_array_function_t = + decltype(std::declval<T&>().start_array(std::declval<std::size_t>())); + +template <typename T> +using end_array_function_t = decltype(std::declval<T&>().end_array()); + +template <typename T, typename Exception> +using parse_error_function_t = decltype(std::declval<T&>().parse_error( + std::declval<std::size_t>(), std::declval<const std::string&>(), + std::declval<const Exception&>())); + +template <typename SAX, typename BasicJsonType> +struct is_sax +{ + private: + static_assert(is_basic_json<BasicJsonType>::value, + "BasicJsonType must be of type basic_json<...>"); + + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using exception_t = typename BasicJsonType::exception; + + public: + static constexpr bool value = + is_detected_exact<bool, null_function_t, SAX>::value && + is_detected_exact<bool, boolean_function_t, SAX>::value && + is_detected_exact<bool, number_integer_function_t, SAX, + number_integer_t>::value && + is_detected_exact<bool, number_unsigned_function_t, SAX, + number_unsigned_t>::value && + is_detected_exact<bool, number_float_function_t, SAX, number_float_t, + string_t>::value && + is_detected_exact<bool, string_function_t, SAX, string_t>::value && + is_detected_exact<bool, start_object_function_t, SAX>::value && + is_detected_exact<bool, key_function_t, SAX, string_t>::value && + is_detected_exact<bool, end_object_function_t, SAX>::value && + is_detected_exact<bool, start_array_function_t, SAX>::value && + is_detected_exact<bool, end_array_function_t, SAX>::value && + is_detected_exact<bool, parse_error_function_t, SAX, exception_t>::value; +}; + +template <typename SAX, typename BasicJsonType> +struct is_sax_static_asserts +{ + private: + static_assert(is_basic_json<BasicJsonType>::value, + "BasicJsonType must be of type basic_json<...>"); + + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using exception_t = typename BasicJsonType::exception; + + public: + static_assert(is_detected_exact<bool, null_function_t, SAX>::value, + "Missing/invalid function: bool null()"); + static_assert(is_detected_exact<bool, boolean_function_t, SAX>::value, + "Missing/invalid function: bool boolean(bool)"); + static_assert(is_detected_exact<bool, boolean_function_t, SAX>::value, + "Missing/invalid function: bool boolean(bool)"); + static_assert( + is_detected_exact<bool, number_integer_function_t, SAX, + number_integer_t>::value, + "Missing/invalid function: bool number_integer(number_integer_t)"); + static_assert( + is_detected_exact<bool, number_unsigned_function_t, SAX, + number_unsigned_t>::value, + "Missing/invalid function: bool number_unsigned(number_unsigned_t)"); + static_assert(is_detected_exact<bool, number_float_function_t, SAX, + number_float_t, string_t>::value, + "Missing/invalid function: bool number_float(number_float_t, const string_t&)"); + static_assert( + is_detected_exact<bool, string_function_t, SAX, string_t>::value, + "Missing/invalid function: bool string(string_t&)"); + static_assert(is_detected_exact<bool, start_object_function_t, SAX>::value, + "Missing/invalid function: bool start_object(std::size_t)"); + static_assert(is_detected_exact<bool, key_function_t, SAX, string_t>::value, + "Missing/invalid function: bool key(string_t&)"); + static_assert(is_detected_exact<bool, end_object_function_t, SAX>::value, + "Missing/invalid function: bool end_object()"); + static_assert(is_detected_exact<bool, start_array_function_t, SAX>::value, + "Missing/invalid function: bool start_array(std::size_t)"); + static_assert(is_detected_exact<bool, end_array_function_t, SAX>::value, + "Missing/invalid function: bool end_array()"); + static_assert( + is_detected_exact<bool, parse_error_function_t, SAX, exception_t>::value, + "Missing/invalid function: bool parse_error(std::size_t, const " + "std::string&, const exception&)"); +}; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/input/input_adapters.hpp> + +// #include <nlohmann/detail/input/json_sax.hpp> + + +#include <cstddef> +#include <string> +#include <vector> + +// #include <nlohmann/detail/input/parser.hpp> + +// #include <nlohmann/detail/exceptions.hpp> + + +namespace nlohmann +{ + +/*! +@brief SAX interface + +This class describes the SAX interface used by @ref nlohmann::json::sax_parse. +Each function is called in different situations while the input is parsed. The +boolean return value informs the parser whether to continue processing the +input. +*/ +template<typename BasicJsonType> +struct json_sax +{ + /// type for (signed) integers + using number_integer_t = typename BasicJsonType::number_integer_t; + /// type for unsigned integers + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + /// type for floating-point numbers + using number_float_t = typename BasicJsonType::number_float_t; + /// type for strings + using string_t = typename BasicJsonType::string_t; + + /*! + @brief a null value was read + @return whether parsing should proceed + */ + virtual bool null() = 0; + + /*! + @brief a boolean value was read + @param[in] val boolean value + @return whether parsing should proceed + */ + virtual bool boolean(bool val) = 0; + + /*! + @brief an integer number was read + @param[in] val integer value + @return whether parsing should proceed + */ + virtual bool number_integer(number_integer_t val) = 0; + + /*! + @brief an unsigned integer number was read + @param[in] val unsigned integer value + @return whether parsing should proceed + */ + virtual bool number_unsigned(number_unsigned_t val) = 0; + + /*! + @brief an floating-point number was read + @param[in] val floating-point value + @param[in] s raw token value + @return whether parsing should proceed + */ + virtual bool number_float(number_float_t val, const string_t& s) = 0; + + /*! + @brief a string was read + @param[in] val string value + @return whether parsing should proceed + @note It is safe to move the passed string. + */ + virtual bool string(string_t& val) = 0; + + /*! + @brief the beginning of an object was read + @param[in] elements number of object elements or -1 if unknown + @return whether parsing should proceed + @note binary formats may report the number of elements + */ + virtual bool start_object(std::size_t elements) = 0; + + /*! + @brief an object key was read + @param[in] val object key + @return whether parsing should proceed + @note It is safe to move the passed string. + */ + virtual bool key(string_t& val) = 0; + + /*! + @brief the end of an object was read + @return whether parsing should proceed + */ + virtual bool end_object() = 0; + + /*! + @brief the beginning of an array was read + @param[in] elements number of array elements or -1 if unknown + @return whether parsing should proceed + @note binary formats may report the number of elements + */ + virtual bool start_array(std::size_t elements) = 0; + + /*! + @brief the end of an array was read + @return whether parsing should proceed + */ + virtual bool end_array() = 0; + + /*! + @brief a parse error occurred + @param[in] position the position in the input where the error occurs + @param[in] last_token the last read token + @param[in] ex an exception object describing the error + @return whether parsing should proceed (must return false) + */ + virtual bool parse_error(std::size_t position, + const std::string& last_token, + const detail::exception& ex) = 0; + + virtual ~json_sax() = default; +}; + + +namespace detail +{ +/*! +@brief SAX implementation to create a JSON value from SAX events + +This class implements the @ref json_sax interface and processes the SAX events +to create a JSON value which makes it basically a DOM parser. The structure or +hierarchy of the JSON value is managed by the stack `ref_stack` which contains +a pointer to the respective array or object for each recursion depth. + +After successful parsing, the value that is passed by reference to the +constructor contains the parsed value. + +@tparam BasicJsonType the JSON type +*/ +template<typename BasicJsonType> +class json_sax_dom_parser +{ + public: + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + + /*! + @param[in, out] r reference to a JSON value that is manipulated while + parsing + @param[in] allow_exceptions_ whether parse errors yield exceptions + */ + explicit json_sax_dom_parser(BasicJsonType& r, const bool allow_exceptions_ = true) + : root(r), allow_exceptions(allow_exceptions_) + {} + + bool null() + { + handle_value(nullptr); + return true; + } + + bool boolean(bool val) + { + handle_value(val); + return true; + } + + bool number_integer(number_integer_t val) + { + handle_value(val); + return true; + } + + bool number_unsigned(number_unsigned_t val) + { + handle_value(val); + return true; + } + + bool number_float(number_float_t val, const string_t& /*unused*/) + { + handle_value(val); + return true; + } + + bool string(string_t& val) + { + handle_value(val); + return true; + } + + bool start_object(std::size_t len) + { + ref_stack.push_back(handle_value(BasicJsonType::value_t::object)); + + if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, + "excessive object size: " + std::to_string(len))); + } + + return true; + } + + bool key(string_t& val) + { + // add null at given key and store the reference for later + object_element = &(ref_stack.back()->m_value.object->operator[](val)); + return true; + } + + bool end_object() + { + ref_stack.pop_back(); + return true; + } + + bool start_array(std::size_t len) + { + ref_stack.push_back(handle_value(BasicJsonType::value_t::array)); + + if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, + "excessive array size: " + std::to_string(len))); + } + + return true; + } + + bool end_array() + { + ref_stack.pop_back(); + return true; + } + + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, + const detail::exception& ex) + { + errored = true; + if (allow_exceptions) + { + // determine the proper exception type from the id + switch ((ex.id / 100) % 100) + { + case 1: + JSON_THROW(*reinterpret_cast<const detail::parse_error*>(&ex)); + case 4: + JSON_THROW(*reinterpret_cast<const detail::out_of_range*>(&ex)); + // LCOV_EXCL_START + case 2: + JSON_THROW(*reinterpret_cast<const detail::invalid_iterator*>(&ex)); + case 3: + JSON_THROW(*reinterpret_cast<const detail::type_error*>(&ex)); + case 5: + JSON_THROW(*reinterpret_cast<const detail::other_error*>(&ex)); + default: + assert(false); + // LCOV_EXCL_STOP + } + } + return false; + } + + constexpr bool is_errored() const + { + return errored; + } + + private: + /*! + @invariant If the ref stack is empty, then the passed value will be the new + root. + @invariant If the ref stack contains a value, then it is an array or an + object to which we can add elements + */ + template<typename Value> + BasicJsonType* handle_value(Value&& v) + { + if (ref_stack.empty()) + { + root = BasicJsonType(std::forward<Value>(v)); + return &root; + } + + assert(ref_stack.back()->is_array() or ref_stack.back()->is_object()); + + if (ref_stack.back()->is_array()) + { + ref_stack.back()->m_value.array->emplace_back(std::forward<Value>(v)); + return &(ref_stack.back()->m_value.array->back()); + } + else + { + assert(object_element); + *object_element = BasicJsonType(std::forward<Value>(v)); + return object_element; + } + } + + /// the parsed JSON value + BasicJsonType& root; + /// stack to model hierarchy of values + std::vector<BasicJsonType*> ref_stack; + /// helper to hold the reference for the next object element + BasicJsonType* object_element = nullptr; + /// whether a syntax error occurred + bool errored = false; + /// whether to throw exceptions in case of errors + const bool allow_exceptions = true; +}; + +template<typename BasicJsonType> +class json_sax_dom_callback_parser +{ + public: + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using parser_callback_t = typename BasicJsonType::parser_callback_t; + using parse_event_t = typename BasicJsonType::parse_event_t; + + json_sax_dom_callback_parser(BasicJsonType& r, + const parser_callback_t cb, + const bool allow_exceptions_ = true) + : root(r), callback(cb), allow_exceptions(allow_exceptions_) + { + keep_stack.push_back(true); + } + + bool null() + { + handle_value(nullptr); + return true; + } + + bool boolean(bool val) + { + handle_value(val); + return true; + } + + bool number_integer(number_integer_t val) + { + handle_value(val); + return true; + } + + bool number_unsigned(number_unsigned_t val) + { + handle_value(val); + return true; + } + + bool number_float(number_float_t val, const string_t& /*unused*/) + { + handle_value(val); + return true; + } + + bool string(string_t& val) + { + handle_value(val); + return true; + } + + bool start_object(std::size_t len) + { + // check callback for object start + const bool keep = callback(static_cast<int>(ref_stack.size()), parse_event_t::object_start, discarded); + keep_stack.push_back(keep); + + auto val = handle_value(BasicJsonType::value_t::object, true); + ref_stack.push_back(val.second); + + // check object limit + if (ref_stack.back()) + { + if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, + "excessive object size: " + std::to_string(len))); + } + } + + return true; + } + + bool key(string_t& val) + { + BasicJsonType k = BasicJsonType(val); + + // check callback for key + const bool keep = callback(static_cast<int>(ref_stack.size()), parse_event_t::key, k); + key_keep_stack.push_back(keep); + + // add discarded value at given key and store the reference for later + if (keep and ref_stack.back()) + { + object_element = &(ref_stack.back()->m_value.object->operator[](val) = discarded); + } + + return true; + } + + bool end_object() + { + if (ref_stack.back()) + { + if (not callback(static_cast<int>(ref_stack.size()) - 1, parse_event_t::object_end, *ref_stack.back())) + { + // discard object + *ref_stack.back() = discarded; + } + } + + assert(not ref_stack.empty()); + assert(not keep_stack.empty()); + ref_stack.pop_back(); + keep_stack.pop_back(); + + if (not ref_stack.empty() and ref_stack.back()) + { + // remove discarded value + if (ref_stack.back()->is_object()) + { + for (auto it = ref_stack.back()->begin(); it != ref_stack.back()->end(); ++it) + { + if (it->is_discarded()) + { + ref_stack.back()->erase(it); + break; + } + } + } + } + + return true; + } + + bool start_array(std::size_t len) + { + const bool keep = callback(static_cast<int>(ref_stack.size()), parse_event_t::array_start, discarded); + keep_stack.push_back(keep); + + auto val = handle_value(BasicJsonType::value_t::array, true); + ref_stack.push_back(val.second); + + // check array limit + if (ref_stack.back()) + { + if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, + "excessive array size: " + std::to_string(len))); + } + } + + return true; + } + + bool end_array() + { + bool keep = true; + + if (ref_stack.back()) + { + keep = callback(static_cast<int>(ref_stack.size()) - 1, parse_event_t::array_end, *ref_stack.back()); + if (not keep) + { + // discard array + *ref_stack.back() = discarded; + } + } + + assert(not ref_stack.empty()); + assert(not keep_stack.empty()); + ref_stack.pop_back(); + keep_stack.pop_back(); + + // remove discarded value + if (not keep and not ref_stack.empty()) + { + if (ref_stack.back()->is_array()) + { + ref_stack.back()->m_value.array->pop_back(); + } + } + + return true; + } + + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, + const detail::exception& ex) + { + errored = true; + if (allow_exceptions) + { + // determine the proper exception type from the id + switch ((ex.id / 100) % 100) + { + case 1: + JSON_THROW(*reinterpret_cast<const detail::parse_error*>(&ex)); + case 4: + JSON_THROW(*reinterpret_cast<const detail::out_of_range*>(&ex)); + // LCOV_EXCL_START + case 2: + JSON_THROW(*reinterpret_cast<const detail::invalid_iterator*>(&ex)); + case 3: + JSON_THROW(*reinterpret_cast<const detail::type_error*>(&ex)); + case 5: + JSON_THROW(*reinterpret_cast<const detail::other_error*>(&ex)); + default: + assert(false); + // LCOV_EXCL_STOP + } + } + return false; + } + + constexpr bool is_errored() const + { + return errored; + } + + private: + /*! + @param[in] v value to add to the JSON value we build during parsing + @param[in] skip_callback whether we should skip calling the callback + function; this is required after start_array() and + start_object() SAX events, because otherwise we would call the + callback function with an empty array or object, respectively. + + @invariant If the ref stack is empty, then the passed value will be the new + root. + @invariant If the ref stack contains a value, then it is an array or an + object to which we can add elements + + @return pair of boolean (whether value should be kept) and pointer (to the + passed value in the ref_stack hierarchy; nullptr if not kept) + */ + template<typename Value> + std::pair<bool, BasicJsonType*> handle_value(Value&& v, const bool skip_callback = false) + { + assert(not keep_stack.empty()); + + // do not handle this value if we know it would be added to a discarded + // container + if (not keep_stack.back()) + { + return {false, nullptr}; + } + + // create value + auto value = BasicJsonType(std::forward<Value>(v)); + + // check callback + const bool keep = skip_callback or callback(static_cast<int>(ref_stack.size()), parse_event_t::value, value); + + // do not handle this value if we just learnt it shall be discarded + if (not keep) + { + return {false, nullptr}; + } + + if (ref_stack.empty()) + { + root = std::move(value); + return {true, &root}; + } + + // skip this value if we already decided to skip the parent + // (https://github.com/nlohmann/json/issues/971#issuecomment-413678360) + if (not ref_stack.back()) + { + return {false, nullptr}; + } + + // we now only expect arrays and objects + assert(ref_stack.back()->is_array() or ref_stack.back()->is_object()); + + if (ref_stack.back()->is_array()) + { + ref_stack.back()->m_value.array->push_back(std::move(value)); + return {true, &(ref_stack.back()->m_value.array->back())}; + } + else + { + // check if we should store an element for the current key + assert(not key_keep_stack.empty()); + const bool store_element = key_keep_stack.back(); + key_keep_stack.pop_back(); + + if (not store_element) + { + return {false, nullptr}; + } + + assert(object_element); + *object_element = std::move(value); + return {true, object_element}; + } + } + + /// the parsed JSON value + BasicJsonType& root; + /// stack to model hierarchy of values + std::vector<BasicJsonType*> ref_stack; + /// stack to manage which values to keep + std::vector<bool> keep_stack; + /// stack to manage which object keys to keep + std::vector<bool> key_keep_stack; + /// helper to hold the reference for the next object element + BasicJsonType* object_element = nullptr; + /// whether a syntax error occurred + bool errored = false; + /// callback function + const parser_callback_t callback = nullptr; + /// whether to throw exceptions in case of errors + const bool allow_exceptions = true; + /// a discarded value for the callback + BasicJsonType discarded = BasicJsonType::value_t::discarded; +}; + +template<typename BasicJsonType> +class json_sax_acceptor +{ + public: + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + + bool null() + { + return true; + } + + bool boolean(bool /*unused*/) + { + return true; + } + + bool number_integer(number_integer_t /*unused*/) + { + return true; + } + + bool number_unsigned(number_unsigned_t /*unused*/) + { + return true; + } + + bool number_float(number_float_t /*unused*/, const string_t& /*unused*/) + { + return true; + } + + bool string(string_t& /*unused*/) + { + return true; + } + + bool start_object(std::size_t /*unused*/ = std::size_t(-1)) + { + return true; + } + + bool key(string_t& /*unused*/) + { + return true; + } + + bool end_object() + { + return true; + } + + bool start_array(std::size_t /*unused*/ = std::size_t(-1)) + { + return true; + } + + bool end_array() + { + return true; + } + + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, const detail::exception& /*unused*/) + { + return false; + } +}; +} // namespace detail + +} // namespace nlohmann + +// #include <nlohmann/detail/input/lexer.hpp> + +// #include <nlohmann/detail/value_t.hpp> + + +namespace nlohmann +{ +namespace detail +{ +//////////// +// parser // +//////////// /*! @brief syntax analysis @@ -2892,6 +4962,7 @@ class parser using number_integer_t = typename BasicJsonType::number_integer_t; using number_unsigned_t = typename BasicJsonType::number_unsigned_t; using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; using lexer_t = lexer<BasicJsonType>; using token_type = typename lexer_t::token_type; @@ -2916,11 +4987,14 @@ class parser std::function<bool(int depth, parse_event_t event, BasicJsonType& parsed)>; /// a parser reading from an input adapter - explicit parser(detail::input_adapter_t adapter, + explicit parser(detail::input_adapter_t&& adapter, const parser_callback_t cb = nullptr, const bool allow_exceptions_ = true) - : callback(cb), m_lexer(adapter), allow_exceptions(allow_exceptions_) - {} + : callback(cb), m_lexer(std::move(adapter)), allow_exceptions(allow_exceptions_) + { + // read first token + get_token(); + } /*! @brief public parser interface @@ -2934,31 +5008,56 @@ class parser */ void parse(const bool strict, BasicJsonType& result) { - // read first token - get_token(); + if (callback) + { + json_sax_dom_callback_parser<BasicJsonType> sdp(result, callback, allow_exceptions); + sax_parse_internal(&sdp); + result.assert_invariant(); - parse_internal(true, result); - result.assert_invariant(); + // in strict mode, input must be completely read + if (strict and (get_token() != token_type::end_of_input)) + { + sdp.parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::end_of_input, "value"))); + } - // in strict mode, input must be completely read - if (strict) - { - get_token(); - expect(token_type::end_of_input); - } + // in case of an error, return discarded value + if (sdp.is_errored()) + { + result = value_t::discarded; + return; + } - // in case of an error, return discarded value - if (errored) - { - result = value_t::discarded; - return; + // set top-level value to null if it was discarded by the callback + // function + if (result.is_discarded()) + { + result = nullptr; + } } - - // set top-level value to null if it was discarded by the callback - // function - if (result.is_discarded()) + else { - result = nullptr; + json_sax_dom_parser<BasicJsonType> sdp(result, allow_exceptions); + sax_parse_internal(&sdp); + result.assert_invariant(); + + // in strict mode, input must be completely read + if (strict and (get_token() != token_type::end_of_input)) + { + sdp.parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::end_of_input, "value"))); + } + + // in case of an error, return discarded value + if (sdp.is_errored()) + { + result = value_t::discarded; + return; + } } } @@ -2970,413 +5069,317 @@ class parser */ bool accept(const bool strict = true) { - // read first token - get_token(); + json_sax_acceptor<BasicJsonType> sax_acceptor; + return sax_parse(&sax_acceptor, strict); + } - if (not accept_internal()) + template <typename SAX> + bool sax_parse(SAX* sax, const bool strict = true) + { + (void)detail::is_sax_static_asserts<SAX, BasicJsonType> {}; + const bool result = sax_parse_internal(sax); + + // strict mode: next byte must be EOF + if (result and strict and (get_token() != token_type::end_of_input)) { - return false; + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::end_of_input, "value"))); } - // strict => last token must be EOF - return not strict or (get_token() == token_type::end_of_input); + return result; } private: - /*! - @brief the actual parser - @throw parse_error.101 in case of an unexpected token - @throw parse_error.102 if to_unicode fails or surrogate error - @throw parse_error.103 if to_unicode fails - */ - void parse_internal(bool keep, BasicJsonType& result) + template <typename SAX> + bool sax_parse_internal(SAX* sax) { - // never parse after a parse error was detected - assert(not errored); + // stack to remember the hierarchy of structured values we are parsing + // true = array; false = object + std::vector<bool> states; + // value to avoid a goto (see comment where set to true) + bool skip_to_state_evaluation = false; - // start with a discarded value - if (not result.is_discarded()) - { - result.m_value.destroy(result.m_type); - result.m_type = value_t::discarded; - } - - switch (last_token) + while (true) { - case token_type::begin_object: + if (not skip_to_state_evaluation) { - if (keep) + // invariant: get_token() was called before each iteration + switch (last_token) { - if (callback) + case token_type::begin_object: { - keep = callback(depth++, parse_event_t::object_start, result); - } + if (JSON_UNLIKELY(not sax->start_object(std::size_t(-1)))) + { + return false; + } - if (not callback or keep) - { - // explicitly set result to object to cope with {} - result.m_type = value_t::object; - result.m_value = value_t::object; - } - } + // closing } -> we are done + if (get_token() == token_type::end_object) + { + if (JSON_UNLIKELY(not sax->end_object())) + { + return false; + } + break; + } - // read next token - get_token(); + // parse key + if (JSON_UNLIKELY(last_token != token_type::value_string)) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::value_string, "object key"))); + } + if (JSON_UNLIKELY(not sax->key(m_lexer.get_string()))) + { + return false; + } - // closing } -> we are done - if (last_token == token_type::end_object) - { - if (keep and callback and not callback(--depth, parse_event_t::object_end, result)) - { - result.m_value.destroy(result.m_type); - result.m_type = value_t::discarded; - } - break; - } + // parse separator (:) + if (JSON_UNLIKELY(get_token() != token_type::name_separator)) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::name_separator, "object separator"))); + } - // parse values - std::string key; - BasicJsonType value; - while (true) - { - // store key - if (not expect(token_type::value_string)) - { - return; + // remember we are now inside an object + states.push_back(false); + + // parse values + get_token(); + continue; } - key = m_lexer.move_string(); - bool keep_tag = false; - if (keep) + case token_type::begin_array: { - if (callback) + if (JSON_UNLIKELY(not sax->start_array(std::size_t(-1)))) { - BasicJsonType k(key); - keep_tag = callback(depth, parse_event_t::key, k); + return false; } - else + + // closing ] -> we are done + if (get_token() == token_type::end_array) { - keep_tag = true; + if (JSON_UNLIKELY(not sax->end_array())) + { + return false; + } + break; } - } - - // parse separator (:) - get_token(); - if (not expect(token_type::name_separator)) - { - return; - } - // parse and add value - get_token(); - value.m_value.destroy(value.m_type); - value.m_type = value_t::discarded; - parse_internal(keep, value); + // remember we are now inside an array + states.push_back(true); - if (JSON_UNLIKELY(errored)) - { - return; + // parse values (no need to call get_token) + continue; } - if (keep and keep_tag and not value.is_discarded()) + case token_type::value_float: { - result.m_value.object->emplace(std::move(key), std::move(value)); - } + const auto res = m_lexer.get_number_float(); - // comma -> next value - get_token(); - if (last_token == token_type::value_separator) - { - get_token(); - continue; + if (JSON_UNLIKELY(not std::isfinite(res))) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + out_of_range::create(406, "number overflow parsing '" + m_lexer.get_token_string() + "'")); + } + else + { + if (JSON_UNLIKELY(not sax->number_float(res, m_lexer.get_string()))) + { + return false; + } + break; + } } - // closing } - if (not expect(token_type::end_object)) + case token_type::literal_false: { - return; + if (JSON_UNLIKELY(not sax->boolean(false))) + { + return false; + } + break; } - break; - } - if (keep and callback and not callback(--depth, parse_event_t::object_end, result)) - { - result.m_value.destroy(result.m_type); - result.m_type = value_t::discarded; - } - break; - } - - case token_type::begin_array: - { - if (keep) - { - if (callback) + case token_type::literal_null: { - keep = callback(depth++, parse_event_t::array_start, result); + if (JSON_UNLIKELY(not sax->null())) + { + return false; + } + break; } - if (not callback or keep) + case token_type::literal_true: { - // explicitly set result to array to cope with [] - result.m_type = value_t::array; - result.m_value = value_t::array; + if (JSON_UNLIKELY(not sax->boolean(true))) + { + return false; + } + break; } - } - - // read next token - get_token(); - // closing ] -> we are done - if (last_token == token_type::end_array) - { - if (callback and not callback(--depth, parse_event_t::array_end, result)) + case token_type::value_integer: { - result.m_value.destroy(result.m_type); - result.m_type = value_t::discarded; + if (JSON_UNLIKELY(not sax->number_integer(m_lexer.get_number_integer()))) + { + return false; + } + break; } - break; - } - // parse values - BasicJsonType value; - while (true) - { - // parse value - value.m_value.destroy(value.m_type); - value.m_type = value_t::discarded; - parse_internal(keep, value); - - if (JSON_UNLIKELY(errored)) + case token_type::value_string: { - return; + if (JSON_UNLIKELY(not sax->string(m_lexer.get_string()))) + { + return false; + } + break; } - if (keep and not value.is_discarded()) + case token_type::value_unsigned: { - result.m_value.array->push_back(std::move(value)); + if (JSON_UNLIKELY(not sax->number_unsigned(m_lexer.get_number_unsigned()))) + { + return false; + } + break; } - // comma -> next value - get_token(); - if (last_token == token_type::value_separator) + case token_type::parse_error: { - get_token(); - continue; + // using "uninitialized" to avoid "expected" message + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::uninitialized, "value"))); } - // closing ] - if (not expect(token_type::end_array)) + default: // the last token was unexpected { - return; + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::literal_or_value, "value"))); } - break; - } - - if (keep and callback and not callback(--depth, parse_event_t::array_end, result)) - { - result.m_value.destroy(result.m_type); - result.m_type = value_t::discarded; } - break; - } - - case token_type::literal_null: - { - result.m_type = value_t::null; - break; - } - - case token_type::value_string: - { - result.m_type = value_t::string; - result.m_value = m_lexer.move_string(); - break; - } - - case token_type::literal_true: - { - result.m_type = value_t::boolean; - result.m_value = true; - break; - } - - case token_type::literal_false: - { - result.m_type = value_t::boolean; - result.m_value = false; - break; } - - case token_type::value_unsigned: + else { - result.m_type = value_t::number_unsigned; - result.m_value = m_lexer.get_number_unsigned(); - break; + skip_to_state_evaluation = false; } - case token_type::value_integer: + // we reached this line after we successfully parsed a value + if (states.empty()) { - result.m_type = value_t::number_integer; - result.m_value = m_lexer.get_number_integer(); - break; + // empty stack: we reached the end of the hierarchy: done + return true; } - - case token_type::value_float: + else { - result.m_type = value_t::number_float; - result.m_value = m_lexer.get_number_float(); - - // throw in case of infinity or NAN - if (JSON_UNLIKELY(not std::isfinite(result.m_value.number_float))) + if (states.back()) // array { - if (allow_exceptions) + // comma -> next value + if (get_token() == token_type::value_separator) { - JSON_THROW(out_of_range::create(406, "number overflow parsing '" + - m_lexer.get_token_string() + "'")); + // parse a new value + get_token(); + continue; } - expect(token_type::uninitialized); - } - break; - } - - case token_type::parse_error: - { - // using "uninitialized" to avoid "expected" message - if (not expect(token_type::uninitialized)) - { - return; - } - break; // LCOV_EXCL_LINE - } - - default: - { - // the last token was unexpected; we expected a value - if (not expect(token_type::literal_or_value)) - { - return; - } - break; // LCOV_EXCL_LINE - } - } - - if (keep and callback and not callback(depth, parse_event_t::value, result)) - { - result.m_type = value_t::discarded; - } - } - - /*! - @brief the actual acceptor - - @invariant 1. The last token is not yet processed. Therefore, the caller - of this function must make sure a token has been read. - 2. When this function returns, the last token is processed. - That is, the last read character was already considered. - - This invariant makes sure that no token needs to be "unput". - */ - bool accept_internal() - { - switch (last_token) - { - case token_type::begin_object: - { - // read next token - get_token(); - // closing } -> we are done - if (last_token == token_type::end_object) - { - return true; - } - - // parse values - while (true) - { - // parse key - if (last_token != token_type::value_string) + // closing ] + if (JSON_LIKELY(last_token == token_type::end_array)) { - return false; - } + if (JSON_UNLIKELY(not sax->end_array())) + { + return false; + } - // parse separator (:) - get_token(); - if (last_token != token_type::name_separator) - { - return false; + // We are done with this array. Before we can parse a + // new value, we need to evaluate the new state first. + // By setting skip_to_state_evaluation to false, we + // are effectively jumping to the beginning of this if. + assert(not states.empty()); + states.pop_back(); + skip_to_state_evaluation = true; + continue; } - - // parse value - get_token(); - if (not accept_internal()) + else { - return false; + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::end_array, "array"))); } - + } + else // object + { // comma -> next value - get_token(); - if (last_token == token_type::value_separator) + if (get_token() == token_type::value_separator) { + // parse key + if (JSON_UNLIKELY(get_token() != token_type::value_string)) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::value_string, "object key"))); + } + else + { + if (JSON_UNLIKELY(not sax->key(m_lexer.get_string()))) + { + return false; + } + } + + // parse separator (:) + if (JSON_UNLIKELY(get_token() != token_type::name_separator)) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::name_separator, "object separator"))); + } + + // parse values get_token(); continue; } // closing } - return (last_token == token_type::end_object); - } - } - - case token_type::begin_array: - { - // read next token - get_token(); - - // closing ] -> we are done - if (last_token == token_type::end_array) - { - return true; - } - - // parse values - while (true) - { - // parse value - if (not accept_internal()) + if (JSON_LIKELY(last_token == token_type::end_object)) { - return false; - } + if (JSON_UNLIKELY(not sax->end_object())) + { + return false; + } - // comma -> next value - get_token(); - if (last_token == token_type::value_separator) - { - get_token(); + // We are done with this object. Before we can parse a + // new value, we need to evaluate the new state first. + // By setting skip_to_state_evaluation to false, we + // are effectively jumping to the beginning of this if. + assert(not states.empty()); + states.pop_back(); + skip_to_state_evaluation = true; continue; } - - // closing ] - return (last_token == token_type::end_array); + else + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::end_object, "object"))); + } } } - - case token_type::value_float: - { - // reject infinity or NAN - return std::isfinite(m_lexer.get_number_float()); - } - - case token_type::literal_false: - case token_type::literal_null: - case token_type::literal_true: - case token_type::value_integer: - case token_type::value_string: - case token_type::value_unsigned: - return true; - - default: // the last token was unexpected - return false; } } @@ -3386,31 +5389,17 @@ class parser return (last_token = m_lexer.scan()); } - /*! - @throw parse_error.101 if expected token did not occur - */ - bool expect(token_type t) + std::string exception_message(const token_type expected, const std::string& context) { - if (JSON_UNLIKELY(t != last_token)) + std::string error_msg = "syntax error "; + + if (not context.empty()) { - errored = true; - expected = t; - if (allow_exceptions) - { - throw_exception(); - } - else - { - return false; - } + error_msg += "while parsing " + context + " "; } - return true; - } + error_msg += "- "; - [[noreturn]] void throw_exception() const - { - std::string error_msg = "syntax error - "; if (last_token == token_type::parse_error) { error_msg += std::string(m_lexer.get_error_message()) + "; last read: '" + @@ -3426,31 +5415,33 @@ class parser error_msg += "; expected " + std::string(lexer_t::token_type_name(expected)); } - JSON_THROW(parse_error::create(101, m_lexer.get_position(), error_msg)); + return error_msg; } private: - /// current level of recursion - int depth = 0; /// callback function const parser_callback_t callback = nullptr; /// the type of the last read token token_type last_token = token_type::uninitialized; /// the lexer lexer_t m_lexer; - /// whether a syntax error occurred - bool errored = false; - /// possible reason for the syntax error - token_type expected = token_type::uninitialized; /// whether to throw exceptions in case of errors const bool allow_exceptions = true; }; +} // namespace detail +} // namespace nlohmann -/////////////// -// iterators // -/////////////// +// #include <nlohmann/detail/iterators/primitive_iterator.hpp> -/*! + +#include <cstddef> // ptrdiff_t +#include <limits> // numeric_limits + +namespace nlohmann +{ +namespace detail +{ +/* @brief an iterator for primitive JSON types This class models an iterator for primitive JSON types (boolean, number, @@ -3461,9 +5452,15 @@ end_value (`1`) models past the end. */ class primitive_iterator_t { - public: + private: using difference_type = std::ptrdiff_t; + static constexpr difference_type begin_value = 0; + static constexpr difference_type end_value = begin_value + 1; + /// iterator as signed integer type + difference_type m_it = (std::numeric_limits<std::ptrdiff_t>::min)(); + + public: constexpr difference_type get_value() const noexcept { return m_it; @@ -3503,10 +5500,10 @@ class primitive_iterator_t return lhs.m_it < rhs.m_it; } - primitive_iterator_t operator+(difference_type i) + primitive_iterator_t operator+(difference_type n) noexcept { auto result = *this; - result += i; + result += n; return result; } @@ -3515,57 +5512,57 @@ class primitive_iterator_t return lhs.m_it - rhs.m_it; } - friend std::ostream& operator<<(std::ostream& os, primitive_iterator_t it) - { - return os << it.m_it; - } - - primitive_iterator_t& operator++() + primitive_iterator_t& operator++() noexcept { ++m_it; return *this; } - primitive_iterator_t const operator++(int) + primitive_iterator_t const operator++(int) noexcept { auto result = *this; - m_it++; + ++m_it; return result; } - primitive_iterator_t& operator--() + primitive_iterator_t& operator--() noexcept { --m_it; return *this; } - primitive_iterator_t const operator--(int) + primitive_iterator_t const operator--(int) noexcept { auto result = *this; - m_it--; + --m_it; return result; } - primitive_iterator_t& operator+=(difference_type n) + primitive_iterator_t& operator+=(difference_type n) noexcept { m_it += n; return *this; } - primitive_iterator_t& operator-=(difference_type n) + primitive_iterator_t& operator-=(difference_type n) noexcept { m_it -= n; return *this; } +}; +} // namespace detail +} // namespace nlohmann - private: - static constexpr difference_type begin_value = 0; - static constexpr difference_type end_value = begin_value + 1; +// #include <nlohmann/detail/iterators/internal_iterator.hpp> + + +// #include <nlohmann/detail/iterators/primitive_iterator.hpp> - /// iterator as signed integer type - difference_type m_it = (std::numeric_limits<std::ptrdiff_t>::min)(); -}; +namespace nlohmann +{ +namespace detail +{ /*! @brief an iterator value @@ -3581,26 +5578,50 @@ template<typename BasicJsonType> struct internal_iterator /// generic iterator for all other types primitive_iterator_t primitive_iterator {}; }; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/iterators/iter_impl.hpp> + + +#include <ciso646> // not +#include <iterator> // iterator, random_access_iterator_tag, bidirectional_iterator_tag, advance, next +#include <type_traits> // conditional, is_const, remove_const + +// #include <nlohmann/detail/exceptions.hpp> + +// #include <nlohmann/detail/iterators/internal_iterator.hpp> +// #include <nlohmann/detail/iterators/primitive_iterator.hpp> + +// #include <nlohmann/detail/macro_scope.hpp> + +// #include <nlohmann/detail/meta/cpp_future.hpp> + +// #include <nlohmann/detail/value_t.hpp> + + +namespace nlohmann +{ +namespace detail +{ +// forward declare, to be able to friend it later on template<typename IteratorType> class iteration_proxy; +template<typename IteratorType> class iteration_proxy_value; /*! @brief a template for a bidirectional iterator for the @ref basic_json class - This class implements a both iterators (iterator and const_iterator) for the @ref basic_json class. - @note An iterator is called *initialized* when a pointer to a JSON value has been set (e.g., by a constructor or a copy assignment). If the iterator is default-constructed, it is *uninitialized* and most methods are undefined. **The library uses assertions to detect calls on uninitialized iterators.** - @requirement The class satisfies the following concept requirements: - -[BidirectionalIterator](http://en.cppreference.com/w/cpp/concept/BidirectionalIterator): +[BidirectionalIterator](https://en.cppreference.com/w/cpp/named_req/BidirectionalIterator): The iterator that can be moved can be moved in both directions (i.e. incremented and decremented). - @since version 1.0.0, simplified in version 2.0.9, change to bidirectional iterators in version 3.0.0 (see https://github.com/nlohmann/json/issues/593) */ @@ -3611,6 +5632,7 @@ class iter_impl friend iter_impl<typename std::conditional<std::is_const<BasicJsonType>::value, typename std::remove_const<BasicJsonType>::type, const BasicJsonType>::type>; friend BasicJsonType; friend iteration_proxy<iter_impl>; + friend iteration_proxy_value<iter_impl>; using object_t = typename BasicJsonType::object_t; using array_t = typename BasicJsonType::array_t; @@ -4149,7 +6171,7 @@ class iter_impl @brief return the key of an object iterator @pre The iterator is initialized; i.e. `m_object != nullptr`. */ - typename object_t::key_type key() const + const typename object_t::key_type& key() const { assert(m_object != nullptr); @@ -4174,94 +6196,26 @@ class iter_impl /// associated JSON instance pointer m_object = nullptr; /// the actual iterator of the associated instance - internal_iterator<typename std::remove_const<BasicJsonType>::type> m_it = {}; + internal_iterator<typename std::remove_const<BasicJsonType>::type> m_it; }; +} // namespace detail +} // namespace nlohmann +// #include <nlohmann/detail/iterators/iteration_proxy.hpp> -/// proxy class for the iterator_wrapper functions -template<typename IteratorType> class iteration_proxy -{ - private: - /// helper class for iteration - class iteration_proxy_internal - { - private: - /// the iterator - IteratorType anchor; - /// an index for arrays (used to create key names) - std::size_t array_index = 0; - - public: - explicit iteration_proxy_internal(IteratorType it) noexcept : anchor(it) {} - - /// dereference operator (needed for range-based for) - iteration_proxy_internal& operator*() - { - return *this; - } - - /// increment operator (needed for range-based for) - iteration_proxy_internal& operator++() - { - ++anchor; - ++array_index; - - return *this; - } - - /// inequality operator (needed for range-based for) - bool operator!=(const iteration_proxy_internal& o) const noexcept - { - return anchor != o.anchor; - } - - /// return key of the iterator - std::string key() const - { - assert(anchor.m_object != nullptr); - - switch (anchor.m_object->type()) - { - // use integer array index as key - case value_t::array: - return std::to_string(array_index); - - // use key from the object - case value_t::object: - return anchor.key(); - - // use an empty key for all primitive types - default: - return ""; - } - } - - /// return value of the iterator - typename IteratorType::reference value() const - { - return anchor.value(); - } - }; - - /// the container to iterate - typename IteratorType::reference container; +// #include <nlohmann/detail/iterators/json_reverse_iterator.hpp> - public: - /// construct iteration proxy from a container - explicit iteration_proxy(typename IteratorType::reference cont) - : container(cont) {} - /// return iterator begin (needed for range-based for) - iteration_proxy_internal begin() noexcept - { - return iteration_proxy_internal(container.begin()); - } +#include <cstddef> // ptrdiff_t +#include <iterator> // reverse_iterator +#include <utility> // declval - /// return iterator end (needed for range-based for) - iteration_proxy_internal end() noexcept - { - return iteration_proxy_internal(container.end()); - } -}; +namespace nlohmann +{ +namespace detail +{ +////////////////////// +// reverse_iterator // +////////////////////// /*! @brief a template for a reverse iterator class @@ -4272,10 +6226,10 @@ create @ref const_reverse_iterator). @requirement The class satisfies the following concept requirements: - -[BidirectionalIterator](http://en.cppreference.com/w/cpp/concept/BidirectionalIterator): +[BidirectionalIterator](https://en.cppreference.com/w/cpp/named_req/BidirectionalIterator): The iterator that can be moved can be moved in both directions (i.e. incremented and decremented). -- [OutputIterator](http://en.cppreference.com/w/cpp/concept/OutputIterator): +- [OutputIterator](https://en.cppreference.com/w/cpp/named_req/OutputIterator): It is possible to write to the pointed-to element (only if @a Base is @ref iterator). @@ -4292,11 +6246,11 @@ class json_reverse_iterator : public std::reverse_iterator<Base> using reference = typename Base::reference; /// create reverse iterator from iterator - json_reverse_iterator(const typename base_iterator::iterator_type& it) noexcept + explicit json_reverse_iterator(const typename base_iterator::iterator_type& it) noexcept : base_iterator(it) {} /// create reverse iterator from base class - json_reverse_iterator(const base_iterator& it) noexcept : base_iterator(it) {} + explicit json_reverse_iterator(const base_iterator& it) noexcept : base_iterator(it) {} /// post-increment (it++) json_reverse_iterator const operator++(int) @@ -4366,11 +6320,25 @@ class json_reverse_iterator : public std::reverse_iterator<Base> return it.operator * (); } }; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/output/output_adapters.hpp> -///////////////////// -// output adapters // -///////////////////// +#include <algorithm> // copy +#include <cstddef> // size_t +#include <ios> // streamsize +#include <iterator> // back_inserter +#include <memory> // shared_ptr, make_shared +#include <ostream> // basic_ostream +#include <string> // basic_string +#include <vector> // vector + +namespace nlohmann +{ +namespace detail +{ /// abstract output adapter interface template<typename CharType> struct output_adapter_protocol { @@ -4388,7 +6356,9 @@ template<typename CharType> class output_vector_adapter : public output_adapter_protocol<CharType> { public: - explicit output_vector_adapter(std::vector<CharType>& vec) : v(vec) {} + explicit output_vector_adapter(std::vector<CharType>& vec) noexcept + : v(vec) + {} void write_character(CharType c) override { @@ -4409,7 +6379,9 @@ template<typename CharType> class output_stream_adapter : public output_adapter_protocol<CharType> { public: - explicit output_stream_adapter(std::basic_ostream<CharType>& s) : stream(s) {} + explicit output_stream_adapter(std::basic_ostream<CharType>& s) noexcept + : stream(s) + {} void write_character(CharType c) override { @@ -4426,11 +6398,13 @@ class output_stream_adapter : public output_adapter_protocol<CharType> }; /// output adapter for basic_string -template<typename CharType> +template<typename CharType, typename StringType = std::basic_string<CharType>> class output_string_adapter : public output_adapter_protocol<CharType> { public: - explicit output_string_adapter(std::basic_string<CharType>& s) : str(s) {} + explicit output_string_adapter(StringType& s) noexcept + : str(s) + {} void write_character(CharType c) override { @@ -4443,10 +6417,10 @@ class output_string_adapter : public output_adapter_protocol<CharType> } private: - std::basic_string<CharType>& str; + StringType& str; }; -template<typename CharType> +template<typename CharType, typename StringType = std::basic_string<CharType>> class output_adapter { public: @@ -4456,8 +6430,8 @@ class output_adapter output_adapter(std::basic_ostream<CharType>& s) : oa(std::make_shared<output_stream_adapter<CharType>>(s)) {} - output_adapter(std::basic_string<CharType>& s) - : oa(std::make_shared<output_string_adapter<CharType>>(s)) {} + output_adapter(StringType& s) + : oa(std::make_shared<output_string_adapter<CharType, StringType>>(s)) {} operator output_adapter_t<CharType>() { @@ -4467,19 +6441,57 @@ class output_adapter private: output_adapter_t<CharType> oa = nullptr; }; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/input/binary_reader.hpp> -////////////////////////////// -// binary reader and writer // -////////////////////////////// + +#include <algorithm> // generate_n +#include <array> // array +#include <cassert> // assert +#include <cmath> // ldexp +#include <cstddef> // size_t +#include <cstdint> // uint8_t, uint16_t, uint32_t, uint64_t +#include <cstdio> // snprintf +#include <cstring> // memcpy +#include <iterator> // back_inserter +#include <limits> // numeric_limits +#include <string> // char_traits, string +#include <utility> // make_pair, move + +// #include <nlohmann/detail/input/input_adapters.hpp> + +// #include <nlohmann/detail/input/json_sax.hpp> + +// #include <nlohmann/detail/exceptions.hpp> + +// #include <nlohmann/detail/macro_scope.hpp> + +// #include <nlohmann/detail/meta/is_sax.hpp> + +// #include <nlohmann/detail/value_t.hpp> + + +namespace nlohmann +{ +namespace detail +{ +/////////////////// +// binary reader // +/////////////////// /*! -@brief deserialization of CBOR and MessagePack values +@brief deserialization of CBOR, MessagePack, and UBJSON values */ -template<typename BasicJsonType> +template<typename BasicJsonType, typename SAX = json_sax_dom_parser<BasicJsonType>> class binary_reader { using number_integer_t = typename BasicJsonType::number_integer_t; using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using json_sax_t = SAX; public: /*! @@ -4489,49 +6501,68 @@ class binary_reader */ explicit binary_reader(input_adapter_t adapter) : ia(std::move(adapter)) { + (void)detail::is_sax_static_asserts<SAX, BasicJsonType> {}; assert(ia); } /*! - @brief create a JSON value from CBOR input - + @param[in] format the binary format to parse + @param[in] sax_ a SAX event processor @param[in] strict whether to expect the input to be consumed completed - @return JSON value created from CBOR input - @throw parse_error.110 if input ended unexpectedly or the end of file was - not reached when @a strict was set to true - @throw parse_error.112 if unsupported byte was read + @return */ - BasicJsonType parse_cbor(const bool strict) + bool sax_parse(const input_format_t format, + json_sax_t* sax_, + const bool strict = true) { - const auto res = parse_cbor_internal(); - if (strict) + sax = sax_; + bool result = false; + + switch (format) { - get(); - check_eof(true); - } - return res; - } + case input_format_t::bson: + result = parse_bson_internal(); + break; - /*! - @brief create a JSON value from MessagePack input + case input_format_t::cbor: + result = parse_cbor_internal(); + break; - @param[in] strict whether to expect the input to be consumed completed - @return JSON value created from MessagePack input + case input_format_t::msgpack: + result = parse_msgpack_internal(); + break; - @throw parse_error.110 if input ended unexpectedly or the end of file was - not reached when @a strict was set to true - @throw parse_error.112 if unsupported byte was read - */ - BasicJsonType parse_msgpack(const bool strict) - { - const auto res = parse_msgpack_internal(); - if (strict) + case input_format_t::ubjson: + result = parse_ubjson_internal(); + break; + + // LCOV_EXCL_START + default: + assert(false); + // LCOV_EXCL_STOP + } + + // strict mode: next byte must be EOF + if (result and strict) { - get(); - check_eof(true); + if (format == input_format_t::ubjson) + { + get_ignore_noop(); + } + else + { + get(); + } + + if (JSON_UNLIKELY(current != std::char_traits<char>::eof())) + { + return sax->parse_error(chars_read, get_token_string(), + parse_error::create(110, chars_read, exception_message(format, "expected end of input; last byte: 0x" + get_token_string(), "value"))); + } } - return res; + + return result; } /*! @@ -4547,18 +6578,239 @@ class binary_reader } private: + ////////// + // BSON // + ////////// + + /*! + @brief Reads in a BSON-object and passes it to the SAX-parser. + @return whether a valid BSON-value was passed to the SAX parser + */ + bool parse_bson_internal() + { + std::int32_t document_size; + get_number<std::int32_t, true>(input_format_t::bson, document_size); + + if (JSON_UNLIKELY(not sax->start_object(std::size_t(-1)))) + { + return false; + } + + if (JSON_UNLIKELY(not parse_bson_element_list(/*is_array*/false))) + { + return false; + } + + return sax->end_object(); + } + + /*! + @brief Parses a C-style string from the BSON input. + @param[in, out] result A reference to the string variable where the read + string is to be stored. + @return `true` if the \x00-byte indicating the end of the string was + encountered before the EOF; false` indicates an unexpected EOF. + */ + bool get_bson_cstr(string_t& result) + { + auto out = std::back_inserter(result); + while (true) + { + get(); + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::bson, "cstring"))) + { + return false; + } + if (current == 0x00) + { + return true; + } + *out++ = static_cast<char>(current); + } + + return true; + } + + /*! + @brief Parses a zero-terminated string of length @a len from the BSON + input. + @param[in] len The length (including the zero-byte at the end) of the + string to be read. + @param[in, out] result A reference to the string variable where the read + string is to be stored. + @tparam NumberType The type of the length @a len + @pre len >= 1 + @return `true` if the string was successfully parsed + */ + template<typename NumberType> + bool get_bson_string(const NumberType len, string_t& result) + { + if (JSON_UNLIKELY(len < 1)) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::bson, "string length must be at least 1, is " + std::to_string(len), "string"))); + } + + return get_string(input_format_t::bson, len - static_cast<NumberType>(1), result) and get() != std::char_traits<char>::eof(); + } + + /*! + @brief Read a BSON document element of the given @a element_type. + @param[in] element_type The BSON element type, c.f. http://bsonspec.org/spec.html + @param[in] element_type_parse_position The position in the input stream, + where the `element_type` was read. + @warning Not all BSON element types are supported yet. An unsupported + @a element_type will give rise to a parse_error.114: + Unsupported BSON record type 0x... + @return whether a valid BSON-object/array was passed to the SAX parser + */ + bool parse_bson_element_internal(const int element_type, + const std::size_t element_type_parse_position) + { + switch (element_type) + { + case 0x01: // double + { + double number; + return get_number<double, true>(input_format_t::bson, number) and sax->number_float(static_cast<number_float_t>(number), ""); + } + + case 0x02: // string + { + std::int32_t len; + string_t value; + return get_number<std::int32_t, true>(input_format_t::bson, len) and get_bson_string(len, value) and sax->string(value); + } + + case 0x03: // object + { + return parse_bson_internal(); + } + + case 0x04: // array + { + return parse_bson_array(); + } + + case 0x08: // boolean + { + return sax->boolean(get() != 0); + } + + case 0x0A: // null + { + return sax->null(); + } + + case 0x10: // int32 + { + std::int32_t value; + return get_number<std::int32_t, true>(input_format_t::bson, value) and sax->number_integer(value); + } + + case 0x12: // int64 + { + std::int64_t value; + return get_number<std::int64_t, true>(input_format_t::bson, value) and sax->number_integer(value); + } + + default: // anything else not supported (yet) + { + char cr[3]; + (std::snprintf)(cr, sizeof(cr), "%.2hhX", static_cast<unsigned char>(element_type)); + return sax->parse_error(element_type_parse_position, std::string(cr), parse_error::create(114, element_type_parse_position, "Unsupported BSON record type 0x" + std::string(cr))); + } + } + } + + /*! + @brief Read a BSON element list (as specified in the BSON-spec) + + The same binary layout is used for objects and arrays, hence it must be + indicated with the argument @a is_array which one is expected + (true --> array, false --> object). + + @param[in] is_array Determines if the element list being read is to be + treated as an object (@a is_array == false), or as an + array (@a is_array == true). + @return whether a valid BSON-object/array was passed to the SAX parser + */ + bool parse_bson_element_list(const bool is_array) + { + string_t key; + while (int element_type = get()) + { + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::bson, "element list"))) + { + return false; + } + + const std::size_t element_type_parse_position = chars_read; + if (JSON_UNLIKELY(not get_bson_cstr(key))) + { + return false; + } + + if (not is_array) + { + if (not sax->key(key)) + { + return false; + } + } + + if (JSON_UNLIKELY(not parse_bson_element_internal(element_type, element_type_parse_position))) + { + return false; + } + + // get_bson_cstr only appends + key.clear(); + } + + return true; + } + + /*! + @brief Reads an array from the BSON input and passes it to the SAX-parser. + @return whether a valid BSON-array was passed to the SAX parser + */ + bool parse_bson_array() + { + std::int32_t document_size; + get_number<std::int32_t, true>(input_format_t::bson, document_size); + + if (JSON_UNLIKELY(not sax->start_array(std::size_t(-1)))) + { + return false; + } + + if (JSON_UNLIKELY(not parse_bson_element_list(/*is_array*/true))) + { + return false; + } + + return sax->end_array(); + } + + ////////// + // CBOR // + ////////// + /*! @param[in] get_char whether a new character should be retrieved from the input (true, default) or whether the last read character should be considered instead + + @return whether a valid CBOR value was passed to the SAX parser */ - BasicJsonType parse_cbor_internal(const bool get_char = true) + bool parse_cbor_internal(const bool get_char = true) { switch (get_char ? get() : current) { // EOF case std::char_traits<char>::eof(): - JSON_THROW(parse_error::create(110, chars_read, "unexpected end of input")); + return unexpect_eof(input_format_t::cbor, "value"); // Integer 0x00..0x17 (0..23) case 0x00: @@ -4585,19 +6837,31 @@ class binary_reader case 0x15: case 0x16: case 0x17: - return static_cast<number_unsigned_t>(current); + return sax->number_unsigned(static_cast<number_unsigned_t>(current)); case 0x18: // Unsigned integer (one-byte uint8_t follows) - return get_number<uint8_t>(); + { + uint8_t number; + return get_number(input_format_t::cbor, number) and sax->number_unsigned(number); + } case 0x19: // Unsigned integer (two-byte uint16_t follows) - return get_number<uint16_t>(); + { + uint16_t number; + return get_number(input_format_t::cbor, number) and sax->number_unsigned(number); + } case 0x1A: // Unsigned integer (four-byte uint32_t follows) - return get_number<uint32_t>(); + { + uint32_t number; + return get_number(input_format_t::cbor, number) and sax->number_unsigned(number); + } case 0x1B: // Unsigned integer (eight-byte uint64_t follows) - return get_number<uint64_t>(); + { + uint64_t number; + return get_number(input_format_t::cbor, number) and sax->number_unsigned(number); + } // Negative integer -1-0x00..-1-0x17 (-1..-24) case 0x20: @@ -4624,28 +6888,31 @@ class binary_reader case 0x35: case 0x36: case 0x37: - return static_cast<int8_t>(0x20 - 1 - current); + return sax->number_integer(static_cast<int8_t>(0x20 - 1 - current)); case 0x38: // Negative integer (one-byte uint8_t follows) { - // must be uint8_t ! - return static_cast<number_integer_t>(-1) - get_number<uint8_t>(); + uint8_t number; + return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast<number_integer_t>(-1) - number); } case 0x39: // Negative integer -1-n (two-byte uint16_t follows) { - return static_cast<number_integer_t>(-1) - get_number<uint16_t>(); + uint16_t number; + return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast<number_integer_t>(-1) - number); } case 0x3A: // Negative integer -1-n (four-byte uint32_t follows) { - return static_cast<number_integer_t>(-1) - get_number<uint32_t>(); + uint32_t number; + return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast<number_integer_t>(-1) - number); } case 0x3B: // Negative integer -1-n (eight-byte uint64_t follows) { - return static_cast<number_integer_t>(-1) - - static_cast<number_integer_t>(get_number<uint64_t>()); + uint64_t number; + return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast<number_integer_t>(-1) + - static_cast<number_integer_t>(number)); } // UTF-8 string (0x00..0x17 bytes follow) @@ -4679,7 +6946,8 @@ class binary_reader case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow) case 0x7F: // UTF-8 string (indefinite length) { - return get_cbor_string(); + string_t s; + return get_cbor_string(s) and sax->string(s); } // array (0x00..0x17 data items follow) @@ -4707,39 +6975,34 @@ class binary_reader case 0x95: case 0x96: case 0x97: - { - return get_cbor_array(current & 0x1F); - } + return get_cbor_array(static_cast<std::size_t>(current & 0x1F)); case 0x98: // array (one-byte uint8_t for n follows) { - return get_cbor_array(get_number<uint8_t>()); + uint8_t len; + return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast<std::size_t>(len)); } case 0x99: // array (two-byte uint16_t for n follow) { - return get_cbor_array(get_number<uint16_t>()); + uint16_t len; + return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast<std::size_t>(len)); } case 0x9A: // array (four-byte uint32_t for n follow) { - return get_cbor_array(get_number<uint32_t>()); + uint32_t len; + return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast<std::size_t>(len)); } case 0x9B: // array (eight-byte uint64_t for n follow) { - return get_cbor_array(get_number<uint64_t>()); + uint64_t len; + return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast<std::size_t>(len)); } case 0x9F: // array (indefinite length) - { - BasicJsonType result = value_t::array; - while (get() != 0xFF) - { - result.push_back(parse_cbor_internal(false)); - } - return result; - } + return get_cbor_array(std::size_t(-1)); // map (0x00..0x17 pairs of data items follow) case 0xA0: @@ -4766,62 +7029,59 @@ class binary_reader case 0xB5: case 0xB6: case 0xB7: - { - return get_cbor_object(current & 0x1F); - } + return get_cbor_object(static_cast<std::size_t>(current & 0x1F)); case 0xB8: // map (one-byte uint8_t for n follows) { - return get_cbor_object(get_number<uint8_t>()); + uint8_t len; + return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast<std::size_t>(len)); } case 0xB9: // map (two-byte uint16_t for n follow) { - return get_cbor_object(get_number<uint16_t>()); + uint16_t len; + return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast<std::size_t>(len)); } case 0xBA: // map (four-byte uint32_t for n follow) { - return get_cbor_object(get_number<uint32_t>()); + uint32_t len; + return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast<std::size_t>(len)); } case 0xBB: // map (eight-byte uint64_t for n follow) { - return get_cbor_object(get_number<uint64_t>()); + uint64_t len; + return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast<std::size_t>(len)); } case 0xBF: // map (indefinite length) - { - BasicJsonType result = value_t::object; - while (get() != 0xFF) - { - auto key = get_cbor_string(); - result[key] = parse_cbor_internal(); - } - return result; - } + return get_cbor_object(std::size_t(-1)); case 0xF4: // false - { - return false; - } + return sax->boolean(false); case 0xF5: // true - { - return true; - } + return sax->boolean(true); case 0xF6: // null - { - return value_t::null; - } + return sax->null(); case 0xF9: // Half-Precision Float (two-byte IEEE 754) { - const int byte1 = get(); - check_eof(); - const int byte2 = get(); - check_eof(); + const int byte1_raw = get(); + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::cbor, "number"))) + { + return false; + } + const int byte2_raw = get(); + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::cbor, "number"))) + { + return false; + } + + const auto byte1 = static_cast<unsigned char>(byte1_raw); + const auto byte2 = static_cast<unsigned char>(byte2_raw); // code from RFC 7049, Appendix D, Figure 3: // As half-precision floating-point numbers were only added @@ -4832,51 +7092,244 @@ class binary_reader // half-precision floating-point numbers in the C language // is shown in Fig. 3. const int half = (byte1 << 8) + byte2; - const int exp = (half >> 10) & 0x1F; - const int mant = half & 0x3FF; - double val; - if (exp == 0) - { - val = std::ldexp(mant, -24); - } - else if (exp != 31) + const double val = [&half] { - val = std::ldexp(mant + 1024, exp - 25); - } - else - { - val = (mant == 0) ? std::numeric_limits<double>::infinity() - : std::numeric_limits<double>::quiet_NaN(); - } - return (half & 0x8000) != 0 ? -val : val; + const int exp = (half >> 10) & 0x1F; + const int mant = half & 0x3FF; + assert(0 <= exp and exp <= 32); + assert(0 <= mant and mant <= 1024); + switch (exp) + { + case 0: + return std::ldexp(mant, -24); + case 31: + return (mant == 0) + ? std::numeric_limits<double>::infinity() + : std::numeric_limits<double>::quiet_NaN(); + default: + return std::ldexp(mant + 1024, exp - 25); + } + }(); + return sax->number_float((half & 0x8000) != 0 + ? static_cast<number_float_t>(-val) + : static_cast<number_float_t>(val), ""); } case 0xFA: // Single-Precision Float (four-byte IEEE 754) { - return get_number<float>(); + float number; + return get_number(input_format_t::cbor, number) and sax->number_float(static_cast<number_float_t>(number), ""); } case 0xFB: // Double-Precision Float (eight-byte IEEE 754) { - return get_number<double>(); + double number; + return get_number(input_format_t::cbor, number) and sax->number_float(static_cast<number_float_t>(number), ""); } default: // anything else (0xFF is handled inside the other types) { - std::stringstream ss; - ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current; - JSON_THROW(parse_error::create(112, chars_read, "error reading CBOR; last byte: 0x" + ss.str())); + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::cbor, "invalid byte: 0x" + last_token, "value"))); } } } - BasicJsonType parse_msgpack_internal() + /*! + @brief reads a CBOR string + + This function first reads starting bytes to determine the expected + string length and then copies this number of bytes into a string. + Additionally, CBOR's strings with indefinite lengths are supported. + + @param[out] result created string + + @return whether string creation completed + */ + bool get_cbor_string(string_t& result) + { + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::cbor, "string"))) + { + return false; + } + + switch (current) + { + // UTF-8 string (0x00..0x17 bytes follow) + case 0x60: + case 0x61: + case 0x62: + case 0x63: + case 0x64: + case 0x65: + case 0x66: + case 0x67: + case 0x68: + case 0x69: + case 0x6A: + case 0x6B: + case 0x6C: + case 0x6D: + case 0x6E: + case 0x6F: + case 0x70: + case 0x71: + case 0x72: + case 0x73: + case 0x74: + case 0x75: + case 0x76: + case 0x77: + { + return get_string(input_format_t::cbor, current & 0x1F, result); + } + + case 0x78: // UTF-8 string (one-byte uint8_t for n follows) + { + uint8_t len; + return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result); + } + + case 0x79: // UTF-8 string (two-byte uint16_t for n follow) + { + uint16_t len; + return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result); + } + + case 0x7A: // UTF-8 string (four-byte uint32_t for n follow) + { + uint32_t len; + return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result); + } + + case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow) + { + uint64_t len; + return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result); + } + + case 0x7F: // UTF-8 string (indefinite length) + { + while (get() != 0xFF) + { + string_t chunk; + if (not get_cbor_string(chunk)) + { + return false; + } + result.append(chunk); + } + return true; + } + + default: + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::cbor, "expected length specification (0x60-0x7B) or indefinite string type (0x7F); last byte: 0x" + last_token, "string"))); + } + } + } + + /*! + @param[in] len the length of the array or std::size_t(-1) for an + array of indefinite size + @return whether array creation completed + */ + bool get_cbor_array(const std::size_t len) + { + if (JSON_UNLIKELY(not sax->start_array(len))) + { + return false; + } + + if (len != std::size_t(-1)) + { + for (std::size_t i = 0; i < len; ++i) + { + if (JSON_UNLIKELY(not parse_cbor_internal())) + { + return false; + } + } + } + else + { + while (get() != 0xFF) + { + if (JSON_UNLIKELY(not parse_cbor_internal(false))) + { + return false; + } + } + } + + return sax->end_array(); + } + + /*! + @param[in] len the length of the object or std::size_t(-1) for an + object of indefinite size + @return whether object creation completed + */ + bool get_cbor_object(const std::size_t len) + { + if (not JSON_UNLIKELY(sax->start_object(len))) + { + return false; + } + + string_t key; + if (len != std::size_t(-1)) + { + for (std::size_t i = 0; i < len; ++i) + { + get(); + if (JSON_UNLIKELY(not get_cbor_string(key) or not sax->key(key))) + { + return false; + } + + if (JSON_UNLIKELY(not parse_cbor_internal())) + { + return false; + } + key.clear(); + } + } + else + { + while (get() != 0xFF) + { + if (JSON_UNLIKELY(not get_cbor_string(key) or not sax->key(key))) + { + return false; + } + + if (JSON_UNLIKELY(not parse_cbor_internal())) + { + return false; + } + key.clear(); + } + } + + return sax->end_object(); + } + + ///////////// + // MsgPack // + ///////////// + + /*! + @return whether a valid MessagePack value was passed to the SAX parser + */ + bool parse_msgpack_internal() { switch (get()) { // EOF case std::char_traits<char>::eof(): - JSON_THROW(parse_error::create(110, chars_read, "unexpected end of input")); + return unexpect_eof(input_format_t::msgpack, "value"); // positive fixint case 0x00: @@ -5007,7 +7460,7 @@ class binary_reader case 0x7D: case 0x7E: case 0x7F: - return static_cast<number_unsigned_t>(current); + return sax->number_unsigned(static_cast<number_unsigned_t>(current)); // fixmap case 0x80: @@ -5026,9 +7479,7 @@ class binary_reader case 0x8D: case 0x8E: case 0x8F: - { - return get_msgpack_object(current & 0x0F); - } + return get_msgpack_object(static_cast<std::size_t>(current & 0x0F)); // fixarray case 0x90: @@ -5047,9 +7498,7 @@ class binary_reader case 0x9D: case 0x9E: case 0x9F: - { - return get_msgpack_array(current & 0x0F); - } + return get_msgpack_array(static_cast<std::size_t>(current & 0x0F)); // fixstr case 0xA0: @@ -5084,73 +7533,113 @@ class binary_reader case 0xBD: case 0xBE: case 0xBF: - return get_msgpack_string(); + { + string_t s; + return get_msgpack_string(s) and sax->string(s); + } case 0xC0: // nil - return value_t::null; + return sax->null(); case 0xC2: // false - return false; + return sax->boolean(false); case 0xC3: // true - return true; + return sax->boolean(true); case 0xCA: // float 32 - return get_number<float>(); + { + float number; + return get_number(input_format_t::msgpack, number) and sax->number_float(static_cast<number_float_t>(number), ""); + } case 0xCB: // float 64 - return get_number<double>(); + { + double number; + return get_number(input_format_t::msgpack, number) and sax->number_float(static_cast<number_float_t>(number), ""); + } case 0xCC: // uint 8 - return get_number<uint8_t>(); + { + uint8_t number; + return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number); + } case 0xCD: // uint 16 - return get_number<uint16_t>(); + { + uint16_t number; + return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number); + } case 0xCE: // uint 32 - return get_number<uint32_t>(); + { + uint32_t number; + return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number); + } case 0xCF: // uint 64 - return get_number<uint64_t>(); + { + uint64_t number; + return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number); + } case 0xD0: // int 8 - return get_number<int8_t>(); + { + int8_t number; + return get_number(input_format_t::msgpack, number) and sax->number_integer(number); + } case 0xD1: // int 16 - return get_number<int16_t>(); + { + int16_t number; + return get_number(input_format_t::msgpack, number) and sax->number_integer(number); + } case 0xD2: // int 32 - return get_number<int32_t>(); + { + int32_t number; + return get_number(input_format_t::msgpack, number) and sax->number_integer(number); + } case 0xD3: // int 64 - return get_number<int64_t>(); + { + int64_t number; + return get_number(input_format_t::msgpack, number) and sax->number_integer(number); + } case 0xD9: // str 8 case 0xDA: // str 16 case 0xDB: // str 32 - return get_msgpack_string(); + { + string_t s; + return get_msgpack_string(s) and sax->string(s); + } case 0xDC: // array 16 { - return get_msgpack_array(get_number<uint16_t>()); + uint16_t len; + return get_number(input_format_t::msgpack, len) and get_msgpack_array(static_cast<std::size_t>(len)); } case 0xDD: // array 32 { - return get_msgpack_array(get_number<uint32_t>()); + uint32_t len; + return get_number(input_format_t::msgpack, len) and get_msgpack_array(static_cast<std::size_t>(len)); } case 0xDE: // map 16 { - return get_msgpack_object(get_number<uint16_t>()); + uint16_t len; + return get_number(input_format_t::msgpack, len) and get_msgpack_object(static_cast<std::size_t>(len)); } case 0xDF: // map 32 { - return get_msgpack_object(get_number<uint32_t>()); + uint32_t len; + return get_number(input_format_t::msgpack, len) and get_msgpack_object(static_cast<std::size_t>(len)); } - // positive fixint + // negative fixint case 0xE0: case 0xE1: case 0xE2: @@ -5183,338 +7672,749 @@ class binary_reader case 0xFD: case 0xFE: case 0xFF: - return static_cast<int8_t>(current); + return sax->number_integer(static_cast<int8_t>(current)); default: // anything else { - std::stringstream ss; - ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current; - JSON_THROW(parse_error::create(112, chars_read, - "error reading MessagePack; last byte: 0x" + ss.str())); + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::msgpack, "invalid byte: 0x" + last_token, "value"))); } } } /*! - @brief get next character from the input + @brief reads a MessagePack string - This function provides the interface to the used input adapter. It does - not throw in case the input reached EOF, but returns a -'ve valued - `std::char_traits<char>::eof()` in that case. + This function first reads starting bytes to determine the expected + string length and then copies this number of bytes into a string. - @return character read from the input + @param[out] result created string + + @return whether string creation completed */ - int get() + bool get_msgpack_string(string_t& result) { - ++chars_read; - return (current = ia->get_character()); - } + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::msgpack, "string"))) + { + return false; + } - /* - @brief read a number from the input + switch (current) + { + // fixstr + case 0xA0: + case 0xA1: + case 0xA2: + case 0xA3: + case 0xA4: + case 0xA5: + case 0xA6: + case 0xA7: + case 0xA8: + case 0xA9: + case 0xAA: + case 0xAB: + case 0xAC: + case 0xAD: + case 0xAE: + case 0xAF: + case 0xB0: + case 0xB1: + case 0xB2: + case 0xB3: + case 0xB4: + case 0xB5: + case 0xB6: + case 0xB7: + case 0xB8: + case 0xB9: + case 0xBA: + case 0xBB: + case 0xBC: + case 0xBD: + case 0xBE: + case 0xBF: + { + return get_string(input_format_t::msgpack, current & 0x1F, result); + } - @tparam NumberType the type of the number + case 0xD9: // str 8 + { + uint8_t len; + return get_number(input_format_t::msgpack, len) and get_string(input_format_t::msgpack, len, result); + } - @return number of type @a NumberType + case 0xDA: // str 16 + { + uint16_t len; + return get_number(input_format_t::msgpack, len) and get_string(input_format_t::msgpack, len, result); + } - @note This function needs to respect the system's endianess, because - bytes in CBOR and MessagePack are stored in network order (big - endian) and therefore need reordering on little endian systems. + case 0xDB: // str 32 + { + uint32_t len; + return get_number(input_format_t::msgpack, len) and get_string(input_format_t::msgpack, len, result); + } + + default: + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::msgpack, "expected length specification (0xA0-0xBF, 0xD9-0xDB); last byte: 0x" + last_token, "string"))); + } + } + } - @throw parse_error.110 if input has less than `sizeof(NumberType)` bytes + /*! + @param[in] len the length of the array + @return whether array creation completed */ - template<typename NumberType> NumberType get_number() + bool get_msgpack_array(const std::size_t len) { - // step 1: read input into array with system's byte order - std::array<uint8_t, sizeof(NumberType)> vec; - for (std::size_t i = 0; i < sizeof(NumberType); ++i) + if (JSON_UNLIKELY(not sax->start_array(len))) { - get(); - check_eof(); + return false; + } - // reverse byte order prior to conversion if necessary - if (is_little_endian) - { - vec[sizeof(NumberType) - i - 1] = static_cast<uint8_t>(current); - } - else + for (std::size_t i = 0; i < len; ++i) + { + if (JSON_UNLIKELY(not parse_msgpack_internal())) { - vec[i] = static_cast<uint8_t>(current); // LCOV_EXCL_LINE + return false; } } - // step 2: convert array into number of type T and return - NumberType result; - std::memcpy(&result, vec.data(), sizeof(NumberType)); - return result; + return sax->end_array(); } /*! - @brief create a string by reading characters from the input + @param[in] len the length of the object + @return whether object creation completed + */ + bool get_msgpack_object(const std::size_t len) + { + if (JSON_UNLIKELY(not sax->start_object(len))) + { + return false; + } - @param[in] len number of bytes to read + string_t key; + for (std::size_t i = 0; i < len; ++i) + { + get(); + if (JSON_UNLIKELY(not get_msgpack_string(key) or not sax->key(key))) + { + return false; + } - @note We can not reserve @a len bytes for the result, because @a len - may be too large. Usually, @ref check_eof() detects the end of - the input before we run out of string memory. + if (JSON_UNLIKELY(not parse_msgpack_internal())) + { + return false; + } + key.clear(); + } - @return string created by reading @a len bytes + return sax->end_object(); + } - @throw parse_error.110 if input has less than @a len bytes + //////////// + // UBJSON // + //////////// + + /*! + @param[in] get_char whether a new character should be retrieved from the + input (true, default) or whether the last read + character should be considered instead + + @return whether a valid UBJSON value was passed to the SAX parser */ - template<typename NumberType> - std::string get_string(const NumberType len) + bool parse_ubjson_internal(const bool get_char = true) { - std::string result; - std::generate_n(std::back_inserter(result), len, [this]() - { - get(); - check_eof(); - return static_cast<char>(current); - }); - return result; + return get_ubjson_value(get_char ? get_ignore_noop() : current); } /*! - @brief reads a CBOR string + @brief reads a UBJSON string - This function first reads starting bytes to determine the expected - string length and then copies this number of bytes into a string. - Additionally, CBOR's strings with indefinite lengths are supported. + This function is either called after reading the 'S' byte explicitly + indicating a string, or in case of an object key where the 'S' byte can be + left out. - @return string + @param[out] result created string + @param[in] get_char whether a new character should be retrieved from the + input (true, default) or whether the last read + character should be considered instead - @throw parse_error.110 if input ended - @throw parse_error.113 if an unexpected byte is read + @return whether string creation completed */ - std::string get_cbor_string() + bool get_ubjson_string(string_t& result, const bool get_char = true) { - check_eof(); + if (get_char) + { + get(); // TODO: may we ignore N here? + } + + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "value"))) + { + return false; + } switch (current) { - // UTF-8 string (0x00..0x17 bytes follow) - case 0x60: - case 0x61: - case 0x62: - case 0x63: - case 0x64: - case 0x65: - case 0x66: - case 0x67: - case 0x68: - case 0x69: - case 0x6A: - case 0x6B: - case 0x6C: - case 0x6D: - case 0x6E: - case 0x6F: - case 0x70: - case 0x71: - case 0x72: - case 0x73: - case 0x74: - case 0x75: - case 0x76: - case 0x77: + case 'U': { - return get_string(current & 0x1F); + uint8_t len; + return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); } - case 0x78: // UTF-8 string (one-byte uint8_t for n follows) + case 'i': { - return get_string(get_number<uint8_t>()); + int8_t len; + return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); } - case 0x79: // UTF-8 string (two-byte uint16_t for n follow) + case 'I': { - return get_string(get_number<uint16_t>()); + int16_t len; + return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); } - case 0x7A: // UTF-8 string (four-byte uint32_t for n follow) + case 'l': { - return get_string(get_number<uint32_t>()); + int32_t len; + return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); } - case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow) + case 'L': { - return get_string(get_number<uint64_t>()); + int64_t len; + return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); } - case 0x7F: // UTF-8 string (indefinite length) + default: + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "expected length type specification (U, i, I, l, L); last byte: 0x" + last_token, "string"))); + } + } + + /*! + @param[out] result determined size + @return whether size determination completed + */ + bool get_ubjson_size_value(std::size_t& result) + { + switch (get_ignore_noop()) + { + case 'U': { - std::string result; - while (get() != 0xFF) + uint8_t number; + if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast<std::size_t>(number); + return true; + } + + case 'i': + { + int8_t number; + if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) { - check_eof(); - result.push_back(static_cast<char>(current)); + return false; } - return result; + result = static_cast<std::size_t>(number); + return true; + } + + case 'I': + { + int16_t number; + if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast<std::size_t>(number); + return true; + } + + case 'l': + { + int32_t number; + if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast<std::size_t>(number); + return true; + } + + case 'L': + { + int64_t number; + if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast<std::size_t>(number); + return true; } default: { - std::stringstream ss; - ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current; - JSON_THROW(parse_error::create(113, chars_read, "expected a CBOR string; last byte: 0x" + ss.str())); + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "expected length type specification (U, i, I, l, L) after '#'; last byte: 0x" + last_token, "size"))); } } } - template<typename NumberType> - BasicJsonType get_cbor_array(const NumberType len) + /*! + @brief determine the type and size for a container + + In the optimized UBJSON format, a type and a size can be provided to allow + for a more compact representation. + + @param[out] result pair of the size and the type + + @return whether pair creation completed + */ + bool get_ubjson_size_type(std::pair<std::size_t, int>& result) { - BasicJsonType result = value_t::array; - std::generate_n(std::back_inserter(*result.m_value.array), len, [this]() + result.first = string_t::npos; // size + result.second = 0; // type + + get_ignore_noop(); + + if (current == '$') { - return parse_cbor_internal(); - }); - return result; - } + result.second = get(); // must not ignore 'N', because 'N' maybe the type + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "type"))) + { + return false; + } - template<typename NumberType> - BasicJsonType get_cbor_object(const NumberType len) - { - BasicJsonType result = value_t::object; - std::generate_n(std::inserter(*result.m_value.object, - result.m_value.object->end()), - len, [this]() + get_ignore_noop(); + if (JSON_UNLIKELY(current != '#')) + { + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "value"))) + { + return false; + } + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::ubjson, "expected '#' after type information; last byte: 0x" + last_token, "size"))); + } + + return get_ubjson_size_value(result.first); + } + else if (current == '#') { - get(); - auto key = get_cbor_string(); - auto val = parse_cbor_internal(); - return std::make_pair(std::move(key), std::move(val)); - }); - return result; + return get_ubjson_size_value(result.first); + } + return true; } /*! - @brief reads a MessagePack string + @param prefix the previously read or set type prefix + @return whether value creation completed + */ + bool get_ubjson_value(const int prefix) + { + switch (prefix) + { + case std::char_traits<char>::eof(): // EOF + return unexpect_eof(input_format_t::ubjson, "value"); - This function first reads starting bytes to determine the expected - string length and then copies this number of bytes into a string. + case 'T': // true + return sax->boolean(true); + case 'F': // false + return sax->boolean(false); + + case 'Z': // null + return sax->null(); - @return string + case 'U': + { + uint8_t number; + return get_number(input_format_t::ubjson, number) and sax->number_unsigned(number); + } + + case 'i': + { + int8_t number; + return get_number(input_format_t::ubjson, number) and sax->number_integer(number); + } - @throw parse_error.110 if input ended - @throw parse_error.113 if an unexpected byte is read + case 'I': + { + int16_t number; + return get_number(input_format_t::ubjson, number) and sax->number_integer(number); + } + + case 'l': + { + int32_t number; + return get_number(input_format_t::ubjson, number) and sax->number_integer(number); + } + + case 'L': + { + int64_t number; + return get_number(input_format_t::ubjson, number) and sax->number_integer(number); + } + + case 'd': + { + float number; + return get_number(input_format_t::ubjson, number) and sax->number_float(static_cast<number_float_t>(number), ""); + } + + case 'D': + { + double number; + return get_number(input_format_t::ubjson, number) and sax->number_float(static_cast<number_float_t>(number), ""); + } + + case 'C': // char + { + get(); + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "char"))) + { + return false; + } + if (JSON_UNLIKELY(current > 127)) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "byte after 'C' must be in range 0x00..0x7F; last byte: 0x" + last_token, "char"))); + } + string_t s(1, static_cast<char>(current)); + return sax->string(s); + } + + case 'S': // string + { + string_t s; + return get_ubjson_string(s) and sax->string(s); + } + + case '[': // array + return get_ubjson_array(); + + case '{': // object + return get_ubjson_object(); + + default: // anything else + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::ubjson, "invalid byte: 0x" + last_token, "value"))); + } + } + } + + /*! + @return whether array creation completed */ - std::string get_msgpack_string() + bool get_ubjson_array() { - check_eof(); + std::pair<std::size_t, int> size_and_type; + if (JSON_UNLIKELY(not get_ubjson_size_type(size_and_type))) + { + return false; + } - switch (current) + if (size_and_type.first != string_t::npos) { - // fixstr - case 0xA0: - case 0xA1: - case 0xA2: - case 0xA3: - case 0xA4: - case 0xA5: - case 0xA6: - case 0xA7: - case 0xA8: - case 0xA9: - case 0xAA: - case 0xAB: - case 0xAC: - case 0xAD: - case 0xAE: - case 0xAF: - case 0xB0: - case 0xB1: - case 0xB2: - case 0xB3: - case 0xB4: - case 0xB5: - case 0xB6: - case 0xB7: - case 0xB8: - case 0xB9: - case 0xBA: - case 0xBB: - case 0xBC: - case 0xBD: - case 0xBE: - case 0xBF: + if (JSON_UNLIKELY(not sax->start_array(size_and_type.first))) { - return get_string(current & 0x1F); + return false; } - case 0xD9: // str 8 + if (size_and_type.second != 0) { - return get_string(get_number<uint8_t>()); + if (size_and_type.second != 'N') + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_UNLIKELY(not get_ubjson_value(size_and_type.second))) + { + return false; + } + } + } + } + else + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_UNLIKELY(not parse_ubjson_internal())) + { + return false; + } + } + } + } + else + { + if (JSON_UNLIKELY(not sax->start_array(std::size_t(-1)))) + { + return false; } - case 0xDA: // str 16 + while (current != ']') { - return get_string(get_number<uint16_t>()); + if (JSON_UNLIKELY(not parse_ubjson_internal(false))) + { + return false; + } + get_ignore_noop(); } + } - case 0xDB: // str 32 + return sax->end_array(); + } + + /*! + @return whether object creation completed + */ + bool get_ubjson_object() + { + std::pair<std::size_t, int> size_and_type; + if (JSON_UNLIKELY(not get_ubjson_size_type(size_and_type))) + { + return false; + } + + string_t key; + if (size_and_type.first != string_t::npos) + { + if (JSON_UNLIKELY(not sax->start_object(size_and_type.first))) { - return get_string(get_number<uint32_t>()); + return false; } - default: + if (size_and_type.second != 0) { - std::stringstream ss; - ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current; - JSON_THROW(parse_error::create(113, chars_read, - "expected a MessagePack string; last byte: 0x" + ss.str())); + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_UNLIKELY(not get_ubjson_string(key) or not sax->key(key))) + { + return false; + } + if (JSON_UNLIKELY(not get_ubjson_value(size_and_type.second))) + { + return false; + } + key.clear(); + } + } + else + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_UNLIKELY(not get_ubjson_string(key) or not sax->key(key))) + { + return false; + } + if (JSON_UNLIKELY(not parse_ubjson_internal())) + { + return false; + } + key.clear(); + } + } + } + else + { + if (JSON_UNLIKELY(not sax->start_object(std::size_t(-1)))) + { + return false; + } + + while (current != '}') + { + if (JSON_UNLIKELY(not get_ubjson_string(key, false) or not sax->key(key))) + { + return false; + } + if (JSON_UNLIKELY(not parse_ubjson_internal())) + { + return false; + } + get_ignore_noop(); + key.clear(); } } + + return sax->end_object(); } - template<typename NumberType> - BasicJsonType get_msgpack_array(const NumberType len) + /////////////////////// + // Utility functions // + /////////////////////// + + /*! + @brief get next character from the input + + This function provides the interface to the used input adapter. It does + not throw in case the input reached EOF, but returns a -'ve valued + `std::char_traits<char>::eof()` in that case. + + @return character read from the input + */ + int get() { - BasicJsonType result = value_t::array; - std::generate_n(std::back_inserter(*result.m_value.array), len, [this]() - { - return parse_msgpack_internal(); - }); - return result; + ++chars_read; + return (current = ia->get_character()); } - template<typename NumberType> - BasicJsonType get_msgpack_object(const NumberType len) + /*! + @return character read from the input after ignoring all 'N' entries + */ + int get_ignore_noop() { - BasicJsonType result = value_t::object; - std::generate_n(std::inserter(*result.m_value.object, - result.m_value.object->end()), - len, [this]() + do { get(); - auto key = get_msgpack_string(); - auto val = parse_msgpack_internal(); - return std::make_pair(std::move(key), std::move(val)); - }); - return result; + } + while (current == 'N'); + + return current; } - /*! - @brief check if input ended - @throw parse_error.110 if input ended + /* + @brief read a number from the input + + @tparam NumberType the type of the number + @param[in] format the current format (for diagnostics) + @param[out] result number of type @a NumberType + + @return whether conversion completed + + @note This function needs to respect the system's endianess, because + bytes in CBOR, MessagePack, and UBJSON are stored in network order + (big endian) and therefore need reordering on little endian systems. */ - void check_eof(const bool expect_eof = false) const + template<typename NumberType, bool InputIsLittleEndian = false> + bool get_number(const input_format_t format, NumberType& result) { - if (expect_eof) + // step 1: read input into array with system's byte order + std::array<uint8_t, sizeof(NumberType)> vec; + for (std::size_t i = 0; i < sizeof(NumberType); ++i) { - if (JSON_UNLIKELY(current != std::char_traits<char>::eof())) + get(); + if (JSON_UNLIKELY(not unexpect_eof(format, "number"))) { - JSON_THROW(parse_error::create(110, chars_read, "expected end of input")); + return false; + } + + // reverse byte order prior to conversion if necessary + if (is_little_endian && !InputIsLittleEndian) + { + vec[sizeof(NumberType) - i - 1] = static_cast<uint8_t>(current); + } + else + { + vec[i] = static_cast<uint8_t>(current); // LCOV_EXCL_LINE } } - else + + // step 2: convert array into number of type T and return + std::memcpy(&result, vec.data(), sizeof(NumberType)); + return true; + } + + /*! + @brief create a string by reading characters from the input + + @tparam NumberType the type of the number + @param[in] format the current format (for diagnostics) + @param[in] len number of characters to read + @param[out] result string created by reading @a len bytes + + @return whether string creation completed + + @note We can not reserve @a len bytes for the result, because @a len + may be too large. Usually, @ref unexpect_eof() detects the end of + the input before we run out of string memory. + */ + template<typename NumberType> + bool get_string(const input_format_t format, + const NumberType len, + string_t& result) + { + bool success = true; + std::generate_n(std::back_inserter(result), len, [this, &success, &format]() { - if (JSON_UNLIKELY(current == std::char_traits<char>::eof())) + get(); + if (JSON_UNLIKELY(not unexpect_eof(format, "string"))) { - JSON_THROW(parse_error::create(110, chars_read, "unexpected end of input")); + success = false; } + return static_cast<char>(current); + }); + return success; + } + + /*! + @param[in] format the current format (for diagnostics) + @param[in] context further context information (for diagnostics) + @return whether the last read character is not EOF + */ + bool unexpect_eof(const input_format_t format, const char* context) const + { + if (JSON_UNLIKELY(current == std::char_traits<char>::eof())) + { + return sax->parse_error(chars_read, "<end of file>", + parse_error::create(110, chars_read, exception_message(format, "unexpected end of input", context))); + } + return true; + } + + /*! + @return a string representation of the last read byte + */ + std::string get_token_string() const + { + char cr[3]; + (std::snprintf)(cr, 3, "%.2hhX", static_cast<unsigned char>(current)); + return std::string{cr}; + } + + /*! + @param[in] format the current format + @param[in] detail a detailed error message + @param[in] context further contect information + @return a message string to use in the parse_error exceptions + */ + std::string exception_message(const input_format_t format, + const std::string& detail, + const std::string& context) const + { + std::string error_msg = "syntax error while parsing "; + + switch (format) + { + case input_format_t::cbor: + error_msg += "CBOR"; + break; + + case input_format_t::msgpack: + error_msg += "MessagePack"; + break; + + case input_format_t::ubjson: + error_msg += "UBJSON"; + break; + + case input_format_t::bson: + error_msg += "BSON"; + break; + + // LCOV_EXCL_START + default: + assert(false); + // LCOV_EXCL_STOP } + + return error_msg + " " + context + ": " + detail; } private: @@ -5529,7 +8429,34 @@ class binary_reader /// whether we can assume little endianess const bool is_little_endian = little_endianess(); + + /// the SAX parser + json_sax_t* sax = nullptr; }; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/output/binary_writer.hpp> + + +#include <algorithm> // reverse +#include <array> // array +#include <cstdint> // uint8_t, uint16_t, uint32_t, uint64_t +#include <cstring> // memcpy +#include <limits> // numeric_limits + +// #include <nlohmann/detail/input/binary_reader.hpp> + +// #include <nlohmann/detail/output/output_adapters.hpp> + + +namespace nlohmann +{ +namespace detail +{ +/////////////////// +// binary writer // +/////////////////// /*! @brief serialization to CBOR and MessagePack values @@ -5537,6 +8464,8 @@ class binary_reader template<typename BasicJsonType, typename CharType> class binary_writer { + using string_t = typename BasicJsonType::string_t; + public: /*! @brief create a binary writer @@ -5549,7 +8478,28 @@ class binary_writer } /*! - @brief[in] j JSON value to serialize + @param[in] j JSON value to serialize + @pre j.type() == value_t::object + */ + void write_bson(const BasicJsonType& j) + { + switch (j.type()) + { + case value_t::object: + { + write_bson_object(*j.m_value.object); + break; + } + + default: + { + JSON_THROW(type_error::create(317, "to serialize to BSON, top-level type must be object, but is " + std::string(j.type_name()))); + } + } + } + + /*! + @param[in] j JSON value to serialize */ void write_cbor(const BasicJsonType& j) { @@ -5557,15 +8507,15 @@ class binary_writer { case value_t::null: { - oa->write_character(static_cast<CharType>(0xF6)); + oa->write_character(to_char_type(0xF6)); break; } case value_t::boolean: { oa->write_character(j.m_value.boolean - ? static_cast<CharType>(0xF5) - : static_cast<CharType>(0xF4)); + ? to_char_type(0xF5) + : to_char_type(0xF4)); break; } @@ -5582,22 +8532,22 @@ class binary_writer } else if (j.m_value.number_integer <= (std::numeric_limits<uint8_t>::max)()) { - oa->write_character(static_cast<CharType>(0x18)); + oa->write_character(to_char_type(0x18)); write_number(static_cast<uint8_t>(j.m_value.number_integer)); } else if (j.m_value.number_integer <= (std::numeric_limits<uint16_t>::max)()) { - oa->write_character(static_cast<CharType>(0x19)); + oa->write_character(to_char_type(0x19)); write_number(static_cast<uint16_t>(j.m_value.number_integer)); } else if (j.m_value.number_integer <= (std::numeric_limits<uint32_t>::max)()) { - oa->write_character(static_cast<CharType>(0x1A)); + oa->write_character(to_char_type(0x1A)); write_number(static_cast<uint32_t>(j.m_value.number_integer)); } else { - oa->write_character(static_cast<CharType>(0x1B)); + oa->write_character(to_char_type(0x1B)); write_number(static_cast<uint64_t>(j.m_value.number_integer)); } } @@ -5612,22 +8562,22 @@ class binary_writer } else if (positive_number <= (std::numeric_limits<uint8_t>::max)()) { - oa->write_character(static_cast<CharType>(0x38)); + oa->write_character(to_char_type(0x38)); write_number(static_cast<uint8_t>(positive_number)); } else if (positive_number <= (std::numeric_limits<uint16_t>::max)()) { - oa->write_character(static_cast<CharType>(0x39)); + oa->write_character(to_char_type(0x39)); write_number(static_cast<uint16_t>(positive_number)); } else if (positive_number <= (std::numeric_limits<uint32_t>::max)()) { - oa->write_character(static_cast<CharType>(0x3A)); + oa->write_character(to_char_type(0x3A)); write_number(static_cast<uint32_t>(positive_number)); } else { - oa->write_character(static_cast<CharType>(0x3B)); + oa->write_character(to_char_type(0x3B)); write_number(static_cast<uint64_t>(positive_number)); } } @@ -5642,30 +8592,30 @@ class binary_writer } else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)()) { - oa->write_character(static_cast<CharType>(0x18)); + oa->write_character(to_char_type(0x18)); write_number(static_cast<uint8_t>(j.m_value.number_unsigned)); } else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)()) { - oa->write_character(static_cast<CharType>(0x19)); + oa->write_character(to_char_type(0x19)); write_number(static_cast<uint16_t>(j.m_value.number_unsigned)); } else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)()) { - oa->write_character(static_cast<CharType>(0x1A)); + oa->write_character(to_char_type(0x1A)); write_number(static_cast<uint32_t>(j.m_value.number_unsigned)); } else { - oa->write_character(static_cast<CharType>(0x1B)); + oa->write_character(to_char_type(0x1B)); write_number(static_cast<uint64_t>(j.m_value.number_unsigned)); } break; } - case value_t::number_float: // Double-Precision Float + case value_t::number_float: { - oa->write_character(static_cast<CharType>(0xFB)); + oa->write_character(get_cbor_float_prefix(j.m_value.number_float)); write_number(j.m_value.number_float); break; } @@ -5678,25 +8628,25 @@ class binary_writer { write_number(static_cast<uint8_t>(0x60 + N)); } - else if (N <= 0xFF) + else if (N <= (std::numeric_limits<uint8_t>::max)()) { - oa->write_character(static_cast<CharType>(0x78)); + oa->write_character(to_char_type(0x78)); write_number(static_cast<uint8_t>(N)); } - else if (N <= 0xFFFF) + else if (N <= (std::numeric_limits<uint16_t>::max)()) { - oa->write_character(static_cast<CharType>(0x79)); + oa->write_character(to_char_type(0x79)); write_number(static_cast<uint16_t>(N)); } - else if (N <= 0xFFFFFFFF) + else if (N <= (std::numeric_limits<uint32_t>::max)()) { - oa->write_character(static_cast<CharType>(0x7A)); + oa->write_character(to_char_type(0x7A)); write_number(static_cast<uint32_t>(N)); } // LCOV_EXCL_START - else if (N <= 0xFFFFFFFFFFFFFFFF) + else if (N <= (std::numeric_limits<uint64_t>::max)()) { - oa->write_character(static_cast<CharType>(0x7B)); + oa->write_character(to_char_type(0x7B)); write_number(static_cast<uint64_t>(N)); } // LCOV_EXCL_STOP @@ -5716,25 +8666,25 @@ class binary_writer { write_number(static_cast<uint8_t>(0x80 + N)); } - else if (N <= 0xFF) + else if (N <= (std::numeric_limits<uint8_t>::max)()) { - oa->write_character(static_cast<CharType>(0x98)); + oa->write_character(to_char_type(0x98)); write_number(static_cast<uint8_t>(N)); } - else if (N <= 0xFFFF) + else if (N <= (std::numeric_limits<uint16_t>::max)()) { - oa->write_character(static_cast<CharType>(0x99)); + oa->write_character(to_char_type(0x99)); write_number(static_cast<uint16_t>(N)); } - else if (N <= 0xFFFFFFFF) + else if (N <= (std::numeric_limits<uint32_t>::max)()) { - oa->write_character(static_cast<CharType>(0x9A)); + oa->write_character(to_char_type(0x9A)); write_number(static_cast<uint32_t>(N)); } // LCOV_EXCL_START - else if (N <= 0xFFFFFFFFFFFFFFFF) + else if (N <= (std::numeric_limits<uint64_t>::max)()) { - oa->write_character(static_cast<CharType>(0x9B)); + oa->write_character(to_char_type(0x9B)); write_number(static_cast<uint64_t>(N)); } // LCOV_EXCL_STOP @@ -5755,25 +8705,25 @@ class binary_writer { write_number(static_cast<uint8_t>(0xA0 + N)); } - else if (N <= 0xFF) + else if (N <= (std::numeric_limits<uint8_t>::max)()) { - oa->write_character(static_cast<CharType>(0xB8)); + oa->write_character(to_char_type(0xB8)); write_number(static_cast<uint8_t>(N)); } - else if (N <= 0xFFFF) + else if (N <= (std::numeric_limits<uint16_t>::max)()) { - oa->write_character(static_cast<CharType>(0xB9)); + oa->write_character(to_char_type(0xB9)); write_number(static_cast<uint16_t>(N)); } - else if (N <= 0xFFFFFFFF) + else if (N <= (std::numeric_limits<uint32_t>::max)()) { - oa->write_character(static_cast<CharType>(0xBA)); + oa->write_character(to_char_type(0xBA)); write_number(static_cast<uint32_t>(N)); } // LCOV_EXCL_START - else if (N <= 0xFFFFFFFFFFFFFFFF) + else if (N <= (std::numeric_limits<uint64_t>::max)()) { - oa->write_character(static_cast<CharType>(0xBB)); + oa->write_character(to_char_type(0xBB)); write_number(static_cast<uint64_t>(N)); } // LCOV_EXCL_STOP @@ -5793,7 +8743,7 @@ class binary_writer } /*! - @brief[in] j JSON value to serialize + @param[in] j JSON value to serialize */ void write_msgpack(const BasicJsonType& j) { @@ -5801,15 +8751,15 @@ class binary_writer { case value_t::null: // nil { - oa->write_character(static_cast<CharType>(0xC0)); + oa->write_character(to_char_type(0xC0)); break; } case value_t::boolean: // true and false { oa->write_character(j.m_value.boolean - ? static_cast<CharType>(0xC3) - : static_cast<CharType>(0xC2)); + ? to_char_type(0xC3) + : to_char_type(0xC2)); break; } @@ -5828,25 +8778,25 @@ class binary_writer else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)()) { // uint 8 - oa->write_character(static_cast<CharType>(0xCC)); + oa->write_character(to_char_type(0xCC)); write_number(static_cast<uint8_t>(j.m_value.number_integer)); } else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)()) { // uint 16 - oa->write_character(static_cast<CharType>(0xCD)); + oa->write_character(to_char_type(0xCD)); write_number(static_cast<uint16_t>(j.m_value.number_integer)); } else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)()) { // uint 32 - oa->write_character(static_cast<CharType>(0xCE)); + oa->write_character(to_char_type(0xCE)); write_number(static_cast<uint32_t>(j.m_value.number_integer)); } else if (j.m_value.number_unsigned <= (std::numeric_limits<uint64_t>::max)()) { // uint 64 - oa->write_character(static_cast<CharType>(0xCF)); + oa->write_character(to_char_type(0xCF)); write_number(static_cast<uint64_t>(j.m_value.number_integer)); } } @@ -5861,28 +8811,28 @@ class binary_writer j.m_value.number_integer <= (std::numeric_limits<int8_t>::max)()) { // int 8 - oa->write_character(static_cast<CharType>(0xD0)); + oa->write_character(to_char_type(0xD0)); write_number(static_cast<int8_t>(j.m_value.number_integer)); } else if (j.m_value.number_integer >= (std::numeric_limits<int16_t>::min)() and j.m_value.number_integer <= (std::numeric_limits<int16_t>::max)()) { // int 16 - oa->write_character(static_cast<CharType>(0xD1)); + oa->write_character(to_char_type(0xD1)); write_number(static_cast<int16_t>(j.m_value.number_integer)); } else if (j.m_value.number_integer >= (std::numeric_limits<int32_t>::min)() and j.m_value.number_integer <= (std::numeric_limits<int32_t>::max)()) { // int 32 - oa->write_character(static_cast<CharType>(0xD2)); + oa->write_character(to_char_type(0xD2)); write_number(static_cast<int32_t>(j.m_value.number_integer)); } else if (j.m_value.number_integer >= (std::numeric_limits<int64_t>::min)() and j.m_value.number_integer <= (std::numeric_limits<int64_t>::max)()) { // int 64 - oa->write_character(static_cast<CharType>(0xD3)); + oa->write_character(to_char_type(0xD3)); write_number(static_cast<int64_t>(j.m_value.number_integer)); } } @@ -5899,33 +8849,33 @@ class binary_writer else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)()) { // uint 8 - oa->write_character(static_cast<CharType>(0xCC)); + oa->write_character(to_char_type(0xCC)); write_number(static_cast<uint8_t>(j.m_value.number_integer)); } else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)()) { // uint 16 - oa->write_character(static_cast<CharType>(0xCD)); + oa->write_character(to_char_type(0xCD)); write_number(static_cast<uint16_t>(j.m_value.number_integer)); } else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)()) { // uint 32 - oa->write_character(static_cast<CharType>(0xCE)); + oa->write_character(to_char_type(0xCE)); write_number(static_cast<uint32_t>(j.m_value.number_integer)); } else if (j.m_value.number_unsigned <= (std::numeric_limits<uint64_t>::max)()) { // uint 64 - oa->write_character(static_cast<CharType>(0xCF)); + oa->write_character(to_char_type(0xCF)); write_number(static_cast<uint64_t>(j.m_value.number_integer)); } break; } - case value_t::number_float: // float 64 + case value_t::number_float: { - oa->write_character(static_cast<CharType>(0xCB)); + oa->write_character(get_msgpack_float_prefix(j.m_value.number_float)); write_number(j.m_value.number_float); break; } @@ -5939,22 +8889,22 @@ class binary_writer // fixstr write_number(static_cast<uint8_t>(0xA0 | N)); } - else if (N <= 255) + else if (N <= (std::numeric_limits<uint8_t>::max)()) { // str 8 - oa->write_character(static_cast<CharType>(0xD9)); + oa->write_character(to_char_type(0xD9)); write_number(static_cast<uint8_t>(N)); } - else if (N <= 65535) + else if (N <= (std::numeric_limits<uint16_t>::max)()) { // str 16 - oa->write_character(static_cast<CharType>(0xDA)); + oa->write_character(to_char_type(0xDA)); write_number(static_cast<uint16_t>(N)); } - else if (N <= 4294967295) + else if (N <= (std::numeric_limits<uint32_t>::max)()) { // str 32 - oa->write_character(static_cast<CharType>(0xDB)); + oa->write_character(to_char_type(0xDB)); write_number(static_cast<uint32_t>(N)); } @@ -5974,16 +8924,16 @@ class binary_writer // fixarray write_number(static_cast<uint8_t>(0x90 | N)); } - else if (N <= 0xFFFF) + else if (N <= (std::numeric_limits<uint16_t>::max)()) { // array 16 - oa->write_character(static_cast<CharType>(0xDC)); + oa->write_character(to_char_type(0xDC)); write_number(static_cast<uint16_t>(N)); } - else if (N <= 0xFFFFFFFF) + else if (N <= (std::numeric_limits<uint32_t>::max)()) { // array 32 - oa->write_character(static_cast<CharType>(0xDD)); + oa->write_character(to_char_type(0xDD)); write_number(static_cast<uint32_t>(N)); } @@ -6004,16 +8954,16 @@ class binary_writer // fixmap write_number(static_cast<uint8_t>(0x80 | (N & 0xF))); } - else if (N <= 65535) + else if (N <= (std::numeric_limits<uint16_t>::max)()) { // map 16 - oa->write_character(static_cast<CharType>(0xDE)); + oa->write_character(to_char_type(0xDE)); write_number(static_cast<uint16_t>(N)); } - else if (N <= 4294967295) + else if (N <= (std::numeric_limits<uint32_t>::max)()) { // map 32 - oa->write_character(static_cast<CharType>(0xDF)); + oa->write_character(to_char_type(0xDF)); write_number(static_cast<uint32_t>(N)); } @@ -6031,25 +8981,745 @@ class binary_writer } } + /*! + @param[in] j JSON value to serialize + @param[in] use_count whether to use '#' prefixes (optimized format) + @param[in] use_type whether to use '$' prefixes (optimized format) + @param[in] add_prefix whether prefixes need to be used for this value + */ + void write_ubjson(const BasicJsonType& j, const bool use_count, + const bool use_type, const bool add_prefix = true) + { + switch (j.type()) + { + case value_t::null: + { + if (add_prefix) + { + oa->write_character(to_char_type('Z')); + } + break; + } + + case value_t::boolean: + { + if (add_prefix) + { + oa->write_character(j.m_value.boolean + ? to_char_type('T') + : to_char_type('F')); + } + break; + } + + case value_t::number_integer: + { + write_number_with_ubjson_prefix(j.m_value.number_integer, add_prefix); + break; + } + + case value_t::number_unsigned: + { + write_number_with_ubjson_prefix(j.m_value.number_unsigned, add_prefix); + break; + } + + case value_t::number_float: + { + write_number_with_ubjson_prefix(j.m_value.number_float, add_prefix); + break; + } + + case value_t::string: + { + if (add_prefix) + { + oa->write_character(to_char_type('S')); + } + write_number_with_ubjson_prefix(j.m_value.string->size(), true); + oa->write_characters( + reinterpret_cast<const CharType*>(j.m_value.string->c_str()), + j.m_value.string->size()); + break; + } + + case value_t::array: + { + if (add_prefix) + { + oa->write_character(to_char_type('[')); + } + + bool prefix_required = true; + if (use_type and not j.m_value.array->empty()) + { + assert(use_count); + const CharType first_prefix = ubjson_prefix(j.front()); + const bool same_prefix = std::all_of(j.begin() + 1, j.end(), + [this, first_prefix](const BasicJsonType & v) + { + return ubjson_prefix(v) == first_prefix; + }); + + if (same_prefix) + { + prefix_required = false; + oa->write_character(to_char_type('$')); + oa->write_character(first_prefix); + } + } + + if (use_count) + { + oa->write_character(to_char_type('#')); + write_number_with_ubjson_prefix(j.m_value.array->size(), true); + } + + for (const auto& el : *j.m_value.array) + { + write_ubjson(el, use_count, use_type, prefix_required); + } + + if (not use_count) + { + oa->write_character(to_char_type(']')); + } + + break; + } + + case value_t::object: + { + if (add_prefix) + { + oa->write_character(to_char_type('{')); + } + + bool prefix_required = true; + if (use_type and not j.m_value.object->empty()) + { + assert(use_count); + const CharType first_prefix = ubjson_prefix(j.front()); + const bool same_prefix = std::all_of(j.begin(), j.end(), + [this, first_prefix](const BasicJsonType & v) + { + return ubjson_prefix(v) == first_prefix; + }); + + if (same_prefix) + { + prefix_required = false; + oa->write_character(to_char_type('$')); + oa->write_character(first_prefix); + } + } + + if (use_count) + { + oa->write_character(to_char_type('#')); + write_number_with_ubjson_prefix(j.m_value.object->size(), true); + } + + for (const auto& el : *j.m_value.object) + { + write_number_with_ubjson_prefix(el.first.size(), true); + oa->write_characters( + reinterpret_cast<const CharType*>(el.first.c_str()), + el.first.size()); + write_ubjson(el.second, use_count, use_type, prefix_required); + } + + if (not use_count) + { + oa->write_character(to_char_type('}')); + } + + break; + } + + default: + break; + } + } + private: + ////////// + // BSON // + ////////// + + /*! + @return The size of a BSON document entry header, including the id marker + and the entry name size (and its null-terminator). + */ + static std::size_t calc_bson_entry_header_size(const string_t& name) + { + const auto it = name.find(static_cast<typename string_t::value_type>(0)); + if (JSON_UNLIKELY(it != BasicJsonType::string_t::npos)) + { + JSON_THROW(out_of_range::create(409, + "BSON key cannot contain code point U+0000 (at byte " + std::to_string(it) + ")")); + } + + return /*id*/ 1ul + name.size() + /*zero-terminator*/1u; + } + + /*! + @brief Writes the given @a element_type and @a name to the output adapter + */ + void write_bson_entry_header(const string_t& name, + const std::uint8_t element_type) + { + oa->write_character(to_char_type(element_type)); // boolean + oa->write_characters( + reinterpret_cast<const CharType*>(name.c_str()), + name.size() + 1u); + } + + /*! + @brief Writes a BSON element with key @a name and boolean value @a value + */ + void write_bson_boolean(const string_t& name, + const bool value) + { + write_bson_entry_header(name, 0x08); + oa->write_character(value ? to_char_type(0x01) : to_char_type(0x00)); + } + + /*! + @brief Writes a BSON element with key @a name and double value @a value + */ + void write_bson_double(const string_t& name, + const double value) + { + write_bson_entry_header(name, 0x01); + write_number<double, true>(value); + } + + /*! + @return The size of the BSON-encoded string in @a value + */ + static std::size_t calc_bson_string_size(const string_t& value) + { + return sizeof(std::int32_t) + value.size() + 1ul; + } + + /*! + @brief Writes a BSON element with key @a name and string value @a value + */ + void write_bson_string(const string_t& name, + const string_t& value) + { + write_bson_entry_header(name, 0x02); + + write_number<std::int32_t, true>(static_cast<std::int32_t>(value.size() + 1ul)); + oa->write_characters( + reinterpret_cast<const CharType*>(value.c_str()), + value.size() + 1); + } + + /*! + @brief Writes a BSON element with key @a name and null value + */ + void write_bson_null(const string_t& name) + { + write_bson_entry_header(name, 0x0A); + } + + /*! + @return The size of the BSON-encoded integer @a value + */ + static std::size_t calc_bson_integer_size(const std::int64_t value) + { + if ((std::numeric_limits<std::int32_t>::min)() <= value and value <= (std::numeric_limits<std::int32_t>::max)()) + { + return sizeof(std::int32_t); + } + else + { + return sizeof(std::int64_t); + } + } + + /*! + @brief Writes a BSON element with key @a name and integer @a value + */ + void write_bson_integer(const string_t& name, + const std::int64_t value) + { + if ((std::numeric_limits<std::int32_t>::min)() <= value and value <= (std::numeric_limits<std::int32_t>::max)()) + { + write_bson_entry_header(name, 0x10); // int32 + write_number<std::int32_t, true>(static_cast<std::int32_t>(value)); + } + else + { + write_bson_entry_header(name, 0x12); // int64 + write_number<std::int64_t, true>(static_cast<std::int64_t>(value)); + } + } + + /*! + @return The size of the BSON-encoded unsigned integer in @a j + */ + static constexpr std::size_t calc_bson_unsigned_size(const std::uint64_t value) noexcept + { + return (value <= static_cast<std::uint64_t>((std::numeric_limits<std::int32_t>::max)())) + ? sizeof(std::int32_t) + : sizeof(std::int64_t); + } + + /*! + @brief Writes a BSON element with key @a name and unsigned @a value + */ + void write_bson_unsigned(const string_t& name, + const std::uint64_t value) + { + if (value <= static_cast<std::uint64_t>((std::numeric_limits<std::int32_t>::max)())) + { + write_bson_entry_header(name, 0x10 /* int32 */); + write_number<std::int32_t, true>(static_cast<std::int32_t>(value)); + } + else if (value <= static_cast<std::uint64_t>((std::numeric_limits<std::int64_t>::max)())) + { + write_bson_entry_header(name, 0x12 /* int64 */); + write_number<std::int64_t, true>(static_cast<std::int64_t>(value)); + } + else + { + JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(value) + " cannot be represented by BSON as it does not fit int64")); + } + } + + /*! + @brief Writes a BSON element with key @a name and object @a value + */ + void write_bson_object_entry(const string_t& name, + const typename BasicJsonType::object_t& value) + { + write_bson_entry_header(name, 0x03); // object + write_bson_object(value); + } + + /*! + @return The size of the BSON-encoded array @a value + */ + static std::size_t calc_bson_array_size(const typename BasicJsonType::array_t& value) + { + std::size_t embedded_document_size = 0ul; + std::size_t array_index = 0ul; + + for (const auto& el : value) + { + embedded_document_size += calc_bson_element_size(std::to_string(array_index++), el); + } + + return sizeof(std::int32_t) + embedded_document_size + 1ul; + } + + /*! + @brief Writes a BSON element with key @a name and array @a value + */ + void write_bson_array(const string_t& name, + const typename BasicJsonType::array_t& value) + { + write_bson_entry_header(name, 0x04); // array + write_number<std::int32_t, true>(static_cast<std::int32_t>(calc_bson_array_size(value))); + + std::size_t array_index = 0ul; + + for (const auto& el : value) + { + write_bson_element(std::to_string(array_index++), el); + } + + oa->write_character(to_char_type(0x00)); + } + + /*! + @brief Calculates the size necessary to serialize the JSON value @a j with its @a name + @return The calculated size for the BSON document entry for @a j with the given @a name. + */ + static std::size_t calc_bson_element_size(const string_t& name, + const BasicJsonType& j) + { + const auto header_size = calc_bson_entry_header_size(name); + switch (j.type()) + { + case value_t::object: + return header_size + calc_bson_object_size(*j.m_value.object); + + case value_t::array: + return header_size + calc_bson_array_size(*j.m_value.array); + + case value_t::boolean: + return header_size + 1ul; + + case value_t::number_float: + return header_size + 8ul; + + case value_t::number_integer: + return header_size + calc_bson_integer_size(j.m_value.number_integer); + + case value_t::number_unsigned: + return header_size + calc_bson_unsigned_size(j.m_value.number_unsigned); + + case value_t::string: + return header_size + calc_bson_string_size(*j.m_value.string); + + case value_t::null: + return header_size + 0ul; + + // LCOV_EXCL_START + default: + assert(false); + return 0ul; + // LCOV_EXCL_STOP + }; + } + + /*! + @brief Serializes the JSON value @a j to BSON and associates it with the + key @a name. + @param name The name to associate with the JSON entity @a j within the + current BSON document + @return The size of the BSON entry + */ + void write_bson_element(const string_t& name, + const BasicJsonType& j) + { + switch (j.type()) + { + case value_t::object: + return write_bson_object_entry(name, *j.m_value.object); + + case value_t::array: + return write_bson_array(name, *j.m_value.array); + + case value_t::boolean: + return write_bson_boolean(name, j.m_value.boolean); + + case value_t::number_float: + return write_bson_double(name, j.m_value.number_float); + + case value_t::number_integer: + return write_bson_integer(name, j.m_value.number_integer); + + case value_t::number_unsigned: + return write_bson_unsigned(name, j.m_value.number_unsigned); + + case value_t::string: + return write_bson_string(name, *j.m_value.string); + + case value_t::null: + return write_bson_null(name); + + // LCOV_EXCL_START + default: + assert(false); + return; + // LCOV_EXCL_STOP + }; + } + + /*! + @brief Calculates the size of the BSON serialization of the given + JSON-object @a j. + @param[in] j JSON value to serialize + @pre j.type() == value_t::object + */ + static std::size_t calc_bson_object_size(const typename BasicJsonType::object_t& value) + { + std::size_t document_size = std::accumulate(value.begin(), value.end(), 0ul, + [](size_t result, const typename BasicJsonType::object_t::value_type & el) + { + return result += calc_bson_element_size(el.first, el.second); + }); + + return sizeof(std::int32_t) + document_size + 1ul; + } + + /*! + @param[in] j JSON value to serialize + @pre j.type() == value_t::object + */ + void write_bson_object(const typename BasicJsonType::object_t& value) + { + write_number<std::int32_t, true>(static_cast<std::int32_t>(calc_bson_object_size(value))); + + for (const auto& el : value) + { + write_bson_element(el.first, el.second); + } + + oa->write_character(to_char_type(0x00)); + } + + ////////// + // CBOR // + ////////// + + static constexpr CharType get_cbor_float_prefix(float /*unused*/) + { + return to_char_type(0xFA); // Single-Precision Float + } + + static constexpr CharType get_cbor_float_prefix(double /*unused*/) + { + return to_char_type(0xFB); // Double-Precision Float + } + + ///////////// + // MsgPack // + ///////////// + + static constexpr CharType get_msgpack_float_prefix(float /*unused*/) + { + return to_char_type(0xCA); // float 32 + } + + static constexpr CharType get_msgpack_float_prefix(double /*unused*/) + { + return to_char_type(0xCB); // float 64 + } + + //////////// + // UBJSON // + //////////// + + // UBJSON: write number (floating point) + template<typename NumberType, typename std::enable_if< + std::is_floating_point<NumberType>::value, int>::type = 0> + void write_number_with_ubjson_prefix(const NumberType n, + const bool add_prefix) + { + if (add_prefix) + { + oa->write_character(get_ubjson_float_prefix(n)); + } + write_number(n); + } + + // UBJSON: write number (unsigned integer) + template<typename NumberType, typename std::enable_if< + std::is_unsigned<NumberType>::value, int>::type = 0> + void write_number_with_ubjson_prefix(const NumberType n, + const bool add_prefix) + { + if (n <= static_cast<uint64_t>((std::numeric_limits<int8_t>::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('i')); // int8 + } + write_number(static_cast<uint8_t>(n)); + } + else if (n <= (std::numeric_limits<uint8_t>::max)()) + { + if (add_prefix) + { + oa->write_character(to_char_type('U')); // uint8 + } + write_number(static_cast<uint8_t>(n)); + } + else if (n <= static_cast<uint64_t>((std::numeric_limits<int16_t>::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('I')); // int16 + } + write_number(static_cast<int16_t>(n)); + } + else if (n <= static_cast<uint64_t>((std::numeric_limits<int32_t>::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('l')); // int32 + } + write_number(static_cast<int32_t>(n)); + } + else if (n <= static_cast<uint64_t>((std::numeric_limits<int64_t>::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('L')); // int64 + } + write_number(static_cast<int64_t>(n)); + } + else + { + JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(n) + " cannot be represented by UBJSON as it does not fit int64")); + } + } + + // UBJSON: write number (signed integer) + template<typename NumberType, typename std::enable_if< + std::is_signed<NumberType>::value and + not std::is_floating_point<NumberType>::value, int>::type = 0> + void write_number_with_ubjson_prefix(const NumberType n, + const bool add_prefix) + { + if ((std::numeric_limits<int8_t>::min)() <= n and n <= (std::numeric_limits<int8_t>::max)()) + { + if (add_prefix) + { + oa->write_character(to_char_type('i')); // int8 + } + write_number(static_cast<int8_t>(n)); + } + else if (static_cast<int64_t>((std::numeric_limits<uint8_t>::min)()) <= n and n <= static_cast<int64_t>((std::numeric_limits<uint8_t>::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('U')); // uint8 + } + write_number(static_cast<uint8_t>(n)); + } + else if ((std::numeric_limits<int16_t>::min)() <= n and n <= (std::numeric_limits<int16_t>::max)()) + { + if (add_prefix) + { + oa->write_character(to_char_type('I')); // int16 + } + write_number(static_cast<int16_t>(n)); + } + else if ((std::numeric_limits<int32_t>::min)() <= n and n <= (std::numeric_limits<int32_t>::max)()) + { + if (add_prefix) + { + oa->write_character(to_char_type('l')); // int32 + } + write_number(static_cast<int32_t>(n)); + } + else if ((std::numeric_limits<int64_t>::min)() <= n and n <= (std::numeric_limits<int64_t>::max)()) + { + if (add_prefix) + { + oa->write_character(to_char_type('L')); // int64 + } + write_number(static_cast<int64_t>(n)); + } + // LCOV_EXCL_START + else + { + JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(n) + " cannot be represented by UBJSON as it does not fit int64")); + } + // LCOV_EXCL_STOP + } + + /*! + @brief determine the type prefix of container values + + @note This function does not need to be 100% accurate when it comes to + integer limits. In case a number exceeds the limits of int64_t, + this will be detected by a later call to function + write_number_with_ubjson_prefix. Therefore, we return 'L' for any + value that does not fit the previous limits. + */ + CharType ubjson_prefix(const BasicJsonType& j) const noexcept + { + switch (j.type()) + { + case value_t::null: + return 'Z'; + + case value_t::boolean: + return j.m_value.boolean ? 'T' : 'F'; + + case value_t::number_integer: + { + if ((std::numeric_limits<int8_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<int8_t>::max)()) + { + return 'i'; + } + if ((std::numeric_limits<uint8_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<uint8_t>::max)()) + { + return 'U'; + } + if ((std::numeric_limits<int16_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<int16_t>::max)()) + { + return 'I'; + } + if ((std::numeric_limits<int32_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<int32_t>::max)()) + { + return 'l'; + } + // no check and assume int64_t (see note above) + return 'L'; + } + + case value_t::number_unsigned: + { + if (j.m_value.number_unsigned <= (std::numeric_limits<int8_t>::max)()) + { + return 'i'; + } + if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)()) + { + return 'U'; + } + if (j.m_value.number_unsigned <= (std::numeric_limits<int16_t>::max)()) + { + return 'I'; + } + if (j.m_value.number_unsigned <= (std::numeric_limits<int32_t>::max)()) + { + return 'l'; + } + // no check and assume int64_t (see note above) + return 'L'; + } + + case value_t::number_float: + return get_ubjson_float_prefix(j.m_value.number_float); + + case value_t::string: + return 'S'; + + case value_t::array: + return '['; + + case value_t::object: + return '{'; + + default: // discarded values + return 'N'; + } + } + + static constexpr CharType get_ubjson_float_prefix(float /*unused*/) + { + return 'd'; // float 32 + } + + static constexpr CharType get_ubjson_float_prefix(double /*unused*/) + { + return 'D'; // float 64 + } + + /////////////////////// + // Utility functions // + /////////////////////// + /* @brief write a number to output input - @param[in] n number of type @a NumberType @tparam NumberType the type of the number + @tparam OutputIsLittleEndian Set to true if output data is + required to be little endian @note This function needs to respect the system's endianess, because bytes - in CBOR and MessagePack are stored in network order (big endian) and - therefore need reordering on little endian systems. + in CBOR, MessagePack, and UBJSON are stored in network order (big + endian) and therefore need reordering on little endian systems. */ - template<typename NumberType> void write_number(NumberType n) + template<typename NumberType, bool OutputIsLittleEndian = false> + void write_number(const NumberType n) { // step 1: write number to array of length NumberType std::array<CharType, sizeof(NumberType)> vec; std::memcpy(vec.data(), &n, sizeof(NumberType)); // step 2: write array to output (with possible reordering) - if (is_little_endian) + if (is_little_endian and not OutputIsLittleEndian) { // reverse byte order prior to conversion if necessary std::reverse(vec.begin(), vec.end()); @@ -6058,6 +9728,47 @@ class binary_writer oa->write_characters(vec.data(), sizeof(NumberType)); } + public: + // The following to_char_type functions are implement the conversion + // between uint8_t and CharType. In case CharType is not unsigned, + // such a conversion is required to allow values greater than 128. + // See <https://github.com/nlohmann/json/issues/1286> for a discussion. + template < typename C = CharType, + enable_if_t < std::is_signed<C>::value and std::is_signed<char>::value > * = nullptr > + static constexpr CharType to_char_type(std::uint8_t x) noexcept + { + return *reinterpret_cast<char*>(&x); + } + + template < typename C = CharType, + enable_if_t < std::is_signed<C>::value and std::is_unsigned<char>::value > * = nullptr > + static CharType to_char_type(std::uint8_t x) noexcept + { + static_assert(sizeof(std::uint8_t) == sizeof(CharType), "size of CharType must be equal to std::uint8_t"); + static_assert(std::is_pod<CharType>::value, "CharType must be POD"); + CharType result; + std::memcpy(&result, &x, sizeof(x)); + return result; + } + + template<typename C = CharType, + enable_if_t<std::is_unsigned<C>::value>* = nullptr> + static constexpr CharType to_char_type(std::uint8_t x) noexcept + { + return x; + } + + template < typename InputCharType, typename C = CharType, + enable_if_t < + std::is_signed<C>::value and + std::is_signed<char>::value and + std::is_same<char, typename std::remove_cv<InputCharType>::type>::value + > * = nullptr > + static constexpr CharType to_char_type(InputCharType x) noexcept + { + return x; + } + private: /// whether we can assume little endianess const bool is_little_endian = binary_reader<BasicJsonType>::little_endianess(); @@ -6065,11 +9776,1150 @@ class binary_writer /// the output output_adapter_t<CharType> oa = nullptr; }; +} // namespace detail +} // namespace nlohmann +// #include <nlohmann/detail/output/serializer.hpp> + + +#include <algorithm> // reverse, remove, fill, find, none_of +#include <array> // array +#include <cassert> // assert +#include <ciso646> // and, or +#include <clocale> // localeconv, lconv +#include <cmath> // labs, isfinite, isnan, signbit +#include <cstddef> // size_t, ptrdiff_t +#include <cstdint> // uint8_t +#include <cstdio> // snprintf +#include <limits> // numeric_limits +#include <string> // string +#include <type_traits> // is_same + +// #include <nlohmann/detail/exceptions.hpp> + +// #include <nlohmann/detail/conversions/to_chars.hpp> + + +#include <cassert> // assert +#include <ciso646> // or, and, not +#include <cmath> // signbit, isfinite +#include <cstdint> // intN_t, uintN_t +#include <cstring> // memcpy, memmove + +namespace nlohmann +{ +namespace detail +{ + +/*! +@brief implements the Grisu2 algorithm for binary to decimal floating-point +conversion. + +This implementation is a slightly modified version of the reference +implementation which may be obtained from +http://florian.loitsch.com/publications (bench.tar.gz). + +The code is distributed under the MIT license, Copyright (c) 2009 Florian Loitsch. + +For a detailed description of the algorithm see: + +[1] Loitsch, "Printing Floating-Point Numbers Quickly and Accurately with + Integers", Proceedings of the ACM SIGPLAN 2010 Conference on Programming + Language Design and Implementation, PLDI 2010 +[2] Burger, Dybvig, "Printing Floating-Point Numbers Quickly and Accurately", + Proceedings of the ACM SIGPLAN 1996 Conference on Programming Language + Design and Implementation, PLDI 1996 +*/ +namespace dtoa_impl +{ + +template <typename Target, typename Source> +Target reinterpret_bits(const Source source) +{ + static_assert(sizeof(Target) == sizeof(Source), "size mismatch"); + + Target target; + std::memcpy(&target, &source, sizeof(Source)); + return target; +} + +struct diyfp // f * 2^e +{ + static constexpr int kPrecision = 64; // = q + + uint64_t f = 0; + int e = 0; + + constexpr diyfp(uint64_t f_, int e_) noexcept : f(f_), e(e_) {} + + /*! + @brief returns x - y + @pre x.e == y.e and x.f >= y.f + */ + static diyfp sub(const diyfp& x, const diyfp& y) noexcept + { + assert(x.e == y.e); + assert(x.f >= y.f); + + return {x.f - y.f, x.e}; + } + + /*! + @brief returns x * y + @note The result is rounded. (Only the upper q bits are returned.) + */ + static diyfp mul(const diyfp& x, const diyfp& y) noexcept + { + static_assert(kPrecision == 64, "internal error"); + + // Computes: + // f = round((x.f * y.f) / 2^q) + // e = x.e + y.e + q + + // Emulate the 64-bit * 64-bit multiplication: + // + // p = u * v + // = (u_lo + 2^32 u_hi) (v_lo + 2^32 v_hi) + // = (u_lo v_lo ) + 2^32 ((u_lo v_hi ) + (u_hi v_lo )) + 2^64 (u_hi v_hi ) + // = (p0 ) + 2^32 ((p1 ) + (p2 )) + 2^64 (p3 ) + // = (p0_lo + 2^32 p0_hi) + 2^32 ((p1_lo + 2^32 p1_hi) + (p2_lo + 2^32 p2_hi)) + 2^64 (p3 ) + // = (p0_lo ) + 2^32 (p0_hi + p1_lo + p2_lo ) + 2^64 (p1_hi + p2_hi + p3) + // = (p0_lo ) + 2^32 (Q ) + 2^64 (H ) + // = (p0_lo ) + 2^32 (Q_lo + 2^32 Q_hi ) + 2^64 (H ) + // + // (Since Q might be larger than 2^32 - 1) + // + // = (p0_lo + 2^32 Q_lo) + 2^64 (Q_hi + H) + // + // (Q_hi + H does not overflow a 64-bit int) + // + // = p_lo + 2^64 p_hi + + const uint64_t u_lo = x.f & 0xFFFFFFFF; + const uint64_t u_hi = x.f >> 32; + const uint64_t v_lo = y.f & 0xFFFFFFFF; + const uint64_t v_hi = y.f >> 32; + + const uint64_t p0 = u_lo * v_lo; + const uint64_t p1 = u_lo * v_hi; + const uint64_t p2 = u_hi * v_lo; + const uint64_t p3 = u_hi * v_hi; + + const uint64_t p0_hi = p0 >> 32; + const uint64_t p1_lo = p1 & 0xFFFFFFFF; + const uint64_t p1_hi = p1 >> 32; + const uint64_t p2_lo = p2 & 0xFFFFFFFF; + const uint64_t p2_hi = p2 >> 32; + + uint64_t Q = p0_hi + p1_lo + p2_lo; + + // The full product might now be computed as + // + // p_hi = p3 + p2_hi + p1_hi + (Q >> 32) + // p_lo = p0_lo + (Q << 32) + // + // But in this particular case here, the full p_lo is not required. + // Effectively we only need to add the highest bit in p_lo to p_hi (and + // Q_hi + 1 does not overflow). + + Q += uint64_t{1} << (64 - 32 - 1); // round, ties up + + const uint64_t h = p3 + p2_hi + p1_hi + (Q >> 32); + + return {h, x.e + y.e + 64}; + } + + /*! + @brief normalize x such that the significand is >= 2^(q-1) + @pre x.f != 0 + */ + static diyfp normalize(diyfp x) noexcept + { + assert(x.f != 0); + + while ((x.f >> 63) == 0) + { + x.f <<= 1; + x.e--; + } + + return x; + } + + /*! + @brief normalize x such that the result has the exponent E + @pre e >= x.e and the upper e - x.e bits of x.f must be zero. + */ + static diyfp normalize_to(const diyfp& x, const int target_exponent) noexcept + { + const int delta = x.e - target_exponent; + + assert(delta >= 0); + assert(((x.f << delta) >> delta) == x.f); + + return {x.f << delta, target_exponent}; + } +}; + +struct boundaries +{ + diyfp w; + diyfp minus; + diyfp plus; +}; + +/*! +Compute the (normalized) diyfp representing the input number 'value' and its +boundaries. + +@pre value must be finite and positive +*/ +template <typename FloatType> +boundaries compute_boundaries(FloatType value) +{ + assert(std::isfinite(value)); + assert(value > 0); + + // Convert the IEEE representation into a diyfp. + // + // If v is denormal: + // value = 0.F * 2^(1 - bias) = ( F) * 2^(1 - bias - (p-1)) + // If v is normalized: + // value = 1.F * 2^(E - bias) = (2^(p-1) + F) * 2^(E - bias - (p-1)) + + static_assert(std::numeric_limits<FloatType>::is_iec559, + "internal error: dtoa_short requires an IEEE-754 floating-point implementation"); + + constexpr int kPrecision = std::numeric_limits<FloatType>::digits; // = p (includes the hidden bit) + constexpr int kBias = std::numeric_limits<FloatType>::max_exponent - 1 + (kPrecision - 1); + constexpr int kMinExp = 1 - kBias; + constexpr uint64_t kHiddenBit = uint64_t{1} << (kPrecision - 1); // = 2^(p-1) + + using bits_type = typename std::conditional< kPrecision == 24, uint32_t, uint64_t >::type; + + const uint64_t bits = reinterpret_bits<bits_type>(value); + const uint64_t E = bits >> (kPrecision - 1); + const uint64_t F = bits & (kHiddenBit - 1); + + const bool is_denormal = (E == 0); + const diyfp v = is_denormal + ? diyfp(F, kMinExp) + : diyfp(F + kHiddenBit, static_cast<int>(E) - kBias); + + // Compute the boundaries m- and m+ of the floating-point value + // v = f * 2^e. + // + // Determine v- and v+, the floating-point predecessor and successor if v, + // respectively. + // + // v- = v - 2^e if f != 2^(p-1) or e == e_min (A) + // = v - 2^(e-1) if f == 2^(p-1) and e > e_min (B) + // + // v+ = v + 2^e + // + // Let m- = (v- + v) / 2 and m+ = (v + v+) / 2. All real numbers _strictly_ + // between m- and m+ round to v, regardless of how the input rounding + // algorithm breaks ties. + // + // ---+-------------+-------------+-------------+-------------+--- (A) + // v- m- v m+ v+ + // + // -----------------+------+------+-------------+-------------+--- (B) + // v- m- v m+ v+ + + const bool lower_boundary_is_closer = (F == 0 and E > 1); + const diyfp m_plus = diyfp(2 * v.f + 1, v.e - 1); + const diyfp m_minus = lower_boundary_is_closer + ? diyfp(4 * v.f - 1, v.e - 2) // (B) + : diyfp(2 * v.f - 1, v.e - 1); // (A) + + // Determine the normalized w+ = m+. + const diyfp w_plus = diyfp::normalize(m_plus); + + // Determine w- = m- such that e_(w-) = e_(w+). + const diyfp w_minus = diyfp::normalize_to(m_minus, w_plus.e); + + return {diyfp::normalize(v), w_minus, w_plus}; +} + +// Given normalized diyfp w, Grisu needs to find a (normalized) cached +// power-of-ten c, such that the exponent of the product c * w = f * 2^e lies +// within a certain range [alpha, gamma] (Definition 3.2 from [1]) +// +// alpha <= e = e_c + e_w + q <= gamma +// +// or +// +// f_c * f_w * 2^alpha <= f_c 2^(e_c) * f_w 2^(e_w) * 2^q +// <= f_c * f_w * 2^gamma +// +// Since c and w are normalized, i.e. 2^(q-1) <= f < 2^q, this implies +// +// 2^(q-1) * 2^(q-1) * 2^alpha <= c * w * 2^q < 2^q * 2^q * 2^gamma +// +// or +// +// 2^(q - 2 + alpha) <= c * w < 2^(q + gamma) +// +// The choice of (alpha,gamma) determines the size of the table and the form of +// the digit generation procedure. Using (alpha,gamma)=(-60,-32) works out well +// in practice: +// +// The idea is to cut the number c * w = f * 2^e into two parts, which can be +// processed independently: An integral part p1, and a fractional part p2: +// +// f * 2^e = ( (f div 2^-e) * 2^-e + (f mod 2^-e) ) * 2^e +// = (f div 2^-e) + (f mod 2^-e) * 2^e +// = p1 + p2 * 2^e +// +// The conversion of p1 into decimal form requires a series of divisions and +// modulos by (a power of) 10. These operations are faster for 32-bit than for +// 64-bit integers, so p1 should ideally fit into a 32-bit integer. This can be +// achieved by choosing +// +// -e >= 32 or e <= -32 := gamma +// +// In order to convert the fractional part +// +// p2 * 2^e = p2 / 2^-e = d[-1] / 10^1 + d[-2] / 10^2 + ... +// +// into decimal form, the fraction is repeatedly multiplied by 10 and the digits +// d[-i] are extracted in order: +// +// (10 * p2) div 2^-e = d[-1] +// (10 * p2) mod 2^-e = d[-2] / 10^1 + ... +// +// The multiplication by 10 must not overflow. It is sufficient to choose +// +// 10 * p2 < 16 * p2 = 2^4 * p2 <= 2^64. +// +// Since p2 = f mod 2^-e < 2^-e, +// +// -e <= 60 or e >= -60 := alpha + +constexpr int kAlpha = -60; +constexpr int kGamma = -32; + +struct cached_power // c = f * 2^e ~= 10^k +{ + uint64_t f; + int e; + int k; +}; + +/*! +For a normalized diyfp w = f * 2^e, this function returns a (normalized) cached +power-of-ten c = f_c * 2^e_c, such that the exponent of the product w * c +satisfies (Definition 3.2 from [1]) + + alpha <= e_c + e + q <= gamma. +*/ +inline cached_power get_cached_power_for_binary_exponent(int e) +{ + // Now + // + // alpha <= e_c + e + q <= gamma (1) + // ==> f_c * 2^alpha <= c * 2^e * 2^q + // + // and since the c's are normalized, 2^(q-1) <= f_c, + // + // ==> 2^(q - 1 + alpha) <= c * 2^(e + q) + // ==> 2^(alpha - e - 1) <= c + // + // If c were an exakt power of ten, i.e. c = 10^k, one may determine k as + // + // k = ceil( log_10( 2^(alpha - e - 1) ) ) + // = ceil( (alpha - e - 1) * log_10(2) ) + // + // From the paper: + // "In theory the result of the procedure could be wrong since c is rounded, + // and the computation itself is approximated [...]. In practice, however, + // this simple function is sufficient." + // + // For IEEE double precision floating-point numbers converted into + // normalized diyfp's w = f * 2^e, with q = 64, + // + // e >= -1022 (min IEEE exponent) + // -52 (p - 1) + // -52 (p - 1, possibly normalize denormal IEEE numbers) + // -11 (normalize the diyfp) + // = -1137 + // + // and + // + // e <= +1023 (max IEEE exponent) + // -52 (p - 1) + // -11 (normalize the diyfp) + // = 960 + // + // This binary exponent range [-1137,960] results in a decimal exponent + // range [-307,324]. One does not need to store a cached power for each + // k in this range. For each such k it suffices to find a cached power + // such that the exponent of the product lies in [alpha,gamma]. + // This implies that the difference of the decimal exponents of adjacent + // table entries must be less than or equal to + // + // floor( (gamma - alpha) * log_10(2) ) = 8. + // + // (A smaller distance gamma-alpha would require a larger table.) + + // NB: + // Actually this function returns c, such that -60 <= e_c + e + 64 <= -34. + + constexpr int kCachedPowersSize = 79; + constexpr int kCachedPowersMinDecExp = -300; + constexpr int kCachedPowersDecStep = 8; + + static constexpr cached_power kCachedPowers[] = + { + { 0xAB70FE17C79AC6CA, -1060, -300 }, + { 0xFF77B1FCBEBCDC4F, -1034, -292 }, + { 0xBE5691EF416BD60C, -1007, -284 }, + { 0x8DD01FAD907FFC3C, -980, -276 }, + { 0xD3515C2831559A83, -954, -268 }, + { 0x9D71AC8FADA6C9B5, -927, -260 }, + { 0xEA9C227723EE8BCB, -901, -252 }, + { 0xAECC49914078536D, -874, -244 }, + { 0x823C12795DB6CE57, -847, -236 }, + { 0xC21094364DFB5637, -821, -228 }, + { 0x9096EA6F3848984F, -794, -220 }, + { 0xD77485CB25823AC7, -768, -212 }, + { 0xA086CFCD97BF97F4, -741, -204 }, + { 0xEF340A98172AACE5, -715, -196 }, + { 0xB23867FB2A35B28E, -688, -188 }, + { 0x84C8D4DFD2C63F3B, -661, -180 }, + { 0xC5DD44271AD3CDBA, -635, -172 }, + { 0x936B9FCEBB25C996, -608, -164 }, + { 0xDBAC6C247D62A584, -582, -156 }, + { 0xA3AB66580D5FDAF6, -555, -148 }, + { 0xF3E2F893DEC3F126, -529, -140 }, + { 0xB5B5ADA8AAFF80B8, -502, -132 }, + { 0x87625F056C7C4A8B, -475, -124 }, + { 0xC9BCFF6034C13053, -449, -116 }, + { 0x964E858C91BA2655, -422, -108 }, + { 0xDFF9772470297EBD, -396, -100 }, + { 0xA6DFBD9FB8E5B88F, -369, -92 }, + { 0xF8A95FCF88747D94, -343, -84 }, + { 0xB94470938FA89BCF, -316, -76 }, + { 0x8A08F0F8BF0F156B, -289, -68 }, + { 0xCDB02555653131B6, -263, -60 }, + { 0x993FE2C6D07B7FAC, -236, -52 }, + { 0xE45C10C42A2B3B06, -210, -44 }, + { 0xAA242499697392D3, -183, -36 }, + { 0xFD87B5F28300CA0E, -157, -28 }, + { 0xBCE5086492111AEB, -130, -20 }, + { 0x8CBCCC096F5088CC, -103, -12 }, + { 0xD1B71758E219652C, -77, -4 }, + { 0x9C40000000000000, -50, 4 }, + { 0xE8D4A51000000000, -24, 12 }, + { 0xAD78EBC5AC620000, 3, 20 }, + { 0x813F3978F8940984, 30, 28 }, + { 0xC097CE7BC90715B3, 56, 36 }, + { 0x8F7E32CE7BEA5C70, 83, 44 }, + { 0xD5D238A4ABE98068, 109, 52 }, + { 0x9F4F2726179A2245, 136, 60 }, + { 0xED63A231D4C4FB27, 162, 68 }, + { 0xB0DE65388CC8ADA8, 189, 76 }, + { 0x83C7088E1AAB65DB, 216, 84 }, + { 0xC45D1DF942711D9A, 242, 92 }, + { 0x924D692CA61BE758, 269, 100 }, + { 0xDA01EE641A708DEA, 295, 108 }, + { 0xA26DA3999AEF774A, 322, 116 }, + { 0xF209787BB47D6B85, 348, 124 }, + { 0xB454E4A179DD1877, 375, 132 }, + { 0x865B86925B9BC5C2, 402, 140 }, + { 0xC83553C5C8965D3D, 428, 148 }, + { 0x952AB45CFA97A0B3, 455, 156 }, + { 0xDE469FBD99A05FE3, 481, 164 }, + { 0xA59BC234DB398C25, 508, 172 }, + { 0xF6C69A72A3989F5C, 534, 180 }, + { 0xB7DCBF5354E9BECE, 561, 188 }, + { 0x88FCF317F22241E2, 588, 196 }, + { 0xCC20CE9BD35C78A5, 614, 204 }, + { 0x98165AF37B2153DF, 641, 212 }, + { 0xE2A0B5DC971F303A, 667, 220 }, + { 0xA8D9D1535CE3B396, 694, 228 }, + { 0xFB9B7CD9A4A7443C, 720, 236 }, + { 0xBB764C4CA7A44410, 747, 244 }, + { 0x8BAB8EEFB6409C1A, 774, 252 }, + { 0xD01FEF10A657842C, 800, 260 }, + { 0x9B10A4E5E9913129, 827, 268 }, + { 0xE7109BFBA19C0C9D, 853, 276 }, + { 0xAC2820D9623BF429, 880, 284 }, + { 0x80444B5E7AA7CF85, 907, 292 }, + { 0xBF21E44003ACDD2D, 933, 300 }, + { 0x8E679C2F5E44FF8F, 960, 308 }, + { 0xD433179D9C8CB841, 986, 316 }, + { 0x9E19DB92B4E31BA9, 1013, 324 }, + }; + + // This computation gives exactly the same results for k as + // k = ceil((kAlpha - e - 1) * 0.30102999566398114) + // for |e| <= 1500, but doesn't require floating-point operations. + // NB: log_10(2) ~= 78913 / 2^18 + assert(e >= -1500); + assert(e <= 1500); + const int f = kAlpha - e - 1; + const int k = (f * 78913) / (1 << 18) + static_cast<int>(f > 0); + + const int index = (-kCachedPowersMinDecExp + k + (kCachedPowersDecStep - 1)) / kCachedPowersDecStep; + assert(index >= 0); + assert(index < kCachedPowersSize); + static_cast<void>(kCachedPowersSize); // Fix warning. + + const cached_power cached = kCachedPowers[index]; + assert(kAlpha <= cached.e + e + 64); + assert(kGamma >= cached.e + e + 64); + + return cached; +} + +/*! +For n != 0, returns k, such that pow10 := 10^(k-1) <= n < 10^k. +For n == 0, returns 1 and sets pow10 := 1. +*/ +inline int find_largest_pow10(const uint32_t n, uint32_t& pow10) +{ + // LCOV_EXCL_START + if (n >= 1000000000) + { + pow10 = 1000000000; + return 10; + } + // LCOV_EXCL_STOP + else if (n >= 100000000) + { + pow10 = 100000000; + return 9; + } + else if (n >= 10000000) + { + pow10 = 10000000; + return 8; + } + else if (n >= 1000000) + { + pow10 = 1000000; + return 7; + } + else if (n >= 100000) + { + pow10 = 100000; + return 6; + } + else if (n >= 10000) + { + pow10 = 10000; + return 5; + } + else if (n >= 1000) + { + pow10 = 1000; + return 4; + } + else if (n >= 100) + { + pow10 = 100; + return 3; + } + else if (n >= 10) + { + pow10 = 10; + return 2; + } + else + { + pow10 = 1; + return 1; + } +} + +inline void grisu2_round(char* buf, int len, uint64_t dist, uint64_t delta, + uint64_t rest, uint64_t ten_k) +{ + assert(len >= 1); + assert(dist <= delta); + assert(rest <= delta); + assert(ten_k > 0); + + // <--------------------------- delta ----> + // <---- dist ---------> + // --------------[------------------+-------------------]-------------- + // M- w M+ + // + // ten_k + // <------> + // <---- rest ----> + // --------------[------------------+----+--------------]-------------- + // w V + // = buf * 10^k + // + // ten_k represents a unit-in-the-last-place in the decimal representation + // stored in buf. + // Decrement buf by ten_k while this takes buf closer to w. + + // The tests are written in this order to avoid overflow in unsigned + // integer arithmetic. + + while (rest < dist + and delta - rest >= ten_k + and (rest + ten_k < dist or dist - rest > rest + ten_k - dist)) + { + assert(buf[len - 1] != '0'); + buf[len - 1]--; + rest += ten_k; + } +} + +/*! +Generates V = buffer * 10^decimal_exponent, such that M- <= V <= M+. +M- and M+ must be normalized and share the same exponent -60 <= e <= -32. +*/ +inline void grisu2_digit_gen(char* buffer, int& length, int& decimal_exponent, + diyfp M_minus, diyfp w, diyfp M_plus) +{ + static_assert(kAlpha >= -60, "internal error"); + static_assert(kGamma <= -32, "internal error"); + + // Generates the digits (and the exponent) of a decimal floating-point + // number V = buffer * 10^decimal_exponent in the range [M-, M+]. The diyfp's + // w, M- and M+ share the same exponent e, which satisfies alpha <= e <= gamma. + // + // <--------------------------- delta ----> + // <---- dist ---------> + // --------------[------------------+-------------------]-------------- + // M- w M+ + // + // Grisu2 generates the digits of M+ from left to right and stops as soon as + // V is in [M-,M+]. + + assert(M_plus.e >= kAlpha); + assert(M_plus.e <= kGamma); + + uint64_t delta = diyfp::sub(M_plus, M_minus).f; // (significand of (M+ - M-), implicit exponent is e) + uint64_t dist = diyfp::sub(M_plus, w ).f; // (significand of (M+ - w ), implicit exponent is e) + + // Split M+ = f * 2^e into two parts p1 and p2 (note: e < 0): + // + // M+ = f * 2^e + // = ((f div 2^-e) * 2^-e + (f mod 2^-e)) * 2^e + // = ((p1 ) * 2^-e + (p2 )) * 2^e + // = p1 + p2 * 2^e + + const diyfp one(uint64_t{1} << -M_plus.e, M_plus.e); + + auto p1 = static_cast<uint32_t>(M_plus.f >> -one.e); // p1 = f div 2^-e (Since -e >= 32, p1 fits into a 32-bit int.) + uint64_t p2 = M_plus.f & (one.f - 1); // p2 = f mod 2^-e + + // 1) + // + // Generate the digits of the integral part p1 = d[n-1]...d[1]d[0] + + assert(p1 > 0); + + uint32_t pow10; + const int k = find_largest_pow10(p1, pow10); + + // 10^(k-1) <= p1 < 10^k, pow10 = 10^(k-1) + // + // p1 = (p1 div 10^(k-1)) * 10^(k-1) + (p1 mod 10^(k-1)) + // = (d[k-1] ) * 10^(k-1) + (p1 mod 10^(k-1)) + // + // M+ = p1 + p2 * 2^e + // = d[k-1] * 10^(k-1) + (p1 mod 10^(k-1)) + p2 * 2^e + // = d[k-1] * 10^(k-1) + ((p1 mod 10^(k-1)) * 2^-e + p2) * 2^e + // = d[k-1] * 10^(k-1) + ( rest) * 2^e + // + // Now generate the digits d[n] of p1 from left to right (n = k-1,...,0) + // + // p1 = d[k-1]...d[n] * 10^n + d[n-1]...d[0] + // + // but stop as soon as + // + // rest * 2^e = (d[n-1]...d[0] * 2^-e + p2) * 2^e <= delta * 2^e + + int n = k; + while (n > 0) + { + // Invariants: + // M+ = buffer * 10^n + (p1 + p2 * 2^e) (buffer = 0 for n = k) + // pow10 = 10^(n-1) <= p1 < 10^n + // + const uint32_t d = p1 / pow10; // d = p1 div 10^(n-1) + const uint32_t r = p1 % pow10; // r = p1 mod 10^(n-1) + // + // M+ = buffer * 10^n + (d * 10^(n-1) + r) + p2 * 2^e + // = (buffer * 10 + d) * 10^(n-1) + (r + p2 * 2^e) + // + assert(d <= 9); + buffer[length++] = static_cast<char>('0' + d); // buffer := buffer * 10 + d + // + // M+ = buffer * 10^(n-1) + (r + p2 * 2^e) + // + p1 = r; + n--; + // + // M+ = buffer * 10^n + (p1 + p2 * 2^e) + // pow10 = 10^n + // + + // Now check if enough digits have been generated. + // Compute + // + // p1 + p2 * 2^e = (p1 * 2^-e + p2) * 2^e = rest * 2^e + // + // Note: + // Since rest and delta share the same exponent e, it suffices to + // compare the significands. + const uint64_t rest = (uint64_t{p1} << -one.e) + p2; + if (rest <= delta) + { + // V = buffer * 10^n, with M- <= V <= M+. + + decimal_exponent += n; + + // We may now just stop. But instead look if the buffer could be + // decremented to bring V closer to w. + // + // pow10 = 10^n is now 1 ulp in the decimal representation V. + // The rounding procedure works with diyfp's with an implicit + // exponent of e. + // + // 10^n = (10^n * 2^-e) * 2^e = ulp * 2^e + // + const uint64_t ten_n = uint64_t{pow10} << -one.e; + grisu2_round(buffer, length, dist, delta, rest, ten_n); + + return; + } + + pow10 /= 10; + // + // pow10 = 10^(n-1) <= p1 < 10^n + // Invariants restored. + } + + // 2) + // + // The digits of the integral part have been generated: + // + // M+ = d[k-1]...d[1]d[0] + p2 * 2^e + // = buffer + p2 * 2^e + // + // Now generate the digits of the fractional part p2 * 2^e. + // + // Note: + // No decimal point is generated: the exponent is adjusted instead. + // + // p2 actually represents the fraction + // + // p2 * 2^e + // = p2 / 2^-e + // = d[-1] / 10^1 + d[-2] / 10^2 + ... + // + // Now generate the digits d[-m] of p1 from left to right (m = 1,2,...) + // + // p2 * 2^e = d[-1]d[-2]...d[-m] * 10^-m + // + 10^-m * (d[-m-1] / 10^1 + d[-m-2] / 10^2 + ...) + // + // using + // + // 10^m * p2 = ((10^m * p2) div 2^-e) * 2^-e + ((10^m * p2) mod 2^-e) + // = ( d) * 2^-e + ( r) + // + // or + // 10^m * p2 * 2^e = d + r * 2^e + // + // i.e. + // + // M+ = buffer + p2 * 2^e + // = buffer + 10^-m * (d + r * 2^e) + // = (buffer * 10^m + d) * 10^-m + 10^-m * r * 2^e + // + // and stop as soon as 10^-m * r * 2^e <= delta * 2^e + + assert(p2 > delta); + + int m = 0; + for (;;) + { + // Invariant: + // M+ = buffer * 10^-m + 10^-m * (d[-m-1] / 10 + d[-m-2] / 10^2 + ...) * 2^e + // = buffer * 10^-m + 10^-m * (p2 ) * 2^e + // = buffer * 10^-m + 10^-m * (1/10 * (10 * p2) ) * 2^e + // = buffer * 10^-m + 10^-m * (1/10 * ((10*p2 div 2^-e) * 2^-e + (10*p2 mod 2^-e)) * 2^e + // + assert(p2 <= UINT64_MAX / 10); + p2 *= 10; + const uint64_t d = p2 >> -one.e; // d = (10 * p2) div 2^-e + const uint64_t r = p2 & (one.f - 1); // r = (10 * p2) mod 2^-e + // + // M+ = buffer * 10^-m + 10^-m * (1/10 * (d * 2^-e + r) * 2^e + // = buffer * 10^-m + 10^-m * (1/10 * (d + r * 2^e)) + // = (buffer * 10 + d) * 10^(-m-1) + 10^(-m-1) * r * 2^e + // + assert(d <= 9); + buffer[length++] = static_cast<char>('0' + d); // buffer := buffer * 10 + d + // + // M+ = buffer * 10^(-m-1) + 10^(-m-1) * r * 2^e + // + p2 = r; + m++; + // + // M+ = buffer * 10^-m + 10^-m * p2 * 2^e + // Invariant restored. + + // Check if enough digits have been generated. + // + // 10^-m * p2 * 2^e <= delta * 2^e + // p2 * 2^e <= 10^m * delta * 2^e + // p2 <= 10^m * delta + delta *= 10; + dist *= 10; + if (p2 <= delta) + { + break; + } + } + + // V = buffer * 10^-m, with M- <= V <= M+. + + decimal_exponent -= m; + + // 1 ulp in the decimal representation is now 10^-m. + // Since delta and dist are now scaled by 10^m, we need to do the + // same with ulp in order to keep the units in sync. + // + // 10^m * 10^-m = 1 = 2^-e * 2^e = ten_m * 2^e + // + const uint64_t ten_m = one.f; + grisu2_round(buffer, length, dist, delta, p2, ten_m); + + // By construction this algorithm generates the shortest possible decimal + // number (Loitsch, Theorem 6.2) which rounds back to w. + // For an input number of precision p, at least + // + // N = 1 + ceil(p * log_10(2)) + // + // decimal digits are sufficient to identify all binary floating-point + // numbers (Matula, "In-and-Out conversions"). + // This implies that the algorithm does not produce more than N decimal + // digits. + // + // N = 17 for p = 53 (IEEE double precision) + // N = 9 for p = 24 (IEEE single precision) +} + +/*! +v = buf * 10^decimal_exponent +len is the length of the buffer (number of decimal digits) +The buffer must be large enough, i.e. >= max_digits10. +*/ +inline void grisu2(char* buf, int& len, int& decimal_exponent, + diyfp m_minus, diyfp v, diyfp m_plus) +{ + assert(m_plus.e == m_minus.e); + assert(m_plus.e == v.e); + + // --------(-----------------------+-----------------------)-------- (A) + // m- v m+ + // + // --------------------(-----------+-----------------------)-------- (B) + // m- v m+ + // + // First scale v (and m- and m+) such that the exponent is in the range + // [alpha, gamma]. + + const cached_power cached = get_cached_power_for_binary_exponent(m_plus.e); + + const diyfp c_minus_k(cached.f, cached.e); // = c ~= 10^-k + + // The exponent of the products is = v.e + c_minus_k.e + q and is in the range [alpha,gamma] + const diyfp w = diyfp::mul(v, c_minus_k); + const diyfp w_minus = diyfp::mul(m_minus, c_minus_k); + const diyfp w_plus = diyfp::mul(m_plus, c_minus_k); + + // ----(---+---)---------------(---+---)---------------(---+---)---- + // w- w w+ + // = c*m- = c*v = c*m+ + // + // diyfp::mul rounds its result and c_minus_k is approximated too. w, w- and + // w+ are now off by a small amount. + // In fact: + // + // w - v * 10^k < 1 ulp + // + // To account for this inaccuracy, add resp. subtract 1 ulp. + // + // --------+---[---------------(---+---)---------------]---+-------- + // w- M- w M+ w+ + // + // Now any number in [M-, M+] (bounds included) will round to w when input, + // regardless of how the input rounding algorithm breaks ties. + // + // And digit_gen generates the shortest possible such number in [M-, M+]. + // Note that this does not mean that Grisu2 always generates the shortest + // possible number in the interval (m-, m+). + const diyfp M_minus(w_minus.f + 1, w_minus.e); + const diyfp M_plus (w_plus.f - 1, w_plus.e ); + + decimal_exponent = -cached.k; // = -(-k) = k + + grisu2_digit_gen(buf, len, decimal_exponent, M_minus, w, M_plus); +} + +/*! +v = buf * 10^decimal_exponent +len is the length of the buffer (number of decimal digits) +The buffer must be large enough, i.e. >= max_digits10. +*/ +template <typename FloatType> +void grisu2(char* buf, int& len, int& decimal_exponent, FloatType value) +{ + static_assert(diyfp::kPrecision >= std::numeric_limits<FloatType>::digits + 3, + "internal error: not enough precision"); + + assert(std::isfinite(value)); + assert(value > 0); + + // If the neighbors (and boundaries) of 'value' are always computed for double-precision + // numbers, all float's can be recovered using strtod (and strtof). However, the resulting + // decimal representations are not exactly "short". + // + // The documentation for 'std::to_chars' (https://en.cppreference.com/w/cpp/utility/to_chars) + // says "value is converted to a string as if by std::sprintf in the default ("C") locale" + // and since sprintf promotes float's to double's, I think this is exactly what 'std::to_chars' + // does. + // On the other hand, the documentation for 'std::to_chars' requires that "parsing the + // representation using the corresponding std::from_chars function recovers value exactly". That + // indicates that single precision floating-point numbers should be recovered using + // 'std::strtof'. + // + // NB: If the neighbors are computed for single-precision numbers, there is a single float + // (7.0385307e-26f) which can't be recovered using strtod. The resulting double precision + // value is off by 1 ulp. +#if 0 + const boundaries w = compute_boundaries(static_cast<double>(value)); +#else + const boundaries w = compute_boundaries(value); +#endif + + grisu2(buf, len, decimal_exponent, w.minus, w.w, w.plus); +} + +/*! +@brief appends a decimal representation of e to buf +@return a pointer to the element following the exponent. +@pre -1000 < e < 1000 +*/ +inline char* append_exponent(char* buf, int e) +{ + assert(e > -1000); + assert(e < 1000); + + if (e < 0) + { + e = -e; + *buf++ = '-'; + } + else + { + *buf++ = '+'; + } + + auto k = static_cast<uint32_t>(e); + if (k < 10) + { + // Always print at least two digits in the exponent. + // This is for compatibility with printf("%g"). + *buf++ = '0'; + *buf++ = static_cast<char>('0' + k); + } + else if (k < 100) + { + *buf++ = static_cast<char>('0' + k / 10); + k %= 10; + *buf++ = static_cast<char>('0' + k); + } + else + { + *buf++ = static_cast<char>('0' + k / 100); + k %= 100; + *buf++ = static_cast<char>('0' + k / 10); + k %= 10; + *buf++ = static_cast<char>('0' + k); + } + + return buf; +} + +/*! +@brief prettify v = buf * 10^decimal_exponent + +If v is in the range [10^min_exp, 10^max_exp) it will be printed in fixed-point +notation. Otherwise it will be printed in exponential notation. + +@pre min_exp < 0 +@pre max_exp > 0 +*/ +inline char* format_buffer(char* buf, int len, int decimal_exponent, + int min_exp, int max_exp) +{ + assert(min_exp < 0); + assert(max_exp > 0); + + const int k = len; + const int n = len + decimal_exponent; + + // v = buf * 10^(n-k) + // k is the length of the buffer (number of decimal digits) + // n is the position of the decimal point relative to the start of the buffer. + + if (k <= n and n <= max_exp) + { + // digits[000] + // len <= max_exp + 2 + + std::memset(buf + k, '0', static_cast<size_t>(n - k)); + // Make it look like a floating-point number (#362, #378) + buf[n + 0] = '.'; + buf[n + 1] = '0'; + return buf + (n + 2); + } + + if (0 < n and n <= max_exp) + { + // dig.its + // len <= max_digits10 + 1 + + assert(k > n); + + std::memmove(buf + (n + 1), buf + n, static_cast<size_t>(k - n)); + buf[n] = '.'; + return buf + (k + 1); + } + + if (min_exp < n and n <= 0) + { + // 0.[000]digits + // len <= 2 + (-min_exp - 1) + max_digits10 + + std::memmove(buf + (2 + -n), buf, static_cast<size_t>(k)); + buf[0] = '0'; + buf[1] = '.'; + std::memset(buf + 2, '0', static_cast<size_t>(-n)); + return buf + (2 + (-n) + k); + } + + if (k == 1) + { + // dE+123 + // len <= 1 + 5 + + buf += 1; + } + else + { + // d.igitsE+123 + // len <= max_digits10 + 1 + 5 + + std::memmove(buf + 2, buf + 1, static_cast<size_t>(k - 1)); + buf[1] = '.'; + buf += 1 + k; + } + + *buf++ = 'e'; + return append_exponent(buf, n - 1); +} + +} // namespace dtoa_impl + +/*! +@brief generates a decimal representation of the floating-point number value in [first, last). + +The format of the resulting decimal representation is similar to printf's %g +format. Returns an iterator pointing past-the-end of the decimal representation. + +@note The input number must be finite, i.e. NaN's and Inf's are not supported. +@note The buffer must be large enough. +@note The result is NOT null-terminated. +*/ +template <typename FloatType> +char* to_chars(char* first, const char* last, FloatType value) +{ + static_cast<void>(last); // maybe unused - fix warning + assert(std::isfinite(value)); + + // Use signbit(value) instead of (value < 0) since signbit works for -0. + if (std::signbit(value)) + { + value = -value; + *first++ = '-'; + } + + if (value == 0) // +-0 + { + *first++ = '0'; + // Make it look like a floating-point number (#362, #378) + *first++ = '.'; + *first++ = '0'; + return first; + } + + assert(last - first >= std::numeric_limits<FloatType>::max_digits10); + + // Compute v = buffer * 10^decimal_exponent. + // The decimal digits are stored in the buffer, which needs to be interpreted + // as an unsigned decimal integer. + // len is the length of the buffer, i.e. the number of decimal digits. + int len = 0; + int decimal_exponent = 0; + dtoa_impl::grisu2(first, len, decimal_exponent, value); + + assert(len <= std::numeric_limits<FloatType>::max_digits10); + + // Format the buffer like printf("%.*g", prec, value) + constexpr int kMinExp = -4; + // Use digits10 here to increase compatibility with version 2. + constexpr int kMaxExp = std::numeric_limits<FloatType>::digits10; + + assert(last - first >= kMaxExp + 2); + assert(last - first >= 2 + (-kMinExp - 1) + std::numeric_limits<FloatType>::max_digits10); + assert(last - first >= std::numeric_limits<FloatType>::max_digits10 + 6); + + return dtoa_impl::format_buffer(first, len, decimal_exponent, kMinExp, kMaxExp); +} + +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/macro_scope.hpp> + +// #include <nlohmann/detail/meta/cpp_future.hpp> + +// #include <nlohmann/detail/output/binary_writer.hpp> + +// #include <nlohmann/detail/output/output_adapters.hpp> + +// #include <nlohmann/detail/value_t.hpp> + + +namespace nlohmann +{ +namespace detail +{ /////////////////// // serialization // /////////////////// +/// how to treat decoding errors +enum class error_handler_t +{ + strict, ///< throw a type_error exception in case of invalid UTF-8 + replace, ///< replace invalid UTF-8 sequences with U+FFFD + ignore ///< ignore invalid UTF-8 sequences +}; + template<typename BasicJsonType> class serializer { @@ -6077,20 +10927,32 @@ class serializer using number_float_t = typename BasicJsonType::number_float_t; using number_integer_t = typename BasicJsonType::number_integer_t; using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + static constexpr uint8_t UTF8_ACCEPT = 0; + static constexpr uint8_t UTF8_REJECT = 1; + public: /*! @param[in] s output stream to serialize to @param[in] ichar indentation character to use + @param[in] error_handler_ how to react on decoding errors */ - serializer(output_adapter_t<char> s, const char ichar) - : o(std::move(s)), loc(std::localeconv()), - thousands_sep(loc->thousands_sep == nullptr ? '\0' : * (loc->thousands_sep)), - decimal_point(loc->decimal_point == nullptr ? '\0' : * (loc->decimal_point)), - indent_char(ichar), indent_string(512, indent_char) {} + serializer(output_adapter_t<char> s, const char ichar, + error_handler_t error_handler_ = error_handler_t::strict) + : o(std::move(s)) + , loc(std::localeconv()) + , thousands_sep(loc->thousands_sep == nullptr ? '\0' : * (loc->thousands_sep)) + , decimal_point(loc->decimal_point == nullptr ? '\0' : * (loc->decimal_point)) + , indent_char(ichar) + , indent_string(512, indent_char) + , error_handler(error_handler_) + {} // delete because of pointer members serializer(const serializer&) = delete; serializer& operator=(const serializer&) = delete; + serializer(serializer&&) = delete; + serializer& operator=(serializer&&) = delete; + ~serializer() = default; /*! @brief internal implementation of the serialization function @@ -6303,323 +11165,249 @@ class serializer private: /*! - @brief returns the number of expected bytes following in UTF-8 string - - @param[in] u the first byte of a UTF-8 string - @return the number of expected bytes following - */ - static constexpr std::size_t bytes_following(const uint8_t u) - { - return ((u <= 127) ? 0 - : ((192 <= u and u <= 223) ? 1 - : ((224 <= u and u <= 239) ? 2 - : ((240 <= u and u <= 247) ? 3 : std::string::npos)))); - } + @brief dump escaped string - /*! - @brief calculates the extra space to escape a JSON string + Escape a string by replacing certain special characters by a sequence of an + escape character (backslash) and another character and other control + characters by a sequence of "\u" followed by a four-digit hex + representation. The escaped string is written to output stream @a o. @param[in] s the string to escape @param[in] ensure_ascii whether to escape non-ASCII characters with \uXXXX sequences - @return the number of characters required to escape string @a s @complexity Linear in the length of string @a s. */ - static std::size_t extra_space(const string_t& s, - const bool ensure_ascii) noexcept + void dump_escaped(const string_t& s, const bool ensure_ascii) { - std::size_t res = 0; + uint32_t codepoint; + uint8_t state = UTF8_ACCEPT; + std::size_t bytes = 0; // number of bytes written to string_buffer + + // number of bytes written at the point of the last valid byte + std::size_t bytes_after_last_accept = 0; + std::size_t undumped_chars = 0; for (std::size_t i = 0; i < s.size(); ++i) { - switch (s[i]) - { - // control characters that can be escaped with a backslash - case '"': - case '\\': - case '\b': - case '\f': - case '\n': - case '\r': - case '\t': - { - // from c (1 byte) to \x (2 bytes) - res += 1; - break; - } + const auto byte = static_cast<uint8_t>(s[i]); - // control characters that need \uxxxx escaping - case 0x00: - case 0x01: - case 0x02: - case 0x03: - case 0x04: - case 0x05: - case 0x06: - case 0x07: - case 0x0B: - case 0x0E: - case 0x0F: - case 0x10: - case 0x11: - case 0x12: - case 0x13: - case 0x14: - case 0x15: - case 0x16: - case 0x17: - case 0x18: - case 0x19: - case 0x1A: - case 0x1B: - case 0x1C: - case 0x1D: - case 0x1E: - case 0x1F: - { - // from c (1 byte) to \uxxxx (6 bytes) - res += 5; - break; - } - - default: + switch (decode(state, codepoint, byte)) + { + case UTF8_ACCEPT: // decode found a new code point { - if (ensure_ascii and (s[i] & 0x80 or s[i] == 0x7F)) + switch (codepoint) { - const auto bytes = bytes_following(static_cast<uint8_t>(s[i])); - // invalid characters will be detected by throw_if_invalid_utf8 - assert (bytes != std::string::npos); - - if (bytes == 3) + case 0x08: // backspace { - // codepoints that need 4 bytes (i.e., 3 additional - // bytes) in UTF-8 need a surrogate pair when \u - // escaping is used: from 4 bytes to \uxxxx\uxxxx - // (12 bytes) - res += (12 - bytes - 1); + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'b'; + break; } - else + + case 0x09: // horizontal tab { - // from x bytes to \uxxxx (6 bytes) - res += (6 - bytes - 1); + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 't'; + break; } - // skip the additional bytes - i += bytes; - } - break; - } - } - } - - return res; - } - - static void escape_codepoint(int codepoint, string_t& result, std::size_t& pos) - { - // expecting a proper codepoint - assert(0x00 <= codepoint and codepoint <= 0x10FFFF); - - // the last written character was the backslash before the 'u' - assert(result[pos] == '\\'); - - // write the 'u' - result[++pos] = 'u'; - - // convert a number 0..15 to its hex representation (0..f) - static const std::array<char, 16> hexify = - { - { - '0', '1', '2', '3', '4', '5', '6', '7', - '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' - } - }; - - if (codepoint < 0x10000) - { - // codepoints U+0000..U+FFFF can be represented as \uxxxx. - result[++pos] = hexify[(codepoint >> 12) & 0x0F]; - result[++pos] = hexify[(codepoint >> 8) & 0x0F]; - result[++pos] = hexify[(codepoint >> 4) & 0x0F]; - result[++pos] = hexify[codepoint & 0x0F]; - } - else - { - // codepoints U+10000..U+10FFFF need a surrogate pair to be - // represented as \uxxxx\uxxxx. - // http://www.unicode.org/faq/utf_bom.html#utf16-4 - codepoint -= 0x10000; - const int high_surrogate = 0xD800 | ((codepoint >> 10) & 0x3FF); - const int low_surrogate = 0xDC00 | (codepoint & 0x3FF); - result[++pos] = hexify[(high_surrogate >> 12) & 0x0F]; - result[++pos] = hexify[(high_surrogate >> 8) & 0x0F]; - result[++pos] = hexify[(high_surrogate >> 4) & 0x0F]; - result[++pos] = hexify[high_surrogate & 0x0F]; - ++pos; // backslash is already in output - result[++pos] = 'u'; - result[++pos] = hexify[(low_surrogate >> 12) & 0x0F]; - result[++pos] = hexify[(low_surrogate >> 8) & 0x0F]; - result[++pos] = hexify[(low_surrogate >> 4) & 0x0F]; - result[++pos] = hexify[low_surrogate & 0x0F]; - } - - ++pos; - } + case 0x0A: // newline + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'n'; + break; + } - /*! - @brief dump escaped string + case 0x0C: // formfeed + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'f'; + break; + } - Escape a string by replacing certain special characters by a sequence of an - escape character (backslash) and another character and other control - characters by a sequence of "\u" followed by a four-digit hex - representation. The escaped string is written to output stream @a o. + case 0x0D: // carriage return + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'r'; + break; + } - @param[in] s the string to escape - @param[in] ensure_ascii whether to escape non-ASCII characters with - \uXXXX sequences + case 0x22: // quotation mark + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = '\"'; + break; + } - @complexity Linear in the length of string @a s. - */ - void dump_escaped(const string_t& s, const bool ensure_ascii) const - { - throw_if_invalid_utf8(s); + case 0x5C: // reverse solidus + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = '\\'; + break; + } - const auto space = extra_space(s, ensure_ascii); - if (space == 0) - { - o->write_characters(s.c_str(), s.size()); - return; - } + default: + { + // escape control characters (0x00..0x1F) or, if + // ensure_ascii parameter is used, non-ASCII characters + if ((codepoint <= 0x1F) or (ensure_ascii and (codepoint >= 0x7F))) + { + if (codepoint <= 0xFFFF) + { + (std::snprintf)(string_buffer.data() + bytes, 7, "\\u%04x", + static_cast<uint16_t>(codepoint)); + bytes += 6; + } + else + { + (std::snprintf)(string_buffer.data() + bytes, 13, "\\u%04x\\u%04x", + static_cast<uint16_t>(0xD7C0 + (codepoint >> 10)), + static_cast<uint16_t>(0xDC00 + (codepoint & 0x3FF))); + bytes += 12; + } + } + else + { + // copy byte to buffer (all previous bytes + // been copied have in default case above) + string_buffer[bytes++] = s[i]; + } + break; + } + } - // create a result string of necessary size - string_t result(s.size() + space, '\\'); - std::size_t pos = 0; + // write buffer and reset index; there must be 13 bytes + // left, as this is the maximal number of bytes to be + // written ("\uxxxx\uxxxx\0") for one code point + if (string_buffer.size() - bytes < 13) + { + o->write_characters(string_buffer.data(), bytes); + bytes = 0; + } - for (std::size_t i = 0; i < s.size(); ++i) - { - switch (s[i]) - { - case '"': // quotation mark (0x22) - { - result[pos + 1] = '"'; - pos += 2; + // remember the byte position of this accept + bytes_after_last_accept = bytes; + undumped_chars = 0; break; } - case '\\': // reverse solidus (0x5C) + case UTF8_REJECT: // decode found invalid UTF-8 byte { - // nothing to change - pos += 2; - break; - } + switch (error_handler) + { + case error_handler_t::strict: + { + std::string sn(3, '\0'); + (std::snprintf)(&sn[0], sn.size(), "%.2X", byte); + JSON_THROW(type_error::create(316, "invalid UTF-8 byte at index " + std::to_string(i) + ": 0x" + sn)); + } - case '\b': // backspace (0x08) - { - result[pos + 1] = 'b'; - pos += 2; - break; - } + case error_handler_t::ignore: + case error_handler_t::replace: + { + // in case we saw this character the first time, we + // would like to read it again, because the byte + // may be OK for itself, but just not OK for the + // previous sequence + if (undumped_chars > 0) + { + --i; + } - case '\f': // formfeed (0x0C) - { - result[pos + 1] = 'f'; - pos += 2; + // reset length buffer to the last accepted index; + // thus removing/ignoring the invalid characters + bytes = bytes_after_last_accept; + + if (error_handler == error_handler_t::replace) + { + // add a replacement character + if (ensure_ascii) + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'u'; + string_buffer[bytes++] = 'f'; + string_buffer[bytes++] = 'f'; + string_buffer[bytes++] = 'f'; + string_buffer[bytes++] = 'd'; + } + else + { + string_buffer[bytes++] = detail::binary_writer<BasicJsonType, char>::to_char_type('\xEF'); + string_buffer[bytes++] = detail::binary_writer<BasicJsonType, char>::to_char_type('\xBF'); + string_buffer[bytes++] = detail::binary_writer<BasicJsonType, char>::to_char_type('\xBD'); + } + bytes_after_last_accept = bytes; + } + + undumped_chars = 0; + + // continue processing the string + state = UTF8_ACCEPT; + break; + } + } break; } - case '\n': // newline (0x0A) + default: // decode found yet incomplete multi-byte code point { - result[pos + 1] = 'n'; - pos += 2; + if (not ensure_ascii) + { + // code point will not be escaped - copy byte to buffer + string_buffer[bytes++] = s[i]; + } + ++undumped_chars; break; } + } + } - case '\r': // carriage return (0x0D) + // we finished processing the string + if (JSON_LIKELY(state == UTF8_ACCEPT)) + { + // write buffer + if (bytes > 0) + { + o->write_characters(string_buffer.data(), bytes); + } + } + else + { + // we finish reading, but do not accept: string was incomplete + switch (error_handler) + { + case error_handler_t::strict: { - result[pos + 1] = 'r'; - pos += 2; - break; + std::string sn(3, '\0'); + (std::snprintf)(&sn[0], sn.size(), "%.2X", static_cast<uint8_t>(s.back())); + JSON_THROW(type_error::create(316, "incomplete UTF-8 string; last byte: 0x" + sn)); } - case '\t': // horizontal tab (0x09) + case error_handler_t::ignore: { - result[pos + 1] = 't'; - pos += 2; + // write all accepted bytes + o->write_characters(string_buffer.data(), bytes_after_last_accept); break; } - default: + case error_handler_t::replace: { - // escape control characters (0x00..0x1F) or, if - // ensure_ascii parameter is used, non-ASCII characters - if ((0x00 <= s[i] and s[i] <= 0x1F) or - (ensure_ascii and (s[i] & 0x80 or s[i] == 0x7F))) + // write all accepted bytes + o->write_characters(string_buffer.data(), bytes_after_last_accept); + // add a replacement character + if (ensure_ascii) { - const auto bytes = bytes_following(static_cast<uint8_t>(s[i])); - // invalid characters will be detected by throw_if_invalid_utf8 - assert (bytes != std::string::npos); - - // check that the additional bytes are present - assert(i + bytes < s.size()); - - // to use \uxxxx escaping, we first need to calculate - // the codepoint from the UTF-8 bytes - int codepoint = 0; - - // bytes is unsigned type: - assert(bytes <= 3); - switch (bytes) - { - case 0: - { - codepoint = s[i] & 0xFF; - break; - } - - case 1: - { - codepoint = ((s[i] & 0x3F) << 6) - + (s[i + 1] & 0x7F); - break; - } - - case 2: - { - codepoint = ((s[i] & 0x1F) << 12) - + ((s[i + 1] & 0x7F) << 6) - + (s[i + 2] & 0x7F); - break; - } - - case 3: - { - codepoint = ((s[i] & 0xF) << 18) - + ((s[i + 1] & 0x7F) << 12) - + ((s[i + 2] & 0x7F) << 6) - + (s[i + 3] & 0x7F); - break; - } - - default: - break; // LCOV_EXCL_LINE - } - - escape_codepoint(codepoint, result, pos); - i += bytes; + o->write_characters("\\ufffd", 6); } else { - // all other characters are added as-is - result[pos++] = s[i]; + o->write_characters("\xEF\xBF\xBD", 3); } break; } } } - - assert(pos == result.size()); - o->write_characters(result.c_str(), result.size()); } /*! @@ -6644,7 +11432,7 @@ class serializer return; } - const bool is_negative = (x <= 0) and (x != 0); // see issue #755 + const bool is_negative = std::is_same<NumberType, number_integer_t>::value and not (x >= 0); // see issue #755 std::size_t i = 0; while (x != 0) @@ -6679,17 +11467,39 @@ class serializer void dump_float(number_float_t x) { // NaN / inf - if (not std::isfinite(x) or std::isnan(x)) + if (not std::isfinite(x)) { o->write_characters("null", 4); return; } - // get number of digits for a text -> float -> text round-trip - static constexpr auto d = std::numeric_limits<number_float_t>::digits10; + // If number_float_t is an IEEE-754 single or double precision number, + // use the Grisu2 algorithm to produce short numbers which are + // guaranteed to round-trip, using strtof and strtod, resp. + // + // NB: The test below works if <long double> == <double>. + static constexpr bool is_ieee_single_or_double + = (std::numeric_limits<number_float_t>::is_iec559 and std::numeric_limits<number_float_t>::digits == 24 and std::numeric_limits<number_float_t>::max_exponent == 128) or + (std::numeric_limits<number_float_t>::is_iec559 and std::numeric_limits<number_float_t>::digits == 53 and std::numeric_limits<number_float_t>::max_exponent == 1024); + + dump_float(x, std::integral_constant<bool, is_ieee_single_or_double>()); + } + + void dump_float(number_float_t x, std::true_type /*is_ieee_single_or_double*/) + { + char* begin = number_buffer.data(); + char* end = ::nlohmann::detail::to_chars(begin, begin + number_buffer.size(), x); + + o->write_characters(begin, static_cast<size_t>(end - begin)); + } + + void dump_float(number_float_t x, std::false_type /*is_ieee_single_or_double*/) + { + // get number of digits for a float -> text -> float round-trip + static constexpr auto d = std::numeric_limits<number_float_t>::max_digits10; // the actual conversion - std::ptrdiff_t len = snprintf(number_buffer.data(), number_buffer.size(), "%.*g", d, x); + std::ptrdiff_t len = (std::snprintf)(number_buffer.data(), number_buffer.size(), "%.*g", d, x); // negative value indicates an error assert(len > 0); @@ -6744,15 +11554,16 @@ class serializer followed. @param[in,out] state the state of the decoding + @param[in,out] codep codepoint (valid only if resulting state is UTF8_ACCEPT) @param[in] byte next byte to decode + @return new state - @note The function has been edited: a std::array is used and the code - point is not calculated. + @note The function has been edited: a std::array is used. @copyright Copyright (c) 2008-2009 Bjoern Hoehrmann <bjoern@hoehrmann.de> @sa http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ */ - static void decode(uint8_t& state, const uint8_t byte) + static uint8_t decode(uint8_t& state, uint32_t& codep, const uint8_t byte) noexcept { static const std::array<uint8_t, 400> utf8d = { @@ -6775,42 +11586,13 @@ class serializer }; const uint8_t type = utf8d[byte]; - state = utf8d[256u + state * 16u + type]; - } - /*! - @brief throw an exception if a string is not UTF-8 encoded - - @param[in] str UTF-8 string to check - @throw type_error.316 if passed string is not UTF-8 encoded + codep = (state != UTF8_ACCEPT) + ? (byte & 0x3fu) | (codep << 6) + : static_cast<uint32_t>(0xff >> type) & (byte); - @since version 3.0.0 - */ - static void throw_if_invalid_utf8(const std::string& str) - { - // start with state 0 (= accept) - uint8_t state = 0; - - for (size_t i = 0; i < str.size(); ++i) - { - const auto byte = static_cast<uint8_t>(str[i]); - decode(state, byte); - if (state == 1) - { - // state 1 means reject - std::stringstream ss; - ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << static_cast<int>(byte); - JSON_THROW(type_error::create(316, "invalid UTF-8 byte at index " + std::to_string(i) + ": 0x" + ss.str())); - } - } - - if (state != 0) - { - // we finish reading, but do not accept: string was incomplete - std::stringstream ss; - ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << static_cast<int>(static_cast<uint8_t>(str.back())); - JSON_THROW(type_error::create(316, "incomplete UTF-8 string; last byte: 0x" + ss.str())); - } + state = utf8d[256u + state * 16u + type]; + return state; } private: @@ -6827,13 +11609,33 @@ class serializer /// the locale's decimal point character const char decimal_point = '\0'; + /// string buffer + std::array<char, 512> string_buffer{{}}; + /// the indentation character const char indent_char; - /// the indentation string string_t indent_string; + + /// error_handler how to react on decoding errors + const error_handler_t error_handler; }; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/json_ref.hpp> + +#include <initializer_list> +#include <utility> + +// #include <nlohmann/detail/meta/type_traits.hpp> + + +namespace nlohmann +{ +namespace detail +{ template<typename BasicJsonType> class json_ref { @@ -6852,15 +11654,19 @@ class json_ref : owned_value(init), value_ref(&owned_value), is_rvalue(true) {} - template<class... Args> - json_ref(Args&& ... args) - : owned_value(std::forward<Args>(args)...), value_ref(&owned_value), is_rvalue(true) - {} + template < + class... Args, + enable_if_t<std::is_constructible<value_type, Args...>::value, int> = 0 > + json_ref(Args && ... args) + : owned_value(std::forward<Args>(args)...), value_ref(&owned_value), + is_rvalue(true) {} // class should be movable only json_ref(json_ref&&) = default; json_ref(const json_ref&) = delete; json_ref& operator=(const json_ref&) = delete; + json_ref& operator=(json_ref&&) = delete; + ~json_ref() = default; value_type moved_or_copied() const { @@ -6886,74 +11692,30 @@ class json_ref value_type* value_ref = nullptr; const bool is_rvalue; }; +} // namespace detail +} // namespace nlohmann -} // namespace detail - -/// namespace to hold default `to_json` / `from_json` functions -namespace -{ -constexpr const auto& to_json = detail::static_const<detail::to_json_fn>::value; -constexpr const auto& from_json = detail::static_const<detail::from_json_fn>::value; -} - - -/*! -@brief default JSONSerializer template argument - -This serializer ignores the template arguments and uses ADL -([argument-dependent lookup](http://en.cppreference.com/w/cpp/language/adl)) -for serialization. -*/ -template<typename, typename> -struct adl_serializer -{ - /*! - @brief convert a JSON value to any value type - - This function is usually called by the `get()` function of the - @ref basic_json class (either explicit or via conversion operators). - - @param[in] j JSON value to read from - @param[in,out] val value to write to - */ - template<typename BasicJsonType, typename ValueType> - static void from_json(BasicJsonType&& j, ValueType& val) noexcept( - noexcept(::nlohmann::from_json(std::forward<BasicJsonType>(j), val))) - { - ::nlohmann::from_json(std::forward<BasicJsonType>(j), val); - } +// #include <nlohmann/detail/json_pointer.hpp> - /*! - @brief convert any value type to a JSON value - This function is usually called by the constructors of the @ref basic_json - class. +#include <cassert> // assert +#include <numeric> // accumulate +#include <string> // string +#include <vector> // vector - @param[in,out] j JSON value to write to - @param[in] val value to read from - */ - template<typename BasicJsonType, typename ValueType> - static void to_json(BasicJsonType& j, ValueType&& val) noexcept( - noexcept(::nlohmann::to_json(j, std::forward<ValueType>(val)))) - { - ::nlohmann::to_json(j, std::forward<ValueType>(val)); - } -}; +// #include <nlohmann/detail/macro_scope.hpp> -/*! -@brief JSON Pointer +// #include <nlohmann/detail/exceptions.hpp> -A JSON pointer defines a string syntax for identifying a specific value -within a JSON document. It can be used with functions `at` and -`operator[]`. Furthermore, JSON pointers are the base for JSON patches. +// #include <nlohmann/detail/value_t.hpp> -@sa [RFC 6901](https://tools.ietf.org/html/rfc6901) -@since version 2.0.0 -*/ +namespace nlohmann +{ +template<typename BasicJsonType> class json_pointer { - /// allow basic_json to access private members + // allow basic_json to access private members NLOHMANN_BASIC_JSON_TPL_DECLARATION friend class basic_json; @@ -6967,19 +11729,21 @@ class json_pointer @param[in] s string representing the JSON pointer; if omitted, the empty string is assumed which references the whole JSON value - @throw parse_error.107 if the given JSON pointer @a s is nonempty and - does not begin with a slash (`/`); see example below + @throw parse_error.107 if the given JSON pointer @a s is nonempty and does + not begin with a slash (`/`); see example below - @throw parse_error.108 if a tilde (`~`) in the given JSON pointer @a s - is not followed by `0` (representing `~`) or `1` (representing `/`); - see example below + @throw parse_error.108 if a tilde (`~`) in the given JSON pointer @a s is + not followed by `0` (representing `~`) or `1` (representing `/`); see + example below - @liveexample{The example shows the construction several valid JSON - pointers as well as the exceptional behavior.,json_pointer} + @liveexample{The example shows the construction several valid JSON pointers + as well as the exceptional behavior.,json_pointer} @since version 2.0.0 */ - explicit json_pointer(const std::string& s = "") : reference_tokens(split(s)) {} + explicit json_pointer(const std::string& s = "") + : reference_tokens(split(s)) + {} /*! @brief return a string representation of the JSON pointer @@ -6996,7 +11760,7 @@ class json_pointer @since version 2.0.0 */ - std::string to_string() const noexcept + std::string to_string() const { return std::accumulate(reference_tokens.begin(), reference_tokens.end(), std::string{}, @@ -7021,7 +11785,7 @@ class json_pointer */ static int array_index(const std::string& s) { - size_t processed_chars = 0; + std::size_t processed_chars = 0; const int res = std::stoi(s, &processed_chars); // check if the string was completely read @@ -7051,7 +11815,7 @@ class json_pointer } /// return whether pointer points to the root document - bool is_root() const + bool is_root() const noexcept { return reference_tokens.empty(); } @@ -7076,8 +11840,66 @@ class json_pointer @throw parse_error.109 if array index is not a number @throw type_error.313 if value cannot be unflattened */ - NLOHMANN_BASIC_JSON_TPL_DECLARATION - NLOHMANN_BASIC_JSON_TPL& get_and_create(NLOHMANN_BASIC_JSON_TPL& j) const; + BasicJsonType& get_and_create(BasicJsonType& j) const + { + using size_type = typename BasicJsonType::size_type; + auto result = &j; + + // in case no reference tokens exist, return a reference to the JSON value + // j which will be overwritten by a primitive value + for (const auto& reference_token : reference_tokens) + { + switch (result->m_type) + { + case detail::value_t::null: + { + if (reference_token == "0") + { + // start a new array if reference token is 0 + result = &result->operator[](0); + } + else + { + // start a new object otherwise + result = &result->operator[](reference_token); + } + break; + } + + case detail::value_t::object: + { + // create an entry in the object + result = &result->operator[](reference_token); + break; + } + + case detail::value_t::array: + { + // create an entry in the array + JSON_TRY + { + result = &result->operator[](static_cast<size_type>(array_index(reference_token))); + } + JSON_CATCH(std::invalid_argument&) + { + JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } + break; + } + + /* + The following code is only reached if there exists a reference + token _and_ the current value is primitive. In this case, we have + an error situation, because primitive values may only occur as + single value; that is, with an empty list of reference tokens. + */ + default: + JSON_THROW(detail::type_error::create(313, "invalid value to unflatten")); + } + } + + return *result; + } /*! @brief return a reference to the pointed to value @@ -7098,8 +11920,75 @@ class json_pointer @throw parse_error.109 if an array index was not a number @throw out_of_range.404 if the JSON pointer can not be resolved */ - NLOHMANN_BASIC_JSON_TPL_DECLARATION - NLOHMANN_BASIC_JSON_TPL& get_unchecked(NLOHMANN_BASIC_JSON_TPL* ptr) const; + BasicJsonType& get_unchecked(BasicJsonType* ptr) const + { + using size_type = typename BasicJsonType::size_type; + for (const auto& reference_token : reference_tokens) + { + // convert null values to arrays or objects before continuing + if (ptr->m_type == detail::value_t::null) + { + // check if reference token is a number + const bool nums = + std::all_of(reference_token.begin(), reference_token.end(), + [](const char x) + { + return (x >= '0' and x <= '9'); + }); + + // change value to array for numbers or "-" or to object otherwise + *ptr = (nums or reference_token == "-") + ? detail::value_t::array + : detail::value_t::object; + } + + switch (ptr->m_type) + { + case detail::value_t::object: + { + // use unchecked object access + ptr = &ptr->operator[](reference_token); + break; + } + + case detail::value_t::array: + { + // error condition (cf. RFC 6901, Sect. 4) + if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0')) + { + JSON_THROW(detail::parse_error::create(106, 0, + "array index '" + reference_token + + "' must not begin with '0'")); + } + + if (reference_token == "-") + { + // explicitly treat "-" as index beyond the end + ptr = &ptr->operator[](ptr->m_value.array->size()); + } + else + { + // convert array index to number; unchecked access + JSON_TRY + { + ptr = &ptr->operator[]( + static_cast<size_type>(array_index(reference_token))); + } + JSON_CATCH(std::invalid_argument&) + { + JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } + } + break; + } + + default: + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); + } + } + + return *ptr; + } /*! @throw parse_error.106 if an array index begins with '0' @@ -7107,8 +11996,57 @@ class json_pointer @throw out_of_range.402 if the array index '-' is used @throw out_of_range.404 if the JSON pointer can not be resolved */ - NLOHMANN_BASIC_JSON_TPL_DECLARATION - NLOHMANN_BASIC_JSON_TPL& get_checked(NLOHMANN_BASIC_JSON_TPL* ptr) const; + BasicJsonType& get_checked(BasicJsonType* ptr) const + { + using size_type = typename BasicJsonType::size_type; + for (const auto& reference_token : reference_tokens) + { + switch (ptr->m_type) + { + case detail::value_t::object: + { + // note: at performs range check + ptr = &ptr->at(reference_token); + break; + } + + case detail::value_t::array: + { + if (JSON_UNLIKELY(reference_token == "-")) + { + // "-" always fails the range check + JSON_THROW(detail::out_of_range::create(402, + "array index '-' (" + std::to_string(ptr->m_value.array->size()) + + ") is out of range")); + } + + // error condition (cf. RFC 6901, Sect. 4) + if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0')) + { + JSON_THROW(detail::parse_error::create(106, 0, + "array index '" + reference_token + + "' must not begin with '0'")); + } + + // note: at performs range check + JSON_TRY + { + ptr = &ptr->at(static_cast<size_type>(array_index(reference_token))); + } + JSON_CATCH(std::invalid_argument&) + { + JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } + break; + } + + default: + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); + } + } + + return *ptr; + } /*! @brief return a const reference to the pointed to value @@ -7123,8 +12061,58 @@ class json_pointer @throw out_of_range.402 if the array index '-' is used @throw out_of_range.404 if the JSON pointer can not be resolved */ - NLOHMANN_BASIC_JSON_TPL_DECLARATION - const NLOHMANN_BASIC_JSON_TPL& get_unchecked(const NLOHMANN_BASIC_JSON_TPL* ptr) const; + const BasicJsonType& get_unchecked(const BasicJsonType* ptr) const + { + using size_type = typename BasicJsonType::size_type; + for (const auto& reference_token : reference_tokens) + { + switch (ptr->m_type) + { + case detail::value_t::object: + { + // use unchecked object access + ptr = &ptr->operator[](reference_token); + break; + } + + case detail::value_t::array: + { + if (JSON_UNLIKELY(reference_token == "-")) + { + // "-" cannot be used for const access + JSON_THROW(detail::out_of_range::create(402, + "array index '-' (" + std::to_string(ptr->m_value.array->size()) + + ") is out of range")); + } + + // error condition (cf. RFC 6901, Sect. 4) + if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0')) + { + JSON_THROW(detail::parse_error::create(106, 0, + "array index '" + reference_token + + "' must not begin with '0'")); + } + + // use unchecked array access + JSON_TRY + { + ptr = &ptr->operator[]( + static_cast<size_type>(array_index(reference_token))); + } + JSON_CATCH(std::invalid_argument&) + { + JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } + break; + } + + default: + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); + } + } + + return *ptr; + } /*! @throw parse_error.106 if an array index begins with '0' @@ -7132,8 +12120,57 @@ class json_pointer @throw out_of_range.402 if the array index '-' is used @throw out_of_range.404 if the JSON pointer can not be resolved */ - NLOHMANN_BASIC_JSON_TPL_DECLARATION - const NLOHMANN_BASIC_JSON_TPL& get_checked(const NLOHMANN_BASIC_JSON_TPL* ptr) const; + const BasicJsonType& get_checked(const BasicJsonType* ptr) const + { + using size_type = typename BasicJsonType::size_type; + for (const auto& reference_token : reference_tokens) + { + switch (ptr->m_type) + { + case detail::value_t::object: + { + // note: at performs range check + ptr = &ptr->at(reference_token); + break; + } + + case detail::value_t::array: + { + if (JSON_UNLIKELY(reference_token == "-")) + { + // "-" always fails the range check + JSON_THROW(detail::out_of_range::create(402, + "array index '-' (" + std::to_string(ptr->m_value.array->size()) + + ") is out of range")); + } + + // error condition (cf. RFC 6901, Sect. 4) + if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0')) + { + JSON_THROW(detail::parse_error::create(106, 0, + "array index '" + reference_token + + "' must not begin with '0'")); + } + + // note: at performs range check + JSON_TRY + { + ptr = &ptr->at(static_cast<size_type>(array_index(reference_token))); + } + JSON_CATCH(std::invalid_argument&) + { + JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } + break; + } + + default: + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); + } + } + + return *ptr; + } /*! @brief split the string input to reference tokens @@ -7170,11 +12207,11 @@ class json_pointer std::size_t slash = reference_string.find_first_of('/', 1), // set the beginning of the first reference token start = 1; - // we can stop if start == string::npos+1 = 0 + // we can stop if start == 0 (if slash == std::string::npos) start != 0; // set the beginning of the next reference token // (will eventually be 0 if slash == std::string::npos) - start = slash + 1, + start = (slash == std::string::npos) ? 0 : slash + 1, // find next slash slash = reference_string.find_first_of('/', start)) { @@ -7230,7 +12267,7 @@ class json_pointer {} } - /// escape "~"" to "~0" and "/" to "~1" + /// escape "~" to "~0" and "/" to "~1" static std::string escape(std::string s) { replace_substring(s, "~", "~0"); @@ -7252,10 +12289,57 @@ class json_pointer @note Empty objects or arrays are flattened to `null`. */ - NLOHMANN_BASIC_JSON_TPL_DECLARATION static void flatten(const std::string& reference_string, - const NLOHMANN_BASIC_JSON_TPL& value, - NLOHMANN_BASIC_JSON_TPL& result); + const BasicJsonType& value, + BasicJsonType& result) + { + switch (value.m_type) + { + case detail::value_t::array: + { + if (value.m_value.array->empty()) + { + // flatten empty array as null + result[reference_string] = nullptr; + } + else + { + // iterate array and use index as reference string + for (std::size_t i = 0; i < value.m_value.array->size(); ++i) + { + flatten(reference_string + "/" + std::to_string(i), + value.m_value.array->operator[](i), result); + } + } + break; + } + + case detail::value_t::object: + { + if (value.m_value.object->empty()) + { + // flatten empty object as null + result[reference_string] = nullptr; + } + else + { + // iterate object and use keys as reference string + for (const auto& element : *value.m_value.object) + { + flatten(reference_string + "/" + escape(element.first), element.second, result); + } + } + break; + } + + default: + { + // add primitive value with its reference string + result[reference_string] = value; + break; + } + } + } /*! @param[in] value flattened JSON @@ -7267,19 +12351,112 @@ class json_pointer @throw type_error.315 if object values are not primitive @throw type_error.313 if value cannot be unflattened */ - NLOHMANN_BASIC_JSON_TPL_DECLARATION - static NLOHMANN_BASIC_JSON_TPL - unflatten(const NLOHMANN_BASIC_JSON_TPL& value); + static BasicJsonType + unflatten(const BasicJsonType& value) + { + if (JSON_UNLIKELY(not value.is_object())) + { + JSON_THROW(detail::type_error::create(314, "only objects can be unflattened")); + } + + BasicJsonType result; + + // iterate the JSON object values + for (const auto& element : *value.m_value.object) + { + if (JSON_UNLIKELY(not element.second.is_primitive())) + { + JSON_THROW(detail::type_error::create(315, "values in object must be primitive")); + } + + // assign value to reference pointed to by JSON pointer; Note that if + // the JSON pointer is "" (i.e., points to the whole value), function + // get_and_create returns a reference to result itself. An assignment + // will then create a primitive value. + json_pointer(element.first).get_and_create(result) = element.second; + } + + return result; + } friend bool operator==(json_pointer const& lhs, - json_pointer const& rhs) noexcept; + json_pointer const& rhs) noexcept + { + return (lhs.reference_tokens == rhs.reference_tokens); + } friend bool operator!=(json_pointer const& lhs, - json_pointer const& rhs) noexcept; + json_pointer const& rhs) noexcept + { + return not (lhs == rhs); + } /// the reference tokens std::vector<std::string> reference_tokens; }; +} // namespace nlohmann + +// #include <nlohmann/adl_serializer.hpp> + + +#include <utility> + +// #include <nlohmann/detail/conversions/from_json.hpp> + +// #include <nlohmann/detail/conversions/to_json.hpp> + + +namespace nlohmann +{ + +template<typename, typename> +struct adl_serializer +{ + /*! + @brief convert a JSON value to any value type + + This function is usually called by the `get()` function of the + @ref basic_json class (either explicit or via conversion operators). + + @param[in] j JSON value to read from + @param[in,out] val value to write to + */ + template<typename BasicJsonType, typename ValueType> + static auto from_json(BasicJsonType&& j, ValueType& val) noexcept( + noexcept(::nlohmann::from_json(std::forward<BasicJsonType>(j), val))) + -> decltype(::nlohmann::from_json(std::forward<BasicJsonType>(j), val), void()) + { + ::nlohmann::from_json(std::forward<BasicJsonType>(j), val); + } + + /*! + @brief convert any value type to a JSON value + + This function is usually called by the constructors of the @ref basic_json + class. + + @param[in,out] j JSON value to write to + @param[in] val value to read from + */ + template <typename BasicJsonType, typename ValueType> + static auto to_json(BasicJsonType& j, ValueType&& val) noexcept( + noexcept(::nlohmann::to_json(j, std::forward<ValueType>(val)))) + -> decltype(::nlohmann::to_json(j, std::forward<ValueType>(val)), void()) + { + ::nlohmann::to_json(j, std::forward<ValueType>(val)); + } +}; + +} // namespace nlohmann + + +/*! +@brief namespace for Niels Lohmann +@see https://github.com/nlohmann +@since version 1.0.0 +*/ +namespace nlohmann +{ /*! @brief a class to store JSON values @@ -7305,42 +12482,42 @@ and `from_json()` (@ref adl_serializer by default) @requirement The class satisfies the following concept requirements: - Basic - - [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible): + - [DefaultConstructible](https://en.cppreference.com/w/cpp/named_req/DefaultConstructible): JSON values can be default constructed. The result will be a JSON null value. - - [MoveConstructible](http://en.cppreference.com/w/cpp/concept/MoveConstructible): + - [MoveConstructible](https://en.cppreference.com/w/cpp/named_req/MoveConstructible): A JSON value can be constructed from an rvalue argument. - - [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible): + - [CopyConstructible](https://en.cppreference.com/w/cpp/named_req/CopyConstructible): A JSON value can be copy-constructed from an lvalue expression. - - [MoveAssignable](http://en.cppreference.com/w/cpp/concept/MoveAssignable): + - [MoveAssignable](https://en.cppreference.com/w/cpp/named_req/MoveAssignable): A JSON value van be assigned from an rvalue argument. - - [CopyAssignable](http://en.cppreference.com/w/cpp/concept/CopyAssignable): + - [CopyAssignable](https://en.cppreference.com/w/cpp/named_req/CopyAssignable): A JSON value can be copy-assigned from an lvalue expression. - - [Destructible](http://en.cppreference.com/w/cpp/concept/Destructible): + - [Destructible](https://en.cppreference.com/w/cpp/named_req/Destructible): JSON values can be destructed. - Layout - - [StandardLayoutType](http://en.cppreference.com/w/cpp/concept/StandardLayoutType): + - [StandardLayoutType](https://en.cppreference.com/w/cpp/named_req/StandardLayoutType): JSON values have - [standard layout](http://en.cppreference.com/w/cpp/language/data_members#Standard_layout): + [standard layout](https://en.cppreference.com/w/cpp/language/data_members#Standard_layout): All non-static data members are private and standard layout types, the class has no virtual functions or (virtual) base classes. - Library-wide - - [EqualityComparable](http://en.cppreference.com/w/cpp/concept/EqualityComparable): + - [EqualityComparable](https://en.cppreference.com/w/cpp/named_req/EqualityComparable): JSON values can be compared with `==`, see @ref operator==(const_reference,const_reference). - - [LessThanComparable](http://en.cppreference.com/w/cpp/concept/LessThanComparable): + - [LessThanComparable](https://en.cppreference.com/w/cpp/named_req/LessThanComparable): JSON values can be compared with `<`, see @ref operator<(const_reference,const_reference). - - [Swappable](http://en.cppreference.com/w/cpp/concept/Swappable): + - [Swappable](https://en.cppreference.com/w/cpp/named_req/Swappable): Any JSON lvalue or rvalue of can be swapped with any lvalue or rvalue of other compatible types, using unqualified function call @ref swap(). - - [NullablePointer](http://en.cppreference.com/w/cpp/concept/NullablePointer): + - [NullablePointer](https://en.cppreference.com/w/cpp/named_req/NullablePointer): JSON values can be compared against `std::nullptr_t` objects which are used to model the `null` value. - Container - - [Container](http://en.cppreference.com/w/cpp/concept/Container): + - [Container](https://en.cppreference.com/w/cpp/named_req/Container): JSON values can be used like STL containers and provide iterator access. - - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer); + - [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer); JSON values can be used like STL containers and provide reverse iterator access. @@ -7367,15 +12544,19 @@ class basic_json { private: template<detail::value_t> friend struct detail::external_constructor; - friend ::nlohmann::json_pointer; + friend ::nlohmann::json_pointer<basic_json>; friend ::nlohmann::detail::parser<basic_json>; friend ::nlohmann::detail::serializer<basic_json>; template<typename BasicJsonType> friend class ::nlohmann::detail::iter_impl; template<typename BasicJsonType, typename CharType> friend class ::nlohmann::detail::binary_writer; - template<typename BasicJsonType> + template<typename BasicJsonType, typename SAX> friend class ::nlohmann::detail::binary_reader; + template<typename BasicJsonType> + friend class ::nlohmann::detail::json_sax_dom_parser; + template<typename BasicJsonType> + friend class ::nlohmann::detail::json_sax_dom_callback_parser; /// workaround type for MSVC using basic_json_t = NLOHMANN_BASIC_JSON_TPL; @@ -7403,13 +12584,19 @@ class basic_json public: using value_t = detail::value_t; - /// @copydoc nlohmann::json_pointer - using json_pointer = ::nlohmann::json_pointer; + /// JSON Pointer, see @ref nlohmann::json_pointer + using json_pointer = ::nlohmann::json_pointer<basic_json>; template<typename T, typename SFINAE> using json_serializer = JSONSerializer<T, SFINAE>; + /// how to treat decoding errors + using error_handler_t = detail::error_handler_t; /// helper type for initializer lists of basic_json values using initializer_list_t = std::initializer_list<detail::json_ref<basic_json>>; + using input_format_t = detail::input_format_t; + /// SAX interface type, see @ref nlohmann::json_sax + using json_sax_t = json_sax<basic_json>; + //////////////// // exceptions // //////////////// @@ -7517,10 +12704,13 @@ class basic_json result["copyright"] = "(C) 2013-2017 Niels Lohmann"; result["name"] = "JSON for Modern C++"; result["url"] = "https://github.com/nlohmann/json"; - result["version"] = - { - {"string", "3.0.1"}, {"major", 3}, {"minor", 0}, {"patch", 1} - }; + result["version"]["string"] = + std::to_string(NLOHMANN_JSON_VERSION_MAJOR) + "." + + std::to_string(NLOHMANN_JSON_VERSION_MINOR) + "." + + std::to_string(NLOHMANN_JSON_VERSION_PATCH); + result["version"]["major"] = NLOHMANN_JSON_VERSION_MAJOR; + result["version"]["minor"] = NLOHMANN_JSON_VERSION_MINOR; + result["version"]["patch"] = NLOHMANN_JSON_VERSION_PATCH; #ifdef _WIN32 result["platform"] = "win32"; @@ -7622,10 +12812,10 @@ class basic_json - When all names are unique, objects will be interoperable in the sense that all software implementations receiving that object will agree on the name-value mappings. - - When the names within an object are not unique, later stored name/value - pairs overwrite previously stored name/value pairs, leaving the used - names unique. For instance, `{"key": 1}` and `{"key": 2, "key": 1}` will - be treated as equal and both stored as `{"key": 1}`. + - When the names within an object are not unique, it is unspecified which + one of the values for a given key will be chosen. For instance, + `{"key": 2, "key": 1}` could be equal to either `{"key": 1}` or + `{"key": 2}`. - Internally, name/value pairs are stored in lexicographical order of the names. Objects will also be serialized (see @ref dump) in this order. For instance, `{"b": 1, "a": 2}` and `{"a": 2, "b": 1}` will be stored @@ -8139,7 +13329,7 @@ class basic_json object = nullptr; // silence warning, see #821 if (JSON_UNLIKELY(t == value_t::null)) { - JSON_THROW(other_error::create(500, "961c151d2e87f2686a955a9be24d316f1362bf21 3.0.1")); // LCOV_EXCL_LINE + JSON_THROW(other_error::create(500, "961c151d2e87f2686a955a9be24d316f1362bf21 3.5.0")); // LCOV_EXCL_LINE } break; } @@ -8182,7 +13372,7 @@ class basic_json array = create<array_t>(std::move(value)); } - void destroy(value_t t) + void destroy(value_t t) noexcept { switch (t) { @@ -8227,7 +13417,7 @@ class basic_json value is changed, because the invariant expresses a relationship between @a m_type and @a m_value. */ - void assert_invariant() const + void assert_invariant() const noexcept { assert(m_type != value_t::object or m_value.object != nullptr); assert(m_type != value_t::array or m_value.array != nullptr); @@ -8307,7 +13497,6 @@ class basic_json */ using parser_callback_t = typename parser::parser_callback_t; - ////////////////// // constructors // ////////////////// @@ -8409,6 +13598,7 @@ class basic_json - @a CompatibleType is not derived from `std::istream`, - @a CompatibleType is not @ref basic_json (to avoid hijacking copy/move constructors), + - @a CompatibleType is not a different @ref basic_json type (i.e. with different template arguments) - @a CompatibleType is not a @ref basic_json nested type (e.g., @ref json_pointer, @ref iterator, etc ...) - @ref @ref json_serializer<U> has a @@ -8432,21 +13622,91 @@ class basic_json @since version 2.1.0 */ - template<typename CompatibleType, typename U = detail::uncvref_t<CompatibleType>, - detail::enable_if_t<not std::is_base_of<std::istream, U>::value and - not std::is_same<U, basic_json_t>::value and - not detail::is_basic_json_nested_type< - basic_json_t, U>::value and - detail::has_to_json<basic_json, U>::value, - int> = 0> - basic_json(CompatibleType && val) noexcept(noexcept(JSONSerializer<U>::to_json( - std::declval<basic_json_t&>(), std::forward<CompatibleType>(val)))) + template <typename CompatibleType, + typename U = detail::uncvref_t<CompatibleType>, + detail::enable_if_t< + not detail::is_basic_json<U>::value and detail::is_compatible_type<basic_json_t, U>::value, int> = 0> + basic_json(CompatibleType && val) noexcept(noexcept( + JSONSerializer<U>::to_json(std::declval<basic_json_t&>(), + std::forward<CompatibleType>(val)))) { JSONSerializer<U>::to_json(*this, std::forward<CompatibleType>(val)); assert_invariant(); } /*! + @brief create a JSON value from an existing one + + This is a constructor for existing @ref basic_json types. + It does not hijack copy/move constructors, since the parameter has different + template arguments than the current ones. + + The constructor tries to convert the internal @ref m_value of the parameter. + + @tparam BasicJsonType a type such that: + - @a BasicJsonType is a @ref basic_json type. + - @a BasicJsonType has different template arguments than @ref basic_json_t. + + @param[in] val the @ref basic_json value to be converted. + + @complexity Usually linear in the size of the passed @a val, also + depending on the implementation of the called `to_json()` + method. + + @exceptionsafety Depends on the called constructor. For types directly + supported by the library (i.e., all types for which no `to_json()` function + was provided), strong guarantee holds: if an exception is thrown, there are + no changes to any JSON value. + + @since version 3.2.0 + */ + template <typename BasicJsonType, + detail::enable_if_t< + detail::is_basic_json<BasicJsonType>::value and not std::is_same<basic_json, BasicJsonType>::value, int> = 0> + basic_json(const BasicJsonType& val) + { + using other_boolean_t = typename BasicJsonType::boolean_t; + using other_number_float_t = typename BasicJsonType::number_float_t; + using other_number_integer_t = typename BasicJsonType::number_integer_t; + using other_number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using other_string_t = typename BasicJsonType::string_t; + using other_object_t = typename BasicJsonType::object_t; + using other_array_t = typename BasicJsonType::array_t; + + switch (val.type()) + { + case value_t::boolean: + JSONSerializer<other_boolean_t>::to_json(*this, val.template get<other_boolean_t>()); + break; + case value_t::number_float: + JSONSerializer<other_number_float_t>::to_json(*this, val.template get<other_number_float_t>()); + break; + case value_t::number_integer: + JSONSerializer<other_number_integer_t>::to_json(*this, val.template get<other_number_integer_t>()); + break; + case value_t::number_unsigned: + JSONSerializer<other_number_unsigned_t>::to_json(*this, val.template get<other_number_unsigned_t>()); + break; + case value_t::string: + JSONSerializer<other_string_t>::to_json(*this, val.template get_ref<const other_string_t&>()); + break; + case value_t::object: + JSONSerializer<other_object_t>::to_json(*this, val.template get_ref<const other_object_t&>()); + break; + case value_t::array: + JSONSerializer<other_array_t>::to_json(*this, val.template get_ref<const other_array_t&>()); + break; + case value_t::null: + *this = nullptr; + break; + case value_t::discarded: + m_type = value_t::discarded; + break; + } + assert_invariant(); + } + + /*! @brief create a container (array or object) from an initializer list Creates a JSON value of type array or object from the passed initializer @@ -8718,7 +13978,7 @@ class basic_json @warning A precondition is enforced with a runtime assertion that will result in calling `std::abort` if this precondition is not met. Assertions can be disabled by defining `NDEBUG` at compile time. - See http://en.cppreference.com/w/cpp/error/assert for more + See https://en.cppreference.com/w/cpp/error/assert for more information. @throw invalid_iterator.201 if iterators @a first and @a last are not @@ -8858,7 +14118,7 @@ class basic_json changes to any JSON value. @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) + [Container](https://en.cppreference.com/w/cpp/named_req/Container) requirements: - The complexity is linear. - As postcondition, it holds: `other == basic_json(other)`. @@ -8943,7 +14203,7 @@ class basic_json exceptions. @requirement This function helps `basic_json` satisfying the - [MoveConstructible](http://en.cppreference.com/w/cpp/concept/MoveConstructible) + [MoveConstructible](https://en.cppreference.com/w/cpp/named_req/MoveConstructible) requirements. @liveexample{The code below shows the move constructor explicitly called @@ -8977,7 +14237,7 @@ class basic_json @complexity Linear. @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) + [Container](https://en.cppreference.com/w/cpp/named_req/Container) requirements: - The complexity is linear. @@ -8988,7 +14248,7 @@ class basic_json @since version 1.0.0 */ - reference& operator=(basic_json other) noexcept ( + basic_json& operator=(basic_json other) noexcept ( std::is_nothrow_move_constructible<value_t>::value and std::is_nothrow_move_assignable<value_t>::value and std::is_nothrow_move_constructible<json_value>::value and @@ -9014,14 +14274,14 @@ class basic_json @complexity Linear. @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) + [Container](https://en.cppreference.com/w/cpp/named_req/Container) requirements: - The complexity is linear. - All stored elements are destroyed and all memory is freed. @since version 1.0.0 */ - ~basic_json() + ~basic_json() noexcept { assert_invariant(); m_value.destroy(m_type); @@ -9054,6 +14314,10 @@ class basic_json @param[in] ensure_ascii If @a ensure_ascii is true, all non-ASCII characters in the output are escaped with `\uXXXX` sequences, and the result consists of ASCII characters only. + @param[in] error_handler how to react on decoding errors; there are three + possible values: `strict` (throws and exception in case a decoding error + occurs; default), `replace` (replace invalid UTF-8 sequences with U+FFFD), + and `ignore` (ignore invalid UTF-8 sequences during serialization). @return string containing the serialization of the JSON value @@ -9072,13 +14336,16 @@ class basic_json @see https://docs.python.org/2/library/json.html#json.dump @since version 1.0.0; indentation character @a indent_char, option - @a ensure_ascii and exceptions added in version 3.0.0 + @a ensure_ascii and exceptions added in version 3.0.0; error + handlers added in version 3.4.0. */ - string_t dump(const int indent = -1, const char indent_char = ' ', - const bool ensure_ascii = false) const + string_t dump(const int indent = -1, + const char indent_char = ' ', + const bool ensure_ascii = false, + const error_handler_t error_handler = error_handler_t::strict) const { string_t result; - serializer s(detail::output_adapter<char>(result), indent_char); + serializer s(detail::output_adapter<char, string_t>(result), indent_char, error_handler); if (indent >= 0) { @@ -9619,11 +14886,34 @@ class basic_json } /*! + @brief get special-case overload + + This overloads converts the current @ref basic_json in a different + @ref basic_json type + + @tparam BasicJsonType == @ref basic_json + + @return a copy of *this, converted into @tparam BasicJsonType + + @complexity Depending on the implementation of the called `from_json()` + method. + + @since version 3.2.0 + */ + template<typename BasicJsonType, detail::enable_if_t< + not std::is_same<BasicJsonType, basic_json>::value and + detail::is_basic_json<BasicJsonType>::value, int> = 0> + BasicJsonType get() const + { + return *this; + } + + /*! @brief get a value (explicit) Explicit type conversion between the JSON value and a compatible value - which is [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible) - and [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible). + which is [CopyConstructible](https://en.cppreference.com/w/cpp/named_req/CopyConstructible) + and [DefaultConstructible](https://en.cppreference.com/w/cpp/named_req/DefaultConstructible). The value is converted by calling the @ref json_serializer<ValueType> `from_json()` method. @@ -9659,7 +14949,7 @@ class basic_json */ template<typename ValueTypeCV, typename ValueType = detail::uncvref_t<ValueTypeCV>, detail::enable_if_t < - not std::is_same<basic_json_t, ValueType>::value and + not detail::is_basic_json<ValueType>::value and detail::has_from_json<basic_json_t, ValueType>::value and not detail::has_non_default_from_json<basic_json_t, ValueType>::value, int> = 0> @@ -9683,8 +14973,8 @@ class basic_json @brief get a value (explicit); special case Explicit type conversion between the JSON value and a compatible value - which is **not** [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible) - and **not** [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible). + which is **not** [CopyConstructible](https://en.cppreference.com/w/cpp/named_req/CopyConstructible) + and **not** [DefaultConstructible](https://en.cppreference.com/w/cpp/named_req/DefaultConstructible). The value is converted by calling the @ref json_serializer<ValueType> `from_json()` method. @@ -9723,17 +15013,64 @@ class basic_json } /*! - @brief get a pointer value (explicit) + @brief get a value (explicit) - Explicit pointer access to the internally stored JSON value. No copies are + Explicit type conversion between the JSON value and a compatible value. + The value is filled into the input parameter by calling the @ref json_serializer<ValueType> + `from_json()` method. + + The function is equivalent to executing + @code {.cpp} + ValueType v; + JSONSerializer<ValueType>::from_json(*this, v); + @endcode + + This overloads is chosen if: + - @a ValueType is not @ref basic_json, + - @ref json_serializer<ValueType> has a `from_json()` method of the form + `void from_json(const basic_json&, ValueType&)`, and + + @tparam ValueType the input parameter type. + + @return the input parameter, allowing chaining calls. + + @throw what @ref json_serializer<ValueType> `from_json()` method throws + + @liveexample{The example below shows several conversions from JSON values + to other types. There a few things to note: (1) Floating-point numbers can + be converted to integers\, (2) A JSON array can be converted to a standard + `std::vector<short>`\, (3) A JSON object can be converted to C++ + associative containers such as `std::unordered_map<std::string\, + json>`.,get_to} + + @since version 3.3.0 + */ + template<typename ValueType, + detail::enable_if_t < + not detail::is_basic_json<ValueType>::value and + detail::has_from_json<basic_json_t, ValueType>::value, + int> = 0> + ValueType & get_to(ValueType& v) const noexcept(noexcept( + JSONSerializer<ValueType>::from_json(std::declval<const basic_json_t&>(), v))) + { + JSONSerializer<ValueType>::from_json(*this, v); + return v; + } + + + /*! + @brief get a pointer value (implicit) + + Implicit pointer access to the internally stored JSON value. No copies are made. - @warning The pointer becomes invalid if the underlying JSON object - changes. + @warning Writing data to the pointee of the result yields an undefined + state. @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, - @ref number_unsigned_t, or @ref number_float_t. + @ref number_unsigned_t, or @ref number_float_t. Enforced by a static + assertion. @return pointer to the internally stored JSON value if the requested pointer type @a PointerType fits to the JSON value; `nullptr` otherwise @@ -9743,45 +15080,43 @@ class basic_json @liveexample{The example below shows how pointers to internal values of a JSON value can be requested. Note that no type conversions are made and a `nullptr` is returned if the value and the requested pointer type does not - match.,get__PointerType} - - @sa @ref get_ptr() for explicit pointer-member access + match.,get_ptr} @since version 1.0.0 */ template<typename PointerType, typename std::enable_if< std::is_pointer<PointerType>::value, int>::type = 0> - PointerType get() noexcept + auto get_ptr() noexcept -> decltype(std::declval<basic_json_t&>().get_impl_ptr(std::declval<PointerType>())) { - // delegate the call to get_ptr - return get_ptr<PointerType>(); + // delegate the call to get_impl_ptr<>() + return get_impl_ptr(static_cast<PointerType>(nullptr)); } /*! - @brief get a pointer value (explicit) - @copydoc get() + @brief get a pointer value (implicit) + @copydoc get_ptr() */ template<typename PointerType, typename std::enable_if< - std::is_pointer<PointerType>::value, int>::type = 0> - constexpr const PointerType get() const noexcept + std::is_pointer<PointerType>::value and + std::is_const<typename std::remove_pointer<PointerType>::type>::value, int>::type = 0> + constexpr auto get_ptr() const noexcept -> decltype(std::declval<const basic_json_t&>().get_impl_ptr(std::declval<PointerType>())) { - // delegate the call to get_ptr - return get_ptr<PointerType>(); + // delegate the call to get_impl_ptr<>() const + return get_impl_ptr(static_cast<PointerType>(nullptr)); } /*! - @brief get a pointer value (implicit) + @brief get a pointer value (explicit) - Implicit pointer access to the internally stored JSON value. No copies are + Explicit pointer access to the internally stored JSON value. No copies are made. - @warning Writing data to the pointee of the result yields an undefined - state. + @warning The pointer becomes invalid if the underlying JSON object + changes. @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, - @ref number_unsigned_t, or @ref number_float_t. Enforced by a static - assertion. + @ref number_unsigned_t, or @ref number_float_t. @return pointer to the internally stored JSON value if the requested pointer type @a PointerType fits to the JSON value; `nullptr` otherwise @@ -9791,59 +15126,30 @@ class basic_json @liveexample{The example below shows how pointers to internal values of a JSON value can be requested. Note that no type conversions are made and a `nullptr` is returned if the value and the requested pointer type does not - match.,get_ptr} + match.,get__PointerType} + + @sa @ref get_ptr() for explicit pointer-member access @since version 1.0.0 */ template<typename PointerType, typename std::enable_if< std::is_pointer<PointerType>::value, int>::type = 0> - PointerType get_ptr() noexcept + auto get() noexcept -> decltype(std::declval<basic_json_t&>().template get_ptr<PointerType>()) { - // get the type of the PointerType (remove pointer and const) - using pointee_t = typename std::remove_const<typename - std::remove_pointer<typename - std::remove_const<PointerType>::type>::type>::type; - // make sure the type matches the allowed types - static_assert( - std::is_same<object_t, pointee_t>::value - or std::is_same<array_t, pointee_t>::value - or std::is_same<string_t, pointee_t>::value - or std::is_same<boolean_t, pointee_t>::value - or std::is_same<number_integer_t, pointee_t>::value - or std::is_same<number_unsigned_t, pointee_t>::value - or std::is_same<number_float_t, pointee_t>::value - , "incompatible pointer type"); - - // delegate the call to get_impl_ptr<>() - return get_impl_ptr(static_cast<PointerType>(nullptr)); + // delegate the call to get_ptr + return get_ptr<PointerType>(); } /*! - @brief get a pointer value (implicit) - @copydoc get_ptr() + @brief get a pointer value (explicit) + @copydoc get() */ template<typename PointerType, typename std::enable_if< - std::is_pointer<PointerType>::value and - std::is_const<typename std::remove_pointer<PointerType>::type>::value, int>::type = 0> - constexpr const PointerType get_ptr() const noexcept + std::is_pointer<PointerType>::value, int>::type = 0> + constexpr auto get() const noexcept -> decltype(std::declval<const basic_json_t&>().template get_ptr<PointerType>()) { - // get the type of the PointerType (remove pointer and const) - using pointee_t = typename std::remove_const<typename - std::remove_pointer<typename - std::remove_const<PointerType>::type>::type>::type; - // make sure the type matches the allowed types - static_assert( - std::is_same<object_t, pointee_t>::value - or std::is_same<array_t, pointee_t>::value - or std::is_same<string_t, pointee_t>::value - or std::is_same<boolean_t, pointee_t>::value - or std::is_same<number_integer_t, pointee_t>::value - or std::is_same<number_unsigned_t, pointee_t>::value - or std::is_same<number_float_t, pointee_t>::value - , "incompatible pointer type"); - - // delegate the call to get_impl_ptr<>() const - return get_impl_ptr(static_cast<PointerType>(nullptr)); + // delegate the call to get_ptr + return get_ptr<PointerType>(); } /*! @@ -9925,13 +15231,16 @@ class basic_json template < typename ValueType, typename std::enable_if < not std::is_pointer<ValueType>::value and not std::is_same<ValueType, detail::json_ref<basic_json>>::value and - not std::is_same<ValueType, typename string_t::value_type>::value + not std::is_same<ValueType, typename string_t::value_type>::value and + not detail::is_basic_json<ValueType>::value + #ifndef _MSC_VER // fix for issue #167 operator<< ambiguity under VS2015 and not std::is_same<ValueType, std::initializer_list<typename string_t::value_type>>::value -#endif -#if defined(JSON_HAS_CPP_17) +#if defined(JSON_HAS_CPP_17) && defined(_MSC_VER) and _MSC_VER <= 1914 and not std::is_same<ValueType, typename std::string_view>::value #endif +#endif + and detail::is_detected<detail::get_template_function, const basic_json_t&, ValueType>::value , int >::type = 0 > operator ValueType() const { @@ -10195,7 +15504,7 @@ class basic_json return m_value.array->operator[](idx); } - JSON_THROW(type_error::create(305, "cannot use operator[] with " + std::string(type_name()))); + JSON_THROW(type_error::create(305, "cannot use operator[] with a numeric argument with " + std::string(type_name()))); } /*! @@ -10225,7 +15534,7 @@ class basic_json return m_value.array->operator[](idx); } - JSON_THROW(type_error::create(305, "cannot use operator[] with " + std::string(type_name()))); + JSON_THROW(type_error::create(305, "cannot use operator[] with a numeric argument with " + std::string(type_name()))); } /*! @@ -10271,7 +15580,7 @@ class basic_json return m_value.object->operator[](key); } - JSON_THROW(type_error::create(305, "cannot use operator[] with " + std::string(type_name()))); + JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name()))); } /*! @@ -10313,7 +15622,7 @@ class basic_json return m_value.object->find(key)->second; } - JSON_THROW(type_error::create(305, "cannot use operator[] with " + std::string(type_name()))); + JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name()))); } /*! @@ -10360,7 +15669,7 @@ class basic_json return m_value.object->operator[](key); } - JSON_THROW(type_error::create(305, "cannot use operator[] with " + std::string(type_name()))); + JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name()))); } /*! @@ -10403,7 +15712,7 @@ class basic_json return m_value.object->find(key)->second; } - JSON_THROW(type_error::create(305, "cannot use operator[] with " + std::string(type_name()))); + JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name()))); } /*! @@ -10476,7 +15785,7 @@ class basic_json /*! @brief overload for a default value of type const char* - @copydoc basic_json::value(const typename object_t::key_type&, ValueType) const + @copydoc basic_json::value(const typename object_t::key_type&, const ValueType&) const */ string_t value(const typename object_t::key_type& key, const char* default_value) const { @@ -10512,7 +15821,7 @@ class basic_json @return copy of the element at key @a key or @a default_value if @a key is not found - @throw type_error.306 if the JSON value is not an objec; in that case, + @throw type_error.306 if the JSON value is not an object; in that case, using `value()` with a key makes no sense. @complexity Logarithmic in the size of the container. @@ -10536,7 +15845,7 @@ class basic_json { return ptr.get_checked(this); } - JSON_CATCH (out_of_range&) + JSON_INTERNAL_CATCH (out_of_range&) { return default_value; } @@ -11047,7 +16356,7 @@ class basic_json @complexity Constant. @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) + [Container](https://en.cppreference.com/w/cpp/named_req/Container) requirements: - The complexity is constant. @@ -11086,7 +16395,7 @@ class basic_json @complexity Constant. @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) + [Container](https://en.cppreference.com/w/cpp/named_req/Container) requirements: - The complexity is constant. - Has the semantics of `const_cast<const basic_json&>(*this).begin()`. @@ -11118,7 +16427,7 @@ class basic_json @complexity Constant. @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) + [Container](https://en.cppreference.com/w/cpp/named_req/Container) requirements: - The complexity is constant. @@ -11157,7 +16466,7 @@ class basic_json @complexity Constant. @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) + [Container](https://en.cppreference.com/w/cpp/named_req/Container) requirements: - The complexity is constant. - Has the semantics of `const_cast<const basic_json&>(*this).end()`. @@ -11187,7 +16496,7 @@ class basic_json @complexity Constant. @requirement This function helps `basic_json` satisfying the - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) + [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer) requirements: - The complexity is constant. - Has the semantics of `reverse_iterator(end())`. @@ -11224,7 +16533,7 @@ class basic_json @complexity Constant. @requirement This function helps `basic_json` satisfying the - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) + [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer) requirements: - The complexity is constant. - Has the semantics of `reverse_iterator(begin())`. @@ -11261,7 +16570,7 @@ class basic_json @complexity Constant. @requirement This function helps `basic_json` satisfying the - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) + [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer) requirements: - The complexity is constant. - Has the semantics of `const_cast<const basic_json&>(*this).rbegin()`. @@ -11290,7 +16599,7 @@ class basic_json @complexity Constant. @requirement This function helps `basic_json` satisfying the - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) + [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer) requirements: - The complexity is constant. - Has the semantics of `const_cast<const basic_json&>(*this).rend()`. @@ -11361,18 +16670,100 @@ class basic_json @note The name of this function is not yet final and may change in the future. + + @deprecated This stream operator is deprecated and will be removed in + future 4.0.0 of the library. Please use @ref items() instead; + that is, replace `json::iterator_wrapper(j)` with `j.items()`. */ - static iteration_proxy<iterator> iterator_wrapper(reference ref) + JSON_DEPRECATED + static iteration_proxy<iterator> iterator_wrapper(reference ref) noexcept { - return iteration_proxy<iterator>(ref); + return ref.items(); } /*! @copydoc iterator_wrapper(reference) */ - static iteration_proxy<const_iterator> iterator_wrapper(const_reference ref) + JSON_DEPRECATED + static iteration_proxy<const_iterator> iterator_wrapper(const_reference ref) noexcept { - return iteration_proxy<const_iterator>(ref); + return ref.items(); + } + + /*! + @brief helper to access iterator member functions in range-based for + + This function allows to access @ref iterator::key() and @ref + iterator::value() during range-based for loops. In these loops, a + reference to the JSON values is returned, so there is no access to the + underlying iterator. + + For loop without `items()` function: + + @code{cpp} + for (auto it = j_object.begin(); it != j_object.end(); ++it) + { + std::cout << "key: " << it.key() << ", value:" << it.value() << '\n'; + } + @endcode + + Range-based for loop without `items()` function: + + @code{cpp} + for (auto it : j_object) + { + // "it" is of type json::reference and has no key() member + std::cout << "value: " << it << '\n'; + } + @endcode + + Range-based for loop with `items()` function: + + @code{cpp} + for (auto& el : j_object.items()) + { + std::cout << "key: " << el.key() << ", value:" << el.value() << '\n'; + } + @endcode + + The `items()` function also allows to use + [structured bindings](https://en.cppreference.com/w/cpp/language/structured_binding) + (C++17): + + @code{cpp} + for (auto& [key, val] : j_object.items()) + { + std::cout << "key: " << key << ", value:" << val << '\n'; + } + @endcode + + @note When iterating over an array, `key()` will return the index of the + element as string (see example). For primitive types (e.g., numbers), + `key()` returns an empty string. + + @return iteration proxy object wrapping @a ref with an interface to use in + range-based for loops + + @liveexample{The following code shows how the function is used.,items} + + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes in the JSON value. + + @complexity Constant. + + @since version 3.1.0, structured bindings support since 3.5.0. + */ + iteration_proxy<iterator> items() noexcept + { + return iteration_proxy<iterator>(*this); + } + + /*! + @copydoc items() + */ + iteration_proxy<const_iterator> items() const noexcept + { + return iteration_proxy<const_iterator>(*this); } /// @} @@ -11417,7 +16808,7 @@ class basic_json false in the case of a string. @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) + [Container](https://en.cppreference.com/w/cpp/named_req/Container) requirements: - The complexity is constant. - Has the semantics of `begin() == end()`. @@ -11488,7 +16879,7 @@ class basic_json the case of a string. @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) + [Container](https://en.cppreference.com/w/cpp/named_req/Container) requirements: - The complexity is constant. - Has the semantics of `std::distance(begin(), end())`. @@ -11558,7 +16949,7 @@ class basic_json @exceptionsafety No-throw guarantee: this function never throws exceptions. @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) + [Container](https://en.cppreference.com/w/cpp/named_req/Container) requirements: - The complexity is constant. - Has the semantics of returning `b.size()` where `b` is the largest @@ -11970,6 +17361,26 @@ class basic_json return {it, res.second}; } + /// Helper for insertion of an iterator + /// @note: This uses std::distance to support GCC 4.8, + /// see https://github.com/nlohmann/json/pull/1257 + template<typename... Args> + iterator insert_iterator(const_iterator pos, Args&& ... args) + { + iterator result(this); + assert(m_value.array != nullptr); + + auto insert_pos = std::distance(m_value.array->begin(), pos.m_it.array_iterator); + m_value.array->insert(pos.m_it.array_iterator, std::forward<Args>(args)...); + result.m_it.array_iterator = m_value.array->begin() + insert_pos; + + // This could have been written as: + // result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, cnt, val); + // but the return value of insert is missing in GCC 4.8, so it is written this way instead. + + return result; + } + /*! @brief inserts element @@ -12004,9 +17415,7 @@ class basic_json } // insert to array and return iterator - iterator result(this); - result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, val); - return result; + return insert_iterator(pos, val); } JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name()))); @@ -12057,9 +17466,7 @@ class basic_json } // insert to array and return iterator - iterator result(this); - result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, cnt, val); - return result; + return insert_iterator(pos, cnt, val); } JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name()))); @@ -12121,12 +17528,7 @@ class basic_json } // insert to array and return iterator - iterator result(this); - result.m_it.array_iterator = m_value.array->insert( - pos.m_it.array_iterator, - first.m_it.array_iterator, - last.m_it.array_iterator); - return result; + return insert_iterator(pos, first.m_it.array_iterator, last.m_it.array_iterator); } /*! @@ -12168,9 +17570,7 @@ class basic_json } // insert to array and return iterator - iterator result(this); - result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, ilist.begin(), ilist.end()); - return result; + return insert_iterator(pos, ilist.begin(), ilist.end()); } /*! @@ -12312,7 +17712,7 @@ class basic_json // passed iterators must belong to objects if (JSON_UNLIKELY(not first.m_object->is_object() - or not first.m_object->is_object())) + or not last.m_object->is_object())) { JSON_THROW(invalid_iterator::create(202, "iterators first and last must point to objects")); } @@ -12943,8 +18343,8 @@ class basic_json /*! @brief serialize to stream - @deprecated This stream operator is deprecated and will be removed in a - future version of the library. Please use + @deprecated This stream operator is deprecated and will be removed in + future 4.0.0 of the library. Please use @ref operator<<(std::ostream&, const basic_json&) instead; that is, replace calls like `j >> o;` with `o << j;`. @since version 1.0.0; deprecated since version 3.0.0 @@ -12999,6 +18399,8 @@ class basic_json @param[in] cb a parser callback function of type @ref parser_callback_t which is used to control the deserialization by filtering unwanted values (optional) + @param[in] allow_exceptions whether to throw exceptions in case of a + parse error (optional, true by default) @return result of the deserialization @@ -13027,19 +18429,7 @@ class basic_json @since version 2.0.3 (contiguous containers) */ - static basic_json parse(detail::input_adapter i, - const parser_callback_t cb = nullptr, - const bool allow_exceptions = true) - { - basic_json result; - parser(i, cb, allow_exceptions).parse(true, result); - return result; - } - - /*! - @copydoc basic_json parse(detail::input_adapter, const parser_callback_t) - */ - static basic_json parse(detail::input_adapter& i, + static basic_json parse(detail::input_adapter&& i, const parser_callback_t cb = nullptr, const bool allow_exceptions = true) { @@ -13048,14 +18438,80 @@ class basic_json return result; } - static bool accept(detail::input_adapter i) + static bool accept(detail::input_adapter&& i) { return parser(i).accept(true); } - static bool accept(detail::input_adapter& i) + /*! + @brief generate SAX events + + The SAX event lister must follow the interface of @ref json_sax. + + This function reads from a compatible input. Examples are: + - an array of 1-byte values + - strings with character/literal type with size of 1 byte + - input streams + - container with contiguous storage of 1-byte values. Compatible container + types include `std::vector`, `std::string`, `std::array`, + `std::valarray`, and `std::initializer_list`. Furthermore, C-style + arrays can be used with `std::begin()`/`std::end()`. User-defined + containers can be used as long as they implement random-access iterators + and a contiguous storage. + + @pre Each element of the container has a size of 1 byte. Violating this + precondition yields undefined behavior. **This precondition is enforced + with a static assertion.** + + @pre The container storage is contiguous. Violating this precondition + yields undefined behavior. **This precondition is enforced with an + assertion.** + @pre Each element of the container has a size of 1 byte. Violating this + precondition yields undefined behavior. **This precondition is enforced + with a static assertion.** + + @warning There is no way to enforce all preconditions at compile-time. If + the function is called with a noncompliant container and with + assertions switched off, the behavior is undefined and will most + likely yield segmentation violation. + + @param[in] i input to read from + @param[in,out] sax SAX event listener + @param[in] format the format to parse (JSON, CBOR, MessagePack, or UBJSON) + @param[in] strict whether the input has to be consumed completely + + @return return value of the last processed SAX event + + @throw parse_error.101 if a parse error occurs; example: `""unexpected end + of input; expected string literal""` + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. The complexity can be higher if the SAX consumer @a sax has + a super-linear complexity. + + @note A UTF-8 byte order mark is silently ignored. + + @liveexample{The example below demonstrates the `sax_parse()` function + reading from string and processing the events with a user-defined SAX + event consumer.,sax_parse} + + @since version 3.2.0 + */ + template <typename SAX> + static bool sax_parse(detail::input_adapter&& i, SAX* sax, + input_format_t format = input_format_t::json, + const bool strict = true) { - return parser(i).accept(true); + assert(sax); + switch (format) + { + case input_format_t::json: + return parser(std::move(i)).sax_parse(sax, strict); + default: + return detail::binary_reader<basic_json, SAX>(std::move(i)).sax_parse(format, sax, strict); + } } /*! @@ -13127,10 +18583,19 @@ class basic_json return parser(detail::input_adapter(first, last)).accept(true); } + template<class IteratorType, class SAX, typename std::enable_if< + std::is_base_of< + std::random_access_iterator_tag, + typename std::iterator_traits<IteratorType>::iterator_category>::value, int>::type = 0> + static bool sax_parse(IteratorType first, IteratorType last, SAX* sax) + { + return parser(detail::input_adapter(first, last)).sax_parse(sax); + } + /*! @brief deserialize from stream - @deprecated This stream operator is deprecated and will be removed in a - future version of the library. Please use + @deprecated This stream operator is deprecated and will be removed in + version 4.0.0 of the library. Please use @ref operator>>(std::istream&, basic_json&) instead; that is, replace calls like `j << i;` with `i >> j;`. @since version 1.0.0; deprecated since version 3.0.0 @@ -13331,9 +18796,11 @@ class basic_json vector in CBOR format.,to_cbor} @sa http://cbor.io - @sa @ref from_cbor(const std::vector<uint8_t>&, const size_t) for the + @sa @ref from_cbor(detail::input_adapter&&, const bool, const bool) for the analogous deserialization @sa @ref to_msgpack(const basic_json&) for the related MessagePack format + @sa @ref to_ubjson(const basic_json&, const bool, const bool) for the + related UBJSON format @since version 2.0.9 */ @@ -13426,9 +18893,10 @@ class basic_json vector in MessagePack format.,to_msgpack} @sa http://msgpack.org - @sa @ref from_msgpack(const std::vector<uint8_t>&, const size_t) for the - analogous deserialization + @sa @ref from_msgpack for the analogous deserialization @sa @ref to_cbor(const basic_json& for the related CBOR format + @sa @ref to_ubjson(const basic_json&, const bool, const bool) for the + related UBJSON format @since version 2.0.9 */ @@ -13450,6 +18918,192 @@ class basic_json } /*! + @brief create a UBJSON serialization of a given JSON value + + Serializes a given JSON value @a j to a byte vector using the UBJSON + (Universal Binary JSON) serialization format. UBJSON aims to be more compact + than JSON itself, yet more efficient to parse. + + The library uses the following mapping from JSON values types to + UBJSON types according to the UBJSON specification: + + JSON value type | value/range | UBJSON type | marker + --------------- | --------------------------------- | ----------- | ------ + null | `null` | null | `Z` + boolean | `true` | true | `T` + boolean | `false` | false | `F` + number_integer | -9223372036854775808..-2147483649 | int64 | `L` + number_integer | -2147483648..-32769 | int32 | `l` + number_integer | -32768..-129 | int16 | `I` + number_integer | -128..127 | int8 | `i` + number_integer | 128..255 | uint8 | `U` + number_integer | 256..32767 | int16 | `I` + number_integer | 32768..2147483647 | int32 | `l` + number_integer | 2147483648..9223372036854775807 | int64 | `L` + number_unsigned | 0..127 | int8 | `i` + number_unsigned | 128..255 | uint8 | `U` + number_unsigned | 256..32767 | int16 | `I` + number_unsigned | 32768..2147483647 | int32 | `l` + number_unsigned | 2147483648..9223372036854775807 | int64 | `L` + number_float | *any value* | float64 | `D` + string | *with shortest length indicator* | string | `S` + array | *see notes on optimized format* | array | `[` + object | *see notes on optimized format* | map | `{` + + @note The mapping is **complete** in the sense that any JSON value type + can be converted to a UBJSON value. + + @note The following values can **not** be converted to a UBJSON value: + - strings with more than 9223372036854775807 bytes (theoretical) + - unsigned integer numbers above 9223372036854775807 + + @note The following markers are not used in the conversion: + - `Z`: no-op values are not created. + - `C`: single-byte strings are serialized with `S` markers. + + @note Any UBJSON output created @ref to_ubjson can be successfully parsed + by @ref from_ubjson. + + @note If NaN or Infinity are stored inside a JSON number, they are + serialized properly. This behavior differs from the @ref dump() + function which serializes NaN or Infinity to `null`. + + @note The optimized formats for containers are supported: Parameter + @a use_size adds size information to the beginning of a container and + removes the closing marker. Parameter @a use_type further checks + whether all elements of a container have the same type and adds the + type marker to the beginning of the container. The @a use_type + parameter must only be used together with @a use_size = true. Note + that @a use_size = true alone may result in larger representations - + the benefit of this parameter is that the receiving side is + immediately informed on the number of elements of the container. + + @param[in] j JSON value to serialize + @param[in] use_size whether to add size annotations to container types + @param[in] use_type whether to add type annotations to container types + (must be combined with @a use_size = true) + @return UBJSON serialization as byte vector + + @complexity Linear in the size of the JSON value @a j. + + @liveexample{The example shows the serialization of a JSON value to a byte + vector in UBJSON format.,to_ubjson} + + @sa http://ubjson.org + @sa @ref from_ubjson(detail::input_adapter&&, const bool, const bool) for the + analogous deserialization + @sa @ref to_cbor(const basic_json& for the related CBOR format + @sa @ref to_msgpack(const basic_json&) for the related MessagePack format + + @since version 3.1.0 + */ + static std::vector<uint8_t> to_ubjson(const basic_json& j, + const bool use_size = false, + const bool use_type = false) + { + std::vector<uint8_t> result; + to_ubjson(j, result, use_size, use_type); + return result; + } + + static void to_ubjson(const basic_json& j, detail::output_adapter<uint8_t> o, + const bool use_size = false, const bool use_type = false) + { + binary_writer<uint8_t>(o).write_ubjson(j, use_size, use_type); + } + + static void to_ubjson(const basic_json& j, detail::output_adapter<char> o, + const bool use_size = false, const bool use_type = false) + { + binary_writer<char>(o).write_ubjson(j, use_size, use_type); + } + + + /*! + @brief Serializes the given JSON object `j` to BSON and returns a vector + containing the corresponding BSON-representation. + + BSON (Binary JSON) is a binary format in which zero or more ordered key/value pairs are + stored as a single entity (a so-called document). + + The library uses the following mapping from JSON values types to BSON types: + + JSON value type | value/range | BSON type | marker + --------------- | --------------------------------- | ----------- | ------ + null | `null` | null | 0x0A + boolean | `true`, `false` | boolean | 0x08 + number_integer | -9223372036854775808..-2147483649 | int64 | 0x12 + number_integer | -2147483648..2147483647 | int32 | 0x10 + number_integer | 2147483648..9223372036854775807 | int64 | 0x12 + number_unsigned | 0..2147483647 | int32 | 0x10 + number_unsigned | 2147483648..9223372036854775807 | int64 | 0x12 + number_unsigned | 9223372036854775808..18446744073709551615| -- | -- + number_float | *any value* | double | 0x01 + string | *any value* | string | 0x02 + array | *any value* | document | 0x04 + object | *any value* | document | 0x03 + + @warning The mapping is **incomplete**, since only JSON-objects (and things + contained therein) can be serialized to BSON. + Also, integers larger than 9223372036854775807 cannot be serialized to BSON, + and the keys may not contain U+0000, since they are serialized a + zero-terminated c-strings. + + @throw out_of_range.407 if `j.is_number_unsigned() && j.get<std::uint64_t>() > 9223372036854775807` + @throw out_of_range.409 if a key in `j` contains a NULL (U+0000) + @throw type_error.317 if `!j.is_object()` + + @pre The input `j` is required to be an object: `j.is_object() == true`. + + @note Any BSON output created via @ref to_bson can be successfully parsed + by @ref from_bson. + + @param[in] j JSON value to serialize + @return BSON serialization as byte vector + + @complexity Linear in the size of the JSON value @a j. + + @liveexample{The example shows the serialization of a JSON value to a byte + vector in BSON format.,to_bson} + + @sa http://bsonspec.org/spec.html + @sa @ref from_bson(detail::input_adapter&&, const bool strict) for the + analogous deserialization + @sa @ref to_ubjson(const basic_json&, const bool, const bool) for the + related UBJSON format + @sa @ref to_cbor(const basic_json&) for the related CBOR format + @sa @ref to_msgpack(const basic_json&) for the related MessagePack format + */ + static std::vector<uint8_t> to_bson(const basic_json& j) + { + std::vector<uint8_t> result; + to_bson(j, result); + return result; + } + + /*! + @brief Serializes the given JSON object `j` to BSON and forwards the + corresponding BSON-representation to the given output_adapter `o`. + @param j The JSON object to convert to BSON. + @param o The output adapter that receives the binary BSON representation. + @pre The input `j` shall be an object: `j.is_object() == true` + @sa @ref to_bson(const basic_json&) + */ + static void to_bson(const basic_json& j, detail::output_adapter<uint8_t> o) + { + binary_writer<uint8_t>(o).write_bson(j); + } + + /*! + @copydoc to_bson(const basic_json&, detail::output_adapter<uint8_t>) + */ + static void to_bson(const basic_json& j, detail::output_adapter<char> o) + { + binary_writer<char>(o).write_bson(j); + } + + + /*! @brief create a JSON value from an input in CBOR format Deserializes a given input @a i to a JSON value using the CBOR (Concise @@ -13490,7 +19144,7 @@ class basic_json map | object | 0xBF False | `false` | 0xF4 True | `true` | 0xF5 - Nill | `null` | 0xF6 + Null | `null` | 0xF6 Half-Precision Float | number_float | 0xF9 Single-Precision Float | number_float | 0xFA Double-Precision Float | number_float | 0xFB @@ -13518,6 +19172,9 @@ class basic_json @param[in] i an input in CBOR format convertible to an input adapter @param[in] strict whether to expect the input to be consumed until EOF (true by default) + @param[in] allow_exceptions whether to throw exceptions in case of a + parse error (optional, true by default) + @return deserialized JSON value @throw parse_error.110 if the given input ends prematurely or the end of @@ -13533,27 +19190,39 @@ class basic_json @sa http://cbor.io @sa @ref to_cbor(const basic_json&) for the analogous serialization - @sa @ref from_msgpack(detail::input_adapter, const bool) for the + @sa @ref from_msgpack(detail::input_adapter&&, const bool, const bool) for the related MessagePack format + @sa @ref from_ubjson(detail::input_adapter&&, const bool, const bool) for the + related UBJSON format @since version 2.0.9; parameter @a start_index since 2.1.1; changed to consume input adapters, removed start_index parameter, and added - @a strict parameter since 3.0.0 + @a strict parameter since 3.0.0; added @a allow_exceptions parameter + since 3.2.0 */ - static basic_json from_cbor(detail::input_adapter i, - const bool strict = true) + static basic_json from_cbor(detail::input_adapter&& i, + const bool strict = true, + const bool allow_exceptions = true) { - return binary_reader(i).parse_cbor(strict); + basic_json result; + detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions); + const bool res = binary_reader(detail::input_adapter(i)).sax_parse(input_format_t::cbor, &sdp, strict); + return res ? result : basic_json(value_t::discarded); } /*! - @copydoc from_cbor(detail::input_adapter, const bool) + @copydoc from_cbor(detail::input_adapter&&, const bool, const bool) */ template<typename A1, typename A2, detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0> - static basic_json from_cbor(A1 && a1, A2 && a2, const bool strict = true) + static basic_json from_cbor(A1 && a1, A2 && a2, + const bool strict = true, + const bool allow_exceptions = true) { - return binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).parse_cbor(strict); + basic_json result; + detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions); + const bool res = binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).sax_parse(input_format_t::cbor, &sdp, strict); + return res ? result : basic_json(value_t::discarded); } /*! @@ -13606,6 +19275,10 @@ class basic_json adapter @param[in] strict whether to expect the input to be consumed until EOF (true by default) + @param[in] allow_exceptions whether to throw exceptions in case of a + parse error (optional, true by default) + + @return deserialized JSON value @throw parse_error.110 if the given input ends prematurely or the end of file was not reached when @a strict was set to true @@ -13620,29 +19293,212 @@ class basic_json @sa http://msgpack.org @sa @ref to_msgpack(const basic_json&) for the analogous serialization - @sa @ref from_cbor(detail::input_adapter, const bool) for the related CBOR - format + @sa @ref from_cbor(detail::input_adapter&&, const bool, const bool) for the + related CBOR format + @sa @ref from_ubjson(detail::input_adapter&&, const bool, const bool) for + the related UBJSON format + @sa @ref from_bson(detail::input_adapter&&, const bool, const bool) for + the related BSON format @since version 2.0.9; parameter @a start_index since 2.1.1; changed to consume input adapters, removed start_index parameter, and added - @a strict parameter since 3.0.0 + @a strict parameter since 3.0.0; added @a allow_exceptions parameter + since 3.2.0 + */ + static basic_json from_msgpack(detail::input_adapter&& i, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions); + const bool res = binary_reader(detail::input_adapter(i)).sax_parse(input_format_t::msgpack, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + /*! + @copydoc from_msgpack(detail::input_adapter&&, const bool, const bool) + */ + template<typename A1, typename A2, + detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0> + static basic_json from_msgpack(A1 && a1, A2 && a2, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions); + const bool res = binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).sax_parse(input_format_t::msgpack, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + /*! + @brief create a JSON value from an input in UBJSON format + + Deserializes a given input @a i to a JSON value using the UBJSON (Universal + Binary JSON) serialization format. + + The library maps UBJSON types to JSON value types as follows: + + UBJSON type | JSON value type | marker + ----------- | --------------------------------------- | ------ + no-op | *no value, next value is read* | `N` + null | `null` | `Z` + false | `false` | `F` + true | `true` | `T` + float32 | number_float | `d` + float64 | number_float | `D` + uint8 | number_unsigned | `U` + int8 | number_integer | `i` + int16 | number_integer | `I` + int32 | number_integer | `l` + int64 | number_integer | `L` + string | string | `S` + char | string | `C` + array | array (optimized values are supported) | `[` + object | object (optimized values are supported) | `{` + + @note The mapping is **complete** in the sense that any UBJSON value can + be converted to a JSON value. + + @param[in] i an input in UBJSON format convertible to an input adapter + @param[in] strict whether to expect the input to be consumed until EOF + (true by default) + @param[in] allow_exceptions whether to throw exceptions in case of a + parse error (optional, true by default) + + @return deserialized JSON value + + @throw parse_error.110 if the given input ends prematurely or the end of + file was not reached when @a strict was set to true + @throw parse_error.112 if a parse error occurs + @throw parse_error.113 if a string could not be parsed successfully + + @complexity Linear in the size of the input @a i. + + @liveexample{The example shows the deserialization of a byte vector in + UBJSON format to a JSON value.,from_ubjson} + + @sa http://ubjson.org + @sa @ref to_ubjson(const basic_json&, const bool, const bool) for the + analogous serialization + @sa @ref from_cbor(detail::input_adapter&&, const bool, const bool) for the + related CBOR format + @sa @ref from_msgpack(detail::input_adapter&&, const bool, const bool) for + the related MessagePack format + @sa @ref from_bson(detail::input_adapter&&, const bool, const bool) for + the related BSON format + + @since version 3.1.0; added @a allow_exceptions parameter since 3.2.0 */ - static basic_json from_msgpack(detail::input_adapter i, - const bool strict = true) + static basic_json from_ubjson(detail::input_adapter&& i, + const bool strict = true, + const bool allow_exceptions = true) { - return binary_reader(i).parse_msgpack(strict); + basic_json result; + detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions); + const bool res = binary_reader(detail::input_adapter(i)).sax_parse(input_format_t::ubjson, &sdp, strict); + return res ? result : basic_json(value_t::discarded); } /*! - @copydoc from_msgpack(detail::input_adapter, const bool) + @copydoc from_ubjson(detail::input_adapter&&, const bool, const bool) */ template<typename A1, typename A2, detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0> - static basic_json from_msgpack(A1 && a1, A2 && a2, const bool strict = true) + static basic_json from_ubjson(A1 && a1, A2 && a2, + const bool strict = true, + const bool allow_exceptions = true) { - return binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).parse_msgpack(strict); + basic_json result; + detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions); + const bool res = binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).sax_parse(input_format_t::ubjson, &sdp, strict); + return res ? result : basic_json(value_t::discarded); } + /*! + @brief Create a JSON value from an input in BSON format + + Deserializes a given input @a i to a JSON value using the BSON (Binary JSON) + serialization format. + + The library maps BSON record types to JSON value types as follows: + + BSON type | BSON marker byte | JSON value type + --------------- | ---------------- | --------------------------- + double | 0x01 | number_float + string | 0x02 | string + document | 0x03 | object + array | 0x04 | array + binary | 0x05 | still unsupported + undefined | 0x06 | still unsupported + ObjectId | 0x07 | still unsupported + boolean | 0x08 | boolean + UTC Date-Time | 0x09 | still unsupported + null | 0x0A | null + Regular Expr. | 0x0B | still unsupported + DB Pointer | 0x0C | still unsupported + JavaScript Code | 0x0D | still unsupported + Symbol | 0x0E | still unsupported + JavaScript Code | 0x0F | still unsupported + int32 | 0x10 | number_integer + Timestamp | 0x11 | still unsupported + 128-bit decimal float | 0x13 | still unsupported + Max Key | 0x7F | still unsupported + Min Key | 0xFF | still unsupported + + @warning The mapping is **incomplete**. The unsupported mappings + are indicated in the table above. + + @param[in] i an input in BSON format convertible to an input adapter + @param[in] strict whether to expect the input to be consumed until EOF + (true by default) + @param[in] allow_exceptions whether to throw exceptions in case of a + parse error (optional, true by default) + + @return deserialized JSON value + + @throw parse_error.114 if an unsupported BSON record type is encountered + + @complexity Linear in the size of the input @a i. + + @liveexample{The example shows the deserialization of a byte vector in + BSON format to a JSON value.,from_bson} + + @sa http://bsonspec.org/spec.html + @sa @ref to_bson(const basic_json&) for the analogous serialization + @sa @ref from_cbor(detail::input_adapter&&, const bool, const bool) for the + related CBOR format + @sa @ref from_msgpack(detail::input_adapter&&, const bool, const bool) for + the related MessagePack format + @sa @ref from_ubjson(detail::input_adapter&&, const bool, const bool) for the + related UBJSON format + */ + static basic_json from_bson(detail::input_adapter&& i, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions); + const bool res = binary_reader(detail::input_adapter(i)).sax_parse(input_format_t::bson, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + /*! + @copydoc from_bson(detail::input_adapter&&, const bool, const bool) + */ + template<typename A1, typename A2, + detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0> + static basic_json from_bson(A1 && a1, A2 && a2, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions); + const bool res = binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).sax_parse(input_format_t::bson, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + + /// @} ////////////////////////// @@ -14008,20 +19864,20 @@ class basic_json // avoid undefined behavior JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range")); } - else - { - // default case: insert add offset - parent.insert(parent.begin() + static_cast<difference_type>(idx), val); - } + + // default case: insert add offset + parent.insert(parent.begin() + static_cast<difference_type>(idx), val); } break; } + // LCOV_EXCL_START default: { // if there exists a parent it cannot be primitive - assert(false); // LCOV_EXCL_LINE + assert(false); } + // LCOV_EXCL_STOP } } }; @@ -14066,7 +19922,7 @@ class basic_json // wrapper to get a value for an operation const auto get_value = [&val](const std::string & op, const std::string & member, - bool string_type) -> basic_json& + bool string_type) -> basic_json & { // find value auto it = val.m_value.object->find(member); @@ -14163,7 +20019,7 @@ class basic_json // the "path" location must exist - use at() success = (result.at(ptr) == get_value("test", "value", false)); } - JSON_CATCH (out_of_range&) + JSON_INTERNAL_CATCH (out_of_range&) { // ignore out of range errors: success remains false } @@ -14216,6 +20072,7 @@ class basic_json diff for two JSON values.,diff} @sa @ref patch -- apply a JSON patch + @sa @ref merge_patch -- apply a JSON Merge Patch @sa [RFC 6902 (JSON Patch)](https://tools.ietf.org/html/rfc6902) @@ -14347,418 +20204,86 @@ class basic_json } /// @} -}; -///////////// -// presets // -///////////// + //////////////////////////////// + // JSON Merge Patch functions // + //////////////////////////////// -/*! -@brief default JSON class - -This type is the default specialization of the @ref basic_json class which -uses the standard template types. - -@since version 1.0.0 -*/ -using json = basic_json<>; - -////////////////// -// json_pointer // -////////////////// - -NLOHMANN_BASIC_JSON_TPL_DECLARATION -NLOHMANN_BASIC_JSON_TPL& -json_pointer::get_and_create(NLOHMANN_BASIC_JSON_TPL& j) const -{ - using size_type = typename NLOHMANN_BASIC_JSON_TPL::size_type; - auto result = &j; - - // in case no reference tokens exist, return a reference to the JSON value - // j which will be overwritten by a primitive value - for (const auto& reference_token : reference_tokens) - { - switch (result->m_type) - { - case detail::value_t::null: - { - if (reference_token == "0") - { - // start a new array if reference token is 0 - result = &result->operator[](0); - } - else - { - // start a new object otherwise - result = &result->operator[](reference_token); - } - break; - } - - case detail::value_t::object: - { - // create an entry in the object - result = &result->operator[](reference_token); - break; - } - - case detail::value_t::array: - { - // create an entry in the array - JSON_TRY - { - result = &result->operator[](static_cast<size_type>(array_index(reference_token))); - } - JSON_CATCH(std::invalid_argument&) - { - JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); - } - break; - } + /// @name JSON Merge Patch functions + /// @{ - /* - The following code is only reached if there exists a reference - token _and_ the current value is primitive. In this case, we have - an error situation, because primitive values may only occur as - single value; that is, with an empty list of reference tokens. - */ - default: - JSON_THROW(detail::type_error::create(313, "invalid value to unflatten")); - } - } + /*! + @brief applies a JSON Merge Patch + + The merge patch format is primarily intended for use with the HTTP PATCH + method as a means of describing a set of modifications to a target + resource's content. This function applies a merge patch to the current + JSON value. + + The function implements the following algorithm from Section 2 of + [RFC 7396 (JSON Merge Patch)](https://tools.ietf.org/html/rfc7396): + + ``` + define MergePatch(Target, Patch): + if Patch is an Object: + if Target is not an Object: + Target = {} // Ignore the contents and set it to an empty Object + for each Name/Value pair in Patch: + if Value is null: + if Name exists in Target: + remove the Name/Value pair from Target + else: + Target[Name] = MergePatch(Target[Name], Value) + return Target + else: + return Patch + ``` + + Thereby, `Target` is the current object; that is, the patch is applied to + the current value. + + @param[in] apply_patch the patch to apply + + @complexity Linear in the lengths of @a patch. + + @liveexample{The following code shows how a JSON Merge Patch is applied to + a JSON document.,merge_patch} - return *result; -} + @sa @ref patch -- apply a JSON patch + @sa [RFC 7396 (JSON Merge Patch)](https://tools.ietf.org/html/rfc7396) -NLOHMANN_BASIC_JSON_TPL_DECLARATION -NLOHMANN_BASIC_JSON_TPL& -json_pointer::get_unchecked(NLOHMANN_BASIC_JSON_TPL* ptr) const -{ - using size_type = typename NLOHMANN_BASIC_JSON_TPL::size_type; - for (const auto& reference_token : reference_tokens) + @since version 3.0.0 + */ + void merge_patch(const basic_json& apply_patch) { - // convert null values to arrays or objects before continuing - if (ptr->m_type == detail::value_t::null) - { - // check if reference token is a number - const bool nums = - std::all_of(reference_token.begin(), reference_token.end(), - [](const char x) - { - return (x >= '0' and x <= '9'); - }); - - // change value to array for numbers or "-" or to object otherwise - *ptr = (nums or reference_token == "-") - ? detail::value_t::array - : detail::value_t::object; - } - - switch (ptr->m_type) + if (apply_patch.is_object()) { - case detail::value_t::object: + if (not is_object()) { - // use unchecked object access - ptr = &ptr->operator[](reference_token); - break; + *this = object(); } - - case detail::value_t::array: + for (auto it = apply_patch.begin(); it != apply_patch.end(); ++it) { - // error condition (cf. RFC 6901, Sect. 4) - if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0')) - { - JSON_THROW(detail::parse_error::create(106, 0, - "array index '" + reference_token + - "' must not begin with '0'")); - } - - if (reference_token == "-") + if (it.value().is_null()) { - // explicitly treat "-" as index beyond the end - ptr = &ptr->operator[](ptr->m_value.array->size()); + erase(it.key()); } else { - // convert array index to number; unchecked access - JSON_TRY - { - ptr = &ptr->operator[]( - static_cast<size_type>(array_index(reference_token))); - } - JSON_CATCH(std::invalid_argument&) - { - JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); - } - } - break; - } - - default: - JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); - } - } - - return *ptr; -} - -NLOHMANN_BASIC_JSON_TPL_DECLARATION -NLOHMANN_BASIC_JSON_TPL& -json_pointer::get_checked(NLOHMANN_BASIC_JSON_TPL* ptr) const -{ - using size_type = typename NLOHMANN_BASIC_JSON_TPL::size_type; - for (const auto& reference_token : reference_tokens) - { - switch (ptr->m_type) - { - case detail::value_t::object: - { - // note: at performs range check - ptr = &ptr->at(reference_token); - break; - } - - case detail::value_t::array: - { - if (JSON_UNLIKELY(reference_token == "-")) - { - // "-" always fails the range check - JSON_THROW(detail::out_of_range::create(402, - "array index '-' (" + std::to_string(ptr->m_value.array->size()) + - ") is out of range")); - } - - // error condition (cf. RFC 6901, Sect. 4) - if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0')) - { - JSON_THROW(detail::parse_error::create(106, 0, - "array index '" + reference_token + - "' must not begin with '0'")); - } - - // note: at performs range check - JSON_TRY - { - ptr = &ptr->at(static_cast<size_type>(array_index(reference_token))); + operator[](it.key()).merge_patch(it.value()); } - JSON_CATCH(std::invalid_argument&) - { - JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); - } - break; } - - default: - JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); } - } - - return *ptr; -} - -NLOHMANN_BASIC_JSON_TPL_DECLARATION -const NLOHMANN_BASIC_JSON_TPL& -json_pointer::get_unchecked(const NLOHMANN_BASIC_JSON_TPL* ptr) const -{ - using size_type = typename NLOHMANN_BASIC_JSON_TPL::size_type; - for (const auto& reference_token : reference_tokens) - { - switch (ptr->m_type) - { - case detail::value_t::object: - { - // use unchecked object access - ptr = &ptr->operator[](reference_token); - break; - } - - case detail::value_t::array: - { - if (JSON_UNLIKELY(reference_token == "-")) - { - // "-" cannot be used for const access - JSON_THROW(detail::out_of_range::create(402, - "array index '-' (" + std::to_string(ptr->m_value.array->size()) + - ") is out of range")); - } - - // error condition (cf. RFC 6901, Sect. 4) - if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0')) - { - JSON_THROW(detail::parse_error::create(106, 0, - "array index '" + reference_token + - "' must not begin with '0'")); - } - - // use unchecked array access - JSON_TRY - { - ptr = &ptr->operator[]( - static_cast<size_type>(array_index(reference_token))); - } - JSON_CATCH(std::invalid_argument&) - { - JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); - } - break; - } - - default: - JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); - } - } - - return *ptr; -} - -NLOHMANN_BASIC_JSON_TPL_DECLARATION -const NLOHMANN_BASIC_JSON_TPL& -json_pointer::get_checked(const NLOHMANN_BASIC_JSON_TPL* ptr) const -{ - using size_type = typename NLOHMANN_BASIC_JSON_TPL::size_type; - for (const auto& reference_token : reference_tokens) - { - switch (ptr->m_type) - { - case detail::value_t::object: - { - // note: at performs range check - ptr = &ptr->at(reference_token); - break; - } - - case detail::value_t::array: - { - if (JSON_UNLIKELY(reference_token == "-")) - { - // "-" always fails the range check - JSON_THROW(detail::out_of_range::create(402, - "array index '-' (" + std::to_string(ptr->m_value.array->size()) + - ") is out of range")); - } - - // error condition (cf. RFC 6901, Sect. 4) - if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0')) - { - JSON_THROW(detail::parse_error::create(106, 0, - "array index '" + reference_token + - "' must not begin with '0'")); - } - - // note: at performs range check - JSON_TRY - { - ptr = &ptr->at(static_cast<size_type>(array_index(reference_token))); - } - JSON_CATCH(std::invalid_argument&) - { - JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); - } - break; - } - - default: - JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); - } - } - - return *ptr; -} - -NLOHMANN_BASIC_JSON_TPL_DECLARATION -void json_pointer::flatten(const std::string& reference_string, - const NLOHMANN_BASIC_JSON_TPL& value, - NLOHMANN_BASIC_JSON_TPL& result) -{ - switch (value.m_type) - { - case detail::value_t::array: - { - if (value.m_value.array->empty()) - { - // flatten empty array as null - result[reference_string] = nullptr; - } - else - { - // iterate array and use index as reference string - for (std::size_t i = 0; i < value.m_value.array->size(); ++i) - { - flatten(reference_string + "/" + std::to_string(i), - value.m_value.array->operator[](i), result); - } - } - break; - } - - case detail::value_t::object: - { - if (value.m_value.object->empty()) - { - // flatten empty object as null - result[reference_string] = nullptr; - } - else - { - // iterate object and use keys as reference string - for (const auto& element : *value.m_value.object) - { - flatten(reference_string + "/" + escape(element.first), element.second, result); - } - } - break; - } - - default: - { - // add primitive value with its reference string - result[reference_string] = value; - break; - } - } -} - -NLOHMANN_BASIC_JSON_TPL_DECLARATION -NLOHMANN_BASIC_JSON_TPL -json_pointer::unflatten(const NLOHMANN_BASIC_JSON_TPL& value) -{ - if (JSON_UNLIKELY(not value.is_object())) - { - JSON_THROW(detail::type_error::create(314, "only objects can be unflattened")); - } - - NLOHMANN_BASIC_JSON_TPL result; - - // iterate the JSON object values - for (const auto& element : *value.m_value.object) - { - if (JSON_UNLIKELY(not element.second.is_primitive())) + else { - JSON_THROW(detail::type_error::create(315, "values in object must be primitive")); + *this = apply_patch; } - - // assign value to reference pointed to by JSON pointer; Note that if - // the JSON pointer is "" (i.e., points to the whole value), function - // get_and_create returns a reference to result itself. An assignment - // will then create a primitive value. - json_pointer(element.first).get_and_create(result) = element.second; } - return result; -} - -inline bool operator==(json_pointer const& lhs, json_pointer const& rhs) noexcept -{ - return (lhs.reference_tokens == rhs.reference_tokens); -} - -inline bool operator!=(json_pointer const& lhs, json_pointer const& rhs) noexcept -{ - return not (lhs == rhs); -} + /// @} +}; } // namespace nlohmann - /////////////////////// // nonmember support // /////////////////////// @@ -14766,20 +20291,6 @@ inline bool operator!=(json_pointer const& lhs, json_pointer const& rhs) noexcep // specialization of std::swap, and std::hash namespace std { -/*! -@brief exchanges the values of two JSON objects - -@since version 1.0.0 -*/ -template<> -inline void swap(nlohmann::json& j1, - nlohmann::json& j2) noexcept( - is_nothrow_move_constructible<nlohmann::json>::value and - is_nothrow_move_assignable<nlohmann::json>::value - ) -{ - j1.swap(j2); -} /// hash value for JSON objects template<> @@ -14815,6 +20326,20 @@ struct less< ::nlohmann::detail::value_t> } }; +/*! +@brief exchanges the values of two JSON objects + +@since version 1.0.0 +*/ +template<> +inline void swap<nlohmann::json>(nlohmann::json& j1, nlohmann::json& j2) noexcept( + is_nothrow_move_constructible<nlohmann::json>::value and + is_nothrow_move_assignable<nlohmann::json>::value +) +{ + j1.swap(j2); +} + } // namespace std /*! @@ -14853,6 +20378,9 @@ inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std return nlohmann::json::json_pointer(std::string(s, n)); } +// #include <nlohmann/detail/macro_unscope.hpp> + + // restore GCC/clang diagnostic settings #if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic pop @@ -14862,13 +20390,17 @@ inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std #endif // clean up +#undef JSON_INTERNAL_CATCH #undef JSON_CATCH #undef JSON_THROW #undef JSON_TRY #undef JSON_LIKELY #undef JSON_UNLIKELY #undef JSON_DEPRECATED +#undef JSON_HAS_CPP_14 +#undef JSON_HAS_CPP_17 #undef NLOHMANN_BASIC_JSON_TPL_DECLARATION #undef NLOHMANN_BASIC_JSON_TPL + #endif diff --git a/src/resolve-system-dependencies/local.mk b/src/resolve-system-dependencies/local.mk index 8792a4a252fa..f9db16268be0 100644 --- a/src/resolve-system-dependencies/local.mk +++ b/src/resolve-system-dependencies/local.mk @@ -6,6 +6,6 @@ resolve-system-dependencies_DIR := $(d) resolve-system-dependencies_INSTALL_DIR := $(libexecdir)/nix -resolve-system-dependencies_LIBS := libstore libmain libutil libformat +resolve-system-dependencies_LIBS := libstore libmain libutil resolve-system-dependencies_SOURCES := $(d)/resolve-system-dependencies.cc diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index 7365a550e57f..eb58ae7c12a8 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -52,9 +52,6 @@ export _NIX_FORCE_HTTP_BINARY_CACHE_STORE=1 basicTests -unset _NIX_FORCE_HTTP_BINARY_CACHE_STORE - - # Test whether Nix notices if the NAR doesn't match the hash in the NAR info. clearStore @@ -79,18 +76,29 @@ if nix-store --substituters "file://$cacheDir" -r $outPath; then fi -# Test whether fallback works if we have cached info but the -# corresponding NAR has disappeared. +# Test whether fallback works if a NAR has disappeared. This does not require --fallback. clearStore -nix-build --substituters "file://$cacheDir" dependencies.nix --dry-run # get info +mv $cacheDir/nar $cacheDir/nar2 + +nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result -mkdir $cacheDir/tmp -mv $cacheDir/*.nar* $cacheDir/tmp/ +mv $cacheDir/nar2 $cacheDir/nar -NIX_DEBUG_SUBST=1 nix-build --substituters "file://$cacheDir" dependencies.nix -o $TEST_ROOT/result --fallback -mv $cacheDir/tmp/* $cacheDir/ +# Test whether fallback works if a NAR is corrupted. This does require --fallback. +clearStore + +mv $cacheDir/nar $cacheDir/nar2 +mkdir $cacheDir/nar +for i in $(cd $cacheDir/nar2 && echo *); do touch $cacheDir/nar/$i; done + +(! nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result) + +nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result --fallback + +rm -rf $cacheDir/nar +mv $cacheDir/nar2 $cacheDir/nar # Test whether building works if the binary cache contains an @@ -107,6 +115,7 @@ if [ -n "$HAVE_SODIUM" ]; then # Create a signed binary cache. clearCache +clearCacheCache declare -a res=($(nix-store --generate-binary-cache-key test.nixos.org-1 $TEST_ROOT/sk1 $TEST_ROOT/pk1 )) publicKey="$(cat $TEST_ROOT/pk1)" @@ -117,7 +126,7 @@ badKey="$(cat $TEST_ROOT/pk2)" res=($(nix-store --generate-binary-cache-key foo.nixos.org-1 $TEST_ROOT/sk3 $TEST_ROOT/pk3)) otherKey="$(cat $TEST_ROOT/pk3)" -nix copy --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath +_NIX_FORCE_HTTP_BINARY_CACHE_STORE= nix copy --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath # Downloading should fail if we don't provide a key. diff --git a/tests/brotli.sh b/tests/brotli.sh index 645dd4214ec6..a3c6e55a8fad 100644 --- a/tests/brotli.sh +++ b/tests/brotli.sh @@ -1,10 +1,5 @@ source common.sh - -# Only test if we found brotli libraries -# (CLI tool is likely unavailable if libraries are missing) -if [ -n "$HAVE_BROTLI" ]; then - clearStore clearCache @@ -24,5 +19,3 @@ nix copy --from $cacheURI $outPath --no-check-sigs HASH2=$(nix hash-path $outPath) [[ $HASH = $HASH2 ]] - -fi # HAVE_BROTLI diff --git a/tests/build-remote.sh b/tests/build-remote.sh index 9bca0f4a3856..ddd68f327a15 100644 --- a/tests/build-remote.sh +++ b/tests/build-remote.sh @@ -11,7 +11,8 @@ rm -rf $TEST_ROOT/store0 $TEST_ROOT/store1 nix build -f build-hook.nix -o $TEST_ROOT/result --max-jobs 0 \ --sandbox-paths /nix/store --sandbox-build-dir /build-tmp \ - --builders "$TEST_ROOT/store0; $TEST_ROOT/store1 - - 1 1 foo" + --builders "$TEST_ROOT/store0; $TEST_ROOT/store1 - - 1 1 foo" \ + --system-features foo outPath=$TEST_ROOT/result diff --git a/tests/check-refs.sh b/tests/check-refs.sh index 34ee22cfc8f7..16bbabc40985 100644 --- a/tests/check-refs.sh +++ b/tests/check-refs.sh @@ -1,5 +1,7 @@ source common.sh +clearStore + RESULT=$TEST_ROOT/result dep=$(nix-build -o $RESULT check-refs.nix -A dep) diff --git a/tests/check-reqs.sh b/tests/check-reqs.sh index 77689215def1..e9f65fc2a6d3 100644 --- a/tests/check-reqs.sh +++ b/tests/check-reqs.sh @@ -1,5 +1,7 @@ source common.sh +clearStore + RESULT=$TEST_ROOT/result nix-build -o $RESULT check-reqs.nix -A test1 diff --git a/tests/common.sh.in b/tests/common.sh.in index 195205988afb..6a523ca9d832 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -31,7 +31,6 @@ export xmllint="@xmllint@" export SHELL="@bash@" export PAGER=cat export HAVE_SODIUM="@HAVE_SODIUM@" -export HAVE_BROTLI="@HAVE_BROTLI@" export version=@PACKAGE_VERSION@ export system=@system@ @@ -86,21 +85,16 @@ killDaemon() { trap "" EXIT } -canUseSandbox() { - if [[ $(uname) != Linux ]]; then return 1; fi +if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then + _canUseSandbox=1 +fi - if [ ! -L /proc/self/ns/user ]; then - echo "Kernel doesn't support user namespaces, skipping this test..." +canUseSandbox() { + if [[ ! $_canUseSandbox ]]; then + echo "Sandboxing not supported, skipping this test..." return 1 fi - if [ -e /proc/sys/kernel/unprivileged_userns_clone ]; then - if [ "$(cat /proc/sys/kernel/unprivileged_userns_clone)" != 1 ]; then - echo "Unprivileged user namespaces disabled by sysctl, skipping this test..." - return 1 - fi - fi - return 0 } diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh index 530ac7bb813c..4c46bdf0465b 100644 --- a/tests/fetchGit.sh +++ b/tests/fetchGit.sh @@ -9,7 +9,7 @@ clearStore repo=$TEST_ROOT/git -rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix/git +rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix/gitv2 git init $repo git -C $repo config user.email "foobar@example.com" @@ -129,7 +129,7 @@ path5=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outP # Nuke the cache -rm -rf $TEST_HOME/.cache/nix/git +rm -rf $TEST_HOME/.cache/nix/gitv2 # Try again, but without 'git' on PATH NIX=$(command -v nix) diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh index 9bbf044f7329..ec3399b08d00 100644 --- a/tests/fetchurl.sh +++ b/tests/fetchurl.sh @@ -18,6 +18,17 @@ outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh cmp $outPath fetchurl.sh +# Now using an SRI hash. +clearStore + +hash=$(nix hash-file ./fetchurl.sh) + +[[ $hash =~ ^sha256- ]] + +outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr hash $hash --no-out-link --hashed-mirrors '') + +cmp $outPath fetchurl.sh + # Test the hashed mirror feature. clearStore diff --git a/tests/gc.sh b/tests/gc.sh index 0adb05bf173a..8b4f8d282184 100644 --- a/tests/gc.sh +++ b/tests/gc.sh @@ -7,7 +7,7 @@ outPath=$(nix-store -rvv "$drvPath") rm -f "$NIX_STATE_DIR"/gcroots/foo ln -sf $outPath "$NIX_STATE_DIR"/gcroots/foo -[ "$(nix-store -q --roots $outPath)" = "$NIX_STATE_DIR"/gcroots/foo ] +[ "$(nix-store -q --roots $outPath)" = "$NIX_STATE_DIR/gcroots/foo -> $outPath" ] nix-store --gc --print-roots | grep $outPath nix-store --gc --print-live | grep $outPath diff --git a/tests/hash.sh b/tests/hash.sh index 9f234bc635b0..4cfc97901012 100644 --- a/tests/hash.sh +++ b/tests/hash.sh @@ -2,7 +2,7 @@ source common.sh try () { printf "%s" "$2" > $TEST_ROOT/vector - hash=$(nix-hash $EXTRA --flat --type "$1" $TEST_ROOT/vector) + hash=$(nix hash-file --base16 $EXTRA --type "$1" $TEST_ROOT/vector) if test "$hash" != "$3"; then echo "hash $1, expected $3, got $hash" exit 1 @@ -33,6 +33,12 @@ EXTRA=--base32 try sha256 "abc" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" EXTRA= +EXTRA=--sri +try sha512 "" "sha512-z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" +try sha512 "abc" "sha512-3a81oZNherrMQXNJriBBMRLm+k6JqX6iCp7u5ktV05ohkpkqJ0/BqDa6PCOj/uu9RU1EI2Q86A4qmslPpUyknw==" +try sha512 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "sha512-IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ==" +try sha256 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "sha256-JI1qYdIGOLjlwCaTDD5gOaM85Flk/yFn9uzt1BnbBsE=" + try2 () { hash=$(nix-hash --type "$1" $TEST_ROOT/hash-path) if test "$hash" != "$2"; then @@ -65,12 +71,16 @@ try2 md5 "f78b733a68f5edbdf9413899339eaa4a" try3() { h64=$(nix to-base64 --type "$1" "$2") [ "$h64" = "$4" ] + sri=$(nix to-sri --type "$1" "$2") + [ "$sri" = "$1-$4" ] h32=$(nix-hash --type "$1" --to-base32 "$2") [ "$h32" = "$3" ] h16=$(nix-hash --type "$1" --to-base16 "$h32") [ "$h16" = "$2" ] h16=$(nix to-base16 --type "$1" "$h64") [ "$h16" = "$2" ] + h16=$(nix to-base16 "$sri") + [ "$h16" = "$2" ] } try3 sha1 "800d59cfcd3c05e900cb4e214be48f6b886a08df" "vw46m23bizj4n8afrc0fj19wrp7mj3c0" "gA1Zz808BekAy04hS+SPa4hqCN8=" try3 sha256 "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=" diff --git a/tests/init.sh b/tests/init.sh index e5353598bcc4..19a12c1e2d9e 100644 --- a/tests/init.sh +++ b/tests/init.sh @@ -16,6 +16,7 @@ mkdir "$NIX_CONF_DIR" cat > "$NIX_CONF_DIR"/nix.conf <<EOF build-users-group = keep-derivations = false +sandbox = false include nix.conf.extra EOF diff --git a/tests/lang/binary-data b/tests/lang/binary-data new file mode 100644 index 000000000000..06d740502001 --- /dev/null +++ b/tests/lang/binary-data Binary files differdiff --git a/tests/lang/eval-fail-hashfile-missing.nix b/tests/lang/eval-fail-hashfile-missing.nix new file mode 100644 index 000000000000..ce098b82380a --- /dev/null +++ b/tests/lang/eval-fail-hashfile-missing.nix @@ -0,0 +1,5 @@ +let + paths = [ ./this-file-is-definitely-not-there-7392097 "/and/neither/is/this/37293620" ]; +in + toString (builtins.concatLists (map (hash: map (builtins.hashFile hash) paths) ["md5" "sha1" "sha256" "sha512"])) + diff --git a/tests/lang/eval-okay-arithmetic.exp b/tests/lang/eval-okay-arithmetic.exp index b195055b7a09..5c54d10b7b47 100644 --- a/tests/lang/eval-okay-arithmetic.exp +++ b/tests/lang/eval-okay-arithmetic.exp @@ -1 +1 @@ -2188 +2216 diff --git a/tests/lang/eval-okay-arithmetic.nix b/tests/lang/eval-okay-arithmetic.nix index bbbbc4691d75..7e9e6a0b666e 100644 --- a/tests/lang/eval-okay-arithmetic.nix +++ b/tests/lang/eval-okay-arithmetic.nix @@ -26,6 +26,10 @@ let { (56088 / 123 / 2) (3 + 4 * const 5 0 - 6 / id 2) + (builtins.bitAnd 12 10) # 0b1100 & 0b1010 = 8 + (builtins.bitOr 12 10) # 0b1100 | 0b1010 = 14 + (builtins.bitXor 12 10) # 0b1100 ^ 0b1010 = 6 + (if 3 < 7 then 1 else err) (if 7 < 3 then err else 1) (if 3 < 3 then err else 1) diff --git a/tests/lang/eval-okay-builtins-add.exp b/tests/lang/eval-okay-builtins-add.exp new file mode 100644 index 000000000000..0350b518a7ec --- /dev/null +++ b/tests/lang/eval-okay-builtins-add.exp @@ -0,0 +1 @@ +[ 5 4 "int" "tt" "float" 4 ] diff --git a/tests/lang/eval-okay-builtins-add.nix b/tests/lang/eval-okay-builtins-add.nix new file mode 100644 index 000000000000..c841816222a5 --- /dev/null +++ b/tests/lang/eval-okay-builtins-add.nix @@ -0,0 +1,8 @@ +[ +(builtins.add 2 3) +(builtins.add 2 2) +(builtins.typeOf (builtins.add 2 2)) +("t" + "t") +(builtins.typeOf (builtins.add 2.0 2)) +(builtins.add 2.0 2) +] diff --git a/tests/lang/eval-okay-concatmap.exp b/tests/lang/eval-okay-concatmap.exp new file mode 100644 index 000000000000..3b8be7739deb --- /dev/null +++ b/tests/lang/eval-okay-concatmap.exp @@ -0,0 +1 @@ +[ [ 1 3 5 7 9 ] [ "a" "z" "b" "z" ] ] diff --git a/tests/lang/eval-okay-concatmap.nix b/tests/lang/eval-okay-concatmap.nix new file mode 100644 index 000000000000..97da5d37a412 --- /dev/null +++ b/tests/lang/eval-okay-concatmap.nix @@ -0,0 +1,5 @@ +with import ./lib.nix; + +[ (builtins.concatMap (x: if x / 2 * 2 == x then [] else [ x ]) (range 0 10)) + (builtins.concatMap (x: [x] ++ ["z"]) ["a" "b"]) +] diff --git a/tests/lang/eval-okay-context-introspection.exp b/tests/lang/eval-okay-context-introspection.exp new file mode 100644 index 000000000000..27ba77ddaf61 --- /dev/null +++ b/tests/lang/eval-okay-context-introspection.exp @@ -0,0 +1 @@ +true diff --git a/tests/lang/eval-okay-context-introspection.nix b/tests/lang/eval-okay-context-introspection.nix new file mode 100644 index 000000000000..43178bd2eef9 --- /dev/null +++ b/tests/lang/eval-okay-context-introspection.nix @@ -0,0 +1,24 @@ +let + drv = derivation { + name = "fail"; + builder = "/bin/false"; + system = "x86_64-linux"; + outputs = [ "out" "foo" ]; + }; + + path = "${./eval-okay-context-introspection.nix}"; + + desired-context = { + "${builtins.unsafeDiscardStringContext path}" = { + path = true; + }; + "${builtins.unsafeDiscardStringContext drv.drvPath}" = { + outputs = [ "foo" "out" ]; + allOutputs = true; + }; + }; + + legit-context = builtins.getContext "${path}${drv.outPath}${drv.foo.outPath}${drv.drvPath}"; + + constructed-context = builtins.getContext (builtins.appendContext "" desired-context); +in legit-context == constructed-context diff --git a/tests/lang/eval-okay-float.exp b/tests/lang/eval-okay-float.exp new file mode 100644 index 000000000000..3c50a8adce86 --- /dev/null +++ b/tests/lang/eval-okay-float.exp @@ -0,0 +1 @@ +[ 3.4 3.5 2.5 1.5 ] diff --git a/tests/lang/eval-okay-float.nix b/tests/lang/eval-okay-float.nix new file mode 100644 index 000000000000..b2702c7b1668 --- /dev/null +++ b/tests/lang/eval-okay-float.nix @@ -0,0 +1,6 @@ +[ + (1.1 + 2.3) + (builtins.add (0.5 + 0.5) (2.0 + 0.5)) + ((0.5 + 0.5) * (2.0 + 0.5)) + ((1.5 + 1.5) / (0.5 * 4.0)) +] diff --git a/tests/lang/eval-okay-fromTOML.exp b/tests/lang/eval-okay-fromTOML.exp new file mode 100644 index 000000000000..5b9d47122cc5 --- /dev/null +++ b/tests/lang/eval-okay-fromTOML.exp @@ -0,0 +1 @@ +[ { clients = { data = [ [ "gamma" "delta" ] [ 1 2 ] ]; hosts = [ "alpha" "omega" ]; }; database = { connection_max = 5000; enabled = true; ports = [ 8001 8001 8002 ]; server = "192.168.1.1"; }; owner = { name = "Tom Preston-Werner"; }; servers = { alpha = { dc = "eqdc10"; ip = "10.0.0.1"; }; beta = { dc = "eqdc10"; ip = "10.0.0.2"; }; }; title = "TOML Example"; } { "'key2'" = "value"; "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; g = { h = { i = { }; }; }; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { "'l'" = { }; }; }; key = "value"; name = "Orange"; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; } { metadata = { "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"; "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"; "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"; "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"; }; package = [ { dependencies = [ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "aho-corasick"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.6.4"; } { name = "ansi_term"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.9.0"; } { dependencies = [ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "atty"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.2.10"; } ]; } ] diff --git a/tests/lang/eval-okay-fromTOML.nix b/tests/lang/eval-okay-fromTOML.nix new file mode 100644 index 000000000000..8e7cbd1c61e9 --- /dev/null +++ b/tests/lang/eval-okay-fromTOML.nix @@ -0,0 +1,184 @@ +[ + + (builtins.fromTOML '' + # This is a TOML document. + + title = "TOML Example" + + [owner] + name = "Tom Preston-Werner" + #dob = 1979-05-27T07:32:00-08:00 # First class dates + + [database] + server = "192.168.1.1" + ports = [ 8001, 8001, 8002 ] + connection_max = 5000 + enabled = true + + [servers] + + # Indentation (tabs and/or spaces) is allowed but not required + [servers.alpha] + ip = "10.0.0.1" + dc = "eqdc10" + + [servers.beta] + ip = "10.0.0.2" + dc = "eqdc10" + + [clients] + data = [ ["gamma", "delta"], [1, 2] ] + + # Line breaks are OK when inside arrays + hosts = [ + "alpha", + "omega" + ] + '') + + (builtins.fromTOML '' + key = "value" + bare_key = "value" + bare-key = "value" + 1234 = "value" + + "127.0.0.1" = "value" + "character encoding" = "value" + "ʎǝʞ" = "value" + 'key2' = "value" + #'quoted "value"' = "value" + + name = "Orange" + + # FIXME: cpptoml doesn't handle dotted keys properly yet. + #physical.color = "orange" + #physical.shape = "round" + #site."google.com" = true + + #a.b.c = 1 + #a.d = 2 + + str = "I'm a string. \"You can quote me\". Name\tJos\u00E9\nLocation\tSF." + + int1 = +99 + int2 = 42 + int3 = 0 + int4 = -17 + int5 = 1_000 + int6 = 5_349_221 + int7 = 1_2_3_4_5 + + # FIXME: cpptoml doesn't support these yet: + + #hex1 = 0xDEADBEEF + #hex2 = 0xdeadbeef + #hex3 = 0xdead_beef + + #oct1 = 0o01234567 + #oct2 = 0o755 + + #bin1 = 0b11010110 + + flt1 = +1.0 + flt2 = 3.1415 + flt3 = -0.01 + flt4 = 5e+22 + flt5 = 1e6 + flt6 = -2E-2 + flt7 = 6.626e-34 + flt8 = 9_224_617.445_991_228_313 + + bool1 = true + bool2 = false + + # FIXME: not supported because Nix doesn't have a date/time type. + #odt1 = 1979-05-27T07:32:00Z + #odt2 = 1979-05-27T00:32:00-07:00 + #odt3 = 1979-05-27T00:32:00.999999-07:00 + #odt4 = 1979-05-27 07:32:00Z + #ldt1 = 1979-05-27T07:32:00 + #ldt2 = 1979-05-27T00:32:00.999999 + #ld1 = 1979-05-27 + #lt1 = 07:32:00 + #lt2 = 00:32:00.999999 + + arr1 = [ 1, 2, 3 ] + arr2 = [ "red", "yellow", "green" ] + arr3 = [ [ 1, 2 ], [3, 4, 5] ] + arr4 = [ "all", 'strings', """are the same""", ''''type''''] + arr5 = [ [ 1, 2 ], ["a", "b", "c"] ] + + arr7 = [ + 1, 2, 3 + ] + + arr8 = [ + 1, + 2, # this is ok + ] + + [table-1] + key1 = "some string" + key2 = 123 + + + [table-2] + key1 = "another string" + key2 = 456 + + #[dog."tater.man"] + #type.name = "pug" + + [a.b.c] + [ d.e.f ] + [ g . h . i ] + [ j . "ʞ" . 'l' ] + [x.y.z.w] + + name = { first = "Tom", last = "Preston-Werner" } + point = { x = 1, y = 2 } + #animal = { type.name = "pug" } + + [[products]] + name = "Hammer" + sku = 738594937 + + [[products]] + + [[products]] + name = "Nail" + sku = 284758393 + color = "gray" + '') + + (builtins.fromTOML '' + [[package]] + name = "aho-corasick" + version = "0.6.4" + source = "registry+https://github.com/rust-lang/crates.io-index" + dependencies = [ + "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + ] + + [[package]] + name = "ansi_term" + version = "0.9.0" + source = "registry+https://github.com/rust-lang/crates.io-index" + + [[package]] + name = "atty" + version = "0.2.10" + source = "registry+https://github.com/rust-lang/crates.io-index" + dependencies = [ + "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", + "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + ] + + [metadata] + "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4" + "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" + "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6" + "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef" + '') +] diff --git a/tests/lang/eval-okay-getattrpos-undefined.exp b/tests/lang/eval-okay-getattrpos-undefined.exp new file mode 100644 index 000000000000..19765bd501b6 --- /dev/null +++ b/tests/lang/eval-okay-getattrpos-undefined.exp @@ -0,0 +1 @@ +null diff --git a/tests/lang/eval-okay-getattrpos-undefined.nix b/tests/lang/eval-okay-getattrpos-undefined.nix new file mode 100644 index 000000000000..14dd38f7734c --- /dev/null +++ b/tests/lang/eval-okay-getattrpos-undefined.nix @@ -0,0 +1 @@ +builtins.unsafeGetAttrPos "abort" builtins diff --git a/tests/lang/eval-okay-hash.exp b/tests/lang/eval-okay-hash.exp index d720a082ddb3..e69de29bb2d1 100644 --- a/tests/lang/eval-okay-hash.exp +++ b/tests/lang/eval-okay-hash.exp @@ -1 +0,0 @@ -[ "d41d8cd98f00b204e9800998ecf8427e" "6c69ee7f211c640419d5366cc076ae46" "bb3438fbabd460ea6dbd27d153e2233b" "da39a3ee5e6b4b0d3255bfef95601890afd80709" "cd54e8568c1b37cf1e5badb0779bcbf382212189" "6d12e10b1d331dad210e47fd25d4f260802b7e77" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" "900a4469df00ccbfd0c145c6d1e4b7953dd0afafadd7534e3a4019e8d38fc663" "ad0387b3bd8652f730ca46d25f9c170af0fd589f42e7f23f5a9e6412d97d7e56" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" "9d0886f8c6b389398a16257bc79780fab9831c7fc11c8ab07fa732cb7b348feade382f92617c9c5305fefba0af02ab5fd39a587d330997ff5bd0db19f7666653" "21644b72aa259e5a588cd3afbafb1d4310f4889680f6c83b9d531596a5a284f34dbebff409d23bcc86aee6bad10c891606f075c6f4755cb536da27db5693f3a7" ] diff --git a/tests/lang/eval-okay-hashfile.exp b/tests/lang/eval-okay-hashfile.exp new file mode 100644 index 000000000000..ff1e8293ef22 --- /dev/null +++ b/tests/lang/eval-okay-hashfile.exp @@ -0,0 +1 @@ +[ "d3b07384d113edec49eaa6238ad5ff00" "0f343b0931126a20f133d67c2b018a3b" "f1d2d2f924e986ac86fdf7b36c94bcdf32beec15" "60cacbf3d72e1e7834203da608037b1bf83b40e8" "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c" "5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef" "0cf9180a764aba863a67b6d72f0918bc131c6772642cb2dce5a34f0a702f9470ddc2bf125c12198b1995c233c34b4afd346c54a2334c350a948a51b6e8b4e6b6" "8efb4f73c5655351c444eb109230c556d39e2c7624e9c11abc9e3fb4b9b9254218cc5085b454a9698d085cfa92198491f07a723be4574adc70617b73eb0b6461" ] diff --git a/tests/lang/eval-okay-hashfile.nix b/tests/lang/eval-okay-hashfile.nix new file mode 100644 index 000000000000..aff5a1856814 --- /dev/null +++ b/tests/lang/eval-okay-hashfile.nix @@ -0,0 +1,4 @@ +let + paths = [ ./data ./binary-data ]; +in + builtins.concatLists (map (hash: map (builtins.hashFile hash) paths) ["md5" "sha1" "sha256" "sha512"]) diff --git a/tests/lang/eval-okay-hashstring.exp b/tests/lang/eval-okay-hashstring.exp new file mode 100644 index 000000000000..d720a082ddb3 --- /dev/null +++ b/tests/lang/eval-okay-hashstring.exp @@ -0,0 +1 @@ +[ "d41d8cd98f00b204e9800998ecf8427e" "6c69ee7f211c640419d5366cc076ae46" "bb3438fbabd460ea6dbd27d153e2233b" "da39a3ee5e6b4b0d3255bfef95601890afd80709" "cd54e8568c1b37cf1e5badb0779bcbf382212189" "6d12e10b1d331dad210e47fd25d4f260802b7e77" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" "900a4469df00ccbfd0c145c6d1e4b7953dd0afafadd7534e3a4019e8d38fc663" "ad0387b3bd8652f730ca46d25f9c170af0fd589f42e7f23f5a9e6412d97d7e56" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" "9d0886f8c6b389398a16257bc79780fab9831c7fc11c8ab07fa732cb7b348feade382f92617c9c5305fefba0af02ab5fd39a587d330997ff5bd0db19f7666653" "21644b72aa259e5a588cd3afbafb1d4310f4889680f6c83b9d531596a5a284f34dbebff409d23bcc86aee6bad10c891606f075c6f4755cb536da27db5693f3a7" ] diff --git a/tests/lang/eval-okay-hash.nix b/tests/lang/eval-okay-hashstring.nix index b0f62b245ca8..b0f62b245ca8 100644 --- a/tests/lang/eval-okay-hash.nix +++ b/tests/lang/eval-okay-hashstring.nix diff --git a/tests/lang/eval-okay-mapattrs.exp b/tests/lang/eval-okay-mapattrs.exp new file mode 100644 index 000000000000..3f113f17bab1 --- /dev/null +++ b/tests/lang/eval-okay-mapattrs.exp @@ -0,0 +1 @@ +{ x = "x-foo"; y = "y-bar"; } diff --git a/tests/lang/eval-okay-mapattrs.nix b/tests/lang/eval-okay-mapattrs.nix new file mode 100644 index 000000000000..f075b6275e5a --- /dev/null +++ b/tests/lang/eval-okay-mapattrs.nix @@ -0,0 +1,3 @@ +with import ./lib.nix; + +builtins.mapAttrs (name: value: name + "-" + value) { x = "foo"; y = "bar"; } diff --git a/tests/lang/eval-okay-nested-with.exp b/tests/lang/eval-okay-nested-with.exp new file mode 100644 index 000000000000..0cfbf08886fc --- /dev/null +++ b/tests/lang/eval-okay-nested-with.exp @@ -0,0 +1 @@ +2 diff --git a/tests/lang/eval-okay-nested-with.nix b/tests/lang/eval-okay-nested-with.nix new file mode 100644 index 000000000000..ba9d79aa79b1 --- /dev/null +++ b/tests/lang/eval-okay-nested-with.nix @@ -0,0 +1,3 @@ +with { x = 1; }; +with { x = 2; }; +x diff --git a/tests/lang/eval-okay-types.exp b/tests/lang/eval-okay-types.exp index 9a8ea0bcbd8a..92a15329935a 100644 --- a/tests/lang/eval-okay-types.exp +++ b/tests/lang/eval-okay-types.exp @@ -1 +1 @@ -[ true false true false true false true false true true true true true true true true true true true false true false "int" "bool" "string" "null" "set" "list" "lambda" "lambda" "lambda" "lambda" ] +[ true false true false true false true false true true true true true true true true true true true false true true true false "int" "bool" "string" "null" "set" "list" "lambda" "lambda" "lambda" "lambda" ] diff --git a/tests/lang/eval-okay-types.nix b/tests/lang/eval-okay-types.nix index a34775f5e602..9b58be5d1dd4 100644 --- a/tests/lang/eval-okay-types.nix +++ b/tests/lang/eval-okay-types.nix @@ -20,6 +20,8 @@ with builtins; (isFloat (1 - 2.0)) (isBool (true && false)) (isBool null) + (isPath /nix/store) + (isPath ./.) (isAttrs { x = 123; }) (isAttrs null) (typeOf (3 * 4)) diff --git a/tests/lang/parse-fail-uft8.nix b/tests/lang/parse-fail-uft8.nix new file mode 100644 index 000000000000..34948d48aed2 --- /dev/null +++ b/tests/lang/parse-fail-uft8.nix @@ -0,0 +1 @@ +123 é 4 diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh index acfd46c54170..52967d07dda2 100644 --- a/tests/linux-sandbox.sh +++ b/tests/linux-sandbox.sh @@ -25,3 +25,6 @@ nix path-info -r $outPath | grep input-2 nix ls-store -R -l $outPath | grep foobar nix cat-store $outPath/foobar | grep FOOBAR + +# Test --check without hash rewriting. +nix-build dependencies.nix --no-out-link --check --sandbox-paths /nix/store diff --git a/tests/local.mk b/tests/local.mk index 9df0adf1bfd8..1ff68348b3c7 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -25,7 +25,8 @@ nix_tests = \ pure-eval.sh \ check.sh \ plugins.sh \ - search.sh + search.sh \ + nix-copy-ssh.sh # parallel.sh install-tests += $(foreach x, $(nix_tests), tests/$(x)) diff --git a/tests/nix-copy-ssh.sh b/tests/nix-copy-ssh.sh new file mode 100644 index 000000000000..6aba667a45a6 --- /dev/null +++ b/tests/nix-copy-ssh.sh @@ -0,0 +1,20 @@ +source common.sh + +clearStore +clearCache + +remoteRoot=$TEST_ROOT/store2 +chmod -R u+w "$remoteRoot" || true +rm -rf "$remoteRoot" + +outPath=$(nix-build dependencies.nix) + +nix copy --to "ssh://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath + +[ -f $remoteRoot$outPath/foobar ] + +clearStore + +nix copy --no-check-sigs --from "ssh://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath + +[ -f $outPath/foobar ] diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh index d25c456cedfb..6024ea399750 100644 --- a/tests/nix-shell.sh +++ b/tests/nix-shell.sh @@ -4,12 +4,19 @@ clearStore # Test nix-shell -A export IMPURE_VAR=foo +export SELECTED_IMPURE_VAR=baz export NIX_BUILD_SHELL=$SHELL output=$(nix-shell --pure shell.nix -A shellDrv --run \ 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') [ "$output" = " - foo - bar" ] +# Test --keep +output=$(nix-shell --pure --keep SELECTED_IMPURE_VAR shell.nix -A shellDrv --run \ + 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $SELECTED_IMPURE_VAR"') + +[ "$output" = " - foo - bar - baz" ] + # Test nix-shell on a .drv [[ $(nix-shell --pure $(nix-instantiate shell.nix -A shellDrv) --run \ 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] diff --git a/tests/plugins/plugintest.cc b/tests/plugins/plugintest.cc index 8da15ebabd7d..c085d33295be 100644 --- a/tests/plugins/plugintest.cc +++ b/tests/plugins/plugintest.cc @@ -1,16 +1,21 @@ -#include "globals.hh" +#include "config.hh" #include "primops.hh" using namespace nix; -static BaseSetting<bool> settingSet{false, "setting-set", +struct MySettings : Config +{ + Setting<bool> settingSet{this, false, "setting-set", "Whether the plugin-defined setting was set"}; +}; + +MySettings mySettings; -static RegisterSetting rs(&settingSet); +static GlobalConfig::Register rs(&mySettings); static void prim_anotherNull (EvalState & state, const Pos & pos, Value ** args, Value & v) { - if (settingSet) + if (mySettings.settingSet) mkNull(v); else mkBool(v, false); diff --git a/tests/remote-builds.nix b/tests/remote-builds.nix index d7a4b21989e5..b867f13b4995 100644 --- a/tests/remote-builds.nix +++ b/tests/remote-builds.nix @@ -8,8 +8,8 @@ makeTest ( let - # The configuration of the build slaves. - slave = + # The configuration of the remote builders. + builder = { config, pkgs, ... }: { services.openssh.enable = true; virtualisation.writableStore = true; @@ -36,21 +36,21 @@ in { nodes = - { slave1 = slave; - slave2 = slave; + { builder1 = builder; + builder2 = builder; client = { config, pkgs, ... }: { nix.maxJobs = 0; # force remote building nix.distributedBuilds = true; nix.buildMachines = - [ { hostName = "slave1"; + [ { hostName = "builder1"; sshUser = "root"; sshKey = "/root/.ssh/id_ed25519"; system = "i686-linux"; maxJobs = 1; } - { hostName = "slave2"; + { hostName = "builder2"; sshUser = "root"; sshKey = "/root/.ssh/id_ed25519"; system = "i686-linux"; @@ -75,33 +75,33 @@ in $client->copyFileFromHost("key", "/root/.ssh/id_ed25519"); $client->succeed("chmod 600 /root/.ssh/id_ed25519"); - # Install the SSH key on the slaves. + # Install the SSH key on the builders. $client->waitForUnit("network.target"); - foreach my $slave ($slave1, $slave2) { - $slave->succeed("mkdir -p -m 700 /root/.ssh"); - $slave->copyFileFromHost("key.pub", "/root/.ssh/authorized_keys"); - $slave->waitForUnit("sshd"); - $client->succeed("ssh -o StrictHostKeyChecking=no " . $slave->name() . " 'echo hello world'"); + foreach my $builder ($builder1, $builder2) { + $builder->succeed("mkdir -p -m 700 /root/.ssh"); + $builder->copyFileFromHost("key.pub", "/root/.ssh/authorized_keys"); + $builder->waitForUnit("sshd"); + $client->succeed("ssh -o StrictHostKeyChecking=no " . $builder->name() . " 'echo hello world'"); } - # Perform a build and check that it was performed on the slave. + # Perform a build and check that it was performed on the builder. my $out = $client->succeed( "nix-build ${expr nodes.client.config 1} 2> build-output", "grep -q Hello build-output" ); - $slave1->succeed("test -e $out"); + $builder1->succeed("test -e $out"); # And a parallel build. my ($out1, $out2) = split /\s/, $client->succeed('nix-store -r $(nix-instantiate ${expr nodes.client.config 2})\!out $(nix-instantiate ${expr nodes.client.config 3})\!out'); - $slave1->succeed("test -e $out1 -o -e $out2"); - $slave2->succeed("test -e $out1 -o -e $out2"); + $builder1->succeed("test -e $out1 -o -e $out2"); + $builder2->succeed("test -e $out1 -o -e $out2"); # And a failing build. $client->fail("nix-build ${expr nodes.client.config 5}"); - # Test whether the build hook automatically skips unavailable slaves. - $slave1->block; + # Test whether the build hook automatically skips unavailable builders. + $builder1->block; $client->succeed("nix-build ${expr nodes.client.config 4}"); ''; diff --git a/tests/restricted.sh b/tests/restricted.sh index a87d8ec2c940..e02becc60e38 100644 --- a/tests/restricted.sh +++ b/tests/restricted.sh @@ -38,3 +38,14 @@ ln -sfn $(pwd)/restricted.nix $TEST_ROOT/restricted.nix nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I $TEST_ROOT -I . [[ $(nix eval --raw --restrict-eval -I . '(builtins.readFile "${import ./simple.nix}/hello")') == 'Hello World!' ]] + +# Check whether we can leak symlink information through directory traversal. +traverseDir="$(pwd)/restricted-traverse-me" +ln -sfn "$(pwd)/restricted-secret" "$(pwd)/restricted-innocent" +mkdir -p "$traverseDir" +goUp="..$(echo "$traverseDir" | sed -e 's,[^/]\+,..,g')" +output="$(nix eval --raw --restrict-eval -I "$traverseDir" \ + "(builtins.readFile \"$traverseDir/$goUp$(pwd)/restricted-innocent\")" \ + 2>&1 || :)" +echo "$output" | grep "is forbidden" +! echo "$output" | grep -F restricted-secret diff --git a/tests/search.sh b/tests/search.sh index d83427247d3e..14da3127b0d5 100644 --- a/tests/search.sh +++ b/tests/search.sh @@ -29,6 +29,11 @@ clearCache # Check search that matches nothing (( $(nix search nosuchpackageexists | wc -l) == 0 )) +# Search for multiple arguments +(( $(nix search hello empty | wc -l) == 3 )) + +# Multiple arguments will not exist +(( $(nix search hello broken | wc -l) == 0 )) ## Search expressions diff --git a/tests/signing.sh b/tests/signing.sh index 46929639199d..9e29e3fbf063 100644 --- a/tests/signing.sh +++ b/tests/signing.sh @@ -62,6 +62,10 @@ outPathCA=$(IMPURE_VAR1=foo IMPURE_VAR2=bar nix-build ./fixed.nix -A good.0 --no nix verify $outPathCA nix verify $outPathCA --sigs-needed 1000 +# Check that signing a content-addressed path doesn't overflow validSigs +nix sign-paths --key-file $TEST_ROOT/sk1 $outPathCA +nix verify -r $outPathCA --sigs-needed 1000 --trusted-public-keys $pk1 + # Copy to a binary cache. nix copy --to file://$cacheDir $outPath2 diff --git a/version b/version deleted file mode 100644 index 415b19fc3623..000000000000 --- a/version +++ /dev/null @@ -1 +0,0 @@ -2.0 \ No newline at end of file |