diff options
Diffstat (limited to 'src')
87 files changed, 4831 insertions, 2317 deletions
diff --git a/src/boost/format/exceptions.hpp b/src/boost/format/exceptions.hpp index 79e452449ef8..a7641458c95e 100644 --- a/src/boost/format/exceptions.hpp +++ b/src/boost/format/exceptions.hpp @@ -33,7 +33,7 @@ namespace io { class format_error : public std::exception { public: - format_error() {} + format_error() { abort(); } virtual const char *what() const throw() { return "boost::format_error: " @@ -44,7 +44,7 @@ public: class bad_format_string : public format_error { public: - bad_format_string() {} + bad_format_string() { abort(); } virtual const char *what() const throw() { return "boost::bad_format_string: " @@ -55,7 +55,7 @@ public: class too_few_args : public format_error { public: - too_few_args() {} + too_few_args() { abort(); } virtual const char *what() const throw() { return "boost::too_few_args: " @@ -66,7 +66,7 @@ public: class too_many_args : public format_error { public: - too_many_args() {} + too_many_args() { abort(); } virtual const char *what() const throw() { return "boost::too_many_args: " @@ -78,7 +78,7 @@ public: class out_of_range : public format_error { public: - out_of_range() {} + out_of_range() { abort(); } virtual const char *what() const throw() { return "boost::out_of_range: " diff --git a/src/bsdiff-4.3/bsdiff.1 b/src/bsdiff-4.3/bsdiff.1 deleted file mode 100644 index ead6c4deb57f..000000000000 --- a/src/bsdiff-4.3/bsdiff.1 +++ /dev/null @@ -1,63 +0,0 @@ -.\"- -.\" Copyright 2003-2005 Colin Percival -.\" All rights reserved -.\" -.\" Redistribution and use in source and binary forms, with or without -.\" modification, are permitted providing that the following conditions -.\" are met: -.\" 1. Redistributions of source code must retain the above copyright -.\" notice, this list of conditions and the following disclaimer. -.\" 2. Redistributions in binary form must reproduce the above copyright -.\" notice, this list of conditions and the following disclaimer in the -.\" documentation and/or other materials provided with the distribution. -.\" -.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR -.\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -.\" WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY -.\" DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -.\" STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING -.\" IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -.\" POSSIBILITY OF SUCH DAMAGE. -.\" -.\" $FreeBSD: src/usr.bin/bsdiff/bsdiff/bsdiff.1,v 1.1 2005/08/06 01:59:05 cperciva Exp $ -.\" -.Dd May 18, 2003 -.Dt BSDIFF 1 -.Os FreeBSD -.Sh NAME -.Nm bsdiff -.Nd generate a patch between two binary files -.Sh SYNOPSIS -.Nm -.Ao Ar oldfile Ac Ao Ar newfile Ac Ao Ar patchfile Ac -.Sh DESCRIPTION -.Nm -compares -.Ao Ar oldfile Ac -to -.Ao Ar newfile Ac -and writes to -.Ao Ar patchfile Ac -a binary patch suitable for use by bspatch(1). -When -.Ao Ar oldfile Ac -and -.Ao Ar newfile Ac -are two versions of an executable program, the -patches produced are on average a factor of five smaller -than those produced by any other binary patch tool known -to the author. -.Pp -.Nm -uses memory equal to 17 times the size of -.Ao Ar oldfile Ac , -and requires -an absolute minimum working set size of 8 times the size of oldfile. -.Sh SEE ALSO -.Xr bspatch 1 -.Sh AUTHORS -.An Colin Percival Aq cperciva@freebsd.org diff --git a/src/bsdiff-4.3/bsdiff.c b/src/bsdiff-4.3/bsdiff.c deleted file mode 100644 index 374ed038fa1f..000000000000 --- a/src/bsdiff-4.3/bsdiff.c +++ /dev/null @@ -1,405 +0,0 @@ -/*- - * Copyright 2003-2005 Colin Percival - * All rights reserved - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted providing that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING - * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#if 0 -__FBSDID("$FreeBSD: src/usr.bin/bsdiff/bsdiff/bsdiff.c,v 1.1 2005/08/06 01:59:05 cperciva Exp $"); -#endif - -#include <sys/types.h> - -#include <bzlib.h> -#include <err.h> -#include <fcntl.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <unistd.h> - -#define MIN(x,y) (((x)<(y)) ? (x) : (y)) - -static void split(off_t *I,off_t *V,off_t start,off_t len,off_t h) -{ - off_t i,j,k,x,tmp,jj,kk; - - if(len<16) { - for(k=start;k<start+len;k+=j) { - j=1;x=V[I[k]+h]; - for(i=1;k+i<start+len;i++) { - if(V[I[k+i]+h]<x) { - x=V[I[k+i]+h]; - j=0; - }; - if(V[I[k+i]+h]==x) { - tmp=I[k+j];I[k+j]=I[k+i];I[k+i]=tmp; - j++; - }; - }; - for(i=0;i<j;i++) V[I[k+i]]=k+j-1; - if(j==1) I[k]=-1; - }; - return; - }; - - x=V[I[start+len/2]+h]; - jj=0;kk=0; - for(i=start;i<start+len;i++) { - if(V[I[i]+h]<x) jj++; - if(V[I[i]+h]==x) kk++; - }; - jj+=start;kk+=jj; - - i=start;j=0;k=0; - while(i<jj) { - if(V[I[i]+h]<x) { - i++; - } else if(V[I[i]+h]==x) { - tmp=I[i];I[i]=I[jj+j];I[jj+j]=tmp; - j++; - } else { - tmp=I[i];I[i]=I[kk+k];I[kk+k]=tmp; - k++; - }; - }; - - while(jj+j<kk) { - if(V[I[jj+j]+h]==x) { - j++; - } else { - tmp=I[jj+j];I[jj+j]=I[kk+k];I[kk+k]=tmp; - k++; - }; - }; - - if(jj>start) split(I,V,start,jj-start,h); - - for(i=0;i<kk-jj;i++) V[I[jj+i]]=kk-1; - if(jj==kk-1) I[jj]=-1; - - if(start+len>kk) split(I,V,kk,start+len-kk,h); -} - -static void qsufsort(off_t *I,off_t *V,u_char *old,off_t oldsize) -{ - off_t buckets[256]; - off_t i,h,len; - - for(i=0;i<256;i++) buckets[i]=0; - for(i=0;i<oldsize;i++) buckets[old[i]]++; - for(i=1;i<256;i++) buckets[i]+=buckets[i-1]; - for(i=255;i>0;i--) buckets[i]=buckets[i-1]; - buckets[0]=0; - - for(i=0;i<oldsize;i++) I[++buckets[old[i]]]=i; - I[0]=oldsize; - for(i=0;i<oldsize;i++) V[i]=buckets[old[i]]; - V[oldsize]=0; - for(i=1;i<256;i++) if(buckets[i]==buckets[i-1]+1) I[buckets[i]]=-1; - I[0]=-1; - - for(h=1;I[0]!=-(oldsize+1);h+=h) { - len=0; - for(i=0;i<oldsize+1;) { - if(I[i]<0) { - len-=I[i]; - i-=I[i]; - } else { - if(len) I[i-len]=-len; - len=V[I[i]]+1-i; - split(I,V,i,len,h); - i+=len; - len=0; - }; - }; - if(len) I[i-len]=-len; - }; - - for(i=0;i<oldsize+1;i++) I[V[i]]=i; -} - -static off_t matchlen(u_char *old,off_t oldsize,u_char *new,off_t newsize) -{ - off_t i; - - for(i=0;(i<oldsize)&&(i<newsize);i++) - if(old[i]!=new[i]) break; - - return i; -} - -static off_t search(off_t *I,u_char *old,off_t oldsize, - u_char *new,off_t newsize,off_t st,off_t en,off_t *pos) -{ - off_t x,y; - - if(en-st<2) { - x=matchlen(old+I[st],oldsize-I[st],new,newsize); - y=matchlen(old+I[en],oldsize-I[en],new,newsize); - - if(x>y) { - *pos=I[st]; - return x; - } else { - *pos=I[en]; - return y; - } - }; - - x=st+(en-st)/2; - if(memcmp(old+I[x],new,MIN(oldsize-I[x],newsize))<0) { - return search(I,old,oldsize,new,newsize,x,en,pos); - } else { - return search(I,old,oldsize,new,newsize,st,x,pos); - }; -} - -static void offtout(off_t x,u_char *buf) -{ - off_t y; - - if(x<0) y=-x; else y=x; - - buf[0]=y%256;y-=buf[0]; - y=y/256;buf[1]=y%256;y-=buf[1]; - y=y/256;buf[2]=y%256;y-=buf[2]; - y=y/256;buf[3]=y%256;y-=buf[3]; - y=y/256;buf[4]=y%256;y-=buf[4]; - y=y/256;buf[5]=y%256;y-=buf[5]; - y=y/256;buf[6]=y%256;y-=buf[6]; - y=y/256;buf[7]=y%256; - - if(x<0) buf[7]|=0x80; -} - -int main(int argc,char *argv[]) -{ - int fd; - u_char *old,*new; - off_t oldsize,newsize; - off_t *I,*V; - off_t scan,pos,len; - off_t lastscan,lastpos,lastoffset; - off_t oldscore,scsc; - off_t s,Sf,lenf,Sb,lenb; - off_t overlap,Ss,lens; - off_t i; - off_t dblen,eblen; - u_char *db,*eb; - u_char buf[8]; - u_char header[32]; - FILE * pf; - BZFILE * pfbz2; - int bz2err; - - if(argc!=4) errx(1,"usage: %s oldfile newfile patchfile\n",argv[0]); - - /* Allocate oldsize+1 bytes instead of oldsize bytes to ensure - that we never try to malloc(0) and get a NULL pointer */ - if(((fd=open(argv[1],O_RDONLY,0))<0) || - ((oldsize=lseek(fd,0,SEEK_END))==-1) || - ((old=malloc(oldsize+1))==NULL) || - (lseek(fd,0,SEEK_SET)!=0) || - (read(fd,old,oldsize)!=oldsize) || - (close(fd)==-1)) err(1,"%s",argv[1]); - - if(((I=malloc((oldsize+1)*sizeof(off_t)))==NULL) || - ((V=malloc((oldsize+1)*sizeof(off_t)))==NULL)) err(1,NULL); - - qsufsort(I,V,old,oldsize); - - free(V); - - /* Allocate newsize+1 bytes instead of newsize bytes to ensure - that we never try to malloc(0) and get a NULL pointer */ - if(((fd=open(argv[2],O_RDONLY,0))<0) || - ((newsize=lseek(fd,0,SEEK_END))==-1) || - ((new=malloc(newsize+1))==NULL) || - (lseek(fd,0,SEEK_SET)!=0) || - (read(fd,new,newsize)!=newsize) || - (close(fd)==-1)) err(1,"%s",argv[2]); - - if(((db=malloc(newsize+1))==NULL) || - ((eb=malloc(newsize+1))==NULL)) err(1,NULL); - dblen=0; - eblen=0; - - /* Create the patch file */ - if ((pf = fopen(argv[3], "w")) == NULL) - err(1, "%s", argv[3]); - - /* Header is - 0 8 "BSDIFF40" - 8 8 length of bzip2ed ctrl block - 16 8 length of bzip2ed diff block - 24 8 length of new file */ - /* File is - 0 32 Header - 32 ?? Bzip2ed ctrl block - ?? ?? Bzip2ed diff block - ?? ?? Bzip2ed extra block */ - memcpy(header,"BSDIFF40",8); - offtout(0, header + 8); - offtout(0, header + 16); - offtout(newsize, header + 24); - if (fwrite(header, 32, 1, pf) != 1) - err(1, "fwrite(%s)", argv[3]); - - /* Compute the differences, writing ctrl as we go */ - if ((pfbz2 = BZ2_bzWriteOpen(&bz2err, pf, 9, 0, 0)) == NULL) - errx(1, "BZ2_bzWriteOpen, bz2err = %d", bz2err); - scan=0;len=0; - lastscan=0;lastpos=0;lastoffset=0; - while(scan<newsize) { - oldscore=0; - - for(scsc=scan+=len;scan<newsize;scan++) { - len=search(I,old,oldsize,new+scan,newsize-scan, - 0,oldsize,&pos); - if (len > 64 * 1024) break; - - for(;scsc<scan+len;scsc++) - if((scsc+lastoffset<oldsize) && - (old[scsc+lastoffset] == new[scsc])) - oldscore++; - - if(((len==oldscore) && (len!=0)) || - (len>oldscore+8)) break; - - if((scan+lastoffset<oldsize) && - (old[scan+lastoffset] == new[scan])) - oldscore--; - }; - - if((len!=oldscore) || (scan==newsize)) { - s=0;Sf=0;lenf=0; - for(i=0;(lastscan+i<scan)&&(lastpos+i<oldsize);) { - if(old[lastpos+i]==new[lastscan+i]) s++; - i++; - if(s*2-i>Sf*2-lenf) { Sf=s; lenf=i; }; - }; - - lenb=0; - if(scan<newsize) { - s=0;Sb=0; - for(i=1;(scan>=lastscan+i)&&(pos>=i);i++) { - if(old[pos-i]==new[scan-i]) s++; - if(s*2-i>Sb*2-lenb) { Sb=s; lenb=i; }; - }; - }; - - if(lastscan+lenf>scan-lenb) { - overlap=(lastscan+lenf)-(scan-lenb); - s=0;Ss=0;lens=0; - for(i=0;i<overlap;i++) { - if(new[lastscan+lenf-overlap+i]== - old[lastpos+lenf-overlap+i]) s++; - if(new[scan-lenb+i]== - old[pos-lenb+i]) s--; - if(s>Ss) { Ss=s; lens=i+1; }; - }; - - lenf+=lens-overlap; - lenb-=lens; - }; - - for(i=0;i<lenf;i++) - db[dblen+i]=new[lastscan+i]-old[lastpos+i]; - for(i=0;i<(scan-lenb)-(lastscan+lenf);i++) - eb[eblen+i]=new[lastscan+lenf+i]; - - dblen+=lenf; - eblen+=(scan-lenb)-(lastscan+lenf); - - offtout(lenf,buf); - BZ2_bzWrite(&bz2err, pfbz2, buf, 8); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWrite, bz2err = %d", bz2err); - - offtout((scan-lenb)-(lastscan+lenf),buf); - BZ2_bzWrite(&bz2err, pfbz2, buf, 8); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWrite, bz2err = %d", bz2err); - - offtout((pos-lenb)-(lastpos+lenf),buf); - BZ2_bzWrite(&bz2err, pfbz2, buf, 8); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWrite, bz2err = %d", bz2err); - - lastscan=scan-lenb; - lastpos=pos-lenb; - lastoffset=pos-scan; - }; - }; - BZ2_bzWriteClose(&bz2err, pfbz2, 0, NULL, NULL); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWriteClose, bz2err = %d", bz2err); - - /* Compute size of compressed ctrl data */ - if ((len = ftello(pf)) == -1) - err(1, "ftello"); - offtout(len-32, header + 8); - - /* Write compressed diff data */ - if ((pfbz2 = BZ2_bzWriteOpen(&bz2err, pf, 9, 0, 0)) == NULL) - errx(1, "BZ2_bzWriteOpen, bz2err = %d", bz2err); - BZ2_bzWrite(&bz2err, pfbz2, db, dblen); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWrite, bz2err = %d", bz2err); - BZ2_bzWriteClose(&bz2err, pfbz2, 0, NULL, NULL); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWriteClose, bz2err = %d", bz2err); - - /* Compute size of compressed diff data */ - if ((newsize = ftello(pf)) == -1) - err(1, "ftello"); - offtout(newsize - len, header + 16); - - /* Write compressed extra data */ - if ((pfbz2 = BZ2_bzWriteOpen(&bz2err, pf, 9, 0, 0)) == NULL) - errx(1, "BZ2_bzWriteOpen, bz2err = %d", bz2err); - BZ2_bzWrite(&bz2err, pfbz2, eb, eblen); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWrite, bz2err = %d", bz2err); - BZ2_bzWriteClose(&bz2err, pfbz2, 0, NULL, NULL); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWriteClose, bz2err = %d", bz2err); - - /* Seek to the beginning, write the header, and close the file */ - if (fseeko(pf, 0, SEEK_SET)) - err(1, "fseeko"); - if (fwrite(header, 32, 1, pf) != 1) - err(1, "fwrite(%s)", argv[3]); - if (fclose(pf)) - err(1, "fclose"); - - /* Free the memory we used */ - free(db); - free(eb); - free(I); - free(old); - free(new); - - return 0; -} diff --git a/src/bsdiff-4.3/bspatch.1 b/src/bsdiff-4.3/bspatch.1 deleted file mode 100644 index 82a2781aa7dc..000000000000 --- a/src/bsdiff-4.3/bspatch.1 +++ /dev/null @@ -1,59 +0,0 @@ -.\"- -.\" Copyright 2003-2005 Colin Percival -.\" All rights reserved -.\" -.\" Redistribution and use in source and binary forms, with or without -.\" modification, are permitted providing that the following conditions -.\" are met: -.\" 1. Redistributions of source code must retain the above copyright -.\" notice, this list of conditions and the following disclaimer. -.\" 2. Redistributions in binary form must reproduce the above copyright -.\" notice, this list of conditions and the following disclaimer in the -.\" documentation and/or other materials provided with the distribution. -.\" -.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR -.\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -.\" WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY -.\" DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -.\" STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING -.\" IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -.\" POSSIBILITY OF SUCH DAMAGE. -.\" -.\" $FreeBSD: src/usr.bin/bsdiff/bspatch/bspatch.1,v 1.1 2005/08/06 01:59:06 cperciva Exp $ -.\" -.Dd May 18, 2003 -.Dt BSPATCH 1 -.Os FreeBSD -.Sh NAME -.Nm bspatch -.Nd apply a patch built with bsdiff(1) -.Sh SYNOPSIS -.Nm -.Ao Ar oldfile Ac Ao Ar newfile Ac Ao Ar patchfile Ac -.Sh DESCRIPTION -.Nm -generates -.Ao Ar newfile Ac -from -.Ao Ar oldfile Ac -and -.Ao Ar patchfile Ac -where -.Ao Ar patchfile Ac -is a binary patch built by bsdiff(1). -.Pp -.Nm -uses memory equal to the size of -.Ao Ar oldfile Ac -plus the size of -.Ao Ar newfile Ac , -but can tolerate a very small working set without a dramatic loss -of performance. -.Sh SEE ALSO -.Xr bsdiff 1 -.Sh AUTHORS -.An Colin Percival Aq cperciva@freebsd.org diff --git a/src/bsdiff-4.3/bspatch.c b/src/bsdiff-4.3/bspatch.c deleted file mode 100644 index f9d33ddd64a2..000000000000 --- a/src/bsdiff-4.3/bspatch.c +++ /dev/null @@ -1,224 +0,0 @@ -/*- - * Copyright 2003-2005 Colin Percival - * All rights reserved - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted providing that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING - * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#if 0 -__FBSDID("$FreeBSD: src/usr.bin/bsdiff/bspatch/bspatch.c,v 1.1 2005/08/06 01:59:06 cperciva Exp $"); -#endif - -#include <bzlib.h> -#include <stdlib.h> -#include <stdio.h> -#include <string.h> -#include <err.h> -#include <errno.h> -#include <unistd.h> -#include <fcntl.h> -#include <sys/types.h> - -static off_t offtin(u_char *buf) -{ - off_t y; - - y=buf[7]&0x7F; - y=y*256;y+=buf[6]; - y=y*256;y+=buf[5]; - y=y*256;y+=buf[4]; - y=y*256;y+=buf[3]; - y=y*256;y+=buf[2]; - y=y*256;y+=buf[1]; - y=y*256;y+=buf[0]; - - if(buf[7]&0x80) y=-y; - - return y; -} - - -void writeFull(const char * name, int fd, - const unsigned char * buf, size_t count) -{ - while (count) { - ssize_t res = write(fd, (char *) buf, count); - if (res == -1) { - if (errno == EINTR) continue; - err(1,"writing to %s",name); - } - count -= res; - buf += res; - } -} - - -int main(int argc,char * argv[]) -{ - FILE * f, * cpf, * dpf, * epf; - BZFILE * cpfbz2, * dpfbz2, * epfbz2; - int cbz2err, dbz2err, ebz2err; - int fd; - ssize_t oldsize,newsize; - ssize_t bzctrllen,bzdatalen; - u_char header[32],buf[8]; - u_char *old, *new; - off_t oldpos,newpos; - off_t ctrl[3]; - off_t lenread; - off_t i; - - if(argc!=4) errx(1,"usage: %s oldfile newfile patchfile\n",argv[0]); - - /* Open patch file */ - if ((f = fopen(argv[3], "r")) == NULL) - err(1, "fopen(%s)", argv[3]); - - /* - File format: - 0 8 "BSDIFF40" - 8 8 X - 16 8 Y - 24 8 sizeof(newfile) - 32 X bzip2(control block) - 32+X Y bzip2(diff block) - 32+X+Y ??? bzip2(extra block) - with control block a set of triples (x,y,z) meaning "add x bytes - from oldfile to x bytes from the diff block; copy y bytes from the - extra block; seek forwards in oldfile by z bytes". - */ - - /* Read header */ - if (fread(header, 1, 32, f) < 32) { - if (feof(f)) - errx(1, "Corrupt patch\n"); - err(1, "fread(%s)", argv[3]); - } - - /* Check for appropriate magic */ - if (memcmp(header, "BSDIFF40", 8) != 0) - errx(1, "Corrupt patch\n"); - - /* Read lengths from header */ - bzctrllen=offtin(header+8); - bzdatalen=offtin(header+16); - newsize=offtin(header+24); - if((bzctrllen<0) || (bzdatalen<0) || (newsize<0)) - errx(1,"Corrupt patch\n"); - - /* Close patch file and re-open it via libbzip2 at the right places */ - if (fclose(f)) - err(1, "fclose(%s)", argv[3]); - if ((cpf = fopen(argv[3], "r")) == NULL) - err(1, "fopen(%s)", argv[3]); - if (fseeko(cpf, 32, SEEK_SET)) - err(1, "fseeko(%s, %lld)", argv[3], - (long long)32); - if ((cpfbz2 = BZ2_bzReadOpen(&cbz2err, cpf, 0, 0, NULL, 0)) == NULL) - errx(1, "BZ2_bzReadOpen, bz2err = %d", cbz2err); - if ((dpf = fopen(argv[3], "r")) == NULL) - err(1, "fopen(%s)", argv[3]); - if (fseeko(dpf, 32 + bzctrllen, SEEK_SET)) - err(1, "fseeko(%s, %lld)", argv[3], - (long long)(32 + bzctrllen)); - if ((dpfbz2 = BZ2_bzReadOpen(&dbz2err, dpf, 0, 0, NULL, 0)) == NULL) - errx(1, "BZ2_bzReadOpen, bz2err = %d", dbz2err); - if ((epf = fopen(argv[3], "r")) == NULL) - err(1, "fopen(%s)", argv[3]); - if (fseeko(epf, 32 + bzctrllen + bzdatalen, SEEK_SET)) - err(1, "fseeko(%s, %lld)", argv[3], - (long long)(32 + bzctrllen + bzdatalen)); - if ((epfbz2 = BZ2_bzReadOpen(&ebz2err, epf, 0, 0, NULL, 0)) == NULL) - errx(1, "BZ2_bzReadOpen, bz2err = %d", ebz2err); - - if(((fd=open(argv[1],O_RDONLY,0))<0) || - ((oldsize=lseek(fd,0,SEEK_END))==-1) || - ((old=malloc(oldsize+1))==NULL) || - (lseek(fd,0,SEEK_SET)!=0) || - (read(fd,old,oldsize)!=oldsize) || - (close(fd)==-1)) err(1,"%s",argv[1]); - if((new=malloc(newsize+1))==NULL) err(1,NULL); - - oldpos=0;newpos=0; - while(newpos<newsize) { - /* Read control data */ - for(i=0;i<=2;i++) { - lenread = BZ2_bzRead(&cbz2err, cpfbz2, buf, 8); - if ((lenread < 8) || ((cbz2err != BZ_OK) && - (cbz2err != BZ_STREAM_END))) - errx(1, "Corrupt patch\n"); - ctrl[i]=offtin(buf); - }; - - /* Sanity-check */ - if(newpos+ctrl[0]>newsize) - errx(1,"Corrupt patch\n"); - - /* Read diff string */ - lenread = BZ2_bzRead(&dbz2err, dpfbz2, new + newpos, ctrl[0]); - if ((lenread < ctrl[0]) || - ((dbz2err != BZ_OK) && (dbz2err != BZ_STREAM_END))) - errx(1, "Corrupt patch\n"); - - /* Add old data to diff string */ - for(i=0;i<ctrl[0];i++) - if((oldpos+i>=0) && (oldpos+i<oldsize)) - new[newpos+i]+=old[oldpos+i]; - - /* Adjust pointers */ - newpos+=ctrl[0]; - oldpos+=ctrl[0]; - - /* Sanity-check */ - if(newpos+ctrl[1]>newsize) - errx(1,"Corrupt patch\n"); - - /* Read extra string */ - lenread = BZ2_bzRead(&ebz2err, epfbz2, new + newpos, ctrl[1]); - if ((lenread < ctrl[1]) || - ((ebz2err != BZ_OK) && (ebz2err != BZ_STREAM_END))) - errx(1, "Corrupt patch\n"); - - /* Adjust pointers */ - newpos+=ctrl[1]; - oldpos+=ctrl[2]; - }; - - /* Clean up the bzip2 reads */ - BZ2_bzReadClose(&cbz2err, cpfbz2); - BZ2_bzReadClose(&dbz2err, dpfbz2); - BZ2_bzReadClose(&ebz2err, epfbz2); - if (fclose(cpf) || fclose(dpf) || fclose(epf)) - err(1, "fclose(%s)", argv[3]); - - /* Write the new file */ - if((fd=open(argv[2],O_CREAT|O_TRUNC|O_WRONLY,0666))<0) - err(1,"%s",argv[2]); - writeFull(argv[2], fd, new, newsize); - if(close(fd)==-1) - err(1,"%s",argv[2]); - - free(new); - free(old); - - return 0; -} diff --git a/src/bsdiff-4.3/compat-include/err.h b/src/bsdiff-4.3/compat-include/err.h deleted file mode 100644 index a851ded6f907..000000000000 --- a/src/bsdiff-4.3/compat-include/err.h +++ /dev/null @@ -1,12 +0,0 @@ -/* Simulate BSD's <err.h> functionality. */ - -#ifndef COMPAT_ERR_H_INCLUDED -#define COMPAT_ERR_H_INCLUDED 1 - -#include <stdio.h> -#include <stdlib.h> - -#define err(rc,...) do { fprintf(stderr,__VA_ARGS__); exit(rc); } while(0) -#define errx(rc,...) do { fprintf(stderr,__VA_ARGS__); exit(rc); } while(0) - -#endif diff --git a/src/bsdiff-4.3/local.mk b/src/bsdiff-4.3/local.mk deleted file mode 100644 index c957ceab0c0f..000000000000 --- a/src/bsdiff-4.3/local.mk +++ /dev/null @@ -1,11 +0,0 @@ -programs += bsdiff bspatch - -bsdiff_DIR := $(d) -bsdiff_SOURCES := $(d)/bsdiff.c -bsdiff_LDFLAGS = -lbz2 $(bsddiff_compat_include) -bsdiff_INSTALL_DIR = $(libexecdir)/nix - -bspatch_DIR := $(d) -bspatch_SOURCES := $(d)/bspatch.c -bspatch_LDFLAGS = -lbz2 $(bsddiff_compat_include) -bspatch_INSTALL_DIR = $(libexecdir)/nix diff --git a/src/libexpr/common-opts.cc b/src/libexpr/common-opts.cc index 68ab4b5cdcbf..8a7989aac663 100644 --- a/src/libexpr/common-opts.cc +++ b/src/libexpr/common-opts.cc @@ -55,7 +55,7 @@ bool parseSearchPathArg(Strings::iterator & i, Path lookupFileArg(EvalState & state, string s) { if (isUri(s)) - return downloadFileCached(state.store, s, true); + return makeDownloader()->downloadCached(state.store, s, true); else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { Path p = s.substr(1, s.size() - 2); return state.findFile(p); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 8ce2f3dfa6af..7ad9a4e46d83 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -5,6 +5,7 @@ #include "derivations.hh" #include "globals.hh" #include "eval-inline.hh" +#include "download.hh" #include <algorithm> #include <cstring> @@ -238,12 +239,38 @@ void initGC() /* Very hacky way to parse $NIX_PATH, which is colon-separated, but can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */ -static Strings parseNixPath(const string & in) +static Strings parseNixPath(const string & s) { - string marker = "\001//"; - auto res = tokenizeString<Strings>(replaceStrings(in, "://", marker), ":"); - for (auto & s : res) - s = replaceStrings(s, marker, "://"); + Strings res; + + auto p = s.begin(); + + while (p != s.end()) { + auto start = p; + auto start2 = p; + + while (p != s.end() && *p != ':') { + if (*p == '=') start2 = p + 1; + ++p; + } + + if (p == s.end()) { + if (p != start) res.push_back(std::string(start, p)); + break; + } + + if (*p == ':') { + if (isUri(std::string(start2, s.end()))) { + ++p; + while (p != s.end() && *p != ':') ++p; + } + res.push_back(std::string(start, p)); + if (p == s.end()) break; + } + + ++p; + } + return res; } @@ -278,7 +305,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store) /* Initialise the Nix expression search path. */ Strings paths = parseNixPath(getEnv("NIX_PATH", "")); - for (auto & i : _searchPath) addToSearchPath(i, true); + for (auto & i : _searchPath) addToSearchPath(i); for (auto & i : paths) addToSearchPath(i); addToSearchPath("nix=" + settings.nixDataDir + "/nix/corepkgs"); @@ -301,11 +328,15 @@ Path EvalState::checkSourcePath(const Path & path_) if (!restricted) return path_; /* Resolve symlinks. */ + debug(format("checking access to ‘%s’") % path_); Path path = canonPath(path_, true); - for (auto & i : searchPath) - if (path == i.second || isInDir(path, i.second)) + for (auto & i : searchPath) { + auto r = resolveSearchPathElem(i); + if (!r.first) continue; + if (path == r.second || isInDir(path, r.second)) return path; + } /* To support import-from-derivation, allow access to anything in the store. FIXME: only allow access to paths that have been diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 40e05712bab1..80e369f2d68f 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -26,9 +26,9 @@ typedef void (* PrimOpFun) (EvalState & state, const Pos & pos, Value * * args, struct PrimOp { PrimOpFun fun; - unsigned int arity; + size_t arity; Symbol name; - PrimOp(PrimOpFun fun, unsigned int arity, Symbol name) + PrimOp(PrimOpFun fun, size_t arity, Symbol name) : fun(fun), arity(arity), name(name) { } }; @@ -56,7 +56,8 @@ typedef std::map<Path, Path> SrcToStore; std::ostream & operator << (std::ostream & str, const Value & v); -typedef list<std::pair<string, Path> > SearchPath; +typedef std::pair<std::string, std::string> SearchPathElem; +typedef std::list<SearchPathElem> SearchPath; /* Initialise the Boehm GC, if applicable. */ @@ -98,12 +99,14 @@ private: SearchPath searchPath; + std::map<std::string, std::pair<bool, std::string>> searchPathResolved; + public: EvalState(const Strings & _searchPath, ref<Store> store); ~EvalState(); - void addToSearchPath(const string & s, bool warn = false); + void addToSearchPath(const string & s); Path checkSourcePath(const Path & path); @@ -125,6 +128,9 @@ public: Path findFile(const string & path); Path findFile(SearchPath & searchPath, const string & path, const Pos & pos = noPos); + /* If the specified search path element is a URI, download it. */ + std::pair<bool, std::string> resolveSearchPathElem(const SearchPathElem & elem); + /* Evaluate an expression to normal form, storing the result in value `v'. */ void eval(Expr * e, Value & v); diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 701c01aff973..f3660ab43723 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -195,5 +195,7 @@ or { return OR_KW; } } +<<EOF>> { data->atEnd = true; return 0; } + %% diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 5de9ccc6d011..620050a13b05 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -4,7 +4,7 @@ libexpr_NAME = libnixexpr libexpr_DIR := $(d) -libexpr_SOURCES := $(wildcard $(d)/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc +libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc libexpr_CXXFLAGS := -Wno-deprecated-register diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 5e7bc40c85c9..d2ca09b3a5bb 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -11,6 +11,7 @@ namespace nix { MakeError(EvalError, Error) MakeError(ParseError, Error) +MakeError(IncompleteParseError, ParseError) MakeError(AssertionError, EvalError) MakeError(ThrownError, AssertionError) MakeError(Abort, EvalError) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index f87aa261935b..20ae1a696097 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -31,10 +31,12 @@ namespace nix { Path basePath; Symbol path; string error; + bool atEnd; Symbol sLetBody; ParseData(EvalState & state) : state(state) , symbols(state.symbols) + , atEnd(false) , sLetBody(symbols.create("<let-body>")) { }; }; @@ -539,7 +541,12 @@ Expr * EvalState::parse(const char * text, int res = yyparse(scanner, &data); yylex_destroy(scanner); - if (res) throw ParseError(data.error); + if (res) { + if (data.atEnd) + throw IncompleteParseError(data.error); + else + throw ParseError(data.error); + } data.result->bindVars(staticEnv); @@ -593,7 +600,7 @@ Expr * EvalState::parseExprFromString(const string & s, const Path & basePath) } -void EvalState::addToSearchPath(const string & s, bool warn) +void EvalState::addToSearchPath(const string & s) { size_t pos = s.find('='); string prefix; @@ -605,16 +612,7 @@ void EvalState::addToSearchPath(const string & s, bool warn) path = string(s, pos + 1); } - if (isUri(path)) - path = downloadFileCached(store, path, true); - - path = absPath(path); - if (pathExists(path)) { - debug(format("adding path ‘%1%’ to the search path") % path); - /* Resolve symlinks in the path to support restricted mode. */ - searchPath.push_back(std::pair<string, Path>(prefix, canonPath(path, true))); - } else if (warn) - printMsg(lvlError, format("warning: Nix search path entry ‘%1%’ does not exist, ignoring") % path); + searchPath.emplace_back(prefix, path); } @@ -627,17 +625,19 @@ Path EvalState::findFile(const string & path) Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos & pos) { for (auto & i : searchPath) { - assert(!isUri(i.second)); - Path res; + std::string suffix; if (i.first.empty()) - res = i.second + "/" + path; + suffix = "/" + path; else { - if (path.compare(0, i.first.size(), i.first) != 0 || - (path.size() > i.first.size() && path[i.first.size()] != '/')) + auto s = i.first.size(); + if (path.compare(0, s, i.first) != 0 || + (path.size() > s && path[s] != '/')) continue; - res = i.second + - (path.size() == i.first.size() ? "" : "/" + string(path, i.first.size())); + suffix = path.size() == s ? "" : "/" + string(path, s); } + auto r = resolveSearchPathElem(i); + if (!r.first) continue; + Path res = r.second + suffix; if (pathExists(res)) return canonPath(res); } format f = format( @@ -648,4 +648,35 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos } +std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathElem & elem) +{ + auto i = searchPathResolved.find(elem.second); + if (i != searchPathResolved.end()) return i->second; + + std::pair<bool, std::string> res; + + if (isUri(elem.second)) { + try { + res = { true, makeDownloader()->downloadCached(store, elem.second, true) }; + } catch (DownloadError & e) { + printMsg(lvlError, format("warning: Nix search path entry ‘%1%’ cannot be downloaded, ignoring") % elem.second); + res = { false, "" }; + } + } else { + auto path = absPath(elem.second); + if (pathExists(path)) + res = { true, path }; + else { + printMsg(lvlError, format("warning: Nix search path entry ‘%1%’ does not exist, ignoring") % elem.second); + res = { false, "" }; + } + } + + debug(format("resolved search path element ‘%s’ to ‘%s’") % elem.second % res.second); + + searchPathResolved[elem.second] = res; + return res; +} + + } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 3c899d769253..51680ad62ee2 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -10,6 +10,7 @@ #include "util.hh" #include "value-to-json.hh" #include "value-to-xml.hh" +#include "primops.hh" #include <sys/types.h> #include <sys/stat.h> @@ -765,7 +766,7 @@ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Va string s = readFile(state.checkSourcePath(path)); if (s.find((char) 0) != string::npos) throw Error(format("the contents of the file ‘%1%’ cannot be represented as a Nix string") % path); - mkString(v, s.c_str()); + mkString(v, s.c_str(), context); } @@ -777,7 +778,6 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va SearchPath searchPath; - PathSet context; for (unsigned int n = 0; n < args[0]->listSize(); ++n) { Value & v2(*args[0]->listElems()[n]); state.forceAttrs(v2, pos); @@ -790,21 +790,23 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va i = v2.attrs->find(state.symbols.create("path")); if (i == v2.attrs->end()) throw EvalError(format("attribute ‘path’ missing, at %1%") % pos); - string path = state.coerceToPath(pos, *i->value, context); - searchPath.push_back(std::pair<string, Path>(prefix, state.checkSourcePath(path))); - } + PathSet context; + string path = state.coerceToString(pos, *i->value, context, false, false); - string path = state.forceStringNoCtx(*args[1], pos); + try { + state.realiseContext(context); + } catch (InvalidPathError & e) { + throw EvalError(format("cannot find ‘%1%’, since path ‘%2%’ is not valid, at %3%") + % path % e.path % pos); + } - try { - state.realiseContext(context); - } catch (InvalidPathError & e) { - throw EvalError(format("cannot find ‘%1%’, since path ‘%2%’ is not valid, at %3%") - % path % e.path % pos); + searchPath.emplace_back(prefix, path); } - mkPath(v, state.findFile(searchPath, path, pos).c_str()); + string path = state.forceStringNoCtx(*args[1], pos); + + mkPath(v, state.checkSourcePath(state.findFile(searchPath, path, pos)).c_str()); } /* Read a directory (without . or ..) */ @@ -1703,7 +1705,7 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, } else url = state.forceStringNoCtx(*args[0], pos); - Path res = downloadFileCached(state.store, url, unpack); + Path res = makeDownloader()->downloadCached(state.store, url, unpack); mkString(v, res, PathSet({res})); } @@ -1725,6 +1727,16 @@ static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args *************************************************************/ +RegisterPrimOp::PrimOps * RegisterPrimOp::primOps; + + +RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun) +{ + if (!primOps) primOps = new PrimOps; + primOps->emplace_back(name, arity, fun); +} + + void EvalState::createBaseEnv() { baseEnv.up = 0; @@ -1889,6 +1901,10 @@ void EvalState::createBaseEnv() } addConstant("__nixPath", v); + if (RegisterPrimOp::primOps) + for (auto & primOp : *RegisterPrimOp::primOps) + addPrimOp(std::get<0>(primOp), std::get<1>(primOp), std::get<2>(primOp)); + /* Now that we've added all primops, sort the `builtins' set, because attribute lookups expect it to be sorted. */ baseEnv.values[0]->attrs->sort(); diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh new file mode 100644 index 000000000000..39d23b04a5ce --- /dev/null +++ b/src/libexpr/primops.hh @@ -0,0 +1,15 @@ +#include "eval.hh" + +#include <tuple> +#include <vector> + +namespace nix { + +struct RegisterPrimOp +{ + typedef std::vector<std::tuple<std::string, size_t, PrimOpFun>> PrimOps; + static PrimOps * primOps; + RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun); +}; + +} diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc new file mode 100644 index 000000000000..9219f380c74f --- /dev/null +++ b/src/libmain/common-args.cc @@ -0,0 +1,38 @@ +#include "common-args.hh" +#include "globals.hh" + +namespace nix { + +MixCommonArgs::MixCommonArgs(const string & programName) + : programName(programName) +{ + mkFlag('v', "verbose", "increase verbosity level", []() { + verbosity = (Verbosity) (verbosity + 1); + }); + + mkFlag(0, "quiet", "decrease verbosity level", []() { + verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; + }); + + mkFlag(0, "debug", "enable debug output", []() { + verbosity = lvlDebug; + }); + + mkFlag1(0, "log-type", "type", "set logging format ('pretty', 'flat', 'systemd')", + [](std::string s) { + if (s == "pretty") logType = ltPretty; + else if (s == "escapes") logType = ltEscapes; + else if (s == "flat") logType = ltFlat; + else if (s == "systemd") logType = ltSystemd; + else throw UsageError("unknown log type"); + }); + + mkFlag(0, "option", {"name", "value"}, "set a Nix configuration option (overriding nix.conf)", 2, + [](Strings ss) { + auto name = ss.front(); ss.pop_front(); + auto value = ss.front(); + settings.set(name, value); + }); +} + +} diff --git a/src/libmain/common-args.hh b/src/libmain/common-args.hh new file mode 100644 index 000000000000..2c0d71edd815 --- /dev/null +++ b/src/libmain/common-args.hh @@ -0,0 +1,23 @@ +#pragma once + +#include "args.hh" + +namespace nix { + +struct MixCommonArgs : virtual Args +{ + string programName; + MixCommonArgs(const string & programName); +}; + +struct MixDryRun : virtual Args +{ + bool dryRun; + + MixDryRun() + { + mkFlag(0, "dry-run", "show what this command would do without doing it", &dryRun); + } +}; + +} diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 8f2aa842036a..ed997052be20 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -1,7 +1,8 @@ #include "config.h" -#include "shared.hh" +#include "common-args.hh" #include "globals.hh" +#include "shared.hh" #include "store-api.hh" #include "util.hh" @@ -23,15 +24,9 @@ namespace nix { -volatile sig_atomic_t blockInt = 0; - - static void sigintHandler(int signo) { - if (!blockInt) { - _isInterrupted = 1; - blockInt = 1; - } + _isInterrupted = 1; } @@ -85,16 +80,6 @@ void printMissing(ref<Store> store, const PathSet & willBuild, } -static void setLogType(string lt) -{ - if (lt == "pretty") logType = ltPretty; - else if (lt == "escapes") logType = ltEscapes; - else if (lt == "flat") logType = ltFlat; - else if (lt == "systemd") logType = ltSystemd; - else throw UsageError("unknown log type"); -} - - string getArg(const string & opt, Strings::iterator & i, const Strings::iterator & end) { @@ -126,8 +111,6 @@ void initNix() std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); #endif - std::ios::sync_with_stdio(false); - if (getEnv("IN_SYSTEMD") == "1") logType = ltSystemd; @@ -180,77 +163,80 @@ void initNix() } -void parseCmdLine(int argc, char * * argv, - std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg) +struct LegacyArgs : public MixCommonArgs { - /* Put the arguments in a vector. */ - Strings args; - argc--; argv++; - while (argc--) args.push_back(*argv++); - - /* Process default options. */ - for (Strings::iterator i = args.begin(); i != args.end(); ++i) { - string arg = *i; - - /* Expand compound dash options (i.e., `-qlf' -> `-q -l -f'). */ - if (arg.length() > 2 && arg[0] == '-' && arg[1] != '-' && isalpha(arg[1])) { - *i = (string) "-" + arg[1]; - auto next = i; ++next; - for (unsigned int j = 2; j < arg.length(); j++) - if (isalpha(arg[j])) - args.insert(next, (string) "-" + arg[j]); - else { - args.insert(next, string(arg, j)); - break; - } - arg = *i; - } + std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg; - if (arg == "--verbose" || arg == "-v") verbosity = (Verbosity) (verbosity + 1); - else if (arg == "--quiet") verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; - else if (arg == "--log-type") { - string s = getArg(arg, i, args.end()); - setLogType(s); - } - else if (arg == "--no-build-output" || arg == "-Q") - settings.buildVerbosity = lvlVomit; - else if (arg == "--print-build-trace") - settings.printBuildTrace = true; - else if (arg == "--keep-failed" || arg == "-K") - settings.keepFailed = true; - else if (arg == "--keep-going" || arg == "-k") - settings.keepGoing = true; - else if (arg == "--fallback") + LegacyArgs(const std::string & programName, + std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg) + : MixCommonArgs(programName), parseArg(parseArg) + { + mkFlag('Q', "no-build-output", "do not show build output", + &settings.buildVerbosity, lvlVomit); + + mkFlag(0, "print-build-trace", "emit special build trace message", + &settings.printBuildTrace); + + mkFlag('K', "keep-failed", "keep temporary directories of failed builds", + &settings.keepFailed); + + mkFlag('k', "keep-going", "keep going after a build fails", + &settings.keepGoing); + + mkFlag(0, "fallback", "build from source if substitution fails", []() { settings.set("build-fallback", "true"); - else if (arg == "--max-jobs" || arg == "-j") - settings.set("build-max-jobs", getArg(arg, i, args.end())); - else if (arg == "--cores") - settings.set("build-cores", getArg(arg, i, args.end())); - else if (arg == "--readonly-mode") - settings.readOnlyMode = true; - else if (arg == "--max-silent-time") - settings.set("build-max-silent-time", getArg(arg, i, args.end())); - else if (arg == "--timeout") - settings.set("build-timeout", getArg(arg, i, args.end())); - else if (arg == "--no-build-hook") - settings.useBuildHook = false; - else if (arg == "--show-trace") - settings.showTrace = true; - else if (arg == "--no-gc-warning") - gcWarning = false; - else if (arg == "--option") { - ++i; if (i == args.end()) throw UsageError("‘--option’ requires two arguments"); - string name = *i; - ++i; if (i == args.end()) throw UsageError("‘--option’ requires two arguments"); - string value = *i; - settings.set(name, value); - } - else { - if (!parseArg(i, args.end())) - throw UsageError(format("unrecognised option ‘%1%’") % *i); - } + }); + + auto intSettingAlias = [&](char shortName, const std::string & longName, + const std::string & description, const std::string & dest) { + mkFlag<unsigned int>(shortName, longName, description, [=](unsigned int n) { + settings.set(dest, std::to_string(n)); + }); + }; + + intSettingAlias('j', "max-jobs", "maximum number of parallel builds", "build-max-jobs"); + intSettingAlias(0, "cores", "maximum number of CPU cores to use inside a build", "build-cores"); + intSettingAlias(0, "max-silent-time", "number of seconds of silence before a build is killed", "build-max-silent-time"); + intSettingAlias(0, "timeout", "number of seconds before a build is killed", "build-timeout"); + + mkFlag(0, "readonly-mode", "do not write to the Nix store", + &settings.readOnlyMode); + + mkFlag(0, "no-build-hook", "disable use of the build hook mechanism", + &settings.useBuildHook, false); + + mkFlag(0, "show-trace", "show Nix expression stack trace in evaluation errors", + &settings.showTrace); + + mkFlag(0, "no-gc-warning", "disable warning about not using ‘--add-root’", + &gcWarning, false); } + bool processFlag(Strings::iterator & pos, Strings::iterator end) override + { + if (MixCommonArgs::processFlag(pos, end)) return true; + bool res = parseArg(pos, end); + if (res) ++pos; + return res; + } + + bool processArgs(const Strings & args, bool finish) override + { + if (args.empty()) return true; + assert(args.size() == 1); + Strings ss(args); + auto pos = ss.begin(); + if (!parseArg(pos, ss.end())) + throw UsageError(format("unexpected argument ‘%1%’") % args.front()); + return true; + } +}; + + +void parseCmdLine(int argc, char * * argv, + std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg) +{ + LegacyArgs(baseNameOf(argv[0]), parseArg).parseCmdline(argvToStrings(argc, argv)); settings.update(); } @@ -295,8 +281,7 @@ int handleExceptions(const string & programName, std::function<void()> fun) condition is discharged before we reach printMsg() below, since otherwise it will throw an (uncaught) exception. */ - blockInt = 1; /* ignore further SIGINTs */ - _isInterrupted = 0; + interruptThrown = true; throw; } } catch (Exit & e) { diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 0682267fa376..6d94a22f788e 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -1,6 +1,7 @@ #pragma once #include "util.hh" +#include "args.hh" #include <signal.h> @@ -9,8 +10,6 @@ namespace nix { -MakeError(UsageError, nix::Error); - class Exit : public std::exception { public: diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc new file mode 100644 index 000000000000..473a0b2614bb --- /dev/null +++ b/src/libstore/binary-cache-store.cc @@ -0,0 +1,438 @@ +#include "archive.hh" +#include "binary-cache-store.hh" +#include "compression.hh" +#include "derivations.hh" +#include "fs-accessor.hh" +#include "globals.hh" +#include "nar-info.hh" +#include "sync.hh" +#include "worker-protocol.hh" +#include "nar-accessor.hh" + +#include <chrono> + +namespace nix { + +BinaryCacheStore::BinaryCacheStore(std::shared_ptr<Store> localStore, + const Path & secretKeyFile) + : localStore(localStore) +{ + if (secretKeyFile != "") { + secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile))); + publicKeys = std::unique_ptr<PublicKeys>(new PublicKeys); + publicKeys->emplace(secretKey->name, secretKey->toPublicKey()); + } + + StringSink sink; + sink << narVersionMagic1; + narMagic = *sink.s; +} + +void BinaryCacheStore::init() +{ + std::string cacheInfoFile = "nix-cache-info"; + if (!fileExists(cacheInfoFile)) + upsertFile(cacheInfoFile, "StoreDir: " + settings.nixStore + "\n"); +} + +void BinaryCacheStore::notImpl() +{ + throw Error("operation not implemented for binary cache stores"); +} + +const BinaryCacheStore::Stats & BinaryCacheStore::getStats() +{ + return stats; +} + +Path BinaryCacheStore::narInfoFileFor(const Path & storePath) +{ + assertStorePath(storePath); + return storePathToHash(storePath) + ".narinfo"; +} + +void BinaryCacheStore::addToCache(const ValidPathInfo & info, + const string & nar) +{ + auto narInfoFile = narInfoFileFor(info.path); + if (fileExists(narInfoFile)) return; + + assert(nar.compare(0, narMagic.size(), narMagic) == 0); + + auto narInfo = make_ref<NarInfo>(info); + + narInfo->narSize = nar.size(); + narInfo->narHash = hashString(htSHA256, nar); + + if (info.narHash.type != htUnknown && info.narHash != narInfo->narHash) + throw Error(format("refusing to copy corrupted path ‘%1%’ to binary cache") % info.path); + + /* Compress the NAR. */ + narInfo->compression = "xz"; + auto now1 = std::chrono::steady_clock::now(); + string narXz = compressXZ(nar); + auto now2 = std::chrono::steady_clock::now(); + narInfo->fileHash = hashString(htSHA256, narXz); + narInfo->fileSize = narXz.size(); + + auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count(); + printMsg(lvlTalkative, format("copying path ‘%1%’ (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache") + % narInfo->path % narInfo->narSize + % ((1.0 - (double) narXz.size() / nar.size()) * 100.0) + % duration); + + /* Atomically write the NAR file. */ + narInfo->url = "nar/" + printHash32(narInfo->fileHash) + ".nar.xz"; + if (!fileExists(narInfo->url)) { + stats.narWrite++; + upsertFile(narInfo->url, narXz); + } else + stats.narWriteAverted++; + + stats.narWriteBytes += nar.size(); + stats.narWriteCompressedBytes += narXz.size(); + stats.narWriteCompressionTimeMs += duration; + + /* Atomically write the NAR info file.*/ + if (secretKey) narInfo->sign(*secretKey); + + upsertFile(narInfoFile, narInfo->to_string()); + + { + auto state_(state.lock()); + state_->narInfoCache.upsert(narInfo->path, narInfo); + stats.narInfoCacheSize = state_->narInfoCache.size(); + } + + stats.narInfoWrite++; +} + +NarInfo BinaryCacheStore::readNarInfo(const Path & storePath) +{ + { + auto state_(state.lock()); + auto res = state_->narInfoCache.get(storePath); + if (res) { + stats.narInfoReadAverted++; + return **res; + } + } + + auto narInfoFile = narInfoFileFor(storePath); + auto narInfo = make_ref<NarInfo>(getFile(narInfoFile), narInfoFile); + if (narInfo->path != storePath) + throw Error(format("NAR info file for store path ‘%1%’ does not match ‘%2%’") % narInfo->path % storePath); + + stats.narInfoRead++; + + if (publicKeys) { + if (!narInfo->checkSignatures(*publicKeys)) + throw Error(format("no good signature on NAR info file ‘%1%’") % narInfoFile); + } + + { + auto state_(state.lock()); + state_->narInfoCache.upsert(storePath, narInfo); + stats.narInfoCacheSize = state_->narInfoCache.size(); + } + + return *narInfo; +} + +bool BinaryCacheStore::isValidPath(const Path & storePath) +{ + { + auto state_(state.lock()); + auto res = state_->narInfoCache.get(storePath); + if (res) { + stats.narInfoReadAverted++; + return true; + } + } + + // FIXME: this only checks whether a .narinfo with a matching hash + // part exists. So ‘f4kb...-foo’ matches ‘f4kb...-bar’, even + // though they shouldn't. Not easily fixed. + return fileExists(narInfoFileFor(storePath)); +} + +void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink) +{ + auto res = readNarInfo(storePath); + + auto nar = getFile(res.url); + + stats.narRead++; + stats.narReadCompressedBytes += nar.size(); + + /* Decompress the NAR. FIXME: would be nice to have the remote + side do this. */ + if (res.compression == "none") + ; + else if (res.compression == "xz") + nar = decompressXZ(nar); + else + throw Error(format("unknown NAR compression type ‘%1%’") % nar); + + stats.narReadBytes += nar.size(); + + printMsg(lvlTalkative, format("exporting path ‘%1%’ (%2% bytes)") % storePath % nar.size()); + + assert(nar.size() % 8 == 0); + + sink((unsigned char *) nar.c_str(), nar.size()); +} + +void BinaryCacheStore::exportPath(const Path & storePath, bool sign, Sink & sink) +{ + assert(!sign); + + auto res = readNarInfo(storePath); + + narFromPath(storePath, sink); + + // FIXME: check integrity of NAR. + + sink << exportMagic << storePath << res.references << res.deriver << 0; +} + +Paths BinaryCacheStore::importPaths(bool requireSignature, Source & source, + std::shared_ptr<FSAccessor> accessor) +{ + assert(!requireSignature); + Paths res; + while (true) { + unsigned long long n = readLongLong(source); + if (n == 0) break; + if (n != 1) throw Error("input doesn't look like something created by ‘nix-store --export’"); + res.push_back(importPath(source, accessor)); + } + return res; +} + +struct TeeSource : Source +{ + Source & readSource; + ref<std::string> data; + TeeSource(Source & readSource) + : readSource(readSource) + , data(make_ref<std::string>()) + { + } + size_t read(unsigned char * data, size_t len) + { + size_t n = readSource.read(data, len); + this->data->append((char *) data, n); + return n; + } +}; + +struct NopSink : ParseSink +{ +}; + +ValidPathInfo BinaryCacheStore::queryPathInfo(const Path & storePath) +{ + return ValidPathInfo(readNarInfo(storePath)); +} + +void BinaryCacheStore::querySubstitutablePathInfos(const PathSet & paths, + SubstitutablePathInfos & infos) +{ + PathSet left; + + if (!localStore) return; + + for (auto & storePath : paths) { + if (!localStore->isValidPath(storePath)) { + left.insert(storePath); + continue; + } + ValidPathInfo info = localStore->queryPathInfo(storePath); + SubstitutablePathInfo sub; + sub.references = info.references; + sub.downloadSize = 0; + sub.narSize = info.narSize; + infos.emplace(storePath, sub); + } + + if (settings.useSubstitutes) + localStore->querySubstitutablePathInfos(left, infos); +} + +Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath, + bool recursive, HashType hashAlgo, PathFilter & filter, bool repair) +{ + // FIXME: some cut&paste from LocalStore::addToStore(). + + /* Read the whole path into memory. This is not a very scalable + method for very large paths, but `copyPath' is mainly used for + small files. */ + StringSink sink; + Hash h; + if (recursive) { + dumpPath(srcPath, sink, filter); + h = hashString(hashAlgo, *sink.s); + } else { + auto s = readFile(srcPath); + dumpString(s, sink); + h = hashString(hashAlgo, s); + } + + ValidPathInfo info; + info.path = makeFixedOutputPath(recursive, hashAlgo, h, name); + + if (repair || !isValidPath(info.path)) + addToCache(info, *sink.s); + + return info.path; +} + +Path BinaryCacheStore::addTextToStore(const string & name, const string & s, + const PathSet & references, bool repair) +{ + ValidPathInfo info; + info.path = computeStorePathForText(name, s, references); + info.references = references; + + if (repair || !isValidPath(info.path)) { + StringSink sink; + dumpString(s, sink); + addToCache(info, *sink.s); + } + + return info.path; +} + +void BinaryCacheStore::buildPaths(const PathSet & paths, BuildMode buildMode) +{ + for (auto & storePath : paths) { + assert(!isDerivation(storePath)); + + if (isValidPath(storePath)) continue; + + if (!localStore) + throw Error(format("don't know how to realise path ‘%1%’ in a binary cache") % storePath); + + localStore->addTempRoot(storePath); + + if (!localStore->isValidPath(storePath)) + localStore->ensurePath(storePath); + + ValidPathInfo info = localStore->queryPathInfo(storePath); + + for (auto & ref : info.references) + if (ref != storePath) + ensurePath(ref); + + StringSink sink; + dumpPath(storePath, sink); + + addToCache(info, *sink.s); + } +} + +void BinaryCacheStore::ensurePath(const Path & path) +{ + buildPaths({path}); +} + +/* Given requests for a path /nix/store/<x>/<y>, this accessor will + first download the NAR for /nix/store/<x> from the binary cache, + build a NAR accessor for that NAR, and use that to access <y>. */ +struct BinaryCacheStoreAccessor : public FSAccessor +{ + ref<BinaryCacheStore> store; + + std::map<Path, ref<FSAccessor>> nars; + + BinaryCacheStoreAccessor(ref<BinaryCacheStore> store) + : store(store) + { + } + + std::pair<ref<FSAccessor>, Path> fetch(const Path & path_) + { + auto path = canonPath(path_); + + auto storePath = toStorePath(path); + std::string restPath = std::string(path, storePath.size()); + + if (!store->isValidPath(storePath)) + throw Error(format("path ‘%1%’ is not a valid store path") % storePath); + + auto i = nars.find(storePath); + if (i != nars.end()) return {i->second, restPath}; + + StringSink sink; + store->exportPath(storePath, false, sink); + + auto accessor = makeNarAccessor(sink.s); + nars.emplace(storePath, accessor); + return {accessor, restPath}; + } + + Stat stat(const Path & path) override + { + auto res = fetch(path); + return res.first->stat(res.second); + } + + StringSet readDirectory(const Path & path) override + { + auto res = fetch(path); + return res.first->readDirectory(res.second); + } + + std::string readFile(const Path & path) override + { + auto res = fetch(path); + return res.first->readFile(res.second); + } + + std::string readLink(const Path & path) override + { + auto res = fetch(path); + return res.first->readLink(res.second); + } +}; + +ref<FSAccessor> BinaryCacheStore::getFSAccessor() +{ + return make_ref<BinaryCacheStoreAccessor>(ref<BinaryCacheStore>( + std::dynamic_pointer_cast<BinaryCacheStore>(shared_from_this()))); +} + +Path BinaryCacheStore::importPath(Source & source, std::shared_ptr<FSAccessor> accessor) +{ + /* FIXME: some cut&paste of LocalStore::importPath(). */ + + /* Extract the NAR from the source. */ + TeeSource tee(source); + NopSink sink; + parseDump(sink, tee); + + uint32_t magic = readInt(source); + if (magic != exportMagic) + throw Error("Nix archive cannot be imported; wrong format"); + + ValidPathInfo info; + info.path = readStorePath(source); + + info.references = readStorePaths<PathSet>(source); + + readString(source); // deriver, don't care + + bool haveSignature = readInt(source) == 1; + assert(!haveSignature); + + addToCache(info, *tee.data); + + auto accessor_ = std::dynamic_pointer_cast<BinaryCacheStoreAccessor>(accessor); + if (accessor_) + accessor_->nars.emplace(info.path, makeNarAccessor(tee.data)); + + return info.path; +} + +} diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh new file mode 100644 index 000000000000..9e7b0ad9a384 --- /dev/null +++ b/src/libstore/binary-cache-store.hh @@ -0,0 +1,172 @@ +#pragma once + +#include "crypto.hh" +#include "store-api.hh" + +#include "lru-cache.hh" +#include "sync.hh" +#include "pool.hh" + +#include <atomic> + +namespace nix { + +struct NarInfo; + +class BinaryCacheStore : public Store +{ +private: + + std::unique_ptr<SecretKey> secretKey; + std::unique_ptr<PublicKeys> publicKeys; + + std::shared_ptr<Store> localStore; + + struct State + { + LRUCache<Path, ref<NarInfo>> narInfoCache{32 * 1024}; + }; + + Sync<State> state; + +protected: + + BinaryCacheStore(std::shared_ptr<Store> localStore, const Path & secretKeyFile); + + [[noreturn]] void notImpl(); + + virtual bool fileExists(const std::string & path) = 0; + + virtual void upsertFile(const std::string & path, const std::string & data) = 0; + + virtual std::string getFile(const std::string & path) = 0; + +public: + + virtual void init(); + + struct Stats + { + std::atomic<uint64_t> narInfoRead{0}; + std::atomic<uint64_t> narInfoReadAverted{0}; + std::atomic<uint64_t> narInfoWrite{0}; + std::atomic<uint64_t> narInfoCacheSize{0}; + std::atomic<uint64_t> narRead{0}; + std::atomic<uint64_t> narReadBytes{0}; + std::atomic<uint64_t> narReadCompressedBytes{0}; + std::atomic<uint64_t> narWrite{0}; + std::atomic<uint64_t> narWriteAverted{0}; + std::atomic<uint64_t> narWriteBytes{0}; + std::atomic<uint64_t> narWriteCompressedBytes{0}; + std::atomic<uint64_t> narWriteCompressionTimeMs{0}; + }; + + const Stats & getStats(); + +private: + + Stats stats; + + std::string narMagic; + + std::string narInfoFileFor(const Path & storePath); + + void addToCache(const ValidPathInfo & info, const string & nar); + +protected: + + NarInfo readNarInfo(const Path & storePath); + +public: + + bool isValidPath(const Path & path) override; + + PathSet queryValidPaths(const PathSet & paths) override + { notImpl(); } + + PathSet queryAllValidPaths() override + { notImpl(); } + + ValidPathInfo queryPathInfo(const Path & path) override; + + Hash queryPathHash(const Path & path) override + { notImpl(); } + + void queryReferrers(const Path & path, + PathSet & referrers) override + { notImpl(); } + + Path queryDeriver(const Path & path) override + { return ""; } + + PathSet queryValidDerivers(const Path & path) override + { return {}; } + + PathSet queryDerivationOutputs(const Path & path) override + { notImpl(); } + + StringSet queryDerivationOutputNames(const Path & path) override + { notImpl(); } + + Path queryPathFromHashPart(const string & hashPart) override + { notImpl(); } + + PathSet querySubstitutablePaths(const PathSet & paths) override + { return {}; } + + void querySubstitutablePathInfos(const PathSet & paths, + SubstitutablePathInfos & infos) override; + + Path addToStore(const string & name, const Path & srcPath, + bool recursive = true, HashType hashAlgo = htSHA256, + PathFilter & filter = defaultPathFilter, bool repair = false) override; + + Path addTextToStore(const string & name, const string & s, + const PathSet & references, bool repair = false) override; + + void narFromPath(const Path & path, Sink & sink) override; + + void exportPath(const Path & path, bool sign, Sink & sink) override; + + Paths importPaths(bool requireSignature, Source & source, + std::shared_ptr<FSAccessor> accessor) override; + + Path importPath(Source & source, std::shared_ptr<FSAccessor> accessor); + + void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) override; + + BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, + BuildMode buildMode = bmNormal) override + { notImpl(); } + + void ensurePath(const Path & path) override; + + void addTempRoot(const Path & path) override + { notImpl(); } + + void addIndirectRoot(const Path & path) override + { notImpl(); } + + void syncWithGC() override + { } + + Roots findRoots() override + { notImpl(); } + + void collectGarbage(const GCOptions & options, GCResults & results) override + { notImpl(); } + + void optimiseStore() override + { } + + bool verifyStore(bool checkContents, bool repair) override + { return true; } + + ref<FSAccessor> getFSAccessor() override; + + void addSignatures(const Path & storePath, const StringSet & sigs) + { notImpl(); } + +}; + +} diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 249ab2335bdf..ba3f3a371d8c 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -22,6 +22,7 @@ #include <sys/stat.h> #include <sys/utsname.h> #include <sys/select.h> +#include <sys/resource.h> #include <fcntl.h> #include <unistd.h> #include <errno.h> @@ -52,7 +53,6 @@ #include <sys/param.h> #include <sys/mount.h> #include <sys/syscall.h> -#include <linux/fs.h> #define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) #endif @@ -240,6 +240,9 @@ private: /* Last time the goals in `waitingForAWhile' where woken up. */ time_t lastWokenUp; + /* Cache for pathContentsGood(). */ + std::map<Path, bool> pathContentsGoodCache; + public: /* Set if at least one derivation had a BuildError (i.e. permanent @@ -305,6 +308,12 @@ public: void waitForInput(); unsigned int exitStatus(); + + /* Check whether the given valid path exists and has the right + contents. */ + bool pathContentsGood(const Path & path); + + void markContentsGood(const Path & path); }; @@ -1039,11 +1048,6 @@ void DerivationGoal::haveDerivation() return; } - /* Check whether any output previously failed to build. If so, - don't bother. */ - for (auto & i : invalidOutputs) - if (pathFailed(i)) return; - /* Reject doing a hash build of anything other than a fixed-output derivation. */ if (buildMode == bmHash) { @@ -1160,7 +1164,7 @@ void DerivationGoal::repairClosure() /* Check each path (slow!). */ PathSet broken; for (auto & i : outputClosure) { - if (worker.store.pathContentsGood(i)) continue; + if (worker.pathContentsGood(i)) continue; printMsg(lvlError, format("found corrupted or missing path ‘%1%’ in the output closure of ‘%2%’") % i % drvPath); Path drvPath2 = outputsToDrv[i]; if (drvPath2 == "") @@ -1310,17 +1314,10 @@ void DerivationGoal::tryToBuild() for (auto & i : drv->outputs) { Path path = i.second.path; if (worker.store.isValidPath(path)) continue; - if (!pathExists(path)) continue; debug(format("removing invalid path ‘%1%’") % path); deletePath(path); } - /* Check again whether any output previously failed to build, - because some other process may have tried and failed before we - acquired the lock. */ - for (auto & i : drv->outputs) - if (pathFailed(i.second.path)) return; - /* Don't do a remote build if the derivation has the attribute `preferLocalBuild' set. Also, check and repair modes are only supported for local builds. */ @@ -1390,8 +1387,7 @@ void replaceValidPath(const Path & storePath, const Path tmpPath) rename(storePath.c_str(), oldPath.c_str()); if (rename(tmpPath.c_str(), storePath.c_str()) == -1) throw SysError(format("moving ‘%1%’ to ‘%2%’") % tmpPath % storePath); - if (pathExists(oldPath)) - deletePath(oldPath); + deletePath(oldPath); } @@ -1490,7 +1486,7 @@ void DerivationGoal::buildDone() /* Delete unused redirected outputs (when doing hash rewriting). */ for (auto & i : redirectedOutputs) - if (pathExists(i.second)) deletePath(i.second); + deletePath(i.second); /* Delete the chroot (if we were using one). */ autoDelChroot.reset(); /* this runs the destructor */ @@ -1543,17 +1539,6 @@ void DerivationGoal::buildDone() statusOk(status) ? BuildResult::OutputRejected : fixedOutput || diskFull ? BuildResult::TransientFailure : BuildResult::PermanentFailure; - - /* Register the outputs of this build as "failed" so we - won't try to build them again (negative caching). - However, don't do this for fixed-output derivations, - since they're likely to fail for transient reasons - (e.g., fetchurl not being able to access the network). - Hook errors (like communication problems with the - remote machine) shouldn't be cached either. */ - if (settings.cacheFailure && !fixedOutput && !diskFull) - for (auto & i : drv->outputs) - worker.store.registerFailedPath(i.second.path); } done(st, e.msg()); @@ -1939,7 +1924,7 @@ void DerivationGoal::startBuilder() to ensure that we can create hard-links to non-directory inputs in the fake Nix store in the chroot (see below). */ chrootRootDir = drvPath + ".chroot"; - if (pathExists(chrootRootDir)) deletePath(chrootRootDir); + deletePath(chrootRootDir); /* Clean up the chroot directory automatically. */ autoDelChroot = std::make_shared<AutoDelete>(chrootRootDir); @@ -2012,7 +1997,7 @@ void DerivationGoal::startBuilder() throw SysError(format("linking ‘%1%’ to ‘%2%’") % p % i); StringSink sink; dumpPath(i, sink); - StringSource source(sink.s); + StringSource source(*sink.s); restorePath(p, source); } } @@ -2389,6 +2374,12 @@ void DerivationGoal::runChild() if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE); #endif + /* Disable core dumps by default. */ + struct rlimit limit = { 0, RLIM_INFINITY }; + setrlimit(RLIMIT_CORE, &limit); + + // FIXME: set other limits to deterministic values? + /* Fill in the environment. */ Strings envStrs; for (auto & i : env) @@ -2514,7 +2505,7 @@ void DerivationGoal::runChild() debug(sandboxProfile); Path sandboxFile = drvPath + ".sb"; - if (pathExists(sandboxFile)) deletePath(sandboxFile); + deletePath(sandboxFile); autoDelSandbox.reset(sandboxFile, false); writeFile(sandboxFile, sandboxProfile); @@ -2669,8 +2660,8 @@ void DerivationGoal::registerOutputs() StringSink sink; dumpPath(actualPath, sink); deletePath(actualPath); - sink.s = rewriteHashes(sink.s, rewritesFromTmp); - StringSource source(sink.s); + sink.s = make_ref<std::string>(rewriteHashes(*sink.s, rewritesFromTmp)); + StringSource source(*sink.s); restorePath(actualPath, source); rewritten = true; @@ -2706,8 +2697,7 @@ void DerivationGoal::registerOutputs() return; if (actualPath != dest) { PathLocks outputLocks({dest}); - if (pathExists(dest)) - deletePath(dest); + deletePath(dest); if (rename(actualPath.c_str(), dest.c_str()) == -1) throw SysError(format("moving ‘%1%’ to ‘%2%’") % actualPath % dest); } @@ -2738,7 +2728,7 @@ void DerivationGoal::registerOutputs() if (hash.first != info.narHash) { if (settings.keepFailed) { Path dst = path + checkSuffix; - if (pathExists(dst)) deletePath(dst); + deletePath(dst); if (rename(actualPath.c_str(), dst.c_str())) throw SysError(format("renaming ‘%1%’ to ‘%2%’") % actualPath % dst); throw Error(format("derivation ‘%1%’ may not be deterministic: output ‘%2%’ differs from ‘%3%’") @@ -2747,6 +2737,15 @@ void DerivationGoal::registerOutputs() throw Error(format("derivation ‘%1%’ may not be deterministic: output ‘%2%’ differs") % drvPath % path); } + + /* Since we verified the build, it's now ultimately + trusted. */ + if (!info.ultimate) { + info.ultimate = true; + worker.store.signPathInfo(info); + worker.store.registerValidPaths({info}); + } + continue; } @@ -2794,7 +2793,7 @@ void DerivationGoal::registerOutputs() if (curRound == nrRounds) { worker.store.optimisePath(path); // FIXME: combine with scanForReferences() - worker.store.markContentsGood(path); + worker.markContentsGood(path); } ValidPathInfo info; @@ -2803,6 +2802,9 @@ void DerivationGoal::registerOutputs() info.narSize = hash.second; info.references = references; info.deriver = drvPath; + info.ultimate = true; + worker.store.signPathInfo(info); + infos.push_back(info); } @@ -2830,7 +2832,7 @@ void DerivationGoal::registerOutputs() if (settings.keepFailed) { for (auto & i : drv->outputs) { Path prev = i.second.path + checkSuffix; - if (pathExists(prev)) deletePath(prev); + deletePath(prev); if (curRound < nrRounds) { Path dst = i.second.path + checkSuffix; if (rename(i.second.path.c_str(), dst.c_str())) @@ -2969,36 +2971,19 @@ PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) if (!wantOutput(i.first, wantedOutputs)) continue; bool good = worker.store.isValidPath(i.second.path) && - (!checkHash || worker.store.pathContentsGood(i.second.path)); + (!checkHash || worker.pathContentsGood(i.second.path)); if (good == returnValid) result.insert(i.second.path); } return result; } -bool DerivationGoal::pathFailed(const Path & path) -{ - if (!settings.cacheFailure) return false; - - if (!worker.store.hasPathFailed(path)) return false; - - printMsg(lvlError, format("builder for ‘%1%’ failed previously (cached)") % path); - - if (settings.printBuildTrace) - printMsg(lvlError, format("@ build-failed %1% - cached") % drvPath); - - done(BuildResult::CachedFailure); - - return true; -} - - Path DerivationGoal::addHashRewrite(const Path & path) { string h1 = string(path, settings.nixStore.size() + 1, 32); string h2 = string(printHash32(hashString(htSHA256, "rewrite:" + drvPath + ":" + path)), 0, 32); Path p = settings.nixStore + "/" + h2 + string(path, settings.nixStore.size() + 33); - if (pathExists(p)) deletePath(p); + deletePath(p); assert(path.size() == p.size()); rewritesToTmp[h1] = h2; rewritesFromTmp[h2] = h1; @@ -3014,7 +2999,7 @@ void DerivationGoal::done(BuildResult::Status status, const string & msg) amDone(result.success() ? ecSuccess : ecFailed); if (result.status == BuildResult::TimedOut) worker.timedOut = true; - if (result.status == BuildResult::PermanentFailure || result.status == BuildResult::CachedFailure) + if (result.status == BuildResult::PermanentFailure) worker.permanentFailure = true; } @@ -3259,8 +3244,7 @@ void SubstitutionGoal::tryToRun() destPath = repair ? storePath + ".tmp" : storePath; /* Remove the (stale) output path if it exists. */ - if (pathExists(destPath)) - deletePath(destPath); + deletePath(destPath); worker.store.setSubstituterEnv(); @@ -3378,7 +3362,7 @@ void SubstitutionGoal::finished() outputLock->setDeletion(true); outputLock.reset(); - worker.store.markContentsGood(storePath); + worker.markContentsGood(storePath); printMsg(lvlChatty, format("substitution of path ‘%1%’ succeeded") % storePath); @@ -3778,6 +3762,32 @@ unsigned int Worker::exitStatus() } +bool Worker::pathContentsGood(const Path & path) +{ + std::map<Path, bool>::iterator i = pathContentsGoodCache.find(path); + if (i != pathContentsGoodCache.end()) return i->second; + printMsg(lvlInfo, format("checking path ‘%1%’...") % path); + ValidPathInfo info = store.queryPathInfo(path); + bool res; + if (!pathExists(path)) + res = false; + else { + HashResult current = hashPath(info.narHash.type, path); + Hash nullHash(htSHA256); + res = info.narHash == nullHash || info.narHash == current.first; + } + pathContentsGoodCache[path] = res; + if (!res) printMsg(lvlError, format("path ‘%1%’ is corrupted or missing!") % path); + return res; +} + + +void Worker::markContentsGood(const Path & path) +{ + pathContentsGoodCache[path] = true; +} + + ////////////////////////////////////////////////////////////////////// diff --git a/src/libstore/builtins.cc b/src/libstore/builtins.cc index a1c4b48bf62e..c22c44f3c7e3 100644 --- a/src/libstore/builtins.cc +++ b/src/libstore/builtins.cc @@ -17,9 +17,9 @@ void builtinFetchurl(const BasicDerivation & drv) options.verifyTLS = false; /* Show a progress indicator, even though stderr is not a tty. */ - options.forceProgress = true; + options.showProgress = DownloadOptions::yes; - auto data = downloadFile(url->second, options); + auto data = makeDownloader()->download(url->second, options); auto out = drv.env.find("out"); if (out == drv.env.end()) throw Error("attribute ‘url’ missing"); diff --git a/src/libstore/crypto.cc b/src/libstore/crypto.cc index c1b57e51d9b4..747483afb30b 100644 --- a/src/libstore/crypto.cc +++ b/src/libstore/crypto.cc @@ -1,5 +1,6 @@ #include "crypto.hh" #include "util.hh" +#include "globals.hh" #if HAVE_SODIUM #include <sodium.h> @@ -37,10 +38,12 @@ SecretKey::SecretKey(const string & s) #endif } +#if !HAVE_SODIUM [[noreturn]] static void noSodium() { throw Error("Nix was not compiled with libsodium, required for signed binary cache support"); } +#endif std::string SecretKey::signDetached(const std::string & data) const { @@ -55,6 +58,17 @@ std::string SecretKey::signDetached(const std::string & data) const #endif } +PublicKey SecretKey::toPublicKey() const +{ +#if HAVE_SODIUM + unsigned char pk[crypto_sign_PUBLICKEYBYTES]; + crypto_sign_ed25519_sk_to_pk(pk, (unsigned char *) key.data()); + return PublicKey(name, std::string((char *) pk, crypto_sign_PUBLICKEYBYTES)); +#else + noSodium(); +#endif +} + PublicKey::PublicKey(const string & s) : Key(s) { @@ -85,4 +99,28 @@ bool verifyDetached(const std::string & data, const std::string & sig, #endif } +PublicKeys getDefaultPublicKeys() +{ + PublicKeys publicKeys; + + // FIXME: filter duplicates + + for (auto s : settings.get("binary-cache-public-keys", Strings())) { + PublicKey key(s); + publicKeys.emplace(key.name, key); + } + + for (auto secretKeyFile : settings.get("secret-key-files", Strings())) { + try { + SecretKey secretKey(readFile(secretKeyFile)); + publicKeys.emplace(secretKey.name, secretKey.toPublicKey()); + } catch (SysError & e) { + /* Ignore unreadable key files. That's normal in a + multi-user installation. */ + } + } + + return publicKeys; +} + } diff --git a/src/libstore/crypto.hh b/src/libstore/crypto.hh index a1489e753649..9110af3aa9e5 100644 --- a/src/libstore/crypto.hh +++ b/src/libstore/crypto.hh @@ -15,19 +15,31 @@ struct Key ‘<name>:<key-in-base64>’. */ Key(const std::string & s); +protected: + Key(const std::string & name, const std::string & key) + : name(name), key(key) { } }; +struct PublicKey; + struct SecretKey : Key { SecretKey(const std::string & s); /* Return a detached signature of the given string. */ std::string signDetached(const std::string & s) const; + + PublicKey toPublicKey() const; }; struct PublicKey : Key { PublicKey(const std::string & data); + +private: + PublicKey(const std::string & name, const std::string & key) + : Key(name, key) { } + friend struct SecretKey; }; typedef std::map<std::string, PublicKey> PublicKeys; @@ -37,4 +49,6 @@ typedef std::map<std::string, PublicKey> PublicKeys; bool verifyDetached(const std::string & data, const std::string & sig, const PublicKeys & publicKeys); +PublicKeys getDefaultPublicKeys(); + } diff --git a/src/libstore/download.cc b/src/libstore/download.cc index e754e82fb27f..7277751b48e7 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -18,7 +18,15 @@ double getTime() return tv.tv_sec + (tv.tv_usec / 1000000.0); } -struct Curl +std::string resolveUri(const std::string & uri) +{ + if (uri.compare(0, 8, "channel:") == 0) + return "https://nixos.org/channels/" + std::string(uri, 8) + "/nixexprs.tar.xz"; + else + return uri; +} + +struct CurlDownloader : public Downloader { CURL * curl; string data; @@ -30,37 +38,40 @@ struct Curl double prevProgressTime{0}, startTime{0}; unsigned int moveBack{1}; - static size_t writeCallback(void * contents, size_t size, size_t nmemb, void * userp) + size_t writeCallback(void * contents, size_t size, size_t nmemb) { - Curl & c(* (Curl *) userp); size_t realSize = size * nmemb; - c.data.append((char *) contents, realSize); + data.append((char *) contents, realSize); return realSize; } - static size_t headerCallback(void * contents, size_t size, size_t nmemb, void * userp) + static size_t writeCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) + { + return ((CurlDownloader *) userp)->writeCallback(contents, size, nmemb); + } + + size_t headerCallback(void * contents, size_t size, size_t nmemb) { - Curl & c(* (Curl *) userp); size_t realSize = size * nmemb; string line = string((char *) contents, realSize); printMsg(lvlVomit, format("got header: %1%") % trim(line)); if (line.compare(0, 5, "HTTP/") == 0) { // new response starts - c.etag = ""; + etag = ""; auto ss = tokenizeString<vector<string>>(line, " "); - c.status = ss.size() >= 2 ? ss[1] : ""; + status = ss.size() >= 2 ? ss[1] : ""; } else { auto i = line.find(':'); if (i != string::npos) { string name = trim(string(line, 0, i)); if (name == "ETag") { // FIXME: case - c.etag = trim(string(line, i + 1)); + etag = trim(string(line, i + 1)); /* Hack to work around a GitHub bug: it sends ETags, but ignores If-None-Match. So if we get the expected ETag on a 200 response, then shut down the connection because we already have the data. */ - printMsg(lvlDebug, format("got ETag: %1%") % c.etag); - if (c.etag == c.expectedETag && c.status == "200") { + printMsg(lvlDebug, format("got ETag: %1%") % etag); + if (etag == expectedETag && status == "200") { printMsg(lvlDebug, format("shutting down on 200 HTTP response with expected ETag")); return 0; } @@ -70,6 +81,11 @@ struct Curl return realSize; } + static size_t headerCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) + { + return ((CurlDownloader *) userp)->headerCallback(contents, size, nmemb); + } + int progressCallback(double dltotal, double dlnow) { if (showProgress) { @@ -88,45 +104,48 @@ struct Curl return _isInterrupted; } - static int progressCallback_(void * userp, double dltotal, double dlnow, double ultotal, double ulnow) + static int progressCallbackWrapper(void * userp, double dltotal, double dlnow, double ultotal, double ulnow) { - Curl & c(* (Curl *) userp); - return c.progressCallback(dltotal, dlnow); + return ((CurlDownloader *) userp)->progressCallback(dltotal, dlnow); } - Curl() + CurlDownloader() { requestHeaders = 0; curl = curl_easy_init(); - if (!curl) throw Error("unable to initialize curl"); + if (!curl) throw nix::Error("unable to initialize curl"); + } + + ~CurlDownloader() + { + if (curl) curl_easy_cleanup(curl); + if (requestHeaders) curl_slist_free_all(requestHeaders); + } + + bool fetch(const string & url, const DownloadOptions & options) + { + showProgress = + options.showProgress == DownloadOptions::yes || + (options.showProgress == DownloadOptions::automatic && isatty(STDERR_FILENO)); + + curl_easy_reset(curl); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); curl_easy_setopt(curl, CURLOPT_USERAGENT, ("Nix/" + nixVersion).c_str()); curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeCallback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) &curl); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeCallbackWrapper); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) this); - curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, headerCallback); - curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void *) &curl); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, headerCallbackWrapper); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void *) this); - curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, progressCallback_); - curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, (void *) &curl); + curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, progressCallbackWrapper); + curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, (void *) this); curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0); curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); - } - - ~Curl() - { - if (curl) curl_easy_cleanup(curl); - if (requestHeaders) curl_slist_free_all(requestHeaders); - } - - bool fetch(const string & url, const DownloadOptions & options) - { - showProgress = options.forceProgress || isatty(STDERR_FILENO); curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); @@ -151,6 +170,9 @@ struct Curl curl_easy_setopt(curl, CURLOPT_HTTPHEADER, requestHeaders); + if (options.head) + curl_easy_setopt(curl, CURLOPT_NOBODY, 1); + if (showProgress) { std::cerr << (format("downloading ‘%1%’... ") % url); std::cerr.flush(); @@ -163,35 +185,45 @@ struct Curl std::cerr << "\n"; checkInterrupt(); if (res == CURLE_WRITE_ERROR && etag == options.expectedETag) return false; - if (res != CURLE_OK) - throw DownloadError(format("unable to download ‘%1%’: %2% (%3%)") - % url % curl_easy_strerror(res) % res); - long httpStatus = 0; + long httpStatus = -1; curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &httpStatus); + + if (res != CURLE_OK) { + Error err = + httpStatus == 404 ? NotFound : + httpStatus == 403 ? Forbidden : Misc; + throw DownloadError(err, format("unable to download ‘%1%’: %2% (%3%)") + % url % curl_easy_strerror(res) % res); + } + if (httpStatus == 304) return false; return true; } -}; + DownloadResult download(string url, const DownloadOptions & options) override + { + DownloadResult res; + if (fetch(resolveUri(url), options)) { + res.cached = false; + res.data = data; + } else + res.cached = true; + res.etag = etag; + return res; + } +}; -DownloadResult downloadFile(string url, const DownloadOptions & options) +ref<Downloader> makeDownloader() { - DownloadResult res; - Curl curl; - if (curl.fetch(url, options)) { - res.cached = false; - res.data = curl.data; - } else - res.cached = true; - res.etag = curl.etag; - return res; + return make_ref<CurlDownloader>(); } - -Path downloadFileCached(ref<Store> store, const string & url, bool unpack) +Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpack) { + auto url = resolveUri(url_); + Path cacheDir = getEnv("XDG_CACHE_HOME", getEnv("HOME", "") + "/.cache") + "/nix/tarballs"; createDirs(cacheDir); @@ -234,7 +266,7 @@ Path downloadFileCached(ref<Store> store, const string & url, bool unpack) try { DownloadOptions options; options.expectedETag = expectedETag; - auto res = downloadFile(url, options); + auto res = download(url, options); if (!res.cached) storePath = store->addTextToStore(name, res.data, PathSet(), false); @@ -276,10 +308,11 @@ Path downloadFileCached(ref<Store> store, const string & url, bool unpack) bool isUri(const string & s) { + if (s.compare(0, 8, "channel:") == 0) return true; size_t pos = s.find("://"); if (pos == string::npos) return false; string scheme(s, 0, pos); - return scheme == "http" || scheme == "https" || scheme == "file"; + return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel"; } diff --git a/src/libstore/download.hh b/src/libstore/download.hh index 7aec8de73e48..5dd2d2c82dec 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -10,7 +10,8 @@ struct DownloadOptions { string expectedETag; bool verifyTLS{true}; - bool forceProgress{false}; + enum { yes, no, automatic } showProgress{yes}; + bool head{false}; }; struct DownloadResult @@ -21,11 +22,25 @@ struct DownloadResult class Store; -DownloadResult downloadFile(string url, const DownloadOptions & options); +struct Downloader +{ + virtual DownloadResult download(string url, const DownloadOptions & options) = 0; + + Path downloadCached(ref<Store> store, const string & url, bool unpack); + + enum Error { NotFound, Forbidden, Misc }; +}; -Path downloadFileCached(ref<Store> store, const string & url, bool unpack); +ref<Downloader> makeDownloader(); -MakeError(DownloadError, Error) +class DownloadError : public Error +{ +public: + Downloader::Error error; + DownloadError(Downloader::Error error, const FormatOrString & fs) + : Error(fs), error(error) + { } +}; bool isUri(const string & s); diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh new file mode 100644 index 000000000000..a67e0775b978 --- /dev/null +++ b/src/libstore/fs-accessor.hh @@ -0,0 +1,30 @@ +#pragma once + +#include "types.hh" + +namespace nix { + +/* An abstract class for accessing a filesystem-like structure, such + as a (possibly remote) Nix store or the contents of a NAR file. */ +class FSAccessor +{ +public: + enum Type { tMissing, tRegular, tSymlink, tDirectory }; + + struct Stat + { + Type type; + uint64_t fileSize; // regular files only + bool isExecutable; // regular files only + }; + + virtual Stat stat(const Path & path) = 0; + + virtual StringSet readDirectory(const Path & path) = 0; + + virtual std::string readFile(const Path & path) = 0; + + virtual std::string readLink(const Path & path) = 0; +}; + +} diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index d19af1cefaf2..52afa1b14e03 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -147,35 +147,36 @@ Path Store::addPermRoot(const Path & _storePath, void LocalStore::addTempRoot(const Path & path) { + auto state(_state.lock()); + /* Create the temporary roots file for this process. */ - if (fdTempRoots == -1) { + if (state->fdTempRoots == -1) { while (1) { Path dir = (format("%1%/%2%") % settings.nixStateDir % tempRootsDir).str(); createDirs(dir); - fnTempRoots = (format("%1%/%2%") - % dir % getpid()).str(); + state->fnTempRoots = (format("%1%/%2%") % dir % getpid()).str(); AutoCloseFD fdGCLock = openGCLock(ltRead); - if (pathExists(fnTempRoots)) + if (pathExists(state->fnTempRoots)) /* It *must* be stale, since there can be no two processes with the same pid. */ - unlink(fnTempRoots.c_str()); + unlink(state->fnTempRoots.c_str()); - fdTempRoots = openLockFile(fnTempRoots, true); + state->fdTempRoots = openLockFile(state->fnTempRoots, true); fdGCLock.close(); - debug(format("acquiring read lock on ‘%1%’") % fnTempRoots); - lockFile(fdTempRoots, ltRead, true); + debug(format("acquiring read lock on ‘%1%’") % state->fnTempRoots); + lockFile(state->fdTempRoots, ltRead, true); /* Check whether the garbage collector didn't get in our way. */ struct stat st; - if (fstat(fdTempRoots, &st) == -1) - throw SysError(format("statting ‘%1%’") % fnTempRoots); + if (fstat(state->fdTempRoots, &st) == -1) + throw SysError(format("statting ‘%1%’") % state->fnTempRoots); if (st.st_size == 0) break; /* The garbage collector deleted this file before we could @@ -187,15 +188,15 @@ void LocalStore::addTempRoot(const Path & path) /* Upgrade the lock to a write lock. This will cause us to block if the garbage collector is holding our lock. */ - debug(format("acquiring write lock on ‘%1%’") % fnTempRoots); - lockFile(fdTempRoots, ltWrite, true); + debug(format("acquiring write lock on ‘%1%’") % state->fnTempRoots); + lockFile(state->fdTempRoots, ltWrite, true); string s = path + '\0'; - writeFull(fdTempRoots, s); + writeFull(state->fdTempRoots, s); /* Downgrade to a read lock. */ - debug(format("downgrading to read lock on ‘%1%’") % fnTempRoots); - lockFile(fdTempRoots, ltRead, true); + debug(format("downgrading to read lock on ‘%1%’") % state->fnTempRoots); + lockFile(state->fdTempRoots, ltRead, true); } @@ -608,6 +609,9 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) state.shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific; + if (state.shouldDelete) + deletePath(reservedPath); + /* Acquire the global GC root. This prevents a) New roots from being added. b) Processes from creating new temporary root files. */ diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index e704837e8798..b6078253319f 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -52,7 +52,6 @@ Settings::Settings() keepLog = true; compressLog = true; maxLogSize = 0; - cacheFailure = false; pollInterval = 5; checkRootReachability = false; gcKeepOutputs = false; @@ -175,7 +174,6 @@ void Settings::update() _get(keepLog, "build-keep-log"); _get(compressLog, "build-compress-log"); _get(maxLogSize, "build-max-log-size"); - _get(cacheFailure, "build-cache-failure"); _get(pollInterval, "build-poll-interval"); _get(checkRootReachability, "gc-check-reachability"); _get(gcKeepOutputs, "gc-keep-outputs"); @@ -196,7 +194,6 @@ void Settings::update() if (getEnv("NIX_OTHER_STORES") != "") substituters.push_back(nixLibexecDir + "/nix/substituters/copy-from-other-stores.pl"); #endif - substituters.push_back(nixLibexecDir + "/nix/substituters/download-using-manifests.pl"); substituters.push_back(nixLibexecDir + "/nix/substituters/download-from-binary-cache.pl"); if (useSshSubstituter && !sshSubstituterHosts.empty()) substituters.push_back(nixLibexecDir + "/nix/substituters/download-via-ssh"); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 60b11afe6088..572fa7188c14 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -168,9 +168,6 @@ struct Settings { before being killed (0 means no limit). */ unsigned long maxLogSize; - /* Whether to cache build failures. */ - bool cacheFailure; - /* How often (in seconds) to poll for locks. */ unsigned int pollInterval; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc new file mode 100644 index 000000000000..8a719db150aa --- /dev/null +++ b/src/libstore/http-binary-cache-store.cc @@ -0,0 +1,82 @@ +#include "binary-cache-store.hh" +#include "download.hh" +#include "globals.hh" + +namespace nix { + +class HttpBinaryCacheStore : public BinaryCacheStore +{ +private: + + Path cacheUri; + + Pool<Downloader> downloaders; + +public: + + HttpBinaryCacheStore(std::shared_ptr<Store> localStore, + const Path & secretKeyFile, const Path & _cacheUri) + : BinaryCacheStore(localStore, secretKeyFile) + , cacheUri(_cacheUri) + , downloaders( + std::numeric_limits<size_t>::max(), + []() { return makeDownloader(); }) + { + if (cacheUri.back() == '/') + cacheUri.pop_back(); + } + + void init() override + { + // FIXME: do this lazily? + if (!fileExists("nix-cache-info")) + throw Error(format("‘%s’ does not appear to be a binary cache") % cacheUri); + } + +protected: + + bool fileExists(const std::string & path) override + { + try { + auto downloader(downloaders.get()); + DownloadOptions options; + options.showProgress = DownloadOptions::no; + options.head = true; + downloader->download(cacheUri + "/" + path, options); + return true; + } catch (DownloadError & e) { + /* S3 buckets return 403 if a file doesn't exist and the + bucket is unlistable, so treat 403 as 404. */ + if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) + return false; + throw; + } + } + + void upsertFile(const std::string & path, const std::string & data) override + { + throw Error("uploading to an HTTP binary cache is not supported"); + } + + std::string getFile(const std::string & path) override + { + auto downloader(downloaders.get()); + DownloadOptions options; + options.showProgress = DownloadOptions::no; + return downloader->download(cacheUri + "/" + path, options).data; + } + +}; + +static RegisterStoreImplementation regStore([](const std::string & uri) -> std::shared_ptr<Store> { + if (std::string(uri, 0, 7) != "http://" && + std::string(uri, 0, 8) != "https://") return 0; + auto store = std::make_shared<HttpBinaryCacheStore>(std::shared_ptr<Store>(0), + settings.get("binary-cache-secret-key-file", string("")), + uri); + store->init(); + return store; +}); + +} + diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc new file mode 100644 index 000000000000..efd6d47254f2 --- /dev/null +++ b/src/libstore/local-binary-cache-store.cc @@ -0,0 +1,83 @@ +#include "binary-cache-store.hh" +#include "globals.hh" + +namespace nix { + +class LocalBinaryCacheStore : public BinaryCacheStore +{ +private: + + Path binaryCacheDir; + +public: + + LocalBinaryCacheStore(std::shared_ptr<Store> localStore, + const Path & secretKeyFile, const Path & binaryCacheDir); + + void init() override; + +protected: + + bool fileExists(const std::string & path) override; + + void upsertFile(const std::string & path, const std::string & data) override; + + std::string getFile(const std::string & path) override; + +}; + +LocalBinaryCacheStore::LocalBinaryCacheStore(std::shared_ptr<Store> localStore, + const Path & secretKeyFile, const Path & binaryCacheDir) + : BinaryCacheStore(localStore, secretKeyFile) + , binaryCacheDir(binaryCacheDir) +{ +} + +void LocalBinaryCacheStore::init() +{ + createDirs(binaryCacheDir + "/nar"); + BinaryCacheStore::init(); +} + +static void atomicWrite(const Path & path, const std::string & s) +{ + Path tmp = path + ".tmp." + std::to_string(getpid()); + AutoDelete del(tmp, false); + writeFile(tmp, s); + if (rename(tmp.c_str(), path.c_str())) + throw SysError(format("renaming ‘%1%’ to ‘%2%’") % tmp % path); + del.cancel(); +} + +bool LocalBinaryCacheStore::fileExists(const std::string & path) +{ + return pathExists(binaryCacheDir + "/" + path); +} + +void LocalBinaryCacheStore::upsertFile(const std::string & path, const std::string & data) +{ + atomicWrite(binaryCacheDir + "/" + path, data); +} + +std::string LocalBinaryCacheStore::getFile(const std::string & path) +{ + return readFile(binaryCacheDir + "/" + path); +} + +ref<Store> openLocalBinaryCacheStore(std::shared_ptr<Store> localStore, + const Path & secretKeyFile, const Path & binaryCacheDir) +{ + auto store = make_ref<LocalBinaryCacheStore>( + localStore, secretKeyFile, binaryCacheDir); + store->init(); + return store; +} + +static RegisterStoreImplementation regStore([](const std::string & uri) -> std::shared_ptr<Store> { + if (std::string(uri, 0, 7) != "file://") return 0; + return openLocalBinaryCacheStore(std::shared_ptr<Store>(0), + settings.get("binary-cache-secret-key-file", string("")), + std::string(uri, 7)); +}); + +} diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc new file mode 100644 index 000000000000..303c3af27b8d --- /dev/null +++ b/src/libstore/local-fs-store.cc @@ -0,0 +1,79 @@ +#include "archive.hh" +#include "fs-accessor.hh" +#include "store-api.hh" + +namespace nix { + +struct LocalStoreAccessor : public FSAccessor +{ + ref<Store> store; + + LocalStoreAccessor(ref<Store> store) : store(store) { } + + void assertStore(const Path & path) + { + Path storePath = toStorePath(path); + if (!store->isValidPath(storePath)) + throw Error(format("path ‘%1%’ is not a valid store path") % storePath); + } + + FSAccessor::Stat stat(const Path & path) override + { + assertStore(path); + + struct stat st; + if (lstat(path.c_str(), &st)) { + if (errno == ENOENT || errno == ENOTDIR) return {Type::tMissing, 0, false}; + throw SysError(format("getting status of ‘%1%’") % path); + } + + if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode)) + throw Error(format("file ‘%1%’ has unsupported type") % path); + + return { + S_ISREG(st.st_mode) ? Type::tRegular : + S_ISLNK(st.st_mode) ? Type::tSymlink : + Type::tDirectory, + S_ISREG(st.st_mode) ? (uint64_t) st.st_size : 0, + S_ISREG(st.st_mode) && st.st_mode & S_IXUSR}; + } + + StringSet readDirectory(const Path & path) override + { + assertStore(path); + + auto entries = nix::readDirectory(path); + + StringSet res; + for (auto & entry : entries) + res.insert(entry.name); + + return res; + } + + std::string readFile(const Path & path) override + { + assertStore(path); + return nix::readFile(path); + } + + std::string readLink(const Path & path) override + { + assertStore(path); + return nix::readLink(path); + } +}; + +ref<FSAccessor> LocalFSStore::getFSAccessor() +{ + return make_ref<LocalStoreAccessor>(ref<Store>(shared_from_this())); +} + +void LocalFSStore::narFromPath(const Path & path, Sink & sink) +{ + if (!isValidPath(path)) + throw Error(format("path ‘%s’ is not valid") % path); + dumpPath(path, sink); +} + +} diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 308aebd73bb4..d6e1e4d7e0d4 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -10,6 +10,7 @@ #include <iostream> #include <algorithm> #include <cstring> +#include <atomic> #include <sys/types.h> #include <sys/stat.h> @@ -36,168 +37,6 @@ namespace nix { -MakeError(SQLiteError, Error); -MakeError(SQLiteBusy, SQLiteError); - - -[[noreturn]] static void throwSQLiteError(sqlite3 * db, const format & f) -{ - int err = sqlite3_errcode(db); - if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) { - if (err == SQLITE_PROTOCOL) - printMsg(lvlError, "warning: SQLite database is busy (SQLITE_PROTOCOL)"); - else { - static bool warned = false; - if (!warned) { - printMsg(lvlError, "warning: SQLite database is busy"); - warned = true; - } - } - /* Sleep for a while since retrying the transaction right away - is likely to fail again. */ -#if HAVE_NANOSLEEP - struct timespec t; - t.tv_sec = 0; - t.tv_nsec = (random() % 100) * 1000 * 1000; /* <= 0.1s */ - nanosleep(&t, 0); -#else - sleep(1); -#endif - throw SQLiteBusy(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); - } - else - throw SQLiteError(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); -} - - -/* Convenience macros for retrying a SQLite transaction. */ -#define retry_sqlite while (1) { try { -#define end_retry_sqlite break; } catch (SQLiteBusy & e) { } } - - -SQLite::~SQLite() -{ - try { - if (db && sqlite3_close(db) != SQLITE_OK) - throwSQLiteError(db, "closing database"); - } catch (...) { - ignoreException(); - } -} - - -void SQLiteStmt::create(sqlite3 * db, const string & s) -{ - checkInterrupt(); - assert(!stmt); - if (sqlite3_prepare_v2(db, s.c_str(), -1, &stmt, 0) != SQLITE_OK) - throwSQLiteError(db, "creating statement"); - this->db = db; -} - - -void SQLiteStmt::reset() -{ - assert(stmt); - /* Note: sqlite3_reset() returns the error code for the most - recent call to sqlite3_step(). So ignore it. */ - sqlite3_reset(stmt); - curArg = 1; -} - - -SQLiteStmt::~SQLiteStmt() -{ - try { - if (stmt && sqlite3_finalize(stmt) != SQLITE_OK) - throwSQLiteError(db, "finalizing statement"); - } catch (...) { - ignoreException(); - } -} - - -void SQLiteStmt::bind(const string & value) -{ - if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); -} - - -void SQLiteStmt::bind(int value) -{ - if (sqlite3_bind_int(stmt, curArg++, value) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); -} - - -void SQLiteStmt::bind64(long long value) -{ - if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); -} - - -void SQLiteStmt::bind() -{ - if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); -} - - -/* Helper class to ensure that prepared statements are reset when - leaving the scope that uses them. Unfinished prepared statements - prevent transactions from being aborted, and can cause locks to be - kept when they should be released. */ -struct SQLiteStmtUse -{ - SQLiteStmt & stmt; - SQLiteStmtUse(SQLiteStmt & stmt) : stmt(stmt) - { - stmt.reset(); - } - ~SQLiteStmtUse() - { - try { - stmt.reset(); - } catch (...) { - ignoreException(); - } - } -}; - - -struct SQLiteTxn -{ - bool active; - sqlite3 * db; - - SQLiteTxn(sqlite3 * db) : active(false) { - this->db = db; - if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "starting transaction"); - active = true; - } - - void commit() - { - if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "committing transaction"); - active = false; - } - - ~SQLiteTxn() - { - try { - if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "aborting transaction"); - } catch (...) { - ignoreException(); - } - } -}; - - void checkStoreNotSymlink() { if (getEnv("NIX_IGNORE_SYMLINK_STORE") == "1") return; @@ -216,20 +55,22 @@ void checkStoreNotSymlink() } -LocalStore::LocalStore(bool reserveSpace) - : didSetSubstituterEnv(false) +LocalStore::LocalStore() + : linksDir(settings.nixStore + "/.links") + , reservedPath(settings.nixDBPath + "/reserved") + , schemaPath(settings.nixDBPath + "/schema") { - schemaPath = settings.nixDBPath + "/schema"; + auto state(_state.lock()); if (settings.readOnlyMode) { - openDB(false); + openDB(*state, false); return; } /* Create missing state directories if they don't already exist. */ createDirs(settings.nixStore); makeStoreWritable(); - createDirs(linksDir = settings.nixStore + "/.links"); + createDirs(linksDir); Path profilesDir = settings.nixStateDir + "/profiles"; createDirs(profilesDir); createDirs(settings.nixStateDir + "/temproots"); @@ -276,25 +117,20 @@ LocalStore::LocalStore(bool reserveSpace) needed, we reserve some dummy space that we can free just before doing a garbage collection. */ try { - Path reservedPath = settings.nixDBPath + "/reserved"; - if (reserveSpace) { - struct stat st; - if (stat(reservedPath.c_str(), &st) == -1 || - st.st_size != settings.reservedSize) - { - AutoCloseFD fd = open(reservedPath.c_str(), O_WRONLY | O_CREAT, 0600); - int res = -1; + struct stat st; + if (stat(reservedPath.c_str(), &st) == -1 || + st.st_size != settings.reservedSize) + { + AutoCloseFD fd = open(reservedPath.c_str(), O_WRONLY | O_CREAT, 0600); + int res = -1; #if HAVE_POSIX_FALLOCATE - res = posix_fallocate(fd, 0, settings.reservedSize); + res = posix_fallocate(fd, 0, settings.reservedSize); #endif - if (res == -1) { - writeFull(fd, string(settings.reservedSize, 'X')); - ftruncate(fd, settings.reservedSize); - } + if (res == -1) { + writeFull(fd, string(settings.reservedSize, 'X')); + ftruncate(fd, settings.reservedSize); } } - else - deletePath(reservedPath); } catch (SysError & e) { /* don't care about errors */ } @@ -306,7 +142,7 @@ LocalStore::LocalStore(bool reserveSpace) } catch (SysError & e) { if (e.errNo != EACCES) throw; settings.readOnlyMode = true; - openDB(false); + openDB(*state, false); return; } @@ -324,7 +160,7 @@ LocalStore::LocalStore(bool reserveSpace) else if (curSchema == 0) { /* new store */ curSchema = nixSchemaVersion; - openDB(true); + openDB(*state, true); writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); } @@ -335,6 +171,12 @@ LocalStore::LocalStore(bool reserveSpace) "which is no longer supported. To convert to the new format,\n" "please upgrade Nix to version 0.12 first."); + if (curSchema < 6) + throw Error( + "Your Nix store has a database in flat file format,\n" + "which is no longer supported. To convert to the new format,\n" + "please upgrade Nix to version 1.11 first."); + if (!lockFile(globalLock, ltWrite, false)) { printMsg(lvlError, "waiting for exclusive access to the Nix store..."); lockFile(globalLock, ltWrite, true); @@ -344,22 +186,41 @@ LocalStore::LocalStore(bool reserveSpace) have performed the upgrade already. */ curSchema = getSchema(); - if (curSchema < 6) upgradeStore6(); - else if (curSchema < 7) { upgradeStore7(); openDB(true); } + if (curSchema < 7) { upgradeStore7(); } + + openDB(*state, false); + + if (curSchema < 8) { + SQLiteTxn txn(state->db); + if (sqlite3_exec(state->db, "alter table ValidPaths add column ultimate integer", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(state->db, "upgrading database schema"); + if (sqlite3_exec(state->db, "alter table ValidPaths add column sigs text", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(state->db, "upgrading database schema"); + txn.commit(); + } + + if (curSchema < 9) { + SQLiteTxn txn(state->db); + if (sqlite3_exec(state->db, "drop table FailedPaths", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(state->db, "upgrading database schema"); + txn.commit(); + } writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); lockFile(globalLock, ltRead, true); } - else openDB(false); + else openDB(*state, false); } LocalStore::~LocalStore() { + auto state(_state.lock()); + try { - for (auto & i : runningSubstituters) { + for (auto & i : state->runningSubstituters) { if (i.second.disabled) continue; i.second.to.close(); i.second.from.close(); @@ -372,9 +233,9 @@ LocalStore::~LocalStore() } try { - if (fdTempRoots != -1) { - fdTempRoots.close(); - unlink(fnTempRoots.c_str()); + if (state->fdTempRoots != -1) { + state->fdTempRoots.close(); + unlink(state->fnTempRoots.c_str()); } } catch (...) { ignoreException(); @@ -400,13 +261,14 @@ bool LocalStore::haveWriteAccess() } -void LocalStore::openDB(bool create) +void LocalStore::openDB(State & state, bool create) { if (!haveWriteAccess()) throw SysError(format("Nix database directory ‘%1%’ is not writable") % settings.nixDBPath); /* Open the Nix database. */ string dbPath = settings.nixDBPath + "/db.sqlite"; + auto & db(state.db); if (sqlite3_open_v2(dbPath.c_str(), &db.db, SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) throw Error(format("cannot open Nix database ‘%1%’") % dbPath); @@ -459,40 +321,31 @@ void LocalStore::openDB(bool create) } /* Prepare SQL statements. */ - stmtRegisterValidPath.create(db, - "insert into ValidPaths (path, hash, registrationTime, deriver, narSize) values (?, ?, ?, ?, ?);"); - stmtUpdatePathInfo.create(db, - "update ValidPaths set narSize = ?, hash = ? where path = ?;"); - stmtAddReference.create(db, + state.stmtRegisterValidPath.create(db, + "insert into ValidPaths (path, hash, registrationTime, deriver, narSize, ultimate, sigs) values (?, ?, ?, ?, ?, ?, ?);"); + state.stmtUpdatePathInfo.create(db, + "update ValidPaths set narSize = ?, hash = ?, ultimate = ?, sigs = ? where path = ?;"); + state.stmtAddReference.create(db, "insert or replace into Refs (referrer, reference) values (?, ?);"); - stmtQueryPathInfo.create(db, - "select id, hash, registrationTime, deriver, narSize from ValidPaths where path = ?;"); - stmtQueryReferences.create(db, + state.stmtQueryPathInfo.create(db, + "select id, hash, registrationTime, deriver, narSize, ultimate, sigs from ValidPaths where path = ?;"); + state.stmtQueryReferences.create(db, "select path from Refs join ValidPaths on reference = id where referrer = ?;"); - stmtQueryReferrers.create(db, + state.stmtQueryReferrers.create(db, "select path from Refs join ValidPaths on referrer = id where reference = (select id from ValidPaths where path = ?);"); - stmtInvalidatePath.create(db, + state.stmtInvalidatePath.create(db, "delete from ValidPaths where path = ?;"); - stmtRegisterFailedPath.create(db, - "insert or ignore into FailedPaths (path, time) values (?, ?);"); - stmtHasPathFailed.create(db, - "select time from FailedPaths where path = ?;"); - stmtQueryFailedPaths.create(db, - "select path from FailedPaths;"); - // If the path is a derivation, then clear its outputs. - stmtClearFailedPath.create(db, - "delete from FailedPaths where ?1 = '*' or path = ?1 " - "or path in (select d.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where v.path = ?1);"); - stmtAddDerivationOutput.create(db, + state.stmtAddDerivationOutput.create(db, "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);"); - stmtQueryValidDerivers.create(db, + state.stmtQueryValidDerivers.create(db, "select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;"); - stmtQueryDerivationOutputs.create(db, + state.stmtQueryDerivationOutputs.create(db, "select id, path from DerivationOutputs where drv = ?;"); // Use "path >= ?" with limit 1 rather than "path like '?%'" to // ensure efficient lookup. - stmtQueryPathFromHashPart.create(db, + state.stmtQueryPathFromHashPart.create(db, "select path from ValidPaths where path >= ? limit 1;"); + state.stmtQueryValidPaths.create(db, "select path from ValidPaths"); } @@ -687,23 +540,19 @@ void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation & } -unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool checkOutputs) +uint64_t LocalStore::addValidPath(State & state, + const ValidPathInfo & info, bool checkOutputs) { - SQLiteStmtUse use(stmtRegisterValidPath); - stmtRegisterValidPath.bind(info.path); - stmtRegisterValidPath.bind("sha256:" + printHash(info.narHash)); - stmtRegisterValidPath.bind(info.registrationTime == 0 ? time(0) : info.registrationTime); - if (info.deriver != "") - stmtRegisterValidPath.bind(info.deriver); - else - stmtRegisterValidPath.bind(); // null - if (info.narSize != 0) - stmtRegisterValidPath.bind64(info.narSize); - else - stmtRegisterValidPath.bind(); // null - if (sqlite3_step(stmtRegisterValidPath) != SQLITE_DONE) - throwSQLiteError(db, format("registering valid path ‘%1%’ in database") % info.path); - unsigned long long id = sqlite3_last_insert_rowid(db); + state.stmtRegisterValidPath.use() + (info.path) + ("sha256:" + printHash(info.narHash)) + (info.registrationTime == 0 ? time(0) : info.registrationTime) + (info.deriver, info.deriver != "") + (info.narSize, info.narSize != 0) + (info.ultimate ? 1 : 0, info.ultimate) + (concatStringsSep(" ", info.sigs), !info.sigs.empty()) + .exec(); + uint64_t id = sqlite3_last_insert_rowid(state.db); /* If this is a derivation, then store the derivation outputs in the database. This is useful for the garbage collector: it can @@ -720,12 +569,11 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool che if (checkOutputs) checkDerivationOutputs(info.path, drv); for (auto & i : drv.outputs) { - SQLiteStmtUse use(stmtAddDerivationOutput); - stmtAddDerivationOutput.bind(id); - stmtAddDerivationOutput.bind(i.first); - stmtAddDerivationOutput.bind(i.second.path); - if (sqlite3_step(stmtAddDerivationOutput) != SQLITE_DONE) - throwSQLiteError(db, format("adding derivation output for ‘%1%’ in database") % info.path); + state.stmtAddDerivationOutput.use() + (id) + (i.first) + (i.second.path) + .exec(); } } @@ -733,79 +581,6 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool che } -void LocalStore::addReference(unsigned long long referrer, unsigned long long reference) -{ - SQLiteStmtUse use(stmtAddReference); - stmtAddReference.bind(referrer); - stmtAddReference.bind(reference); - if (sqlite3_step(stmtAddReference) != SQLITE_DONE) - throwSQLiteError(db, "adding reference to database"); -} - - -void LocalStore::registerFailedPath(const Path & path) -{ - retry_sqlite { - SQLiteStmtUse use(stmtRegisterFailedPath); - stmtRegisterFailedPath.bind(path); - stmtRegisterFailedPath.bind(time(0)); - if (sqlite3_step(stmtRegisterFailedPath) != SQLITE_DONE) - throwSQLiteError(db, format("registering failed path ‘%1%’") % path); - } end_retry_sqlite; -} - - -bool LocalStore::hasPathFailed(const Path & path) -{ - retry_sqlite { - SQLiteStmtUse use(stmtHasPathFailed); - stmtHasPathFailed.bind(path); - int res = sqlite3_step(stmtHasPathFailed); - if (res != SQLITE_DONE && res != SQLITE_ROW) - throwSQLiteError(db, "querying whether path failed"); - return res == SQLITE_ROW; - } end_retry_sqlite; -} - - -PathSet LocalStore::queryFailedPaths() -{ - retry_sqlite { - SQLiteStmtUse use(stmtQueryFailedPaths); - - PathSet res; - int r; - while ((r = sqlite3_step(stmtQueryFailedPaths)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryFailedPaths, 0); - assert(s); - res.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, "error querying failed paths"); - - return res; - } end_retry_sqlite; -} - - -void LocalStore::clearFailedPaths(const PathSet & paths) -{ - retry_sqlite { - SQLiteTxn txn(db); - - for (auto & i : paths) { - SQLiteStmtUse use(stmtClearFailedPath); - stmtClearFailedPath.bind(i); - if (sqlite3_step(stmtClearFailedPath) != SQLITE_DONE) - throwSQLiteError(db, format("clearing failed path ‘%1%’ in database") % i); - } - - txn.commit(); - } end_retry_sqlite; -} - - Hash parseHashField(const Path & path, const string & s) { string::size_type colon = s.find(':'); @@ -827,153 +602,117 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path) assertStorePath(path); - retry_sqlite { + return retrySQLite<ValidPathInfo>([&]() { + auto state(_state.lock()); /* Get the path info. */ - SQLiteStmtUse use1(stmtQueryPathInfo); - - stmtQueryPathInfo.bind(path); + auto useQueryPathInfo(state->stmtQueryPathInfo.use()(path)); - int r = sqlite3_step(stmtQueryPathInfo); - if (r == SQLITE_DONE) throw Error(format("path ‘%1%’ is not valid") % path); - if (r != SQLITE_ROW) throwSQLiteError(db, "querying path in database"); + if (!useQueryPathInfo.next()) + throw Error(format("path ‘%1%’ is not valid") % path); - info.id = sqlite3_column_int(stmtQueryPathInfo, 0); + info.id = useQueryPathInfo.getInt(0); - const char * s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 1); - assert(s); - info.narHash = parseHashField(path, s); + info.narHash = parseHashField(path, useQueryPathInfo.getStr(1)); - info.registrationTime = sqlite3_column_int(stmtQueryPathInfo, 2); + info.registrationTime = useQueryPathInfo.getInt(2); - s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 3); + auto s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 3); if (s) info.deriver = s; /* Note that narSize = NULL yields 0. */ - info.narSize = sqlite3_column_int64(stmtQueryPathInfo, 4); + info.narSize = useQueryPathInfo.getInt(4); - /* Get the references. */ - SQLiteStmtUse use2(stmtQueryReferences); + info.ultimate = useQueryPathInfo.getInt(5) == 1; - stmtQueryReferences.bind(info.id); + s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 6); + if (s) info.sigs = tokenizeString<StringSet>(s, " "); - while ((r = sqlite3_step(stmtQueryReferences)) == SQLITE_ROW) { - s = (const char *) sqlite3_column_text(stmtQueryReferences, 0); - assert(s); - info.references.insert(s); - } + /* Get the references. */ + auto useQueryReferences(state->stmtQueryReferences.use()(info.id)); - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting references of ‘%1%’") % path); + while (useQueryReferences.next()) + info.references.insert(useQueryReferences.getStr(0)); return info; - } end_retry_sqlite; + }); } -/* Update path info in the database. Currently only updates the - narSize field. */ -void LocalStore::updatePathInfo(const ValidPathInfo & info) +/* Update path info in the database. */ +void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { - SQLiteStmtUse use(stmtUpdatePathInfo); - if (info.narSize != 0) - stmtUpdatePathInfo.bind64(info.narSize); - else - stmtUpdatePathInfo.bind(); // null - stmtUpdatePathInfo.bind("sha256:" + printHash(info.narHash)); - stmtUpdatePathInfo.bind(info.path); - if (sqlite3_step(stmtUpdatePathInfo) != SQLITE_DONE) - throwSQLiteError(db, format("updating info of path ‘%1%’ in database") % info.path); + state.stmtUpdatePathInfo.use() + (info.narSize, info.narSize != 0) + ("sha256:" + printHash(info.narHash)) + (info.ultimate ? 1 : 0, info.ultimate) + (concatStringsSep(" ", info.sigs), !info.sigs.empty()) + (info.path) + .exec(); } -unsigned long long LocalStore::queryValidPathId(const Path & path) +uint64_t LocalStore::queryValidPathId(State & state, const Path & path) { - SQLiteStmtUse use(stmtQueryPathInfo); - stmtQueryPathInfo.bind(path); - int res = sqlite3_step(stmtQueryPathInfo); - if (res == SQLITE_ROW) return sqlite3_column_int(stmtQueryPathInfo, 0); - if (res == SQLITE_DONE) throw Error(format("path ‘%1%’ is not valid") % path); - throwSQLiteError(db, "querying path in database"); + auto use(state.stmtQueryPathInfo.use()(path)); + if (!use.next()) + throw Error(format("path ‘%1%’ is not valid") % path); + return use.getInt(0); } -bool LocalStore::isValidPath_(const Path & path) +bool LocalStore::isValidPath(State & state, const Path & path) { - SQLiteStmtUse use(stmtQueryPathInfo); - stmtQueryPathInfo.bind(path); - int res = sqlite3_step(stmtQueryPathInfo); - if (res != SQLITE_DONE && res != SQLITE_ROW) - throwSQLiteError(db, "querying path in database"); - return res == SQLITE_ROW; + return state.stmtQueryPathInfo.use()(path).next(); } bool LocalStore::isValidPath(const Path & path) { - retry_sqlite { - return isValidPath_(path); - } end_retry_sqlite; + return retrySQLite<bool>([&]() { + auto state(_state.lock()); + return isValidPath(*state, path); + }); } PathSet LocalStore::queryValidPaths(const PathSet & paths) { - retry_sqlite { - PathSet res; - for (auto & i : paths) - if (isValidPath_(i)) res.insert(i); - return res; - } end_retry_sqlite; + PathSet res; + for (auto & i : paths) + if (isValidPath(i)) res.insert(i); + return res; } PathSet LocalStore::queryAllValidPaths() { - retry_sqlite { - SQLiteStmt stmt; - stmt.create(db, "select path from ValidPaths"); - + return retrySQLite<PathSet>([&]() { + auto state(_state.lock()); + auto use(state->stmtQueryValidPaths.use()); PathSet res; - int r; - while ((r = sqlite3_step(stmt)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmt, 0); - assert(s); - res.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, "error getting valid paths"); - + while (use.next()) res.insert(use.getStr(0)); return res; - } end_retry_sqlite; + }); } -void LocalStore::queryReferrers_(const Path & path, PathSet & referrers) +void LocalStore::queryReferrers(State & state, const Path & path, PathSet & referrers) { - SQLiteStmtUse use(stmtQueryReferrers); + auto useQueryReferrers(state.stmtQueryReferrers.use()(path)); - stmtQueryReferrers.bind(path); - - int r; - while ((r = sqlite3_step(stmtQueryReferrers)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryReferrers, 0); - assert(s); - referrers.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting references of ‘%1%’") % path); + while (useQueryReferrers.next()) + referrers.insert(useQueryReferrers.getStr(0)); } void LocalStore::queryReferrers(const Path & path, PathSet & referrers) { assertStorePath(path); - retry_sqlite { - queryReferrers_(path, referrers); - } end_retry_sqlite; + return retrySQLite<void>([&]() { + auto state(_state.lock()); + queryReferrers(*state, path, referrers); + }); } @@ -987,67 +726,51 @@ PathSet LocalStore::queryValidDerivers(const Path & path) { assertStorePath(path); - retry_sqlite { - SQLiteStmtUse use(stmtQueryValidDerivers); - stmtQueryValidDerivers.bind(path); + return retrySQLite<PathSet>([&]() { + auto state(_state.lock()); - PathSet derivers; - int r; - while ((r = sqlite3_step(stmtQueryValidDerivers)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryValidDerivers, 1); - assert(s); - derivers.insert(s); - } + auto useQueryValidDerivers(state->stmtQueryValidDerivers.use()(path)); - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting valid derivers of ‘%1%’") % path); + PathSet derivers; + while (useQueryValidDerivers.next()) + derivers.insert(useQueryValidDerivers.getStr(1)); return derivers; - } end_retry_sqlite; + }); } PathSet LocalStore::queryDerivationOutputs(const Path & path) { - retry_sqlite { - SQLiteStmtUse use(stmtQueryDerivationOutputs); - stmtQueryDerivationOutputs.bind(queryValidPathId(path)); + return retrySQLite<PathSet>([&]() { + auto state(_state.lock()); - PathSet outputs; - int r; - while ((r = sqlite3_step(stmtQueryDerivationOutputs)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryDerivationOutputs, 1); - assert(s); - outputs.insert(s); - } + auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use() + (queryValidPathId(*state, path))); - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting outputs of ‘%1%’") % path); + PathSet outputs; + while (useQueryDerivationOutputs.next()) + outputs.insert(useQueryDerivationOutputs.getStr(1)); return outputs; - } end_retry_sqlite; + }); } StringSet LocalStore::queryDerivationOutputNames(const Path & path) { - retry_sqlite { - SQLiteStmtUse use(stmtQueryDerivationOutputs); - stmtQueryDerivationOutputs.bind(queryValidPathId(path)); + return retrySQLite<StringSet>([&]() { + auto state(_state.lock()); - StringSet outputNames; - int r; - while ((r = sqlite3_step(stmtQueryDerivationOutputs)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryDerivationOutputs, 0); - assert(s); - outputNames.insert(s); - } + auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use() + (queryValidPathId(*state, path))); - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting output names of ‘%1%’") % path); + StringSet outputNames; + while (useQueryDerivationOutputs.next()) + outputNames.insert(useQueryDerivationOutputs.getStr(0)); return outputNames; - } end_retry_sqlite; + }); } @@ -1057,29 +780,28 @@ Path LocalStore::queryPathFromHashPart(const string & hashPart) Path prefix = settings.nixStore + "/" + hashPart; - retry_sqlite { - SQLiteStmtUse use(stmtQueryPathFromHashPart); - stmtQueryPathFromHashPart.bind(prefix); + return retrySQLite<Path>([&]() { + auto state(_state.lock()); + + auto useQueryPathFromHashPart(state->stmtQueryPathFromHashPart.use()(prefix)); - int res = sqlite3_step(stmtQueryPathFromHashPart); - if (res == SQLITE_DONE) return ""; - if (res != SQLITE_ROW) throwSQLiteError(db, "finding path in database"); + if (!useQueryPathFromHashPart.next()) return ""; - const char * s = (const char *) sqlite3_column_text(stmtQueryPathFromHashPart, 0); + const char * s = (const char *) sqlite3_column_text(state->stmtQueryPathFromHashPart, 0); return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s : ""; - } end_retry_sqlite; + }); } void LocalStore::setSubstituterEnv() { - if (didSetSubstituterEnv) return; + static std::atomic_flag done; + + if (done.test_and_set()) return; /* Pass configuration options (including those overridden with --option) to substituters. */ setenv("_NIX_OPTIONS", settings.pack().c_str(), 1); - - didSetSubstituterEnv = true; } @@ -1201,10 +923,12 @@ template<class T> T LocalStore::getIntLineFromSubstituter(RunningSubstituter & r PathSet LocalStore::querySubstitutablePaths(const PathSet & paths) { + auto state(_state.lock()); + PathSet res; for (auto & i : settings.substituters) { if (res.size() == paths.size()) break; - RunningSubstituter & run(runningSubstituters[i]); + RunningSubstituter & run(state->runningSubstituters[i]); startSubstituter(i, run); if (run.disabled) continue; string s = "have "; @@ -1221,6 +945,7 @@ PathSet LocalStore::querySubstitutablePaths(const PathSet & paths) res.insert(path); } } + return res; } @@ -1228,7 +953,9 @@ PathSet LocalStore::querySubstitutablePaths(const PathSet & paths) void LocalStore::querySubstitutablePathInfos(const Path & substituter, PathSet & paths, SubstitutablePathInfos & infos) { - RunningSubstituter & run(runningSubstituters[substituter]); + auto state(_state.lock()); + + RunningSubstituter & run(state->runningSubstituters[substituter]); startSubstituter(substituter, run); if (run.disabled) return; @@ -1285,28 +1012,31 @@ void LocalStore::registerValidPath(const ValidPathInfo & info) void LocalStore::registerValidPaths(const ValidPathInfos & infos) { - /* SQLite will fsync by default, but the new valid paths may not be fsync-ed. - * So some may want to fsync them before registering the validity, at the - * expense of some speed of the path registering operation. */ + /* SQLite will fsync by default, but the new valid paths may not + be fsync-ed. So some may want to fsync them before registering + the validity, at the expense of some speed of the path + registering operation. */ if (settings.syncBeforeRegistering) sync(); - retry_sqlite { - SQLiteTxn txn(db); + return retrySQLite<void>([&]() { + auto state(_state.lock()); + + SQLiteTxn txn(state->db); PathSet paths; for (auto & i : infos) { assert(i.narHash.type == htSHA256); - if (isValidPath_(i.path)) - updatePathInfo(i); + if (isValidPath(*state, i.path)) + updatePathInfo(*state, i); else - addValidPath(i, false); + addValidPath(*state, i, false); paths.insert(i.path); } for (auto & i : infos) { - unsigned long long referrer = queryValidPathId(i.path); + auto referrer = queryValidPathId(*state, i.path); for (auto & j : i.references) - addReference(referrer, queryValidPathId(j)); + state->stmtAddReference.use()(referrer)(queryValidPathId(*state, j)).exec(); } /* Check that the derivation outputs are correct. We can't do @@ -1327,24 +1057,17 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) topoSortPaths(paths); txn.commit(); - } end_retry_sqlite; + }); } /* Invalidate a path. The caller is responsible for checking that there are no referrers. */ -void LocalStore::invalidatePath(const Path & path) +void LocalStore::invalidatePath(State & state, const Path & path) { debug(format("invalidating path ‘%1%’") % path); - drvHashes.erase(path); - - SQLiteStmtUse use(stmtInvalidatePath); - - stmtInvalidatePath.bind(path); - - if (sqlite3_step(stmtInvalidatePath) != SQLITE_DONE) - throwSQLiteError(db, format("invalidating path ‘%1%’ in database") % path); + state.stmtInvalidatePath.use()(path).exec(); /* Note that the foreign key constraints on the Refs table take care of deleting the references entries for `path'. */ @@ -1369,7 +1092,7 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name, if (repair || !isValidPath(dstPath)) { - if (pathExists(dstPath)) deletePath(dstPath); + deletePath(dstPath); if (recursive) { StringSource source(dump); @@ -1396,6 +1119,7 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name, info.path = dstPath; info.narHash = hash.first; info.narSize = hash.second; + info.ultimate = true; registerValidPath(info); } @@ -1410,7 +1134,6 @@ Path LocalStore::addToStore(const string & name, const Path & _srcPath, bool recursive, HashType hashAlgo, PathFilter & filter, bool repair) { Path srcPath(absPath(_srcPath)); - debug(format("adding ‘%1%’ to the store") % srcPath); /* Read the whole path into memory. This is not a very scalable method for very large paths, but `copyPath' is mainly used for @@ -1419,9 +1142,9 @@ Path LocalStore::addToStore(const string & name, const Path & _srcPath, if (recursive) dumpPath(srcPath, sink, filter); else - sink.s = readFile(srcPath); + sink.s = make_ref<std::string>(readFile(srcPath)); - return addToStoreFromDump(sink.s, name, recursive, hashAlgo, repair); + return addToStoreFromDump(*sink.s, name, recursive, hashAlgo, repair); } @@ -1438,21 +1161,24 @@ Path LocalStore::addTextToStore(const string & name, const string & s, if (repair || !isValidPath(dstPath)) { - if (pathExists(dstPath)) deletePath(dstPath); + deletePath(dstPath); writeFile(dstPath, s); canonicalisePathMetaData(dstPath, -1); - HashResult hash = hashPath(htSHA256, dstPath); + StringSink sink; + dumpString(s, sink); + auto hash = hashString(htSHA256, *sink.s); optimisePath(dstPath); ValidPathInfo info; info.path = dstPath; - info.narHash = hash.first; - info.narSize = hash.second; + info.narHash = hash; + info.narSize = sink.s->size(); info.references = references; + info.ultimate = true; registerValidPath(info); } @@ -1661,7 +1387,7 @@ Path LocalStore::importPath(bool requireSignature, Source & source) if (!isValidPath(dstPath)) { - if (pathExists(dstPath)) deletePath(dstPath); + deletePath(dstPath); if (rename(unpacked.c_str(), dstPath.c_str()) == -1) throw SysError(format("cannot move ‘%1%’ to ‘%2%’") @@ -1691,7 +1417,8 @@ Path LocalStore::importPath(bool requireSignature, Source & source) } -Paths LocalStore::importPaths(bool requireSignature, Source & source) +Paths LocalStore::importPaths(bool requireSignature, Source & source, + std::shared_ptr<FSAccessor> accessor) { Paths res; while (true) { @@ -1708,20 +1435,22 @@ void LocalStore::invalidatePathChecked(const Path & path) { assertStorePath(path); - retry_sqlite { - SQLiteTxn txn(db); + retrySQLite<void>([&]() { + auto state(_state.lock()); - if (isValidPath_(path)) { - PathSet referrers; queryReferrers_(path, referrers); + SQLiteTxn txn(state->db); + + if (isValidPath(*state, path)) { + PathSet referrers; queryReferrers(*state, path, referrers); referrers.erase(path); /* ignore self-references */ if (!referrers.empty()) throw PathInUse(format("cannot delete path ‘%1%’ because it is in use by %2%") % path % showPaths(referrers)); - invalidatePath(path); + invalidatePath(*state, path); } txn.commit(); - } end_retry_sqlite; + }); } @@ -1786,7 +1515,10 @@ bool LocalStore::verifyStore(bool checkContents, bool repair) update = true; } - if (update) updatePathInfo(info); + if (update) { + auto state(_state.lock()); + updatePathInfo(*state, info); + } } @@ -1816,7 +1548,8 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store, if (!isStorePath(path)) { printMsg(lvlError, format("path ‘%1%’ is not in the Nix store") % path); - invalidatePath(path); + auto state(_state.lock()); + invalidatePath(*state, path); return; } @@ -1834,7 +1567,8 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store, if (canInvalidate) { printMsg(lvlError, format("path ‘%1%’ disappeared, removing from database...") % path); - invalidatePath(path); + auto state(_state.lock()); + invalidatePath(*state, path); } else { printMsg(lvlError, format("path ‘%1%’ disappeared, but it still has valid referrers!") % path); if (repair) @@ -1854,114 +1588,6 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store, } -bool LocalStore::pathContentsGood(const Path & path) -{ - std::map<Path, bool>::iterator i = pathContentsGoodCache.find(path); - if (i != pathContentsGoodCache.end()) return i->second; - printMsg(lvlInfo, format("checking path ‘%1%’...") % path); - ValidPathInfo info = queryPathInfo(path); - bool res; - if (!pathExists(path)) - res = false; - else { - HashResult current = hashPath(info.narHash.type, path); - Hash nullHash(htSHA256); - res = info.narHash == nullHash || info.narHash == current.first; - } - pathContentsGoodCache[path] = res; - if (!res) printMsg(lvlError, format("path ‘%1%’ is corrupted or missing!") % path); - return res; -} - - -void LocalStore::markContentsGood(const Path & path) -{ - pathContentsGoodCache[path] = true; -} - - -/* Functions for upgrading from the pre-SQLite database. */ - -PathSet LocalStore::queryValidPathsOld() -{ - PathSet paths; - for (auto & i : readDirectory(settings.nixDBPath + "/info")) - if (i.name.at(0) != '.') paths.insert(settings.nixStore + "/" + i.name); - return paths; -} - - -ValidPathInfo LocalStore::queryPathInfoOld(const Path & path) -{ - ValidPathInfo res; - res.path = path; - - /* Read the info file. */ - string baseName = baseNameOf(path); - Path infoFile = (format("%1%/info/%2%") % settings.nixDBPath % baseName).str(); - if (!pathExists(infoFile)) - throw Error(format("path ‘%1%’ is not valid") % path); - string info = readFile(infoFile); - - /* Parse it. */ - Strings lines = tokenizeString<Strings>(info, "\n"); - - for (auto & i : lines) { - string::size_type p = i.find(':'); - if (p == string::npos) - throw Error(format("corrupt line in ‘%1%’: %2%") % infoFile % i); - string name(i, 0, p); - string value(i, p + 2); - if (name == "References") { - Strings refs = tokenizeString<Strings>(value, " "); - res.references = PathSet(refs.begin(), refs.end()); - } else if (name == "Deriver") { - res.deriver = value; - } else if (name == "Hash") { - res.narHash = parseHashField(path, value); - } else if (name == "Registered-At") { - int n = 0; - string2Int(value, n); - res.registrationTime = n; - } - } - - return res; -} - - -/* Upgrade from schema 5 (Nix 0.12) to schema 6 (Nix >= 0.15). */ -void LocalStore::upgradeStore6() -{ - printMsg(lvlError, "upgrading Nix store to new schema (this may take a while)..."); - - openDB(true); - - PathSet validPaths = queryValidPathsOld(); - - SQLiteTxn txn(db); - - for (auto & i : validPaths) { - addValidPath(queryPathInfoOld(i), false); - std::cerr << "."; - } - - std::cerr << "|"; - - for (auto & i : validPaths) { - ValidPathInfo info = queryPathInfoOld(i); - unsigned long long referrer = queryValidPathId(i); - for (auto & j : info.references) - addReference(referrer, queryValidPathId(j)); - std::cerr << "."; - } - - std::cerr << "\n"; - - txn.commit(); -} - - #if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && defined(FS_IMMUTABLE_FL) static void makeMutable(const Path & path) @@ -2016,8 +1642,41 @@ void LocalStore::upgradeStore7() void LocalStore::vacuumDB() { - if (sqlite3_exec(db, "vacuum;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "vacuuming SQLite database"); + auto state(_state.lock()); + + if (sqlite3_exec(state->db, "vacuum;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(state->db, "vacuuming SQLite database"); +} + + +void LocalStore::addSignatures(const Path & storePath, const StringSet & sigs) +{ + retrySQLite<void>([&]() { + auto state(_state.lock()); + + SQLiteTxn txn(state->db); + + auto info = queryPathInfo(storePath); + + info.sigs.insert(sigs.begin(), sigs.end()); + + updatePathInfo(*state, info); + + txn.commit(); + }); +} + + +void LocalStore::signPathInfo(ValidPathInfo & info) +{ + // FIXME: keep secret keys in memory. + + auto secretKeyFiles = settings.get("secret-key-files", Strings()); + + for (auto & secretKeyFile : secretKeyFiles) { + SecretKey secretKey(readFile(secretKeyFile)); + info.sign(secretKey); + } } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index a96000d9fbeb..14ff92c35cc5 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -1,15 +1,14 @@ #pragma once -#include <string> -#include <unordered_set> +#include "sqlite.hh" +#include "pathlocks.hh" #include "store-api.hh" +#include "sync.hh" #include "util.hh" -#include "pathlocks.hh" - -class sqlite3; -class sqlite3_stmt; +#include <string> +#include <unordered_set> namespace nix { @@ -18,8 +17,8 @@ namespace nix { /* Nix store and database schema version. Version 1 (or 0) was Nix <= 0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10. Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is - Nix 1.0. Version 7 is Nix 1.3. */ -const int nixSchemaVersion = 7; + Nix 1.0. Version 7 is Nix 1.3. Version 9 is 1.12. */ +const int nixSchemaVersion = 9; extern string drvsLogDir; @@ -52,47 +51,52 @@ struct RunningSubstituter }; -/* Wrapper object to close the SQLite database automatically. */ -struct SQLite -{ - sqlite3 * db; - SQLite() { db = 0; } - ~SQLite(); - operator sqlite3 * () { return db; } -}; - - -/* Wrapper object to create and destroy SQLite prepared statements. */ -struct SQLiteStmt -{ - sqlite3 * db; - sqlite3_stmt * stmt; - unsigned int curArg; - SQLiteStmt() { stmt = 0; } - void create(sqlite3 * db, const string & s); - void reset(); - ~SQLiteStmt(); - operator sqlite3_stmt * () { return stmt; } - void bind(const string & value); - void bind(int value); - void bind64(long long value); - void bind(); -}; - - -class LocalStore : public Store +class LocalStore : public LocalFSStore { private: - typedef std::map<Path, RunningSubstituter> RunningSubstituters; - RunningSubstituters runningSubstituters; - Path linksDir; + /* Lock file used for upgrading. */ + AutoCloseFD globalLock; + + struct State + { + /* The SQLite database object. */ + SQLite db; + + /* Some precompiled SQLite statements. */ + SQLiteStmt stmtRegisterValidPath; + SQLiteStmt stmtUpdatePathInfo; + SQLiteStmt stmtAddReference; + SQLiteStmt stmtQueryPathInfo; + SQLiteStmt stmtQueryReferences; + SQLiteStmt stmtQueryReferrers; + SQLiteStmt stmtInvalidatePath; + SQLiteStmt stmtAddDerivationOutput; + SQLiteStmt stmtQueryValidDerivers; + SQLiteStmt stmtQueryDerivationOutputs; + SQLiteStmt stmtQueryPathFromHashPart; + SQLiteStmt stmtQueryValidPaths; + + /* The file to which we write our temporary roots. */ + Path fnTempRoots; + AutoCloseFD fdTempRoots; + + typedef std::map<Path, RunningSubstituter> RunningSubstituters; + RunningSubstituters runningSubstituters; + + }; + + Sync<State, std::recursive_mutex> _state; + + const Path linksDir; + const Path reservedPath; + const Path schemaPath; public: /* Initialise the local store, upgrading the schema if necessary. */ - LocalStore(bool reserveSpace = true); + LocalStore(); ~LocalStore(); @@ -145,7 +149,8 @@ public: void exportPath(const Path & path, bool sign, Sink & sink) override; - Paths importPaths(bool requireSignature, Source & source) override; + Paths importPaths(bool requireSignature, Source & source, + std::shared_ptr<FSAccessor> accessor) override; void buildPaths(const PathSet & paths, BuildMode buildMode) override; @@ -168,14 +173,11 @@ public: files with the same contents. */ void optimiseStore(OptimiseStats & stats); - /* Generic variant of the above method. */ void optimiseStore() override; /* Optimise a single store path. */ void optimisePath(const Path & path); - /* Check the integrity of the Nix store. Returns true if errors - remain. */ bool verifyStore(bool checkContents, bool repair) override; /* Register the validity of a path, i.e., that `path' exists, that @@ -188,90 +190,31 @@ public: void registerValidPaths(const ValidPathInfos & infos); - /* Register that the build of a derivation with output `path' has - failed. */ - void registerFailedPath(const Path & path); - - /* Query whether `path' previously failed to build. */ - bool hasPathFailed(const Path & path); - - PathSet queryFailedPaths() override; - - void clearFailedPaths(const PathSet & paths) override; - void vacuumDB(); /* Repair the contents of the given path by redownloading it using a substituter (if available). */ void repairPath(const Path & path); - /* Check whether the given valid path exists and has the right - contents. */ - bool pathContentsGood(const Path & path); - - void markContentsGood(const Path & path); - void setSubstituterEnv(); -private: - - Path schemaPath; - - /* Lock file used for upgrading. */ - AutoCloseFD globalLock; - - /* The SQLite database object. */ - SQLite db; - - /* Some precompiled SQLite statements. */ - SQLiteStmt stmtRegisterValidPath; - SQLiteStmt stmtUpdatePathInfo; - SQLiteStmt stmtAddReference; - SQLiteStmt stmtQueryPathInfo; - SQLiteStmt stmtQueryReferences; - SQLiteStmt stmtQueryReferrers; - SQLiteStmt stmtInvalidatePath; - SQLiteStmt stmtRegisterFailedPath; - SQLiteStmt stmtHasPathFailed; - SQLiteStmt stmtQueryFailedPaths; - SQLiteStmt stmtClearFailedPath; - SQLiteStmt stmtAddDerivationOutput; - SQLiteStmt stmtQueryValidDerivers; - SQLiteStmt stmtQueryDerivationOutputs; - SQLiteStmt stmtQueryPathFromHashPart; - - /* Cache for pathContentsGood(). */ - std::map<Path, bool> pathContentsGoodCache; - - bool didSetSubstituterEnv; - - /* The file to which we write our temporary roots. */ - Path fnTempRoots; - AutoCloseFD fdTempRoots; - - int getSchema(); - -public: + void addSignatures(const Path & storePath, const StringSet & sigs) override; static bool haveWriteAccess(); private: - void openDB(bool create); - - void makeStoreWritable(); - - unsigned long long queryValidPathId(const Path & path); + int getSchema(); - unsigned long long addValidPath(const ValidPathInfo & info, bool checkOutputs = true); + void openDB(State & state, bool create); - void addReference(unsigned long long referrer, unsigned long long reference); + void makeStoreWritable(); - void appendReferrer(const Path & from, const Path & to, bool lock); + uint64_t queryValidPathId(State & state, const Path & path); - void rewriteReferrers(const Path & path, bool purge, PathSet referrers); + uint64_t addValidPath(State & state, const ValidPathInfo & info, bool checkOutputs = true); - void invalidatePath(const Path & path); + void invalidatePath(State & state, const Path & path); /* Delete a path from the Nix store. */ void invalidatePathChecked(const Path & path); @@ -279,7 +222,7 @@ private: void verifyPath(const Path & path, const PathSet & store, PathSet & done, PathSet & validPaths, bool repair, bool & errors); - void updatePathInfo(const ValidPathInfo & info); + void updatePathInfo(State & state, const ValidPathInfo & info); void upgradeStore6(); void upgradeStore7(); @@ -327,8 +270,14 @@ private: void optimisePath_(OptimiseStats & stats, const Path & path, InodeHash & inodeHash); // Internal versions that are not wrapped in retry_sqlite. - bool isValidPath_(const Path & path); - void queryReferrers_(const Path & path, PathSet & referrers); + bool isValidPath(State & state, const Path & path); + void queryReferrers(State & state, const Path & path, PathSet & referrers); + + /* Add signatures to a ValidPathInfo using the secret keys + specified by the ‘secret-key-files’ option. */ + void signPathInfo(ValidPathInfo & info); + + friend class DerivationGoal; }; diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc new file mode 100644 index 000000000000..8896862be149 --- /dev/null +++ b/src/libstore/nar-accessor.cc @@ -0,0 +1,141 @@ +#include "nar-accessor.hh" +#include "archive.hh" + +#include <map> + +namespace nix { + +struct NarMember +{ + FSAccessor::Type type; + + bool isExecutable; + + /* If this is a regular file, position of the contents of this + file in the NAR. */ + size_t start, size; + + std::string target; +}; + +struct NarIndexer : ParseSink, StringSource +{ + // FIXME: should store this as a tree. Now we're vulnerable to + // O(nm) memory consumption (e.g. for x_0/.../x_n/{y_0..y_m}). + typedef std::map<Path, NarMember> Members; + Members members; + + Path currentPath; + std::string currentStart; + bool isExec; + + NarIndexer(const std::string & nar) : StringSource(nar) + { + } + + void createDirectory(const Path & path) override + { + members.emplace(path, + NarMember{FSAccessor::Type::tDirectory, false, 0, 0}); + } + + void createRegularFile(const Path & path) override + { + currentPath = path; + } + + void isExecutable() override + { + isExec = true; + } + + void preallocateContents(unsigned long long size) override + { + currentStart = string(s, pos, 16); + members.emplace(currentPath, + NarMember{FSAccessor::Type::tRegular, isExec, pos, size}); + } + + void receiveContents(unsigned char * data, unsigned int len) override + { + // Sanity check + if (!currentStart.empty()) { + assert(len < 16 || currentStart == string((char *) data, 16)); + currentStart.clear(); + } + } + + void createSymlink(const Path & path, const string & target) override + { + members.emplace(path, + NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target}); + } + + Members::iterator find(const Path & path) + { + auto i = members.find(path); + if (i == members.end()) + throw Error(format("NAR file does not contain path ‘%1%’") % path); + return i; + } +}; + +struct NarAccessor : public FSAccessor +{ + ref<const std::string> nar; + NarIndexer indexer; + + NarAccessor(ref<const std::string> nar) : nar(nar), indexer(*nar) + { + parseDump(indexer, indexer); + } + + Stat stat(const Path & path) override + { + auto i = indexer.members.find(path); + if (i == indexer.members.end()) + return {FSAccessor::Type::tMissing, 0, false}; + return {i->second.type, i->second.size, i->second.isExecutable}; + } + + StringSet readDirectory(const Path & path) override + { + auto i = indexer.find(path); + + if (i->second.type != FSAccessor::Type::tDirectory) + throw Error(format("path ‘%1%’ inside NAR file is not a directory") % path); + + ++i; + StringSet res; + while (i != indexer.members.end() && isInDir(i->first, path)) { + // FIXME: really bad performance. + if (i->first.find('/', path.size() + 1) == std::string::npos) + res.insert(std::string(i->first, path.size() + 1)); + ++i; + } + return res; + } + + std::string readFile(const Path & path) override + { + auto i = indexer.find(path); + if (i->second.type != FSAccessor::Type::tRegular) + throw Error(format("path ‘%1%’ inside NAR file is not a regular file") % path); + return std::string(*nar, i->second.start, i->second.size); + } + + std::string readLink(const Path & path) override + { + auto i = indexer.find(path); + if (i->second.type != FSAccessor::Type::tSymlink) + throw Error(format("path ‘%1%’ inside NAR file is not a symlink") % path); + return i->second.target; + } +}; + +ref<FSAccessor> makeNarAccessor(ref<const std::string> nar) +{ + return make_ref<NarAccessor>(nar); +} + +} diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh new file mode 100644 index 000000000000..83c570be4c7b --- /dev/null +++ b/src/libstore/nar-accessor.hh @@ -0,0 +1,11 @@ +#pragma once + +#include "fs-accessor.hh" + +namespace nix { + +/* Return an object that provides access to the contents of a NAR + file. */ +ref<FSAccessor> makeNarAccessor(ref<const std::string> nar); + +} diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index e9260a09bf5a..680facdcfeb8 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -1,4 +1,3 @@ -#include "crypto.hh" #include "globals.hh" #include "nar-info.hh" @@ -66,7 +65,7 @@ NarInfo::NarInfo(const std::string & s, const std::string & whence) else if (name == "System") system = value; else if (name == "Sig") - sig = value; + sigs.insert(value); pos = eol + 1; } @@ -98,21 +97,12 @@ std::string NarInfo::to_string() const if (!system.empty()) res += "System: " + system + "\n"; - if (!sig.empty()) + for (auto sig : sigs) res += "Sig: " + sig + "\n"; return res; } -std::string NarInfo::fingerprint() const -{ - return - "1;" + path + ";" - + printHashType(narHash.type) + ":" + printHash32(narHash) + ";" - + std::to_string(narSize) + ";" - + concatStringsSep(",", references); -} - Strings NarInfo::shortRefs() const { Strings refs; @@ -121,14 +111,4 @@ Strings NarInfo::shortRefs() const return refs; } -void NarInfo::sign(const SecretKey & secretKey) -{ - sig = secretKey.signDetached(fingerprint()); -} - -bool NarInfo::checkSignature(const PublicKeys & publicKeys) const -{ - return sig != "" && verifyDetached(fingerprint(), sig, publicKeys); -} - } diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 22e27cb42ebf..3c783cf83fef 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -13,7 +13,6 @@ struct NarInfo : ValidPathInfo Hash fileHash; uint64_t fileSize = 0; std::string system; - std::string sig; // FIXME: support multiple signatures NarInfo() { } NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } @@ -21,20 +20,6 @@ struct NarInfo : ValidPathInfo std::string to_string() const; - /* Return a fingerprint of the store path to be used in binary - cache signatures. It contains the store path, the base-32 - SHA-256 hash of the NAR serialisation of the path, the size of - the NAR, and the sorted references. The size field is strictly - speaking superfluous, but might prevent endless/excessive data - attacks. */ - std::string fingerprint() const; - - void sign(const SecretKey & secretKey); - - /* Return true iff this .narinfo is signed by one of the specified - keys. */ - bool checkSignature(const PublicKeys & publicKeys) const; - private: Strings shortRefs() const; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index ab2ebb9aecc3..761e835481a8 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -6,6 +6,7 @@ #include "affinity.hh" #include "globals.hh" #include "derivations.hh" +#include "pool.hh" #include <sys/types.h> #include <sys/stat.h> @@ -13,9 +14,8 @@ #include <sys/un.h> #include <errno.h> #include <fcntl.h> - -#include <iostream> #include <unistd.h> + #include <cstring> namespace nix { @@ -39,183 +39,158 @@ template<class T> T readStorePaths(Source & from) template PathSet readStorePaths(Source & from); -RemoteStore::RemoteStore() +RemoteStore::RemoteStore(size_t maxConnections) + : connections(make_ref<Pool<Connection>>( + maxConnections, + [this]() { return openConnection(); }, + [](const ref<Connection> & r) { return r->to.good() && r->from.good(); } + )) { - initialised = false; } -void RemoteStore::openConnection(bool reserveSpace) +ref<RemoteStore::Connection> RemoteStore::openConnection() { - if (initialised) return; - initialised = true; + auto conn = make_ref<Connection>(); /* Connect to a daemon that does the privileged work for us. */ - connectToDaemon(); + conn->fd = socket(PF_UNIX, SOCK_STREAM, 0); + if (conn->fd == -1) + throw SysError("cannot create Unix domain socket"); + closeOnExec(conn->fd); + + string socketPath = settings.nixDaemonSocketFile; + + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + if (socketPath.size() + 1 >= sizeof(addr.sun_path)) + throw Error(format("socket path ‘%1%’ is too long") % socketPath); + strcpy(addr.sun_path, socketPath.c_str()); + + if (connect(conn->fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError(format("cannot connect to daemon at ‘%1%’") % socketPath); - from.fd = fdSocket; - to.fd = fdSocket; + conn->from.fd = conn->fd; + conn->to.fd = conn->fd; /* Send the magic greeting, check for the reply. */ try { - to << WORKER_MAGIC_1; - to.flush(); - unsigned int magic = readInt(from); + conn->to << WORKER_MAGIC_1; + conn->to.flush(); + unsigned int magic = readInt(conn->from); if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch"); - daemonVersion = readInt(from); - if (GET_PROTOCOL_MAJOR(daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION)) + conn->daemonVersion = readInt(conn->from); + if (GET_PROTOCOL_MAJOR(conn->daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION)) throw Error("Nix daemon protocol version not supported"); - to << PROTOCOL_VERSION; + conn->to << PROTOCOL_VERSION; - if (GET_PROTOCOL_MINOR(daemonVersion) >= 14) { + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 14) { int cpu = settings.lockCPU ? lockToCurrentCPU() : -1; if (cpu != -1) - to << 1 << cpu; + conn->to << 1 << cpu; else - to << 0; + conn->to << 0; } - if (GET_PROTOCOL_MINOR(daemonVersion) >= 11) - to << reserveSpace; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 11) + conn->to << false; - processStderr(); + conn->processStderr(); } catch (Error & e) { throw Error(format("cannot start daemon worker: %1%") % e.msg()); } - setOptions(); -} - - -void RemoteStore::connectToDaemon() -{ - fdSocket = socket(PF_UNIX, SOCK_STREAM, 0); - if (fdSocket == -1) - throw SysError("cannot create Unix domain socket"); - closeOnExec(fdSocket); - - string socketPath = settings.nixDaemonSocketFile; - - /* Urgh, sockaddr_un allows path names of only 108 characters. So - chdir to the socket directory so that we can pass a relative - path name. !!! this is probably a bad idea in multi-threaded - applications... */ - AutoCloseFD fdPrevDir = open(".", O_RDONLY); - if (fdPrevDir == -1) throw SysError("couldn't open current directory"); - if (chdir(dirOf(socketPath).c_str()) == -1) throw SysError(format("couldn't change to directory of ‘%1%’") % socketPath); - Path socketPathRel = "./" + baseNameOf(socketPath); - - struct sockaddr_un addr; - addr.sun_family = AF_UNIX; - if (socketPathRel.size() >= sizeof(addr.sun_path)) - throw Error(format("socket path ‘%1%’ is too long") % socketPathRel); - using namespace std; - strcpy(addr.sun_path, socketPathRel.c_str()); - - if (connect(fdSocket, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError(format("cannot connect to daemon at ‘%1%’") % socketPath); + setOptions(conn); - if (fchdir(fdPrevDir) == -1) - throw SysError("couldn't change back to previous directory"); + return conn; } -RemoteStore::~RemoteStore() +void RemoteStore::setOptions(ref<Connection> conn) { - try { - to.flush(); - fdSocket.close(); - } catch (...) { - ignoreException(); - } -} - - -void RemoteStore::setOptions() -{ - to << wopSetOptions + conn->to << wopSetOptions << settings.keepFailed << settings.keepGoing << settings.tryFallback << verbosity << settings.maxBuildJobs << settings.maxSilentTime; - if (GET_PROTOCOL_MINOR(daemonVersion) >= 2) - to << settings.useBuildHook; - if (GET_PROTOCOL_MINOR(daemonVersion) >= 4) - to << settings.buildVerbosity + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 2) + conn->to << settings.useBuildHook; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 4) + conn->to << settings.buildVerbosity << logType << settings.printBuildTrace; - if (GET_PROTOCOL_MINOR(daemonVersion) >= 6) - to << settings.buildCores; - if (GET_PROTOCOL_MINOR(daemonVersion) >= 10) - to << settings.useSubstitutes; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 6) + conn->to << settings.buildCores; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 10) + conn->to << settings.useSubstitutes; - if (GET_PROTOCOL_MINOR(daemonVersion) >= 12) { + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 12) { Settings::SettingsMap overrides = settings.getOverrides(); if (overrides["ssh-auth-sock"] == "") overrides["ssh-auth-sock"] = getEnv("SSH_AUTH_SOCK"); - to << overrides.size(); + conn->to << overrides.size(); for (auto & i : overrides) - to << i.first << i.second; + conn->to << i.first << i.second; } - processStderr(); + conn->processStderr(); } bool RemoteStore::isValidPath(const Path & path) { - openConnection(); - to << wopIsValidPath << path; - processStderr(); - unsigned int reply = readInt(from); + auto conn(connections->get()); + conn->to << wopIsValidPath << path; + conn->processStderr(); + unsigned int reply = readInt(conn->from); return reply != 0; } PathSet RemoteStore::queryValidPaths(const PathSet & paths) { - openConnection(); - if (GET_PROTOCOL_MINOR(daemonVersion) < 12) { + auto conn(connections->get()); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { PathSet res; for (auto & i : paths) if (isValidPath(i)) res.insert(i); return res; } else { - to << wopQueryValidPaths << paths; - processStderr(); - return readStorePaths<PathSet>(from); + conn->to << wopQueryValidPaths << paths; + conn->processStderr(); + return readStorePaths<PathSet>(conn->from); } } PathSet RemoteStore::queryAllValidPaths() { - openConnection(); - to << wopQueryAllValidPaths; - processStderr(); - return readStorePaths<PathSet>(from); + auto conn(connections->get()); + conn->to << wopQueryAllValidPaths; + conn->processStderr(); + return readStorePaths<PathSet>(conn->from); } PathSet RemoteStore::querySubstitutablePaths(const PathSet & paths) { - openConnection(); - if (GET_PROTOCOL_MINOR(daemonVersion) < 12) { + auto conn(connections->get()); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { PathSet res; for (auto & i : paths) { - to << wopHasSubstitutes << i; - processStderr(); - if (readInt(from)) res.insert(i); + conn->to << wopHasSubstitutes << i; + conn->processStderr(); + if (readInt(conn->from)) res.insert(i); } return res; } else { - to << wopQuerySubstitutablePaths << paths; - processStderr(); - return readStorePaths<PathSet>(from); + conn->to << wopQuerySubstitutablePaths << paths; + conn->processStderr(); + return readStorePaths<PathSet>(conn->from); } } @@ -225,39 +200,39 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths, { if (paths.empty()) return; - openConnection(); + auto conn(connections->get()); - if (GET_PROTOCOL_MINOR(daemonVersion) < 3) return; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 3) return; - if (GET_PROTOCOL_MINOR(daemonVersion) < 12) { + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { for (auto & i : paths) { SubstitutablePathInfo info; - to << wopQuerySubstitutablePathInfo << i; - processStderr(); - unsigned int reply = readInt(from); + conn->to << wopQuerySubstitutablePathInfo << i; + conn->processStderr(); + unsigned int reply = readInt(conn->from); if (reply == 0) continue; - info.deriver = readString(from); + info.deriver = readString(conn->from); if (info.deriver != "") assertStorePath(info.deriver); - info.references = readStorePaths<PathSet>(from); - info.downloadSize = readLongLong(from); - info.narSize = GET_PROTOCOL_MINOR(daemonVersion) >= 7 ? readLongLong(from) : 0; + info.references = readStorePaths<PathSet>(conn->from); + info.downloadSize = readLongLong(conn->from); + info.narSize = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 7 ? readLongLong(conn->from) : 0; infos[i] = info; } } else { - to << wopQuerySubstitutablePathInfos << paths; - processStderr(); - unsigned int count = readInt(from); + conn->to << wopQuerySubstitutablePathInfos << paths; + conn->processStderr(); + unsigned int count = readInt(conn->from); for (unsigned int n = 0; n < count; n++) { - Path path = readStorePath(from); + Path path = readStorePath(conn->from); SubstitutablePathInfo & info(infos[path]); - info.deriver = readString(from); + info.deriver = readString(conn->from); if (info.deriver != "") assertStorePath(info.deriver); - info.references = readStorePaths<PathSet>(from); - info.downloadSize = readLongLong(from); - info.narSize = readLongLong(from); + info.references = readStorePaths<PathSet>(conn->from); + info.downloadSize = readLongLong(conn->from); + info.narSize = readLongLong(conn->from); } } @@ -266,27 +241,31 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths, ValidPathInfo RemoteStore::queryPathInfo(const Path & path) { - openConnection(); - to << wopQueryPathInfo << path; - processStderr(); + auto conn(connections->get()); + conn->to << wopQueryPathInfo << path; + conn->processStderr(); ValidPathInfo info; info.path = path; - info.deriver = readString(from); + info.deriver = readString(conn->from); if (info.deriver != "") assertStorePath(info.deriver); - info.narHash = parseHash(htSHA256, readString(from)); - info.references = readStorePaths<PathSet>(from); - info.registrationTime = readInt(from); - info.narSize = readLongLong(from); + info.narHash = parseHash(htSHA256, readString(conn->from)); + info.references = readStorePaths<PathSet>(conn->from); + info.registrationTime = readInt(conn->from); + info.narSize = readLongLong(conn->from); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { + info.ultimate = readInt(conn->from) != 0; + info.sigs = readStrings<StringSet>(conn->from); + } return info; } Hash RemoteStore::queryPathHash(const Path & path) { - openConnection(); - to << wopQueryPathHash << path; - processStderr(); - string hash = readString(from); + auto conn(connections->get()); + conn->to << wopQueryPathHash << path; + conn->processStderr(); + string hash = readString(conn->from); return parseHash(htSHA256, hash); } @@ -294,10 +273,10 @@ Hash RemoteStore::queryPathHash(const Path & path) void RemoteStore::queryReferences(const Path & path, PathSet & references) { - openConnection(); - to << wopQueryReferences << path; - processStderr(); - PathSet references2 = readStorePaths<PathSet>(from); + auto conn(connections->get()); + conn->to << wopQueryReferences << path; + conn->processStderr(); + PathSet references2 = readStorePaths<PathSet>(conn->from); references.insert(references2.begin(), references2.end()); } @@ -305,20 +284,20 @@ void RemoteStore::queryReferences(const Path & path, void RemoteStore::queryReferrers(const Path & path, PathSet & referrers) { - openConnection(); - to << wopQueryReferrers << path; - processStderr(); - PathSet referrers2 = readStorePaths<PathSet>(from); + auto conn(connections->get()); + conn->to << wopQueryReferrers << path; + conn->processStderr(); + PathSet referrers2 = readStorePaths<PathSet>(conn->from); referrers.insert(referrers2.begin(), referrers2.end()); } Path RemoteStore::queryDeriver(const Path & path) { - openConnection(); - to << wopQueryDeriver << path; - processStderr(); - Path drvPath = readString(from); + auto conn(connections->get()); + conn->to << wopQueryDeriver << path; + conn->processStderr(); + Path drvPath = readString(conn->from); if (drvPath != "") assertStorePath(drvPath); return drvPath; } @@ -326,37 +305,37 @@ Path RemoteStore::queryDeriver(const Path & path) PathSet RemoteStore::queryValidDerivers(const Path & path) { - openConnection(); - to << wopQueryValidDerivers << path; - processStderr(); - return readStorePaths<PathSet>(from); + auto conn(connections->get()); + conn->to << wopQueryValidDerivers << path; + conn->processStderr(); + return readStorePaths<PathSet>(conn->from); } PathSet RemoteStore::queryDerivationOutputs(const Path & path) { - openConnection(); - to << wopQueryDerivationOutputs << path; - processStderr(); - return readStorePaths<PathSet>(from); + auto conn(connections->get()); + conn->to << wopQueryDerivationOutputs << path; + conn->processStderr(); + return readStorePaths<PathSet>(conn->from); } PathSet RemoteStore::queryDerivationOutputNames(const Path & path) { - openConnection(); - to << wopQueryDerivationOutputNames << path; - processStderr(); - return readStrings<PathSet>(from); + auto conn(connections->get()); + conn->to << wopQueryDerivationOutputNames << path; + conn->processStderr(); + return readStrings<PathSet>(conn->from); } Path RemoteStore::queryPathFromHashPart(const string & hashPart) { - openConnection(); - to << wopQueryPathFromHashPart << hashPart; - processStderr(); - Path path = readString(from); + auto conn(connections->get()); + conn->to << wopQueryPathFromHashPart << hashPart; + conn->processStderr(); + Path path = readString(conn->from); if (!path.empty()) assertStorePath(path); return path; } @@ -367,32 +346,32 @@ Path RemoteStore::addToStore(const string & name, const Path & _srcPath, { if (repair) throw Error("repairing is not supported when building through the Nix daemon"); - openConnection(); + auto conn(connections->get()); Path srcPath(absPath(_srcPath)); - to << wopAddToStore << name + conn->to << wopAddToStore << name << ((hashAlgo == htSHA256 && recursive) ? 0 : 1) /* backwards compatibility hack */ << (recursive ? 1 : 0) << printHashType(hashAlgo); try { - to.written = 0; - to.warn = true; - dumpPath(srcPath, to, filter); - to.warn = false; - processStderr(); + conn->to.written = 0; + conn->to.warn = true; + dumpPath(srcPath, conn->to, filter); + conn->to.warn = false; + conn->processStderr(); } catch (SysError & e) { /* Daemon closed while we were sending the path. Probably OOM or I/O error. */ if (e.errNo == EPIPE) try { - processStderr(); + conn->processStderr(); } catch (EndOfFile & e) { } throw; } - return readStorePath(from); + return readStorePath(conn->from); } @@ -401,43 +380,44 @@ Path RemoteStore::addTextToStore(const string & name, const string & s, { if (repair) throw Error("repairing is not supported when building through the Nix daemon"); - openConnection(); - to << wopAddTextToStore << name << s << references; + auto conn(connections->get()); + conn->to << wopAddTextToStore << name << s << references; - processStderr(); - return readStorePath(from); + conn->processStderr(); + return readStorePath(conn->from); } void RemoteStore::exportPath(const Path & path, bool sign, Sink & sink) { - openConnection(); - to << wopExportPath << path << (sign ? 1 : 0); - processStderr(&sink); /* sink receives the actual data */ - readInt(from); + auto conn(connections->get()); + conn->to << wopExportPath << path << (sign ? 1 : 0); + conn->processStderr(&sink); /* sink receives the actual data */ + readInt(conn->from); } -Paths RemoteStore::importPaths(bool requireSignature, Source & source) +Paths RemoteStore::importPaths(bool requireSignature, Source & source, + std::shared_ptr<FSAccessor> accessor) { - openConnection(); - to << wopImportPaths; + auto conn(connections->get()); + conn->to << wopImportPaths; /* We ignore requireSignature, since the worker forces it to true anyway. */ - processStderr(0, &source); - return readStorePaths<Paths>(from); + conn->processStderr(0, &source); + return readStorePaths<Paths>(conn->from); } void RemoteStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) { - openConnection(); - to << wopBuildPaths; - if (GET_PROTOCOL_MINOR(daemonVersion) >= 13) { - to << drvPaths; - if (GET_PROTOCOL_MINOR(daemonVersion) >= 15) - to << buildMode; + auto conn(connections->get()); + conn->to << wopBuildPaths; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13) { + conn->to << drvPaths; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) + conn->to << buildMode; else /* Old daemons did not take a 'buildMode' parameter, so we need to validate it here on the client side. */ @@ -449,22 +429,22 @@ void RemoteStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) PathSet drvPaths2; for (auto & i : drvPaths) drvPaths2.insert(string(i, 0, i.find('!'))); - to << drvPaths2; + conn->to << drvPaths2; } - processStderr(); - readInt(from); + conn->processStderr(); + readInt(conn->from); } BuildResult RemoteStore::buildDerivation(const Path & drvPath, const BasicDerivation & drv, BuildMode buildMode) { - openConnection(); - to << wopBuildDerivation << drvPath << drv << buildMode; - processStderr(); + auto conn(connections->get()); + conn->to << wopBuildDerivation << drvPath << drv << buildMode; + conn->processStderr(); BuildResult res; unsigned int status; - from >> status >> res.errorMsg; + conn->from >> status >> res.errorMsg; res.status = (BuildResult::Status) status; return res; } @@ -472,50 +452,50 @@ BuildResult RemoteStore::buildDerivation(const Path & drvPath, const BasicDeriva void RemoteStore::ensurePath(const Path & path) { - openConnection(); - to << wopEnsurePath << path; - processStderr(); - readInt(from); + auto conn(connections->get()); + conn->to << wopEnsurePath << path; + conn->processStderr(); + readInt(conn->from); } void RemoteStore::addTempRoot(const Path & path) { - openConnection(); - to << wopAddTempRoot << path; - processStderr(); - readInt(from); + auto conn(connections->get()); + conn->to << wopAddTempRoot << path; + conn->processStderr(); + readInt(conn->from); } void RemoteStore::addIndirectRoot(const Path & path) { - openConnection(); - to << wopAddIndirectRoot << path; - processStderr(); - readInt(from); + auto conn(connections->get()); + conn->to << wopAddIndirectRoot << path; + conn->processStderr(); + readInt(conn->from); } void RemoteStore::syncWithGC() { - openConnection(); - to << wopSyncWithGC; - processStderr(); - readInt(from); + auto conn(connections->get()); + conn->to << wopSyncWithGC; + conn->processStderr(); + readInt(conn->from); } Roots RemoteStore::findRoots() { - openConnection(); - to << wopFindRoots; - processStderr(); - unsigned int count = readInt(from); + auto conn(connections->get()); + conn->to << wopFindRoots; + conn->processStderr(); + unsigned int count = readInt(conn->from); Roots result; while (count--) { - Path link = readString(from); - Path target = readStorePath(from); + Path link = readString(conn->from); + Path target = readStorePath(conn->from); result[link] = target; } return result; @@ -524,56 +504,61 @@ Roots RemoteStore::findRoots() void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) { - openConnection(false); + auto conn(connections->get()); - to << wopCollectGarbage << options.action << options.pathsToDelete << options.ignoreLiveness + conn->to << wopCollectGarbage << options.action << options.pathsToDelete << options.ignoreLiveness << options.maxFreed << 0; - if (GET_PROTOCOL_MINOR(daemonVersion) >= 5) + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 5) /* removed options */ - to << 0 << 0; + conn->to << 0 << 0; - processStderr(); + conn->processStderr(); - results.paths = readStrings<PathSet>(from); - results.bytesFreed = readLongLong(from); - readLongLong(from); // obsolete + results.paths = readStrings<PathSet>(conn->from); + results.bytesFreed = readLongLong(conn->from); + readLongLong(conn->from); // obsolete } -PathSet RemoteStore::queryFailedPaths() +void RemoteStore::optimiseStore() { - openConnection(); - to << wopQueryFailedPaths; - processStderr(); - return readStorePaths<PathSet>(from); + auto conn(connections->get()); + conn->to << wopOptimiseStore; + conn->processStderr(); + readInt(conn->from); } -void RemoteStore::clearFailedPaths(const PathSet & paths) +bool RemoteStore::verifyStore(bool checkContents, bool repair) { - openConnection(); - to << wopClearFailedPaths << paths; - processStderr(); - readInt(from); + auto conn(connections->get()); + conn->to << wopVerifyStore << checkContents << repair; + conn->processStderr(); + return readInt(conn->from) != 0; } -void RemoteStore::optimiseStore() + +void RemoteStore::addSignatures(const Path & storePath, const StringSet & sigs) { - openConnection(); - to << wopOptimiseStore; - processStderr(); - readInt(from); + auto conn(connections->get()); + conn->to << wopAddSignatures << storePath << sigs; + conn->processStderr(); + readInt(conn->from); } -bool RemoteStore::verifyStore(bool checkContents, bool repair) + +RemoteStore::Connection::~Connection() { - openConnection(); - to << wopVerifyStore << checkContents << repair; - processStderr(); - return readInt(from) != 0; + try { + to.flush(); + fd.close(); + } catch (...) { + ignoreException(); + } } -void RemoteStore::processStderr(Sink * sink, Source * source) + +void RemoteStore::Connection::processStderr(Sink * sink, Source * source) { to.flush(); unsigned int msg; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index f15182285e8c..45bc41804ccf 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -1,5 +1,6 @@ #pragma once +#include <limits> #include <string> #include "store-api.hh" @@ -12,15 +13,16 @@ class Pipe; class Pid; struct FdSink; struct FdSource; +template<typename T> class Pool; -class RemoteStore : public Store +/* FIXME: RemoteStore is a misnomer - should be something like + DaemonStore. */ +class RemoteStore : public LocalFSStore { public: - RemoteStore(); - - ~RemoteStore(); + RemoteStore(size_t maxConnections = std::numeric_limits<size_t>::max()); /* Implementations of abstract store API methods. */ @@ -63,7 +65,8 @@ public: void exportPath(const Path & path, bool sign, Sink & sink) override; - Paths importPaths(bool requireSignature, Source & source) override; + Paths importPaths(bool requireSignature, Source & source, + std::shared_ptr<FSAccessor> accessor) override; void buildPaths(const PathSet & paths, BuildMode buildMode) override; @@ -82,28 +85,31 @@ public: void collectGarbage(const GCOptions & options, GCResults & results) override; - PathSet queryFailedPaths() override; - - void clearFailedPaths(const PathSet & paths) override; - void optimiseStore() override; bool verifyStore(bool checkContents, bool repair) override; + void addSignatures(const Path & storePath, const StringSet & sigs) override; + private: - AutoCloseFD fdSocket; - FdSink to; - FdSource from; - unsigned int daemonVersion; - bool initialised; - void openConnection(bool reserveSpace = true); + struct Connection + { + AutoCloseFD fd; + FdSink to; + FdSource from; + unsigned int daemonVersion; + + ~Connection(); + + void processStderr(Sink * sink = 0, Source * source = 0); + }; - void processStderr(Sink * sink = 0, Source * source = 0); + ref<Pool<Connection>> connections; - void connectToDaemon(); + ref<Connection> openConnection(); - void setOptions(); + void setOptions(ref<Connection> conn); }; diff --git a/src/libstore/schema.sql b/src/libstore/schema.sql index c1b4a689afcb..91878af1580d 100644 --- a/src/libstore/schema.sql +++ b/src/libstore/schema.sql @@ -4,7 +4,9 @@ create table if not exists ValidPaths ( hash text not null, registrationTime integer not null, deriver text, - narSize integer + narSize integer, + ultimate integer, -- null implies "false" + sigs text -- space-separated ); create table if not exists Refs ( @@ -37,8 +39,3 @@ create table if not exists DerivationOutputs ( ); create index if not exists IndexDerivationOutputs on DerivationOutputs(path); - -create table if not exists FailedPaths ( - path text primary key not null, - time integer not null -); diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc new file mode 100644 index 000000000000..f93fa0857588 --- /dev/null +++ b/src/libstore/sqlite.cc @@ -0,0 +1,167 @@ +#include "sqlite.hh" +#include "util.hh" + +#include <sqlite3.h> + +namespace nix { + +[[noreturn]] void throwSQLiteError(sqlite3 * db, const format & f) +{ + int err = sqlite3_errcode(db); + if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) { + if (err == SQLITE_PROTOCOL) + printMsg(lvlError, "warning: SQLite database is busy (SQLITE_PROTOCOL)"); + else { + static bool warned = false; + if (!warned) { + printMsg(lvlError, "warning: SQLite database is busy"); + warned = true; + } + } + /* Sleep for a while since retrying the transaction right away + is likely to fail again. */ + checkInterrupt(); +#if HAVE_NANOSLEEP + struct timespec t; + t.tv_sec = 0; + t.tv_nsec = (random() % 100) * 1000 * 1000; /* <= 0.1s */ + nanosleep(&t, 0); +#else + sleep(1); +#endif + throw SQLiteBusy(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); + } + else + throw SQLiteError(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); +} + +SQLite::~SQLite() +{ + try { + if (db && sqlite3_close(db) != SQLITE_OK) + throwSQLiteError(db, "closing database"); + } catch (...) { + ignoreException(); + } +} + +void SQLiteStmt::create(sqlite3 * db, const string & s) +{ + checkInterrupt(); + assert(!stmt); + if (sqlite3_prepare_v2(db, s.c_str(), -1, &stmt, 0) != SQLITE_OK) + throwSQLiteError(db, "creating statement"); + this->db = db; +} + +SQLiteStmt::~SQLiteStmt() +{ + try { + if (stmt && sqlite3_finalize(stmt) != SQLITE_OK) + throwSQLiteError(db, "finalizing statement"); + } catch (...) { + ignoreException(); + } +} + +SQLiteStmt::Use::Use(SQLiteStmt & stmt) + : stmt(stmt) +{ + assert(stmt.stmt); + /* Note: sqlite3_reset() returns the error code for the most + recent call to sqlite3_step(). So ignore it. */ + sqlite3_reset(stmt); +} + +SQLiteStmt::Use::~Use() +{ + sqlite3_reset(stmt); +} + +SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool notNull) +{ + if (notNull) { + if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + throwSQLiteError(stmt.db, "binding argument"); + } else + bind(); + return *this; +} + +SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull) +{ + if (notNull) { + if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK) + throwSQLiteError(stmt.db, "binding argument"); + } else + bind(); + return *this; +} + +SQLiteStmt::Use & SQLiteStmt::Use::bind() +{ + if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK) + throwSQLiteError(stmt.db, "binding argument"); + return *this; +} + +int SQLiteStmt::Use::step() +{ + return sqlite3_step(stmt); +} + +void SQLiteStmt::Use::exec() +{ + int r = step(); + assert(r != SQLITE_ROW); + if (r != SQLITE_DONE) + throwSQLiteError(stmt.db, "executing SQLite statement"); +} + +bool SQLiteStmt::Use::next() +{ + int r = step(); + if (r != SQLITE_DONE && r != SQLITE_ROW) + throwSQLiteError(stmt.db, "executing SQLite query"); + return r == SQLITE_ROW; +} + +std::string SQLiteStmt::Use::getStr(int col) +{ + auto s = (const char *) sqlite3_column_text(stmt, col); + assert(s); + return s; +} + +int64_t SQLiteStmt::Use::getInt(int col) +{ + // FIXME: detect nulls? + return sqlite3_column_int64(stmt, col); +} + +SQLiteTxn::SQLiteTxn(sqlite3 * db) +{ + this->db = db; + if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "starting transaction"); + active = true; +} + +void SQLiteTxn::commit() +{ + if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "committing transaction"); + active = false; +} + +SQLiteTxn::~SQLiteTxn() +{ + try { + if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "aborting transaction"); + } catch (...) { + ignoreException(); + } +} + +} diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh new file mode 100644 index 000000000000..326e4a4855b7 --- /dev/null +++ b/src/libstore/sqlite.hh @@ -0,0 +1,102 @@ +#pragma once + +#include <functional> +#include <string> + +#include "types.hh" + +class sqlite3; +class sqlite3_stmt; + +namespace nix { + +/* RAII wrapper to close a SQLite database automatically. */ +struct SQLite +{ + sqlite3 * db; + SQLite() { db = 0; } + ~SQLite(); + operator sqlite3 * () { return db; } +}; + +/* RAII wrapper to create and destroy SQLite prepared statements. */ +struct SQLiteStmt +{ + sqlite3 * db = 0; + sqlite3_stmt * stmt = 0; + SQLiteStmt() { } + void create(sqlite3 * db, const std::string & s); + ~SQLiteStmt(); + operator sqlite3_stmt * () { return stmt; } + + /* Helper for binding / executing statements. */ + class Use + { + friend struct SQLiteStmt; + private: + SQLiteStmt & stmt; + unsigned int curArg = 1; + Use(SQLiteStmt & stmt); + + public: + + ~Use(); + + /* Bind the next parameter. */ + Use & operator () (const std::string & value, bool notNull = true); + Use & operator () (int64_t value, bool notNull = true); + Use & bind(); // null + + int step(); + + /* Execute a statement that does not return rows. */ + void exec(); + + /* For statements that return 0 or more rows. Returns true iff + a row is available. */ + bool next(); + + std::string getStr(int col); + int64_t getInt(int col); + }; + + Use use() + { + return Use(*this); + } +}; + +/* RAII helper that ensures transactions are aborted unless explicitly + committed. */ +struct SQLiteTxn +{ + bool active = false; + sqlite3 * db; + + SQLiteTxn(sqlite3 * db); + + void commit(); + + ~SQLiteTxn(); +}; + + +MakeError(SQLiteError, Error); +MakeError(SQLiteBusy, SQLiteError); + +[[noreturn]] void throwSQLiteError(sqlite3 * db, const format & f); + +/* Convenience function for retrying a SQLite transaction when the + database is busy. */ +template<typename T> +T retrySQLite(std::function<T()> fun) +{ + while (true) { + try { + return fun(); + } catch (SQLiteBusy & e) { + } + } +} + +} diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 82872cc335e3..cc91ed287768 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1,5 +1,6 @@ -#include "store-api.hh" +#include "crypto.hh" #include "globals.hh" +#include "store-api.hh" #include "util.hh" @@ -135,14 +136,14 @@ void checkStoreName(const string & name) if <type> = "source": the serialisation of the path from which this store path is copied, as returned by hashPath() - if <type> = "output:out": + if <type> = "output:<id>": for non-fixed derivation outputs: the derivation (see hashDerivationModulo() in primops.cc) for paths copied by addToStore() or produced by fixed-output derivations: the string "fixed:out:<rec><algo>:<hash>:", where - <rec> = "r:" for recursive (path) hashes, or "" or flat + <rec> = "r:" for recursive (path) hashes, or "" for flat (file) hashes <algo> = "md5", "sha1" or "sha256" <hash> = base-16 representation of the path or flat hash of @@ -309,22 +310,78 @@ void Store::exportPaths(const Paths & paths, } +std::string ValidPathInfo::fingerprint() const +{ + if (narSize == 0 || narHash.type == htUnknown) + throw Error(format("cannot calculate fingerprint of path ‘%s’ because its size/hash is not known") + % path); + return + "1;" + path + ";" + + printHashType(narHash.type) + ":" + printHash32(narHash) + ";" + + std::to_string(narSize) + ";" + + concatStringsSep(",", references); +} + + +void ValidPathInfo::sign(const SecretKey & secretKey) +{ + sigs.insert(secretKey.signDetached(fingerprint())); +} + + +unsigned int ValidPathInfo::checkSignatures(const PublicKeys & publicKeys) const +{ + unsigned int good = 0; + for (auto & sig : sigs) + if (checkSignature(publicKeys, sig)) + good++; + return good; +} + + +bool ValidPathInfo::checkSignature(const PublicKeys & publicKeys, const std::string & sig) const +{ + return verifyDetached(fingerprint(), sig, publicKeys); +} + + } #include "local-store.hh" -#include "serialise.hh" #include "remote-store.hh" namespace nix { -ref<Store> openStore(bool reserveSpace) +RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0; + + +ref<Store> openStoreAt(const std::string & uri) { + for (auto fun : *RegisterStoreImplementation::implementations) { + auto store = fun(uri); + if (store) return ref<Store>(store); + } + + throw Error(format("don't know how to open Nix store ‘%s’") % uri); +} + + +ref<Store> openStore() +{ + return openStoreAt(getEnv("NIX_REMOTE")); +} + + +static RegisterStoreImplementation regStore([](const std::string & uri) -> std::shared_ptr<Store> { enum { mDaemon, mLocal, mAuto } mode; - mode = getEnv("NIX_REMOTE") == "daemon" ? mDaemon : mAuto; + if (uri == "daemon") mode = mDaemon; + else if (uri == "local") mode = mLocal; + else if (uri == "") mode = mAuto; + else return 0; if (mode == mAuto) { if (LocalStore::haveWriteAccess()) @@ -336,9 +393,9 @@ ref<Store> openStore(bool reserveSpace) } return mode == mDaemon - ? (ref<Store>) make_ref<RemoteStore>() - : (ref<Store>) make_ref<LocalStore>(reserveSpace); -} + ? std::shared_ptr<Store>(std::make_shared<RemoteStore>()) + : std::shared_ptr<Store>(std::make_shared<LocalStore>()); +}); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 6f50e3c55aba..ae5631ba0b7c 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -2,6 +2,7 @@ #include "hash.hh" #include "serialise.hh" +#include "crypto.hh" #include <string> #include <limits> @@ -95,8 +96,15 @@ struct ValidPathInfo Hash narHash; PathSet references; time_t registrationTime = 0; - unsigned long long narSize = 0; // 0 = unknown - unsigned long long id; // internal use only + uint64_t narSize = 0; // 0 = unknown + uint64_t id; // internal use only + + /* Whether the path is ultimately trusted, that is, it was built + locally or is content-addressable (e.g. added via addToStore() + or the result of a fixed-output derivation). */ + bool ultimate = false; + + StringSet sigs; // note: not necessarily verified bool operator == (const ValidPathInfo & i) const { @@ -105,6 +113,23 @@ struct ValidPathInfo && narHash == i.narHash && references == i.references; } + + /* Return a fingerprint of the store path to be used in binary + cache signatures. It contains the store path, the base-32 + SHA-256 hash of the NAR serialisation of the path, the size of + the NAR, and the sorted references. The size field is strictly + speaking superfluous, but might prevent endless/excessive data + attacks. */ + std::string fingerprint() const; + + void sign(const SecretKey & secretKey); + + /* Return the number of signatures on this .narinfo that were + produced by one of the specified keys. */ + unsigned int checkSignatures(const PublicKeys & publicKeys) const; + + /* Verify a single signature. */ + bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const; }; typedef list<ValidPathInfo> ValidPathInfos; @@ -123,7 +148,6 @@ struct BuildResult InputRejected, OutputRejected, TransientFailure, // possibly transient - CachedFailure, TimedOut, MiscFailure, DependencyFailed, @@ -140,9 +164,10 @@ struct BuildResult struct BasicDerivation; struct Derivation; +class FSAccessor; -class Store +class Store : public std::enable_shared_from_this<Store> { public: @@ -214,6 +239,9 @@ public: virtual Path addTextToStore(const string & name, const string & s, const PathSet & references, bool repair = false) = 0; + /* Write a NAR dump of a store path. */ + virtual void narFromPath(const Path & path, Sink & sink) = 0; + /* Export a store path, that is, create a NAR dump of the store path and append its references and its deriver. Optionally, a cryptographic signature (created by OpenSSL) of the preceding @@ -226,8 +254,11 @@ public: void exportPaths(const Paths & paths, bool sign, Sink & sink); /* Import a sequence of NAR dumps created by exportPaths() into - the Nix store. */ - virtual Paths importPaths(bool requireSignature, Source & source) = 0; + the Nix store. Optionally, the contents of the NARs are + preloaded into the specified FS accessor to speed up subsequent + access. */ + virtual Paths importPaths(bool requireSignature, Source & source, + std::shared_ptr<FSAccessor> accessor) = 0; /* For each path, if it's a derivation, build it. Building a derivation means ensuring that the output paths are valid. If @@ -293,13 +324,6 @@ public: /* Perform a garbage collection. */ virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0; - /* Return the set of paths that have failed to build.*/ - virtual PathSet queryFailedPaths() = 0; - - /* Clear the "failed" status of the given paths. The special - value `*' causes all failed paths to be cleared. */ - virtual void clearFailedPaths(const PathSet & paths) = 0; - /* Return a string representing information about the path that can be loaded into the database using `nix-store --load-db' or `nix-store --register-validity'. */ @@ -314,6 +338,13 @@ public: remain. */ virtual bool verifyStore(bool checkContents, bool repair) = 0; + /* Return an object to access files in the Nix store. */ + virtual ref<FSAccessor> getFSAccessor() = 0; + + /* Add signatures to the specified store path. The signatures are + not verified. */ + virtual void addSignatures(const Path & storePath, const StringSet & sigs) = 0; + /* Utility functions. */ /* Read a derivation, after ensuring its existence through @@ -345,6 +376,14 @@ public: }; +class LocalFSStore : public Store +{ +public: + void narFromPath(const Path & path, Sink & sink) override; + ref<FSAccessor> getFSAccessor() override; +}; + + /* !!! These should be part of the store API, I guess. */ /* Throw an exception if `path' is not directly in the Nix store. */ @@ -419,9 +458,46 @@ Path computeStorePathForText(const string & name, const string & s, void removeTempRoots(); -/* Factory method: open the Nix database, either through the local or - remote implementation. */ -ref<Store> openStore(bool reserveSpace = true); +/* Return a Store object to access the Nix store denoted by + ‘uri’ (slight misnomer...). Supported values are: + + * ‘direct’: The Nix store in /nix/store and database in + /nix/var/nix/db, accessed directly. + + * ‘daemon’: The Nix store accessed via a Unix domain socket + connection to nix-daemon. + + * ‘file://<path>’: A binary cache stored in <path>. + + If ‘uri’ is empty, it defaults to ‘direct’ or ‘daemon’ depending on + whether the user has write access to the local Nix store/database. + set to true *unless* you're going to collect garbage. */ +ref<Store> openStoreAt(const std::string & uri); + + +/* Open the store indicated by the ‘NIX_REMOTE’ environment variable. */ +ref<Store> openStore(); + + +ref<Store> openLocalBinaryCacheStore(std::shared_ptr<Store> localStore, + const Path & secretKeyFile, const Path & binaryCacheDir); + + +/* Store implementation registration. */ +typedef std::function<std::shared_ptr<Store>(const std::string & uri)> OpenStore; + +struct RegisterStoreImplementation +{ + typedef std::vector<OpenStore> Implementations; + static Implementations * implementations; + + RegisterStoreImplementation(OpenStore fun) + { + if (!implementations) implementations = new Implementations; + implementations->push_back(fun); + } +}; + /* Display a set of paths in human-readable form (i.e., between quotes diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 7d9bcb58a249..c10598d5d301 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -6,7 +6,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x10f +#define PROTOCOL_VERSION 0x110 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -45,6 +45,7 @@ typedef enum { wopOptimiseStore = 34, wopVerifyStore = 35, wopBuildDerivation = 36, + wopAddSignatures = 37, } WorkerOp; diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 6ee7981432b6..5363496c272e 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -29,7 +29,7 @@ bool useCaseHack = false; #endif -static string archiveVersion1 = "nix-archive-1"; +const std::string narVersionMagic1 = "nix-archive-1"; static string caseHackSuffix = "~nix~case~hack~"; @@ -113,11 +113,17 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter) void dumpPath(const Path & path, Sink & sink, PathFilter & filter) { - sink << archiveVersion1; + sink << narVersionMagic1; dump(path, sink, filter); } +void dumpString(const std::string & s, Sink & sink) +{ + sink << narVersionMagic1 << "(" << "type" << "regular" << "contents" << s << ")"; +} + + static SerialisationError badArchive(string s) { return SerialisationError("bad archive: " + s); @@ -214,7 +220,8 @@ static void parse(ParseSink & sink, Source & source, const Path & path) } else if (s == "executable" && type == tpRegular) { - readString(source); + auto s = readString(source); + if (s != "") throw badArchive("executable marker has non-empty value"); sink.isExecutable(); } @@ -275,7 +282,7 @@ void parseDump(ParseSink & sink, Source & source) /* This generally means the integer at the start couldn't be decoded. Ignore and throw the exception below. */ } - if (version != archiveVersion1) + if (version != narVersionMagic1) throw badArchive("input doesn't look like a Nix archive"); parse(sink, source, ""); } diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index c216e9768fd1..d58b91df0461 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -55,6 +55,9 @@ extern PathFilter defaultPathFilter; void dumpPath(const Path & path, Sink & sink, PathFilter & filter = defaultPathFilter); +void dumpString(const std::string & s, Sink & sink); + +/* FIXME: fix this API, it sucks. */ struct ParseSink { virtual void createDirectory(const Path & path) { }; @@ -76,4 +79,7 @@ void restorePath(const Path & path, Source & source); extern bool useCaseHack; +extern const std::string narVersionMagic1; + + } diff --git a/src/libutil/args.cc b/src/libutil/args.cc new file mode 100644 index 000000000000..6e4b82a279ce --- /dev/null +++ b/src/libutil/args.cc @@ -0,0 +1,179 @@ +#include "args.hh" +#include "hash.hh" + +namespace nix { + +void Args::parseCmdline(const Strings & _cmdline) +{ + Strings pendingArgs; + bool dashDash = false; + + Strings cmdline(_cmdline); + + for (auto pos = cmdline.begin(); pos != cmdline.end(); ) { + + auto arg = *pos; + + /* Expand compound dash options (i.e., `-qlf' -> `-q -l -f', + `-j3` -> `-j 3`). */ + if (!dashDash && arg.length() > 2 && arg[0] == '-' && arg[1] != '-' && isalpha(arg[1])) { + *pos = (string) "-" + arg[1]; + auto next = pos; ++next; + for (unsigned int j = 2; j < arg.length(); j++) + if (isalpha(arg[j])) + cmdline.insert(next, (string) "-" + arg[j]); + else { + cmdline.insert(next, string(arg, j)); + break; + } + arg = *pos; + } + + if (!dashDash && arg == "--") { + dashDash = true; + ++pos; + } + else if (!dashDash && std::string(arg, 0, 1) == "-") { + if (!processFlag(pos, cmdline.end())) + throw UsageError(format("unrecognised flag ‘%1%’") % arg); + } + else { + pendingArgs.push_back(*pos++); + if (processArgs(pendingArgs, false)) + pendingArgs.clear(); + } + } + + processArgs(pendingArgs, true); +} + +void Args::printHelp(const string & programName, std::ostream & out) +{ + std::cout << "Usage: " << programName << " <FLAGS>..."; + for (auto & exp : expectedArgs) { + std::cout << renderLabels({exp.label}); + // FIXME: handle arity > 1 + if (exp.arity == 0) std::cout << "..."; + } + std::cout << "\n"; + + auto s = description(); + if (s != "") + std::cout << "\nSummary: " << s << ".\n"; + + if (longFlags.size()) { + std::cout << "\n"; + std::cout << "Flags:\n"; + printFlags(out); + } +} + +void Args::printFlags(std::ostream & out) +{ + Table2 table; + for (auto & flags : longFlags) + table.push_back(std::make_pair( + "--" + flags.first + renderLabels(flags.second.labels), + flags.second.description)); + printTable(out, table); +} + +bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) +{ + assert(pos != end); + + auto process = [&](const std::string & name, const Flag & flag) -> bool { + ++pos; + Strings args; + for (size_t n = 0 ; n < flag.arity; ++n) { + if (pos == end) + throw UsageError(format("flag ‘%1%’ requires %2% argument(s)") + % name % flag.arity); + args.push_back(*pos++); + } + flag.handler(args); + return true; + }; + + if (string(*pos, 0, 2) == "--") { + auto i = longFlags.find(string(*pos, 2)); + if (i == longFlags.end()) return false; + return process("--" + i->first, i->second); + } + + if (string(*pos, 0, 1) == "-" && pos->size() == 2) { + auto c = (*pos)[1]; + auto i = shortFlags.find(c); + if (i == shortFlags.end()) return false; + return process(std::string("-") + c, i->second); + } + + return false; +} + +bool Args::processArgs(const Strings & args, bool finish) +{ + if (expectedArgs.empty()) { + if (!args.empty()) + throw UsageError(format("unexpected argument ‘%1%’") % args.front()); + return true; + } + + auto & exp = expectedArgs.front(); + + bool res = false; + + if ((exp.arity == 0 && finish) || + (exp.arity > 0 && args.size() == exp.arity)) + { + exp.handler(args); + expectedArgs.pop_front(); + res = true; + } + + if (finish && !expectedArgs.empty()) + throw UsageError("more arguments are required"); + + return res; +} + +void Args::mkHashTypeFlag(const std::string & name, HashType * ht) +{ + mkFlag1(0, name, "TYPE", "hash algorithm (‘md5’, ‘sha1’, ‘sha256’, or ‘sha512’)", [=](std::string s) { + *ht = parseHashType(s); + if (*ht == htUnknown) + throw UsageError(format("unknown hash type ‘%1%’") % s); + }); +} + +Strings argvToStrings(int argc, char * * argv) +{ + Strings args; + argc--; argv++; + while (argc--) args.push_back(*argv++); + return args; +} + +std::string renderLabels(const Strings & labels) +{ + std::string res; + for (auto label : labels) { + for (auto & c : label) c = std::toupper(c); + res += " <" + label + ">"; + } + return res; +} + +void printTable(std::ostream & out, const Table2 & table) +{ + size_t max = 0; + for (auto & row : table) + max = std::max(max, row.first.size()); + for (auto & row : table) { + out << " " << row.first + << std::string(max - row.first.size() + 2, ' ') + << row.second << "\n"; + } +} + +} diff --git a/src/libutil/args.hh b/src/libutil/args.hh new file mode 100644 index 000000000000..4469a046d28a --- /dev/null +++ b/src/libutil/args.hh @@ -0,0 +1,162 @@ +#pragma once + +#include <iostream> +#include <map> +#include <memory> + +#include "util.hh" + +namespace nix { + +MakeError(UsageError, nix::Error); + +enum HashType : char; + +class Args +{ +public: + + /* Parse the command line, throwing a UsageError if something goes + wrong. */ + void parseCmdline(const Strings & cmdline); + + virtual void printHelp(const string & programName, std::ostream & out); + + virtual std::string description() { return ""; } + +protected: + + /* Flags. */ + struct Flag + { + std::string description; + Strings labels; + size_t arity; + std::function<void(Strings)> handler; + }; + + std::map<std::string, Flag> longFlags; + std::map<char, Flag> shortFlags; + + virtual bool processFlag(Strings::iterator & pos, Strings::iterator end); + + void printFlags(std::ostream & out); + + /* Positional arguments. */ + struct ExpectedArg + { + std::string label; + size_t arity; // 0 = any + std::function<void(Strings)> handler; + }; + + std::list<ExpectedArg> expectedArgs; + + virtual bool processArgs(const Strings & args, bool finish); + +public: + + /* Helper functions for constructing flags / positional + arguments. */ + + void mkFlag(char shortName, const std::string & longName, + const Strings & labels, const std::string & description, + size_t arity, std::function<void(Strings)> handler) + { + auto flag = Flag{description, labels, arity, handler}; + if (shortName) shortFlags[shortName] = flag; + longFlags[longName] = flag; + } + + void mkFlag(char shortName, const std::string & longName, + const std::string & description, std::function<void()> fun) + { + mkFlag(shortName, longName, {}, description, 0, std::bind(fun)); + } + + void mkFlag1(char shortName, const std::string & longName, + const std::string & label, const std::string & description, + std::function<void(std::string)> fun) + { + mkFlag(shortName, longName, {label}, description, 1, [=](Strings ss) { + fun(ss.front()); + }); + } + + void mkFlag(char shortName, const std::string & name, + const std::string & description, bool * dest) + { + mkFlag(shortName, name, description, dest, true); + } + + void mkFlag(char shortName, const std::string & longName, + const std::string & label, const std::string & description, + string * dest) + { + mkFlag1(shortName, longName, label, description, [=](std::string s) { + *dest = s; + }); + } + + void mkHashTypeFlag(const std::string & name, HashType * ht); + + template<class T> + void mkFlag(char shortName, const std::string & longName, const std::string & description, + T * dest, const T & value) + { + mkFlag(shortName, longName, {}, description, 0, [=](Strings ss) { + *dest = value; + }); + } + + template<class I> + void mkIntFlag(char shortName, const std::string & longName, + const std::string & description, I * dest) + { + mkFlag<I>(shortName, longName, description, [=](I n) { + *dest = n; + }); + } + + template<class I> + void mkFlag(char shortName, const std::string & longName, + const std::string & description, std::function<void(I)> fun) + { + mkFlag(shortName, longName, {"N"}, description, 1, [=](Strings ss) { + I n; + if (!string2Int(ss.front(), n)) + throw UsageError(format("flag ‘--%1%’ requires a integer argument") % longName); + fun(n); + }); + } + + /* Expect a string argument. */ + void expectArg(const std::string & label, string * dest) + { + expectedArgs.push_back(ExpectedArg{label, 1, [=](Strings ss) { + *dest = ss.front(); + }}); + } + + /* Expect 0 or more arguments. */ + void expectArgs(const std::string & label, Strings * dest) + { + expectedArgs.push_back(ExpectedArg{label, 0, [=](Strings ss) { + *dest = ss; + }}); + } + + friend class MultiCommand; +}; + +Strings argvToStrings(int argc, char * * argv); + +/* Helper function for rendering argument labels. */ +std::string renderLabels(const Strings & labels); + +/* Helper function for printing 2-column tables. */ +typedef std::vector<std::pair<std::string, std::string>> Table2; + +void printTable(std::ostream & out, const Table2 & table); + +} diff --git a/src/libutil/local.mk b/src/libutil/local.mk index 4dae3305433f..2e5d2672e5f0 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -6,6 +6,6 @@ libutil_DIR := $(d) libutil_SOURCES := $(wildcard $(d)/*.cc) -libutil_LDFLAGS = -llzma $(OPENSSL_LIBS) +libutil_LDFLAGS = -llzma -pthread $(OPENSSL_LIBS) libutil_LIBS = libformat diff --git a/src/libutil/lru-cache.hh b/src/libutil/lru-cache.hh new file mode 100644 index 000000000000..4344d6601bc8 --- /dev/null +++ b/src/libutil/lru-cache.hh @@ -0,0 +1,84 @@ +#pragma once + +#include <map> +#include <list> + +namespace nix { + +/* A simple least-recently used cache. Not thread-safe. */ +template<typename Key, typename Value> +class LRUCache +{ +private: + + size_t maxSize; + + // Stupid wrapper to get around circular dependency between Data + // and LRU. + struct LRUIterator; + + using Data = std::map<Key, std::pair<LRUIterator, Value>>; + using LRU = std::list<typename Data::iterator>; + + struct LRUIterator { typename LRU::iterator it; }; + + Data data; + LRU lru; + +public: + + LRUCache(size_t maxSize) : maxSize(maxSize) { } + + /* Insert or upsert an item in the cache. */ + void upsert(const Key & key, const Value & value) + { + erase(key); + + if (data.size() >= maxSize) { + /* Retire the oldest item. */ + auto oldest = lru.begin(); + data.erase(*oldest); + lru.erase(oldest); + } + + auto res = data.emplace(key, std::make_pair(LRUIterator(), value)); + assert(res.second); + auto & i(res.first); + + auto j = lru.insert(lru.end(), i); + + i->second.first.it = j; + } + + bool erase(const Key & key) + { + auto i = data.find(key); + if (i == data.end()) return false; + lru.erase(i->second.first.it); + data.erase(i); + return true; + } + + /* Look up an item in the cache. If it exists, it becomes the most + recently used item. */ + // FIXME: use boost::optional? + Value * get(const Key & key) + { + auto i = data.find(key); + if (i == data.end()) return 0; + + /* Move this item to the back of the LRU list. */ + lru.erase(i->second.first.it); + auto j = lru.insert(lru.end(), i); + i->second.first.it = j; + + return &i->second.second; + } + + size_t size() + { + return data.size(); + } +}; + +} diff --git a/src/libutil/pool.hh b/src/libutil/pool.hh new file mode 100644 index 000000000000..f291cd578388 --- /dev/null +++ b/src/libutil/pool.hh @@ -0,0 +1,151 @@ +#pragma once + +#include <functional> +#include <limits> +#include <list> +#include <memory> +#include <cassert> + +#include "sync.hh" +#include "ref.hh" + +namespace nix { + +/* This template class implements a simple pool manager of resources + of some type R, such as database connections. It is used as + follows: + + class Connection { ... }; + + Pool<Connection> pool; + + { + auto conn(pool.get()); + conn->exec("select ..."); + } + + Here, the Connection object referenced by ‘conn’ is automatically + returned to the pool when ‘conn’ goes out of scope. +*/ + +template <class R> +class Pool +{ +public: + + /* A function that produces new instances of R on demand. */ + typedef std::function<ref<R>()> Factory; + + /* A function that checks whether an instance of R is still + usable. Unusable instances are removed from the pool. */ + typedef std::function<bool(const ref<R> &)> Validator; + +private: + + Factory factory; + Validator validator; + + struct State + { + size_t inUse = 0; + size_t max; + std::vector<ref<R>> idle; + }; + + Sync<State> state; + + std::condition_variable wakeup; + +public: + + Pool(size_t max = std::numeric_limits<size_t>::max(), + const Factory & factory = []() { return make_ref<R>(); }, + const Validator & validator = [](ref<R> r) { return true; }) + : factory(factory) + , validator(validator) + { + auto state_(state.lock()); + state_->max = max; + } + + ~Pool() + { + auto state_(state.lock()); + assert(!state_->inUse); + state_->max = 0; + state_->idle.clear(); + } + + class Handle + { + private: + Pool & pool; + std::shared_ptr<R> r; + + friend Pool; + + Handle(Pool & pool, std::shared_ptr<R> r) : pool(pool), r(r) { } + + public: + Handle(Handle && h) : pool(h.pool), r(h.r) { h.r.reset(); } + + Handle(const Handle & l) = delete; + + ~Handle() + { + if (!r) return; + { + auto state_(pool.state.lock()); + state_->idle.push_back(ref<R>(r)); + assert(state_->inUse); + state_->inUse--; + } + pool.wakeup.notify_one(); + } + + R * operator -> () { return &*r; } + R & operator * () { return *r; } + }; + + Handle get() + { + { + auto state_(state.lock()); + + /* If we're over the maximum number of instance, we need + to wait until a slot becomes available. */ + while (state_->idle.empty() && state_->inUse >= state_->max) + state_.wait(wakeup); + + while (!state_->idle.empty()) { + auto p = state_->idle.back(); + state_->idle.pop_back(); + if (validator(p)) { + state_->inUse++; + return Handle(*this, p); + } + } + + state_->inUse++; + } + + /* We need to create a new instance. Because that might take a + while, we don't hold the lock in the meantime. */ + try { + Handle h(*this, factory()); + return h; + } catch (...) { + auto state_(state.lock()); + state_->inUse--; + throw; + } + } + + unsigned int count() + { + auto state_(state.lock()); + return state_->idle.size() + state_->inUse; + } +}; + +} diff --git a/src/libutil/ref.hh b/src/libutil/ref.hh new file mode 100644 index 000000000000..349f24f7c488 --- /dev/null +++ b/src/libutil/ref.hh @@ -0,0 +1,68 @@ +#pragma once + +#include <memory> +#include <exception> +#include <stdexcept> + +namespace nix { + +/* A simple non-nullable reference-counted pointer. Actually a wrapper + around std::shared_ptr that prevents non-null constructions. */ +template<typename T> +class ref +{ +private: + + std::shared_ptr<T> p; + +public: + + ref<T>(const ref<T> & r) + : p(r.p) + { } + + explicit ref<T>(const std::shared_ptr<T> & p) + : p(p) + { + if (!p) + throw std::invalid_argument("null pointer cast to ref"); + } + + T* operator ->() const + { + return &*p; + } + + T& operator *() const + { + return *p; + } + + operator std::shared_ptr<T> () + { + return p; + } + + template<typename T2> + operator ref<T2> () + { + return ref<T2>((std::shared_ptr<T2>) p); + } + +private: + + template<typename T2, typename... Args> + friend ref<T2> + make_ref(Args&&... args); + +}; + +template<typename T, typename... Args> +inline ref<T> +make_ref(Args&&... args) +{ + auto p = std::make_shared<T>(std::forward<Args>(args)...); + return ref<T>(p); +} + +} diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index f136a13248ba..5c45c890f7b6 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -64,15 +64,25 @@ static void warnLargeDump() void FdSink::write(const unsigned char * data, size_t len) { + written += len; static bool warned = false; if (warn && !warned) { - written += len; if (written > threshold) { warnLargeDump(); warned = true; } } - writeFull(fd, data, len); + try { + writeFull(fd, data, len); + } catch (SysError & e) { + _good = true; + } +} + + +bool FdSink::good() +{ + return _good; } @@ -119,12 +129,19 @@ size_t FdSource::readUnbuffered(unsigned char * data, size_t len) checkInterrupt(); n = ::read(fd, (char *) data, bufSize); } while (n == -1 && errno == EINTR); - if (n == -1) throw SysError("reading from file"); - if (n == 0) throw EndOfFile("unexpected end-of-file"); + if (n == -1) { _good = false; throw SysError("reading from file"); } + if (n == 0) { _good = false; throw EndOfFile("unexpected end-of-file"); } + read += n; return n; } +bool FdSource::good() +{ + return _good; +} + + size_t StringSource::read(unsigned char * data, size_t len) { if (pos == s.size()) throw EndOfFile("end of string reached"); @@ -271,11 +288,11 @@ template PathSet readStrings(Source & source); void StringSink::operator () (const unsigned char * data, size_t len) { static bool warned = false; - if (!warned && s.size() > threshold) { + if (!warned && s->size() > threshold) { warnLargeDump(); warned = true; } - s.append((const char *) data, len); + s->append((const char *) data, len); } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 979ff849fcaf..9ba6391f817a 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -12,6 +12,7 @@ struct Sink { virtual ~Sink() { } virtual void operator () (const unsigned char * data, size_t len) = 0; + virtual bool good() { return true; } }; @@ -25,7 +26,7 @@ struct BufferedSink : Sink : bufSize(bufSize), bufPos(0), buffer(0) { } ~BufferedSink(); - void operator () (const unsigned char * data, size_t len); + void operator () (const unsigned char * data, size_t len) override; void flush(); @@ -47,6 +48,8 @@ struct Source return the number of bytes stored. If blocks until at least one byte is available. */ virtual size_t read(unsigned char * data, size_t len) = 0; + + virtual bool good() { return true; } }; @@ -60,7 +63,7 @@ struct BufferedSource : Source : bufSize(bufSize), bufPosIn(0), bufPosOut(0), buffer(0) { } ~BufferedSource(); - size_t read(unsigned char * data, size_t len); + size_t read(unsigned char * data, size_t len) override; /* Underlying read call, to be overridden. */ virtual size_t readUnbuffered(unsigned char * data, size_t len) = 0; @@ -73,14 +76,19 @@ struct BufferedSource : Source struct FdSink : BufferedSink { int fd; - bool warn; - size_t written; + bool warn = false; + size_t written = 0; - FdSink() : fd(-1), warn(false), written(0) { } - FdSink(int fd) : fd(fd), warn(false), written(0) { } + FdSink() : fd(-1) { } + FdSink(int fd) : fd(fd) { } ~FdSink(); - void write(const unsigned char * data, size_t len); + void write(const unsigned char * data, size_t len) override; + + bool good() override; + +private: + bool _good = true; }; @@ -88,17 +96,24 @@ struct FdSink : BufferedSink struct FdSource : BufferedSource { int fd; + size_t read = 0; + FdSource() : fd(-1) { } FdSource(int fd) : fd(fd) { } - size_t readUnbuffered(unsigned char * data, size_t len); + size_t readUnbuffered(unsigned char * data, size_t len) override; + bool good() override; +private: + bool _good = true; }; /* A sink that writes data to a string. */ struct StringSink : Sink { - string s; - void operator () (const unsigned char * data, size_t len); + ref<std::string> s; + StringSink() : s(make_ref<std::string>()) { }; + StringSink(ref<std::string> s) : s(s) { }; + void operator () (const unsigned char * data, size_t len) override; }; @@ -108,7 +123,7 @@ struct StringSource : Source const string & s; size_t pos; StringSource(const string & _s) : s(_s), pos(0) { } - size_t read(unsigned char * data, size_t len); + size_t read(unsigned char * data, size_t len) override; }; diff --git a/src/libutil/sync.hh b/src/libutil/sync.hh new file mode 100644 index 000000000000..ebe64ffbdab7 --- /dev/null +++ b/src/libutil/sync.hh @@ -0,0 +1,78 @@ +#pragma once + +#include <mutex> +#include <condition_variable> +#include <cassert> + +namespace nix { + +/* This template class ensures synchronized access to a value of type + T. It is used as follows: + + struct Data { int x; ... }; + + Sync<Data> data; + + { + auto data_(data.lock()); + data_->x = 123; + } + + Here, "data" is automatically unlocked when "data_" goes out of + scope. +*/ + +template<class T, class M = std::mutex> +class Sync +{ +private: + M mutex; + T data; + +public: + + Sync() { } + Sync(const T & data) : data(data) { } + + class Lock + { + private: + Sync * s; + std::unique_lock<M> lk; + friend Sync; + Lock(Sync * s) : s(s), lk(s->mutex) { } + public: + Lock(Lock && l) : s(l.s) { abort(); } + Lock(const Lock & l) = delete; + ~Lock() { } + T * operator -> () { return &s->data; } + T & operator * () { return s->data; } + + void wait(std::condition_variable & cv) + { + assert(s); + cv.wait(lk); + } + + template<class Rep, class Period, class Predicate> + bool wait_for(std::condition_variable & cv, + const std::chrono::duration<Rep, Period> & duration, + Predicate pred) + { + assert(s); + return cv.wait_for(lk, duration, pred); + } + + template<class Clock, class Duration> + std::cv_status wait_until(std::condition_variable & cv, + const std::chrono::time_point<Clock, Duration> & duration) + { + assert(s); + return cv.wait_until(lk, duration); + } + }; + + Lock lock() { return Lock(this); } +}; + +} diff --git a/src/libutil/thread-pool.cc b/src/libutil/thread-pool.cc new file mode 100644 index 000000000000..743038b588a7 --- /dev/null +++ b/src/libutil/thread-pool.cc @@ -0,0 +1,82 @@ +#include "thread-pool.hh" + +namespace nix { + +ThreadPool::ThreadPool(size_t _nrThreads) + : nrThreads(_nrThreads) +{ + if (!nrThreads) { + nrThreads = std::thread::hardware_concurrency(); + if (!nrThreads) nrThreads = 1; + } +} + +void ThreadPool::enqueue(const work_t & t) +{ + auto state_(state.lock()); + state_->left.push(t); + wakeup.notify_one(); +} + +void ThreadPool::process() +{ + printMsg(lvlDebug, format("starting pool of %d threads") % nrThreads); + + std::vector<std::thread> workers; + + for (size_t n = 0; n < nrThreads; n++) + workers.push_back(std::thread([&]() { + bool first = true; + + while (true) { + work_t work; + { + auto state_(state.lock()); + if (state_->exception) return; + if (!first) { + assert(state_->pending); + state_->pending--; + } + first = false; + while (state_->left.empty()) { + if (!state_->pending) { + wakeup.notify_all(); + return; + } + if (state_->exception) return; + state_.wait(wakeup); + } + work = state_->left.front(); + state_->left.pop(); + state_->pending++; + } + + try { + work(); + } catch (std::exception & e) { + auto state_(state.lock()); + if (state_->exception) { + if (!dynamic_cast<Interrupted*>(&e)) + printMsg(lvlError, format("error: %s") % e.what()); + } else { + state_->exception = std::current_exception(); + wakeup.notify_all(); + } + } + } + + })); + + for (auto & thr : workers) + thr.join(); + + { + auto state_(state.lock()); + if (state_->exception) + std::rethrow_exception(state_->exception); + } +} + +} + + diff --git a/src/libutil/thread-pool.hh b/src/libutil/thread-pool.hh new file mode 100644 index 000000000000..77641d88ba4e --- /dev/null +++ b/src/libutil/thread-pool.hh @@ -0,0 +1,52 @@ +#pragma once + +#include "sync.hh" +#include "util.hh" + +#include <queue> +#include <functional> +#include <thread> + +namespace nix { + +/* A simple thread pool that executes a queue of work items + (lambdas). */ +class ThreadPool +{ +public: + + ThreadPool(size_t nrThreads = 0); + + // FIXME: use std::packaged_task? + typedef std::function<void()> work_t; + + /* Enqueue a function to be executed by the thread pool. */ + void enqueue(const work_t & t); + + /* Execute work items until the queue is empty. Note that work + items are allowed to add new items to the queue; this is + handled correctly. Queue processing stops prematurely if any + work item throws an exception. This exception is propagated to + the calling thread. If multiple work items throw an exception + concurrently, only one item is propagated; the others are + printed on stderr and otherwise ignored. */ + void process(); + +private: + + size_t nrThreads; + + struct State + { + std::queue<work_t> left; + size_t pending = 0; + std::exception_ptr exception; + }; + + Sync<State> state; + + std::condition_variable wakeup; + +}; + +} diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 0eae46c5fe93..33aaf5fc9c4d 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -2,6 +2,8 @@ #include "config.h" +#include "ref.hh" + #include <string> #include <list> #include <set> @@ -97,63 +99,4 @@ typedef enum { } Verbosity; -/* A simple non-nullable reference-counted pointer. Actually a wrapper - around std::shared_ptr that prevents non-null constructions. */ -template<typename T> -class ref -{ -private: - - std::shared_ptr<T> p; - -public: - - ref<T>(const ref<T> & r) - : p(r.p) - { } - - explicit ref<T>(const std::shared_ptr<T> & p) - : p(p) - { - if (!p) - throw std::invalid_argument("null pointer cast to ref"); - } - - T* operator ->() const - { - return &*p; - } - - T& operator *() const - { - return *p; - } - - operator std::shared_ptr<T> () - { - return p; - } - - template<typename T2> - operator ref<T2> () - { - return ref<T2>((std::shared_ptr<T2>) p); - } - -private: - - template<typename T2, typename... Args> - friend ref<T2> - make_ref(Args&&... args); - -}; - -template<typename T, typename... Args> -inline ref<T> -make_ref(Args&&... args) -{ - auto p = std::make_shared<T>(std::forward<Args>(args)...); - return ref<T>(p); -} - } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index def0525abc18..55d490992108 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -209,7 +209,7 @@ Path readLink(const Path & path) else if (rlsize > st.st_size) throw Error(format("symbolic link ‘%1%’ size overflow %2% > %3%") % path % rlsize % st.st_size); - return string(buf, st.st_size); + return string(buf, rlsize); } @@ -320,9 +320,11 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed) { checkInterrupt(); - printMsg(lvlVomit, format("%1%") % path); - - struct stat st = lstat(path); + struct stat st; + if (lstat(path.c_str(), &st) == -1) { + if (errno == ENOENT) return; + throw SysError(format("getting status of ‘%1%’") % path); + } if (!S_ISDIR(st.st_mode) && st.st_nlink == 1) bytesFreed += st.st_blocks * 512; @@ -338,8 +340,10 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed) _deletePath(path + "/" + i.name, bytesFreed); } - if (remove(path.c_str()) == -1) + if (remove(path.c_str()) == -1) { + if (errno == ENOENT) return; throw SysError(format("cannot unlink ‘%1%’") % path); + } } @@ -544,7 +548,7 @@ void writeToStderr(const string & s) } -void (*_writeToStderr) (const unsigned char * buf, size_t count) = 0; +std::function<void(const unsigned char * buf, size_t count)> _writeToStderr; void readFull(int fd, unsigned char * buf, size_t count) @@ -1058,13 +1062,15 @@ void restoreSIGPIPE() volatile sig_atomic_t _isInterrupted = 0; +thread_local bool interruptThrown = false; + void _interrupted() { /* Block user interrupts while an exception is being handled. Throwing an exception while another exception is being handled kills the program! */ - if (!std::uncaught_exception()) { - _isInterrupted = 0; + if (!interruptThrown && !std::uncaught_exception()) { + interruptThrown = true; throw Interrupted("interrupted by the user"); } } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 9eebb67fdf3a..20bd62a0e752 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -92,8 +92,8 @@ string readLine(int fd); void writeLine(int fd, string s); /* Delete a path; i.e., in the case of a directory, it is deleted - recursively. Don't use this at home, kids. The second variant - returns the number of bytes and blocks freed. */ + recursively. It's not an error if the path does not exist. The + second variant returns the number of bytes and blocks freed. */ void deletePath(const Path & path); void deletePath(const Path & path, unsigned long long & bytesFreed); @@ -167,7 +167,7 @@ void warnOnce(bool & haveWarned, const FormatOrString & fs); void writeToStderr(const string & s); -extern void (*_writeToStderr) (const unsigned char * buf, size_t count); +extern std::function<void(const unsigned char * buf, size_t count)> _writeToStderr; /* Wrappers arount read()/write() that read/write exactly the @@ -316,6 +316,8 @@ void restoreSIGPIPE(); extern volatile sig_atomic_t _isInterrupted; +extern thread_local bool interruptThrown; + void _interrupted(); void inline checkInterrupt() diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index b9ccafb751c1..3aa348581b19 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -82,7 +82,7 @@ int main(int argc, char * * argv) // Run the actual garbage collector. if (!dryRun) { - auto store = openStore(false); + auto store = openStore(); options.action = GCOptions::gcDeleteDead; GCResults results; PrintFreed freed(true, results); diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 27694eac1a8c..c3cdb8395093 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -310,7 +310,7 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe case wopImportPaths: { startWork(); TunnelSource source(from); - Paths paths = store->importPaths(!trusted, source); + Paths paths = store->importPaths(!trusted, source, 0); stopWork(); to << paths; break; @@ -322,8 +322,8 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { mode = (BuildMode)readInt(from); - /* Repairing is not atomic, so disallowed for "untrusted" - clients. */ + /* Repairing is not atomic, so disallowed for "untrusted" + clients. */ if (mode == bmRepair && !trusted) throw Error("repairing is not supported when building through the Nix daemon"); } @@ -493,23 +493,6 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe break; } - case wopQueryFailedPaths: { - startWork(); - PathSet paths = store->queryFailedPaths(); - stopWork(); - to << paths; - break; - } - - case wopClearFailedPaths: { - PathSet paths = readStrings<PathSet>(from); - startWork(); - store->clearFailedPaths(paths); - stopWork(); - to << 1; - break; - } - case wopQueryPathInfo: { Path path = readStorePath(from); startWork(); @@ -517,6 +500,10 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe stopWork(); to << info.deriver << printHash(info.narHash) << info.references << info.registrationTime << info.narSize; + if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { + to << info.ultimate + << info.sigs; + } break; } @@ -539,6 +526,18 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe break; } + case wopAddSignatures: { + Path path = readStorePath(from); + StringSet sigs = readStrings<StringSet>(from); + startWork(); + if (!trusted) + throw Error("you are not privileged to add signatures"); + store->addSignatures(path, sigs); + stopWork(); + to << 1; + break; + } + default: throw Error(format("invalid operation %1%") % op); } @@ -562,9 +561,8 @@ static void processConnection(bool trusted) if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) setAffinityTo(readInt(from)); - bool reserveSpace = true; if (GET_PROTOCOL_MINOR(clientVersion) >= 11) - reserveSpace = readInt(from) != 0; + readInt(from); // obsolete reserveSpace /* Send startup error messages to the client. */ startWork(); @@ -582,7 +580,7 @@ static void processConnection(bool trusted) #endif /* Open the store. */ - auto store = make_ref<LocalStore>(reserveSpace); + auto store = make_ref<LocalStore>(); stopWork(); to.flush(); diff --git a/src/nix-hash/local.mk b/src/nix-hash/local.mk deleted file mode 100644 index 7c290ca8466e..000000000000 --- a/src/nix-hash/local.mk +++ /dev/null @@ -1,7 +0,0 @@ -programs += nix-hash - -nix-hash_DIR := $(d) - -nix-hash_SOURCES := $(d)/nix-hash.cc - -nix-hash_LIBS = libmain libstore libutil libformat diff --git a/src/nix-hash/nix-hash.cc b/src/nix-hash/nix-hash.cc deleted file mode 100644 index 8035162aea37..000000000000 --- a/src/nix-hash/nix-hash.cc +++ /dev/null @@ -1,63 +0,0 @@ -#include "hash.hh" -#include "shared.hh" - -#include <iostream> - -using namespace nix; - - -int main(int argc, char * * argv) -{ - HashType ht = htMD5; - bool flat = false; - bool base32 = false; - bool truncate = false; - enum { opHash, opTo32, opTo16 } op = opHash; - - Strings ss; - - return handleExceptions(argv[0], [&]() { - initNix(); - - parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { - if (*arg == "--help") - showManPage("nix-hash"); - else if (*arg == "--version") - printVersion("nix-hash"); - else if (*arg == "--flat") flat = true; - else if (*arg == "--base32") base32 = true; - else if (*arg == "--truncate") truncate = true; - else if (*arg == "--type") { - string s = getArg(*arg, arg, end); - ht = parseHashType(s); - if (ht == htUnknown) - throw UsageError(format("unknown hash type ‘%1%’") % s); - } - else if (*arg == "--to-base16") op = opTo16; - else if (*arg == "--to-base32") op = opTo32; - else if (*arg != "" && arg->at(0) == '-') - return false; - else - ss.push_back(*arg); - return true; - }); - - if (op == opHash) { - for (auto & i : ss) { - Hash h = flat ? hashFile(ht, i) : hashPath(ht, i).first; - if (truncate && h.hashSize > 20) h = compressHash(h, 20); - std::cout << format("%1%\n") % - (base32 ? printHash32(h) : printHash(h)); - } - } - - else { - for (auto & i : ss) { - Hash h = parseHash16or32(ht, i); - std::cout << format("%1%\n") % - (op == opTo16 ? printHash(h) : printHash32(h)); - } - } - }); -} - diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index c0c05a60bd78..c65961a15720 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -158,7 +158,7 @@ int main(int argc, char * * argv) auto actualUri = resolveMirrorUri(state, uri); /* Download the file. */ - auto result = downloadFile(actualUri, DownloadOptions()); + auto result = makeDownloader()->download(actualUri, DownloadOptions()); AutoDelete tmpDir(createTempDir(), true); Path tmpFile = (Path) tmpDir + "/tmp"; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 7ec61eb625b0..179015b52bfe 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -732,7 +732,7 @@ static void opImport(Strings opFlags, Strings opArgs) if (!opArgs.empty()) throw UsageError("no arguments expected"); FdSource source(STDIN_FILENO); - Paths paths = store->importPaths(requireSignature, source); + Paths paths = store->importPaths(requireSignature, source, 0); for (auto & i : paths) cout << format("%1%\n") % i << std::flush; @@ -783,7 +783,9 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) Path path = followLinksToStorePath(i); printMsg(lvlTalkative, format("checking path ‘%1%’...") % path); ValidPathInfo info = store->queryPathInfo(path); - HashResult current = hashPath(info.narHash.type, path); + HashSink sink(info.narHash.type); + store->narFromPath(path, sink); + auto current = sink.finish(); if (current.first != info.narHash) { printMsg(lvlError, format("path ‘%1%’ was modified! expected hash ‘%2%’, got ‘%3%’") @@ -819,24 +821,6 @@ static void opOptimise(Strings opFlags, Strings opArgs) store->optimiseStore(); } -static void opQueryFailedPaths(Strings opFlags, Strings opArgs) -{ - if (!opArgs.empty() || !opFlags.empty()) - throw UsageError("no arguments expected"); - PathSet failed = store->queryFailedPaths(); - for (auto & i : failed) - cout << format("%1%\n") % i; -} - - -static void opClearFailedPaths(Strings opFlags, Strings opArgs) -{ - if (!opFlags.empty()) - throw UsageError("no flags expected"); - store->clearFailedPaths(PathSet(opArgs.begin(), opArgs.end())); -} - - /* Serve the nix store in a way usable by a restricted ssh user. */ static void opServe(Strings opFlags, Strings opArgs) { @@ -935,7 +919,7 @@ static void opServe(Strings opFlags, Strings opArgs) case cmdImportPaths: { if (!writeAllowed) throw Error("importing paths is not allowed"); - store->importPaths(false, in); + store->importPaths(false, in, 0); out << 1; // indicate success break; } @@ -1100,10 +1084,6 @@ int main(int argc, char * * argv) op = opRepairPath; else if (*arg == "--optimise" || *arg == "--optimize") op = opOptimise; - else if (*arg == "--query-failed-paths") - op = opQueryFailedPaths; - else if (*arg == "--clear-failed-paths") - op = opClearFailedPaths; else if (*arg == "--serve") op = opServe; else if (*arg == "--generate-binary-cache-key") @@ -1131,7 +1111,7 @@ int main(int argc, char * * argv) if (!op) throw UsageError("no operation specified"); if (op != opDump && op != opRestore) /* !!! hack */ - store = openStore(op != opGC); + store = openStore(); op(opFlags, opArgs); }); diff --git a/src/nix/build.cc b/src/nix/build.cc new file mode 100644 index 000000000000..812464d7582b --- /dev/null +++ b/src/nix/build.cc @@ -0,0 +1,46 @@ +#include "command.hh" +#include "common-args.hh" +#include "installables.hh" +#include "shared.hh" +#include "store-api.hh" + +using namespace nix; + +struct CmdBuild : StoreCommand, MixDryRun, MixInstallables +{ + CmdBuild() + { + } + + std::string name() override + { + return "build"; + } + + std::string description() override + { + return "build a derivation or fetch a store path"; + } + + void run(ref<Store> store) override + { + auto elems = evalInstallables(store); + + PathSet pathsToBuild; + + for (auto & elem : elems) { + if (elem.isDrv) + pathsToBuild.insert(elem.drvPath); + else + pathsToBuild.insert(elem.outPaths.begin(), elem.outPaths.end()); + } + + printMissing(store, pathsToBuild); + + if (dryRun) return; + + store->buildPaths(pathsToBuild); + } +}; + +static RegisterCommand r1(make_ref<CmdBuild>()); diff --git a/src/nix/cat.cc b/src/nix/cat.cc new file mode 100644 index 000000000000..2405a8cb44ef --- /dev/null +++ b/src/nix/cat.cc @@ -0,0 +1,74 @@ +#include "command.hh" +#include "store-api.hh" +#include "fs-accessor.hh" +#include "nar-accessor.hh" + +using namespace nix; + +struct MixCat : virtual Args +{ + std::string path; + + void cat(ref<FSAccessor> accessor) + { + auto st = accessor->stat(path); + if (st.type == FSAccessor::Type::tMissing) + throw Error(format("path ‘%1%’ does not exist") % path); + if (st.type != FSAccessor::Type::tRegular) + throw Error(format("path ‘%1%’ is not a regular file") % path); + + std::cout << accessor->readFile(path); + } +}; + +struct CmdCatStore : StoreCommand, MixCat +{ + CmdCatStore() + { + expectArg("path", &path); + } + + std::string name() override + { + return "cat-store"; + } + + std::string description() override + { + return "print the contents of a store file on stdout"; + } + + void run(ref<Store> store) override + { + cat(store->getFSAccessor()); + } +}; + +struct CmdCatNar : StoreCommand, MixCat +{ + Path narPath; + + CmdCatNar() + { + expectArg("nar", &narPath); + expectArg("path", &path); + } + + std::string name() override + { + return "cat-nar"; + } + + std::string description() override + { + return "print the contents of a file inside a NAR file"; + } + + void run(ref<Store> store) override + { + cat(makeNarAccessor(make_ref<std::string>(readFile(narPath)))); + } +}; + +static RegisterCommand r1(make_ref<CmdCatStore>()); +static RegisterCommand r2(make_ref<CmdCatNar>()); diff --git a/src/nix/command.cc b/src/nix/command.cc new file mode 100644 index 000000000000..a89246a937c1 --- /dev/null +++ b/src/nix/command.cc @@ -0,0 +1,93 @@ +#include "command.hh" +#include "store-api.hh" + +namespace nix { + +Commands * RegisterCommand::commands = 0; + +MultiCommand::MultiCommand(const Commands & _commands) + : commands(_commands) +{ + expectedArgs.push_back(ExpectedArg{"command", 1, [=](Strings ss) { + assert(!command); + auto i = commands.find(ss.front()); + if (i == commands.end()) + throw UsageError(format("‘%1%’ is not a recognised command") % ss.front()); + command = i->second; + }}); +} + +void MultiCommand::printHelp(const string & programName, std::ostream & out) +{ + if (command) { + command->printHelp(programName + " " + command->name(), out); + return; + } + + out << "Usage: " << programName << " <COMMAND> <FLAGS>... <ARGS>...\n"; + + out << "\n"; + out << "Common flags:\n"; + printFlags(out); + + out << "\n"; + out << "Available commands:\n"; + + Table2 table; + for (auto & command : commands) + table.push_back(std::make_pair(command.second->name(), command.second->description())); + printTable(out, table); + + out << "\n"; + out << "For full documentation, run ‘man " << programName << "’ or ‘man " << programName << "-<COMMAND>’.\n"; +} + +bool MultiCommand::processFlag(Strings::iterator & pos, Strings::iterator end) +{ + if (Args::processFlag(pos, end)) return true; + if (command && command->processFlag(pos, end)) return true; + return false; +} + +bool MultiCommand::processArgs(const Strings & args, bool finish) +{ + if (command) + return command->processArgs(args, finish); + else + return Args::processArgs(args, finish); +} + +StoreCommand::StoreCommand() +{ + storeUri = getEnv("NIX_REMOTE"); + + mkFlag(0, "store", "store-uri", "URI of the Nix store to use", &storeUri); +} + +void StoreCommand::run() +{ + run(openStoreAt(storeUri)); +} + +StorePathsCommand::StorePathsCommand() +{ + expectArgs("paths", &storePaths); + mkFlag('r', "recursive", "apply operation to closure of the specified paths", &recursive); +} + +void StorePathsCommand::run(ref<Store> store) +{ + for (auto & storePath : storePaths) + storePath = followLinksToStorePath(storePath); + + if (recursive) { + PathSet closure; + for (auto & storePath : storePaths) + store->computeFSClosure(storePath, closure, false, false); + storePaths = store->topoSortPaths(closure); + } + + run(store, storePaths); +} + +} diff --git a/src/nix/command.hh b/src/nix/command.hh new file mode 100644 index 000000000000..8397244ca177 --- /dev/null +++ b/src/nix/command.hh @@ -0,0 +1,76 @@ +#pragma once + +#include "args.hh" + +namespace nix { + +/* A command is an argument parser that can be executed by calling its + run() method. */ +struct Command : virtual Args +{ + virtual std::string name() = 0; + virtual void prepare() { }; + virtual void run() = 0; +}; + +class Store; + +/* A command that require a Nix store. */ +struct StoreCommand : virtual Command +{ + std::string storeUri; + StoreCommand(); + void run() override; + virtual void run(ref<Store>) = 0; +}; + +/* A command that operates on zero or more store paths. */ +struct StorePathsCommand : public StoreCommand +{ +private: + + Paths storePaths; + bool recursive = false; + +public: + + StorePathsCommand(); + + virtual void run(ref<Store> store, Paths storePaths) = 0; + + void run(ref<Store> store) override; +}; + +typedef std::map<std::string, ref<Command>> Commands; + +/* An argument parser that supports multiple subcommands, + i.e. ‘<command> <subcommand>’. */ +class MultiCommand : virtual Args +{ +public: + Commands commands; + + std::shared_ptr<Command> command; + + MultiCommand(const Commands & commands); + + void printHelp(const string & programName, std::ostream & out) override; + + bool processFlag(Strings::iterator & pos, Strings::iterator end) override; + + bool processArgs(const Strings & args, bool finish) override; +}; + +/* A helper class for registering commands globally. */ +struct RegisterCommand +{ + static Commands * commands; + + RegisterCommand(ref<Command> command) + { + if (!commands) commands = new Commands; + commands->emplace(command->name(), command); + } +}; + +} diff --git a/src/nix/hash.cc b/src/nix/hash.cc new file mode 100644 index 000000000000..5dd891e8add3 --- /dev/null +++ b/src/nix/hash.cc @@ -0,0 +1,140 @@ +#include "command.hh" +#include "hash.hh" +#include "legacy.hh" +#include "shared.hh" + +using namespace nix; + +struct CmdHash : Command +{ + enum Mode { mFile, mPath }; + Mode mode; + bool base32 = false; + bool truncate = false; + HashType ht = htSHA512; + Strings paths; + + CmdHash(Mode mode) : mode(mode) + { + mkFlag(0, "base32", "print hash in base-32", &base32); + mkFlag(0, "base16", "print hash in base-16", &base32, false); + mkHashTypeFlag("type", &ht); + expectArgs("paths", &paths); + } + + std::string name() override + { + return mode == mFile ? "hash-file" : "hash-path"; + } + + std::string description() override + { + return mode == mFile + ? "print cryptographic hash of a regular file" + : "print cryptographic hash of the NAR serialisation of a path"; + } + + void run() override + { + for (auto path : paths) { + Hash h = mode == mFile ? hashFile(ht, path) : hashPath(ht, path).first; + if (truncate && h.hashSize > 20) h = compressHash(h, 20); + std::cout << format("%1%\n") % + (base32 ? printHash32(h) : printHash(h)); + } + } +}; + +static RegisterCommand r1(make_ref<CmdHash>(CmdHash::mFile)); +static RegisterCommand r2(make_ref<CmdHash>(CmdHash::mPath)); + +struct CmdToBase : Command +{ + bool toBase32; + HashType ht = htSHA512; + Strings args; + + CmdToBase(bool toBase32) : toBase32(toBase32) + { + mkHashTypeFlag("type", &ht); + expectArgs("strings", &args); + } + + std::string name() override + { + return toBase32 ? "to-base32" : "to-base16"; + } + + std::string description() override + { + return toBase32 + ? "convert a hash to base-32 representation" + : "convert a hash to base-16 representation"; + } + + void run() override + { + for (auto s : args) { + Hash h = parseHash16or32(ht, s); + std::cout << format("%1%\n") % + (toBase32 ? printHash32(h) : printHash(h)); + } + } +}; + +static RegisterCommand r3(make_ref<CmdToBase>(false)); +static RegisterCommand r4(make_ref<CmdToBase>(true)); + +/* Legacy nix-hash command. */ +static int compatNixHash(int argc, char * * argv) +{ + HashType ht = htMD5; + bool flat = false; + bool base32 = false; + bool truncate = false; + enum { opHash, opTo32, opTo16 } op = opHash; + Strings ss; + + parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { + if (*arg == "--help") + showManPage("nix-hash"); + else if (*arg == "--version") + printVersion("nix-hash"); + else if (*arg == "--flat") flat = true; + else if (*arg == "--base32") base32 = true; + else if (*arg == "--truncate") truncate = true; + else if (*arg == "--type") { + string s = getArg(*arg, arg, end); + ht = parseHashType(s); + if (ht == htUnknown) + throw UsageError(format("unknown hash type ‘%1%’") % s); + } + else if (*arg == "--to-base16") op = opTo16; + else if (*arg == "--to-base32") op = opTo32; + else if (*arg != "" && arg->at(0) == '-') + return false; + else + ss.push_back(*arg); + return true; + }); + + if (op == opHash) { + CmdHash cmd(flat ? CmdHash::mFile : CmdHash::mPath); + cmd.ht = ht; + cmd.base32 = base32; + cmd.truncate = truncate; + cmd.paths = ss; + cmd.run(); + } + + else { + CmdToBase cmd(op == opTo32); + cmd.args = ss; + cmd.ht = ht; + cmd.run(); + } + + return 0; +} + +static RegisterLegacyCommand s1("nix-hash", compatNixHash); diff --git a/src/nix/installables.cc b/src/nix/installables.cc new file mode 100644 index 000000000000..fb5a515825aa --- /dev/null +++ b/src/nix/installables.cc @@ -0,0 +1,75 @@ +#include "attr-path.hh" +#include "common-opts.hh" +#include "derivations.hh" +#include "eval-inline.hh" +#include "eval.hh" +#include "get-drvs.hh" +#include "installables.hh" +#include "store-api.hh" + +namespace nix { + +UserEnvElems MixInstallables::evalInstallables(ref<Store> store) +{ + UserEnvElems res; + + for (auto & installable : installables) { + + if (std::string(installable, 0, 1) == "/") { + + if (isStorePath(installable)) { + + if (isDerivation(installable)) { + UserEnvElem elem; + // FIXME: handle empty case, drop version + elem.attrPath = {storePathToName(installable)}; + elem.isDrv = true; + elem.drvPath = installable; + res.push_back(elem); + } + + else { + UserEnvElem elem; + // FIXME: handle empty case, drop version + elem.attrPath = {storePathToName(installable)}; + elem.isDrv = false; + elem.outPaths = {installable}; + res.push_back(elem); + } + } + + else + throw UsageError(format("don't know what to do with ‘%1%’") % installable); + } + + else { + + EvalState state({}, store); + + Expr * e = state.parseExprFromFile(resolveExprPath(lookupFileArg(state, file))); + + Value vRoot; + state.eval(e, vRoot); + + std::map<string, string> autoArgs_; + Bindings & autoArgs(*evalAutoArgs(state, autoArgs_)); + + Value & v(*findAlongAttrPath(state, installable, autoArgs, vRoot)); + state.forceValue(v); + + DrvInfos drvs; + getDerivations(state, v, "", autoArgs, drvs, false); + + for (auto & i : drvs) { + UserEnvElem elem; + elem.isDrv = true; + elem.drvPath = i.queryDrvPath(); + res.push_back(elem); + } + } + } + + return res; +} + +} diff --git a/src/nix/installables.hh b/src/nix/installables.hh new file mode 100644 index 000000000000..5eb897d46148 --- /dev/null +++ b/src/nix/installables.hh @@ -0,0 +1,38 @@ +#pragma once + +#include "args.hh" + +namespace nix { + +struct UserEnvElem +{ + Strings attrPath; + + // FIXME: should use boost::variant or so. + bool isDrv; + + // Derivation case: + Path drvPath; + StringSet outputNames; + + // Non-derivation case: + PathSet outPaths; +}; + +typedef std::vector<UserEnvElem> UserEnvElems; + +struct MixInstallables : virtual Args +{ + Strings installables; + Path file = "<nixpkgs>"; + + MixInstallables() + { + mkFlag('f', "file", "file", "evaluate FILE rather than the default", &file); + expectArgs("installables", &installables); + } + + UserEnvElems evalInstallables(ref<Store> store); +}; + +} diff --git a/src/nix/legacy.cc b/src/nix/legacy.cc new file mode 100644 index 000000000000..6df09ee37a5e --- /dev/null +++ b/src/nix/legacy.cc @@ -0,0 +1,7 @@ +#include "legacy.hh" + +namespace nix { + +RegisterLegacyCommand::Commands * RegisterLegacyCommand::commands = 0; + +} diff --git a/src/nix/legacy.hh b/src/nix/legacy.hh new file mode 100644 index 000000000000..f503b0da3e1a --- /dev/null +++ b/src/nix/legacy.hh @@ -0,0 +1,23 @@ +#pragma once + +#include <functional> +#include <map> +#include <string> + +namespace nix { + +typedef std::function<void(int, char * *)> MainFunction; + +struct RegisterLegacyCommand +{ + typedef std::map<std::string, MainFunction> Commands; + static Commands * commands; + + RegisterLegacyCommand(const std::string & name, MainFunction fun) + { + if (!commands) commands = new Commands; + (*commands)[name] = fun; + } +}; + +} diff --git a/src/nix/local.mk b/src/nix/local.mk new file mode 100644 index 000000000000..f6e7073b6e7d --- /dev/null +++ b/src/nix/local.mk @@ -0,0 +1,9 @@ +programs += nix + +nix_DIR := $(d) + +nix_SOURCES := $(wildcard $(d)/*.cc) + +nix_LIBS = libexpr libmain libstore libutil libformat + +$(eval $(call install-symlink, nix, $(bindir)/nix-hash)) diff --git a/src/nix/ls.cc b/src/nix/ls.cc new file mode 100644 index 000000000000..3476dfb05287 --- /dev/null +++ b/src/nix/ls.cc @@ -0,0 +1,123 @@ +#include "command.hh" +#include "store-api.hh" +#include "fs-accessor.hh" +#include "nar-accessor.hh" + +using namespace nix; + +struct MixLs : virtual Args +{ + std::string path; + + bool recursive = false; + bool verbose = false; + bool showDirectory = false; + + MixLs() + { + mkFlag('R', "recursive", "list subdirectories recursively", &recursive); + mkFlag('l', "long", "show more file information", &verbose); + mkFlag('d', "directory", "show directories rather than their contents", &showDirectory); + } + + void list(ref<FSAccessor> accessor) + { + std::function<void(const FSAccessor::Stat &, const Path &, const std::string &, bool)> doPath; + + auto showFile = [&](const Path & curPath, const std::string & relPath) { + if (verbose) { + auto st = accessor->stat(curPath); + std::string tp = + st.type == FSAccessor::Type::tRegular ? + (st.isExecutable ? "-r-xr-xr-x" : "-r--r--r--") : + st.type == FSAccessor::Type::tSymlink ? "lrwxrwxrwx" : + "dr-xr-xr-x"; + std::cout << + (format("%s %20d %s") % tp % st.fileSize % relPath); + if (st.type == FSAccessor::Type::tSymlink) + std::cout << " -> " << accessor->readLink(curPath) + ; + std::cout << "\n"; + if (recursive && st.type == FSAccessor::Type::tDirectory) + doPath(st, curPath, relPath, false); + } else { + std::cout << relPath << "\n"; + if (recursive) { + auto st = accessor->stat(curPath); + if (st.type == FSAccessor::Type::tDirectory) + doPath(st, curPath, relPath, false); + } + } + }; + + doPath = [&](const FSAccessor::Stat & st , const Path & curPath, + const std::string & relPath, bool showDirectory) + { + if (st.type == FSAccessor::Type::tDirectory && !showDirectory) { + auto names = accessor->readDirectory(curPath); + for (auto & name : names) + showFile(curPath + "/" + name, relPath + "/" + name); + } else + showFile(curPath, relPath); + }; + + auto st = accessor->stat(path); + if (st.type == FSAccessor::Type::tMissing) + throw Error(format("path ‘%1%’ does not exist") % path); + doPath(st, path, + st.type == FSAccessor::Type::tDirectory ? "." : baseNameOf(path), + showDirectory); + } +}; + +struct CmdLsStore : StoreCommand, MixLs +{ + CmdLsStore() + { + expectArg("path", &path); + } + + std::string name() override + { + return "ls-store"; + } + + std::string description() override + { + return "show information about a store path"; + } + + void run(ref<Store> store) override + { + list(store->getFSAccessor()); + } +}; + +struct CmdLsNar : Command, MixLs +{ + Path narPath; + + CmdLsNar() + { + expectArg("nar", &narPath); + expectArg("path", &path); + } + + std::string name() override + { + return "ls-nar"; + } + + std::string description() override + { + return "show information about the contents of a NAR file"; + } + + void run() override + { + list(makeNarAccessor(make_ref<std::string>(readFile(narPath)))); + } +}; + +static RegisterCommand r1(make_ref<CmdLsStore>()); +static RegisterCommand r2(make_ref<CmdLsNar>()); diff --git a/src/nix/main.cc b/src/nix/main.cc new file mode 100644 index 000000000000..2005ec5f9a6d --- /dev/null +++ b/src/nix/main.cc @@ -0,0 +1,56 @@ +#include <algorithm> + +#include "command.hh" +#include "common-args.hh" +#include "eval.hh" +#include "globals.hh" +#include "legacy.hh" +#include "shared.hh" +#include "store-api.hh" + +namespace nix { + +struct NixArgs : virtual MultiCommand, virtual MixCommonArgs +{ + NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix") + { + mkFlag('h', "help", "show usage information", [=]() { + printHelp(programName, std::cout); + std::cout << "\nNote: this program is EXPERIMENTAL and subject to change.\n"; + throw Exit(); + }); + + mkFlag(0, "version", "show version information", std::bind(printVersion, programName)); + } +}; + +void mainWrapped(int argc, char * * argv) +{ + initNix(); + initGC(); + + string programName = baseNameOf(argv[0]); + + { + auto legacy = (*RegisterLegacyCommand::commands)[programName]; + if (legacy) return legacy(argc, argv); + } + + NixArgs args; + + args.parseCmdline(argvToStrings(argc, argv)); + + assert(args.command); + + args.command->prepare(); + args.command->run(); +} + +} + +int main(int argc, char * * argv) +{ + return nix::handleExceptions(argv[0], [&]() { + nix::mainWrapped(argc, argv); + }); +} diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc new file mode 100644 index 000000000000..ed7b578e2f49 --- /dev/null +++ b/src/nix/progress-bar.cc @@ -0,0 +1,72 @@ +#include "progress-bar.hh" + +#include <iostream> + +namespace nix { + +ProgressBar::ProgressBar() +{ + _writeToStderr = [&](const unsigned char * buf, size_t count) { + auto state_(state.lock()); + assert(!state_->done); + std::cerr << "\r\e[K" << std::string((const char *) buf, count); + render(*state_); + }; +} + +ProgressBar::~ProgressBar() +{ + done(); +} + +void ProgressBar::updateStatus(const std::string & s) +{ + auto state_(state.lock()); + assert(!state_->done); + state_->status = s; + render(*state_); +} + +void ProgressBar::done() +{ + auto state_(state.lock()); + assert(state_->activities.empty()); + state_->done = true; + std::cerr << "\r\e[K"; + std::cerr.flush(); + _writeToStderr = decltype(_writeToStderr)(); +} + +void ProgressBar::render(State & state_) +{ + std::cerr << '\r' << state_.status; + if (!state_.activities.empty()) { + if (!state_.status.empty()) std::cerr << ' '; + std::cerr << *state_.activities.rbegin(); + } + std::cerr << "\e[K"; + std::cerr.flush(); +} + + +ProgressBar::Activity ProgressBar::startActivity(const FormatOrString & fs) +{ + return Activity(*this, fs); +} + +ProgressBar::Activity::Activity(ProgressBar & pb, const FormatOrString & fs) + : pb(pb) +{ + auto state_(pb.state.lock()); + state_->activities.push_back(fs.s); + it = state_->activities.end(); --it; + pb.render(*state_); +} + +ProgressBar::Activity::~Activity() +{ + auto state_(pb.state.lock()); + state_->activities.erase(it); +} + +} diff --git a/src/nix/progress-bar.hh b/src/nix/progress-bar.hh new file mode 100644 index 000000000000..2dda24346c90 --- /dev/null +++ b/src/nix/progress-bar.hh @@ -0,0 +1,49 @@ +#pragma once + +#include "sync.hh" +#include "util.hh" + +namespace nix { + +class ProgressBar +{ +private: + struct State + { + std::string status; + bool done = false; + std::list<std::string> activities; + }; + + Sync<State> state; + +public: + + ProgressBar(); + + ~ProgressBar(); + + void updateStatus(const std::string & s); + + void done(); + + class Activity + { + friend class ProgressBar; + private: + ProgressBar & pb; + std::list<std::string>::iterator it; + Activity(ProgressBar & pb, const FormatOrString & fs); + public: + ~Activity(); + }; + + Activity startActivity(const FormatOrString & fs); + +private: + + void render(State & state_); + +}; + +} diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc new file mode 100644 index 000000000000..bcc46c3e7d4f --- /dev/null +++ b/src/nix/sigs.cc @@ -0,0 +1,181 @@ +#include "affinity.hh" // FIXME +#include "command.hh" +#include "progress-bar.hh" +#include "shared.hh" +#include "store-api.hh" +#include "thread-pool.hh" + +#include <atomic> + +using namespace nix; + +struct CmdCopySigs : StorePathsCommand +{ + Strings substituterUris; + + CmdCopySigs() + { + mkFlag('s', "substituter", {"store-uri"}, "use signatures from specified store", 1, + [&](Strings ss) { substituterUris.push_back(ss.front()); }); + } + + std::string name() override + { + return "copy-sigs"; + } + + std::string description() override + { + return "copy path signatures from substituters (like binary caches)"; + } + + void run(ref<Store> store, Paths storePaths) override + { + restoreAffinity(); // FIXME + + if (substituterUris.empty()) + throw UsageError("you must specify at least one substituter using ‘-s’"); + + // FIXME: factor out commonality with MixVerify. + std::vector<ref<Store>> substituters; + for (auto & s : substituterUris) + substituters.push_back(openStoreAt(s)); + + ProgressBar progressBar; + + ThreadPool pool; + + std::atomic<size_t> done{0}; + std::atomic<size_t> added{0}; + + auto showProgress = [&]() { + return (format("[%d/%d done]") % done % storePaths.size()).str(); + }; + + progressBar.updateStatus(showProgress()); + + auto doPath = [&](const Path & storePath) { + auto activity(progressBar.startActivity(format("getting signatures for ‘%s’") % storePath)); + + checkInterrupt(); + + auto info = store->queryPathInfo(storePath); + + StringSet newSigs; + + for (auto & store2 : substituters) { + if (!store2->isValidPath(storePath)) continue; + auto info2 = store2->queryPathInfo(storePath); + + /* Don't import signatures that don't match this + binary. */ + if (info.narHash != info2.narHash || + info.narSize != info2.narSize || + info.references != info2.references) + continue; + + for (auto & sig : info2.sigs) + if (!info.sigs.count(sig)) + newSigs.insert(sig); + } + + if (!newSigs.empty()) { + store->addSignatures(storePath, newSigs); + added += newSigs.size(); + } + + done++; + progressBar.updateStatus(showProgress()); + }; + + for (auto & storePath : storePaths) + pool.enqueue(std::bind(doPath, storePath)); + + pool.process(); + + progressBar.done(); + + printMsg(lvlInfo, format("imported %d signatures") % added); + } +}; + +static RegisterCommand r1(make_ref<CmdCopySigs>()); + +struct CmdQueryPathSigs : StorePathsCommand +{ + CmdQueryPathSigs() + { + } + + std::string name() override + { + return "query-path-sigs"; + } + + std::string description() override + { + return "print store path signatures"; + } + + void run(ref<Store> store, Paths storePaths) override + { + for (auto & storePath : storePaths) { + auto info = store->queryPathInfo(storePath); + std::cout << storePath << " "; + if (info.ultimate) std::cout << "ultimate "; + for (auto & sig : info.sigs) + std::cout << sig << " "; + std::cout << "\n"; + } + } +}; + +static RegisterCommand r2(make_ref<CmdQueryPathSigs>()); + +struct CmdSignPaths : StorePathsCommand +{ + Path secretKeyFile; + + CmdSignPaths() + { + mkFlag('k', "key-file", {"file"}, "file containing the secret signing key", &secretKeyFile); + } + + std::string name() override + { + return "sign-paths"; + } + + std::string description() override + { + return "sign the specified paths"; + } + + void run(ref<Store> store, Paths storePaths) override + { + if (secretKeyFile.empty()) + throw UsageError("you must specify a secret key file using ‘-k’"); + + SecretKey secretKey(readFile(secretKeyFile)); + + size_t added{0}; + + for (auto & storePath : storePaths) { + auto info = store->queryPathInfo(storePath); + + auto info2(info); + info2.sigs.clear(); + info2.sign(secretKey); + assert(!info2.sigs.empty()); + + if (!info.sigs.count(*info2.sigs.begin())) { + store->addSignatures(storePath, info2.sigs); + added++; + } + } + + printMsg(lvlInfo, format("added %d signatures") % added); + } +}; + +static RegisterCommand r3(make_ref<CmdSignPaths>()); diff --git a/src/nix/verify.cc b/src/nix/verify.cc new file mode 100644 index 000000000000..9214d3b651d1 --- /dev/null +++ b/src/nix/verify.cc @@ -0,0 +1,211 @@ +#include "affinity.hh" // FIXME +#include "command.hh" +#include "progress-bar.hh" +#include "shared.hh" +#include "store-api.hh" +#include "sync.hh" +#include "thread-pool.hh" + +#include <atomic> + +using namespace nix; + +struct MixVerify : virtual Args +{ + bool noContents = false; + bool noTrust = false; + Strings substituterUris; + size_t sigsNeeded; + + MixVerify() + { + mkFlag(0, "no-contents", "do not verify the contents of each store path", &noContents); + mkFlag(0, "no-trust", "do not verify whether each store path is trusted", &noTrust); + mkFlag('s', "substituter", {"store-uri"}, "use signatures from specified store", 1, + [&](Strings ss) { substituterUris.push_back(ss.front()); }); + mkIntFlag('n', "sigs-needed", "require that each path has at least N valid signatures", &sigsNeeded); + } + + void verifyPaths(ref<Store> store, const Paths & storePaths) + { + restoreAffinity(); // FIXME + + std::vector<ref<Store>> substituters; + for (auto & s : substituterUris) + substituters.push_back(openStoreAt(s)); + + auto publicKeys = getDefaultPublicKeys(); + + std::atomic<size_t> untrusted{0}; + std::atomic<size_t> corrupted{0}; + std::atomic<size_t> done{0}; + std::atomic<size_t> failed{0}; + + ProgressBar progressBar; + + auto showProgress = [&](bool final) { + std::string s; + if (final) + s = (format("checked %d paths") % storePaths.size()).str(); + else + s = (format("[%d/%d checked") % done % storePaths.size()).str(); + if (corrupted > 0) + s += (format(", %d corrupted") % corrupted).str(); + if (untrusted > 0) + s += (format(", %d untrusted") % untrusted).str(); + if (failed > 0) + s += (format(", %d failed") % failed).str(); + if (!final) s += "]"; + return s; + }; + + progressBar.updateStatus(showProgress(false)); + + ThreadPool pool; + + auto doPath = [&](const Path & storePath) { + try { + checkInterrupt(); + + auto activity(progressBar.startActivity(format("checking ‘%s’") % storePath)); + + auto info = store->queryPathInfo(storePath); + + if (!noContents) { + + HashSink sink(info.narHash.type); + store->narFromPath(storePath, sink); + + auto hash = sink.finish(); + + if (hash.first != info.narHash) { + corrupted = 1; + printMsg(lvlError, + format("path ‘%s’ was modified! expected hash ‘%s’, got ‘%s’") + % storePath % printHash(info.narHash) % printHash(hash.first)); + } + + } + + if (!noTrust) { + + bool good = false; + + if (info.ultimate && !sigsNeeded) + good = true; + + else { + + StringSet sigsSeen; + size_t actualSigsNeeded = sigsNeeded ? sigsNeeded : 1; + size_t validSigs = 0; + + auto doSigs = [&](StringSet sigs) { + for (auto sig : sigs) { + if (sigsSeen.count(sig)) continue; + sigsSeen.insert(sig); + if (info.checkSignature(publicKeys, sig)) + validSigs++; + } + }; + + doSigs(info.sigs); + + for (auto & store2 : substituters) { + if (validSigs >= actualSigsNeeded) break; + try { + if (!store2->isValidPath(storePath)) continue; + doSigs(store2->queryPathInfo(storePath).sigs); + } catch (Error & e) { + printMsg(lvlError, format(ANSI_RED "error:" ANSI_NORMAL " %s") % e.what()); + } + } + + if (validSigs >= actualSigsNeeded) + good = true; + } + + if (!good) { + untrusted++; + printMsg(lvlError, format("path ‘%s’ is untrusted") % storePath); + } + + } + + done++; + + progressBar.updateStatus(showProgress(false)); + + } catch (Error & e) { + printMsg(lvlError, format(ANSI_RED "error:" ANSI_NORMAL " %s") % e.what()); + failed++; + } + }; + + for (auto & storePath : storePaths) + pool.enqueue(std::bind(doPath, storePath)); + + pool.process(); + + progressBar.done(); + + printMsg(lvlInfo, showProgress(true)); + + throw Exit( + (corrupted ? 1 : 0) | + (untrusted ? 2 : 0) | + (failed ? 4 : 0)); + } +}; + +struct CmdVerifyPaths : StorePathsCommand, MixVerify +{ + CmdVerifyPaths() + { + } + + std::string name() override + { + return "verify-paths"; + } + + std::string description() override + { + return "verify the integrity of store paths"; + } + + void run(ref<Store> store, Paths storePaths) override + { + verifyPaths(store, storePaths); + } +}; + +static RegisterCommand r1(make_ref<CmdVerifyPaths>()); + +struct CmdVerifyStore : StoreCommand, MixVerify +{ + CmdVerifyStore() + { + } + + std::string name() override + { + return "verify-store"; + } + + std::string description() override + { + return "verify the integrity of all paths in the Nix store"; + } + + void run(ref<Store> store) override + { + // FIXME: use store->verifyStore()? + + PathSet validPaths = store->queryAllValidPaths(); + + verifyPaths(store, Paths(validPaths.begin(), validPaths.end())); + } +}; + +static RegisterCommand r2(make_ref<CmdVerifyStore>()); |