Detect stack overflows

Previously, if the Nix evaluator gets a stack overflow due to a deep
or infinite recursion in the Nix expression, the user gets an
unhelpful message ("Segmentation fault") that doesn't indicate that
the problem is in the user's code rather than Nix itself.  Now it
prints:

  error: stack overflow (possible infinite recursion)

This only works on x86_64-linux and i686-linux.

Fixes #35.

3 files changed, 80 insertions, 1 deletions

diff --git a/src/libmain/Makefile.am b/src/libmain/Makefile.am
index 6a2d7f5f58e5..75b9d83dcf73 100644
--- a/src/libmain/Makefile.am
+++ b/src/libmain/Makefile.am
@@ -1,6 +1,6 @@
 pkglib_LTLIBRARIES = libmain.la
 
-libmain_la_SOURCES = shared.cc
+libmain_la_SOURCES = shared.cc stack.cc
 
 libmain_la_LIBADD = ../libstore/libstore.la @BDW_GC_LIBS@
 
diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 4796629dc48c..ee0dccc9d11d 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -13,6 +13,7 @@
 #include <sys/time.h>
 #include <sys/stat.h>
 #include <unistd.h>
+#include <signal.h>
 
 #if HAVE_BOEHMGC
 #include <gc/gc.h>
@@ -100,6 +101,9 @@ string getArg(const string & opt,
 }
 
 
+void detectStackOverflow();
+
+
 /* Initialize and reorder arguments, then call the actual argument
    processor. */
 static void initAndRun(int argc, char * * argv)
@@ -131,6 +135,9 @@ static void initAndRun(int argc, char * * argv)
     if (sigaction(SIGCHLD, &act, 0))
         throw SysError("resetting SIGCHLD");
 
+    /* Register a SIGSEGV handler to detect stack overflows. */
+    detectStackOverflow();
+
     /* There is no privacy in the Nix system ;-)  At least not for
        now.  In particular, store objects should be readable by
        everybody. */
diff --git a/src/libmain/stack.cc b/src/libmain/stack.cc
new file mode 100644
index 000000000000..0ea80f18e3b0
--- /dev/null
+++ b/src/libmain/stack.cc
@@ -0,0 +1,72 @@
+#include "config.h"
+
+#include "types.hh"
+
+#include <cstring>
+#include <cstddef>
+#include <cstdlib>
+
+#include <unistd.h>
+#include <signal.h>
+
+namespace nix {
+
+
+static void sigsegvHandler(int signo, siginfo_t * info, void * ctx)
+{
+    /* Detect stack overflows by comparing the faulting address with
+       the stack pointer.  Unfortunately, getting the stack pointer is
+       not portable. */
+    bool haveSP = true;
+    char * sp;
+#if defined(__x86_64__) && defined(REG_RSP)
+    sp = (char *) ((ucontext *) ctx)->uc_mcontext.gregs[REG_RSP];
+#elif defined(REG_ESP)
+    sp = (char *) ((ucontext *) ctx)->uc_mcontext.gregs[REG_ESP];
+#else
+    haveSP = false;
+#endif
+
+    if (haveSP) {
+        ptrdiff_t diff = (char *) info->si_addr - sp;
+        if (diff < 0) diff = -diff;
+        if (diff < 4096) {
+            char msg[] = "error: stack overflow (possible infinite recursion)\n";
+            write(2, msg, strlen(msg));
+            _exit(1); // maybe abort instead?
+        }
+    }
+
+    /* Restore default behaviour (i.e. segfault and dump core). */
+    struct sigaction act;
+    sigfillset(&act.sa_mask);
+    act.sa_handler = SIG_DFL;
+    act.sa_flags = 0;
+    if (sigaction(SIGSEGV, &act, 0)) abort();
+}
+
+
+void detectStackOverflow()
+{
+#if defined(SA_SIGINFO) && defined (SA_ONSTACK)
+    /* Install a SIGSEGV handler to detect stack overflows.  This
+       requires an alternative stack, otherwise the signal cannot be
+       delivered when we're out of stack space. */
+    stack_t stack;
+    stack.ss_size = 4096 * 4;
+    stack.ss_sp = new char[stack.ss_size];
+    if (!stack.ss_sp) throw Error("cannot allocate alternative stack");
+    stack.ss_flags = 0;
+    if (sigaltstack(&stack, 0) == -1) throw SysError("cannot set alternative stack");
+
+    struct sigaction act;
+    sigfillset(&act.sa_mask);
+    act.sa_sigaction = sigsegvHandler;
+    act.sa_flags = SA_SIGINFO | SA_ONSTACK;
+    if (sigaction(SIGSEGV, &act, 0))
+        throw SysError("resetting SIGCHLD");
+#endif
+}
+
+
+}