HttpBinaryCacheStore: Retry on transient HTTP errors

This makes us more robust against 500 errors from CloudFront or S3
(assuming the 500 error isn't cached by CloudFront...).

1 files changed, 26 insertions, 9 deletions

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index ce5584188a..95c7d2255a 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -8,6 +8,7 @@
 #include <curl/curl.h>
 
 #include <iostream>
+#include <thread>
 
 
 namespace nix {
@@ -194,7 +195,11 @@ struct CurlDownloader : public Downloader
         if (res != CURLE_OK) {
             Error err =
                 httpStatus == 404 ? NotFound :
-                httpStatus == 403 ? Forbidden : Misc;
+                httpStatus == 403 ? Forbidden :
+                (httpStatus == 408 || httpStatus == 500 || httpStatus == 503
+                 || httpStatus == 504  || httpStatus == 522 || httpStatus == 524
+                 || res == CURLE_COULDNT_RESOLVE_HOST) ? Transient :
+                Misc;
             if (res == CURLE_HTTP_RETURNED_ERROR && httpStatus != -1)
                 throw DownloadError(err, format("unable to download ‘%s’: HTTP error %d")
                     % url % httpStatus);
@@ -210,14 +215,26 @@ struct CurlDownloader : public Downloader
 
     DownloadResult download(string url, const DownloadOptions & options) override
     {
-        DownloadResult res;
-        if (fetch(resolveUri(url), options)) {
-            res.cached = false;
-            res.data = data;
-        } else
-            res.cached = true;
-        res.etag = etag;
-        return res;
+        size_t attempt = 0;
+
+        while (true) {
+            try {
+                DownloadResult res;
+                if (fetch(resolveUri(url), options)) {
+                    res.cached = false;
+                    res.data = data;
+                } else
+                    res.cached = true;
+                res.etag = etag;
+                return res;
+            } catch (DownloadError & e) {
+                attempt++;
+                if (e.error != Transient || attempt >= options.tries) throw;
+                auto ms = 25 * (1 << (attempt - 1));
+                printMsg(lvlError, format("warning: %s; retrying in %d ms") % e.what() % ms);
+                std::this_thread::sleep_for(std::chrono::milliseconds(ms));
+            }
+        }
     }
 };