about summary refs log tree commit diff
path: root/ci/retry-command.sh
blob: 190bce4817121e3348342c8f0853180643c53367 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
#!/usr/bin/env bash
# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -eu

# Make three attempts to install the dependencies. It is rare, but from time to
# time the downloading the packages, building the Docker images, or an installer
# Bazel, or the Google Cloud SDK fails. To make the CI build more robust, try
# again when that happens.

if (( $# < 1 )); then
  echo "Usage: $(basename "$0") program [arguments]"
  exit 1
fi
readonly PROGRAM=${1}
shift

# Initially, wait at least 2 minutes (the times are in seconds), because it
# makes no sense to try faster. This used to be 180 seconds, but that ends with
# sleeps close to 10 minutes, and Travis aborts builds that do not output in
# 10m.
min_wait=120
# Do not exit on failures for this loop.
set +e
for i in 1 2 3; do
  if "${PROGRAM}" "$@"; then
    exit 0
  fi
  # Sleep for a few minutes before trying again.
  period=$(( (RANDOM % 60) + min_wait ))
  echo "${PROGRAM} failed (attempt ${i}); trying again in ${period} seconds."
  sleep ${period}s
  min_wait=$(( min_wait * 2 ))
done

exit 1