#!/bin/bash
# Repeatedly try a command until it succeeds.
sleeptime=1
max=256
verbose=false
while [[ $# -gt 0 ]]; do
    case "$1" in
        -s) sleeptime=$2; shift; shift;;
        -m) max=$2; shift; shift;;
        -v) verbose=true; shift;;
        -*) cat <<EOF; exit 1;;
Usage: $(basename "$0") [-s initialsleep] [-m maxsleep] [-v] command

Repeatedly runs command until it is successful, exponentially
increasing the time between retries from initialsleep (currently
$sleeptime) up to maxsleep (currently $max).

This is especially useful with long-running but eventually convergent
commands like \`rsync\`.

EOF
        *) break
    esac
done

while true; do
    $verbose && echo "$0: $@"
    # Is there a way to block SIGINT, but just for the shell script,
    # and just while this command is executing?  `trap '' INT` blocks
    # it for the command too!
    "$@" && exit 1
    $verbose && echo $0: sleep "$sleeptime"
    sleep "$sleeptime"
    sleeptime=$(($sleeptime*2))
    if [[ "$sleeptime" -gt "$max" ]]; then
        sleeptime=$max
    fi
done