Default to exponential backoff retry (#741)
Co-authored-by: UltralyticsAssistant <[email protected]>
This commit is contained in:
Glenn Jocher
co-authored by
UltralyticsAssistant
UltralyticsAssistant
parent
2de6732fb8
commit
000482b900
@@ -68,6 +68,116 @@ jobs:
|
||||
echo "Running line 2 with TEST_VAR=$TEST_VAR"
|
||||
false
|
||||
|
||||
- name: Record start (exponential)
|
||||
id: start_exp
|
||||
run: echo "t=$(date +%s)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Test exponential backoff timing
|
||||
uses: ./retry
|
||||
continue-on-error: true
|
||||
with:
|
||||
retries: 3
|
||||
retry_delay_seconds: 2
|
||||
backoff: exponential
|
||||
jitter: false
|
||||
run: |
|
||||
# Waits should be 2s, 4s, 8s. Total ~14s between 4 attempts.
|
||||
echo "Attempt at $(date +%s)"
|
||||
false
|
||||
|
||||
- name: Assert exponential timing (~14s expected)
|
||||
run: |
|
||||
elapsed=$(( $(date +%s) - ${{ steps.start_exp.outputs.t }} ))
|
||||
# Expected waits: 2 + 4 + 8 = 14s (no jitter). Allow generous upper bound for runner overhead.
|
||||
if [ "$elapsed" -lt 13 ] || [ "$elapsed" -gt 40 ]; then
|
||||
echo "::error::Exponential timing off: ${elapsed}s, expected ~14s (13-40s range)"
|
||||
exit 1
|
||||
fi
|
||||
echo "OK: ${elapsed}s elapsed (expected ~14s)"
|
||||
|
||||
- name: Record start (fixed)
|
||||
id: start_fixed
|
||||
run: echo "t=$(date +%s)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Test fixed backoff
|
||||
uses: ./retry
|
||||
continue-on-error: true
|
||||
with:
|
||||
retries: 2
|
||||
retry_delay_seconds: 3
|
||||
backoff: fixed
|
||||
jitter: false
|
||||
run: |
|
||||
echo "Attempt at $(date +%s)"
|
||||
false
|
||||
|
||||
- name: Assert fixed timing (~6s expected)
|
||||
run: |
|
||||
elapsed=$(( $(date +%s) - ${{ steps.start_fixed.outputs.t }} ))
|
||||
# Expected waits: 3 + 3 = 6s (no jitter). Allow runner overhead.
|
||||
if [ "$elapsed" -lt 6 ] || [ "$elapsed" -gt 25 ]; then
|
||||
echo "::error::Fixed timing off: ${elapsed}s, expected ~6s (6-25s range)"
|
||||
exit 1
|
||||
fi
|
||||
echo "OK: ${elapsed}s elapsed (expected ~6s)"
|
||||
|
||||
- name: Record start (jitter floor)
|
||||
id: start_jitter
|
||||
run: echo "t=$(date +%s)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Test jitter never collapses small delay to 0
|
||||
uses: ./retry
|
||||
continue-on-error: true
|
||||
with:
|
||||
retries: 4
|
||||
retry_delay_seconds: 1
|
||||
backoff: fixed
|
||||
jitter: true
|
||||
run: |
|
||||
echo "Attempt at $(date +%s)"
|
||||
false
|
||||
|
||||
- name: Assert jitter floor (>=4s for 4 retries of >=1s each)
|
||||
run: |
|
||||
elapsed=$(( $(date +%s) - ${{ steps.start_jitter.outputs.t }} ))
|
||||
# Without the floor, jitter would round 1s to 0s and the 4 retries would run instantly.
|
||||
if [ "$elapsed" -lt 4 ]; then
|
||||
echo "::error::Jitter collapsed small delays: ${elapsed}s elapsed, expected >=4s"
|
||||
exit 1
|
||||
fi
|
||||
echo "OK: ${elapsed}s elapsed (>=4s confirmed)"
|
||||
|
||||
- name: Test invalid backoff value rejected
|
||||
uses: ./retry
|
||||
continue-on-error: true
|
||||
id: invalid_backoff
|
||||
with:
|
||||
retries: 0
|
||||
backoff: linear
|
||||
run: echo "should not run"
|
||||
|
||||
- name: Assert invalid backoff failed
|
||||
run: |
|
||||
if [ "${{ steps.invalid_backoff.outcome }}" != "failure" ]; then
|
||||
echo "::error::Expected invalid backoff to fail, got ${{ steps.invalid_backoff.outcome }}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Test invalid numeric input rejected
|
||||
uses: ./retry
|
||||
continue-on-error: true
|
||||
id: invalid_retries
|
||||
with:
|
||||
retries: "-1"
|
||||
run: echo "should not run"
|
||||
|
||||
- name: Assert invalid numeric input failed
|
||||
run: |
|
||||
if [ "${{ steps.invalid_retries.outcome }}" != "failure" ]; then
|
||||
echo "::error::Expected invalid retries to fail, got ${{ steps.invalid_retries.outcome }}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
test-python:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
|
||||
@@ -115,13 +115,13 @@ Reusable composite actions for common CI/CD tasks. Each can be used independentl
|
||||
|
||||
### 1. Retry Action
|
||||
|
||||
Retry failed commands with exponential backoff.
|
||||
Retry failed commands with exponential backoff and jitter.
|
||||
|
||||
```yaml
|
||||
- uses: ultralytics/actions/retry@main
|
||||
with:
|
||||
command: npm install
|
||||
max_attempts: 3
|
||||
run: npm install
|
||||
retries: 3
|
||||
timeout_minutes: 5
|
||||
```
|
||||
|
||||
|
||||
+16
-12
@@ -19,7 +19,7 @@ steps:
|
||||
|
||||
### Advanced Usage
|
||||
|
||||
Full configuration with custom retries, timeout, and delay:
|
||||
Full configuration with custom retries, timeout, backoff, and jitter:
|
||||
|
||||
```yaml
|
||||
steps:
|
||||
@@ -29,8 +29,10 @@ steps:
|
||||
python setup.py install
|
||||
pytest tests/
|
||||
retries: 2 # Retry twice after initial attempt (3 total runs)
|
||||
timeout_minutes: 30 # Each attempt times out after 30 minutes
|
||||
retry_delay_seconds: 60 # Wait 60 seconds between retries
|
||||
timeout_minutes: 30 # Total timeout across all attempts
|
||||
retry_delay_seconds: 10 # Base delay between retries
|
||||
backoff: exponential # exponential (10s, 20s, 40s, ...) or fixed
|
||||
jitter: true # Randomize delay to 80-120% to avoid thundering herd
|
||||
shell: bash # Use python or bash shell
|
||||
```
|
||||
|
||||
@@ -50,18 +52,20 @@ steps:
|
||||
|
||||
## 📋 Inputs
|
||||
|
||||
| Input | Description | Required | Default |
|
||||
| --------------------- | ---------------------------------------- | -------- | ------- |
|
||||
| `run` | Command to run | Yes | - |
|
||||
| `retries` | Number of retry attempts after initial | No | `3` |
|
||||
| `timeout_minutes` | Maximum time in minutes for each attempt | No | `60` |
|
||||
| `retry_delay_seconds` | Delay between retries in seconds | No | `30` |
|
||||
| `shell` | Shell to use (bash or python) | No | `bash` |
|
||||
| Input | Description | Required | Default |
|
||||
| --------------------- | ------------------------------------------------------------ | -------- | ------------- |
|
||||
| `run` | Command to run | Yes | - |
|
||||
| `retries` | Number of retry attempts after initial run | No | `3` |
|
||||
| `timeout_minutes` | Maximum total time in minutes for all attempts combined | No | `360` |
|
||||
| `retry_delay_seconds` | Base delay between retries in seconds | No | `10` |
|
||||
| `backoff` | Backoff strategy: `exponential` (base \* 2^n) or `fixed` | No | `exponential` |
|
||||
| `jitter` | Randomize delay to 80-120% of value to avoid thundering herd | No | `true` |
|
||||
| `shell` | Shell to use (`bash` or `python`) | No | `bash` |
|
||||
|
||||
## ✨ Features
|
||||
|
||||
- Preserves environment variables and step context
|
||||
- Configurable timeout per attempt
|
||||
- Exponential backoff with retry delay
|
||||
- Exponential backoff with ±20% jitter (best-practice defaults)
|
||||
- Configurable total timeout across all attempts (individual sleeps auto-capped to remaining budget)
|
||||
- GitHub Actions grouping for retry attempts
|
||||
- Supports both Bash and Python shells
|
||||
|
||||
+53
-8
@@ -15,10 +15,12 @@
|
||||
# run: |
|
||||
# python setup.py install
|
||||
# pytest tests/
|
||||
# retries: 2 # Retry twice after initial attempt (3 total runs)
|
||||
# timeout_minutes: 30 # Total timeout for all attempts combined
|
||||
# retry_delay_seconds: 60 # Wait 60 seconds between retries
|
||||
# shell: bash # Use python or bash shell
|
||||
# retries: 2 # Retry twice after initial attempt (3 total runs)
|
||||
# timeout_minutes: 30 # Total timeout for all attempts combined
|
||||
# retry_delay_seconds: 10 # Base delay between retries in seconds
|
||||
# backoff: exponential # exponential (10s, 20s, 40s, ...) or fixed
|
||||
# jitter: true # Randomize delay to 80-120% to avoid thundering herd
|
||||
# shell: bash # Use python or bash shell
|
||||
|
||||
name: "Step-Level Retry"
|
||||
description: "Retries a step while preserving its full context"
|
||||
@@ -32,9 +34,17 @@ inputs:
|
||||
required: false
|
||||
default: "3"
|
||||
retry_delay_seconds:
|
||||
description: "Delay between retries in seconds"
|
||||
description: "Base delay between retries in seconds"
|
||||
required: false
|
||||
default: "30"
|
||||
default: "10"
|
||||
backoff:
|
||||
description: "Backoff strategy: exponential (base * 2^n) or fixed. Delays are bounded by timeout_minutes."
|
||||
required: false
|
||||
default: "exponential"
|
||||
jitter:
|
||||
description: "Randomize delay to 80-120% of computed value to spread retries and avoid thundering herd"
|
||||
required: false
|
||||
default: "true"
|
||||
run:
|
||||
description: "Command to run"
|
||||
required: true
|
||||
@@ -60,10 +70,23 @@ runs:
|
||||
max_attempts=$(( 1 + ${{ inputs.retries }} )) # Initial run + retries
|
||||
retries=${{ inputs.retries }}
|
||||
retry_delay=${{ inputs.retry_delay_seconds }}
|
||||
backoff="${{ inputs.backoff }}"
|
||||
jitter="${{ inputs.jitter }}"
|
||||
timeout_mins=${{ inputs.timeout_minutes }}
|
||||
shell_type="${{ inputs.shell }}"
|
||||
exit_code=0
|
||||
|
||||
# Validate backoff strategy
|
||||
if [ "$backoff" != "exponential" ] && [ "$backoff" != "fixed" ]; then
|
||||
echo "::error::Invalid backoff '$backoff'; must be 'exponential' or 'fixed'"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Validate numeric inputs are non-negative integers
|
||||
case "$retries$retry_delay$timeout_mins" in
|
||||
''|*[!0-9]*) echo "::error::retries, retry_delay_seconds, timeout_minutes must be non-negative integers"; exit 1 ;;
|
||||
esac
|
||||
|
||||
# Create temporary script file with appropriate extension
|
||||
if [ "$shell_type" = "python" ]; then
|
||||
if ! TEMP_SCRIPT=$(mktemp --suffix=.py 2>/dev/null); then
|
||||
@@ -137,7 +160,29 @@ runs:
|
||||
exit "$exit_code"
|
||||
fi
|
||||
|
||||
echo "Retrying in $retry_delay seconds..."
|
||||
sleep "$retry_delay"
|
||||
# Compute delay for the upcoming retry
|
||||
if [ "$backoff" = "exponential" ]; then
|
||||
delay=$((retry_delay * (2 ** (attempt - 1))))
|
||||
else
|
||||
delay=$retry_delay
|
||||
fi
|
||||
|
||||
# Apply jitter: random 80-120% of computed delay (±20%)
|
||||
if [ "$jitter" = "true" ] && [ "$delay" -gt 0 ]; then
|
||||
low=$((delay * 8 / 10))
|
||||
span=$((delay * 4 / 10))
|
||||
rand=$((RANDOM % (span + 1)))
|
||||
delay=$((low + rand))
|
||||
[ "$delay" -lt 1 ] && delay=1 # integer division floor; never collapse to 0
|
||||
fi
|
||||
|
||||
# Cap sleep at remaining timeout budget; no point sleeping past deadline.
|
||||
# Refresh timestamp — the attempt may have taken significant time.
|
||||
remaining=$((timeout_seconds - ($(date +%s) - start_time)))
|
||||
[ "$delay" -gt "$remaining" ] && delay=$remaining
|
||||
[ "$delay" -lt 0 ] && delay=0
|
||||
|
||||
echo "Retrying in $delay seconds..."
|
||||
sleep "$delay"
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
|
||||
Reference in New Issue
Block a user