Retry fix (#686)
This commit is contained in:
Glenn Jocher
@@ -16,16 +16,18 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Cleanup any leftover test files
|
||||
run: rm -f /tmp/retry_attempt /tmp/retry_attempt_py
|
||||
|
||||
- name: Test successful command
|
||||
uses: ultralytics/actions/retry@main
|
||||
uses: ./retry
|
||||
with:
|
||||
run: |
|
||||
echo "This should succeed first try"
|
||||
true
|
||||
|
||||
- name: Test retry with eventual success
|
||||
uses: ultralytics/actions/retry@main
|
||||
continue-on-error: true
|
||||
uses: ./retry
|
||||
env:
|
||||
TEST_VAR: "hello"
|
||||
with:
|
||||
@@ -33,10 +35,28 @@ jobs:
|
||||
retry_delay_seconds: 5
|
||||
run: |
|
||||
echo "Running attempt with TEST_VAR=$TEST_VAR"
|
||||
exit 1
|
||||
# Validate env var is passed correctly
|
||||
if [ "$TEST_VAR" != "hello" ]; then
|
||||
echo "ERROR: TEST_VAR not passed correctly, got '$TEST_VAR'"
|
||||
exit 1
|
||||
fi
|
||||
# Fail first two times, succeed on third attempt
|
||||
if [ ! -f /tmp/retry_attempt ]; then
|
||||
echo "1" > /tmp/retry_attempt
|
||||
echo "First attempt - failing"
|
||||
exit 1
|
||||
fi
|
||||
count=$(cat /tmp/retry_attempt)
|
||||
if [ "$count" -lt 2 ]; then
|
||||
echo "$((count + 1))" > /tmp/retry_attempt
|
||||
echo "Attempt $((count + 1)) - failing"
|
||||
exit 1
|
||||
fi
|
||||
rm -f /tmp/retry_attempt
|
||||
echo "Success on attempt $((count + 1))!"
|
||||
|
||||
- name: Test multi-line failure
|
||||
uses: ultralytics/actions/retry@main
|
||||
uses: ./retry
|
||||
continue-on-error: true
|
||||
env:
|
||||
TEST_VAR: "hello"
|
||||
@@ -53,16 +73,18 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Cleanup any leftover test files
|
||||
run: rm -f /tmp/retry_attempt /tmp/retry_attempt_py
|
||||
|
||||
- name: Test successful Python command
|
||||
uses: ultralytics/actions/retry@main
|
||||
uses: ./retry
|
||||
with:
|
||||
shell: python
|
||||
run: |
|
||||
print("This should succeed first try")
|
||||
|
||||
- name: Test Python retry with eventual success
|
||||
uses: ultralytics/actions/retry@main
|
||||
continue-on-error: true
|
||||
uses: ./retry
|
||||
env:
|
||||
TEST_VAR: "hello"
|
||||
with:
|
||||
@@ -71,11 +93,27 @@ jobs:
|
||||
retry_delay_seconds: 5
|
||||
run: |
|
||||
import os
|
||||
from pathlib import Path
|
||||
print(f"Running attempt with TEST_VAR={os.environ.get('TEST_VAR')}")
|
||||
raise Exception("Forced failure")
|
||||
# Validate env var is passed correctly
|
||||
if os.environ.get('TEST_VAR') != 'hello':
|
||||
raise Exception(f"TEST_VAR not passed correctly, got '{os.environ.get('TEST_VAR')}'")
|
||||
# Fail first two times, succeed on third attempt
|
||||
counter_file = Path("/tmp/retry_attempt_py")
|
||||
if not counter_file.exists():
|
||||
counter_file.write_text("1")
|
||||
print("First attempt - failing")
|
||||
raise Exception("Intentional failure")
|
||||
count = int(counter_file.read_text())
|
||||
if count < 2:
|
||||
counter_file.write_text(str(count + 1))
|
||||
print(f"Attempt {count + 1} - failing")
|
||||
raise Exception("Intentional failure")
|
||||
counter_file.unlink()
|
||||
print(f"Success on attempt {count + 1}!")
|
||||
|
||||
- name: Test Python multi-line failure
|
||||
uses: ultralytics/actions/retry@main
|
||||
uses: ./retry
|
||||
continue-on-error: true
|
||||
env:
|
||||
TEST_VAR: "hello"
|
||||
|
||||
+76
-91
@@ -16,7 +16,7 @@
|
||||
# python setup.py install
|
||||
# pytest tests/
|
||||
# retries: 2 # Retry twice after initial attempt (3 total runs)
|
||||
# timeout_minutes: 30 # Each attempt times out after 30 minutes
|
||||
# timeout_minutes: 30 # Total timeout for all attempts combined
|
||||
# retry_delay_seconds: 60 # Wait 60 seconds between retries
|
||||
# shell: bash # Use python or bash shell
|
||||
|
||||
@@ -24,7 +24,7 @@ name: "Step-Level Retry"
|
||||
description: "Retries a step while preserving its full context"
|
||||
inputs:
|
||||
timeout_minutes:
|
||||
description: "Maximum time in minutes for each attempt"
|
||||
description: "Maximum total time in minutes for all attempts"
|
||||
required: false
|
||||
default: "360"
|
||||
retries:
|
||||
@@ -46,113 +46,98 @@ inputs:
|
||||
runs:
|
||||
using: "composite"
|
||||
steps:
|
||||
- name: Execute with retry (Python)
|
||||
if: inputs.shell == 'python'
|
||||
shell: python
|
||||
env: ${{ env }}
|
||||
run: |
|
||||
import time
|
||||
import sys
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
start_time = time.time()
|
||||
timeout_seconds = ${{ inputs.timeout_minutes }} * 60
|
||||
attempt = 1
|
||||
max_attempts = 1 + ${{ inputs.retries }} # Initial run + retries
|
||||
|
||||
# Create temporary script file
|
||||
temp_path = Path(tempfile.gettempdir()) / 'retry_script.py'
|
||||
temp_path.write_text("""${{ inputs.run }}""")
|
||||
|
||||
while attempt <= max_attempts:
|
||||
if attempt > 1:
|
||||
print(f"::group::Retry {attempt-1} of ${{ inputs.retries }}")
|
||||
|
||||
if (time.time() - start_time) > timeout_seconds:
|
||||
print(f"::error::Step timed out after ${{ inputs.timeout_minutes }} minutes")
|
||||
temp_path.unlink(missing_ok=True)
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
exec(temp_path.read_text())
|
||||
if attempt > 1:
|
||||
print("::endgroup::")
|
||||
temp_path.unlink(missing_ok=True)
|
||||
sys.exit(0)
|
||||
except Exception as e:
|
||||
if attempt > 1:
|
||||
print(f"Retry {attempt-1} failed with error: {str(e)}")
|
||||
print("::endgroup::")
|
||||
else:
|
||||
print(f"Initial attempt failed with error: {str(e)}")
|
||||
|
||||
if attempt == max_attempts:
|
||||
print(f"::error::Step failed after initial attempt and {attempt-1} retries")
|
||||
temp_path.unlink(missing_ok=True)
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Retrying in ${{ inputs.retry_delay_seconds }} seconds...")
|
||||
time.sleep(${{ inputs.retry_delay_seconds }})
|
||||
attempt += 1
|
||||
|
||||
- name: Execute with retry (Bash)
|
||||
if: inputs.shell != 'python'
|
||||
- name: Execute with retry
|
||||
shell: bash
|
||||
env: ${{ env }}
|
||||
env:
|
||||
# Note: Workflow env vars are automatically inherited. Only RETRY_COMMANDS is added here.
|
||||
RETRY_COMMANDS: ${{ inputs.run }}
|
||||
run: |
|
||||
set +e
|
||||
set +e # Don't exit on error - we handle errors manually
|
||||
|
||||
start_time=$(date +%s)
|
||||
timeout_seconds=$(( ${{ inputs.timeout_minutes }} * 60 ))
|
||||
attempt=1
|
||||
max_attempts=$(( 1 + ${{ inputs.retries }} )) # Initial run + retries
|
||||
retries=${{ inputs.retries }}
|
||||
retry_delay=${{ inputs.retry_delay_seconds }}
|
||||
timeout_mins=${{ inputs.timeout_minutes }}
|
||||
shell_type="${{ inputs.shell }}"
|
||||
exit_code=0
|
||||
|
||||
# Create a temporary script file with proper shebang
|
||||
TEMP_SCRIPT=$(mktemp)
|
||||
cat << 'EOF' > "$TEMP_SCRIPT"
|
||||
#!/bin/bash
|
||||
set -e
|
||||
${{ inputs.run }}
|
||||
EOF
|
||||
chmod +x "$TEMP_SCRIPT"
|
||||
|
||||
while [ $attempt -le $max_attempts ]; do
|
||||
if [ $attempt -gt 1 ]; then
|
||||
echo "::group::Retry $((attempt-1)) of ${{ inputs.retries }}"
|
||||
# Create temporary script file with appropriate extension
|
||||
if [ "$shell_type" = "python" ]; then
|
||||
if ! TEMP_SCRIPT=$(mktemp --suffix=.py 2>/dev/null); then
|
||||
# Fallback: create temp file then rename it
|
||||
if ! TEMP_SCRIPT=$(mktemp 2>/dev/null); then
|
||||
echo "::error::Failed to create temporary file"
|
||||
exit 1
|
||||
fi
|
||||
if ! mv "$TEMP_SCRIPT" "${TEMP_SCRIPT}.py"; then
|
||||
rm -f "$TEMP_SCRIPT"
|
||||
echo "::error::Failed to rename temporary file"
|
||||
exit 1
|
||||
fi
|
||||
TEMP_SCRIPT="${TEMP_SCRIPT}.py"
|
||||
fi
|
||||
|
||||
current_time=$(date +%s)
|
||||
if [ $((current_time - start_time)) -gt $timeout_seconds ]; then
|
||||
echo "::error::Step timed out after ${{ inputs.timeout_minutes }} minutes"
|
||||
rm "$TEMP_SCRIPT"
|
||||
else
|
||||
if ! TEMP_SCRIPT=$(mktemp 2>/dev/null); then
|
||||
echo "::error::Failed to create temporary file"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Execute the temporary script
|
||||
"$TEMP_SCRIPT"
|
||||
fi
|
||||
|
||||
# Ensure cleanup on exit
|
||||
trap 'rm -f "$TEMP_SCRIPT"' EXIT
|
||||
|
||||
# Write the user's commands to the temp script
|
||||
printf '%s\n' "$RETRY_COMMANDS" > "$TEMP_SCRIPT"
|
||||
|
||||
# Validate script was written
|
||||
if [ ! -s "$TEMP_SCRIPT" ]; then
|
||||
echo "::error::No commands provided to execute"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
while [ "$attempt" -le "$max_attempts" ]; do
|
||||
if [ "$attempt" -gt 1 ]; then
|
||||
echo "::group::Retry $((attempt - 1)) of $retries"
|
||||
fi
|
||||
|
||||
current_time=$(date +%s)
|
||||
elapsed=$((current_time - start_time))
|
||||
if [ "$elapsed" -gt "$timeout_seconds" ]; then
|
||||
echo "::error::Step timed out after $timeout_mins minutes"
|
||||
[ "$attempt" -gt 1 ] && echo "::endgroup::"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Execute the script with appropriate interpreter
|
||||
if [ "$shell_type" = "python" ]; then
|
||||
python3 "$TEMP_SCRIPT"
|
||||
else
|
||||
bash -e "$TEMP_SCRIPT"
|
||||
fi
|
||||
exit_code=$?
|
||||
|
||||
if [ $exit_code -eq 0 ]; then
|
||||
[ $attempt -gt 1 ] && echo "::endgroup::"
|
||||
rm "$TEMP_SCRIPT"
|
||||
|
||||
if [ "$exit_code" -eq 0 ]; then
|
||||
[ "$attempt" -gt 1 ] && echo "::endgroup::"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ $attempt -gt 1 ]; then
|
||||
echo "Retry $((attempt-1)) failed with exit code $exit_code"
|
||||
|
||||
# Handle failure
|
||||
if [ "$attempt" -gt 1 ]; then
|
||||
echo "Retry $((attempt - 1)) failed with exit code $exit_code"
|
||||
echo "::endgroup::"
|
||||
else
|
||||
echo "Initial attempt failed with exit code $exit_code"
|
||||
fi
|
||||
|
||||
if [ $attempt -eq $max_attempts ]; then
|
||||
echo "::error::Step failed after initial attempt and $((attempt-1)) retries"
|
||||
rm "$TEMP_SCRIPT"
|
||||
exit $exit_code
|
||||
|
||||
if [ "$attempt" -eq "$max_attempts" ]; then
|
||||
echo "::error::Step failed after initial attempt and $((attempt - 1)) retries"
|
||||
exit "$exit_code"
|
||||
fi
|
||||
|
||||
echo "Retrying in ${{ inputs.retry_delay_seconds }} seconds..."
|
||||
sleep ${{ inputs.retry_delay_seconds }}
|
||||
|
||||
echo "Retrying in $retry_delay seconds..."
|
||||
sleep "$retry_delay"
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
|
||||
Reference in New Issue
Block a user