From 216add1ba67b741caa2c56854c505e6b14e1acff Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 14 Dec 2025 23:13:43 +0000 Subject: [PATCH] Retry fix (#686) --- .github/workflows/test-retry.yml | 58 +++++++++-- retry/action.yml | 167 ++++++++++++++----------------- 2 files changed, 124 insertions(+), 101 deletions(-) diff --git a/.github/workflows/test-retry.yml b/.github/workflows/test-retry.yml index 862207d..0330953 100644 --- a/.github/workflows/test-retry.yml +++ b/.github/workflows/test-retry.yml @@ -16,16 +16,18 @@ jobs: steps: - uses: actions/checkout@v6 + - name: Cleanup any leftover test files + run: rm -f /tmp/retry_attempt /tmp/retry_attempt_py + - name: Test successful command - uses: ultralytics/actions/retry@main + uses: ./retry with: run: | echo "This should succeed first try" true - name: Test retry with eventual success - uses: ultralytics/actions/retry@main - continue-on-error: true + uses: ./retry env: TEST_VAR: "hello" with: @@ -33,10 +35,28 @@ jobs: retry_delay_seconds: 5 run: | echo "Running attempt with TEST_VAR=$TEST_VAR" - exit 1 + # Validate env var is passed correctly + if [ "$TEST_VAR" != "hello" ]; then + echo "ERROR: TEST_VAR not passed correctly, got '$TEST_VAR'" + exit 1 + fi + # Fail first two times, succeed on third attempt + if [ ! -f /tmp/retry_attempt ]; then + echo "1" > /tmp/retry_attempt + echo "First attempt - failing" + exit 1 + fi + count=$(cat /tmp/retry_attempt) + if [ "$count" -lt 2 ]; then + echo "$((count + 1))" > /tmp/retry_attempt + echo "Attempt $((count + 1)) - failing" + exit 1 + fi + rm -f /tmp/retry_attempt + echo "Success on attempt $((count + 1))!" - name: Test multi-line failure - uses: ultralytics/actions/retry@main + uses: ./retry continue-on-error: true env: TEST_VAR: "hello" @@ -53,16 +73,18 @@ jobs: steps: - uses: actions/checkout@v6 + - name: Cleanup any leftover test files + run: rm -f /tmp/retry_attempt /tmp/retry_attempt_py + - name: Test successful Python command - uses: ultralytics/actions/retry@main + uses: ./retry with: shell: python run: | print("This should succeed first try") - name: Test Python retry with eventual success - uses: ultralytics/actions/retry@main - continue-on-error: true + uses: ./retry env: TEST_VAR: "hello" with: @@ -71,11 +93,27 @@ jobs: retry_delay_seconds: 5 run: | import os + from pathlib import Path print(f"Running attempt with TEST_VAR={os.environ.get('TEST_VAR')}") - raise Exception("Forced failure") + # Validate env var is passed correctly + if os.environ.get('TEST_VAR') != 'hello': + raise Exception(f"TEST_VAR not passed correctly, got '{os.environ.get('TEST_VAR')}'") + # Fail first two times, succeed on third attempt + counter_file = Path("/tmp/retry_attempt_py") + if not counter_file.exists(): + counter_file.write_text("1") + print("First attempt - failing") + raise Exception("Intentional failure") + count = int(counter_file.read_text()) + if count < 2: + counter_file.write_text(str(count + 1)) + print(f"Attempt {count + 1} - failing") + raise Exception("Intentional failure") + counter_file.unlink() + print(f"Success on attempt {count + 1}!") - name: Test Python multi-line failure - uses: ultralytics/actions/retry@main + uses: ./retry continue-on-error: true env: TEST_VAR: "hello" diff --git a/retry/action.yml b/retry/action.yml index b6351c4..142eafb 100644 --- a/retry/action.yml +++ b/retry/action.yml @@ -16,7 +16,7 @@ # python setup.py install # pytest tests/ # retries: 2 # Retry twice after initial attempt (3 total runs) -# timeout_minutes: 30 # Each attempt times out after 30 minutes +# timeout_minutes: 30 # Total timeout for all attempts combined # retry_delay_seconds: 60 # Wait 60 seconds between retries # shell: bash # Use python or bash shell @@ -24,7 +24,7 @@ name: "Step-Level Retry" description: "Retries a step while preserving its full context" inputs: timeout_minutes: - description: "Maximum time in minutes for each attempt" + description: "Maximum total time in minutes for all attempts" required: false default: "360" retries: @@ -46,113 +46,98 @@ inputs: runs: using: "composite" steps: - - name: Execute with retry (Python) - if: inputs.shell == 'python' - shell: python - env: ${{ env }} - run: | - import time - import sys - from pathlib import Path - import tempfile - - start_time = time.time() - timeout_seconds = ${{ inputs.timeout_minutes }} * 60 - attempt = 1 - max_attempts = 1 + ${{ inputs.retries }} # Initial run + retries - - # Create temporary script file - temp_path = Path(tempfile.gettempdir()) / 'retry_script.py' - temp_path.write_text("""${{ inputs.run }}""") - - while attempt <= max_attempts: - if attempt > 1: - print(f"::group::Retry {attempt-1} of ${{ inputs.retries }}") - - if (time.time() - start_time) > timeout_seconds: - print(f"::error::Step timed out after ${{ inputs.timeout_minutes }} minutes") - temp_path.unlink(missing_ok=True) - sys.exit(1) - - try: - exec(temp_path.read_text()) - if attempt > 1: - print("::endgroup::") - temp_path.unlink(missing_ok=True) - sys.exit(0) - except Exception as e: - if attempt > 1: - print(f"Retry {attempt-1} failed with error: {str(e)}") - print("::endgroup::") - else: - print(f"Initial attempt failed with error: {str(e)}") - - if attempt == max_attempts: - print(f"::error::Step failed after initial attempt and {attempt-1} retries") - temp_path.unlink(missing_ok=True) - sys.exit(1) - - print(f"Retrying in ${{ inputs.retry_delay_seconds }} seconds...") - time.sleep(${{ inputs.retry_delay_seconds }}) - attempt += 1 - - - name: Execute with retry (Bash) - if: inputs.shell != 'python' + - name: Execute with retry shell: bash - env: ${{ env }} + env: + # Note: Workflow env vars are automatically inherited. Only RETRY_COMMANDS is added here. + RETRY_COMMANDS: ${{ inputs.run }} run: | - set +e + set +e # Don't exit on error - we handle errors manually start_time=$(date +%s) timeout_seconds=$(( ${{ inputs.timeout_minutes }} * 60 )) attempt=1 max_attempts=$(( 1 + ${{ inputs.retries }} )) # Initial run + retries + retries=${{ inputs.retries }} + retry_delay=${{ inputs.retry_delay_seconds }} + timeout_mins=${{ inputs.timeout_minutes }} + shell_type="${{ inputs.shell }}" + exit_code=0 - # Create a temporary script file with proper shebang - TEMP_SCRIPT=$(mktemp) - cat << 'EOF' > "$TEMP_SCRIPT" - #!/bin/bash - set -e - ${{ inputs.run }} - EOF - chmod +x "$TEMP_SCRIPT" - - while [ $attempt -le $max_attempts ]; do - if [ $attempt -gt 1 ]; then - echo "::group::Retry $((attempt-1)) of ${{ inputs.retries }}" + # Create temporary script file with appropriate extension + if [ "$shell_type" = "python" ]; then + if ! TEMP_SCRIPT=$(mktemp --suffix=.py 2>/dev/null); then + # Fallback: create temp file then rename it + if ! TEMP_SCRIPT=$(mktemp 2>/dev/null); then + echo "::error::Failed to create temporary file" + exit 1 + fi + if ! mv "$TEMP_SCRIPT" "${TEMP_SCRIPT}.py"; then + rm -f "$TEMP_SCRIPT" + echo "::error::Failed to rename temporary file" + exit 1 + fi + TEMP_SCRIPT="${TEMP_SCRIPT}.py" fi - - current_time=$(date +%s) - if [ $((current_time - start_time)) -gt $timeout_seconds ]; then - echo "::error::Step timed out after ${{ inputs.timeout_minutes }} minutes" - rm "$TEMP_SCRIPT" + else + if ! TEMP_SCRIPT=$(mktemp 2>/dev/null); then + echo "::error::Failed to create temporary file" exit 1 fi - - # Execute the temporary script - "$TEMP_SCRIPT" + fi + + # Ensure cleanup on exit + trap 'rm -f "$TEMP_SCRIPT"' EXIT + + # Write the user's commands to the temp script + printf '%s\n' "$RETRY_COMMANDS" > "$TEMP_SCRIPT" + + # Validate script was written + if [ ! -s "$TEMP_SCRIPT" ]; then + echo "::error::No commands provided to execute" + exit 1 + fi + + while [ "$attempt" -le "$max_attempts" ]; do + if [ "$attempt" -gt 1 ]; then + echo "::group::Retry $((attempt - 1)) of $retries" + fi + + current_time=$(date +%s) + elapsed=$((current_time - start_time)) + if [ "$elapsed" -gt "$timeout_seconds" ]; then + echo "::error::Step timed out after $timeout_mins minutes" + [ "$attempt" -gt 1 ] && echo "::endgroup::" + exit 1 + fi + + # Execute the script with appropriate interpreter + if [ "$shell_type" = "python" ]; then + python3 "$TEMP_SCRIPT" + else + bash -e "$TEMP_SCRIPT" + fi exit_code=$? - - if [ $exit_code -eq 0 ]; then - [ $attempt -gt 1 ] && echo "::endgroup::" - rm "$TEMP_SCRIPT" + + if [ "$exit_code" -eq 0 ]; then + [ "$attempt" -gt 1 ] && echo "::endgroup::" exit 0 fi - - if [ $attempt -gt 1 ]; then - echo "Retry $((attempt-1)) failed with exit code $exit_code" + + # Handle failure + if [ "$attempt" -gt 1 ]; then + echo "Retry $((attempt - 1)) failed with exit code $exit_code" echo "::endgroup::" else echo "Initial attempt failed with exit code $exit_code" fi - - if [ $attempt -eq $max_attempts ]; then - echo "::error::Step failed after initial attempt and $((attempt-1)) retries" - rm "$TEMP_SCRIPT" - exit $exit_code + + if [ "$attempt" -eq "$max_attempts" ]; then + echo "::error::Step failed after initial attempt and $((attempt - 1)) retries" + exit "$exit_code" fi - - echo "Retrying in ${{ inputs.retry_delay_seconds }} seconds..." - sleep ${{ inputs.retry_delay_seconds }} + + echo "Retrying in $retry_delay seconds..." + sleep "$retry_delay" attempt=$((attempt + 1)) done