grm/ansible/roles/gitea_runner/templates/runner-healthcheck.sh.j2

#!/bin/bash
# Health check for gitea-runner: verifies Docker daemon and runner service.
# Exits 0 if healthy, 1 if Docker is down (triggers restart), 2 if runner is down.
set -euo pipefail

DOCKER_HOST="unix:///run/user/{{ gitea_runner_uid }}/docker.sock"
XDG_RUNTIME_DIR="/run/user/{{ gitea_runner_uid }}"
export DOCKER_HOST XDG_RUNTIME_DIR

# 1. Check Docker daemon responsiveness (with timeout — a bare `docker info` can
#    hang indefinitely on a stuck daemon, blocking the healthcheck itself).
if ! timeout 10 docker info >/dev/null 2>&1; then
    echo "ERROR: Docker daemon not responding at ${DOCKER_HOST} (timed out after 10s)"
    systemctl --user restart docker.service
    sleep 3
    if ! timeout 10 docker info >/dev/null 2>&1; then
        echo "CRITICAL: Docker daemon still down after restart"
        exit 1
    fi
    echo "RECOVERED: Docker daemon restarted successfully"
fi

# 1b. Clean up stuck containers — containers in "removing" or "stopping" state
#     for too long cause "cannot kill container: did not receive an exit event"
#     errors in molecule destroy phases. Force-remove them so subsequent CI jobs
#     don't inherit the stuck state.
stuck_containers=$(timeout 10 docker ps -a --filter "status=removing" --filter "status=stopping" --format '{% raw %}{{.ID}}{% endraw %}' 2>/dev/null || true)
if [[ -n "$stuck_containers" ]]; then
    echo "WARN: Found stuck containers (removing/stopping), force-cleaning"
    echo "$stuck_containers" | xargs -r docker rm -f 2>/dev/null || true
fi

# 2. Check gitea-runner service is active
runner_state=$(systemctl --user is-active gitea-runner.service 2>/dev/null || true)
if [[ "$runner_state" != "active" ]]; then
    echo "ERROR: gitea-runner service is ${runner_state}, restarting"
    systemctl --user restart gitea-runner.service
    sleep 2
    runner_state=$(systemctl --user is-active gitea-runner.service 2>/dev/null || true)
    if [[ "$runner_state" != "active" ]]; then
        echo "CRITICAL: gitea-runner service still down after restart"
        exit 2
    fi
    echo "RECOVERED: gitea-runner service restarted successfully"
fi

# 2b. Detect unregistered runner state. When Gitea no longer recognizes the
# runner (e.g., server restore, runner record deleted, Gitea restart with
# token salt change), the runner logs "unregistered runner" every few seconds.
# A service restart will not fix this; re-registration is required.
{% if gitea_runner_auto_recover_api_token %}
# Auto-recovery is enabled: fetch a new registration token from the Gitea API
# and re-register the runner automatically. A cooldown prevents infinite loops.
GITEA_API_TOKEN_FILE="{{ gitea_runner_config_dir }}/auto-recover.token"
COOLDOWN_FILE="{{ gitea_runner_data_dir }}/auto-recover.cooldown"
COOLDOWN_SEC={{ gitea_runner_auto_recover_cooldown_sec }}
GITEA_URL="{{ gitea_url }}"
RUNNER_NAME="{{ gitea_runner_name }}"
RUNNER_LABELS="{{ gitea_runner_labels }}"
BINARY="{{ gitea_runner_binary_path }}"
{% endif %}
recent_errors=$(journalctl --user -u gitea-runner.service --since "5 minutes ago" --no-pager -q 2>/dev/null | grep -c "unregistered runner" || true)
if [[ "$recent_errors" -ge 3 ]]; then
    echo "CRITICAL: runner is unregistered in Gitea (re-login failed $recent_errors times in 5 minutes)."
{% if gitea_runner_auto_recover_api_token %}
    # Check cooldown — skip if we recently attempted recovery
    if [[ -f "$COOLDOWN_FILE" ]]; then
        last_attempt=$(cat "$COOLDOWN_FILE" 2>/dev/null || echo 0)
        now=$(date +%s)
        elapsed=$((now - last_attempt))
        if [[ "$elapsed" -lt "$COOLDOWN_SEC" ]]; then
            echo "SKIP: auto-recovery cooldown active (${elapsed}s < ${COOLDOWN_SEC}s). Will retry later."
            exit 3
        fi
    fi

    # Mark attempt time BEFORE trying (so failures also get cooldown)
    date +%s > "$COOLDOWN_FILE" 2>/dev/null || true

    # Read the API token
    if [[ ! -f "$GITEA_API_TOKEN_FILE" ]]; then
        echo "ERROR: auto-recover token file not found at $GITEA_API_TOKEN_FILE. Manual re-registration required."
        exit 3
    fi
    API_TOKEN=$(cat "$GITEA_API_TOKEN_FILE" 2>/dev/null || true)
    if [[ -z "$API_TOKEN" ]]; then
        echo "ERROR: auto-recover token file is empty. Manual re-registration required."
        exit 3
    fi

    echo "ATTEMPT: auto-recovering by fetching new registration token and re-registering..."

    # Fetch a new registration token from the Gitea API
    # Try org-level first (for org-scoped runners), then instance-level
    REG_TOKEN=""
    for endpoint in \
        "api/v1/orgs/{{ gitea_runner_org | default('oblachno') }}/actions/runners/registration-token" \
        "api/v1/admin/actions/runners/registration-token"; do
        REG_TOKEN=$(curl -sf --connect-timeout 5 --max-time 10 -X POST \
            -H "Authorization: token $API_TOKEN" \
            "${GITEA_URL}/${endpoint}" 2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin).get('token',''))" 2>/dev/null || true)
        if [[ -n "$REG_TOKEN" ]]; then
            echo "INFO: fetched registration token from ${endpoint}"
            break
        fi
    done

    if [[ -z "$REG_TOKEN" ]]; then
        echo "ERROR: failed to fetch registration token from Gitea API. Is Gitea reachable?"
        exit 3
    fi

    # Stop the runner service
    systemctl --user stop gitea-runner.service 2>/dev/null || true
    sleep 1

    # Remove the stale .runner file
    rm -f "{{ gitea_runner_data_dir }}/.runner" 2>/dev/null || true

    # Re-register
    cd "{{ gitea_runner_data_dir }}"
    if "$BINARY" register \
        --token "$REG_TOKEN" \
        --name "$RUNNER_NAME" \
        --instance "$GITEA_URL" \
        --labels "$RUNNER_LABELS" \
        --no-interactive 2>&1; then
        echo "RECOVERED: runner re-registered successfully"
    else
        echo "ERROR: re-registration failed. Manual intervention required."
        exit 3
    fi

    # Start the runner service
    systemctl --user start gitea-runner.service
    sleep 3

    # Verify recovery — check if unregistered errors stopped
    new_errors=$(journalctl --user -u gitea-runner.service --since "10 seconds ago" --no-pager -q 2>/dev/null | grep -c "unregistered runner" || true)
    if [[ "$new_errors" -eq 0 ]]; then
        echo "OK: runner recovered and no longer reporting unregistered errors"
        # Clear cooldown on success
        rm -f "$COOLDOWN_FILE" 2>/dev/null || true
    else
        echo "WARN: runner still showing unregistered errors after re-registration. Will retry after cooldown."
        exit 3
    fi
{% else %}
    echo "Manual re-registration required: re-run gitea_runner role with gitea_runner_force_reregister=true."
    # Restart the service once in case it is a transient token refresh issue,
    # but this cannot recover an unregistered runner without re-registration.
    systemctl --user restart gitea-runner.service
    sleep 2
    exit 3
{% endif %}
fi

# 3. Check disk space — prune aggressively if below threshold
disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')
if [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_threshold }} ]]; then
    echo "WARN: Disk usage at ${disk_pct}%, pruning all runner resources"
    # Force-remove stale containers (including running ones from failed molecule tests).
    # "docker container prune -f" only removes stopped containers, so running
    # containers from crashed CI jobs accumulate and consume disk/memory.
    # Exclude CI job containers (name starts with GITEA-ACTIONS-TASK).
    docker ps -a --format '{% raw %}{{.ID}} {{.Names}}{% endraw %}' 2>/dev/null \
        | grep -v 'GITEA-ACTIONS-TASK' \
        | awk '{print $1}' \
        | xargs -r docker rm -f 2>/dev/null || true
    docker system prune -af --filter "until=1h" --volumes || true
    docker network prune -f || true
    disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')
    echo "INFO: Disk usage after prune: ${disk_pct}%"
fi

echo "OK: runner healthy, disk at ${disk_pct}%"
exit 0