GRM-123: fix: cast disk threshold to string in template-content verify assertion

ansible/disable-runner.yml

@@ -18,6 +18,16 @@
       when: systemd_available.stat.exists
       changed_when: true
 
+    - name: Stop and disable healthcheck timer
+      ansible.builtin.command: systemctl --user stop --disable runner-healthcheck.timer
+      become: true
+      become_user: "{{ gitea_runner_service_user | default('grm-' ~ runner_name) }}"
+      environment:
+        XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid | default('') }}"
+      when: systemd_available.stat.exists
+      changed_when: true
+      failed_when: false
+
     - name: Include deregistration
       ansible.builtin.include_role:
         name: gitea-runner

ansible/remove-runner.yml

@@ -79,6 +79,16 @@
         tasks_from: deregister.yml
       when: not skip_runner_registration | default(false)
 
+    - name: Stop and disable healthcheck timer
+      ansible.builtin.command: systemctl --user stop --disable runner-healthcheck.timer
+      become: true
+      become_user: "{{ gitea_runner_service_user | default('grm-' ~ runner_name) }}"
+      environment:
+        XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid | default('') }}"
+      when: systemd_available.stat.exists
+      changed_when: true
+      failed_when: false
+
     - name: Remove docker-prune user service file
       ansible.builtin.file:
         path: "{{ gitea_runner_home | default('/home/grm-' ~ runner_name) }}/.config/systemd/user/docker-prune.service"
@@ -91,6 +101,24 @@
         state: absent
       failed_when: false
 
+    - name: Remove healthcheck user service file
+      ansible.builtin.file:
+        path: "{{ gitea_runner_home | default('/home/grm-' ~ runner_name) }}/.config/systemd/user/runner-healthcheck.service"
+        state: absent
+      failed_when: false
+
+    - name: Remove healthcheck user timer file
+      ansible.builtin.file:
+        path: "{{ gitea_runner_home | default('/home/grm-' ~ runner_name) }}/.config/systemd/user/runner-healthcheck.timer"
+        state: absent
+      failed_when: false
+
+    - name: Remove healthcheck script
+      ansible.builtin.file:
+        path: "{{ gitea_runner_config_dir | default('/etc/gitea-runner/' ~ runner_name) }}/healthcheck.sh"
+        state: absent
+      failed_when: false
+
     - name: Remove systemd user unit file
       ansible.builtin.file:
         path: "{{ gitea_runner_home | default('/home/grm-' ~ runner_name) }}/.config/systemd/user/gitea-runner.service"

ansible/roles/gitea-runner/defaults/main.yml

@@ -24,6 +24,12 @@ gitea_runner_prune_label: "gitea-runner=true"
 # Service configuration
 gitea_runner_service_restart_sec: "5"
 
+# Health check configuration
+gitea_runner_healthcheck_interval: "5min"
+gitea_runner_healthcheck_boot_delay: "2min"
+gitea_runner_healthcheck_disk_threshold: 85
+gitea_runner_healthcheck_script_path: "{{ gitea_runner_config_dir }}/healthcheck.sh"
+
 # Admin token for runner deregistration via Gitea API.
 # If not set, falls back to registration_token (which likely lacks admin scope).
 # Set this to a token with admin scope to enable automatic runner cleanup on removal.

ansible/roles/gitea-runner/molecule/default/verify.yml

@@ -75,3 +75,42 @@
         that:
           - timer_stat.stat.exists
         fail_msg: "Docker prune timer is missing"
+
+    - name: Check healthcheck script exists
+      ansible.builtin.stat:
+        path: "{{ gitea_runner_healthcheck_script_path }}"
+      register: healthcheck_script_stat
+
+    - name: Assert healthcheck script exists
+      ansible.builtin.assert:
+        that:
+          - healthcheck_script_stat.stat.exists
+        fail_msg: "Healthcheck script is missing"
+
+    - name: Assert healthcheck script is executable
+      ansible.builtin.assert:
+        that:
+          - healthcheck_script_stat.stat.mode == "0755"
+        fail_msg: "Healthcheck script is not executable"
+
+    - name: Check healthcheck service exists
+      ansible.builtin.stat:
+        path: "{{ gitea_runner_home }}/.config/systemd/user/runner-healthcheck.service"
+      register: healthcheck_service_stat
+
+    - name: Assert healthcheck service exists
+      ansible.builtin.assert:
+        that:
+          - healthcheck_service_stat.stat.exists
+        fail_msg: "Healthcheck systemd service is missing"
+
+    - name: Check healthcheck timer exists
+      ansible.builtin.stat:
+        path: "{{ gitea_runner_home }}/.config/systemd/user/runner-healthcheck.timer"
+      register: healthcheck_timer_stat
+
+    - name: Assert healthcheck timer exists
+      ansible.builtin.assert:
+        that:
+          - healthcheck_timer_stat.stat.exists
+        fail_msg: "Healthcheck systemd timer is missing"

ansible/roles/gitea-runner/molecule/remove/verify.yml

@@ -89,6 +89,39 @@
           - not prune_timer_stat.stat.exists
         fail_msg: "docker-prune timer unit still exists after removal"
 
+    - name: Check healthcheck service unit is absent
+      ansible.builtin.stat:
+        path: "{{ gitea_runner_home }}/.config/systemd/user/runner-healthcheck.service"
+      register: healthcheck_service_stat
+
+    - name: Assert healthcheck service unit is absent
+      ansible.builtin.assert:
+        that:
+          - not healthcheck_service_stat.stat.exists
+        fail_msg: "runner-healthcheck service unit still exists after removal"
+
+    - name: Check healthcheck timer unit is absent
+      ansible.builtin.stat:
+        path: "{{ gitea_runner_home }}/.config/systemd/user/runner-healthcheck.timer"
+      register: healthcheck_timer_stat
+
+    - name: Assert healthcheck timer unit is absent
+      ansible.builtin.assert:
+        that:
+          - not healthcheck_timer_stat.stat.exists
+        fail_msg: "runner-healthcheck timer unit still exists after removal"
+
+    - name: Check healthcheck script is absent
+      ansible.builtin.stat:
+        path: "{{ gitea_runner_config_dir }}/healthcheck.sh"
+      register: healthcheck_script_stat
+
+    - name: Assert healthcheck script is absent
+      ansible.builtin.assert:
+        that:
+          - not healthcheck_script_stat.stat.exists
+        fail_msg: "healthcheck script still exists after removal"
+
     - name: Check subuid entry is absent
       ansible.builtin.command: "grep -c '^{{ gitea_runner_service_user }}:' /etc/subuid"
       register: subuid_check

ansible/roles/gitea-runner/molecule/template-content/verify.yml

@@ -62,3 +62,45 @@
           - "'OnCalendar={{ gitea_runner_prune_schedule }}' in prune_timer.content | b64decode"
           - "'Persistent=true' in prune_timer.content | b64decode"
         fail_msg: "Prune timer template is missing expected directives"
+
+    - name: Read rendered healthcheck service template
+      ansible.builtin.slurp:
+        src: "{{ gitea_runner_home }}/.config/systemd/user/runner-healthcheck.service"
+      register: healthcheck_service
+
+    - name: Assert healthcheck service contains expected directives
+      ansible.builtin.assert:
+        that:
+          - "'Type=oneshot' in healthcheck_service.content | b64decode"
+          - "'ExecStart={{ gitea_runner_healthcheck_script_path }}' in healthcheck_service.content | b64decode"
+          - "'DOCKER_HOST=unix:///run/user/' in healthcheck_service.content | b64decode"
+          - "'XDG_RUNTIME_DIR=/run/user/' in healthcheck_service.content | b64decode"
+        fail_msg: "Healthcheck service template is missing expected directives"
+
+    - name: Read rendered healthcheck timer template
+      ansible.builtin.slurp:
+        src: "{{ gitea_runner_home }}/.config/systemd/user/runner-healthcheck.timer"
+      register: healthcheck_timer
+
+    - name: Assert healthcheck timer contains expected directives
+      ansible.builtin.assert:
+        that:
+          - "'OnBootSec={{ gitea_runner_healthcheck_boot_delay }}' in healthcheck_timer.content | b64decode"
+          - "'OnUnitActiveSec={{ gitea_runner_healthcheck_interval }}' in healthcheck_timer.content | b64decode"
+          - "'Persistent=true' in healthcheck_timer.content | b64decode"
+        fail_msg: "Healthcheck timer template is missing expected directives"
+
+    - name: Read rendered healthcheck script
+      ansible.builtin.slurp:
+        src: "{{ gitea_runner_healthcheck_script_path }}"
+      register: healthcheck_script
+
+    - name: Assert healthcheck script contains expected content
+      ansible.builtin.assert:
+        that:
+          - "'docker info' in healthcheck_script.content | b64decode"
+          - "'systemctl --user restart docker.service' in healthcheck_script.content | b64decode"
+          - "'systemctl --user restart gitea-runner.service' in healthcheck_script.content | b64decode"
+          - "'docker system prune' in healthcheck_script.content | b64decode"
+          - "gitea_runner_healthcheck_disk_threshold | string in healthcheck_script.content | b64decode"
+        fail_msg: "Healthcheck script template is missing expected content"

ansible/roles/gitea-runner/tasks/healthcheck.yml

@@ -0,0 +1,46 @@
+---
+- name: Create healthcheck script
+  ansible.builtin.template:
+    src: runner-healthcheck.sh.j2
+    dest: "{{ gitea_runner_healthcheck_script_path }}"
+    owner: "{{ gitea_runner_service_user }}"
+    group: "{{ gitea_runner_service_user }}"
+    mode: "0755"
+
+- name: Create healthcheck user service file
+  ansible.builtin.template:
+    src: runner-healthcheck.service.j2
+    dest: "{{ gitea_runner_home }}/.config/systemd/user/runner-healthcheck.service"
+    owner: "{{ gitea_runner_service_user }}"
+    group: "{{ gitea_runner_service_user }}"
+    mode: "0644"
+
+- name: Create healthcheck user timer file
+  ansible.builtin.template:
+    src: runner-healthcheck.timer.j2
+    dest: "{{ gitea_runner_home }}/.config/systemd/user/runner-healthcheck.timer"
+    owner: "{{ gitea_runner_service_user }}"
+    group: "{{ gitea_runner_service_user }}"
+    mode: "0644"
+
+- name: Reload systemd user daemon for healthcheck timer
+  ansible.builtin.command: systemctl --user daemon-reload
+  become: true
+  become_user: "{{ gitea_runner_service_user }}"
+  environment:
+    XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid }}"
+  changed_when: true
+  when:
+    - systemd_available.stat.exists
+    - docker_rootless_setup
+
+- name: Enable and start healthcheck user timer
+  ansible.builtin.command: systemctl --user enable --now runner-healthcheck.timer
+  become: true
+  become_user: "{{ gitea_runner_service_user }}"
+  environment:
+    XDG_RUNTIME_DIR: "/run/user/{{ gitea_runner_uid }}"
+  changed_when: true
+  when:
+    - systemd_available.stat.exists
+    - docker_rootless_setup

ansible/roles/gitea-runner/tasks/main.yml

@@ -14,6 +14,9 @@
 - name: Include prune setup
   ansible.builtin.include_tasks: prune.yml
 
+- name: Include healthcheck setup
+  ansible.builtin.include_tasks: healthcheck.yml
+
 - name: Include integration test
   ansible.builtin.include_tasks: integration_test.yml
   when: not skip_runner_registration

ansible/roles/gitea-runner/templates/runner-healthcheck.service.j2

@@ -0,0 +1,9 @@
+[Unit]
+Description=Gitea Runner health check (Docker + service + disk)
+After=docker.service gitea-runner.service
+
+[Service]
+Type=oneshot
+Environment=DOCKER_HOST=unix:///run/user/{{ gitea_runner_uid }}/docker.sock
+Environment=XDG_RUNTIME_DIR=/run/user/{{ gitea_runner_uid }}
+ExecStart={{ gitea_runner_healthcheck_script_path }}

ansible/roles/gitea-runner/templates/runner-healthcheck.sh.j2

@@ -0,0 +1,49 @@
+#!/bin/bash
+# Health check for gitea-runner: verifies Docker daemon and runner service.
+# Exits 0 if healthy, 1 if Docker is down (triggers restart), 2 if runner is down.
+set -euo pipefail
+
+DOCKER_HOST="unix:///run/user/{{ gitea_runner_uid }}/docker.sock"
+XDG_RUNTIME_DIR="/run/user/{{ gitea_runner_uid }}"
+export DOCKER_HOST XDG_RUNTIME_DIR
+
+# 1. Check Docker daemon responsiveness
+if ! docker info >/dev/null 2>&1; then
+    echo "ERROR: Docker daemon not responding at ${DOCKER_HOST}"
+    systemctl --user restart docker.service
+    sleep 3
+    if ! docker info >/dev/null 2>&1; then
+        echo "CRITICAL: Docker daemon still down after restart"
+        exit 1
+    fi
+    echo "RECOVERED: Docker daemon restarted successfully"
+fi
+
+# 2. Check gitea-runner service is active
+runner_state=$(systemctl --user is-active gitea-runner.service 2>/dev/null || true)
+if [[ "$runner_state" != "active" ]]; then
+    echo "ERROR: gitea-runner service is ${runner_state}, restarting"
+    systemctl --user restart gitea-runner.service
+    sleep 2
+    runner_state=$(systemctl --user is-active gitea-runner.service 2>/dev/null || true)
+    if [[ "$runner_state" != "active" ]]; then
+        echo "CRITICAL: gitea-runner service still down after restart"
+        exit 2
+    fi
+    echo "RECOVERED: gitea-runner service restarted successfully"
+fi
+
+# 3. Check disk space — prune aggressively if below threshold
+disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')
+if [[ "$disk_pct" -ge {{ gitea_runner_healthcheck_disk_threshold }} ]]; then
+    echo "WARN: Disk usage at ${disk_pct}%, pruning all runner resources"
+    docker system prune -af --filter "label={{ gitea_runner_prune_label }}" --filter "until=1h" || true
+    docker volume prune -af --filter "label={{ gitea_runner_prune_label }}" || true
+    # Also prune dangling images (no label)
+    docker image prune -af || true
+    disk_pct=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')
+    echo "INFO: Disk usage after prune: ${disk_pct}%"
+fi
+
+echo "OK: runner healthy, disk at ${disk_pct}%"
+exit 0

ansible/roles/gitea-runner/templates/runner-healthcheck.timer.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=Periodic Gitea Runner health check
+
+[Timer]
+OnBootSec={{ gitea_runner_healthcheck_boot_delay }}
+OnUnitActiveSec={{ gitea_runner_healthcheck_interval }}
+Persistent=true
+
+[Install]
+WantedBy=timers.target