126 lines
5.4 KiB
Django/Jinja
126 lines
5.4 KiB
Django/Jinja
#!/bin/bash
|
|
# Scoped Docker cleanup for gitea-runner hosts.
|
|
# Implements: REQ-1..REQ-3 (GRM-173) — ownership leases, tiered watermarks,
|
|
# keep-images. Single entry point shared by docker-prune.service (routine)
|
|
# and runner-healthcheck.sh (pressure/critical).
|
|
# No `set -e`: a failing prune must not abort the remaining cleanup.
|
|
set -uo pipefail
|
|
|
|
DOCKER_HOST="unix:///run/user/{{ gitea_runner_uid }}/docker.sock"
|
|
XDG_RUNTIME_DIR="/run/user/{{ gitea_runner_uid }}"
|
|
export DOCKER_HOST XDG_RUNTIME_DIR
|
|
|
|
# REQ-1 label contract: `org.oblachno.lease-until` (epoch) protects an object
|
|
# while in the future; `org.oblachno.owner` records the owning run.
|
|
TIER="${1:-routine}"
|
|
LEASE_UNTIL_LABEL="org.oblachno.lease-until"
|
|
KEEP_IMAGES_RE="{{ gitea_runner_keep_images | join('|') }}"
|
|
now_epoch=$(date +%s)
|
|
|
|
# Implements: REQ-1 — a lease whose `lease-until` epoch lies in the future
|
|
# protects its object from every removal path in this script.
|
|
lease_active() {
|
|
local until="$1"
|
|
[[ -n "$until" && "$until" =~ ^[0-9]+$ && "$until" -gt "$now_epoch" ]]
|
|
}
|
|
|
|
# Remove stopped containers. $1 = "aged" (only >1h, RunningFor heuristic)
|
|
# or "all". CI job containers and valid leases are never removed.
|
|
remove_stopped_containers() {
|
|
local mode="$1"
|
|
# Implements: REQ-1/REQ-3 — pipe-separated fields; RunningFor contains
|
|
# spaces, so whitespace-splitting would break the age gate.
|
|
{ timeout 30 docker ps -a --filter "status=exited" --filter "status=dead" \
|
|
--format '{% raw %}{{.ID}}|{{.Names}}|{{.RunningFor}}|{{.Label "org.oblachno.lease-until"}}{% endraw %}' \
|
|
2>/dev/null || true; } \
|
|
| while IFS='|' read -r cid cname running_for lease_until; do
|
|
[[ -z "$cid" ]] && continue
|
|
case "$cname" in GITEA-ACTIONS-TASK*) continue ;; esac
|
|
lease_active "$lease_until" && continue
|
|
if [[ "$mode" != "all" ]] \
|
|
&& ! grep -qE '(hour|day|week|month|year)s? ago' <<<"$running_for"; then
|
|
continue
|
|
fi
|
|
docker rm -f "$cid" >/dev/null 2>&1 || true
|
|
done
|
|
}
|
|
|
|
# Remove unused images older than $1 ("all" = no age limit). The keep-list
|
|
# (warm base layers) and leased images are never removed; images referenced
|
|
# by any container are refused by the daemon anyway. `docker images` has no
|
|
# label formatter, so the lease is checked via inspect per candidate.
|
|
remove_old_images() {
|
|
local until="$1"
|
|
docker image prune -f >/dev/null 2>&1 || true
|
|
local filters=(--filter "dangling=false")
|
|
[[ "$until" != "all" ]] && filters+=(--filter "until=${until}")
|
|
{ timeout 30 docker images "${filters[@]}" \
|
|
--format '{% raw %}{{.ID}}|{{.Repository}}:{{.Tag}}{% endraw %}' \
|
|
2>/dev/null || true; } \
|
|
| while IFS='|' read -r iid ref; do
|
|
[[ -z "$iid" || "$ref" == *"<none>"* ]] && continue
|
|
[[ -n "$KEEP_IMAGES_RE" && "$ref" =~ $KEEP_IMAGES_RE ]] && continue
|
|
local lease_until
|
|
lease_until=$(docker image inspect "$iid" \
|
|
--format '{% raw %}{{index .Config.Labels "org.oblachno.lease-until"}}{% endraw %}' \
|
|
2>/dev/null || true)
|
|
lease_active "$lease_until" && continue
|
|
docker image rm "$iid" >/dev/null 2>&1 || true
|
|
done
|
|
}
|
|
|
|
# Reclaim volumes/networks whose lease expired. The prune filters below
|
|
# skip every leased object (label!=); this pass removes the expired ones.
|
|
reclaim_expired_leases() {
|
|
timeout 20 docker volume ls -q --filter "label=${LEASE_UNTIL_LABEL}" 2>/dev/null \
|
|
| while read -r vol; do
|
|
lease_until=$(docker volume inspect "$vol" \
|
|
--format '{% raw %}{{index .Labels "org.oblachno.lease-until"}}{% endraw %}' \
|
|
2>/dev/null || true)
|
|
lease_active "$lease_until" || docker volume rm "$vol" >/dev/null 2>&1 || true
|
|
done
|
|
timeout 20 docker network ls -q --filter "label=${LEASE_UNTIL_LABEL}" 2>/dev/null \
|
|
| while read -r net; do
|
|
lease_until=$(docker network inspect "$net" \
|
|
--format '{% raw %}{{index .Labels "org.oblachno.lease-until"}}{% endraw %}' \
|
|
2>/dev/null || true)
|
|
lease_active "$lease_until" || docker network rm "$net" >/dev/null 2>&1 || true
|
|
done
|
|
}
|
|
|
|
# Anonymous volumes only at routine/pressure tiers — a named volume may
|
|
# belong to a job between create/attach steps. Critical removes all unused.
|
|
prune_volumes_networks() {
|
|
local vol_all="$1" net_until="$2"
|
|
docker volume prune ${vol_all:+$vol_all} -f --filter "label!=${LEASE_UNTIL_LABEL}" >/dev/null 2>&1 || true
|
|
docker network prune -f --filter "label!=${LEASE_UNTIL_LABEL}" ${net_until:+--filter until=${net_until}} >/dev/null 2>&1 || true
|
|
}
|
|
|
|
case "$TIER" in
|
|
routine)
|
|
remove_stopped_containers aged
|
|
remove_old_images "{{ gitea_runner_prune_until }}"
|
|
prune_volumes_networks "" "{{ gitea_runner_prune_until }}"
|
|
docker builder prune -f --filter "until=24h" >/dev/null 2>&1 || true
|
|
;;
|
|
pressure)
|
|
remove_stopped_containers aged
|
|
remove_old_images "1h"
|
|
prune_volumes_networks "" "1h"
|
|
docker builder prune -f --filter "until=24h" >/dev/null 2>&1 || true
|
|
;;
|
|
critical)
|
|
# Implements: REQ-3 — age limits dropped, ownership still honored.
|
|
remove_stopped_containers all
|
|
remove_old_images all
|
|
prune_volumes_networks "-a" ""
|
|
docker builder prune -af >/dev/null 2>&1 || true
|
|
;;
|
|
*)
|
|
echo "ERROR: unknown cleanup tier '$TIER' (expected routine|pressure|critical)" >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
|
|
reclaim_expired_leases
|