DEVX-68: feat: add pre-built Docker runner images and tested image build/push tools

This commit is contained in:
emil
2026-06-27 01:34:05 +00:00
parent d4ddbd7e7a
commit c0238e75df
14 changed files with 2034 additions and 389 deletions
+55
View File
@@ -249,3 +249,58 @@ devx-clean:
@find . -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true
@find . -type f -name "*.pyc" -delete 2>/dev/null || true
@rm -rf .coverage htmlcov/ dist/ build/ *.egg-info/ .molecule/ 2>/dev/null || true
# ── Pre-built image setup ─────────────────────────────────────────────────────
#
# When running inside a pre-built Docker runner image (ci-base, ci-quality,
# ci-full), all deps are already installed in /opt/venv. This target links
# the venv and installs the project itself (no-deps, fast).
# Falls back to devx-setup-ci if /opt/venv is not present (local dev).
devx-setup-image:
@if [ -d /opt/venv ]; then \
ln -sf /opt/venv $(DEVX_VENV); \
. $(DEVX_BIN)/activate && pip install -e . --no-deps 2>/dev/null; \
echo "[devx-setup-image] Linked /opt/venv and installed project (no-deps)."; \
else \
echo "[devx-setup-image] /opt/venv not found — falling back to devx-setup-ci"; \
$(MAKE) devx-setup-ci; \
fi
# ── Docker image build / push / cleanup ───────────────────────────────────────
#
# Variables:
# DEVX_GITEA_REGISTRY — registry URL (default: git.oblachno.oblachno.fyi)
# DEVX_IMAGE_MANIFEST — path to JSON manifest (default: docker/images.json)
# DEVX_IMAGE_OWNER — package owner for cleanup (default: oblachno-oss)
DEVX_GITEA_REGISTRY ?= git.oblachno.oblachno.fyi
DEVX_IMAGE_MANIFEST ?= docker/images.json
DEVX_IMAGE_OWNER ?= oblachno-oss
# Build all images from manifest (no push)
devx-build-images:
@$(DEVX_PYTHON) -m devx.tools.build_image --manifest $(DEVX_IMAGE_MANIFEST) --pull
# Build and push all images to the Gitea registry
devx-push-images:
@$(DEVX_PYTHON) -m devx.tools.build_image \
--manifest $(DEVX_IMAGE_MANIFEST) \
--registry $(DEVX_GITEA_REGISTRY) \
--push --pull
# Dry-run: show what would be built/pushed
devx-build-images-dry-run:
@$(DEVX_PYTHON) -m devx.tools.build_image \
--manifest $(DEVX_IMAGE_MANIFEST) \
--registry $(DEVX_GITEA_REGISTRY) \
--push --dry-run
# Clean up old image versions (keep last 2 + latest)
devx-clean-images:
@$(DEVX_PYTHON) -m devx.tools.clean_images \
--owner $(DEVX_IMAGE_OWNER) \
--name oblachno-oss/runner-images/ci-base \
--name oblachno-oss/runner-images/ci-quality \
--name oblachno-oss/runner-images/ci-full \
--keep 2
+334
View File
@@ -0,0 +1,334 @@
#!/usr/bin/env python3
"""Build and push Docker images to a Gitea container registry.
Replaces raw ``docker build`` / ``docker push`` shell commands with a
tested Python tool. Supports:
- Building from any Dockerfile with a configurable context directory
- Tagging with multiple tags (e.g. ``latest`` + version)
- Optional push to a Gitea registry (with login)
- Dry-run mode (prints commands without executing)
Usage::
# Build a single image
python3 -m devx.tools.build_image \\
--dockerfile docker/ci-base/Dockerfile \\
--tag ci-base:latest \\
--tag ci-base:0.19.3
# Build and push to registry
python3 -m devx.tools.build_image \\
--dockerfile docker/ci-base/Dockerfile \\
--tag ci-base:latest \\
--tag ci-base:0.19.3 \\
--registry git.oblachno.oblachno.fyi \\
--push
# Build multiple images (from a manifest file)
python3 -m devx.tools.build_image --manifest docker/images.json --push
The manifest file is a JSON list of dicts, each with:
- ``name``: image name (e.g. ``ci-base``)
- ``dockerfile``: path to Dockerfile (relative to repo root)
- ``context``: build context directory (optional, defaults to repo root)
- ``tags``: list of tags (optional, defaults to ``["latest"]``)
Registry authentication uses ``REPO_TOKEN`` (or ``GITEA_REGISTRY_TOKEN``)
and ``REGISTRY_USERNAME`` (or ``GITEA_REGISTRY_USERNAME``) environment
variables, matching the existing CI workflow patterns.
"""
from __future__ import annotations
import json
import os
import subprocess # nosec B404
from dataclasses import dataclass, field
from pathlib import Path
import click
from devx.i18n import _
@dataclass
class ImageSpec:
"""Specification for a single Docker image to build."""
name: str
dockerfile: str
context: str = "."
tags: list[str] = field(default_factory=lambda: ["latest"])
@classmethod
def from_dict(cls, data: dict[str, object]) -> ImageSpec:
"""Create an ImageSpec from a dict (e.g. from a JSON manifest)."""
name = str(data.get("name", ""))
if not name:
raise ValueError(_("Image manifest entry missing 'name'"))
dockerfile = str(data.get("dockerfile", ""))
if not dockerfile:
raise ValueError(_("Image manifest entry missing 'dockerfile'"))
context = str(data.get("context", "."))
tags_raw = data.get("tags", ["latest"])
if not isinstance(tags_raw, list):
raise ValueError(_("Image 'tags' must be a list"))
tags = [str(t) for t in tags_raw] if tags_raw else ["latest"]
return cls(name=name, dockerfile=dockerfile, context=context, tags=tags)
def load_manifest(path: str | Path) -> list[ImageSpec]:
"""Load a JSON manifest file describing images to build.
The file must contain a JSON list of dicts with at least ``name`` and
``dockerfile`` keys. ``context`` and ``tags`` are optional.
Returns a list of :class:`ImageSpec` instances.
"""
p = Path(path)
if not p.is_file():
raise click.ClickException(_("Manifest file not found: {path}", path=p))
with p.open() as f: # noqa: PTH123
data = json.load(f)
if not isinstance(data, list):
raise click.ClickException(_("Manifest must be a JSON list"))
return [ImageSpec.from_dict(entry) for entry in data]
def build_full_tag(registry: str | None, name: str, tag: str) -> str:
"""Build a full image tag, optionally prefixed with a registry.
>>> build_full_tag(None, "ci-base", "latest")
'ci-base:latest'
>>> build_full_tag("git.example.com", "ci-base", "0.1.0")
'git.example.com/ci-base:0.1.0'
"""
if registry:
return f"{registry}/{name}:{tag}"
return f"{name}:{tag}"
def registry_login(
registry: str,
username: str,
token: str,
*,
dry_run: bool = False,
) -> bool:
"""Log in to a Docker registry.
Returns True on success, False on failure.
In dry-run mode, prints the command without executing.
"""
cmd = ["docker", "login", registry, "-u", username, "--password-stdin"]
if dry_run:
click.echo(f"[dry-run] {' '.join(cmd)}")
return True
result = subprocess.run( # nosec B603
cmd,
input=token,
text=True,
capture_output=True,
check=False,
)
if result.returncode != 0:
click.echo(
_("Registry login failed: {error}", error=result.stderr.strip()),
err=True,
)
return False
click.echo(f"Logged in to {registry}")
return True
def build_image(
spec: ImageSpec,
registry: str | None = None,
*,
dry_run: bool = False,
pull: bool = False,
) -> bool:
"""Build a Docker image from a Dockerfile.
Tags the image with all specified tags, optionally prefixed with the
registry. Returns True on success, False on failure.
"""
if not Path(spec.dockerfile).is_file():
click.echo(
_("Dockerfile not found: {path}", path=spec.dockerfile),
err=True,
)
return False
full_tags = [build_full_tag(registry, spec.name, t) for t in spec.tags]
cmd = ["docker", "build"]
if pull:
cmd.append("--pull")
for ft in full_tags:
cmd.extend(["-t", ft])
cmd.extend(["-f", spec.dockerfile, spec.context])
if dry_run:
click.echo(f"[dry-run] {' '.join(cmd)}")
return True
click.echo(f"Building {spec.name} ({len(full_tags)} tag(s))...")
result = subprocess.run( # nosec B603
cmd,
check=False,
)
if result.returncode != 0:
click.echo(_("Build failed for {name}", name=spec.name), err=True)
return False
click.echo(f"Built {spec.name}")
return True
def push_image(
spec: ImageSpec,
registry: str,
*,
dry_run: bool = False,
) -> bool:
"""Push all tags of a Docker image to the registry.
Returns True if all pushes succeed, False if any fail.
"""
full_tags = [build_full_tag(registry, spec.name, t) for t in spec.tags]
all_ok = True
for ft in full_tags:
cmd = ["docker", "push", ft]
if dry_run:
click.echo(f"[dry-run] {' '.join(cmd)}")
continue
click.echo(f"Pushing {ft}...")
result = subprocess.run( # nosec B603
cmd,
capture_output=True,
text=True,
check=False,
)
if result.returncode != 0:
click.echo(
_("Push failed for {tag}: {error}", tag=ft, error=result.stderr.strip()),
err=True,
)
all_ok = False
else:
click.echo(f"Pushed {ft}")
return all_ok
def _get_registry_creds() -> tuple[str, str]:
"""Get registry credentials from environment variables.
Supports both REPO_TOKEN/GITEA_REGISTRY_TOKEN and
REGISTRY_USERNAME/GITEA_REGISTRY_USERNAME patterns.
"""
token = os.environ.get("REPO_TOKEN") or os.environ.get("GITEA_REGISTRY_TOKEN", "")
username = os.environ.get("REGISTRY_USERNAME") or os.environ.get("GITEA_REGISTRY_USERNAME", "")
return username, token
@click.command()
@click.option(
"--dockerfile",
"dockerfile",
default=None,
help="Path to Dockerfile (for single-image build).",
)
@click.option(
"--context",
"context",
default=".",
help="Build context directory (for single-image build).",
)
@click.option(
"--name",
"name",
default=None,
help="Image name (for single-image build).",
)
@click.option(
"--tag",
"tags",
multiple=True,
help="Tag(s) for the image. Can be repeated. Defaults to 'latest'.",
)
@click.option(
"--manifest",
"manifest",
default=None,
help="Path to JSON manifest file listing images to build.",
)
@click.option(
"--registry",
"registry",
default=None,
help="Registry URL (e.g. git.example.com). If set with --push, images are tagged and pushed there.",
)
@click.option(
"--push",
is_flag=True,
default=False,
help="Push images to the registry after building.",
)
@click.option(
"--dry-run",
is_flag=True,
default=False,
help="Print commands without executing.",
)
@click.option(
"--pull",
is_flag=True,
default=False,
help="Pass --pull to docker build (always fetch latest base image).",
)
def main(
dockerfile: str | None,
context: str,
name: str | None,
tags: tuple[str, ...],
manifest: str | None,
registry: str | None,
push: bool,
dry_run: bool,
pull: bool,
) -> None:
"""Build and optionally push Docker images to a Gitea registry."""
if manifest:
specs = load_manifest(manifest)
elif dockerfile and name:
tag_list = list(tags) if tags else ["latest"]
specs = [ImageSpec(name=name, dockerfile=dockerfile, context=context, tags=tag_list)]
else:
raise click.ClickException(_("Provide --manifest or both --dockerfile and --name"))
if push:
if not registry:
raise click.ClickException(_("--push requires --registry"))
username, token = _get_registry_creds()
if not token or not username:
raise click.ClickException(
_("Registry credentials required: set REPO_TOKEN and REGISTRY_USERNAME env vars")
)
if not registry_login(registry, username, token, dry_run=dry_run):
raise click.ClickException(_("Registry login failed"))
failed: list[str] = []
for spec in specs:
if not build_image(spec, registry, dry_run=dry_run, pull=pull):
failed.append(spec.name)
continue
if push and not push_image(spec, registry, dry_run=dry_run): # type: ignore[arg-type]
failed.append(spec.name)
if failed:
raise click.ClickException(_("Failed images: {names}", names=", ".join(failed)))
click.echo(f"\nDone. {len(specs)} image(s) processed.")
if __name__ == "__main__": # pragma: no cover
main() # pragma: no cover
+221
View File
@@ -0,0 +1,221 @@
#!/usr/bin/env python3
"""Clean up old Docker images from a Gitea container registry.
Queries the Gitea API for all versions of a package (container type) and
deletes all but the most recent N versions. The ``latest`` tag is always
preserved if present.
Usage::
# Clean up ci-base images, keep last 2 versions
python3 -m devx.tools.clean_images \\
--owner oblachno-oss \\
--name ci-base \\
--keep 2
# Clean up multiple images
python3 -m devx.tools.clean_images \\
--owner oblachno-oss \\
--name ci-base \\
--name ci-quality \\
--name ci-full \\
--keep 2
# Dry run (list what would be deleted)
python3 -m devx.tools.clean_images \\
--owner oblachno-oss \\
--name ci-base \\
--keep 2 \\
--dry-run
Authentication uses ``REPO_TOKEN`` environment variable.
"""
from __future__ import annotations
import os
from typing import Any
import click
import requests
from devx.config import GITEA_API_URL
from devx.i18n import _
def list_package_versions(
api_url: str,
owner: str,
name: str,
token: str,
*,
timeout: int = 30,
) -> list[dict[str, Any]]:
"""List all versions of a container package from the Gitea API.
Returns a list of version dicts, each containing at least ``version``
and ``created_at`` fields.
"""
url = f"{api_url}/packages/{owner}?type=container&name={name}"
headers = {"Authorization": f"token {token}"}
all_versions: list[dict[str, Any]] = []
page = 1
while True:
resp = requests.get(
f"{url}&page={page}&limit=50",
headers=headers,
timeout=timeout,
)
resp.raise_for_status()
data = resp.json()
if not data:
break
all_versions.extend(data)
if len(data) < 50:
break
page += 1
return all_versions
def delete_package_version(
api_url: str,
owner: str,
name: str,
version: str,
token: str,
*,
timeout: int = 30,
) -> bool:
"""Delete a specific version of a container package.
Returns True on success, False on failure.
"""
url = f"{api_url}/packages/{owner}/{name}/{version}"
headers = {"Authorization": f"token {token}"}
resp = requests.delete(url, headers=headers, timeout=timeout)
return resp.status_code in (204, 200)
def sort_versions_by_date(
versions: list[dict[str, Any]],
) -> list[dict[str, Any]]:
"""Sort package versions by creation date, newest first.
Falls back to version string comparison if created_at is missing.
"""
def _sort_key(v: dict[str, Any]) -> str:
return str(v.get("created_at", v.get("version", "")))
return sorted(versions, key=_sort_key, reverse=True)
def select_for_deletion(
versions: list[dict[str, Any]],
keep: int,
) -> list[dict[str, Any]]:
"""Select versions to delete, keeping the most recent ``keep`` versions.
Versions named ``latest`` are always preserved.
"""
sorted_versions = sort_versions_by_date(versions)
to_delete = sorted_versions[keep:]
# Always preserve 'latest' tag
to_delete = [v for v in to_delete if v.get("version") != "latest"]
return to_delete
@click.command()
@click.option(
"--owner",
required=True,
help="Package owner (user or org).",
)
@click.option(
"--name",
"names",
multiple=True,
required=True,
help="Package name(s). Can be repeated.",
)
@click.option(
"--keep",
default=2,
type=int,
show_default=True,
help="Number of recent versions to keep (excluding 'latest').",
)
@click.option(
"--dry-run",
is_flag=True,
default=False,
help="List versions that would be deleted without actually deleting.",
)
@click.option(
"--api-url",
default=None,
help="Gitea API URL (defaults to DEVX_GITEA_API_URL or built-in default).",
)
def main(
owner: str,
names: tuple[str, ...],
keep: int,
dry_run: bool,
api_url: str | None,
) -> None:
"""Clean up old Docker image versions from a Gitea registry."""
token = os.environ.get("REPO_TOKEN", "")
if not token:
raise click.ClickException(_("REPO_TOKEN environment variable required"))
base_url = api_url or GITEA_API_URL
total_deleted = 0
total_kept = 0
for name in names:
click.echo(f"\n{'=' * 60}")
click.echo(f"Package: {owner}/{name}")
click.echo(f"{'=' * 60}")
try:
versions = list_package_versions(base_url, owner, name, token)
except requests.RequestException as exc:
click.echo(
_("Failed to list versions for {name}: {error}", name=name, error=exc),
err=True,
)
continue
if not versions:
click.echo(_("No versions found."))
continue
click.echo(f"Found {len(versions)} version(s):")
for v in sort_versions_by_date(versions):
click.echo(f" {v.get('version', '?')} (created: {v.get('created_at', '?')})")
to_delete = select_for_deletion(versions, keep)
kept_count = len(versions) - len(to_delete)
click.echo(f"\nKeeping {kept_count}, would delete {len(to_delete)}")
if dry_run:
for v in to_delete:
click.echo(f" [dry-run] Would delete: {v.get('version', '?')}")
total_kept += kept_count
continue
deleted_count = 0
for v in to_delete:
version = str(v.get("version", ""))
if delete_package_version(base_url, owner, name, version, token):
click.echo(f" Deleted: {version}")
deleted_count += 1
else:
click.echo(f" FAILED to delete: {version}", err=True)
total_deleted += deleted_count
total_kept += kept_count
click.echo(f"\nDone. Deleted {total_deleted}, kept {total_kept}.")
if __name__ == "__main__": # pragma: no cover
main() # pragma: no cover
+517 -381
View File
File diff suppressed because it is too large Load Diff