Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
18632bd543 | ||
|
|
1fc7a03c1a | ||
|
|
808e7a2e42 | ||
|
|
dd8e6c69e9 | ||
|
|
ccb7023965 | ||
|
|
7dd15f1461 | ||
|
|
d4e4621fa1 | ||
|
|
a6f814c446 | ||
|
|
04aa5acb1f | ||
|
|
6973f9d851 | ||
|
|
2669a0ea73 | ||
|
|
03f057b55a | ||
|
|
706d6dafe0 | ||
|
|
03ddce427c | ||
|
|
9642d6884c | ||
|
|
b2074d6635 | ||
|
|
a6dddf25e7 | ||
|
|
01130a7385 | ||
|
|
07580c9280 | ||
|
|
6601d90bee | ||
|
|
0df79fed53 | ||
|
|
cf8287e683 | ||
|
|
9f1bdc4cf1 |
@@ -145,6 +145,7 @@ src/devx/
|
||||
└── molecule/ # Optional molecule testing helpers (for Ansible projects)
|
||||
├── discover_runners.py # Dynamic Gitea runner discovery
|
||||
├── distribute_molecule.py # Distribute molecule scenarios across runners (LPT scheduling, --roles-root for multi-role)
|
||||
├── molecule_ci_guard.py # Run molecule with cross-runner fail-fast (--roles-root)
|
||||
├── molecule_all.py # Run all molecule scenarios locally
|
||||
├── molecule_changed.py # Detect which Ansible roles changed and output molecule scenarios
|
||||
├── start_docker.py # Ensure Docker daemon is running for molecule tests
|
||||
|
||||
+68
-50
@@ -2,96 +2,114 @@
|
||||
|
||||
All notable changes to this project will be documented in this file.
|
||||
|
||||
## [0.50.1] - 2026-08-15
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Prefer rootless Docker socket over low-space inner DinD daemon
|
||||
- Check /run/host-docker.sock for host Docker daemon
|
||||
- Use /dev/shm/docker as data-root for inner dockerd
|
||||
- Kill existing dockerd before starting /dev/shm/docker daemon
|
||||
- Use /dev/shm/docker.sock socket for local dockerd
|
||||
- Add --iptables=false to local dockerd
|
||||
- Disable bridge and ip6tables for local dockerd
|
||||
- Use host Docker when root dir is inaccessible (free=0)
|
||||
- Prefer /run/host-docker.sock over inner dockerd
|
||||
- Start local dockerd instead of using low-space inner dockerd
|
||||
- Use container overlay for local dockerd data root
|
||||
- Kill inner dockerd with SIGKILL, use alt socket if alive
|
||||
- Kill dockerd by PID when pkill fails, use /dev/shm for alive daemon
|
||||
- Trust /var/run/docker.sock with free=0 when no inner dockerd exists
|
||||
|
||||
## [0.50.0] - 2026-08-09
|
||||
|
||||
### Features
|
||||
|
||||
- Add 5 standalone lint scripts from infra
|
||||
|
||||
## [0.49.0] - 2026-08-09
|
||||
## [0.50.0] - 2026-08-12
|
||||
|
||||
### Features
|
||||
|
||||
- Sync missing features from v0.49.x line to master
|
||||
## [0.49.5] - 2026-08-07
|
||||
|
||||
## [0.48.2] - 2026-08-09
|
||||
### Performance
|
||||
|
||||
- Skip dep resolution in setup-image with --no-deps
|
||||
## [0.49.4] - 2026-08-07
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Remove dead translation keys and add missing one
|
||||
|
||||
## [0.48.1] - 2026-08-08
|
||||
- Add container.credentials for private registry auth
|
||||
## [0.49.3] - 2026-08-07
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- *(setup)* Extract version from filename for mirror installs
|
||||
- Retry ansible-galaxy collection install on transient timeouts
|
||||
## [0.49.2] - 2026-08-07
|
||||
|
||||
## [0.48.0] - 2026-08-08
|
||||
### Bug Fixes
|
||||
|
||||
- Add fallback URL for tea download
|
||||
## [0.49.1] - 2026-08-07
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Add container images to build-images workflow
|
||||
## [0.49.0] - 2026-08-07
|
||||
|
||||
### Features
|
||||
|
||||
- *(setup)* Mirror Ansible collections from Gitea registry with auth
|
||||
- Add --include-roles and --exclude-roles to distribute_molecule
|
||||
## [0.48.0] - 2026-07-22
|
||||
|
||||
## [0.47.10] - 2026-08-05
|
||||
### Features
|
||||
|
||||
- Extract reusable components from infra and grm into devx
|
||||
|
||||
## [0.49.5] - 2026-08-07
|
||||
|
||||
### Performance
|
||||
|
||||
- Skip dep resolution in setup-image with --no-deps
|
||||
|
||||
## [0.49.4] - 2026-08-07
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Unique molecule container names per CI runner
|
||||
- Add container.credentials for private registry auth
|
||||
|
||||
## [0.47.9] - 2026-08-03
|
||||
## [0.49.3] - 2026-08-07
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Unique molecule container names per CI runner
|
||||
- Retry ansible-galaxy collection install on transient timeouts
|
||||
|
||||
## [0.47.8] - 2026-08-03
|
||||
## [0.49.2] - 2026-08-07
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Increase CI_SCALE_FACTOR default from 4 to 6
|
||||
- Add fallback URL for tea download
|
||||
|
||||
## [0.47.7] - 2026-08-03
|
||||
## [0.49.1] - 2026-08-07
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Scale check_test_speed limits on CI runners
|
||||
- Add container images to build-images workflow
|
||||
|
||||
## [0.47.6] - 2026-08-03
|
||||
## [0.49.0] - 2026-08-07
|
||||
|
||||
### Bug Fixes
|
||||
### Features
|
||||
|
||||
- Configure git auth in setup_image for git+https deps
|
||||
- Add --include-roles and --exclude-roles to distribute_molecule
|
||||
## [0.48.0] - 2026-07-22
|
||||
|
||||
## [0.47.5] - 2026-08-03
|
||||
### Features
|
||||
|
||||
### Bug Fixes
|
||||
- Extract reusable components from infra and grm into devx
|
||||
|
||||
- Push wiki to main branch instead of master
|
||||
## [0.48.0] - 2026-07-22
|
||||
|
||||
## [0.47.4] - 2026-08-03
|
||||
### Features
|
||||
|
||||
### Bug Fixes
|
||||
- Extract reusable components from infra and grm into devx
|
||||
|
||||
- Add User-Agent header to _download in install_tools
|
||||
## [Unreleased]
|
||||
|
||||
### Features
|
||||
|
||||
- Extract reusable components from infra and grm into devx:
|
||||
- `devx.utils.ui.say()` — unified click.echo + logging output
|
||||
- `devx.utils.api.APIClient` — base HTTP API client class with retry logic
|
||||
- `devx.utils.jinja` — Jinja2 environment helpers with Ansible-compatible filters
|
||||
- `devx.i18n.configure_i18n()` — configurable `lang_env_var` and `translations_path_env_var`
|
||||
- `devx.ci.cancel_superseded_runs` — cancel in-flight CI runs for the same PR branch
|
||||
- `devx.ci.check_workflow_artifact_deps` — verify artifact download jobs depend on upload jobs
|
||||
- `devx.ci.check_workflow_tofu_init` — verify tofu-state jobs have a tofu-init step
|
||||
- `devx.tools.check_docker_init` — check Docker Compose services with healthchecks have init: true
|
||||
- `devx.tools.check_ansible_set_fact_to_json` — check set_fact tasks don't misuse to_json
|
||||
- `devx.tools.check_alert_rules` — validate Prometheus alert rules with promtool
|
||||
- Add `jinja2` and `pyyaml` as core dependencies (previously in `deploy` extras only)
|
||||
- Register new CLI commands: `devx ci cancel-superseded-runs`, `devx ci check-workflow-artifact-deps`,
|
||||
`devx ci check-workflow-tofu-init`, `devx tools check-docker-init`,
|
||||
`devx tools check-ansible-set-fact-to-json`, `devx tools check-alert-rules`
|
||||
- Add Makefile targets for all new check tools
|
||||
|
||||
## [0.47.3] - 2026-07-17
|
||||
|
||||
|
||||
@@ -16,12 +16,12 @@ quality badges.
|
||||
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/devx/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/devx/src/branch/master/LICENSE)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/devx/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/devx/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/devx/wiki)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/devx/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/devx/releases)
|
||||
[](https://www.python.org/downloads/)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/devx/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/devx/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/devx/wiki)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/devx/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/devx/releases)
|
||||
[](https://www.python.org/downloads/)
|
||||
|
||||
## Why devx?
|
||||
|
||||
@@ -87,7 +87,7 @@ extra index and list devx in your dependencies:
|
||||
```toml
|
||||
[project]
|
||||
dependencies = [
|
||||
"devx>=0.50.1",
|
||||
"devx>=0.50.0",
|
||||
]
|
||||
|
||||
[tool.pip]
|
||||
@@ -101,8 +101,8 @@ pip install -e .
|
||||
```
|
||||
|
||||
> **Note:** If your project requires a specific devx version, pin it in
|
||||
> `dependencies` (for example, `"devx==0.50.1"`) or use a version constraint
|
||||
> (for example, `"devx>=0.50.1,<0.51"`).
|
||||
> `dependencies` (for example, `"devx==0.50.0"`) or use a version constraint
|
||||
> (for example, `"devx>=0.50.0,<0.51"`).
|
||||
|
||||
### Optional extras
|
||||
|
||||
|
||||
+8
-8
@@ -12,12 +12,12 @@ project to be reusable across all oblachno-oss repositories.
|
||||
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/devx/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/devx/src/branch/master/LICENSE)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/devx/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/devx/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/devx/wiki)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/devx/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/devx/releases)
|
||||
[](https://www.python.org/downloads/)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/devx/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/devx/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/devx/wiki)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/devx/actions)
|
||||
[](https://git.oblachno.oblachno.fyi/oblachno-oss/devx/releases)
|
||||
[](https://www.python.org/downloads/)
|
||||
|
||||
## Overview
|
||||
|
||||
@@ -74,14 +74,14 @@ Add devx to your `pyproject.toml` dependencies and configure the registry:
|
||||
```toml
|
||||
[project]
|
||||
dependencies = [
|
||||
"devx>=0.50.1",
|
||||
"devx>=0.50.0",
|
||||
]
|
||||
|
||||
[tool.pip]
|
||||
extra-index-url = "https://git.oblachno.oblachno.fyi/api/packages/oblachno-oss/pypi/simple"
|
||||
```
|
||||
|
||||
Pin a specific version if needed: `"devx==0.50.1"` or `"devx>=0.50.1,<0.51"`.
|
||||
Pin a specific version if needed: `"devx==0.50.0"` or `"devx>=0.50.0,<0.51"`.
|
||||
|
||||
### Optional extras
|
||||
|
||||
|
||||
@@ -50,6 +50,7 @@ src/devx/
|
||||
├── __init__.py
|
||||
├── discover_runners.py # Dynamic Gitea runner discovery
|
||||
├── distribute_molecule.py # Distribute scenarios across runners
|
||||
├── molecule_ci_guard.py # Run molecule with cross-runner fail-fast
|
||||
├── molecule_all.py # Run all molecule scenarios locally
|
||||
├── start_docker.py # Ensure Docker is available for molecule
|
||||
└── platforms.py # Supported molecule platforms
|
||||
@@ -299,9 +300,9 @@ Distributes files matching a glob pattern across N parallel runners
|
||||
|
||||
### `integration_guard.py`
|
||||
|
||||
Runs pytest with cross-runner failure detection. A background thread polls
|
||||
the Gitea API. If any other integration-tests matrix runner reports failure,
|
||||
the current pytest subprocess is killed and this runner exits early.
|
||||
Runs pytest with the same cross-runner failure detection mechanism used by
|
||||
`molecule_ci_guard`. If any other integration-tests matrix runner reports
|
||||
failure, the current pytest subprocess is killed and this runner exits early.
|
||||
|
||||
## Developer tools (`devx.tools`)
|
||||
|
||||
@@ -381,6 +382,13 @@ the supported OS platform matrix. Supports `--roles-root` for multi-role
|
||||
repositories, `--list` to list scenarios, and `--list-platforms` to list
|
||||
platforms.
|
||||
|
||||
### `molecule_ci_guard.py`
|
||||
|
||||
Runs molecule tests sequentially while polling the Gitea API for other runner
|
||||
failures. If any other molecule matrix runner reports failure, the current
|
||||
molecule subprocess is killed and this runner exits early. Supports both
|
||||
single-role (4-part) and multi-role (5-part) pair encoding.
|
||||
|
||||
### `molecule_all.py`
|
||||
|
||||
Runs all molecule scenarios on all supported OS platforms sequentially.
|
||||
|
||||
@@ -478,6 +478,15 @@ python -m devx.molecule.distribute_molecule --list
|
||||
python -m devx.molecule.distribute_molecule --list-platforms
|
||||
```
|
||||
|
||||
### `molecule_ci_guard.py`
|
||||
|
||||
Runs molecule tests sequentially while polling the Gitea API for other runner
|
||||
failures. Aborts early if another runner fails the same job.
|
||||
|
||||
```bash
|
||||
python -m devx.molecule.molecule_ci_guard [--roles-root <dir>] pair1 pair2 ...
|
||||
```
|
||||
|
||||
### `validate_commit_msg.py`
|
||||
|
||||
Validates commit messages. On feature branches: conventional commits only
|
||||
|
||||
@@ -631,3 +631,29 @@ Options:
|
||||
- `--list-platforms` — list all platforms, one per line
|
||||
- `--roles-root <dir>` — roles root directory for multi-role repos (default:
|
||||
`ansible/roles`)
|
||||
|
||||
### `devx molecule guard`
|
||||
|
||||
Run molecule tests sequentially with CI failure polling. A background thread
|
||||
polls the Gitea API. If any other molecule matrix runner reports failure, the
|
||||
current molecule subprocess is killed and this runner exits early with code 1.
|
||||
|
||||
```bash
|
||||
devx molecule guard pair1 pair2 pair3
|
||||
devx molecule guard --roles-root ansible/roles pair1 pair2
|
||||
```
|
||||
|
||||
Each pair is encoded as:
|
||||
- **Single-role (4-part):** `scenario|platform_name|platform_image|platform_command`
|
||||
- **Multi-role (5-part):** `role|scenario|platform_name|platform_image|platform_command`
|
||||
|
||||
Options:
|
||||
- `--roles-root <dir>` — roles root directory for multi-role repos
|
||||
|
||||
Environment variables:
|
||||
- `GITEA_URL` — base URL of the Gitea instance
|
||||
- `CI_GITEA_TOKEN` — API token with repo access
|
||||
- `RUN_ID` — workflow run ID (`GITHUB_RUN_ID`)
|
||||
- `JOB_NAME` — base job name (`GITHUB_JOB`)
|
||||
- `MATRIX_INDEX` — current matrix index (runner-index)
|
||||
- `GITEA_REPOSITORY` — repository in `owner/repo` format
|
||||
|
||||
@@ -48,12 +48,12 @@ Add devx to your `pyproject.toml`:
|
||||
```toml
|
||||
[project]
|
||||
dependencies = [
|
||||
"devx>=0.50.1",
|
||||
"devx>=0.50.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
"devx>=0.50.1",
|
||||
"devx>=0.50.0",
|
||||
]
|
||||
```
|
||||
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
"""devx — reusable development and CI/CD tools for oblachno-oss projects."""
|
||||
|
||||
__version__ = "0.50.1"
|
||||
__version__ = "0.50.0"
|
||||
|
||||
@@ -52,6 +52,7 @@ REQUIRED_SCRIPTS = [
|
||||
"detect_release_commit.py",
|
||||
"push_badges.py",
|
||||
"distribute_molecule.py",
|
||||
"molecule_ci_guard.py",
|
||||
"validate_commit_msg.py",
|
||||
]
|
||||
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run integration tests with cross-runner failure detection.
|
||||
|
||||
Wraps ``pytest`` with Gitea API polling. If any other integration-tests
|
||||
matrix runner reports failure, the current pytest subprocess is killed
|
||||
and this runner exits early with code 1.
|
||||
Wraps ``pytest`` with the same Gitea API polling mechanism used by
|
||||
``molecule_ci_guard``. If any other integration-tests matrix runner
|
||||
reports failure, the current pytest subprocess is killed and this runner
|
||||
exits early with code 1.
|
||||
|
||||
Usage::
|
||||
|
||||
@@ -34,62 +35,17 @@ import threading
|
||||
import time
|
||||
|
||||
import click
|
||||
import requests
|
||||
|
||||
from devx.config import REPO_NAME, REPO_OWNER
|
||||
from devx.i18n import _
|
||||
from devx.molecule.molecule_ci_guard import (
|
||||
poll_for_other_failures,
|
||||
)
|
||||
from devx.tokens import get_ci_token
|
||||
|
||||
POLL_INTERVAL = 10
|
||||
|
||||
|
||||
def get_running_jobs(gitea_url: str, owner: str, repo: str, token: str, run_id: int) -> list[dict]:
|
||||
"""Return jobs for the given workflow run."""
|
||||
url = f"{gitea_url}/api/v1/repos/{owner}/{repo}/actions/runs/{run_id}/jobs"
|
||||
headers = {"Authorization": f"token {token}"}
|
||||
response = requests.get(url, headers=headers, timeout=30)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
return data.get("jobs", [])
|
||||
|
||||
|
||||
def any_other_runner_failed(jobs: list[dict], current_job_name: str, current_index: int) -> bool:
|
||||
"""Return True if any other matrix job has failed."""
|
||||
for job in jobs:
|
||||
name = job.get("name", "")
|
||||
if not name.startswith(current_job_name):
|
||||
continue
|
||||
if name == f"{current_job_name} ({current_index})" or name == current_job_name:
|
||||
continue
|
||||
if job.get("conclusion") == "failure":
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def poll_for_other_failures(
|
||||
gitea_url: str,
|
||||
owner: str,
|
||||
repo: str,
|
||||
token: str,
|
||||
run_id: int,
|
||||
job_name: str,
|
||||
current_index: int,
|
||||
stop_event: threading.Event,
|
||||
failed_event: threading.Event,
|
||||
) -> None:
|
||||
"""Background thread: poll API and signal if another runner fails."""
|
||||
while not stop_event.is_set():
|
||||
try:
|
||||
jobs = get_running_jobs(gitea_url, owner, repo, token, run_id)
|
||||
if any_other_runner_failed(jobs, job_name, current_index):
|
||||
click.echo(_("Another runner failed. Stopping this runner early."))
|
||||
failed_event.set()
|
||||
return
|
||||
except requests.RequestException as exc:
|
||||
click.echo(_("API poll warning: {exc}", exc=exc))
|
||||
stop_event.wait(POLL_INTERVAL)
|
||||
|
||||
|
||||
@click.command(context_settings={"ignore_unknown_options": True})
|
||||
@click.argument("pytest_args", nargs=-1, type=click.UNPROCESSED, required=True)
|
||||
def cli(pytest_args: tuple[str, ...]) -> None:
|
||||
|
||||
@@ -252,7 +252,7 @@ def commit_and_push(wiki_dir: Path, wiki_url: str, dry_run: bool) -> bool:
|
||||
|
||||
# Push
|
||||
result = subprocess.run( # nosec
|
||||
["git", "push", "--force", wiki_url, "HEAD:main"],
|
||||
["git", "push", "--force", wiki_url, "HEAD:master"],
|
||||
cwd=wiki_dir,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
|
||||
@@ -301,6 +301,13 @@ def molecule_discover_runners(args: tuple[str, ...]) -> None:
|
||||
_run_module("devx.molecule.discover_runners", list(args))
|
||||
|
||||
|
||||
@molecule.command("guard")
|
||||
@click.argument("args", nargs=-1)
|
||||
def molecule_guard(args: tuple[str, ...]) -> None:
|
||||
"""Run molecule tests sequentially with CI failure polling."""
|
||||
_run_module("devx.molecule.molecule_ci_guard", list(args))
|
||||
|
||||
|
||||
@molecule.command("all")
|
||||
@click.argument("args", nargs=-1)
|
||||
def molecule_all(args: tuple[str, ...]) -> None:
|
||||
|
||||
@@ -0,0 +1,284 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run molecule tests sequentially while polling Gitea for other runner failures.
|
||||
|
||||
Each pair is encoded as one of:
|
||||
|
||||
- **Single-role (4-part):** ``scenario|platform_name|platform_image|platform_command``
|
||||
- **Multi-role (5-part):** ``role|scenario|platform_name|platform_image|platform_command``
|
||||
|
||||
Pairs are executed one at a time (molecule scenarios share temp directories and
|
||||
Docker networks, so parallel execution within a single runner is unsafe).
|
||||
|
||||
A background thread polls the Gitea API. If any other molecule matrix runner
|
||||
reports failure, the current molecule subprocess is killed and this runner
|
||||
exits early with code 1.
|
||||
|
||||
Usage::
|
||||
|
||||
# Single-role
|
||||
python3 -m devx.molecule.molecule_ci_guard pair1 pair2 ...
|
||||
# Multi-role
|
||||
python3 -m devx.molecule.molecule_ci_guard --roles-root ansible/roles pair1 pair2 ...
|
||||
|
||||
Environment variables:
|
||||
GITEA_URL Base URL of the Gitea instance.
|
||||
CI_GITEA_API_TOKEN API token with repo access (CI_GITEA_TOKEN accepted for legacy).
|
||||
RUN_ID Workflow run ID (GITHUB_RUN_ID).
|
||||
JOB_NAME Base job name (GITHUB_JOB), e.g. "molecule-tests".
|
||||
MATRIX_INDEX Current matrix index (runner-index).
|
||||
GITEA_REPOSITORY Repository in "owner/repo" format.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import os
|
||||
import signal
|
||||
import subprocess # nosec B404
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
import requests
|
||||
|
||||
from devx.config import REPO_NAME, REPO_OWNER
|
||||
from devx.i18n import _
|
||||
from devx.tokens import get_ci_token
|
||||
|
||||
POLL_INTERVAL = 10
|
||||
|
||||
|
||||
def get_running_jobs(gitea_url: str, owner: str, repo: str, token: str, run_id: int) -> list[dict]:
|
||||
"""Return jobs for the given workflow run."""
|
||||
url = f"{gitea_url}/api/v1/repos/{owner}/{repo}/actions/runs/{run_id}/jobs"
|
||||
headers = {"Authorization": f"token {token}"}
|
||||
response = requests.get(url, headers=headers, timeout=30)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
return data.get("jobs", [])
|
||||
|
||||
|
||||
def any_other_runner_failed(jobs: list[dict], current_job_name: str, current_index: int) -> bool:
|
||||
"""Return True if any other molecule matrix job has failed."""
|
||||
for job in jobs:
|
||||
name = job.get("name", "")
|
||||
if not name.startswith(current_job_name):
|
||||
continue
|
||||
if name == f"{current_job_name} ({current_index})" or name == current_job_name:
|
||||
continue
|
||||
if job.get("conclusion") == "failure":
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def poll_for_other_failures(
|
||||
gitea_url: str,
|
||||
owner: str,
|
||||
repo: str,
|
||||
token: str,
|
||||
run_id: int,
|
||||
job_name: str,
|
||||
current_index: int,
|
||||
stop_event: threading.Event,
|
||||
failed_event: threading.Event,
|
||||
) -> None:
|
||||
"""Background thread: poll API and signal if another runner fails."""
|
||||
while not stop_event.is_set():
|
||||
try:
|
||||
jobs = get_running_jobs(gitea_url, owner, repo, token, run_id)
|
||||
if any_other_runner_failed(jobs, job_name, current_index):
|
||||
click.echo(_("Another molecule runner failed. Stopping this runner early."))
|
||||
failed_event.set()
|
||||
return
|
||||
except requests.RequestException as exc:
|
||||
click.echo(_("API poll warning: {exc}", exc=exc))
|
||||
stop_event.wait(POLL_INTERVAL)
|
||||
|
||||
|
||||
def build_molecule_cmd(scenario: str) -> list[str]:
|
||||
"""Build the molecule command for a scenario."""
|
||||
cmd = ["molecule", "test"]
|
||||
if scenario != "default":
|
||||
cmd.extend(["-s", scenario])
|
||||
return cmd
|
||||
|
||||
|
||||
def parse_pair(pair: str) -> tuple[str, str, str, str, str]:
|
||||
"""Parse a pair string into (role, scenario, platform_name, platform_image, platform_command).
|
||||
|
||||
Supports both 4-part (single-role) and 5-part (multi-role) formats.
|
||||
For 4-part pairs, role is empty (caller uses default role dir).
|
||||
Spaces in the command field are encoded as ``__SPACE__`` to survive
|
||||
shell word-splitting when ``$TEST_PAIRS`` is expanded unquoted.
|
||||
"""
|
||||
parts = pair.split("|")
|
||||
if len(parts) == 4:
|
||||
return "", parts[0], parts[1], parts[2], parts[3].replace("__SPACE__", " ")
|
||||
if len(parts) == 5:
|
||||
return parts[0], parts[1], parts[2], parts[3], parts[4].replace("__SPACE__", " ")
|
||||
raise click.ClickException(f"Invalid pair format: {pair!r} (expected 4 or 5 pipe-delimited parts)")
|
||||
|
||||
|
||||
def build_env_for_pair(pair: str, base_env: dict[str, str]) -> dict[str, str]:
|
||||
"""Build environment for a single molecule pair."""
|
||||
_role, _scenario, platform_name, platform_image, platform_command = parse_pair(pair)
|
||||
env = base_env.copy()
|
||||
env["MOLECULE_PLATFORM_NAME"] = platform_name
|
||||
env["MOLECULE_PLATFORM_IMAGE"] = platform_image
|
||||
if platform_command:
|
||||
env["MOLECULE_PLATFORM_COMMAND"] = platform_command
|
||||
elif "MOLECULE_PLATFORM_COMMAND" in env:
|
||||
del env["MOLECULE_PLATFORM_COMMAND"]
|
||||
env["ANSIBLE_ALLOW_BROKEN_CONDITIONALS"] = "true"
|
||||
# Use a fresh MOLECULE_HOME per pair to avoid stale config cache
|
||||
# from previous CI runs (causes "Instances missing" errors).
|
||||
if "MOLECULE_HOME" not in env:
|
||||
import tempfile
|
||||
|
||||
env["MOLECULE_HOME"] = tempfile.mkdtemp(prefix="molecule-ci-")
|
||||
return env
|
||||
|
||||
|
||||
def resolve_role_dir(role: str, roles_root: Path | None, repo_root: Path) -> Path:
|
||||
"""Resolve the working directory for a molecule pair.
|
||||
|
||||
For multi-role pairs (role non-empty), uses ``roles_root/role``.
|
||||
For single-role pairs, auto-discovers the first role with a molecule/
|
||||
subdirectory under ``repo_root/ansible/roles/``.
|
||||
"""
|
||||
if role:
|
||||
if roles_root is None:
|
||||
roles_root = repo_root / "ansible" / "roles"
|
||||
return roles_root / role
|
||||
roles_dir = repo_root / "ansible" / "roles"
|
||||
if roles_dir.is_dir():
|
||||
role_dirs = sorted(d for d in roles_dir.iterdir() if (d / "molecule").is_dir())
|
||||
if role_dirs:
|
||||
return role_dirs[0]
|
||||
return roles_dir / "role" # will produce a clear "not found" error
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.argument("pairs", nargs=-1, required=True)
|
||||
@click.option(
|
||||
"--roles-root",
|
||||
type=click.Path(exists=True, file_okay=False, path_type=Path),
|
||||
default=None,
|
||||
help="Root directory for multi-role pairs (e.g. ansible/roles). Required when pairs use 5-part format.",
|
||||
)
|
||||
def cli(pairs: tuple[str, ...], roles_root: Path | None) -> None:
|
||||
"""Run molecule pairs sequentially, stop if another CI runner fails."""
|
||||
gitea_url = os.environ.get("GITEA_URL", "")
|
||||
try:
|
||||
token = get_ci_token()
|
||||
except click.ClickException:
|
||||
token = None
|
||||
run_id = int(os.environ.get("RUN_ID", "0"))
|
||||
job_name = os.environ.get("JOB_NAME", "molecule-tests")
|
||||
current_index = int(os.environ.get("MATRIX_INDEX", "0"))
|
||||
repository = os.environ.get("GITEA_REPOSITORY", "")
|
||||
owner, _sep, repo = repository.partition("/")
|
||||
if not owner or not repo:
|
||||
owner, repo = REPO_OWNER, REPO_NAME
|
||||
|
||||
if not all([gitea_url, token, run_id]):
|
||||
click.echo(_("GITEA_URL/CI_GITEA_TOKEN/RUN_ID not set; running without cross-runner cancellation."))
|
||||
|
||||
# When devx is installed as a pip package, __file__ resolves to the
|
||||
# site-packages directory, not the repo root. Use GITHUB_WORKSPACE
|
||||
# (set by Gitea Actions) or cwd as the repo root.
|
||||
repo_root = Path(os.environ.get("GITHUB_WORKSPACE", os.getcwd())).resolve()
|
||||
|
||||
base_env = os.environ.copy()
|
||||
base_env.setdefault("DOCKER_HOST", f"unix:///run/user/{os.getuid()}/docker.sock")
|
||||
base_env.setdefault("ANSIBLE_INJECT_INVOCATION", "1")
|
||||
|
||||
stop_event = threading.Event()
|
||||
failed_event = threading.Event()
|
||||
|
||||
if gitea_url and token and run_id:
|
||||
poller = threading.Thread(
|
||||
target=poll_for_other_failures,
|
||||
args=(
|
||||
gitea_url,
|
||||
owner,
|
||||
repo,
|
||||
token,
|
||||
run_id,
|
||||
job_name,
|
||||
current_index,
|
||||
stop_event,
|
||||
failed_event,
|
||||
),
|
||||
daemon=True,
|
||||
)
|
||||
poller.start()
|
||||
|
||||
try:
|
||||
for pair in pairs:
|
||||
if failed_event.is_set():
|
||||
sys.exit(1)
|
||||
|
||||
role, scenario, platform_name, _img, _cmd = parse_pair(pair)
|
||||
click.echo(_("Running: {scenario} on {platform}", scenario=scenario, platform=platform_name))
|
||||
|
||||
cmd = build_molecule_cmd(scenario)
|
||||
env = build_env_for_pair(pair, base_env)
|
||||
cwd = resolve_role_dir(role, roles_root, repo_root)
|
||||
|
||||
process = subprocess.Popen( # nosec B603
|
||||
cmd,
|
||||
cwd=str(cwd),
|
||||
env=env,
|
||||
preexec_fn=os.setsid,
|
||||
)
|
||||
|
||||
try:
|
||||
while process.poll() is None:
|
||||
if failed_event.is_set():
|
||||
with contextlib.suppress(ProcessLookupError):
|
||||
os.killpg(os.getpgid(process.pid), signal.SIGTERM)
|
||||
try:
|
||||
process.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
with contextlib.suppress(ProcessLookupError):
|
||||
os.killpg(os.getpgid(process.pid), signal.SIGKILL)
|
||||
process.wait()
|
||||
sys.exit(1)
|
||||
time.sleep(1)
|
||||
except KeyboardInterrupt:
|
||||
with contextlib.suppress(ProcessLookupError):
|
||||
os.killpg(os.getpgid(process.pid), signal.SIGTERM)
|
||||
process.wait()
|
||||
sys.exit(1)
|
||||
|
||||
rc = process.returncode
|
||||
|
||||
if rc != 0:
|
||||
click.echo(_("FAILED: {pair} exited with code {code}", pair=pair, code=rc))
|
||||
sys.exit(rc)
|
||||
|
||||
click.echo(_("PASSED: {pair}", pair=pair))
|
||||
|
||||
# Prune Docker data between scenarios to prevent disk exhaustion
|
||||
# in Docker-in-Docker molecule containers (each scenario pulls
|
||||
# hundreds of MB of images that accumulate across pairs).
|
||||
with contextlib.suppress(subprocess.SubprocessError, OSError):
|
||||
subprocess.run( # nosec B603, B607
|
||||
["docker", "system", "prune", "-af", "--volumes"],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
click.echo(_("All molecule tests passed."))
|
||||
finally:
|
||||
stop_event.set()
|
||||
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
cli()
|
||||
@@ -10,12 +10,6 @@ If the host socket is not available, it tries the rootless socket, then
|
||||
starts a local ``dockerd`` with the vfs storage driver (requires
|
||||
privileged container).
|
||||
|
||||
When the host socket IS available but has limited disk space (e.g. an
|
||||
inner DinD daemon writing to a 38 GB container overlay), the script
|
||||
prefers a rootless socket that has more available space. This prevents
|
||||
"no space left on device" errors during molecule tests that pull images
|
||||
and create containers via the Docker daemon.
|
||||
|
||||
Usage::
|
||||
|
||||
python3 -m devx.molecule.start_docker [--timeout 30]
|
||||
@@ -23,10 +17,8 @@ Usage::
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import glob
|
||||
import os
|
||||
import shutil
|
||||
import subprocess # nosec B404
|
||||
import sys
|
||||
import tempfile
|
||||
@@ -40,16 +32,6 @@ DEFAULT_TIMEOUT = 30
|
||||
DOCKER_SOCK = "/var/run/docker.sock"
|
||||
# Rootless socket fallback (e.g. /run/user/994/docker.sock)
|
||||
ROOTLESS_SOCK = f"/run/user/{os.getuid()}/docker.sock"
|
||||
# Host Docker socket mounted by gitea_runner config (see runner config
|
||||
# ``options: "-v /run/user/<uid>/docker.sock:/run/host-docker.sock"``).
|
||||
# This gives CI containers access to the host's rootless Docker daemon,
|
||||
# which has the full host filesystem (e.g. 455 GB) instead of the
|
||||
# container's limited overlay (e.g. 38 GB).
|
||||
HOST_DOCKER_SOCK = "/run/host-docker.sock"
|
||||
# Minimum free bytes for a Docker daemon to be considered usable.
|
||||
# Below this, image pulls and container creation will fail with ENOSPC.
|
||||
# 20 GB leaves room for molecule-test-base (~500 MB) + a few containers.
|
||||
MIN_FREE_BYTES = 20 * 1024**3 # 20 GB
|
||||
|
||||
|
||||
def is_docker_ready() -> bool:
|
||||
@@ -64,46 +46,6 @@ def is_docker_ready() -> bool:
|
||||
return result.returncode == 0
|
||||
|
||||
|
||||
def _get_docker_free_bytes() -> int:
|
||||
"""Get free disk space (bytes) at the Docker daemon's data root.
|
||||
|
||||
Returns 0 if the daemon is not reachable or the data root cannot be
|
||||
determined.
|
||||
"""
|
||||
docker_host = os.environ.get("DOCKER_HOST", f"unix://{DOCKER_SOCK}")
|
||||
try:
|
||||
result = subprocess.run( # nosec B603 B607
|
||||
[
|
||||
"docker",
|
||||
"info",
|
||||
"--format",
|
||||
"{{.DockerRootDir}}",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
check=False,
|
||||
env={**os.environ, "DOCKER_HOST": docker_host},
|
||||
)
|
||||
if result.returncode != 0 or not result.stdout.strip():
|
||||
return 0
|
||||
data_root = result.stdout.strip()
|
||||
if not os.path.exists(data_root):
|
||||
return 0
|
||||
return shutil.disk_usage(data_root).free
|
||||
except (FileNotFoundError, subprocess.TimeoutExpired, OSError):
|
||||
return 0
|
||||
|
||||
|
||||
def _try_socket(sock_path: str) -> bool:
|
||||
"""Set DOCKER_HOST to *sock_path* and check if the daemon is ready.
|
||||
|
||||
Returns ``True`` if the daemon responds, ``False`` otherwise.
|
||||
"""
|
||||
os.environ["DOCKER_HOST"] = f"unix://{sock_path}"
|
||||
return is_docker_ready()
|
||||
|
||||
|
||||
def _diagnose_socket() -> None:
|
||||
"""Print diagnostic info about the Docker socket."""
|
||||
click.echo(f"DOCKER_HOST = {os.environ.get('DOCKER_HOST', '(not set)')}")
|
||||
@@ -155,177 +97,50 @@ def _diagnose_socket() -> None:
|
||||
def start_docker_daemon(timeout: int = DEFAULT_TIMEOUT) -> bool:
|
||||
"""Ensure Docker is ready for molecule tests.
|
||||
|
||||
Tries sockets in this order, preferring ones with enough disk space:
|
||||
|
||||
1. Host rootless socket (``/run/host-docker.sock``) — mounted by the
|
||||
gitea runner config, has access to the host's full filesystem
|
||||
(e.g. 455 GB). Preferred over the inner dockerd.
|
||||
2. Default socket (``/var/run/docker.sock``) — may be an inner dockerd
|
||||
started by the CI image (v29.5.3) with data root on the container's
|
||||
limited overlay (e.g. 38 GB, often 100 % full).
|
||||
3. Other rootless sockets (``/run/user/*/docker.sock``).
|
||||
4. Local ``dockerd`` with vfs storage driver — last resort.
|
||||
First tries the host socket. If that works, sets ``DOCKER_HOST`` and
|
||||
returns immediately. If not, tries the rootless socket. If neither
|
||||
works, starts a local ``dockerd`` with vfs storage driver (requires
|
||||
privileged container).
|
||||
|
||||
Returns ``True`` if Docker is ready, ``False`` if it failed to
|
||||
start within the timeout.
|
||||
"""
|
||||
# Point Docker CLI and Python library to the socket explicitly
|
||||
os.environ["DOCKER_HOST"] = f"unix://{DOCKER_SOCK}"
|
||||
|
||||
# Diagnose socket state
|
||||
click.echo("--- Docker socket diagnostics ---")
|
||||
_diagnose_socket()
|
||||
click.echo("--- End diagnostics ---")
|
||||
|
||||
# Collect candidate sockets in priority order.
|
||||
# The host's rootless Docker socket (mounted at /run/host-docker.sock
|
||||
# by the gitea runner config) is preferred — it has access to the
|
||||
# host's full filesystem instead of the container's limited overlay.
|
||||
candidates: list[str] = []
|
||||
if os.path.exists(HOST_DOCKER_SOCK):
|
||||
candidates.append(HOST_DOCKER_SOCK)
|
||||
if os.path.exists(DOCKER_SOCK):
|
||||
candidates.append(DOCKER_SOCK)
|
||||
if os.path.exists(ROOTLESS_SOCK):
|
||||
candidates.append(ROOTLESS_SOCK)
|
||||
for sock in sorted(glob.glob("/run/user/*/docker.sock")):
|
||||
if sock not in candidates:
|
||||
candidates.append(sock)
|
||||
# Check if host Docker is already available
|
||||
if is_docker_ready():
|
||||
click.echo(_("Docker daemon already running"))
|
||||
return True
|
||||
|
||||
# Try each candidate socket — prefer one with enough free space
|
||||
for sock in candidates:
|
||||
click.echo(f"Trying socket: {sock}")
|
||||
if not _try_socket(sock):
|
||||
# Try rootless socket (e.g. /run/user/994/docker.sock)
|
||||
click.echo(f"Trying rootless socket: {ROOTLESS_SOCK}")
|
||||
os.environ["DOCKER_HOST"] = f"unix://{ROOTLESS_SOCK}"
|
||||
if os.path.exists(ROOTLESS_SOCK) and is_docker_ready():
|
||||
click.echo(_("Docker daemon already running"))
|
||||
return True
|
||||
|
||||
# Scan for any rootless sockets at other UIDs
|
||||
for sock in sorted(glob.glob("/run/user/*/docker.sock")):
|
||||
if sock == ROOTLESS_SOCK:
|
||||
continue
|
||||
free_bytes = _get_docker_free_bytes()
|
||||
free_gb = free_bytes / 1024**3
|
||||
click.echo(f" Docker daemon ready (free space: {free_gb:.1f} GB)")
|
||||
if free_bytes >= MIN_FREE_BYTES:
|
||||
click.echo(f"Trying alternative rootless socket: {sock}")
|
||||
os.environ["DOCKER_HOST"] = f"unix://{sock}"
|
||||
if is_docker_ready():
|
||||
click.echo(_("Docker daemon already running"))
|
||||
return True
|
||||
# If free_bytes is 0, the Docker root dir is on the host filesystem
|
||||
# (not accessible from inside the container). This is expected for
|
||||
# the host's rootless Docker — it has the full host disk.
|
||||
# Only trust this for /run/host-docker.sock (known host socket).
|
||||
# For other sockets (e.g. inner dockerd), free_bytes == 0 means
|
||||
# the data root path doesn't exist inside the container — the
|
||||
# inner dockerd may be using the container's full overlay.
|
||||
if free_bytes == 0 and sock == HOST_DOCKER_SOCK:
|
||||
click.echo("Host rootless Docker root dir not accessible from container, using it")
|
||||
return True
|
||||
# If free_bytes is 0 and there are no dockerd processes inside the
|
||||
# container, the socket is the host's Docker (mounted from outside).
|
||||
# The data root is on the host filesystem and has plenty of space.
|
||||
if free_bytes == 0 and sock == DOCKER_SOCK:
|
||||
has_inner_dockerd = False
|
||||
with contextlib.suppress(Exception):
|
||||
pgrep_result = subprocess.run( # nosec B603 B607
|
||||
["pgrep", "-f", "dockerd"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=5,
|
||||
)
|
||||
has_inner_dockerd = pgrep_result.returncode == 0
|
||||
if not has_inner_dockerd:
|
||||
click.echo("No inner dockerd found, socket is host Docker (data root on host), using it")
|
||||
return True
|
||||
click.echo(f" Insufficient space ({free_gb:.1f} GB), trying next...")
|
||||
|
||||
# No socket with sufficient space found.
|
||||
# Don't fall back to the low-space inner dockerd — it will fail
|
||||
# on image pulls. Instead, kill the inner dockerd, clean up its
|
||||
# data root to free space, and start a new dockerd using the
|
||||
# freed space on the container's overlay.
|
||||
click.echo(_("Host Docker not available, starting local dockerd..."))
|
||||
|
||||
# Kill the inner dockerd (started by the CI image) to free its
|
||||
# data root and socket. The inner dockerd uses the container's
|
||||
# overlay (38G, often 100% full). Killing it frees up the
|
||||
# socket and any space used by its containers/volumes.
|
||||
# Use SIGKILL (-9) since the inner dockerd may not respond to SIGTERM.
|
||||
# Try multiple approaches to ensure the inner dockerd is killed.
|
||||
with contextlib.suppress(Exception):
|
||||
result = subprocess.run( # nosec B603 B607
|
||||
["pgrep", "-af", "dockerd"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=5,
|
||||
)
|
||||
if result.stdout.strip():
|
||||
click.echo(f" dockerd processes before kill: {result.stdout.strip()}")
|
||||
|
||||
for pattern in ["dockerd", "dockerd-entrypoint.sh", "containerd"]:
|
||||
with contextlib.suppress(Exception):
|
||||
subprocess.run( # nosec B603 B607
|
||||
["pkill", "-9", "-f", pattern],
|
||||
check=False,
|
||||
timeout=5,
|
||||
)
|
||||
time.sleep(3)
|
||||
|
||||
# Check if dockerd processes are still alive
|
||||
with contextlib.suppress(Exception):
|
||||
result = subprocess.run( # nosec B603 B607
|
||||
["pgrep", "-af", "dockerd"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=5,
|
||||
)
|
||||
if result.stdout.strip():
|
||||
click.echo(f" dockerd processes after kill: {result.stdout.strip()}")
|
||||
# Try killing by PID directly
|
||||
for pid_str in result.stdout.split("\n"):
|
||||
pid = pid_str.split()[0] if pid_str.strip() else ""
|
||||
if pid:
|
||||
with contextlib.suppress(Exception):
|
||||
os.kill(int(pid), 9)
|
||||
time.sleep(2)
|
||||
|
||||
# Verify the inner dockerd is actually dead. If we can still
|
||||
# connect to /var/run/docker.sock, the old daemon is still running
|
||||
# and we need to use a different socket path.
|
||||
old_daemon_alive = False
|
||||
with contextlib.suppress(Exception):
|
||||
result = subprocess.run( # nosec B603 B607
|
||||
["docker", "info"],
|
||||
env={**os.environ, "DOCKER_HOST": f"unix://{DOCKER_SOCK}"},
|
||||
capture_output=True,
|
||||
timeout=5,
|
||||
)
|
||||
old_daemon_alive = result.returncode == 0
|
||||
|
||||
if old_daemon_alive:
|
||||
click.echo(" Inner dockerd still alive, using alternate socket")
|
||||
local_sock = "/dev/shm/docker.sock" # nosec B108
|
||||
else:
|
||||
local_sock = DOCKER_SOCK
|
||||
|
||||
# Clean up the inner dockerd's data root to free space.
|
||||
# The inner dockerd stores images, containers, and volumes here.
|
||||
# Removing them frees up ~2.4GB on the container's overlay.
|
||||
inner_data_root = "/home/grm-ci-runner-*/.local/share/docker"
|
||||
rm_paths = " ".join(f"{inner_data_root}/{d}" for d in ("overlay2", "image", "volumes", "containers"))
|
||||
with contextlib.suppress(Exception):
|
||||
subprocess.run( # nosec B603 B607
|
||||
["sh", "-c", f"rm -rf {rm_paths}"],
|
||||
check=False,
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
# Use a fresh data root. If the inner dockerd is dead, use the
|
||||
# container's overlay (38G, with freed space). If the inner
|
||||
# dockerd is still alive, use /dev/shm (16G tmpfs) — the overlay
|
||||
# is still full because the inner dockerd's data can't be cleaned.
|
||||
docker_data_root = "/dev/shm/docker" if old_daemon_alive else "/tmp/docker-data" # nosec B108
|
||||
|
||||
# Remove stale socket if present
|
||||
with contextlib.suppress(OSError):
|
||||
os.unlink(local_sock)
|
||||
|
||||
os.environ["DOCKER_HOST"] = f"unix://{local_sock}"
|
||||
# Reset DOCKER_HOST to host socket for local dockerd
|
||||
os.environ["DOCKER_HOST"] = f"unix://{DOCKER_SOCK}"
|
||||
|
||||
# Start local dockerd (requires privileged container)
|
||||
# Use /tmp/docker-data as data root on the container's overlay.
|
||||
# The inner dockerd's data root has been cleaned up, freeing ~2.4GB.
|
||||
# vfs storage driver is used since overlay2 may not work inside
|
||||
# a Docker-in-Docker container without --privileged.
|
||||
log_file = tempfile.NamedTemporaryFile( # noqa: SIM115
|
||||
mode="w", suffix="dockerd.log", delete=False
|
||||
)
|
||||
@@ -335,13 +150,8 @@ def start_docker_daemon(timeout: int = DEFAULT_TIMEOUT) -> bool:
|
||||
"dockerd",
|
||||
"--storage-driver",
|
||||
"vfs",
|
||||
"--data-root",
|
||||
docker_data_root,
|
||||
"--iptables=false",
|
||||
"--ip6tables=false",
|
||||
"--bridge=none",
|
||||
"-H",
|
||||
f"unix://{local_sock}",
|
||||
f"unix://{DOCKER_SOCK}",
|
||||
],
|
||||
stdout=log_file,
|
||||
stderr=subprocess.STDOUT,
|
||||
|
||||
@@ -11,18 +11,6 @@ Usage:
|
||||
The module runs ``make test-unit`` with ``PYTEST_ADDOPTS=--durations=0`` so
|
||||
that pytest emits per-test timing lines alongside the summary. Both the
|
||||
total wall-clock time and individual test durations are parsed and validated.
|
||||
|
||||
CI runner scaling
|
||||
-----------------
|
||||
CI runners (Gitea Actions Docker containers) are typically 5-8x slower than
|
||||
local development machines due to shared CPU, fewer cores, and container
|
||||
overhead. When the ``CI`` environment variable is set (standard CI
|
||||
convention), both the total and per-test limits are multiplied by
|
||||
``CI_SCALE_FACTOR`` (default 6) to account for this. This keeps the local
|
||||
budget strict while preventing false failures on slower CI runners.
|
||||
|
||||
The scale factor can be overridden via the ``DEVX_CI_SCALE_FACTOR``
|
||||
environment variable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -39,12 +27,6 @@ DEFAULT_MAX_SECONDS = 10.0
|
||||
DEFAULT_MAX_SINGLE_SECONDS = 0.5
|
||||
TEST_COMMAND = ["make", "test-unit"]
|
||||
|
||||
# CI runners are typically 5-8x slower than local machines (shared CPU,
|
||||
# fewer cores, container overhead). Scale limits up when running on CI
|
||||
# so the gate catches real regressions, not infrastructure slowness.
|
||||
CI_SCALE_FACTOR = float(os.environ.get("DEVX_CI_SCALE_FACTOR", "6"))
|
||||
_IS_CI = bool(os.environ.get("CI") or os.environ.get("GITEA_ACTIONS"))
|
||||
|
||||
# Matches pytest summary line: "234 passed in 0.70s"
|
||||
_TIMING_RE = re.compile(r"(\d+) passed.* in ([0-9.]+)s")
|
||||
|
||||
@@ -56,13 +38,6 @@ _TIMING_RE = re.compile(r"(\d+) passed.* in ([0-9.]+)s")
|
||||
_DURATION_LINE_RE = re.compile(r"^(\d+\.?\d*)s\s+call\s+(.+)$")
|
||||
|
||||
|
||||
def _ci_scale_limit(limit: float) -> float:
|
||||
"""Scale a time limit by the CI factor when running on CI."""
|
||||
if _IS_CI:
|
||||
return limit * CI_SCALE_FACTOR
|
||||
return limit
|
||||
|
||||
|
||||
def run_tests() -> tuple[str, str]:
|
||||
"""Execute the unit-test suite and return (stdout, stderr).
|
||||
|
||||
@@ -148,38 +123,21 @@ def check_per_test_speed(
|
||||
|
||||
def main(max_seconds: float, max_single_seconds: float) -> None:
|
||||
"""Run tests, parse timings, and enforce both budgets."""
|
||||
# Scale limits for CI runners (slower CPU, fewer workers).
|
||||
effective_max = _ci_scale_limit(max_seconds)
|
||||
effective_single = _ci_scale_limit(max_single_seconds)
|
||||
|
||||
if _IS_CI:
|
||||
click.echo(
|
||||
_(
|
||||
"[check-test-speed] CI environment detected — scaling limits by {factor}x "
|
||||
"(total: {orig}s → {eff}s, per-test: {orig_s}s → {eff_s}s)",
|
||||
factor=CI_SCALE_FACTOR,
|
||||
orig=max_seconds,
|
||||
eff=effective_max,
|
||||
orig_s=max_single_seconds,
|
||||
eff_s=effective_single,
|
||||
)
|
||||
)
|
||||
|
||||
stdout, stderr = run_tests()
|
||||
combined = stdout + "\n" + stderr
|
||||
click.echo(combined, err=False)
|
||||
|
||||
duration = parse_duration(combined)
|
||||
check_speed(duration, effective_max)
|
||||
check_speed(duration, max_seconds)
|
||||
|
||||
if effective_single > 0:
|
||||
if max_single_seconds > 0:
|
||||
per_test = parse_per_test_durations(combined)
|
||||
violations = check_per_test_speed(per_test, effective_single)
|
||||
violations = check_per_test_speed(per_test, max_single_seconds)
|
||||
if violations:
|
||||
msg = _(
|
||||
"Per-test speed check FAILED: {count} test(s) exceed {limit}s limit.",
|
||||
count=len(violations),
|
||||
limit=effective_single,
|
||||
limit=max_single_seconds,
|
||||
)
|
||||
click.echo(f"\n{msg}", err=True)
|
||||
for v in violations:
|
||||
@@ -190,8 +148,8 @@ def main(max_seconds: float, max_single_seconds: float) -> None:
|
||||
_(
|
||||
"Unit tests passed in {duration:.2f}s (under {max}s limit, all tests under {single}s per-test limit).",
|
||||
duration=duration,
|
||||
max=effective_max,
|
||||
single=effective_single,
|
||||
max=max_seconds,
|
||||
single=max_single_seconds,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -65,13 +65,8 @@ def _ensure_target_dir() -> Path:
|
||||
|
||||
|
||||
def _download(url: str, dest: Path) -> None:
|
||||
"""Download a file from ``url`` to ``dest`` with a 60s timeout.
|
||||
|
||||
A User-Agent header is set because some CDNs (e.g. dl.gitea.com)
|
||||
return 403 to requests with Python's default User-Agent.
|
||||
"""
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "devx/install-tools"})
|
||||
with urllib.request.urlopen(req, timeout=60) as resp, open(dest, "wb") as f: # nosec B310
|
||||
"""Download a file from ``url`` to ``dest`` with a 60s timeout."""
|
||||
with urllib.request.urlopen(url, timeout=60) as resp, open(dest, "wb") as f: # nosec B310
|
||||
shutil.copyfileobj(resp, f)
|
||||
|
||||
|
||||
|
||||
@@ -83,17 +83,6 @@ def _install_in_image(
|
||||
username,
|
||||
token,
|
||||
)
|
||||
# Configure git URL rewrite so git+https dependencies can authenticate
|
||||
subprocess.run( # nosec B603, B607
|
||||
[
|
||||
"git",
|
||||
"config",
|
||||
"--global",
|
||||
f"url.https://{username}:{token}@{gitea_host}/.insteadOf",
|
||||
f"https://{gitea_host}/",
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
|
||||
click.echo(f"[setup-image] Linked {opt_venv}" + (f" with [{extras}]" if extras else "") + ".")
|
||||
subprocess.run(cmd, check=True, env=env) # nosec B603
|
||||
|
||||
+39
-15
@@ -783,6 +783,14 @@
|
||||
"ru": "Additional directory to scan (default: scripts, tests). Can be repeated.",
|
||||
"zh": "Additional directory to scan (default: scripts, tests). Can be repeated."
|
||||
},
|
||||
"All molecule tests passed.": {
|
||||
"bg": "All molecule tests passed.",
|
||||
"de": "All molecule tests passed.",
|
||||
"en": "All molecule tests passed.",
|
||||
"pl": "Wszystkie testy molecule zakończone pomyślnie.",
|
||||
"ru": "All molecule tests passed.",
|
||||
"zh": "All molecule tests passed."
|
||||
},
|
||||
"Allow empty tag (PR mode where SHA is concrete).": {
|
||||
"bg": "Позволи празен таг (PR режим, където SHA е конкретен).",
|
||||
"de": "Leeren Tag zulassen (PR-Modus, in dem SHA konkret ist).",
|
||||
@@ -791,13 +799,13 @@
|
||||
"ru": "Разрешить пустой тег (режим PR, где SHA конкретен).",
|
||||
"zh": "允许空标签(SHA 为具体值的 PR 模式)。"
|
||||
},
|
||||
"Another runner failed. Stopping this runner early.": {
|
||||
"bg": "Друг runner се провали. Спиране на този runner по-рано.",
|
||||
"de": "Ein anderer Runner ist fehlgeschlagen. Dieser Runner wird vorzeitig gestoppt.",
|
||||
"en": "Another runner failed. Stopping this runner early.",
|
||||
"pl": "Inny runner zakończył się niepowodzeniem. Wczesne zatrzymanie tego runnera.",
|
||||
"ru": "Другой runner завершился с ошибкой. Останавливаю этот runner досрочно.",
|
||||
"zh": "另一个 runner 失败。提前停止此 runner。"
|
||||
"Another molecule runner failed. Stopping this runner early.": {
|
||||
"bg": "Another molecule runner failed. Stopping this runner early.",
|
||||
"de": "Another molecule runner failed. Stopping this runner early.",
|
||||
"en": "Another molecule runner failed. Stopping this runner early.",
|
||||
"pl": "Inny runner molecule zakończył się niepowodzeniem. Wczesne zatrzymanie tego runnera.",
|
||||
"ru": "Another molecule runner failed. Stopping this runner early.",
|
||||
"zh": "Another molecule runner failed. Stopping this runner early."
|
||||
},
|
||||
"Assigned {count} files to runner {runner_index}": {
|
||||
"bg": "Assigned {count} files to runner {runner_index}",
|
||||
@@ -1487,6 +1495,14 @@
|
||||
"ru": "FAILED: {count} undocumented dependency/ies",
|
||||
"zh": "FAILED: {count} undocumented dependency/ies"
|
||||
},
|
||||
"FAILED: {pair} exited with code {code}": {
|
||||
"bg": "FAILED: {pair} exited with code {code}",
|
||||
"de": "FAILED: {pair} exited with code {code}",
|
||||
"en": "FAILED: {pair} exited with code {code}",
|
||||
"pl": "NIEUDANE: {pair} zakończone kodem {code}",
|
||||
"ru": "FAILED: {pair} exited with code {code}",
|
||||
"zh": "FAILED: {pair} exited with code {code}"
|
||||
},
|
||||
"Failed images: {names}": {
|
||||
"bg": "Failed images: {names}",
|
||||
"de": "Failed images: {names}",
|
||||
@@ -2247,6 +2263,14 @@
|
||||
"ru": "PASS: All documentation checks passed!",
|
||||
"zh": "PASS: All documentation checks passed!"
|
||||
},
|
||||
"PASSED: {pair}": {
|
||||
"bg": "PASSED: {pair}",
|
||||
"de": "PASSED: {pair}",
|
||||
"en": "PASSED: {pair}",
|
||||
"pl": "UDANE: {pair}",
|
||||
"ru": "PASSED: {pair}",
|
||||
"zh": "PASSED: {pair}"
|
||||
},
|
||||
"PR #{pr} rebased successfully. A new CI run will start automatically.\nIf auto-merge is enabled (ready-to-merge label), the next CI run\nwill attempt to merge this PR.": {
|
||||
"bg": "PR #{pr} rebased successfully. A new CI run will start automatically.\nIf auto-merge is enabled (ready-to-merge label), the next CI run\nwill attempt to merge this PR.",
|
||||
"de": "PR #{pr} rebased successfully. A new CI run will start automatically.\nIf auto-merge is enabled (ready-to-merge label), the next CI run\nwill attempt to merge this PR.",
|
||||
@@ -2719,6 +2743,14 @@
|
||||
"ru": "Running: {cmd}",
|
||||
"zh": "Running: {cmd}"
|
||||
},
|
||||
"Running: {scenario} on {platform}": {
|
||||
"bg": "Running: {scenario} on {platform}",
|
||||
"de": "Running: {scenario} on {platform}",
|
||||
"en": "Running: {scenario} on {platform}",
|
||||
"pl": "Uruchamianie: {scenario} na {platform}",
|
||||
"ru": "Running: {scenario} on {platform}",
|
||||
"zh": "Running: {scenario} on {platform}"
|
||||
},
|
||||
"SSH key set up successfully": {
|
||||
"bg": "SSH ключът е настроен успешно",
|
||||
"de": "SSH-Schlüssel erfolgreich eingerichtet",
|
||||
@@ -3790,13 +3822,5 @@
|
||||
"pl": "importlib.reload({mod}) called {n} time(s) in test '{test}' — odd count leaves module in modified state. Add a final reload to restore defaults or wrap in try/finally.",
|
||||
"ru": "importlib.reload({mod}) called {n} time(s) in test '{test}' — odd count leaves module in modified state. Add a final reload to restore defaults or wrap in try/finally.",
|
||||
"zh": "importlib.reload({mod}) called {n} time(s) in test '{test}' — odd count leaves module in modified state. Add a final reload to restore defaults or wrap in try/finally."
|
||||
},
|
||||
"[check-test-speed] CI environment detected — scaling limits by {factor}x (total: {orig}s → {eff}s, per-test: {orig_s}s → {eff_s}s)": {
|
||||
"en": "[check-test-speed] CI environment detected — scaling limits by {factor}x (total: {orig}s → {eff}s, per-test: {orig_s}s → {eff_s}s)",
|
||||
"bg": "[check-test-speed] CI environment detected — scaling limits by {factor}x (total: {orig}s → {eff}s, per-test: {orig_s}s → {eff_s}s)",
|
||||
"de": "[check-test-speed] CI environment detected — scaling limits by {factor}x (total: {orig}s → {eff}s, per-test: {orig_s}s → {eff_s}s)",
|
||||
"pl": "[check-test-speed] CI environment detected — scaling limits by {factor}x (total: {orig}s → {eff}s, per-test: {orig_s}s → {eff_s}s)",
|
||||
"ru": "[check-test-speed] CI environment detected — scaling limits by {factor}x (total: {orig}s → {eff}s, per-test: {orig_s}s → {eff_s}s)",
|
||||
"zh": "[check-test-speed] CI environment detected — scaling limits by {factor}x (total: {orig}s → {eff}s, per-test: {orig_s}s → {eff_s}s)"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,7 +10,6 @@ from devx.tools.check_test_speed import (
|
||||
DEFAULT_MAX_SECONDS,
|
||||
DEFAULT_MAX_SINGLE_SECONDS,
|
||||
TEST_COMMAND,
|
||||
_ci_scale_limit,
|
||||
check_per_test_speed,
|
||||
check_speed,
|
||||
cli,
|
||||
@@ -145,26 +144,7 @@ def test_main_module_block() -> None:
|
||||
mock_cli.assert_called_once_with([])
|
||||
|
||||
|
||||
class TestCiScaleLimit:
|
||||
def test_no_scaling_when_not_ci(self) -> None:
|
||||
with patch("devx.tools.check_test_speed._IS_CI", False):
|
||||
assert _ci_scale_limit(10.0) == 10.0
|
||||
assert _ci_scale_limit(0.5) == 0.5
|
||||
|
||||
def test_scales_when_ci(self) -> None:
|
||||
with patch("devx.tools.check_test_speed._IS_CI", True):
|
||||
with patch("devx.tools.check_test_speed.CI_SCALE_FACTOR", 4.0):
|
||||
assert _ci_scale_limit(10.0) == 40.0
|
||||
assert _ci_scale_limit(0.5) == 2.0
|
||||
|
||||
def test_custom_scale_factor(self) -> None:
|
||||
with patch("devx.tools.check_test_speed._IS_CI", True):
|
||||
with patch("devx.tools.check_test_speed.CI_SCALE_FACTOR", 2.5):
|
||||
assert _ci_scale_limit(10.0) == 25.0
|
||||
|
||||
|
||||
class TestMain:
|
||||
@patch("devx.tools.check_test_speed._IS_CI", False)
|
||||
@patch("devx.tools.check_test_speed.run_tests")
|
||||
@patch("devx.tools.check_test_speed.parse_duration")
|
||||
@patch("devx.tools.check_test_speed.check_speed")
|
||||
@@ -194,7 +174,6 @@ class TestMain:
|
||||
mock_parse_per.assert_called_once()
|
||||
mock_check_per.assert_called_once_with([], DEFAULT_MAX_SINGLE_SECONDS)
|
||||
|
||||
@patch("devx.tools.check_test_speed._IS_CI", False)
|
||||
@patch("devx.tools.check_test_speed.run_tests")
|
||||
@patch("devx.tools.check_test_speed.parse_duration")
|
||||
def test_slow_total_exits(
|
||||
@@ -210,7 +189,6 @@ class TestMain:
|
||||
assert result.exit_code == 1
|
||||
assert "too slow" in result.output.lower()
|
||||
|
||||
@patch("devx.tools.check_test_speed._IS_CI", False)
|
||||
@patch("devx.tools.check_test_speed.run_tests")
|
||||
@patch("devx.tools.check_test_speed.parse_duration")
|
||||
@patch("devx.tools.check_test_speed.check_speed")
|
||||
@@ -235,7 +213,6 @@ class TestMain:
|
||||
assert "Per-test speed check FAILED" in result.output
|
||||
assert "test_slow" in result.output
|
||||
|
||||
@patch("devx.tools.check_test_speed._IS_CI", False)
|
||||
@patch("devx.tools.check_test_speed.run_tests")
|
||||
def test_parse_failure_exits(
|
||||
self,
|
||||
@@ -248,7 +225,6 @@ class TestMain:
|
||||
assert result.exit_code == 1
|
||||
assert "Could not parse" in result.output
|
||||
|
||||
@patch("devx.tools.check_test_speed._IS_CI", False)
|
||||
@patch("devx.tools.check_test_speed.run_tests")
|
||||
@patch("devx.tools.check_test_speed.parse_duration")
|
||||
@patch("devx.tools.check_test_speed.check_speed")
|
||||
@@ -272,7 +248,6 @@ class TestMain:
|
||||
assert result.exit_code == 0
|
||||
mock_check.assert_called_once_with(0.5, 1.5)
|
||||
|
||||
@patch("devx.tools.check_test_speed._IS_CI", False)
|
||||
@patch("devx.tools.check_test_speed.run_tests")
|
||||
@patch("devx.tools.check_test_speed.parse_duration")
|
||||
@patch("devx.tools.check_test_speed.check_speed")
|
||||
@@ -295,7 +270,6 @@ class TestMain:
|
||||
mock_parse_per.assert_not_called()
|
||||
mock_check_per.assert_not_called()
|
||||
|
||||
@patch("devx.tools.check_test_speed._IS_CI", False)
|
||||
@patch("devx.tools.check_test_speed.run_tests")
|
||||
@patch("devx.tools.check_test_speed.parse_duration")
|
||||
@patch("devx.tools.check_test_speed.check_speed")
|
||||
@@ -318,32 +292,3 @@ class TestMain:
|
||||
result = runner.invoke(cli, ["--max-single-seconds", "1.0"])
|
||||
assert result.exit_code == 0
|
||||
mock_check_per.assert_called_once_with([], 1.0)
|
||||
|
||||
@patch("devx.tools.check_test_speed._IS_CI", True)
|
||||
@patch("devx.tools.check_test_speed.CI_SCALE_FACTOR", 4.0)
|
||||
@patch("devx.tools.check_test_speed.run_tests")
|
||||
@patch("devx.tools.check_test_speed.parse_duration")
|
||||
@patch("devx.tools.check_test_speed.check_speed")
|
||||
@patch("devx.tools.check_test_speed.parse_per_test_durations")
|
||||
@patch("devx.tools.check_test_speed.check_per_test_speed")
|
||||
def test_ci_scales_limits(
|
||||
self,
|
||||
mock_check_per: MagicMock,
|
||||
mock_parse_per: MagicMock,
|
||||
mock_check: MagicMock,
|
||||
mock_parse: MagicMock,
|
||||
mock_run: MagicMock,
|
||||
) -> None:
|
||||
mock_run.return_value = ("out\n", "err\n")
|
||||
mock_parse.return_value = 30.0 # would fail local (10s) but pass CI (40s)
|
||||
mock_parse_per.return_value = []
|
||||
mock_check_per.return_value = []
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, [])
|
||||
assert result.exit_code == 0
|
||||
assert "CI environment detected" in result.output
|
||||
assert "scaling limits by 4.0x" in result.output
|
||||
# check_speed called with scaled limit
|
||||
mock_check.assert_called_once_with(30.0, 40.0)
|
||||
mock_check_per.assert_called_once_with([], 2.0)
|
||||
|
||||
@@ -40,6 +40,7 @@ class TestCliGroups:
|
||||
result = runner.invoke(cli, ["molecule", "--help"])
|
||||
assert result.exit_code == 0
|
||||
assert "distribute" in result.output
|
||||
assert "guard" in result.output
|
||||
assert "all" in result.output
|
||||
|
||||
|
||||
@@ -230,6 +231,13 @@ class TestMoleculeCommands:
|
||||
assert result.exit_code == 0
|
||||
mock_run.assert_called_once_with("devx.molecule.discover_runners", [])
|
||||
|
||||
@patch("devx.cli._run_module")
|
||||
def test_molecule_guard(self, mock_run: MagicMock) -> None:
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["molecule", "guard"])
|
||||
assert result.exit_code == 0
|
||||
mock_run.assert_called_once_with("devx.molecule.molecule_ci_guard", [])
|
||||
|
||||
@patch("devx.cli._run_module")
|
||||
def test_molecule_all(self, mock_run: MagicMock) -> None:
|
||||
runner = CliRunner()
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import platform
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
@@ -67,10 +66,6 @@ class TestDownload:
|
||||
with patch("urllib.request.urlopen", return_value=_FakeResponse()) as mock_urlopen:
|
||||
install_tools._download("https://example.com/file", dest)
|
||||
mock_urlopen.assert_called_once()
|
||||
call_args = mock_urlopen.call_args
|
||||
req = call_args.args[0]
|
||||
assert isinstance(req, urllib.request.Request)
|
||||
assert req.get_header("User-agent") == "devx/install-tools"
|
||||
assert dest.read_bytes() == b"data"
|
||||
|
||||
|
||||
|
||||
@@ -7,7 +7,6 @@ import subprocess # nosec B404
|
||||
import time
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from click.testing import CliRunner
|
||||
|
||||
from devx.ci.integration_guard import cli
|
||||
@@ -112,8 +111,9 @@ class TestCli:
|
||||
clear=True,
|
||||
),
|
||||
patch("devx.ci.integration_guard.POLL_INTERVAL", 0.01),
|
||||
patch("devx.molecule.molecule_ci_guard.POLL_INTERVAL", 0.01),
|
||||
patch("devx.ci.integration_guard.subprocess.Popen") as mock_popen,
|
||||
patch("devx.ci.integration_guard.get_running_jobs", side_effect=get_jobs_side_effect),
|
||||
patch("devx.molecule.molecule_ci_guard.get_running_jobs", side_effect=get_jobs_side_effect),
|
||||
patch("os.killpg") as mock_killpg,
|
||||
patch("os.getpgid") as mock_getpgid,
|
||||
patch("time.sleep", side_effect=lambda x: real_sleep(0)),
|
||||
@@ -158,8 +158,9 @@ class TestCli:
|
||||
clear=True,
|
||||
),
|
||||
patch("devx.ci.integration_guard.POLL_INTERVAL", 0.01),
|
||||
patch("devx.molecule.molecule_ci_guard.POLL_INTERVAL", 0.01),
|
||||
patch("devx.ci.integration_guard.subprocess.Popen") as mock_popen,
|
||||
patch("devx.ci.integration_guard.get_running_jobs", side_effect=get_jobs_side_effect),
|
||||
patch("devx.molecule.molecule_ci_guard.get_running_jobs", side_effect=get_jobs_side_effect),
|
||||
patch("os.killpg", side_effect=ProcessLookupError("no such process")),
|
||||
patch("os.getpgid") as mock_getpgid,
|
||||
patch("time.sleep", side_effect=lambda x: real_sleep(0)),
|
||||
@@ -202,8 +203,9 @@ class TestCli:
|
||||
clear=True,
|
||||
),
|
||||
patch("devx.ci.integration_guard.POLL_INTERVAL", 0.01),
|
||||
patch("devx.molecule.molecule_ci_guard.POLL_INTERVAL", 0.01),
|
||||
patch("devx.ci.integration_guard.subprocess.Popen") as mock_popen,
|
||||
patch("devx.ci.integration_guard.get_running_jobs", side_effect=get_jobs_side_effect),
|
||||
patch("devx.molecule.molecule_ci_guard.get_running_jobs", side_effect=get_jobs_side_effect),
|
||||
patch("os.killpg") as mock_killpg,
|
||||
patch("os.getpgid") as mock_getpgid,
|
||||
patch("time.sleep", side_effect=lambda x: real_sleep(0)),
|
||||
@@ -287,125 +289,3 @@ def test_main_module_block() -> None:
|
||||
namespace = dict(ig.__dict__)
|
||||
exec(compile(source, ig.__file__, "exec"), namespace)
|
||||
assert callable(namespace["cli"])
|
||||
|
||||
|
||||
class TestGetRunningJobs:
|
||||
def test_returns_jobs(self) -> None:
|
||||
with patch("devx.ci.integration_guard.requests.get") as mock_get:
|
||||
mock_response = MagicMock()
|
||||
mock_response.json.return_value = {
|
||||
"jobs": [
|
||||
{"name": "integration-tests (0)", "conclusion": "success"},
|
||||
{"name": "integration-tests (1)", "conclusion": "failure"},
|
||||
]
|
||||
}
|
||||
mock_response.raise_for_status.return_value = None
|
||||
mock_get.return_value = mock_response
|
||||
|
||||
from devx.ci.integration_guard import get_running_jobs
|
||||
|
||||
jobs = get_running_jobs("https://gitea.example", "owner", "repo", "token", 123)
|
||||
assert len(jobs) == 2
|
||||
mock_get.assert_called_once()
|
||||
|
||||
def test_raises_on_request_error(self) -> None:
|
||||
import requests
|
||||
|
||||
with patch("devx.ci.integration_guard.requests.get") as mock_get:
|
||||
mock_get.side_effect = requests.RequestException("boom")
|
||||
with pytest.raises(requests.RequestException):
|
||||
from devx.ci.integration_guard import get_running_jobs
|
||||
|
||||
get_running_jobs("https://gitea.example", "owner", "repo", "token", 123)
|
||||
|
||||
|
||||
class TestAnyOtherRunnerFailed:
|
||||
def test_detects_other_failure(self) -> None:
|
||||
from devx.ci.integration_guard import any_other_runner_failed
|
||||
|
||||
jobs = [
|
||||
{"name": "integration-tests (0)", "conclusion": "success"},
|
||||
{"name": "integration-tests (1)", "conclusion": "failure"},
|
||||
{"name": "integration-tests (2)", "conclusion": "running"},
|
||||
]
|
||||
assert any_other_runner_failed(jobs, "integration-tests", 0) is True
|
||||
|
||||
def test_ignores_current_runner(self) -> None:
|
||||
from devx.ci.integration_guard import any_other_runner_failed
|
||||
|
||||
jobs = [
|
||||
{"name": "integration-tests (0)", "conclusion": "failure"},
|
||||
{"name": "integration-tests (1)", "conclusion": "success"},
|
||||
]
|
||||
assert any_other_runner_failed(jobs, "integration-tests", 0) is False
|
||||
|
||||
def test_ignores_non_matching_jobs(self) -> None:
|
||||
from devx.ci.integration_guard import any_other_runner_failed
|
||||
|
||||
jobs = [
|
||||
{"name": "quality", "conclusion": "failure"},
|
||||
{"name": "integration-tests (1)", "conclusion": "success"},
|
||||
]
|
||||
assert any_other_runner_failed(jobs, "integration-tests", 0) is False
|
||||
|
||||
|
||||
class TestPollForOtherFailures:
|
||||
def test_sets_failed_event_when_other_runner_fails(self) -> None:
|
||||
from devx.ci.integration_guard import poll_for_other_failures
|
||||
|
||||
stop_event = MagicMock()
|
||||
failed_event = MagicMock()
|
||||
|
||||
def side_effect(*args, **kwargs):
|
||||
if stop_event.wait.call_count < 1:
|
||||
return [
|
||||
{"name": "integration-tests (0)", "conclusion": "success"},
|
||||
{"name": "integration-tests (1)", "conclusion": "failure"},
|
||||
]
|
||||
return []
|
||||
|
||||
with patch("devx.ci.integration_guard.get_running_jobs") as mock_get_jobs:
|
||||
mock_get_jobs.side_effect = side_effect
|
||||
stop_event.is_set.side_effect = [False, False]
|
||||
stop_event.wait.return_value = True
|
||||
|
||||
poll_for_other_failures(
|
||||
"https://gitea.example",
|
||||
"owner",
|
||||
"repo",
|
||||
"token",
|
||||
123,
|
||||
"integration-tests",
|
||||
0,
|
||||
stop_event,
|
||||
failed_event,
|
||||
)
|
||||
|
||||
failed_event.set.assert_called_once()
|
||||
|
||||
def test_poll_warns_on_api_error(self) -> None:
|
||||
import requests
|
||||
|
||||
from devx.ci.integration_guard import poll_for_other_failures
|
||||
|
||||
stop_event = MagicMock()
|
||||
failed_event = MagicMock()
|
||||
|
||||
with patch("devx.ci.integration_guard.get_running_jobs") as mock_get_jobs:
|
||||
mock_get_jobs.side_effect = requests.RequestException("boom")
|
||||
stop_event.is_set.side_effect = [False, True]
|
||||
stop_event.wait.return_value = True
|
||||
|
||||
poll_for_other_failures(
|
||||
"https://gitea.example",
|
||||
"owner",
|
||||
"repo",
|
||||
"token",
|
||||
123,
|
||||
"integration-tests",
|
||||
0,
|
||||
stop_event,
|
||||
failed_event,
|
||||
)
|
||||
|
||||
failed_event.set.assert_not_called()
|
||||
|
||||
@@ -0,0 +1,581 @@
|
||||
"""Unit tests for scripts/ci/molecule_ci_guard.py."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess # nosec B404
|
||||
import time
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import click
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from devx.molecule.molecule_ci_guard import (
|
||||
any_other_runner_failed,
|
||||
build_env_for_pair,
|
||||
build_molecule_cmd,
|
||||
cli,
|
||||
get_running_jobs,
|
||||
parse_pair,
|
||||
poll_for_other_failures,
|
||||
resolve_role_dir,
|
||||
)
|
||||
|
||||
|
||||
class TestGetRunningJobs:
|
||||
def test_returns_jobs(self) -> None:
|
||||
with patch("devx.molecule.molecule_ci_guard.requests.get") as mock_get:
|
||||
mock_response = MagicMock()
|
||||
mock_response.json.return_value = {
|
||||
"jobs": [
|
||||
{"name": "molecule-tests (0)", "conclusion": "success"},
|
||||
{"name": "molecule-tests (1)", "conclusion": "failure"},
|
||||
]
|
||||
}
|
||||
mock_response.raise_for_status.return_value = None
|
||||
mock_get.return_value = mock_response
|
||||
|
||||
jobs = get_running_jobs("https://gitea.example", "owner", "repo", "token", 123)
|
||||
assert len(jobs) == 2
|
||||
mock_get.assert_called_once()
|
||||
|
||||
def test_raises_on_request_error(self) -> None:
|
||||
with patch("devx.molecule.molecule_ci_guard.requests.get") as mock_get:
|
||||
mock_get.side_effect = requests.RequestException("boom")
|
||||
with pytest.raises(requests.RequestException):
|
||||
get_running_jobs("https://gitea.example", "owner", "repo", "token", 123)
|
||||
|
||||
|
||||
class TestAnyOtherRunnerFailed:
|
||||
def test_detects_other_failure(self) -> None:
|
||||
jobs = [
|
||||
{"name": "molecule-tests (0)", "conclusion": "success"},
|
||||
{"name": "molecule-tests (1)", "conclusion": "failure"},
|
||||
{"name": "molecule-tests (2)", "conclusion": "running"},
|
||||
]
|
||||
assert any_other_runner_failed(jobs, "molecule-tests", 0) is True
|
||||
|
||||
def test_ignores_current_runner(self) -> None:
|
||||
jobs = [
|
||||
{"name": "molecule-tests (0)", "conclusion": "failure"},
|
||||
{"name": "molecule-tests (1)", "conclusion": "success"},
|
||||
]
|
||||
assert any_other_runner_failed(jobs, "molecule-tests", 0) is False
|
||||
|
||||
def test_ignores_non_molecule_jobs(self) -> None:
|
||||
jobs = [
|
||||
{"name": "quality", "conclusion": "failure"},
|
||||
{"name": "molecule-tests (1)", "conclusion": "success"},
|
||||
]
|
||||
assert any_other_runner_failed(jobs, "molecule-tests", 0) is False
|
||||
|
||||
|
||||
class TestBuildMoleculeCmd:
|
||||
def test_default_scenario(self) -> None:
|
||||
assert build_molecule_cmd("default") == ["molecule", "test"]
|
||||
|
||||
def test_named_scenario(self) -> None:
|
||||
assert build_molecule_cmd("lifecycle") == ["molecule", "test", "-s", "lifecycle"]
|
||||
|
||||
|
||||
class TestBuildEnvForPair:
|
||||
def test_with_command(self) -> None:
|
||||
env = build_env_for_pair("default|ubuntu-2204|img:latest|/lib/systemd/systemd", {})
|
||||
assert env["MOLECULE_PLATFORM_NAME"] == "ubuntu-2204"
|
||||
assert env["MOLECULE_PLATFORM_IMAGE"] == "img:latest"
|
||||
assert env["MOLECULE_PLATFORM_COMMAND"] == "/lib/systemd/systemd"
|
||||
assert env["ANSIBLE_ALLOW_BROKEN_CONDITIONALS"] == "true"
|
||||
|
||||
def test_without_command(self) -> None:
|
||||
env = build_env_for_pair("default|ubuntu-2204|img:latest|", {})
|
||||
assert env["MOLECULE_PLATFORM_NAME"] == "ubuntu-2204"
|
||||
assert "MOLECULE_PLATFORM_COMMAND" not in env
|
||||
|
||||
def test_without_command_removes_existing(self) -> None:
|
||||
env = build_env_for_pair("default|ubuntu-2204|img:latest|", {"MOLECULE_PLATFORM_COMMAND": "old"})
|
||||
assert "MOLECULE_PLATFORM_COMMAND" not in env
|
||||
|
||||
def test_preserves_existing_molecule_home(self) -> None:
|
||||
"""When MOLECULE_HOME is already set, it is not overridden."""
|
||||
env = build_env_for_pair("default|ubuntu-2204|img:latest|", {"MOLECULE_HOME": "/custom/home"})
|
||||
assert env["MOLECULE_HOME"] == "/custom/home"
|
||||
|
||||
|
||||
class TestPollForOtherFailures:
|
||||
def test_sets_failed_event_when_other_runner_fails(self) -> None:
|
||||
stop_event = MagicMock()
|
||||
failed_event = MagicMock()
|
||||
|
||||
def side_effect(*args, **kwargs):
|
||||
if stop_event.wait.call_count < 1:
|
||||
return [
|
||||
{"name": "molecule-tests (0)", "conclusion": "success"},
|
||||
{"name": "molecule-tests (1)", "conclusion": "failure"},
|
||||
]
|
||||
return []
|
||||
|
||||
with patch("devx.molecule.molecule_ci_guard.get_running_jobs") as mock_get_jobs:
|
||||
mock_get_jobs.side_effect = side_effect
|
||||
stop_event.is_set.side_effect = [False, False]
|
||||
stop_event.wait.return_value = True
|
||||
|
||||
poll_for_other_failures(
|
||||
"https://gitea.example",
|
||||
"owner",
|
||||
"repo",
|
||||
"token",
|
||||
123,
|
||||
"molecule-tests",
|
||||
0,
|
||||
stop_event,
|
||||
failed_event,
|
||||
)
|
||||
|
||||
failed_event.set.assert_called_once()
|
||||
|
||||
def test_poll_warns_on_api_error(self) -> None:
|
||||
stop_event = MagicMock()
|
||||
failed_event = MagicMock()
|
||||
|
||||
with patch("devx.molecule.molecule_ci_guard.get_running_jobs") as mock_get_jobs:
|
||||
mock_get_jobs.side_effect = requests.RequestException("boom")
|
||||
stop_event.is_set.side_effect = [False, True]
|
||||
stop_event.wait.return_value = True
|
||||
|
||||
poll_for_other_failures(
|
||||
"https://gitea.example",
|
||||
"owner",
|
||||
"repo",
|
||||
"token",
|
||||
123,
|
||||
"molecule-tests",
|
||||
0,
|
||||
stop_event,
|
||||
failed_event,
|
||||
)
|
||||
|
||||
failed_event.set.assert_not_called()
|
||||
|
||||
|
||||
class TestCli:
|
||||
def test_all_pass(self) -> None:
|
||||
from click.testing import CliRunner
|
||||
|
||||
with (
|
||||
patch("devx.molecule.molecule_ci_guard.subprocess.Popen") as mock_popen,
|
||||
patch("devx.molecule.molecule_ci_guard.subprocess.run"),
|
||||
patch("devx.molecule.molecule_ci_guard.subprocess.run") as mock_run,
|
||||
patch("time.sleep"),
|
||||
):
|
||||
proc = MagicMock()
|
||||
proc.poll.return_value = 0
|
||||
proc.returncode = 0
|
||||
mock_popen.return_value = proc
|
||||
mock_run.return_value = MagicMock(returncode=0)
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"])
|
||||
assert result.exit_code == 0
|
||||
assert "All molecule tests passed" in result.output
|
||||
# Verify Docker prune was called between scenarios
|
||||
mock_run.assert_called_once_with(
|
||||
["docker", "system", "prune", "-af", "--volumes"],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
@patch("devx.molecule.molecule_ci_guard.get_ci_token", side_effect=click.ClickException("no token"))
|
||||
def test_missing_token_runs_without_polling(self, mock_token: MagicMock) -> None:
|
||||
"""When no token is available, cross-runner polling is skipped."""
|
||||
from click.testing import CliRunner
|
||||
|
||||
with (
|
||||
patch("devx.molecule.molecule_ci_guard.subprocess.Popen") as mock_popen,
|
||||
patch("devx.molecule.molecule_ci_guard.subprocess.run"),
|
||||
patch("devx.molecule.molecule_ci_guard.subprocess.run") as mock_run,
|
||||
patch("devx.molecule.molecule_ci_guard.poll_for_other_failures") as mock_poll,
|
||||
patch("time.sleep"),
|
||||
):
|
||||
proc = MagicMock()
|
||||
proc.poll.return_value = 0
|
||||
proc.returncode = 0
|
||||
mock_popen.return_value = proc
|
||||
mock_run.return_value = MagicMock(returncode=0)
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"])
|
||||
assert result.exit_code == 0
|
||||
mock_poll.assert_not_called()
|
||||
|
||||
def test_invalid_pair_format_raises(self) -> None:
|
||||
"""Pair with fewer than 2 parts should raise."""
|
||||
from click.testing import CliRunner
|
||||
|
||||
with (
|
||||
patch("devx.molecule.molecule_ci_guard.subprocess.Popen"),
|
||||
patch("devx.molecule.molecule_ci_guard.subprocess.run"),
|
||||
):
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["invalid_no_pipe"])
|
||||
assert result.exit_code != 0
|
||||
assert "Invalid pair format" in result.output
|
||||
|
||||
def test_failure_exits_nonzero(self) -> None:
|
||||
from click.testing import CliRunner
|
||||
|
||||
with (
|
||||
patch("devx.molecule.molecule_ci_guard.subprocess.Popen") as mock_popen,
|
||||
patch("devx.molecule.molecule_ci_guard.subprocess.run"),
|
||||
patch("devx.molecule.molecule_ci_guard.subprocess.run"),
|
||||
patch("time.sleep"),
|
||||
):
|
||||
proc = MagicMock()
|
||||
proc.poll.return_value = 1
|
||||
proc.returncode = 1
|
||||
mock_popen.return_value = proc
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"])
|
||||
assert result.exit_code == 1
|
||||
assert "FAILED" in result.output
|
||||
|
||||
def test_exits_before_starting_when_already_failed(self) -> None:
|
||||
from click.testing import CliRunner
|
||||
|
||||
with (
|
||||
patch.dict(
|
||||
os.environ,
|
||||
{
|
||||
"GITEA_URL": "https://gitea.example",
|
||||
"CI_GITEA_TOKEN": "token",
|
||||
"RUN_ID": "123",
|
||||
"JOB_NAME": "molecule-tests",
|
||||
"MATRIX_INDEX": "0",
|
||||
"GITEA_REPOSITORY": "my-org/my-repo",
|
||||
"PATH": os.environ.get("PATH", ""),
|
||||
},
|
||||
clear=True,
|
||||
),
|
||||
patch("devx.molecule.molecule_ci_guard.POLL_INTERVAL", 0.01),
|
||||
patch("devx.molecule.molecule_ci_guard.subprocess.Popen") as mock_popen,
|
||||
patch("devx.molecule.molecule_ci_guard.subprocess.run"),
|
||||
patch("devx.molecule.molecule_ci_guard.subprocess.run"),
|
||||
patch("devx.molecule.molecule_ci_guard.get_running_jobs") as mock_get_jobs,
|
||||
patch("time.sleep"),
|
||||
):
|
||||
mock_get_jobs.return_value = [
|
||||
{"name": "molecule-tests (0)", "conclusion": "running"},
|
||||
{"name": "molecule-tests (1)", "conclusion": "failure"},
|
||||
]
|
||||
proc = MagicMock()
|
||||
proc.poll.return_value = 0
|
||||
proc.returncode = 0
|
||||
mock_popen.return_value = proc
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"])
|
||||
assert result.exit_code == 1
|
||||
assert "Another molecule runner failed" in result.output
|
||||
|
||||
def test_keyboard_interrupt_kills_process(self) -> None:
|
||||
from click.testing import CliRunner
|
||||
|
||||
with (
|
||||
patch("devx.molecule.molecule_ci_guard.subprocess.Popen") as mock_popen,
|
||||
patch("devx.molecule.molecule_ci_guard.subprocess.run"),
|
||||
patch("devx.molecule.molecule_ci_guard.subprocess.run"),
|
||||
patch("time.sleep", side_effect=KeyboardInterrupt),
|
||||
patch("os.killpg") as mock_killpg,
|
||||
patch("os.getpgid") as mock_getpgid,
|
||||
):
|
||||
mock_getpgid.return_value = 123
|
||||
proc = MagicMock()
|
||||
proc.poll.return_value = None
|
||||
proc.wait.return_value = 0
|
||||
mock_popen.return_value = proc
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"])
|
||||
assert result.exit_code == 1
|
||||
mock_killpg.assert_called()
|
||||
|
||||
def test_exits_when_other_runner_fails(self) -> None:
|
||||
from click.testing import CliRunner
|
||||
|
||||
real_sleep = time.sleep
|
||||
call_count = [0]
|
||||
|
||||
def get_jobs_side_effect(*args, **kwargs):
|
||||
call_count[0] += 1
|
||||
if call_count[0] < 2:
|
||||
return [{"name": "molecule-tests (1)", "conclusion": "running"}]
|
||||
return [
|
||||
{"name": "molecule-tests (0)", "conclusion": "running"},
|
||||
{"name": "molecule-tests (1)", "conclusion": "failure"},
|
||||
]
|
||||
|
||||
with (
|
||||
patch.dict(
|
||||
os.environ,
|
||||
{
|
||||
"GITEA_URL": "https://gitea.example",
|
||||
"CI_GITEA_TOKEN": "token",
|
||||
"RUN_ID": "123",
|
||||
"JOB_NAME": "molecule-tests",
|
||||
"MATRIX_INDEX": "0",
|
||||
"GITEA_REPOSITORY": "my-org/my-repo",
|
||||
"PATH": os.environ.get("PATH", ""),
|
||||
},
|
||||
clear=True,
|
||||
),
|
||||
patch("devx.molecule.molecule_ci_guard.POLL_INTERVAL", 0.01),
|
||||
patch("devx.molecule.molecule_ci_guard.subprocess.Popen") as mock_popen,
|
||||
patch("devx.molecule.molecule_ci_guard.subprocess.run"),
|
||||
patch("devx.molecule.molecule_ci_guard.get_running_jobs", side_effect=get_jobs_side_effect),
|
||||
patch("os.killpg") as mock_killpg,
|
||||
patch("os.getpgid") as mock_getpgid,
|
||||
patch("time.sleep", side_effect=lambda x: real_sleep(0)),
|
||||
):
|
||||
mock_getpgid.return_value = 123
|
||||
proc = MagicMock()
|
||||
proc.poll.return_value = None
|
||||
proc.wait.return_value = 0
|
||||
mock_popen.return_value = proc
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"])
|
||||
assert result.exit_code == 1
|
||||
mock_killpg.assert_called()
|
||||
|
||||
def test_default_owner_repo_fallback(self) -> None:
|
||||
from click.testing import CliRunner
|
||||
|
||||
real_sleep = time.sleep
|
||||
|
||||
with (
|
||||
patch.dict(
|
||||
os.environ,
|
||||
{
|
||||
"GITEA_URL": "https://gitea.example",
|
||||
"CI_GITEA_TOKEN": "token",
|
||||
"RUN_ID": "123",
|
||||
"JOB_NAME": "molecule-tests",
|
||||
"MATRIX_INDEX": "0",
|
||||
"GITEA_REPOSITORY": "invalid",
|
||||
"PATH": os.environ.get("PATH", ""),
|
||||
},
|
||||
clear=True,
|
||||
),
|
||||
patch("devx.molecule.molecule_ci_guard.POLL_INTERVAL", 0.01),
|
||||
patch("devx.molecule.molecule_ci_guard.subprocess.Popen") as mock_popen,
|
||||
patch("devx.molecule.molecule_ci_guard.subprocess.run"),
|
||||
patch("devx.molecule.molecule_ci_guard.subprocess.run") as mock_run,
|
||||
patch("devx.molecule.molecule_ci_guard.get_running_jobs") as mock_get_jobs,
|
||||
patch("time.sleep", side_effect=lambda x: real_sleep(0)),
|
||||
):
|
||||
mock_get_jobs.return_value = [{"name": "molecule-tests (1)", "conclusion": "success"}]
|
||||
proc = MagicMock()
|
||||
proc.poll.return_value = 0
|
||||
proc.returncode = 0
|
||||
mock_popen.return_value = proc
|
||||
mock_run.return_value = MagicMock(returncode=0)
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"])
|
||||
assert result.exit_code == 0
|
||||
|
||||
def test_exits_when_other_runner_fails_with_process_lookup_error(self) -> None:
|
||||
from click.testing import CliRunner
|
||||
|
||||
real_sleep = time.sleep
|
||||
call_count = [0]
|
||||
|
||||
def get_jobs_side_effect(*args, **kwargs):
|
||||
call_count[0] += 1
|
||||
if call_count[0] < 2:
|
||||
return [{"name": "molecule-tests (1)", "conclusion": "running"}]
|
||||
return [
|
||||
{"name": "molecule-tests (0)", "conclusion": "running"},
|
||||
{"name": "molecule-tests (1)", "conclusion": "failure"},
|
||||
]
|
||||
|
||||
with (
|
||||
patch.dict(
|
||||
os.environ,
|
||||
{
|
||||
"GITEA_URL": "https://gitea.example",
|
||||
"CI_GITEA_TOKEN": "token",
|
||||
"RUN_ID": "123",
|
||||
"JOB_NAME": "molecule-tests",
|
||||
"MATRIX_INDEX": "0",
|
||||
"GITEA_REPOSITORY": "my-org/my-repo",
|
||||
"PATH": os.environ.get("PATH", ""),
|
||||
},
|
||||
clear=True,
|
||||
),
|
||||
patch("devx.molecule.molecule_ci_guard.POLL_INTERVAL", 0.01),
|
||||
patch("devx.molecule.molecule_ci_guard.subprocess.Popen") as mock_popen,
|
||||
patch("devx.molecule.molecule_ci_guard.subprocess.run"),
|
||||
patch("devx.molecule.molecule_ci_guard.get_running_jobs", side_effect=get_jobs_side_effect),
|
||||
patch("os.killpg") as mock_killpg,
|
||||
patch("os.getpgid") as mock_getpgid,
|
||||
patch("time.sleep", side_effect=lambda x: real_sleep(0)),
|
||||
):
|
||||
mock_getpgid.return_value = 123
|
||||
mock_killpg.side_effect = ProcessLookupError("no such process")
|
||||
proc = MagicMock()
|
||||
proc.poll.return_value = None
|
||||
proc.wait.return_value = 0
|
||||
mock_popen.return_value = proc
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"])
|
||||
assert result.exit_code == 1
|
||||
|
||||
def test_exits_when_other_runner_fails_with_timeout(self) -> None:
|
||||
from click.testing import CliRunner
|
||||
|
||||
real_sleep = time.sleep
|
||||
call_count = [0]
|
||||
|
||||
def get_jobs_side_effect(*args, **kwargs):
|
||||
call_count[0] += 1
|
||||
if call_count[0] < 2:
|
||||
return [{"name": "molecule-tests (1)", "conclusion": "running"}]
|
||||
return [
|
||||
{"name": "molecule-tests (0)", "conclusion": "running"},
|
||||
{"name": "molecule-tests (1)", "conclusion": "failure"},
|
||||
]
|
||||
|
||||
with (
|
||||
patch.dict(
|
||||
os.environ,
|
||||
{
|
||||
"GITEA_URL": "https://gitea.example",
|
||||
"CI_GITEA_TOKEN": "token",
|
||||
"RUN_ID": "123",
|
||||
"JOB_NAME": "molecule-tests",
|
||||
"MATRIX_INDEX": "0",
|
||||
"GITEA_REPOSITORY": "my-org/my-repo",
|
||||
"PATH": os.environ.get("PATH", ""),
|
||||
},
|
||||
clear=True,
|
||||
),
|
||||
patch("devx.molecule.molecule_ci_guard.POLL_INTERVAL", 0.01),
|
||||
patch("devx.molecule.molecule_ci_guard.subprocess.Popen") as mock_popen,
|
||||
patch("devx.molecule.molecule_ci_guard.subprocess.run"),
|
||||
patch("devx.molecule.molecule_ci_guard.get_running_jobs", side_effect=get_jobs_side_effect),
|
||||
patch("os.killpg") as mock_killpg,
|
||||
patch("os.getpgid") as mock_getpgid,
|
||||
patch("time.sleep", side_effect=lambda x: real_sleep(0)),
|
||||
):
|
||||
mock_getpgid.return_value = 123
|
||||
mock_killpg.side_effect = [None, ProcessLookupError("no such process")]
|
||||
proc = MagicMock()
|
||||
proc.poll.return_value = None
|
||||
proc.wait.side_effect = [subprocess.TimeoutExpired("cmd", 10)]
|
||||
mock_popen.return_value = proc
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"])
|
||||
assert result.exit_code == 1
|
||||
|
||||
|
||||
def test_main_module_block() -> None:
|
||||
import devx.molecule.molecule_ci_guard as mg
|
||||
|
||||
with open(mg.__file__) as f:
|
||||
source = f.read()
|
||||
source = source.replace('if __name__ == "__main__":\n cli()\n', "")
|
||||
namespace = dict(mg.__dict__)
|
||||
exec(compile(source, mg.__file__, "exec"), namespace)
|
||||
assert callable(namespace["cli"])
|
||||
|
||||
|
||||
class TestParsePair:
|
||||
def test_single_role_4_part(self) -> None:
|
||||
role, scenario, name, image, cmd = parse_pair("default|ubuntu-2204|ubuntu:22.04|")
|
||||
assert role == ""
|
||||
assert scenario == "default"
|
||||
assert name == "ubuntu-2204"
|
||||
assert image == "ubuntu:22.04"
|
||||
assert cmd == ""
|
||||
|
||||
def test_multi_role_5_part(self) -> None:
|
||||
role, scenario, name, image, cmd = parse_pair("gitea-runner|default|ubuntu-2204|ubuntu:22.04|")
|
||||
assert role == "gitea-runner"
|
||||
assert scenario == "default"
|
||||
assert name == "ubuntu-2204"
|
||||
assert image == "ubuntu:22.04"
|
||||
assert cmd == ""
|
||||
|
||||
def test_multi_role_with_command(self) -> None:
|
||||
role, scenario, name, image, cmd = parse_pair(
|
||||
"docker-base|lifecycle|archlinux|archlinux:latest|/usr/lib/systemd/systemd"
|
||||
)
|
||||
assert role == "docker-base"
|
||||
assert scenario == "lifecycle"
|
||||
assert cmd == "/usr/lib/systemd/systemd"
|
||||
|
||||
def test_invalid_pair_raises(self) -> None:
|
||||
with pytest.raises(click.ClickException, match="Invalid pair format"):
|
||||
parse_pair("only|two|parts")
|
||||
|
||||
def test_too_many_parts_raises(self) -> None:
|
||||
with pytest.raises(click.ClickException, match="Invalid pair format"):
|
||||
parse_pair("a|b|c|d|e|f")
|
||||
|
||||
|
||||
class TestResolveRoleDir:
|
||||
def test_multi_role_with_roles_root(self, tmp_path: Path) -> None:
|
||||
roles_root = tmp_path / "ansible" / "roles"
|
||||
roles_root.mkdir(parents=True)
|
||||
result = resolve_role_dir("gitea-runner", roles_root, tmp_path)
|
||||
assert result == roles_root / "gitea-runner"
|
||||
|
||||
def test_multi_role_default_roles_root(self, tmp_path: Path) -> None:
|
||||
result = resolve_role_dir("docker-base", None, tmp_path)
|
||||
assert result == tmp_path / "ansible" / "roles" / "docker-base"
|
||||
|
||||
def test_single_role_auto_discovers(self, tmp_path: Path) -> None:
|
||||
"""Single-role mode auto-discovers first role with molecule/ dir."""
|
||||
roles_dir = tmp_path / "ansible" / "roles"
|
||||
(roles_dir / "my_role" / "molecule").mkdir(parents=True)
|
||||
result = resolve_role_dir("", None, tmp_path)
|
||||
assert result == roles_dir / "my_role"
|
||||
|
||||
def test_single_role_no_roles_returns_fallback(self, tmp_path: Path) -> None:
|
||||
"""When no roles exist, returns a fallback path (will error at runtime)."""
|
||||
result = resolve_role_dir("", None, tmp_path)
|
||||
assert "roles" in str(result)
|
||||
|
||||
|
||||
class TestCliMultiRole:
|
||||
def test_multi_role_pair_passes(self, tmp_path: Path) -> None:
|
||||
from click.testing import CliRunner
|
||||
|
||||
roles_root = tmp_path / "ansible" / "roles"
|
||||
(roles_root / "gitea-runner").mkdir(parents=True)
|
||||
|
||||
with (
|
||||
patch("devx.molecule.molecule_ci_guard.subprocess.Popen") as mock_popen,
|
||||
patch("devx.molecule.molecule_ci_guard.subprocess.run"),
|
||||
patch("devx.molecule.molecule_ci_guard.subprocess.run") as mock_run,
|
||||
patch("time.sleep"),
|
||||
):
|
||||
proc = MagicMock()
|
||||
proc.poll.return_value = 0
|
||||
proc.returncode = 0
|
||||
mock_popen.return_value = proc
|
||||
mock_run.return_value = MagicMock(returncode=0)
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(
|
||||
cli,
|
||||
["--roles-root", str(roles_root), "gitea-runner|default|ubuntu-2204|ubuntu:22.04|"],
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
assert "All molecule tests passed" in result.output
|
||||
+40
-405
@@ -7,32 +7,13 @@ from click.testing import CliRunner
|
||||
|
||||
from devx.molecule.start_docker import (
|
||||
DOCKER_SOCK,
|
||||
HOST_DOCKER_SOCK,
|
||||
ROOTLESS_SOCK,
|
||||
_diagnose_socket,
|
||||
_get_docker_free_bytes,
|
||||
_try_socket,
|
||||
is_docker_ready,
|
||||
main,
|
||||
start_docker_daemon,
|
||||
)
|
||||
|
||||
|
||||
def _pgrep_empty() -> MagicMock:
|
||||
"""Mock for pgrep returning no dockerd processes."""
|
||||
return MagicMock(stdout="", returncode=1)
|
||||
|
||||
|
||||
def _docker_info_alive() -> MagicMock:
|
||||
"""Mock for docker info showing daemon alive."""
|
||||
return MagicMock(returncode=0)
|
||||
|
||||
|
||||
def _docker_info_dead() -> MagicMock:
|
||||
"""Mock for docker info showing daemon dead."""
|
||||
return MagicMock(returncode=1)
|
||||
|
||||
|
||||
class TestIsDockerReady:
|
||||
@patch("devx.molecule.start_docker.subprocess.run")
|
||||
def test_ready(self, mock_run: MagicMock) -> None:
|
||||
@@ -59,42 +40,6 @@ class TestIsDockerReady:
|
||||
assert call_kwargs.kwargs["env"]["DOCKER_HOST"] == rootless
|
||||
|
||||
|
||||
class TestGetDockerFreeBytes:
|
||||
@patch("devx.molecule.start_docker.shutil.disk_usage")
|
||||
@patch("devx.molecule.start_docker.os.path.exists", return_value=True)
|
||||
@patch("devx.molecule.start_docker.subprocess.run")
|
||||
def test_returns_free_bytes(self, mock_run: MagicMock, mock_exists: MagicMock, mock_du: MagicMock) -> None:
|
||||
mock_run.return_value = MagicMock(stdout="/var/lib/docker\n", returncode=0, text="")
|
||||
mock_du.return_value = MagicMock(free=50 * 1024**3)
|
||||
with patch.dict("os.environ", {"DOCKER_HOST": f"unix://{DOCKER_SOCK}"}, clear=False):
|
||||
assert _get_docker_free_bytes() == 50 * 1024**3
|
||||
|
||||
@patch("devx.molecule.start_docker.subprocess.run")
|
||||
def test_daemon_not_reachable(self, mock_run: MagicMock) -> None:
|
||||
mock_run.return_value = MagicMock(returncode=1, stdout="", text="")
|
||||
assert _get_docker_free_bytes() == 0
|
||||
|
||||
@patch("devx.molecule.start_docker.os.path.exists", return_value=False)
|
||||
@patch("devx.molecule.start_docker.subprocess.run")
|
||||
def test_data_root_not_accessible(self, mock_run: MagicMock, mock_exists: MagicMock) -> None:
|
||||
mock_run.return_value = MagicMock(stdout="/some/path\n", returncode=0, text="")
|
||||
assert _get_docker_free_bytes() == 0
|
||||
|
||||
@patch("devx.molecule.start_docker.subprocess.run", side_effect=FileNotFoundError)
|
||||
def test_subprocess_not_found(self, mock_run: MagicMock) -> None:
|
||||
assert _get_docker_free_bytes() == 0
|
||||
|
||||
|
||||
class TestTrySocket:
|
||||
@patch("devx.molecule.start_docker.is_docker_ready", return_value=True)
|
||||
def test_ready(self, mock_ready: MagicMock) -> None:
|
||||
assert _try_socket("/run/user/999/docker.sock") is True
|
||||
|
||||
@patch("devx.molecule.start_docker.is_docker_ready", return_value=False)
|
||||
def test_not_ready(self, mock_ready: MagicMock) -> None:
|
||||
assert _try_socket("/run/user/999/docker.sock") is False
|
||||
|
||||
|
||||
class TestDiagnoseSocket:
|
||||
@patch("devx.molecule.start_docker.os.stat")
|
||||
@patch("devx.molecule.start_docker.os.path.exists", return_value=True)
|
||||
@@ -141,373 +86,81 @@ class TestDiagnoseSocket:
|
||||
mock_exists.assert_called_with(DOCKER_SOCK)
|
||||
|
||||
|
||||
def _exists_map(paths: set[str]) -> MagicMock:
|
||||
"""Return a mock os.path.exists that returns True only for *paths*."""
|
||||
return MagicMock(side_effect=lambda p: p in paths)
|
||||
|
||||
|
||||
class TestStartDockerDaemon:
|
||||
@patch("devx.molecule.start_docker._diagnose_socket")
|
||||
@patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=100 * 1024**3)
|
||||
@patch("devx.molecule.start_docker.is_docker_ready", return_value=True)
|
||||
@patch("devx.molecule.start_docker.os.path.exists", side_effect=_exists_map({DOCKER_SOCK}))
|
||||
@patch("devx.molecule.start_docker.glob.glob", return_value=[])
|
||||
def test_host_socket_available_with_space(
|
||||
self,
|
||||
mock_glob: MagicMock,
|
||||
mock_exists: MagicMock,
|
||||
mock_ready: MagicMock,
|
||||
mock_free: MagicMock,
|
||||
mock_diag: MagicMock,
|
||||
) -> None:
|
||||
"""Should return immediately if /var/run/docker.sock has enough space."""
|
||||
def test_host_socket_available(self, mock_ready: MagicMock, mock_diag: MagicMock) -> None:
|
||||
"""Should return immediately if host Docker is available."""
|
||||
assert start_docker_daemon(timeout=5) is True
|
||||
mock_ready.assert_called_once()
|
||||
mock_diag.assert_called_once()
|
||||
|
||||
@patch("devx.molecule.start_docker._diagnose_socket")
|
||||
@patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=0)
|
||||
@patch("devx.molecule.start_docker.is_docker_ready", return_value=True)
|
||||
@patch("devx.molecule.start_docker.os.path.exists", side_effect=_exists_map({HOST_DOCKER_SOCK}))
|
||||
@patch("devx.molecule.start_docker.glob.glob", return_value=[])
|
||||
def test_host_rootless_inaccessible_root_uses_host(
|
||||
self,
|
||||
mock_glob: MagicMock,
|
||||
mock_exists: MagicMock,
|
||||
mock_ready: MagicMock,
|
||||
mock_free: MagicMock,
|
||||
mock_diag: MagicMock,
|
||||
) -> None:
|
||||
"""Should use host rootless Docker when root dir is inaccessible (free=0)."""
|
||||
assert start_docker_daemon(timeout=5) is True
|
||||
|
||||
@patch("devx.molecule.start_docker._diagnose_socket")
|
||||
@patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=0)
|
||||
@patch("devx.molecule.start_docker.is_docker_ready", return_value=True)
|
||||
@patch("devx.molecule.start_docker.os.path.exists", side_effect=_exists_map({DOCKER_SOCK}))
|
||||
@patch("devx.molecule.start_docker.glob.glob", return_value=[])
|
||||
@patch("devx.molecule.start_docker.subprocess.run")
|
||||
@patch("devx.molecule.start_docker.subprocess.Popen")
|
||||
@patch("devx.molecule.start_docker.tempfile.NamedTemporaryFile")
|
||||
@patch("devx.molecule.start_docker.time.sleep")
|
||||
def test_inner_dockerd_free_zero_starts_local(
|
||||
self,
|
||||
mock_sleep: MagicMock,
|
||||
mock_ntf: MagicMock,
|
||||
mock_popen: MagicMock,
|
||||
mock_run: MagicMock,
|
||||
mock_glob: MagicMock,
|
||||
mock_exists: MagicMock,
|
||||
mock_ready: MagicMock,
|
||||
mock_free: MagicMock,
|
||||
mock_diag: MagicMock,
|
||||
) -> None:
|
||||
"""Inner dockerd with free=0 (not host socket) should NOT be trusted — start local."""
|
||||
mock_ntf.return_value = MagicMock(name="/tmp/dockerd.log")
|
||||
# is_docker_ready: first check (inner dockerd) True, then local daemon checks
|
||||
mock_ready.side_effect = [True, False, False, False, False, True]
|
||||
# subprocess.run: pgrep(socket check, finds dockerd), pgrep(before kill),
|
||||
# pkill x3, pgrep(after kill), docker info(dead), rm
|
||||
mock_run.side_effect = [
|
||||
MagicMock(returncode=0, stdout="123 /usr/bin/dockerd\n"), # socket check pgrep
|
||||
_pgrep_empty(),
|
||||
MagicMock(),
|
||||
MagicMock(),
|
||||
MagicMock(),
|
||||
_pgrep_empty(),
|
||||
_docker_info_dead(),
|
||||
MagicMock(),
|
||||
]
|
||||
assert start_docker_daemon(timeout=5) is True
|
||||
mock_popen.assert_called_once()
|
||||
|
||||
@patch("devx.molecule.start_docker._diagnose_socket")
|
||||
@patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=0)
|
||||
@patch("devx.molecule.start_docker.is_docker_ready", return_value=True)
|
||||
@patch("devx.molecule.start_docker.os.path.exists", side_effect=_exists_map({DOCKER_SOCK}))
|
||||
@patch("devx.molecule.start_docker.glob.glob", return_value=[])
|
||||
@patch("devx.molecule.start_docker.subprocess.run")
|
||||
def test_no_inner_dockerd_trusts_host_socket(
|
||||
self,
|
||||
mock_run: MagicMock,
|
||||
mock_glob: MagicMock,
|
||||
mock_exists: MagicMock,
|
||||
mock_ready: MagicMock,
|
||||
mock_free: MagicMock,
|
||||
mock_diag: MagicMock,
|
||||
) -> None:
|
||||
"""Should trust /var/run/docker.sock with free=0 when no inner dockerd exists."""
|
||||
# pgrep finds no dockerd processes (returncode=1)
|
||||
mock_run.return_value = MagicMock(returncode=1, stdout="")
|
||||
assert start_docker_daemon(timeout=5) is True
|
||||
assert os.environ.get("DOCKER_HOST") == f"unix://{DOCKER_SOCK}"
|
||||
|
||||
@patch("devx.molecule.start_docker._diagnose_socket")
|
||||
@patch("devx.molecule.start_docker._get_docker_free_bytes")
|
||||
@patch("devx.molecule.start_docker.os.path.exists", return_value=True)
|
||||
@patch("devx.molecule.start_docker.is_docker_ready")
|
||||
@patch("devx.molecule.start_docker.os.path.exists", side_effect=_exists_map({DOCKER_SOCK}))
|
||||
@patch("devx.molecule.start_docker.glob.glob", return_value=[])
|
||||
@patch("devx.molecule.start_docker.subprocess.run")
|
||||
@patch("devx.molecule.start_docker.subprocess.Popen")
|
||||
@patch("devx.molecule.start_docker.tempfile.NamedTemporaryFile")
|
||||
@patch("devx.molecule.start_docker.time.sleep")
|
||||
def test_host_socket_low_space_starts_local(
|
||||
self,
|
||||
mock_sleep: MagicMock,
|
||||
mock_ntf: MagicMock,
|
||||
mock_popen: MagicMock,
|
||||
mock_run: MagicMock,
|
||||
mock_glob: MagicMock,
|
||||
mock_exists: MagicMock,
|
||||
mock_ready: MagicMock,
|
||||
mock_free: MagicMock,
|
||||
mock_diag: MagicMock,
|
||||
def test_rootless_socket_available(
|
||||
self, mock_ready: MagicMock, mock_exists: MagicMock, mock_diag: MagicMock
|
||||
) -> None:
|
||||
"""Should start local dockerd if host Docker has low space and no rootless sockets."""
|
||||
mock_ntf.return_value = MagicMock(name="/tmp/dockerd.log")
|
||||
mock_ready.side_effect = [True, False, False, False, False, True]
|
||||
mock_free.return_value = 5 * 1024**3
|
||||
# subprocess.run: pgrep(before), pkill x3, pgrep(after), docker info(dead), rm
|
||||
mock_run.side_effect = [
|
||||
_pgrep_empty(),
|
||||
MagicMock(),
|
||||
MagicMock(),
|
||||
MagicMock(),
|
||||
_pgrep_empty(),
|
||||
_docker_info_dead(),
|
||||
MagicMock(),
|
||||
]
|
||||
assert start_docker_daemon(timeout=5) is True
|
||||
mock_popen.assert_called_once()
|
||||
|
||||
@patch("devx.molecule.start_docker._diagnose_socket")
|
||||
@patch("devx.molecule.start_docker._get_docker_free_bytes")
|
||||
@patch("devx.molecule.start_docker.is_docker_ready")
|
||||
@patch("devx.molecule.start_docker.os.path.exists", side_effect=_exists_map({DOCKER_SOCK}))
|
||||
@patch("devx.molecule.start_docker.glob.glob", return_value=[])
|
||||
@patch("devx.molecule.start_docker.subprocess.run")
|
||||
@patch("devx.molecule.start_docker.subprocess.Popen")
|
||||
@patch("devx.molecule.start_docker.tempfile.NamedTemporaryFile")
|
||||
@patch("devx.molecule.start_docker.time.sleep")
|
||||
def test_inner_dockerd_alive_uses_alt_sock(
|
||||
self,
|
||||
mock_sleep: MagicMock,
|
||||
mock_ntf: MagicMock,
|
||||
mock_popen: MagicMock,
|
||||
mock_run: MagicMock,
|
||||
mock_glob: MagicMock,
|
||||
mock_exists: MagicMock,
|
||||
mock_ready: MagicMock,
|
||||
mock_free: MagicMock,
|
||||
mock_diag: MagicMock,
|
||||
) -> None:
|
||||
"""Should use /dev/shm/docker.sock if inner dockerd can't be killed."""
|
||||
mock_ntf.return_value = MagicMock(name="/tmp/dockerd.log")
|
||||
mock_ready.side_effect = [True, False, False, False, False, True]
|
||||
mock_free.return_value = 5 * 1024**3
|
||||
# subprocess.run: pgrep(before), pkill x3, pgrep(after), docker info(alive), rm
|
||||
mock_run.side_effect = [
|
||||
_pgrep_empty(),
|
||||
MagicMock(),
|
||||
MagicMock(),
|
||||
MagicMock(),
|
||||
_pgrep_empty(),
|
||||
_docker_info_alive(),
|
||||
MagicMock(),
|
||||
]
|
||||
assert start_docker_daemon(timeout=5) is True
|
||||
mock_popen.assert_called_once()
|
||||
assert os.environ.get("DOCKER_HOST") == "unix:///dev/shm/docker.sock"
|
||||
|
||||
@patch("devx.molecule.start_docker.os.kill")
|
||||
@patch("devx.molecule.start_docker._diagnose_socket")
|
||||
@patch("devx.molecule.start_docker._get_docker_free_bytes")
|
||||
@patch("devx.molecule.start_docker.is_docker_ready")
|
||||
@patch("devx.molecule.start_docker.os.path.exists", side_effect=_exists_map({DOCKER_SOCK}))
|
||||
@patch("devx.molecule.start_docker.glob.glob", return_value=[])
|
||||
@patch("devx.molecule.start_docker.subprocess.run")
|
||||
@patch("devx.molecule.start_docker.subprocess.Popen")
|
||||
@patch("devx.molecule.start_docker.tempfile.NamedTemporaryFile")
|
||||
@patch("devx.molecule.start_docker.time.sleep")
|
||||
def test_inner_dockerd_killed_by_pid(
|
||||
self,
|
||||
mock_sleep: MagicMock,
|
||||
mock_ntf: MagicMock,
|
||||
mock_popen: MagicMock,
|
||||
mock_run: MagicMock,
|
||||
mock_glob: MagicMock,
|
||||
mock_exists: MagicMock,
|
||||
mock_ready: MagicMock,
|
||||
mock_free: MagicMock,
|
||||
mock_diag: MagicMock,
|
||||
mock_kill: MagicMock,
|
||||
) -> None:
|
||||
"""Should kill dockerd by PID when pkill fails and pgrep finds processes."""
|
||||
mock_ntf.return_value = MagicMock(name="/tmp/dockerd.log")
|
||||
mock_ready.side_effect = [True, False, False, False, False, True]
|
||||
mock_free.return_value = 5 * 1024**3
|
||||
# pgrep(before) empty, pkill x3, pgrep(after) finds PID 12345, docker info(dead), rm
|
||||
mock_run.side_effect = [
|
||||
_pgrep_empty(),
|
||||
MagicMock(),
|
||||
MagicMock(),
|
||||
MagicMock(),
|
||||
MagicMock(stdout="12345 /usr/bin/dockererd\n", returncode=0),
|
||||
_docker_info_dead(),
|
||||
MagicMock(),
|
||||
]
|
||||
assert start_docker_daemon(timeout=5) is True
|
||||
mock_popen.assert_called_once()
|
||||
mock_kill.assert_called_once_with(12345, 9)
|
||||
|
||||
@patch("devx.molecule.start_docker._diagnose_socket")
|
||||
@patch("devx.molecule.start_docker._get_docker_free_bytes")
|
||||
@patch("devx.molecule.start_docker.is_docker_ready")
|
||||
@patch("devx.molecule.start_docker.os.path.exists", side_effect=_exists_map({DOCKER_SOCK, HOST_DOCKER_SOCK}))
|
||||
@patch("devx.molecule.start_docker.glob.glob", return_value=[])
|
||||
def test_host_rootless_preferred_over_inner_dockerd(
|
||||
self,
|
||||
mock_glob: MagicMock,
|
||||
mock_exists: MagicMock,
|
||||
mock_ready: MagicMock,
|
||||
mock_free: MagicMock,
|
||||
mock_diag: MagicMock,
|
||||
) -> None:
|
||||
"""Should prefer /run/host-docker.sock over /var/run/docker.sock."""
|
||||
# host rootless ready with space, inner dockerd never tried
|
||||
mock_ready.side_effect = [True]
|
||||
mock_free.side_effect = [200 * 1024**3]
|
||||
assert start_docker_daemon(timeout=5) is True
|
||||
# DOCKER_HOST should be set to host socket
|
||||
assert os.environ.get("DOCKER_HOST") == f"unix://{HOST_DOCKER_SOCK}"
|
||||
|
||||
@patch("devx.molecule.start_docker._diagnose_socket")
|
||||
@patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=100 * 1024**3)
|
||||
@patch("devx.molecule.start_docker.is_docker_ready")
|
||||
@patch("devx.molecule.start_docker.os.path.exists", side_effect=_exists_map({DOCKER_SOCK, HOST_DOCKER_SOCK}))
|
||||
@patch("devx.molecule.start_docker.glob.glob", return_value=[])
|
||||
def test_host_rootless_not_ready_falls_to_inner(
|
||||
self,
|
||||
mock_glob: MagicMock,
|
||||
mock_exists: MagicMock,
|
||||
mock_ready: MagicMock,
|
||||
mock_free: MagicMock,
|
||||
mock_diag: MagicMock,
|
||||
) -> None:
|
||||
"""Should fall through to inner dockerd if host rootless socket is not ready."""
|
||||
"""Should use rootless socket if host socket fails."""
|
||||
# First check (host) fails, second check (rootless) succeeds
|
||||
mock_ready.side_effect = [False, True]
|
||||
assert start_docker_daemon(timeout=5) is True
|
||||
assert os.environ.get("DOCKER_HOST") == f"unix://{DOCKER_SOCK}"
|
||||
with patch("devx.molecule.start_docker.glob.glob", return_value=[]):
|
||||
assert start_docker_daemon(timeout=5) is True
|
||||
|
||||
@patch("devx.molecule.start_docker._diagnose_socket")
|
||||
@patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=100 * 1024**3)
|
||||
@patch("devx.molecule.start_docker.is_docker_ready", return_value=True)
|
||||
@patch(
|
||||
"devx.molecule.start_docker.os.path.exists",
|
||||
side_effect=_exists_map({DOCKER_SOCK, HOST_DOCKER_SOCK, ROOTLESS_SOCK, "/run/user/999/docker.sock"}),
|
||||
)
|
||||
@patch("devx.molecule.start_docker.glob.glob", return_value=["/run/user/999/docker.sock"])
|
||||
def test_glob_finds_extra_rootless_socket(
|
||||
self,
|
||||
mock_glob: MagicMock,
|
||||
mock_exists: MagicMock,
|
||||
mock_ready: MagicMock,
|
||||
mock_free: MagicMock,
|
||||
mock_diag: MagicMock,
|
||||
) -> None:
|
||||
"""Should include rootless sockets found via glob scan."""
|
||||
assert start_docker_daemon(timeout=5) is True
|
||||
|
||||
@patch("devx.molecule.start_docker._diagnose_socket")
|
||||
@patch("devx.molecule.start_docker._get_docker_free_bytes")
|
||||
@patch("devx.molecule.start_docker.os.path.exists", return_value=True)
|
||||
@patch("devx.molecule.start_docker.is_docker_ready")
|
||||
@patch("devx.molecule.start_docker.os.path.exists", side_effect=_exists_map({DOCKER_SOCK, HOST_DOCKER_SOCK}))
|
||||
@patch("devx.molecule.start_docker.glob.glob", return_value=[])
|
||||
def test_inner_dockerd_fallback_when_host_rootless_low_space(
|
||||
self,
|
||||
mock_glob: MagicMock,
|
||||
mock_exists: MagicMock,
|
||||
mock_ready: MagicMock,
|
||||
mock_free: MagicMock,
|
||||
mock_diag: MagicMock,
|
||||
def test_alt_rootless_socket_found(
|
||||
self, mock_ready: MagicMock, mock_exists: MagicMock, mock_diag: MagicMock
|
||||
) -> None:
|
||||
"""Should fall back to inner dockerd if host rootless has low space."""
|
||||
# host rootless ready low space, inner dockerd ready with space
|
||||
mock_ready.side_effect = [True, True]
|
||||
mock_free.side_effect = [5 * 1024**3, 200 * 1024**3]
|
||||
assert start_docker_daemon(timeout=5) is True
|
||||
assert os.environ.get("DOCKER_HOST") == f"unix://{DOCKER_SOCK}"
|
||||
"""Should find rootless socket at a different UID via glob scan."""
|
||||
# Host fails, own rootless fails, alt rootless succeeds
|
||||
mock_ready.side_effect = [False, False, True]
|
||||
alt_sock = "/run/user/999/docker.sock"
|
||||
with patch("devx.molecule.start_docker.glob.glob", return_value=[alt_sock]):
|
||||
assert start_docker_daemon(timeout=5) is True
|
||||
|
||||
@patch("devx.molecule.start_docker._diagnose_socket")
|
||||
@patch("devx.molecule.start_docker._get_docker_free_bytes")
|
||||
@patch("devx.molecule.start_docker.os.path.exists", return_value=True)
|
||||
@patch("devx.molecule.start_docker.is_docker_ready")
|
||||
@patch("devx.molecule.start_docker.os.path.exists", side_effect=_exists_map({DOCKER_SOCK}))
|
||||
@patch("devx.molecule.start_docker.glob.glob", return_value=[])
|
||||
@patch("devx.molecule.start_docker.subprocess.run")
|
||||
@patch("devx.molecule.start_docker.subprocess.Popen")
|
||||
@patch("devx.molecule.start_docker.tempfile.NamedTemporaryFile")
|
||||
@patch("devx.molecule.start_docker.time.sleep")
|
||||
def test_all_sockets_low_space_starts_local(
|
||||
self,
|
||||
mock_sleep: MagicMock,
|
||||
mock_ntf: MagicMock,
|
||||
mock_popen: MagicMock,
|
||||
mock_run: MagicMock,
|
||||
mock_glob: MagicMock,
|
||||
mock_exists: MagicMock,
|
||||
mock_ready: MagicMock,
|
||||
mock_free: MagicMock,
|
||||
mock_diag: MagicMock,
|
||||
def test_alt_rootless_socket_skips_own(
|
||||
self, mock_ready: MagicMock, mock_exists: MagicMock, mock_diag: MagicMock
|
||||
) -> None:
|
||||
"""Should start local dockerd if all sockets have low space."""
|
||||
mock_ntf.return_value = MagicMock(name="/tmp/dockerd.log")
|
||||
mock_ready.side_effect = [True, False, False, False, False, True]
|
||||
mock_free.return_value = 5 * 1024**3
|
||||
# subprocess.run: pgrep(before), pkill x3, pgrep(after), docker info(dead), rm
|
||||
mock_run.side_effect = [
|
||||
_pgrep_empty(),
|
||||
MagicMock(),
|
||||
MagicMock(),
|
||||
MagicMock(),
|
||||
_pgrep_empty(),
|
||||
_docker_info_dead(),
|
||||
MagicMock(),
|
||||
]
|
||||
assert start_docker_daemon(timeout=5) is True
|
||||
mock_popen.assert_called_once()
|
||||
"""Should skip the own rootless socket in glob scan (already tried)."""
|
||||
# Host fails, own rootless fails, alt rootless also fails, dockerd fails
|
||||
mock_ready.side_effect = [False, False, False, False, False, False]
|
||||
own_sock = f"/run/user/{os.getuid()}/docker.sock"
|
||||
alt_sock = "/run/user/999/docker.sock"
|
||||
with (
|
||||
patch("devx.molecule.start_docker.glob.glob", return_value=[own_sock, alt_sock]),
|
||||
patch("devx.molecule.start_docker.time.sleep"),
|
||||
patch("devx.molecule.start_docker.subprocess.Popen"),
|
||||
patch("devx.molecule.start_docker.tempfile.NamedTemporaryFile") as mock_ntf,
|
||||
patch("builtins.open", mock_open(read_data="err")),
|
||||
):
|
||||
mock_ntf.return_value = MagicMock(name="/tmp/dockerd.log")
|
||||
assert start_docker_daemon(timeout=2) is False
|
||||
|
||||
@patch("devx.molecule.start_docker.glob.glob", return_value=[])
|
||||
@patch("devx.molecule.start_docker._diagnose_socket")
|
||||
@patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=0)
|
||||
@patch("devx.molecule.start_docker.os.path.exists", return_value=False)
|
||||
@patch("devx.molecule.start_docker.is_docker_ready", return_value=False)
|
||||
@patch("devx.molecule.start_docker.time.sleep")
|
||||
@patch("devx.molecule.start_docker.subprocess.run")
|
||||
@patch("devx.molecule.start_docker.subprocess.Popen")
|
||||
@patch("devx.molecule.start_docker.tempfile.NamedTemporaryFile")
|
||||
def test_starts_local_daemon(
|
||||
self,
|
||||
mock_ntf: MagicMock,
|
||||
mock_popen: MagicMock,
|
||||
mock_run: MagicMock,
|
||||
mock_sleep: MagicMock,
|
||||
mock_ready: MagicMock,
|
||||
mock_exists: MagicMock,
|
||||
mock_free: MagicMock,
|
||||
mock_diag: MagicMock,
|
||||
mock_glob: MagicMock,
|
||||
) -> None:
|
||||
mock_ntf.return_value = MagicMock(name="/tmp/dockerd.log")
|
||||
# No sockets exist, local daemon starts
|
||||
# Host fails, rootless doesn't exist, local daemon starts
|
||||
mock_ready.side_effect = [False, False, False, False, True]
|
||||
# subprocess.run: pgrep(before), pkill x3, pgrep(after), rm
|
||||
mock_run.side_effect = [
|
||||
_pgrep_empty(),
|
||||
MagicMock(),
|
||||
MagicMock(),
|
||||
MagicMock(),
|
||||
_pgrep_empty(),
|
||||
MagicMock(),
|
||||
]
|
||||
assert start_docker_daemon(timeout=5) is True
|
||||
mock_popen.assert_called_once()
|
||||
popen_args = mock_popen.call_args.args[0]
|
||||
@@ -518,22 +171,18 @@ class TestStartDockerDaemon:
|
||||
|
||||
@patch("devx.molecule.start_docker.glob.glob", return_value=[])
|
||||
@patch("devx.molecule.start_docker._diagnose_socket")
|
||||
@patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=0)
|
||||
@patch("devx.molecule.start_docker.os.path.exists", return_value=False)
|
||||
@patch("devx.molecule.start_docker.is_docker_ready", return_value=False)
|
||||
@patch("devx.molecule.start_docker.time.sleep")
|
||||
@patch("devx.molecule.start_docker.subprocess.run")
|
||||
@patch("devx.molecule.start_docker.subprocess.Popen")
|
||||
@patch("devx.molecule.start_docker.tempfile.NamedTemporaryFile")
|
||||
def test_fails_after_timeout(
|
||||
self,
|
||||
mock_ntf: MagicMock,
|
||||
mock_popen: MagicMock,
|
||||
mock_run: MagicMock,
|
||||
mock_sleep: MagicMock,
|
||||
mock_ready: MagicMock,
|
||||
mock_exists: MagicMock,
|
||||
mock_free: MagicMock,
|
||||
mock_diag: MagicMock,
|
||||
mock_glob: MagicMock,
|
||||
) -> None:
|
||||
@@ -541,26 +190,22 @@ class TestStartDockerDaemon:
|
||||
with patch("builtins.open", mock_open(read_data="dockerd error log")):
|
||||
assert start_docker_daemon(timeout=3) is False
|
||||
mock_popen.assert_called_once()
|
||||
assert mock_sleep.call_count == 5 # 1 after pkill + 1 after pgrep + 3 timeout retries
|
||||
assert mock_sleep.call_count == 3
|
||||
|
||||
@patch("devx.molecule.start_docker.glob.glob", return_value=[])
|
||||
@patch("devx.molecule.start_docker._diagnose_socket")
|
||||
@patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=0)
|
||||
@patch("devx.molecule.start_docker.os.path.exists", return_value=False)
|
||||
@patch("devx.molecule.start_docker.is_docker_ready", return_value=False)
|
||||
@patch("devx.molecule.start_docker.time.sleep")
|
||||
@patch("devx.molecule.start_docker.subprocess.run")
|
||||
@patch("devx.molecule.start_docker.subprocess.Popen")
|
||||
@patch("devx.molecule.start_docker.tempfile.NamedTemporaryFile")
|
||||
def test_fails_log_read_error(
|
||||
self,
|
||||
mock_ntf: MagicMock,
|
||||
mock_popen: MagicMock,
|
||||
mock_run: MagicMock,
|
||||
mock_sleep: MagicMock,
|
||||
mock_ready: MagicMock,
|
||||
mock_exists: MagicMock,
|
||||
mock_free: MagicMock,
|
||||
mock_diag: MagicMock,
|
||||
mock_glob: MagicMock,
|
||||
) -> None:
|
||||
@@ -571,45 +216,35 @@ class TestStartDockerDaemon:
|
||||
|
||||
@patch("devx.molecule.start_docker.glob.glob", return_value=[])
|
||||
@patch("devx.molecule.start_docker._diagnose_socket")
|
||||
@patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=0)
|
||||
@patch("devx.molecule.start_docker.os.path.exists", return_value=False)
|
||||
@patch("devx.molecule.start_docker.is_docker_ready")
|
||||
@patch("devx.molecule.start_docker.time.sleep")
|
||||
@patch("devx.molecule.start_docker.subprocess.run")
|
||||
@patch("devx.molecule.start_docker.subprocess.Popen")
|
||||
@patch("devx.molecule.start_docker.tempfile.NamedTemporaryFile")
|
||||
def test_local_daemon_ready_on_first_check(
|
||||
self,
|
||||
mock_ntf: MagicMock,
|
||||
mock_popen: MagicMock,
|
||||
mock_run: MagicMock,
|
||||
mock_sleep: MagicMock,
|
||||
mock_ready: MagicMock,
|
||||
mock_exists: MagicMock,
|
||||
mock_free: MagicMock,
|
||||
mock_diag: MagicMock,
|
||||
mock_glob: MagicMock,
|
||||
) -> None:
|
||||
mock_ntf.return_value = MagicMock(name="/tmp/dockerd.log")
|
||||
# No sockets exist, local ready on second loop check
|
||||
# Host fails, rootless doesn't exist, local ready on first loop check
|
||||
mock_ready.side_effect = [False, False, True]
|
||||
assert start_docker_daemon(timeout=5) is True
|
||||
assert mock_popen.call_count == 1
|
||||
assert mock_sleep.call_count == 4 # 1 after pkill + 1 after pgrep + 2 loop retries
|
||||
assert mock_sleep.call_count == 1
|
||||
|
||||
@patch("devx.molecule.start_docker._diagnose_socket")
|
||||
@patch("devx.molecule.start_docker._get_docker_free_bytes", return_value=100 * 1024**3)
|
||||
@patch("devx.molecule.start_docker.os.environ")
|
||||
@patch("devx.molecule.start_docker.is_docker_ready", return_value=True)
|
||||
@patch("devx.molecule.start_docker.os.path.exists", side_effect=_exists_map({DOCKER_SOCK}))
|
||||
@patch("devx.molecule.start_docker.glob.glob", return_value=[])
|
||||
def test_sets_docker_host(
|
||||
self,
|
||||
mock_glob: MagicMock,
|
||||
mock_exists: MagicMock,
|
||||
mock_ready: MagicMock,
|
||||
mock_environ: MagicMock,
|
||||
mock_free: MagicMock,
|
||||
mock_diag: MagicMock,
|
||||
) -> None:
|
||||
"""DOCKER_HOST must be set so molecule connects to correct socket."""
|
||||
|
||||
Reference in New Issue
Block a user