diff --git a/scripts/molecule_ci_guard.py b/scripts/molecule_ci_guard.py index f5c8c0a..f0f8383 100644 --- a/scripts/molecule_ci_guard.py +++ b/scripts/molecule_ci_guard.py @@ -1,20 +1,29 @@ #!/usr/bin/env python3 -"""Run molecule tests while polling Gitea for other runner failures. +"""Run molecule tests sequentially while polling Gitea for other runner failures. + +Each pair is encoded as ``scenario|platform_name|platform_image|platform_command``. +Pairs are executed one at a time (molecule scenarios share temp directories and +Docker networks, so parallel execution within a single runner is unsafe). + +A background thread polls the Gitea API. If any other molecule matrix runner +reports failure, the current molecule subprocess is killed and this runner +exits early with code 1. Usage: python3 scripts/molecule_ci_guard.py ... Environment variables: - GITEA_URL Base URL of the Gitea instance. - REPO_TOKEN API token with repo access. - RUN_ID Workflow run ID (GITHUB_RUN_ID). - JOB_NAME Base job name (GITHUB_JOB), e.g. "molecule-tests". - MATRIX_INDEX Current matrix index (runner-index). - OWNER, REPO Repository owner/name. + GITEA_URL Base URL of the Gitea instance. + REPO_TOKEN API token with repo access. + RUN_ID Workflow run ID (GITHUB_RUN_ID). + JOB_NAME Base job name (GITHUB_JOB), e.g. "molecule-tests". + MATRIX_INDEX Current matrix index (runner-index). + GITEA_REPOSITORY Repository in "owner/repo" format. """ from __future__ import annotations +import contextlib import os import signal import subprocess # nosec B404 @@ -51,7 +60,6 @@ def any_other_runner_failed( name = job.get("name", "") if not name.startswith(current_job_name): continue - # Skip the current runner instance if name == f"{current_job_name} ({current_index})" or name == current_job_name: continue if job.get("conclusion") == "failure": @@ -87,10 +95,32 @@ def poll_for_other_failures( stop_event.wait(POLL_INTERVAL) +def build_molecule_cmd(scenario: str) -> list[str]: + """Build the molecule command for a scenario.""" + cmd = ["molecule", "test"] + if scenario != "default": + cmd.extend(["-s", scenario]) + return cmd + + +def build_env_for_pair(pair: str, base_env: dict[str, str]) -> dict[str, str]: + """Build environment for a single molecule pair.""" + scenario, platform_name, platform_image, platform_command = pair.split("|") + env = base_env.copy() + env["MOLECULE_PLATFORM_NAME"] = platform_name + env["MOLECULE_PLATFORM_IMAGE"] = platform_image + if platform_command: + env["MOLECULE_PLATFORM_COMMAND"] = platform_command + elif "MOLECULE_PLATFORM_COMMAND" in env: + del env["MOLECULE_PLATFORM_COMMAND"] + env["ANSIBLE_ALLOW_BROKEN_CONDITIONALS"] = "true" + return env + + @click.command() @click.argument("pairs", nargs=-1, required=True) def cli(pairs: tuple[str, ...]) -> None: - """Run molecule pairs and stop if another CI runner fails.""" + """Run molecule pairs sequentially, stop if another CI runner fails.""" gitea_url = os.environ.get("GITEA_URL", "") token = os.environ.get("REPO_TOKEN", "") run_id = int(os.environ.get("RUN_ID", "0")) @@ -109,21 +139,11 @@ def cli(pairs: tuple[str, ...]) -> None: ) repo_root = Path(__file__).resolve().parent.parent - script_path = repo_root / "scripts" / "run_molecule_parallel.py" + role_dir = repo_root / "ansible" / "roles" / "gitea-runner" - env = os.environ.copy() - env.setdefault("DOCKER_HOST", f"unix:///run/user/{os.getuid()}/docker.sock") - env.setdefault("ANSIBLE_INJECT_INVOCATION", "1") - - click.echo(_("Starting molecule runner with {count} pairs.", count=len(pairs))) - - process = subprocess.Popen( # nosec B603 - [sys.executable, str(script_path), *pairs], - env=env, - stdout=None, - stderr=None, - preexec_fn=os.setsid, - ) + base_env = os.environ.copy() + base_env.setdefault("DOCKER_HOST", f"unix:///run/user/{os.getuid()}/docker.sock") + base_env.setdefault("ANSIBLE_INJECT_INVOCATION", "1") stop_event = threading.Event() failed_event = threading.Event() @@ -147,26 +167,59 @@ def cli(pairs: tuple[str, ...]) -> None: poller.start() try: - while process.poll() is None: + for pair in pairs: if failed_event.is_set(): - try: - os.killpg(os.getpgid(process.pid), signal.SIGTERM) - except ProcessLookupError: - pass - try: - process.wait(timeout=10) - except subprocess.TimeoutExpired: - try: - os.killpg(os.getpgid(process.pid), signal.SIGKILL) - except ProcessLookupError: - pass - process.wait() sys.exit(1) - time.sleep(1) + + scenario = pair.split("|")[0] + platform_name = pair.split("|")[1] + click.echo( + _("Running: {scenario} on {platform}", scenario=scenario, platform=platform_name) + ) + + cmd = build_molecule_cmd(scenario) + env = build_env_for_pair(pair, base_env) + + process = subprocess.Popen( # nosec B603 + cmd, + cwd=str(role_dir), + env=env, + preexec_fn=os.setsid, + ) + + try: + while process.poll() is None: + if failed_event.is_set(): + with contextlib.suppress(ProcessLookupError): + os.killpg(os.getpgid(process.pid), signal.SIGTERM) + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + with contextlib.suppress(ProcessLookupError): + os.killpg(os.getpgid(process.pid), signal.SIGKILL) + process.wait() + sys.exit(1) + time.sleep(1) + except KeyboardInterrupt: + with contextlib.suppress(ProcessLookupError): + os.killpg(os.getpgid(process.pid), signal.SIGTERM) + process.wait() + sys.exit(1) + + rc = process.returncode + if rc != 0: + click.echo( + _("FAILED: {pair} exited with code {code}", pair=pair, code=rc) + ) + sys.exit(rc) + + click.echo(_("PASSED: {pair}", pair=pair)) + + click.echo(_("All molecule tests passed.")) finally: stop_event.set() - sys.exit(process.returncode) + sys.exit(0) if __name__ == "__main__": # pragma: no cover diff --git a/tests/unit/test_molecule_ci_guard.py b/tests/unit/test_molecule_ci_guard.py index 23174d2..16cb24f 100644 --- a/tests/unit/test_molecule_ci_guard.py +++ b/tests/unit/test_molecule_ci_guard.py @@ -4,6 +4,7 @@ from __future__ import annotations import os import subprocess # nosec B404 +import time from unittest.mock import MagicMock, patch import pytest @@ -11,6 +12,8 @@ import requests from scripts.molecule_ci_guard import ( any_other_runner_failed, + build_env_for_pair, + build_molecule_cmd, cli, get_running_jobs, poll_for_other_failures, @@ -65,6 +68,32 @@ class TestAnyOtherRunnerFailed: assert any_other_runner_failed(jobs, "molecule-tests", 0) is False +class TestBuildMoleculeCmd: + def test_default_scenario(self) -> None: + assert build_molecule_cmd("default") == ["molecule", "test"] + + def test_named_scenario(self) -> None: + assert build_molecule_cmd("lifecycle") == ["molecule", "test", "-s", "lifecycle"] + + +class TestBuildEnvForPair: + def test_with_command(self) -> None: + env = build_env_for_pair("default|ubuntu-2204|img:latest|/lib/systemd/systemd", {}) + assert env["MOLECULE_PLATFORM_NAME"] == "ubuntu-2204" + assert env["MOLECULE_PLATFORM_IMAGE"] == "img:latest" + assert env["MOLECULE_PLATFORM_COMMAND"] == "/lib/systemd/systemd" + assert env["ANSIBLE_ALLOW_BROKEN_CONDITIONALS"] == "true" + + def test_without_command(self) -> None: + env = build_env_for_pair("default|ubuntu-2204|img:latest|", {}) + assert env["MOLECULE_PLATFORM_NAME"] == "ubuntu-2204" + assert "MOLECULE_PLATFORM_COMMAND" not in env + + def test_without_command_removes_existing(self) -> None: + env = build_env_for_pair("default|ubuntu-2204|img:latest|", {"MOLECULE_PLATFORM_COMMAND": "old"}) + assert "MOLECULE_PLATFORM_COMMAND" not in env + + class TestPollForOtherFailures: def test_sets_failed_event_when_other_runner_fails(self) -> None: stop_event = MagicMock() @@ -122,13 +151,12 @@ class TestPollForOtherFailures: class TestCli: - def test_runs_without_api_env(self) -> None: + def test_all_pass(self) -> None: from click.testing import CliRunner - env = {"PATH": os.environ.get("PATH", "")} with ( - patch.dict(os.environ, env, clear=True), patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen, + patch("time.sleep"), ): proc = MagicMock() proc.poll.return_value = 0 @@ -138,8 +166,26 @@ class TestCli: runner = CliRunner() result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"]) assert result.exit_code == 0 + assert "All molecule tests passed" in result.output - def test_exits_when_other_runner_fails(self) -> None: + def test_failure_exits_nonzero(self) -> None: + from click.testing import CliRunner + + with ( + patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen, + patch("time.sleep"), + ): + proc = MagicMock() + proc.poll.return_value = 1 + proc.returncode = 1 + mock_popen.return_value = proc + + runner = CliRunner() + result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"]) + assert result.exit_code == 1 + assert "FAILED" in result.output + + def test_exits_before_starting_when_already_failed(self) -> None: from click.testing import CliRunner with ( @@ -156,19 +202,85 @@ class TestCli: }, clear=True, ), + patch("scripts.molecule_ci_guard.POLL_INTERVAL", 0.01), patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen, patch("scripts.molecule_ci_guard.get_running_jobs") as mock_get_jobs, - patch("os.killpg") as mock_killpg, - patch("os.getpgid") as mock_getpgid, + patch("time.sleep"), ): - mock_getpgid.return_value = 123 mock_get_jobs.return_value = [ {"name": "molecule-tests (0)", "conclusion": "running"}, {"name": "molecule-tests (1)", "conclusion": "failure"}, ] + proc = MagicMock() + proc.poll.return_value = 0 + proc.returncode = 0 + mock_popen.return_value = proc + runner = CliRunner() + result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"]) + assert result.exit_code == 1 + assert "Another molecule runner failed" in result.output + + def test_keyboard_interrupt_kills_process(self) -> None: + from click.testing import CliRunner + + with ( + patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen, + patch("time.sleep", side_effect=KeyboardInterrupt), + patch("os.killpg") as mock_killpg, + patch("os.getpgid") as mock_getpgid, + ): + mock_getpgid.return_value = 123 proc = MagicMock() proc.poll.return_value = None + proc.wait.return_value = 0 + mock_popen.return_value = proc + + runner = CliRunner() + result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"]) + assert result.exit_code == 1 + mock_killpg.assert_called() + + def test_exits_when_other_runner_fails(self) -> None: + from click.testing import CliRunner + + real_sleep = time.sleep + call_count = [0] + + def get_jobs_side_effect(*args, **kwargs): + call_count[0] += 1 + if call_count[0] < 2: + return [{"name": "molecule-tests (1)", "conclusion": "running"}] + return [ + {"name": "molecule-tests (0)", "conclusion": "running"}, + {"name": "molecule-tests (1)", "conclusion": "failure"}, + ] + + with ( + patch.dict( + os.environ, + { + "GITEA_URL": "https://gitea.example", + "REPO_TOKEN": "token", + "RUN_ID": "123", + "JOB_NAME": "molecule-tests", + "MATRIX_INDEX": "0", + "GITEA_REPOSITORY": "oblachno-oss/grm", + "PATH": os.environ.get("PATH", ""), + }, + clear=True, + ), + patch("scripts.molecule_ci_guard.POLL_INTERVAL", 0.01), + patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen, + patch("scripts.molecule_ci_guard.get_running_jobs", side_effect=get_jobs_side_effect), + patch("os.killpg") as mock_killpg, + patch("os.getpgid") as mock_getpgid, + patch("time.sleep", side_effect=lambda x: real_sleep(0.1)), + ): + mock_getpgid.return_value = 123 + proc = MagicMock() + proc.poll.return_value = None + proc.wait.return_value = 0 mock_popen.return_value = proc runner = CliRunner() @@ -179,6 +291,8 @@ class TestCli: def test_default_owner_repo_fallback(self) -> None: from click.testing import CliRunner + real_sleep = time.sleep + with ( patch.dict( os.environ, @@ -193,8 +307,10 @@ class TestCli: }, clear=True, ), + patch("scripts.molecule_ci_guard.POLL_INTERVAL", 0.01), patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen, patch("scripts.molecule_ci_guard.get_running_jobs") as mock_get_jobs, + patch("time.sleep", side_effect=lambda x: real_sleep(0.05)), ): mock_get_jobs.return_value = [{"name": "molecule-tests (1)", "conclusion": "success"}] proc = MagicMock() @@ -206,69 +322,47 @@ class TestCli: result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"]) assert result.exit_code == 0 - def test_sleeps_while_waiting_for_process(self) -> None: - from click.testing import CliRunner - - with ( - patch.dict( - os.environ, - { - "GITEA_URL": "https://gitea.example", - "REPO_TOKEN": "token", - "RUN_ID": "123", - "JOB_NAME": "molecule-tests", - "MATRIX_INDEX": "0", - "GITEA_REPOSITORY": "oblachno-oss/grm", - "PATH": os.environ.get("PATH", ""), - }, - clear=True, - ), - patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen, - patch("scripts.molecule_ci_guard.get_running_jobs") as mock_get_jobs, - patch("time.sleep") as mock_sleep, - ): - mock_get_jobs.return_value = [{"name": "molecule-tests (1)", "conclusion": "success"}] - proc = MagicMock() - proc.poll.side_effect = [None, 0] - proc.returncode = 0 - mock_popen.return_value = proc - - runner = CliRunner() - result = runner.invoke(cli, ["default|ubuntu-2204|img:latest|"]) - assert result.exit_code == 0 - mock_sleep.assert_called() - def test_exits_when_other_runner_fails_with_process_lookup_error(self) -> None: from click.testing import CliRunner - with ( - patch.dict( - os.environ, - { - "GITEA_URL": "https://gitea.example", - "REPO_TOKEN": "token", - "RUN_ID": "123", - "JOB_NAME": "molecule-tests", - "MATRIX_INDEX": "0", - "GITEA_REPOSITORY": "oblachno-oss/grm", - "PATH": os.environ.get("PATH", ""), - }, - clear=True, - ), - patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen, - patch("scripts.molecule_ci_guard.get_running_jobs") as mock_get_jobs, - patch("os.killpg") as mock_killpg, - patch("os.getpgid") as mock_getpgid, - ): - mock_getpgid.return_value = 123 - mock_killpg.side_effect = ProcessLookupError("no such process") - mock_get_jobs.return_value = [ + real_sleep = time.sleep + call_count = [0] + + def get_jobs_side_effect(*args, **kwargs): + call_count[0] += 1 + if call_count[0] < 2: + return [{"name": "molecule-tests (1)", "conclusion": "running"}] + return [ {"name": "molecule-tests (0)", "conclusion": "running"}, {"name": "molecule-tests (1)", "conclusion": "failure"}, ] + with ( + patch.dict( + os.environ, + { + "GITEA_URL": "https://gitea.example", + "REPO_TOKEN": "token", + "RUN_ID": "123", + "JOB_NAME": "molecule-tests", + "MATRIX_INDEX": "0", + "GITEA_REPOSITORY": "oblachno-oss/grm", + "PATH": os.environ.get("PATH", ""), + }, + clear=True, + ), + patch("scripts.molecule_ci_guard.POLL_INTERVAL", 0.01), + patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen, + patch("scripts.molecule_ci_guard.get_running_jobs", side_effect=get_jobs_side_effect), + patch("os.killpg") as mock_killpg, + patch("os.getpgid") as mock_getpgid, + patch("time.sleep", side_effect=lambda x: real_sleep(0.1)), + ): + mock_getpgid.return_value = 123 + mock_killpg.side_effect = ProcessLookupError("no such process") proc = MagicMock() proc.poll.return_value = None + proc.wait.return_value = 0 mock_popen.return_value = proc runner = CliRunner() @@ -278,6 +372,18 @@ class TestCli: def test_exits_when_other_runner_fails_with_timeout(self) -> None: from click.testing import CliRunner + real_sleep = time.sleep + call_count = [0] + + def get_jobs_side_effect(*args, **kwargs): + call_count[0] += 1 + if call_count[0] < 2: + return [{"name": "molecule-tests (1)", "conclusion": "running"}] + return [ + {"name": "molecule-tests (0)", "conclusion": "running"}, + {"name": "molecule-tests (1)", "conclusion": "failure"}, + ] + with ( patch.dict( os.environ, @@ -292,18 +398,15 @@ class TestCli: }, clear=True, ), + patch("scripts.molecule_ci_guard.POLL_INTERVAL", 0.01), patch("scripts.molecule_ci_guard.subprocess.Popen") as mock_popen, - patch("scripts.molecule_ci_guard.get_running_jobs") as mock_get_jobs, + patch("scripts.molecule_ci_guard.get_running_jobs", side_effect=get_jobs_side_effect), patch("os.killpg") as mock_killpg, patch("os.getpgid") as mock_getpgid, + patch("time.sleep", side_effect=lambda x: real_sleep(0.1)), ): mock_getpgid.return_value = 123 mock_killpg.side_effect = [None, ProcessLookupError("no such process")] - mock_get_jobs.return_value = [ - {"name": "molecule-tests (0)", "conclusion": "running"}, - {"name": "molecule-tests (1)", "conclusion": "failure"}, - ] - proc = MagicMock() proc.poll.return_value = None proc.wait.side_effect = [subprocess.TimeoutExpired("cmd", 10)]