fix: add GitHub mirror fallback for actionlint and vale downloads
CI image builds failed repeatedly because GitHub releases dropped connections (RemoteDisconnected) on all 5 retry attempts. Add ghproxy.com as a fallback URL for _download_and_extract_tarball so the build can fall through to a mirror when GitHub is flaky. - Add fallback_urls parameter to _download_and_extract_tarball - Add ghproxy.com mirror fallback for actionlint and vale - Add tests for fallback success and all-URLs-fail paths DEVX-159 Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
parent
f9cdbec86a
commit
deb9399232
@@ -131,26 +131,37 @@ def _download_with_fallback(urls: list[str], binary_name: str) -> Path:
|
||||
raise click.ClickException(f"Failed to download {binary_name} from all URLs: {'; '.join(errors)}")
|
||||
|
||||
|
||||
def _download_and_extract_tarball(url: str, binary_name: str) -> Path:
|
||||
def _download_and_extract_tarball(url: str, binary_name: str, *, fallback_urls: list[str] | None = None) -> Path:
|
||||
"""Download a tarball, extract the binary, and install it to TARGET_DIR.
|
||||
|
||||
Returns the path to the installed binary.
|
||||
Returns the path to the installed binary. Falls back to ``fallback_urls``
|
||||
if the primary ``url`` fails all retries.
|
||||
"""
|
||||
target_dir = _ensure_target_dir()
|
||||
dest = target_dir / binary_name
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
tarball = Path(tmpdir) / "archive.tar.gz"
|
||||
_download(url, tarball)
|
||||
with tarfile.open(tarball, "r:gz") as tar:
|
||||
tar.extractall(tmpdir) # nosec B202
|
||||
# Find the binary in the extracted tree
|
||||
extracted = Path(tmpdir).rglob(binary_name)
|
||||
found = next(extracted, None)
|
||||
if found is None:
|
||||
raise click.ClickException(f"Binary {binary_name} not found in archive from {url}")
|
||||
shutil.copy2(found, dest)
|
||||
dest.chmod(0o755)
|
||||
return dest
|
||||
urls = [url, *(fallback_urls or [])]
|
||||
errors: list[str] = []
|
||||
for try_url in urls:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
tarball = Path(tmpdir) / "archive.tar.gz"
|
||||
try:
|
||||
_download(try_url, tarball)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
errors.append(f"{try_url}: {exc}")
|
||||
click.echo(f" {binary_name}: fallback — {exc}")
|
||||
continue
|
||||
with tarfile.open(tarball, "r:gz") as tar:
|
||||
tar.extractall(tmpdir) # nosec B202
|
||||
# Find the binary in the extracted tree
|
||||
extracted = Path(tmpdir).rglob(binary_name)
|
||||
found = next(extracted, None)
|
||||
if found is None:
|
||||
errors.append(f"{try_url}: binary not found in archive")
|
||||
continue
|
||||
shutil.copy2(found, dest)
|
||||
dest.chmod(0o755)
|
||||
return dest
|
||||
raise click.ClickException(f"Failed to download {binary_name} from all URLs: {'; '.join(errors)}")
|
||||
|
||||
|
||||
def _download_binary(url: str, binary_name: str) -> Path:
|
||||
@@ -178,11 +189,12 @@ def install_actionlint() -> bool:
|
||||
click.echo("actionlint: already installed")
|
||||
return True
|
||||
arch = _arch()
|
||||
url = (
|
||||
f"https://github.com/rhysd/actionlint/releases/download/"
|
||||
f"v{ACTIONLINT_VERSION}/actionlint_{ACTIONLINT_VERSION}_linux_{arch}.tar.gz"
|
||||
path = (
|
||||
f"rhysd/actionlint/releases/download/v{ACTIONLINT_VERSION}/actionlint_{ACTIONLINT_VERSION}_linux_{arch}.tar.gz"
|
||||
)
|
||||
dest = _download_and_extract_tarball(url, "actionlint")
|
||||
url = f"https://github.com/{path}"
|
||||
fallback = [f"https://ghproxy.com/{path}"]
|
||||
dest = _download_and_extract_tarball(url, "actionlint", fallback_urls=fallback)
|
||||
click.echo(f"actionlint: installed to {dest}")
|
||||
return True
|
||||
|
||||
@@ -276,8 +288,10 @@ def install_vale() -> bool:
|
||||
return True
|
||||
machine = platform.machine().lower()
|
||||
arch = "64-bit" if machine in {"x86_64", "amd64"} else "arm64"
|
||||
url = f"https://github.com/errata-ai/vale/releases/download/v{VALE_VERSION}/vale_{VALE_VERSION}_Linux_{arch}.tar.gz"
|
||||
dest = _download_and_extract_tarball(url, "vale")
|
||||
path = f"errata-ai/vale/releases/download/v{VALE_VERSION}/vale_{VALE_VERSION}_Linux_{arch}.tar.gz"
|
||||
url = f"https://github.com/{path}"
|
||||
fallback = [f"https://ghproxy.com/{path}"]
|
||||
dest = _download_and_extract_tarball(url, "vale", fallback_urls=fallback)
|
||||
click.echo(f"vale: installed to {dest}")
|
||||
return True
|
||||
|
||||
|
||||
@@ -201,6 +201,50 @@ class TestDownloadAndExtractTarball:
|
||||
with pytest.raises(ClickException, match="not found in archive"):
|
||||
install_tools._download_and_extract_tarball("https://example.com/actionlint.tar.gz", "actionlint")
|
||||
|
||||
def test_fallback_url_succeeds(self, tmp_path: Path) -> None:
|
||||
import io
|
||||
import tarfile
|
||||
|
||||
tarball_path = tmp_path / "archive.tar.gz"
|
||||
binary_content = b"fake binary"
|
||||
with tarfile.open(tarball_path, "w:gz") as tar:
|
||||
info = tarfile.TarInfo(name="actionlint")
|
||||
info.size = len(binary_content)
|
||||
tar.addfile(info, io.BytesIO(binary_content))
|
||||
|
||||
target_dir = tmp_path / "bin"
|
||||
target_dir.mkdir()
|
||||
tarball_bytes = tarball_path.read_bytes()
|
||||
|
||||
def fake_download(url: str, dest: Path) -> None:
|
||||
if "primary" in url:
|
||||
raise OSError("connection refused")
|
||||
Path(dest).write_bytes(tarball_bytes)
|
||||
|
||||
with patch.object(install_tools, "TARGET_DIR", target_dir):
|
||||
with patch.object(install_tools, "_download", side_effect=fake_download):
|
||||
result = install_tools._download_and_extract_tarball(
|
||||
"https://primary.com/actionlint.tar.gz",
|
||||
"actionlint",
|
||||
fallback_urls=["https://fallback.com/actionlint.tar.gz"],
|
||||
)
|
||||
|
||||
assert result == target_dir / "actionlint"
|
||||
assert result.read_bytes() == binary_content
|
||||
|
||||
def test_all_urls_fail(self, tmp_path: Path) -> None:
|
||||
target_dir = tmp_path / "bin"
|
||||
target_dir.mkdir()
|
||||
|
||||
with patch.object(install_tools, "TARGET_DIR", target_dir):
|
||||
with patch.object(install_tools, "_download", side_effect=OSError("connection refused")):
|
||||
with pytest.raises(ClickException, match="Failed to download"):
|
||||
install_tools._download_and_extract_tarball(
|
||||
"https://primary.com/actionlint.tar.gz",
|
||||
"actionlint",
|
||||
fallback_urls=["https://fallback.com/actionlint.tar.gz"],
|
||||
)
|
||||
|
||||
|
||||
class TestInstallActionlint:
|
||||
def test_already_installed(self) -> None:
|
||||
|
||||
Reference in New Issue
Block a user