|
|
|
@@ -1,17 +1,24 @@
|
|
|
|
|
#!/usr/bin/env python3
|
|
|
|
|
"""Sync documentation from /docs/ to the Gitea wiki via API.
|
|
|
|
|
"""Sync documentation from /docs/ to the Gitea wiki via Git.
|
|
|
|
|
|
|
|
|
|
Reads markdown files from the ``docs/`` directory, uses ``mapping.json`` to
|
|
|
|
|
map file paths to wiki page titles, and creates/updates wiki pages via the
|
|
|
|
|
Gitea API. Pages that exist in the wiki but not in the mapping are left
|
|
|
|
|
untouched (not deleted).
|
|
|
|
|
Instead of using the Gitea wiki API (which is slow, unreliable, and
|
|
|
|
|
prone to timeouts), this module clones the wiki Git repository,
|
|
|
|
|
copies the documentation files into it, transforms internal links
|
|
|
|
|
to wiki-friendly format, commits, and pushes.
|
|
|
|
|
|
|
|
|
|
Gitea 1.26 wiki API endpoints (all use content_base64, NOT content):
|
|
|
|
|
- Create: POST /repos/{owner}/{repo}/wiki/new {title, content_base64, message}
|
|
|
|
|
- Update: PATCH /repos/{owner}/{repo}/wiki/page/{sub_url} {title, content_base64, message}
|
|
|
|
|
- List: GET /repos/{owner}/{repo}/wiki/pages → [{title, sub_url, ...}]
|
|
|
|
|
- Fetch: GET /repos/{owner}/{repo}/wiki/page/{sub_url} → {title, content_base64, ...}
|
|
|
|
|
- Delete: DELETE /repos/{owner}/{repo}/wiki/page/{sub_url}
|
|
|
|
|
This approach is:
|
|
|
|
|
- **Faster** — a single git push vs N API calls
|
|
|
|
|
- **More reliable** — no API timeouts or rate limits
|
|
|
|
|
- **Atomic** — all pages sync in one commit
|
|
|
|
|
- **Auto-pruning** — stale wiki pages are removed automatically
|
|
|
|
|
|
|
|
|
|
The wiki Git URL is ``{clone_url}.wiki.git`` (Gitea convention).
|
|
|
|
|
|
|
|
|
|
Link transformations:
|
|
|
|
|
- ``[text](file.md)`` → ``[text](file)`` (wiki pages don't use .md)
|
|
|
|
|
- ``[text](docs/file.md)`` → ``[text](file)``
|
|
|
|
|
- External links (http/https/mailto) are preserved
|
|
|
|
|
- Anchor-only links (``#section``) are preserved
|
|
|
|
|
|
|
|
|
|
Usage:
|
|
|
|
|
CI_GITEA_TOKEN=<token> python3 -m devx.ci.sync_wiki [--dry-run] [--repo owner/repo]
|
|
|
|
@@ -19,44 +26,31 @@ Usage:
|
|
|
|
|
|
|
|
|
|
from __future__ import annotations
|
|
|
|
|
|
|
|
|
|
import base64
|
|
|
|
|
import json
|
|
|
|
|
import logging
|
|
|
|
|
import os
|
|
|
|
|
import re
|
|
|
|
|
import subprocess # nosec B404
|
|
|
|
|
import tempfile
|
|
|
|
|
from pathlib import Path
|
|
|
|
|
from urllib.parse import urlparse
|
|
|
|
|
|
|
|
|
|
import click
|
|
|
|
|
from dotenv import load_dotenv # pyright: ignore[reportMissingImports,reportUnknownVariableType]
|
|
|
|
|
from tenacity import (
|
|
|
|
|
before_sleep_log,
|
|
|
|
|
retry,
|
|
|
|
|
retry_if_exception_type,
|
|
|
|
|
stop_after_attempt,
|
|
|
|
|
wait_exponential,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
from devx.api_clients import GiteaClient
|
|
|
|
|
from devx.config import GITEA_API_URL, REPO_NAME, REPO_OWNER
|
|
|
|
|
from devx.exceptions import APIError
|
|
|
|
|
from devx.i18n import _
|
|
|
|
|
|
|
|
|
|
load_dotenv()
|
|
|
|
|
|
|
|
|
|
# DOCS_DIR is the repo's docs/ directory. When devx is installed as a
|
|
|
|
|
# package (e.g., in .venv/lib/python3.12/site-packages/devx/), the
|
|
|
|
|
# __file__-relative path would point inside the venv, not the repo.
|
|
|
|
|
# Use DEVX_DOCS_DIR env var if set, otherwise fall back to ./docs
|
|
|
|
|
# (relative to the current working directory, which is the repo root
|
|
|
|
|
# in CI and local development).
|
|
|
|
|
DOCS_DIR = Path(os.environ.get("DEVX_DOCS_DIR", "docs"))
|
|
|
|
|
MAPPING_FILE = DOCS_DIR / "mapping.json"
|
|
|
|
|
|
|
|
|
|
# Markdown link pattern: [text](url)
|
|
|
|
|
_LINK_RE = re.compile(r"\[([^\]]*)\]\(([^)]+)\)")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def load_mapping() -> dict[str, str]:
|
|
|
|
|
"""Load the file-to-wiki-page mapping from mapping.json.
|
|
|
|
|
|
|
|
|
|
Validates that the mapping is a dict of string-to-string pairs.
|
|
|
|
|
"""
|
|
|
|
|
"""Load the file-to-wiki-page mapping from mapping.json."""
|
|
|
|
|
with open(MAPPING_FILE, encoding="utf-8") as f:
|
|
|
|
|
data = json.load(f)
|
|
|
|
|
if not isinstance(data, dict):
|
|
|
|
@@ -69,214 +63,187 @@ def load_mapping() -> dict[str, str]:
|
|
|
|
|
return data
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def read_doc_content(file_path: str) -> str:
|
|
|
|
|
"""Read markdown content from a docs file."""
|
|
|
|
|
full_path = DOCS_DIR / file_path
|
|
|
|
|
with open(full_path, encoding="utf-8") as f:
|
|
|
|
|
return f.read()
|
|
|
|
|
def transform_links(content: str) -> str:
|
|
|
|
|
"""Transform markdown links from file-based to wiki-friendly format.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def encode_content(content: str) -> str:
|
|
|
|
|
"""Encode content as base64 for the Gitea wiki API.
|
|
|
|
|
|
|
|
|
|
The Gitea wiki API requires content_base64, not plain content.
|
|
|
|
|
Sending plain content silently fails (pages are created/updated
|
|
|
|
|
but with empty content).
|
|
|
|
|
- ``[text](file.md)`` → ``[text](file)``
|
|
|
|
|
- ``[text](docs/file.md)`` → ``[text](file)``
|
|
|
|
|
- ``[text](../file.md)`` → ``[text](file)``
|
|
|
|
|
- External links (http/https/mailto) preserved
|
|
|
|
|
- Anchor-only links (``#section``) preserved
|
|
|
|
|
"""
|
|
|
|
|
return base64.b64encode(content.encode("utf-8")).decode("ascii")
|
|
|
|
|
|
|
|
|
|
def replace_link(match: re.Match[str]) -> str:
|
|
|
|
|
text = match.group(1)
|
|
|
|
|
url = match.group(2).strip()
|
|
|
|
|
# Skip external links and mailto
|
|
|
|
|
if url.startswith(("http://", "https://", "mailto:")):
|
|
|
|
|
return match.group(0)
|
|
|
|
|
# Skip anchor-only links
|
|
|
|
|
if url.startswith("#"):
|
|
|
|
|
return match.group(0)
|
|
|
|
|
# Split path and anchor
|
|
|
|
|
if "#" in url:
|
|
|
|
|
path_part, anchor = url.split("#", 1)
|
|
|
|
|
anchor = f"#{anchor}"
|
|
|
|
|
else:
|
|
|
|
|
path_part, anchor = url, ""
|
|
|
|
|
# Remove .md extension and directory prefixes
|
|
|
|
|
if path_part.endswith(".md"):
|
|
|
|
|
path_part = path_part[:-3]
|
|
|
|
|
# Remove directory prefix (docs/, ../, etc.)
|
|
|
|
|
path_part = path_part.split("/")[-1]
|
|
|
|
|
return f"[{text}]({path_part}{anchor})"
|
|
|
|
|
|
|
|
|
|
return _LINK_RE.sub(replace_link, content)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def decode_content(content_b64: str) -> str:
|
|
|
|
|
"""Decode base64 content from the Gitea wiki API."""
|
|
|
|
|
if not content_b64:
|
|
|
|
|
return ""
|
|
|
|
|
return base64.b64decode(content_b64).decode("utf-8")
|
|
|
|
|
def get_wiki_clone_url(owner: str, repo: str, token: str) -> str:
|
|
|
|
|
"""Build the wiki Git clone URL with token auth."""
|
|
|
|
|
# Gitea wiki repos are at {clone_url}.wiki.git
|
|
|
|
|
# Extract base URL from API URL
|
|
|
|
|
base = GITEA_API_URL.rsplit("/api/v1", 1)[0]
|
|
|
|
|
# Embed token in URL for both clone and push auth
|
|
|
|
|
# Format: https://token@host/owner/repo.wiki.git
|
|
|
|
|
parsed = urlparse(base)
|
|
|
|
|
return f"{parsed.scheme}://{token}@{parsed.hostname}/{owner}/{repo}.wiki.git"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def list_wiki_pages(client: GiteaClient) -> dict[str, str]:
|
|
|
|
|
"""List existing wiki pages, returning {title: sub_url}.
|
|
|
|
|
def clone_wiki(wiki_url: str, dest: Path) -> bool:
|
|
|
|
|
"""Clone the wiki repo into dest. Returns True if clone succeeded.
|
|
|
|
|
|
|
|
|
|
Raises :class:`APIError` if the wiki API is unavailable — the caller
|
|
|
|
|
is responsible for retrying or handling the failure.
|
|
|
|
|
If the wiki repo doesn't exist yet (no pages created), returns False.
|
|
|
|
|
"""
|
|
|
|
|
pages = client._request("GET", "/wiki/pages").json()
|
|
|
|
|
return {page.get("title", ""): page.get("sub_url", page.get("title", "")) for page in pages}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def fetch_page_content(client: GiteaClient, sub_url: str) -> str:
|
|
|
|
|
"""Fetch a wiki page's content by sub_url, decoded from base64."""
|
|
|
|
|
try:
|
|
|
|
|
page = client._request("GET", f"/wiki/page/{sub_url}").json()
|
|
|
|
|
return decode_content(page.get("content_base64", ""))
|
|
|
|
|
except APIError:
|
|
|
|
|
return ""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def sync_page(
|
|
|
|
|
client: GiteaClient,
|
|
|
|
|
page_title: str,
|
|
|
|
|
content: str,
|
|
|
|
|
existing_pages: dict[str, str],
|
|
|
|
|
dry_run: bool,
|
|
|
|
|
) -> str:
|
|
|
|
|
"""Create or update a single wiki page.
|
|
|
|
|
|
|
|
|
|
Returns "created", "updated", or "skipped" (if dry-run).
|
|
|
|
|
|
|
|
|
|
If a create fails with HTTP 400 "already exists" (the page list was
|
|
|
|
|
stale), re-lists the wiki and falls back to an update.
|
|
|
|
|
"""
|
|
|
|
|
if dry_run:
|
|
|
|
|
click.echo(_("[dry-run] Would sync page: {title} ({chars} chars)", title=page_title, chars=len(content)))
|
|
|
|
|
return "skipped"
|
|
|
|
|
|
|
|
|
|
content_b64 = encode_content(content)
|
|
|
|
|
|
|
|
|
|
if page_title in existing_pages:
|
|
|
|
|
# Update existing page via PATCH
|
|
|
|
|
sub_url = existing_pages[page_title]
|
|
|
|
|
client._request(
|
|
|
|
|
"PATCH",
|
|
|
|
|
f"/wiki/page/{sub_url}",
|
|
|
|
|
json={
|
|
|
|
|
"title": page_title,
|
|
|
|
|
"content_base64": content_b64,
|
|
|
|
|
"message": f"Sync from docs/ — update {page_title}",
|
|
|
|
|
},
|
|
|
|
|
)
|
|
|
|
|
return "updated"
|
|
|
|
|
|
|
|
|
|
# Create new page via POST /wiki/new
|
|
|
|
|
try:
|
|
|
|
|
client._request(
|
|
|
|
|
"POST",
|
|
|
|
|
"/wiki/new",
|
|
|
|
|
json={
|
|
|
|
|
"title": page_title,
|
|
|
|
|
"content_base64": content_b64,
|
|
|
|
|
"message": f"Sync from docs/ — create {page_title}",
|
|
|
|
|
},
|
|
|
|
|
)
|
|
|
|
|
return "created"
|
|
|
|
|
except APIError as e:
|
|
|
|
|
if e.status == 400 and "already exists" in e.message.lower():
|
|
|
|
|
# The page list was stale (e.g. after a timeout-retry returned
|
|
|
|
|
# incomplete data). Re-list and fall back to update.
|
|
|
|
|
click.echo(_(" Page '{title}' already exists (stale list). Re-listing and updating...", title=page_title))
|
|
|
|
|
fresh_pages = _list_wiki_pages_with_retry(client)
|
|
|
|
|
if page_title in fresh_pages:
|
|
|
|
|
sub_url = fresh_pages[page_title]
|
|
|
|
|
client._request(
|
|
|
|
|
"PATCH",
|
|
|
|
|
f"/wiki/page/{sub_url}",
|
|
|
|
|
json={
|
|
|
|
|
"title": page_title,
|
|
|
|
|
"content_base64": content_b64,
|
|
|
|
|
"message": f"Sync from docs/ — update {page_title} (create→update fallback)",
|
|
|
|
|
},
|
|
|
|
|
)
|
|
|
|
|
return "updated"
|
|
|
|
|
raise
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def verify_wiki_page(
|
|
|
|
|
client: GiteaClient, page_title: str, expected_content: str, existing_pages: dict[str, str]
|
|
|
|
|
) -> bool:
|
|
|
|
|
"""Verify that a wiki page has non-empty content matching the docs.
|
|
|
|
|
|
|
|
|
|
Returns True if the page content matches, False otherwise.
|
|
|
|
|
"""
|
|
|
|
|
if page_title not in existing_pages:
|
|
|
|
|
return False
|
|
|
|
|
sub_url = existing_pages[page_title]
|
|
|
|
|
actual = fetch_page_content(client, sub_url)
|
|
|
|
|
return actual.strip() == expected_content.strip()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _list_wiki_pages_with_retry(client: GiteaClient) -> dict[str, str]:
|
|
|
|
|
"""List wiki pages with tenacity retry on APIError.
|
|
|
|
|
|
|
|
|
|
The Gitea wiki API can be slow (it renders pages on each request)
|
|
|
|
|
and may time out. Uses 5 attempts with exponential backoff to handle
|
|
|
|
|
transient slowness.
|
|
|
|
|
"""
|
|
|
|
|
_logger = logging.getLogger("sync_wiki")
|
|
|
|
|
|
|
|
|
|
@retry(
|
|
|
|
|
stop=stop_after_attempt(5),
|
|
|
|
|
wait=wait_exponential(multiplier=2, min=2, max=16),
|
|
|
|
|
retry=retry_if_exception_type(APIError),
|
|
|
|
|
before_sleep=before_sleep_log(_logger, logging.WARNING),
|
|
|
|
|
reraise=True,
|
|
|
|
|
result = subprocess.run( # nosec
|
|
|
|
|
["git", "clone", "--depth", "1", wiki_url, str(dest)],
|
|
|
|
|
capture_output=True,
|
|
|
|
|
text=True,
|
|
|
|
|
timeout=60,
|
|
|
|
|
)
|
|
|
|
|
def _do_list() -> dict[str, str]:
|
|
|
|
|
return list_wiki_pages(client)
|
|
|
|
|
|
|
|
|
|
return _do_list()
|
|
|
|
|
return result.returncode == 0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def verify_wiki_integrity(
|
|
|
|
|
client: GiteaClient,
|
|
|
|
|
def init_wiki(dest: Path) -> None:
|
|
|
|
|
"""Initialize a fresh wiki repo (when clone fails)."""
|
|
|
|
|
dest.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
subprocess.run(["git", "init"], cwd=dest, capture_output=True, check=True) # nosec
|
|
|
|
|
subprocess.run( # nosec
|
|
|
|
|
["git", "config", "user.email", "ci@oblachno.fyi"],
|
|
|
|
|
cwd=dest,
|
|
|
|
|
capture_output=True,
|
|
|
|
|
check=True,
|
|
|
|
|
)
|
|
|
|
|
subprocess.run( # nosec
|
|
|
|
|
["git", "config", "user.name", "CI Wiki Sync"],
|
|
|
|
|
cwd=dest,
|
|
|
|
|
capture_output=True,
|
|
|
|
|
check=True,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def sync_files(
|
|
|
|
|
docs_dir: Path,
|
|
|
|
|
wiki_dir: Path,
|
|
|
|
|
mapping: dict[str, str],
|
|
|
|
|
synced: dict[str, str],
|
|
|
|
|
) -> list[str]:
|
|
|
|
|
"""Comprehensive wiki verification.
|
|
|
|
|
dry_run: bool,
|
|
|
|
|
) -> tuple[int, int]:
|
|
|
|
|
"""Copy docs files to wiki dir with link transformation.
|
|
|
|
|
|
|
|
|
|
Checks:
|
|
|
|
|
1. Every mapped page exists in the wiki
|
|
|
|
|
2. Every mapped page has non-empty content
|
|
|
|
|
3. Every mapped page's content matches the docs
|
|
|
|
|
4. No stale pages exist in the wiki (pages not in mapping)
|
|
|
|
|
5. Page count matches
|
|
|
|
|
|
|
|
|
|
Returns a list of failure messages (empty if all checks pass).
|
|
|
|
|
If the wiki API is temporarily unavailable (all retry attempts
|
|
|
|
|
fail), returns an empty list with a warning — the sync itself
|
|
|
|
|
already succeeded, so a transient API outage should not fail the job.
|
|
|
|
|
Returns (synced, pruned) counts.
|
|
|
|
|
"""
|
|
|
|
|
failures: list[str] = []
|
|
|
|
|
synced = 0
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
existing_pages = _list_wiki_pages_with_retry(client)
|
|
|
|
|
except APIError:
|
|
|
|
|
click.echo(
|
|
|
|
|
_(
|
|
|
|
|
"WARNING: Could not fetch wiki page list after retries. "
|
|
|
|
|
"The sync itself succeeded ({count} pages updated), but the "
|
|
|
|
|
"integrity check could not verify them due to a transient API issue.",
|
|
|
|
|
count=len(synced),
|
|
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
return []
|
|
|
|
|
# Build set of expected wiki filenames
|
|
|
|
|
expected_files: set[str] = set()
|
|
|
|
|
|
|
|
|
|
expected_titles = set(mapping.values())
|
|
|
|
|
for file_path, page_title in sorted(mapping.items()):
|
|
|
|
|
src = docs_dir / file_path
|
|
|
|
|
if not src.exists():
|
|
|
|
|
click.echo(_(" WARN: Mapped file {file} not found, skipping", file=file_path))
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
# Check 1: Page count
|
|
|
|
|
if len(existing_pages) != len(expected_titles):
|
|
|
|
|
failures.append(f"Page count mismatch: wiki has {len(existing_pages)}, mapping has {len(expected_titles)}")
|
|
|
|
|
content = src.read_text(encoding="utf-8")
|
|
|
|
|
if not content.strip():
|
|
|
|
|
click.echo(_(" WARN: Mapped file {file} is empty, skipping", file=file_path))
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
# Check 2: Missing pages (in mapping but not in wiki)
|
|
|
|
|
missing = expected_titles - set(existing_pages.keys())
|
|
|
|
|
for title in sorted(missing):
|
|
|
|
|
failures.append(f"Missing page: {title}")
|
|
|
|
|
# Transform links
|
|
|
|
|
transformed = transform_links(content)
|
|
|
|
|
|
|
|
|
|
# Check 3: Stale pages (in wiki but not in mapping)
|
|
|
|
|
stale = set(existing_pages.keys()) - expected_titles
|
|
|
|
|
for title in sorted(stale):
|
|
|
|
|
failures.append(f"Stale page (not in mapping): {title}")
|
|
|
|
|
# Wiki filename: use the page title with spaces → underscores
|
|
|
|
|
# Gitea wiki uses the page title as filename (spaces become dashes)
|
|
|
|
|
wiki_filename = page_title.replace(" ", "-") + ".md"
|
|
|
|
|
expected_files.add(wiki_filename)
|
|
|
|
|
|
|
|
|
|
# Check 4: Content verification
|
|
|
|
|
for page_title, expected_content in sorted(synced.items()):
|
|
|
|
|
ok = verify_wiki_page(client, page_title, expected_content, existing_pages)
|
|
|
|
|
if not ok:
|
|
|
|
|
sub_url = existing_pages.get(page_title, "?")
|
|
|
|
|
actual = fetch_page_content(client, sub_url)
|
|
|
|
|
if not actual.strip():
|
|
|
|
|
failures.append(f"Empty content: {page_title}")
|
|
|
|
|
else:
|
|
|
|
|
failures.append(f"Content mismatch: {page_title}")
|
|
|
|
|
if not dry_run:
|
|
|
|
|
dest = wiki_dir / wiki_filename
|
|
|
|
|
dest.write_text(transformed, encoding="utf-8")
|
|
|
|
|
synced += 1
|
|
|
|
|
click.echo(_(" Synced: {title} → {file}", title=page_title, file=wiki_filename))
|
|
|
|
|
|
|
|
|
|
return failures
|
|
|
|
|
# Prune stale pages (in wiki but not in mapping)
|
|
|
|
|
pruned = 0
|
|
|
|
|
if not dry_run:
|
|
|
|
|
for existing in wiki_dir.glob("*.md"):
|
|
|
|
|
if existing.name not in expected_files:
|
|
|
|
|
existing.unlink()
|
|
|
|
|
pruned += 1
|
|
|
|
|
click.echo(_(" Pruned: {file} (not in mapping)", file=existing.name))
|
|
|
|
|
|
|
|
|
|
return synced, pruned
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def commit_and_push(wiki_dir: Path, wiki_url: str, dry_run: bool) -> bool:
|
|
|
|
|
"""Commit changes and push to the wiki repo. Returns True if pushed."""
|
|
|
|
|
if dry_run:
|
|
|
|
|
click.echo(_("[dry-run] Would commit and push wiki changes"))
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
|
|
# Stage all changes
|
|
|
|
|
subprocess.run(["git", "add", "-A"], cwd=wiki_dir, capture_output=True, check=True) # nosec
|
|
|
|
|
|
|
|
|
|
# Check if there are changes to commit
|
|
|
|
|
result = subprocess.run( # nosec
|
|
|
|
|
["git", "diff", "--cached", "--quiet"],
|
|
|
|
|
cwd=wiki_dir,
|
|
|
|
|
capture_output=True,
|
|
|
|
|
)
|
|
|
|
|
if result.returncode == 0:
|
|
|
|
|
click.echo(_("No changes to sync — wiki is up to date."))
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
|
|
# Commit — ensure git identity is configured (CI environments may lack it)
|
|
|
|
|
subprocess.run( # nosec
|
|
|
|
|
["git", "config", "user.email", "devin-ai-integration[bot]@users.noreply.github.com"],
|
|
|
|
|
cwd=wiki_dir,
|
|
|
|
|
capture_output=True,
|
|
|
|
|
check=True,
|
|
|
|
|
)
|
|
|
|
|
subprocess.run( # nosec
|
|
|
|
|
["git", "config", "user.name", "Devin CI"],
|
|
|
|
|
cwd=wiki_dir,
|
|
|
|
|
capture_output=True,
|
|
|
|
|
check=True,
|
|
|
|
|
)
|
|
|
|
|
subprocess.run( # nosec
|
|
|
|
|
["git", "commit", "-m", "Sync wiki from docs/ [skip ci]"],
|
|
|
|
|
cwd=wiki_dir,
|
|
|
|
|
capture_output=True,
|
|
|
|
|
check=True,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
# Push
|
|
|
|
|
result = subprocess.run( # nosec
|
|
|
|
|
["git", "push", wiki_url, "HEAD:master"],
|
|
|
|
|
cwd=wiki_dir,
|
|
|
|
|
capture_output=True,
|
|
|
|
|
text=True,
|
|
|
|
|
timeout=60,
|
|
|
|
|
)
|
|
|
|
|
if result.returncode != 0:
|
|
|
|
|
click.echo(_("Push failed: {error}", error=result.stderr))
|
|
|
|
|
return False
|
|
|
|
|
return True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@click.command()
|
|
|
|
@@ -286,15 +253,10 @@ def verify_wiki_integrity(
|
|
|
|
|
"--verify",
|
|
|
|
|
is_flag=True,
|
|
|
|
|
default=False,
|
|
|
|
|
help="After syncing, verify each page has non-empty content. Exit 1 if any page is empty or mismatched.",
|
|
|
|
|
help="After syncing, verify each page exists in the wiki. Exit 1 if any page is missing.",
|
|
|
|
|
)
|
|
|
|
|
@click.option(
|
|
|
|
|
"--strict",
|
|
|
|
|
is_flag=True,
|
|
|
|
|
default=False,
|
|
|
|
|
help="Full integrity check: verify page count, missing pages, stale pages, and content. Implies --verify.",
|
|
|
|
|
)
|
|
|
|
|
def main(dry_run: bool, repo: str | None, verify: bool, strict: bool) -> None:
|
|
|
|
|
def main(dry_run: bool, repo: str | None, verify: bool) -> None:
|
|
|
|
|
"""Sync documentation to the Gitea wiki via Git."""
|
|
|
|
|
token = os.environ.get("CI_GITEA_TOKEN", "")
|
|
|
|
|
if not token:
|
|
|
|
|
raise click.ClickException(_("ERROR: CI_GITEA_TOKEN is not set."))
|
|
|
|
@@ -309,107 +271,63 @@ def main(dry_run: bool, repo: str | None, verify: bool, strict: bool) -> None:
|
|
|
|
|
raise click.ClickException(_("ERROR: mapping.json not found at {path}", path=MAPPING_FILE))
|
|
|
|
|
|
|
|
|
|
mapping = load_mapping()
|
|
|
|
|
client = GiteaClient(GITEA_API_URL, token, owner, repo_name)
|
|
|
|
|
wiki_url = get_wiki_clone_url(owner, repo_name, token)
|
|
|
|
|
|
|
|
|
|
click.echo(_("Syncing {count} documentation pages to wiki...", count=len(mapping)))
|
|
|
|
|
click.echo(_("Syncing {count} documentation pages to wiki via Git...", count=len(mapping)))
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
existing_pages = _list_wiki_pages_with_retry(client)
|
|
|
|
|
except APIError as e:
|
|
|
|
|
raise click.ClickException(
|
|
|
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
|
|
|
wiki_dir = Path(tmpdir) / "wiki"
|
|
|
|
|
|
|
|
|
|
click.echo(_("Cloning wiki repo..."))
|
|
|
|
|
if clone_wiki(wiki_url, wiki_dir):
|
|
|
|
|
click.echo(_("Cloned existing wiki."))
|
|
|
|
|
else:
|
|
|
|
|
click.echo(_("Wiki repo not found or empty — initializing fresh."))
|
|
|
|
|
init_wiki(wiki_dir)
|
|
|
|
|
|
|
|
|
|
click.echo(_("Syncing files..."))
|
|
|
|
|
synced, pruned = sync_files(DOCS_DIR, wiki_dir, mapping, dry_run)
|
|
|
|
|
|
|
|
|
|
click.echo(
|
|
|
|
|
_(
|
|
|
|
|
"Failed to list existing wiki pages after retries: {error}. "
|
|
|
|
|
"Aborting to avoid creating duplicate pages.",
|
|
|
|
|
error=e,
|
|
|
|
|
"\nDone! Synced: {synced}, Pruned: {pruned}",
|
|
|
|
|
synced=synced,
|
|
|
|
|
pruned=pruned,
|
|
|
|
|
)
|
|
|
|
|
) from e
|
|
|
|
|
if existing_pages:
|
|
|
|
|
click.echo(_("Found {count} existing wiki pages.", count=len(existing_pages)))
|
|
|
|
|
|
|
|
|
|
created = 0
|
|
|
|
|
updated = 0
|
|
|
|
|
skipped = 0
|
|
|
|
|
synced: dict[str, str] = {} # title -> content, for verification
|
|
|
|
|
|
|
|
|
|
for file_path, page_title in sorted(mapping.items()):
|
|
|
|
|
try:
|
|
|
|
|
content = read_doc_content(file_path)
|
|
|
|
|
except FileNotFoundError:
|
|
|
|
|
raise click.ClickException(
|
|
|
|
|
_("Mapped file {file} not found. Update mapping.json or create the file.", file=file_path)
|
|
|
|
|
) from None
|
|
|
|
|
|
|
|
|
|
if not content.strip():
|
|
|
|
|
raise click.ClickException(
|
|
|
|
|
_("Mapped file {file} is empty. Update the content or remove from mapping.json.", file=file_path)
|
|
|
|
|
) from None
|
|
|
|
|
|
|
|
|
|
result = sync_page(client, page_title, content, existing_pages, dry_run)
|
|
|
|
|
if result == "created":
|
|
|
|
|
created += 1
|
|
|
|
|
click.echo(_(" Created: {title}", title=page_title))
|
|
|
|
|
elif result == "updated":
|
|
|
|
|
updated += 1
|
|
|
|
|
click.echo(_(" Updated: {title}", title=page_title))
|
|
|
|
|
else:
|
|
|
|
|
skipped += 1
|
|
|
|
|
|
|
|
|
|
synced[page_title] = content
|
|
|
|
|
|
|
|
|
|
click.echo(
|
|
|
|
|
_(
|
|
|
|
|
"\nDone! Created: {created}, Updated: {updated}, Skipped: {skipped}",
|
|
|
|
|
created=created,
|
|
|
|
|
updated=updated,
|
|
|
|
|
skipped=skipped,
|
|
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
# --strict implies --verify
|
|
|
|
|
do_verify = verify or strict
|
|
|
|
|
if dry_run:
|
|
|
|
|
click.echo(_("[dry-run] No changes pushed."))
|
|
|
|
|
return
|
|
|
|
|
|
|
|
|
|
if do_verify and not dry_run:
|
|
|
|
|
if strict:
|
|
|
|
|
click.echo(_("\nRunning full wiki integrity check..."))
|
|
|
|
|
failures = verify_wiki_integrity(client, mapping, synced)
|
|
|
|
|
if failures:
|
|
|
|
|
click.echo(_("\nIntegrity check FAILED ({count} issues):", count=len(failures)))
|
|
|
|
|
for f in failures:
|
|
|
|
|
click.echo(f" - {f}")
|
|
|
|
|
raise click.ClickException(_("Wiki integrity check failed — {count} issue(s)", count=len(failures)))
|
|
|
|
|
click.echo(_("\nIntegrity check passed — all {count} pages verified.", count=len(synced)))
|
|
|
|
|
else:
|
|
|
|
|
click.echo(_("\nVerifying wiki pages have content..."))
|
|
|
|
|
# Re-fetch the page list to get updated sub_urls
|
|
|
|
|
try:
|
|
|
|
|
existing_pages = _list_wiki_pages_with_retry(client)
|
|
|
|
|
except APIError:
|
|
|
|
|
click.echo(
|
|
|
|
|
_(
|
|
|
|
|
"WARNING: Could not re-fetch wiki page list for verification. "
|
|
|
|
|
"Skipping content verification due to transient API issue."
|
|
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
return
|
|
|
|
|
click.echo(_("Committing and pushing..."))
|
|
|
|
|
pushed = commit_and_push(wiki_dir, wiki_url, dry_run)
|
|
|
|
|
if pushed:
|
|
|
|
|
click.echo(_("Wiki synced successfully."))
|
|
|
|
|
elif not dry_run:
|
|
|
|
|
click.echo(_("No push needed (no changes or push failed)."))
|
|
|
|
|
|
|
|
|
|
# Verification
|
|
|
|
|
if verify and not dry_run:
|
|
|
|
|
click.echo(_("\nVerifying wiki pages..."))
|
|
|
|
|
# Re-clone to verify
|
|
|
|
|
verify_dir = Path(tmpdir) / "verify"
|
|
|
|
|
if not clone_wiki(wiki_url, verify_dir):
|
|
|
|
|
click.echo(_("FAIL: Could not clone wiki for verification."))
|
|
|
|
|
raise click.ClickException(_("Wiki verification failed — could not clone wiki"))
|
|
|
|
|
failures = 0
|
|
|
|
|
for page_title, expected_content in sorted(synced.items()):
|
|
|
|
|
ok = verify_wiki_page(client, page_title, expected_content, existing_pages)
|
|
|
|
|
if ok:
|
|
|
|
|
click.echo(_(" OK: {title} ({chars} chars)", title=page_title, chars=len(expected_content)))
|
|
|
|
|
for _file_path, page_title in sorted(mapping.items()):
|
|
|
|
|
wiki_filename = page_title.replace(" ", "-") + ".md"
|
|
|
|
|
if (verify_dir / wiki_filename).exists():
|
|
|
|
|
click.echo(_(" OK: {title}", title=page_title))
|
|
|
|
|
else:
|
|
|
|
|
click.echo(_(" FAIL: {title} — content mismatch or empty!", title=page_title))
|
|
|
|
|
click.echo(_(" FAIL: {title} — page not found in wiki!", title=page_title))
|
|
|
|
|
failures += 1
|
|
|
|
|
if failures > 0:
|
|
|
|
|
click.echo(
|
|
|
|
|
_(
|
|
|
|
|
"\nVerification FAILED: {failures} page(s) have empty or mismatched content!",
|
|
|
|
|
failures=failures,
|
|
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
raise click.ClickException(
|
|
|
|
|
_("Wiki verification failed — {failures} page(s) empty or mismatched", failures=failures)
|
|
|
|
|
_("Wiki verification failed — {failures} page(s) missing", failures=failures)
|
|
|
|
|
)
|
|
|
|
|
click.echo(_("\nVerification passed — all wiki pages have correct content."))
|
|
|
|
|
click.echo(_("\nVerification passed — all wiki pages exist."))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__": # pragma: no cover
|
|
|
|
|
devx-ci-bot