From 146d051cb89ba2eca3653c3c29e49433b9868a34 Mon Sep 17 00:00:00 2001 From: Richard Macias Date: Sat, 28 Feb 2026 10:33:31 -0600 Subject: [PATCH] add scheduled daily test workflow with auto issue management. add --json flag to test-apps.py --- .github/workflows/scheduled-test.yml | 49 +++++++ justfile | 4 + scripts/process-test-results.py | 187 +++++++++++++++++++++++++++ scripts/test-apps.py | 103 +++++++++++---- 4 files changed, 319 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/scheduled-test.yml create mode 100644 scripts/process-test-results.py diff --git a/.github/workflows/scheduled-test.yml b/.github/workflows/scheduled-test.yml new file mode 100644 index 0000000..70b41fe --- /dev/null +++ b/.github/workflows/scheduled-test.yml @@ -0,0 +1,49 @@ +name: Scheduled App Tests + +on: + schedule: + - cron: "0 6 * * *" # Daily at 06:00 UTC + workflow_dispatch: + +permissions: + issues: write + +jobs: + test: + name: Live Test All Apps + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Run tests + id: test + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + python scripts/test-apps.py --json > test-results.json 2>&1 || true + cat test-results.json + + - name: Process results and manage issues + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + python scripts/process-test-results.py \ + test-results.json \ + --run-url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + + - name: Fail if any tests failed + run: | + python -c " + import json, sys + with open('test-results.json') as f: + data = json.load(f) + failed = data.get('summary', {}).get('failed', 0) + if failed > 0: + print(f'{failed} app(s) failed') + sys.exit(1) + print('All apps passed') + " diff --git a/justfile b/justfile index 7615597..6c987be 100644 --- a/justfile +++ b/justfile @@ -33,6 +33,10 @@ normalize: test *args: @python scripts/test-apps.py {{ args }} +# Dry-run the scheduled test workflow (no issues created) +test-cron *args: + @{{ if args == "-h" { "python scripts/test-apps.py -h" } else if args == "--help" { "python scripts/test-apps.py -h" } else { "python scripts/test-apps.py --json " + args + " > /tmp/test-results.json && python scripts/process-test-results.py /tmp/test-results.json --dry-run --run-url local" } }} + # Generate output files generate *args: @{{ if args == "help" { "just _generate-help" } else if args == "-h" { "just _generate-help" } else if args == "--help" { "just _generate-help" } else if args == "table" { "just _generate-table" } else if args == "readme" { "just _generate-readme" } else if args == "standard" { "just _generate-standard" } else if args == "dual-screen" { "just _generate-dual-screen" } else { "just _generate-all" } }} diff --git a/scripts/process-test-results.py b/scripts/process-test-results.py new file mode 100644 index 0000000..4caeba3 --- /dev/null +++ b/scripts/process-test-results.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +"""Process test-apps.py JSON output and manage GitHub issues. + +Creates issues for newly failing apps, closes issues for recovered apps. +Designed to run in GitHub Actions as part of the scheduled test workflow. +""" + +import argparse +import json +import subprocess +import sys +from typing import Any + +from help_formatter import StyledHelpFormatter + +ISSUE_LABEL = "automated-test-failure" +TITLE_PREFIX = "[Automated Test Failure]" + + +def _run_gh(args: list[str]) -> subprocess.CompletedProcess: + return subprocess.run( + ["gh", *args], + capture_output=True, + text=True, + ) + + +def _parse_gh_json(result: subprocess.CompletedProcess) -> list | None: + if result.returncode != 0 or not result.stdout.strip(): + return None + try: + return json.loads(result.stdout) + except json.JSONDecodeError: + return None + + +def _ensure_label_exists() -> None: + result = _run_gh(["label", "list", "--search", ISSUE_LABEL, "--json", "name"]) + labels = _parse_gh_json(result) + if labels is None: + _run_gh([ + "label", "create", ISSUE_LABEL, + "--description", "Automatically created when a scheduled app test fails", + "--color", "d93f0b", + ]) + return + if not any(label["name"] == ISSUE_LABEL for label in labels): + _run_gh([ + "label", "create", ISSUE_LABEL, + "--description", "Automatically created when a scheduled app test fails", + "--color", "d93f0b", + ]) + + +def _find_open_issue(app_name: str) -> int | None: + """Search for an open issue matching this app. Returns issue number or None.""" + search_title = f"{TITLE_PREFIX} {app_name}" + result = _run_gh([ + "issue", "list", + "--label", ISSUE_LABEL, + "--state", "open", + "--search", f"{search_title} in:title", + "--json", "number,title", + ]) + issues = _parse_gh_json(result) + if issues is None: + return None + for issue in issues: + if app_name in issue.get("title", ""): + return issue["number"] + return None + + +def _create_issue(app: dict[str, Any], run_url: str) -> None: + title = f"{TITLE_PREFIX} {app['app_name']}" + body = ( + f"The scheduled test run detected a failure for **{app['app_name']}**.\n\n" + f"| Field | Value |\n" + f"|-------|-------|\n" + f"| App ID | `{app['app_id']}` |\n" + f"| Source | {app['source']} |\n" + f"| URL | {app['url']} |\n" + f"| Error | {app.get('error', 'unknown')} |\n\n" + ) + if app.get("warnings"): + body += "**Warnings:**\n" + for w in app["warnings"]: + body += f"- {w}\n" + body += "\n" + body += f"[Workflow run]({run_url})\n" + + _run_gh([ + "issue", "create", + "--title", title, + "--body", body, + "--label", ISSUE_LABEL, + ]) + print(f" Created issue: {title}") + + +def _close_issue(issue_number: int, app_name: str, run_url: str) -> None: + comment = ( + f"**{app_name}** is passing again in the latest scheduled test run.\n\n" + f"[Workflow run]({run_url})" + ) + _run_gh(["issue", "close", str(issue_number), "--comment", comment]) + print(f" Closed issue #{issue_number}: {app_name} recovered") + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Process test results JSON and manage GitHub issues.", + formatter_class=StyledHelpFormatter, + ) + parser.add_argument( + "results_file", + help="Path to test-results.json from test-apps.py --json", + ) + parser.add_argument( + "--run-url", + default="", + help="URL of the GitHub Actions workflow run", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print planned actions without creating or closing issues", + ) + args = parser.parse_args() + + try: + with open(args.results_file, "r", encoding="utf-8") as f: + data = json.load(f) + except (json.JSONDecodeError, FileNotFoundError) as e: + print(f"Error loading results file: {e}") + return 1 + + if "error" in data: + print(f"Test run error: {data['error']}") + return 1 + + results = data.get("results", []) + if not results: + print("No test results to process.") + return 0 + + dry_run = args.dry_run + + if dry_run: + print("DRY RUN: no issues will be created or closed\n") + else: + _ensure_label_exists() + + failed = [r for r in results if not r["passed"]] + passed = [r for r in results if r["passed"]] + + summary = data.get("summary", {}) + print( + f"Processing {summary.get('total', len(results))} results: " + f"{summary.get('passed', len(passed))} passed, " + f"{summary.get('failed', len(failed))} failed" + ) + + for app in failed: + if dry_run: + print(f" Would create issue: {TITLE_PREFIX} {app['app_name']}") + print(f" Error: {app.get('error', 'unknown')}") + else: + existing = _find_open_issue(app["app_name"]) + if existing: + print(f" Skipped {app['app_name']}: open issue #{existing} already exists") + else: + _create_issue(app, args.run_url) + + for app in passed: + if dry_run: + print(f" Would check/close issue for: {app['app_name']} (passing)") + else: + existing = _find_open_issue(app["app_name"]) + if existing: + _close_issue(existing, app["app_name"], args.run_url) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test-apps.py b/scripts/test-apps.py index 92f03cd..fe18fcf 100644 --- a/scripts/test-apps.py +++ b/scripts/test-apps.py @@ -181,6 +181,22 @@ class TestResult: self.warnings: list[str] = [] self.duration_ms = 0 + def to_dict(self) -> dict[str, Any]: + return { + "app_name": self.app_name, + "app_id": self.app_id, + "source": self.source, + "url": self.url, + "passed": self.passed, + "version": self.version, + "apk_count": self.apk_count, + "apk_urls": self.apk_urls, + "preferred_apk_index": self.preferred_apk_index, + "error": self.error, + "warnings": self.warnings, + "duration_ms": self.duration_ms, + } + def __repr__(self) -> str: status = "PASS" if self.passed else "FAIL" return f"{status}: {self.app_name} ({self.source})" @@ -571,6 +587,15 @@ def print_result( print(f" APK: {url}") +def _print_json_error(message: str) -> None: + output = { + "summary": {"total": 0, "passed": 0, "failed": 0, "warned": 0, "wall_time_ms": 0, "cumulative_time_ms": 0}, + "results": [], + "error": message, + } + print(json.dumps(output, indent=2)) + + def main() -> int: load_dotenv() @@ -609,6 +634,11 @@ def main() -> int: default=8, help="Number of parallel workers (default: 8, use 1 for serial)", ) + parser.add_argument( + "--json", + action="store_true", + help="Output results as JSON (for CI/scripting)", + ) args = parser.parse_args() json_file = args.file @@ -617,12 +647,16 @@ def main() -> int: verbose = args.verbose show_apks = args.apks workers = max(args.jobs, 1) + json_output = args.json try: with open(json_file, "r", encoding="utf-8") as f: data = json.load(f) except (json.JSONDecodeError, FileNotFoundError) as e: - print(f"Error loading {json_file}: {e}") + if json_output: + _print_json_error(f"Error loading {json_file}: {e}") + else: + print(f"Error loading {json_file}: {e}") return 1 apps = data.get("apps", []) @@ -632,21 +666,27 @@ def main() -> int: apps = [a for a in apps if name_filter in a.get("name", "").lower()] if not apps: - print("No apps matched the filter.") + if json_output: + _print_json_error("No apps matched the filter.") + else: + print("No apps matched the filter.") return 1 - has_token = bool(os.environ.get("GITHUB_TOKEN")) - github_count = sum(1 for a in apps if _effective_source(a) == "GitHub") - if github_count > 0 and not has_token: - print( - f"\033[33mNote\033[0m: {github_count} GitHub apps to test, " - "but GITHUB_TOKEN is not set. You may hit rate limits.\n" - " Set it with: export GITHUB_TOKEN=\n" - ) + if not json_output: + has_token = bool(os.environ.get("GITHUB_TOKEN")) + github_count = sum(1 for a in apps if _effective_source(a) == "GitHub") + if github_count > 0 and not has_token: + print( + f"\033[33mNote\033[0m: {github_count} GitHub apps to test, " + "but GITHUB_TOKEN is not set. You may hit rate limits.\n" + " Set it with: export GITHUB_TOKEN=\n" + ) serial = workers == 1 or len(apps) == 1 - mode = "serial" if serial else f"{workers} workers" - print(f"Testing {len(apps)} app(s) ({mode})...\n") + + if not json_output: + mode = "serial" if serial else f"{workers} workers" + print(f"Testing {len(apps)} app(s) ({mode})...\n") wall_start = time.monotonic() @@ -655,7 +695,8 @@ def main() -> int: for app in apps: result = test_app(app) results.append(result) - print_result(result, verbose=verbose, show_apks=show_apks) + if not json_output: + print_result(result, verbose=verbose, show_apks=show_apks) else: result_map: dict[str, TestResult] = {} with ThreadPoolExecutor(max_workers=workers) as pool: @@ -663,10 +704,10 @@ def main() -> int: for future in as_completed(futures): result = future.result() result_map[result.app_id] = result - # Print in original order results = [result_map[app["id"]] for app in apps] - for result in results: - print_result(result, verbose=verbose, show_apks=show_apks) + if not json_output: + for result in results: + print_result(result, verbose=verbose, show_apks=show_apks) wall_ms = int((time.monotonic() - wall_start) * 1000) passed = sum(1 for r in results if r.passed) @@ -674,15 +715,29 @@ def main() -> int: warned = sum(1 for r in results if r.warnings) sum_time = sum(r.duration_ms for r in results) - print(f"\n{'=' * 60}") - print(f"Results: {passed} passed, {failed} failed, {warned} with warnings") - print(f"Time: {wall_ms / 1000:.1f}s wall, {sum_time / 1000:.1f}s cumulative") + if json_output: + output = { + "summary": { + "total": len(results), + "passed": passed, + "failed": failed, + "warned": warned, + "wall_time_ms": wall_ms, + "cumulative_time_ms": sum_time, + }, + "results": [r.to_dict() for r in results], + } + print(json.dumps(output, indent=2)) + else: + print(f"\n{'=' * 60}") + print(f"Results: {passed} passed, {failed} failed, {warned} with warnings") + print(f"Time: {wall_ms / 1000:.1f}s wall, {sum_time / 1000:.1f}s cumulative") - if failed > 0: - print(f"\nFailed apps:") - for r in results: - if not r.passed: - print(f" - {r.app_name}: {r.error}") + if failed > 0: + print(f"\nFailed apps:") + for r in results: + if not r.passed: + print(f" - {r.app_name}: {r.error}") return 1 if failed > 0 else 0