add justfile as primary task runner, slim Makefile to CI-only. add styled argparse help formatter, parallel test execution (-j8 default), argparse for test-apps.py. release.py accepts --dryrun alias

This commit is contained in:
Richard Macias
2026-02-27 18:13:17 -06:00
parent cd20e372dc
commit 1b003007ea
6 changed files with 287 additions and 102 deletions

View File

@@ -1,71 +1,39 @@
.PHONY: help readme validate add-app normalize build publish publish-dry-run publish-from-file test test-app test-apks test-verbose # These targets exist for CI (GitHub Actions) only.
default: help # For local development, use the justfile: run `just` to see all commands.
help: # Show help for each of the makefile recipes. .PHONY: help validate test build normalize table readme minify minify-dual-screen
@width=$$(grep -E '^[a-zA-Z0-9 -]+:.*#' Makefile | cut -f1 -d':' | awk '{print length}' | sort -rn | head -1); \ .DEFAULT_GOAL := help
grep -E '^[a-zA-Z0-9 -]+:.*#' Makefile | sort | while read -r l; \
do printf "\033[1;32m%-$${width}s\033[00m %s\n" "$$(echo $$l | cut -f 1 -d':')" "$$(echo $$l | cut -f 2- -d'#')"; done
# --------------------------------------------------------------------------- help: ## Show available targets
# Apps @echo "CI-only build targets. For local development, run 'just' to see all commands."
# --------------------------------------------------------------------------- @echo ""
@grep -E '^[a-z].*:.*##' Makefile | sed 's/:.*## /\t/' | while IFS=$$'\t' read -r target desc; do \
printf "\033[1;32m%-20s\033[0m %s\n" "$$target" "$$desc"; \
done
add-app: # Interactive CLI to add a new app validate: ## Validate applications.json for errors
@python scripts/add-app.py
validate: # Validate applications.json for errors (structure, regex syntax, source types)
@python scripts/validate-json.py src/applications.json @python scripts/validate-json.py src/applications.json
normalize: # Normalize key order and add missing defaults in applications.json test: ## Live-test all app configs resolve to downloadable APKs
@python scripts/test-apps.py
build: validate normalize readme minify minify-dual-screen ## Build all artifacts
normalize: ## Normalize key order and add missing defaults
@python scripts/normalize-json.py src/applications.json @python scripts/normalize-json.py src/applications.json
test: # Live-test that all app configs can resolve to downloadable APKs table: ## Generate markdown table for the README
@python scripts/test-apps.py src/applications.json
test-app: # Live-test a single app by name (e.g. make test-app APP=Dolphin)
@python scripts/test-apps.py src/applications.json --verbose --apks $(APP)
test-apks: # Live-test all apps and show numbered APK list for index selection
@python scripts/test-apps.py src/applications.json --apks
test-verbose: # Live-test with APK URL details shown
@python scripts/test-apps.py src/applications.json --verbose
# ---------------------------------------------------------------------------
# Build
# ---------------------------------------------------------------------------
build: validate normalize readme minify minify-dual-screen # Build all artifacts: validate, normalize, readme, and both release JSONs.
minify: # Generate standard release JSON
@python scripts/minify-json.py src/applications.json obtainium-emulation-pack-latest.json --variant standard
minify-dual-screen: # Generate dual screen release JSON
@python scripts/minify-json.py src/applications.json obtainium-emulation-pack-dual-screen-latest.json --variant dual-screen
# ---------------------------------------------------------------------------
# Docs
# ---------------------------------------------------------------------------
table: # Generate a table of obtainium links for the README.
@python scripts/generate-table.py src/applications.json ./pages/table.md @python scripts/generate-table.py src/applications.json ./pages/table.md
readme: table # Generate the readme file. Why? Because editing that table every change is tedious. readme: table ## Generate the README from page sections
@python scripts/generate-readme.py \ @python scripts/generate-readme.py \
./pages/header.md \ ./pages/header.md \
./pages/table.md \ ./pages/table.md \
./pages/faq.md \ ./pages/faq.md \
./pages/footer.md ./pages/footer.md
# --------------------------------------------------------------------------- minify: ## Generate standard release JSON
# Publish @python scripts/minify-json.py src/applications.json obtainium-emulation-pack-latest.json --variant standard
# ---------------------------------------------------------------------------
publish: # Tag, push, and create a GitHub release (opens $EDITOR for notes) minify-dual-screen: ## Generate dual-screen release JSON
@python scripts/release.py @python scripts/minify-json.py src/applications.json obtainium-emulation-pack-dual-screen-latest.json --variant dual-screen
publish-dry-run: # Preview release notes as a markdown file without publishing
@python scripts/release.py --dry-run
publish-from-file: # Publish using a previously edited release notes file (e.g. from publish-dry)
@python scripts/release.py --notes-file $(FILE)

38
justfile Normal file
View File

@@ -0,0 +1,38 @@
import 'utility.just'
[private]
default:
@echo '{{YELLOW}}Tip:{{NORMAL}} Recipes with {{BOLD}}*args{{NORMAL}} accept {{BOLD}}-h{{NORMAL}} for help.'
@just --list
# Interactive CLI to add a new app
[group('CLI Tools')]
add-app:
@python scripts/add-app.py
# Test, validate, normalize, and generate all output files
[group('Release')]
build: test validate normalize generate
# Tag, push, and create a GitHub release
[group('Release')]
release *args:
@python scripts/release.py {{ args }}
# Validate applications.json for errors (structure, regex syntax, source types)
[group('Formatting')]
validate:
@python scripts/validate-json.py src/applications.json
# Normalize key order and add missing defaults in applications.json
[group('Formatting')]
normalize:
@python scripts/normalize-json.py src/applications.json
# Live-test app configs
test *args:
@python scripts/test-apps.py {{ args }}
# Generate output files
generate *args:
@{{ if args == "help" { "just _generate-help" } else if args == "-h" { "just _generate-help" } else if args == "--help" { "just _generate-help" } else if args == "table" { "just _generate-table" } else if args == "readme" { "just _generate-readme" } else if args == "standard" { "just _generate-standard" } else if args == "dual-screen" { "just _generate-dual-screen" } else { "just _generate-all" } }}

114
scripts/help_formatter.py Normal file
View File

@@ -0,0 +1,114 @@
"""Styled argparse help formatter with ANSI colors."""
import argparse
import re
import sys
BOLD = "\033[1m"
DIM = "\033[2m"
YELLOW = "\033[33m"
CYAN = "\033[36m"
GREEN = "\033[32m"
RESET = "\033[0m"
ANSI_ESCAPE = re.compile(r"\033\[[0-9;]*m")
def _supports_color() -> bool:
return hasattr(sys.stdout, "isatty") and sys.stdout.isatty()
def _visible_len(s: str) -> int:
return len(ANSI_ESCAPE.sub("", s))
class StyledHelpFormatter(argparse.HelpFormatter):
def __init__(self, prog: str, **kwargs) -> None:
kwargs.setdefault("max_help_position", 36)
super().__init__(prog, **kwargs)
self._color = _supports_color()
def _format_usage(self, usage, actions, groups, prefix):
if prefix is None:
prefix = "usage: "
if self._color:
prefix = f"{YELLOW}{prefix}{RESET}"
return super()._format_usage(usage, actions, groups, prefix)
def start_section(self, heading):
if self._color and heading:
heading = f"{BOLD}{heading}{RESET}"
super().start_section(heading)
def _format_action_invocation(self, action):
if not action.option_strings:
# Positional arg: just the metavar
result = self._metavar_formatter(action, action.dest)(1)[0]
if self._color:
result = f"{CYAN}{result}{RESET}"
return result
# Sort: short flags (-v) before long flags (--version)
short = [s for s in action.option_strings if not s.startswith("--")]
long = [s for s in action.option_strings if s.startswith("--")]
parts = short + long
# Append metavar once at the end (not after each flag)
if action.nargs != 0:
metavar = self._metavar_formatter(action, action.dest.upper())(1)[0]
result = ", ".join(parts) + " " + metavar
else:
result = ", ".join(parts)
# Pad with leading spaces when no short flag, to align with those that have one
if not short:
result = " " + result
if self._color:
result = f"{GREEN}{result}{RESET}"
return result
def _format_action(self, action):
help_position = min(self._action_max_length + 2,
self._max_help_position)
help_width = max(self._width - help_position, 11)
action_width = help_position - self._current_indent - 2
action_header = self._format_action_invocation(action)
# Use visible length (ignoring ANSI codes) for layout decisions
visible = _visible_len(action_header) if self._color else len(action_header)
indent_first = 0
if not action.help:
tup = self._current_indent, '', action_header
action_header = '%*s%s\n' % tup
elif visible <= action_width:
# Pad based on visible width so columns align despite ANSI codes
ansi_pad = len(action_header) - visible
tup = self._current_indent, '', action_width + ansi_pad, action_header
action_header = '%*s%-*s ' % tup
indent_first = 0
else:
tup = self._current_indent, '', action_header
action_header = '%*s%s\n' % tup
indent_first = help_position
parts = [action_header]
if action.help and action.help.strip():
help_text = self._expand_help(action)
if help_text:
if self._color:
help_text = f"{DIM}{help_text}{RESET}"
help_lines = self._split_lines(help_text, help_width)
parts.append('%*s%s\n' % (indent_first, '', help_lines[0]))
for line in help_lines[1:]:
parts.append('%*s%s\n' % (help_position, '', line))
elif not action_header.endswith('\n'):
parts.append('\n')
for subaction in self._iter_indented_subactions(action):
parts.append(self._format_action(subaction))
return self._join_parts(parts)

View File

@@ -31,6 +31,7 @@ from pathlib import Path
from typing import Any from typing import Any
from constants import GITHUB_NOREPLY_SUFFIX from constants import GITHUB_NOREPLY_SUFFIX
from help_formatter import StyledHelpFormatter
from utils import get_application_url, get_display_name, load_dotenv, make_obtainium_link, should_include_app from utils import get_application_url, get_display_name, load_dotenv, make_obtainium_link, should_include_app
REPO_ROOT = Path(__file__).resolve().parent.parent REPO_ROOT = Path(__file__).resolve().parent.parent
@@ -434,7 +435,8 @@ def get_app_count(json_path: Path) -> int:
def main() -> None: def main() -> None:
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Create a GitHub release for Obtainium Emulation Pack" description="Create a GitHub release for Obtainium Emulation Pack",
formatter_class=StyledHelpFormatter,
) )
parser.add_argument( parser.add_argument(
"--version", "-v", "--version", "-v",
@@ -449,7 +451,7 @@ def main() -> None:
help="Path to a file containing release notes. Skips generation and editor.", help="Path to a file containing release notes. Skips generation and editor.",
) )
parser.add_argument( parser.add_argument(
"--dry-run", "--dry-run", "--dryrun",
action="store_true", action="store_true",
help="Show what would happen without making changes.", help="Show what would happen without making changes.",
) )

View File

@@ -1,25 +1,23 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
"""Live validation that app configs can resolve to downloadable APKs. """Live validation that app configs can resolve to downloadable APKs.
Usage:
python scripts/test-apps.py src/applications.json
python scripts/test-apps.py src/applications.json Dolphin
python scripts/test-apps.py src/applications.json --id org.dolphinemu.dolphinemu
Set GITHUB_TOKEN in .env or environment to avoid API rate limits. Set GITHUB_TOKEN in .env or environment to avoid API rate limits.
""" """
import argparse
import json import json
import os import os
import re import re
import ssl import ssl
import sys import sys
import time import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from html.parser import HTMLParser from html.parser import HTMLParser
from typing import Any from typing import Any
from urllib.parse import urljoin, urlparse from urllib.parse import urljoin, urlparse
from urllib.request import Request, urlopen from urllib.request import Request, urlopen
from help_formatter import StyledHelpFormatter
from utils import load_dotenv from utils import load_dotenv
USER_AGENT = ( USER_AGENT = (
@@ -560,39 +558,49 @@ def print_result(
def main() -> int: def main() -> int:
load_dotenv() load_dotenv()
if len(sys.argv) < 2: parser = argparse.ArgumentParser(
print("Usage: python test-apps.py <json_file> [name_filter] [--id <app_id>] [--verbose] [--apks]") description="Live-test app configs resolve to downloadable APKs.",
print() formatter_class=StyledHelpFormatter,
print("Examples:") )
print(" python test-apps.py src/applications.json # test all apps") parser.add_argument(
print(" python test-apps.py src/applications.json Dolphin # filter by name") "name",
print(" python test-apps.py src/applications.json --id org.dolphinemu.dolphinemu") nargs="?",
print(" python test-apps.py src/applications.json --verbose # show APK URLs") help="Filter by app name (case-insensitive substring match)",
print(" python test-apps.py src/applications.json --apks # show numbered APK list") )
return 1 parser.add_argument(
"-f", "--file",
default="src/applications.json",
help="Path to applications.json (default: src/applications.json)",
)
parser.add_argument(
"--id",
dest="id_filter",
help="Filter by exact app ID",
)
parser.add_argument(
"--verbose",
action="store_true",
help="Show APK download URLs",
)
parser.add_argument(
"--apks",
action="store_true",
help="Show numbered APK list with preferredApkIndex marker",
)
parser.add_argument(
"-j", "--jobs",
type=int,
default=8,
help="Number of parallel workers (default: 8, use 1 for serial)",
)
args = parser.parse_args()
json_file = sys.argv[1] json_file = args.file
args = sys.argv[2:] name_filter = args.name.lower() if args.name else None
id_filter = args.id_filter
verbose = "--verbose" in args verbose = args.verbose
if verbose: show_apks = args.apks
args.remove("--verbose") workers = max(args.jobs, 1)
show_apks = "--apks" in args
if show_apks:
args.remove("--apks")
id_filter = None
if "--id" in args:
idx = args.index("--id")
if idx + 1 < len(args):
id_filter = args[idx + 1]
args = args[:idx] + args[idx + 2:]
else:
print("Error: --id requires an argument")
return 1
name_filter = " ".join(args).lower() if args else None
try: try:
with open(json_file, "r", encoding="utf-8") as f: with open(json_file, "r", encoding="utf-8") as f:
@@ -620,22 +628,39 @@ def main() -> int:
" Set it with: export GITHUB_TOKEN=<your_token>\n" " Set it with: export GITHUB_TOKEN=<your_token>\n"
) )
print(f"Testing {len(apps)} app(s)...\n") serial = workers == 1 or len(apps) == 1
mode = "serial" if serial else f"{workers} workers"
print(f"Testing {len(apps)} app(s) ({mode})...\n")
wall_start = time.monotonic()
if serial:
results = [] results = []
for app in apps: for app in apps:
result = test_app(app) result = test_app(app)
results.append(result) results.append(result)
print_result(result, verbose=verbose, show_apks=show_apks) print_result(result, verbose=verbose, show_apks=show_apks)
else:
result_map: dict[str, TestResult] = {}
with ThreadPoolExecutor(max_workers=workers) as pool:
futures = {pool.submit(test_app, app): app for app in apps}
for future in as_completed(futures):
result = future.result()
result_map[result.app_id] = result
# Print in original order
results = [result_map[app["id"]] for app in apps]
for result in results:
print_result(result, verbose=verbose, show_apks=show_apks)
wall_ms = int((time.monotonic() - wall_start) * 1000)
passed = sum(1 for r in results if r.passed) passed = sum(1 for r in results if r.passed)
failed = sum(1 for r in results if not r.passed) failed = sum(1 for r in results if not r.passed)
warned = sum(1 for r in results if r.warnings) warned = sum(1 for r in results if r.warnings)
total_time = sum(r.duration_ms for r in results) sum_time = sum(r.duration_ms for r in results)
print(f"\n{'=' * 60}") print(f"\n{'=' * 60}")
print(f"Results: {passed} passed, {failed} failed, {warned} with warnings") print(f"Results: {passed} passed, {failed} failed, {warned} with warnings")
print(f"Time: {total_time / 1000:.1f}s total") print(f"Time: {wall_ms / 1000:.1f}s wall, {sum_time / 1000:.1f}s cumulative")
if failed > 0: if failed > 0:
print(f"\nFailed apps:") print(f"\nFailed apps:")

38
utility.just Normal file
View File

@@ -0,0 +1,38 @@
DIM := "\u{1B}[2m"
[private]
_generate-help:
@echo '{{YELLOW}}usage:{{NORMAL}} just generate [subcommand]'
@echo ''
@echo 'Generate release artifacts and documentation from applications.json.'
@echo ''
@echo '{{BOLD}}subcommands:{{NORMAL}}'
@echo ' {{GREEN}}just generate{{NORMAL}} {{DIM}}Generate all output files{{NORMAL}}'
@echo ' {{GREEN}}just generate help{{NORMAL}} {{DIM}}Show this help{{NORMAL}}'
@echo ' {{GREEN}}just generate table{{NORMAL}} {{DIM}}Generate markdown table{{NORMAL}}'
@echo ' {{GREEN}}just generate readme{{NORMAL}} {{DIM}}Generate README (includes table){{NORMAL}}'
@echo ' {{GREEN}}just generate standard{{NORMAL}} {{DIM}}Generate standard release JSON{{NORMAL}}'
@echo ' {{GREEN}}just generate dual-screen{{NORMAL}} {{DIM}}Generate dual-screen release JSON{{NORMAL}}'
[private]
_generate-all: _generate-readme _generate-standard _generate-dual-screen
[private]
_generate-table:
@python scripts/generate-table.py src/applications.json ./pages/table.md
[private]
_generate-readme: _generate-table
@python scripts/generate-readme.py \
./pages/header.md \
./pages/table.md \
./pages/faq.md \
./pages/footer.md
[private]
_generate-standard:
@python scripts/minify-json.py src/applications.json obtainium-emulation-pack-latest.json --variant standard
[private]
_generate-dual-screen:
@python scripts/minify-json.py src/applications.json obtainium-emulation-pack-dual-screen-latest.json --variant dual-screen