hoborg/Obtainium-Emulation-Pack
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

refactor: single SETTINGS_SCHEMA in constants.py, sparse additionalSettings in source file, hydrate on export

Browse Source
This commit is contained in:
Richard Macias
2026-02-28 00:21:47 -06:00
parent 5a0ff4f4de
commit 2194787230
12 changed files with 322 additions and 1613 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
scripts/add-app.py
View File

@@ -9,35 +9,6 @@ from urllib.parse import urlparse
from utils import load_dotenv
# Default Obtainium settings for GitHub apps
DEFAULT_ADDITIONAL_SETTINGS = {
"includePrereleases": False,
"fallbackToOlderReleases": True,
"filterReleaseTitlesByRegEx": "",
"filterReleaseNotesByRegEx": "",
"verifyLatestTag": False,
"sortMethodChoice": "date",
"useLatestAssetDateAsReleaseDate": False,
"releaseTitleAsVersion": False,
"trackOnly": False,
"versionExtractionRegEx": "",
"matchGroupToUse": "",
"versionDetection": True,
"releaseDateAsVersion": False,
"useVersionCodeAsOSVersion": False,
"apkFilterRegEx": "",
"invertAPKFilter": False,
"autoApkFilterByArch": True,
"appName": "",
"appAuthor": "",
"shizukuPretendToBeGooglePlay": False,
"allowInsecure": False,
"exemptFromBackgroundUpdates": False,
"skipUpdateNotifications": False,
"about": "",
"refreshBeforeDownload": False,
}
CATEGORIES = [
"Emulator",
"Frontend",
@@ -177,7 +148,7 @@ def generate_app_entry(
app_name_override: str | None = None,
url_override: str | None = None,
) -> dict:
settings = DEFAULT_ADDITIONAL_SETTINGS.copy()
settings: dict[str, object] = {}
if "Track Only" in categories:
settings["trackOnly"] = True
if include_prereleases:
@@ -357,7 +328,7 @@ def main() -> int:
print()
print("Next steps:")
print(" 1. Run 'make build' to regenerate all files")
print(" 1. Run 'just build' to regenerate all files")
print(" 2. Review the diff before committing")
return 0

225
scripts/constants.py
View File

@@ -1,5 +1,7 @@
"""Shared constants for Obtainium Emulation Pack scripts."""
from typing import Any, NamedTuple
SRC_FILE = "src/applications.json"
PAGES_DIR = "pages"
TABLE_FILE = "pages/table.md"
@@ -11,12 +13,6 @@ VARIANTS = ("standard", "dual-screen")
GITHUB_NOREPLY_SUFFIX = "@users.noreply.github.com"
# ---------------------------------------------------------------------------
# Obtainium source types and settings schema
# Derived from Obtainium source code: lib/app_sources/*.dart
# Reference: ~/code/Obtainium
# ---------------------------------------------------------------------------
# Valid overrideSource values (runtime type names from Obtainium)
VALID_SOURCES = {
"GitHub",
@@ -71,107 +67,6 @@ SOURCE_HOST_MAP = {
"sourceforge.net": "SourceForge",
}
# Common additionalSettings keys valid for all source types
COMMON_SETTINGS_KEYS = {
"trackOnly",
"versionExtractionRegEx",
"matchGroupToUse",
"versionDetection",
"useVersionCodeAsOSVersion",
"apkFilterRegEx",
"invertAPKFilter",
"autoApkFilterByArch",
"appName",
"appAuthor",
"shizukuPretendToBeGooglePlay",
"allowInsecure",
"exemptFromBackgroundUpdates",
"skipUpdateNotifications",
"about",
"refreshBeforeDownload",
"releaseDateAsVersion",
"includeZips",
"zippedApkFilterRegEx",
}
# Source-specific additionalSettings keys
SOURCE_SPECIFIC_KEYS: dict[str, set[str]] = {
"GitHub": {
"includePrereleases",
"fallbackToOlderReleases",
"filterReleaseTitlesByRegEx",
"filterReleaseNotesByRegEx",
"verifyLatestTag",
"sortMethodChoice",
"useLatestAssetDateAsReleaseDate",
"releaseTitleAsVersion",
"github-creds",
"GHReqPrefix",
},
"GitLab": {
"fallbackToOlderReleases",
"gitlab-creds",
},
"Codeberg": {
# Inherits GitHub's settings
"includePrereleases",
"fallbackToOlderReleases",
"filterReleaseTitlesByRegEx",
"filterReleaseNotesByRegEx",
"verifyLatestTag",
"sortMethodChoice",
"useLatestAssetDateAsReleaseDate",
"releaseTitleAsVersion",
},
"FDroid": {
"filterVersionsByRegEx",
"trySelectingSuggestedVersionCode",
"autoSelectHighestVersionCode",
},
"IzzyOnDroid": {
# Inherits FDroid's settings
"filterVersionsByRegEx",
"trySelectingSuggestedVersionCode",
"autoSelectHighestVersionCode",
},
"FDroidRepo": {
"appIdOrName",
"pickHighestVersionCode",
"trySelectingSuggestedVersionCode",
},
"SourceHut": {
"fallbackToOlderReleases",
},
"APKPure": {
"fallbackToOlderReleases",
"stayOneVersionBehind",
"useFirstApkOfVersion",
},
"APKMirror": {
"fallbackToOlderReleases",
"filterReleaseTitlesByRegEx",
},
"Farsroid": {
"useFirstApkOfVersion",
},
"HTML": {
"intermediateLink",
"customLinkFilterRegex",
"filterByLinkText",
"matchLinksOutsideATags",
"skipSort",
"reverseSort",
"sortByLastLinkSegment",
"versionExtractWholePage",
"requestHeader",
"defaultPseudoVersioningMethod",
},
"DirectAPKLink": {
"requestHeader",
"defaultPseudoVersioningMethod",
},
}
# Deprecated keys still accepted for backward compatibility.
# Obtainium auto-migrates these on load (see appJSONCompatibilityModifiers
# in lib/providers/source_provider.dart). New configs should use the
@@ -182,13 +77,111 @@ DEPRECATED_SETTINGS_KEYS: dict[str, str] = {
"sortByFileNamesNotLinks": "sortByLastLinkSegment",
}
# Settings keys that contain regex patterns (should be validated)
REGEX_SETTINGS_KEYS = {
"apkFilterRegEx",
"versionExtractionRegEx",
"filterReleaseTitlesByRegEx",
"filterReleaseNotesByRegEx",
"customLinkFilterRegex",
"filterVersionsByRegEx",
"zippedApkFilterRegEx",
# ---------------------------------------------------------------------------
# Obtainium additionalSettings schema
# Single source of truth for key metadata: default value, applicable sources,
# whether the value is a regex pattern. Derived from Obtainium source code:
# lib/app_sources/*.dart. Reference: ~/code/Obtainium
#
# Dict insertion order defines the canonical key ordering used by
# normalize-json.py and export hydration.
# ---------------------------------------------------------------------------
ALL_SOURCES = frozenset(VALID_SOURCES)
_GITHUB_LIKE = frozenset({"GitHub", "Codeberg"})
_DEFAULT_USER_AGENT_HEADER = [
{"requestHeader": "User-Agent: Mozilla/5.0 (Linux; Android 10; K) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/114.0.0.0 Mobile Safari/537.36"}
]
class SettingDef(NamedTuple):
default: Any
sources: frozenset[str]
is_regex: bool = False
SETTINGS_SCHEMA: dict[str, SettingDef] = {
# --- GitHub / Codeberg source-specific ---
"includePrereleases": SettingDef(False, _GITHUB_LIKE),
"fallbackToOlderReleases": SettingDef(True, frozenset({"GitHub", "Codeberg", "GitLab", "SourceHut", "APKPure", "APKMirror"})),
"filterReleaseTitlesByRegEx": SettingDef("", frozenset({"GitHub", "Codeberg", "APKMirror"}), is_regex=True),
"filterReleaseNotesByRegEx": SettingDef("", _GITHUB_LIKE, is_regex=True),
"verifyLatestTag": SettingDef(False, _GITHUB_LIKE),
"sortMethodChoice": SettingDef("date", _GITHUB_LIKE),
"useLatestAssetDateAsReleaseDate": SettingDef(False, _GITHUB_LIKE),
"releaseTitleAsVersion": SettingDef(False, _GITHUB_LIKE),
"github-creds": SettingDef("", frozenset({"GitHub"})),
"GHReqPrefix": SettingDef("", frozenset({"GitHub"})),
# --- GitLab source-specific ---
"gitlab-creds": SettingDef("", frozenset({"GitLab"})),
# --- FDroid / IzzyOnDroid source-specific ---
"filterVersionsByRegEx": SettingDef("", frozenset({"FDroid", "IzzyOnDroid"}), is_regex=True),
"trySelectingSuggestedVersionCode": SettingDef(True, frozenset({"FDroid", "IzzyOnDroid", "FDroidRepo"})),
"autoSelectHighestVersionCode": SettingDef(False, frozenset({"FDroid", "IzzyOnDroid"})),
# --- FDroidRepo source-specific ---
"appIdOrName": SettingDef("", frozenset({"FDroidRepo"})),
"pickHighestVersionCode": SettingDef(False, frozenset({"FDroidRepo"})),
# --- APKPure source-specific ---
"stayOneVersionBehind": SettingDef(False, frozenset({"APKPure"})),
"useFirstApkOfVersion": SettingDef(True, frozenset({"APKPure", "Farsroid"})),
# --- HTML source-specific ---
"intermediateLink": SettingDef([], frozenset({"HTML"})),
"customLinkFilterRegex": SettingDef("", frozenset({"HTML"}), is_regex=True),
"filterByLinkText": SettingDef(False, frozenset({"HTML"})),
"matchLinksOutsideATags": SettingDef(False, frozenset({"HTML"})),
"skipSort": SettingDef(False, frozenset({"HTML"})),
"reverseSort": SettingDef(False, frozenset({"HTML"})),
"sortByLastLinkSegment": SettingDef(False, frozenset({"HTML"})),
"versionExtractWholePage": SettingDef(False, frozenset({"HTML"})),
"requestHeader": SettingDef(_DEFAULT_USER_AGENT_HEADER, frozenset({"HTML", "DirectAPKLink"})),
"defaultPseudoVersioningMethod": SettingDef("partialAPKHash", frozenset({"HTML", "DirectAPKLink"})),
# --- Common keys (all sources) ---
"trackOnly": SettingDef(False, ALL_SOURCES),
"versionExtractionRegEx": SettingDef("", ALL_SOURCES, is_regex=True),
"matchGroupToUse": SettingDef("", ALL_SOURCES),
"versionDetection": SettingDef(True, ALL_SOURCES),
"releaseDateAsVersion": SettingDef(False, ALL_SOURCES),
"useVersionCodeAsOSVersion": SettingDef(False, ALL_SOURCES),
"apkFilterRegEx": SettingDef("", ALL_SOURCES, is_regex=True),
"invertAPKFilter": SettingDef(False, ALL_SOURCES),
"autoApkFilterByArch": SettingDef(True, ALL_SOURCES),
"appName": SettingDef("", ALL_SOURCES),
"appAuthor": SettingDef("", ALL_SOURCES),
"shizukuPretendToBeGooglePlay": SettingDef(False, ALL_SOURCES),
"allowInsecure": SettingDef(False, ALL_SOURCES),
"exemptFromBackgroundUpdates": SettingDef(False, ALL_SOURCES),
"skipUpdateNotifications": SettingDef(False, ALL_SOURCES),
"about": SettingDef("", ALL_SOURCES),
"refreshBeforeDownload": SettingDef(False, ALL_SOURCES),
"includeZips": SettingDef(False, ALL_SOURCES),
"zippedApkFilterRegEx": SettingDef("", ALL_SOURCES, is_regex=True),
}
# ---------------------------------------------------------------------------
# Derived views - computed from SETTINGS_SCHEMA so there's one place to update
# ---------------------------------------------------------------------------
COMMON_SETTINGS_KEYS: set[str] = {
key for key, s in SETTINGS_SCHEMA.items() if s.sources == ALL_SOURCES
}
SOURCE_SPECIFIC_KEYS: dict[str, set[str]] = {}
for _source in VALID_SOURCES:
_keys = {key for key, s in SETTINGS_SCHEMA.items() if _source in s.sources and s.sources != ALL_SOURCES}
if _keys:
SOURCE_SPECIFIC_KEYS[_source] = _keys
REGEX_SETTINGS_KEYS: set[str] = {
key for key, s in SETTINGS_SCHEMA.items() if s.is_regex
}

6
scripts/minify-json.py
View File

@@ -5,7 +5,7 @@ import json
import sys
from typing import Any
from utils import should_include_app, stringify_additional_settings
from utils import get_additional_settings, should_include_app, stringify_additional_settings
def minify_json(input_file: str, output_file: str, variant: str = "standard") -> None:
@@ -19,7 +19,9 @@ def minify_json(input_file: str, output_file: str, variant: str = "standard") ->
if should_include_app(app, variant):
app_copy = app.copy()
app_copy.pop("meta", None)
app_copy["additionalSettings"] = stringify_additional_settings(app_copy)
settings = get_additional_settings(app_copy)
source = app_copy.get("overrideSource")
app_copy["additionalSettings"] = stringify_additional_settings(settings, source)
filtered_apps.append(app_copy)
data["apps"] = filtered_apps

58
scripts/normalize-json.py
View File

@@ -5,9 +5,9 @@ import sys
from pathlib import Path
from typing import Any
from constants import SRC_FILE
from constants import SETTINGS_SCHEMA, SRC_FILE
# Canonical key order - matches the output of generate_app_entry() in add-app.py
# Canonical top-level key order for each app entry
KEY_ORDER = [
"id",
"url",
@@ -21,59 +21,14 @@ KEY_ORDER = [
"meta",
]
# Canonical key order for additionalSettings - source-specific keys first,
# then common keys, grouped logically. Matches DEFAULT_ADDITIONAL_SETTINGS
# in add-app.py with source-specific keys prepended.
SETTINGS_KEY_ORDER = [
# GitHub/Codeberg source-specific
"includePrereleases",
"fallbackToOlderReleases",
"filterReleaseTitlesByRegEx",
"filterReleaseNotesByRegEx",
"verifyLatestTag",
"sortMethodChoice",
"useLatestAssetDateAsReleaseDate",
"releaseTitleAsVersion",
"github-creds",
"GHReqPrefix",
# HTML source-specific
"intermediateLink",
"customLinkFilterRegex",
"filterByLinkText",
"matchLinksOutsideATags",
"skipSort",
"reverseSort",
"sortByLastLinkSegment",
"versionExtractWholePage",
"requestHeader",
"defaultPseudoVersioningMethod",
# Common keys
"trackOnly",
"versionExtractionRegEx",
"matchGroupToUse",
"versionDetection",
"releaseDateAsVersion",
"useVersionCodeAsOSVersion",
"apkFilterRegEx",
"invertAPKFilter",
"autoApkFilterByArch",
"appName",
"appAuthor",
"shizukuPretendToBeGooglePlay",
"allowInsecure",
"exemptFromBackgroundUpdates",
"skipUpdateNotifications",
"about",
"refreshBeforeDownload",
"includeZips",
"zippedApkFilterRegEx",
]
# Fields to backfill with defaults when missing
DEFAULTS: dict[str, object] = {
"allowIdChange": False,
}
# Settings key order derived from SETTINGS_SCHEMA insertion order
_SETTINGS_KEY_ORDER = list(SETTINGS_SCHEMA.keys())
def _order_dict(d: dict[str, Any], key_order: list[str]) -> dict[str, Any]:
ordered: dict[str, Any] = {}
@@ -92,10 +47,9 @@ def normalize_app(app: dict) -> dict:
if key not in app:
app[key] = default
# Normalize additionalSettings key order if it's a dict
settings = app.get("additionalSettings")
if isinstance(settings, dict):
app["additionalSettings"] = _order_dict(settings, SETTINGS_KEY_ORDER)
app["additionalSettings"] = _order_dict(settings, _SETTINGS_KEY_ORDER)
return _order_dict(app, KEY_ORDER)

11
scripts/release.py
View File

@@ -32,7 +32,7 @@ from typing import Any
from constants import GITHUB_NOREPLY_SUFFIX
from help_formatter import StyledHelpFormatter
from utils import get_application_url, get_display_name, load_dotenv, make_obtainium_link, should_include_app
from utils import get_additional_settings, get_application_url, get_display_name, load_dotenv, make_obtainium_link, should_include_app
REPO_ROOT = Path(__file__).resolve().parent.parent
@@ -171,14 +171,7 @@ def load_apps_from_file() -> dict[str, dict[str, Any]]:
def normalize_app_for_comparison(app: dict[str, Any]) -> dict[str, Any]:
"""Parse additionalSettings and strip meta so formatting-only changes are ignored."""
normalized = {k: v for k, v in app.items() if k != "meta"}
settings = normalized.get("additionalSettings")
if isinstance(settings, str):
try:
normalized["additionalSettings"] = json.loads(settings)
except json.JSONDecodeError:
pass
normalized["additionalSettings"] = get_additional_settings(normalized)
return normalized

5
scripts/test-apps.py
View File

@@ -18,7 +18,7 @@ from urllib.parse import urljoin, urlparse
from urllib.request import Request, urlopen
from help_formatter import StyledHelpFormatter
from utils import get_additional_settings, load_dotenv
from utils import get_additional_settings, hydrate_settings, load_dotenv
USER_AGENT = (
"Mozilla/5.0 (Linux; Android 10; K) "
@@ -496,7 +496,8 @@ def test_app(app: dict[str, Any]) -> TestResult:
source = _effective_source(app)
try:
settings = get_additional_settings(app)
sparse = get_additional_settings(app)
settings = hydrate_settings(sparse, source)
except (json.JSONDecodeError, TypeError):
result = TestResult(app.get("name", "?"), app.get("id", "?"), source, app.get("url", "?"))
result.error = "Cannot parse additionalSettings"

39
scripts/utils.py
View File

@@ -1,12 +1,13 @@
"""Shared utility functions for Obtainium Emulation Pack scripts."""
import copy
import json
import os
import urllib.parse
from pathlib import Path
from typing import Any
from constants import OBTAINIUM_SCHEME, REDIRECT_URL
from constants import OBTAINIUM_SCHEME, REDIRECT_URL, SETTINGS_SCHEMA
def load_dotenv() -> None:
@@ -56,15 +57,37 @@ def get_additional_settings(app: dict[str, Any]) -> dict[str, Any]:
return {}
def stringify_additional_settings(app: dict[str, Any]) -> str:
"""Return additionalSettings as a compact JSON string for Obtainium consumption."""
raw = app.get("additionalSettings", {})
if isinstance(raw, str):
return raw
return json.dumps(raw, separators=(",", ":"))
def get_defaults_for_source(source: str | None) -> dict[str, Any]:
"""Return the full default settings dict for a given source type."""
defaults: dict[str, Any] = {}
for key, defn in SETTINGS_SCHEMA.items():
if source is None or source in defn.sources:
defaults[key] = copy.deepcopy(defn.default)
return defaults
def hydrate_settings(sparse: dict[str, Any], source: str | None) -> dict[str, Any]:
"""Merge sparse settings with schema defaults to produce a full settings dict.
Keys are ordered according to SETTINGS_SCHEMA insertion order.
"""
defaults = get_defaults_for_source(source)
defaults.update(sparse)
return defaults
def stringify_additional_settings(
settings: dict[str, Any],
source: str | None = None,
) -> str:
"""Hydrate and stringify settings for Obtainium consumption."""
hydrated = hydrate_settings(settings, source)
return json.dumps(hydrated, separators=(",", ":"))
def make_obtainium_link(app: dict[str, Any]) -> str:
settings = get_additional_settings(app)
source = app.get("overrideSource")
payload = {
"id": app["id"],
"url": app["url"],
@@ -73,7 +96,7 @@ def make_obtainium_link(app: dict[str, Any]) -> str:
"otherAssetUrls": app.get("otherAssetUrls"),
"apkUrls": app.get("apkUrls"),
"preferredApkIndex": app.get("preferredApkIndex"),
"additionalSettings": stringify_additional_settings(app),
"additionalSettings": stringify_additional_settings(settings, source),
"categories": app.get("categories"),
"overrideSource": app.get("overrideSource"),
"allowIdChange": app.get("allowIdChange"),