Files
aostats-custom/scripts/fetch_gamedata.py
bionickatana 7279b3c207
Build / Build source distribution (push) Failing after 13s
Build / Execute unit tests (push) Failing after 8s
Build / Verify generated game data is current (push) Failing after 7s
Build / Upload Release Asset (push) Has been skipped
Build / upload_pypi (push) Has been skipped
Working advanced tracking code
2026-10-13 09:08:38 -06:00

142 lines
4.6 KiB
Python

"""Fetch the large game data dumps that generated tables are built from.
The app ships small, generated lookup tables (see ``src/static/vendor``). The
dumps they are derived from are far too large to version, so they are fetched
on demand into a gitignored cache.
This is a shallow, blobless, sparse clone, so it transfers only the handful of
files listed below rather than the repository's full history.
Usage:
python scripts/fetch_gamedata.py [--cache DIR] [--force] [--check]
"""
import argparse
import hashlib
import json
import os
import shutil
import subprocess
import sys
UPSTREAM_URL = 'https://github.com/ao-data/ao-bin-dumps.git'
PINNED_COMMIT = '47e4f5aca4d30b7495afa5a625ce0d5795e32050'
# Only what the generators actually read. `localization.json` is deliberately
# absent: it is 90 MB, and it cannot supply a display name for every mob
# anyway (883 of 5479 mob definitions have no namelocatag). Nothing in
# attribution depends on it. See src/static/vendor/PROVENANCE.md.
REQUIRED_FILES = (
'items.json',
'spells.json',
'mobs.json',
'gamedata.json',
# Source for the vendored src/static/vendor/item_ids.txt.
'formatted/items.txt',
)
# Anchored to the repo root. Without the leading slash gitcone matches the
# basename anywhere in the tree, which drags in the redundant 24 MB
# formatted/items.json alongside the root one.
SPARSE_PATTERNS = tuple('/' + name for name in REQUIRED_FILES)
DEFAULT_CACHE = os.path.join('.cache', 'gamedata')
def _run(args, cwd=None):
result = subprocess.run(
args,
cwd=cwd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
if result.returncode != 0:
sys.stderr.write(result.stdout.decode('utf-8', 'replace'))
raise SystemExit('command failed: {}'.format(' '.join(args)))
return result.stdout.decode('utf-8', 'replace')
def _checksums(directory):
"""Map of filename -> sha256 for the required files."""
digests = {}
for name in REQUIRED_FILES:
path = os.path.join(directory, name)
digest = hashlib.sha256()
with open(path, 'rb') as handle:
for chunk in iter(lambda: handle.read(1 << 20), b''):
digest.update(chunk)
digests[name] = digest.hexdigest()
return digests
def is_populated(directory):
return all(os.path.exists(os.path.join(directory, name))
for name in REQUIRED_FILES)
def fetch(cache_dir, force=False):
"""Ensure the dumps are present in ``cache_dir`` and return that path."""
if force and os.path.isdir(cache_dir):
shutil.rmtree(cache_dir)
elif is_populated(cache_dir):
return cache_dir
parent = os.path.dirname(os.path.abspath(cache_dir)) or '.'
os.makedirs(parent, exist_ok=True)
if os.path.isdir(cache_dir):
shutil.rmtree(cache_dir)
# --filter=blob:none avoids the full history; --sparse plus the final
# checkout limits us to the files we name.
_run([
'git', 'clone',
'--filter=blob:none',
'--no-checkout',
'--depth', '1',
UPSTREAM_URL,
cache_dir,
])
_run(['git', 'sparse-checkout', 'set', '--no-cone', *SPARSE_PATTERNS],
cwd=cache_dir)
_run(['git', 'checkout', PINNED_COMMIT], cwd=cache_dir)
missing = [name for name in REQUIRED_FILES
if not os.path.exists(os.path.join(cache_dir, name))]
if missing:
raise SystemExit('fetch incomplete, missing: {}'.format(missing))
return cache_dir
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--cache', default=DEFAULT_CACHE,
help='cache directory (default: %(default)s)')
parser.add_argument('--force', action='store_true',
help='re-fetch even if the cache is already populated')
parser.add_argument('--check', action='store_true',
help='verify the cache matches the pinned commit, '
'without fetching')
args = parser.parse_args(argv)
if args.check:
if not is_populated(args.cache):
print('cache is not populated: {}'.format(args.cache))
return 1
head = _run(['git', 'rev-parse', 'HEAD'], cwd=args.cache).strip()
if head != PINNED_COMMIT:
print('cache at {}, expected {}'.format(head, PINNED_COMMIT))
return 1
print('cache OK at {}'.format(PINNED_COMMIT))
print(json.dumps(_checksums(args.cache), indent=2, sort_keys=True))
return 0
fetch(args.cache, force=args.force)
print('game data ready in {}'.format(args.cache))
return 0
if __name__ == '__main__':
raise SystemExit(main())