"""Fetch the large game data dumps that generated tables are built from. The app ships small, generated lookup tables (see ``src/static/vendor``). The dumps they are derived from are far too large to version, so they are fetched on demand into a gitignored cache. This is a shallow, blobless, sparse clone, so it transfers only the handful of files listed below rather than the repository's full history. Usage: python scripts/fetch_gamedata.py [--cache DIR] [--force] [--check] """ import argparse import hashlib import json import os import shutil import subprocess import sys UPSTREAM_URL = 'https://github.com/ao-data/ao-bin-dumps.git' PINNED_COMMIT = '47e4f5aca4d30b7495afa5a625ce0d5795e32050' # Only what the generators actually read. `localization.json` is deliberately # absent: it is 90 MB, and it cannot supply a display name for every mob # anyway (883 of 5479 mob definitions have no namelocatag). Nothing in # attribution depends on it. See src/static/vendor/PROVENANCE.md. REQUIRED_FILES = ( 'items.json', 'spells.json', 'mobs.json', 'gamedata.json', # Source for the vendored src/static/vendor/item_ids.txt. 'formatted/items.txt', ) # Anchored to the repo root. Without the leading slash gitcone matches the # basename anywhere in the tree, which drags in the redundant 24 MB # formatted/items.json alongside the root one. SPARSE_PATTERNS = tuple('/' + name for name in REQUIRED_FILES) DEFAULT_CACHE = os.path.join('.cache', 'gamedata') def _run(args, cwd=None): result = subprocess.run( args, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) if result.returncode != 0: sys.stderr.write(result.stdout.decode('utf-8', 'replace')) raise SystemExit('command failed: {}'.format(' '.join(args))) return result.stdout.decode('utf-8', 'replace') def _checksums(directory): """Map of filename -> sha256 for the required files.""" digests = {} for name in REQUIRED_FILES: path = os.path.join(directory, name) digest = hashlib.sha256() with open(path, 'rb') as handle: for chunk in iter(lambda: handle.read(1 << 20), b''): digest.update(chunk) digests[name] = digest.hexdigest() return digests def is_populated(directory): return all(os.path.exists(os.path.join(directory, name)) for name in REQUIRED_FILES) def fetch(cache_dir, force=False): """Ensure the dumps are present in ``cache_dir`` and return that path.""" if force and os.path.isdir(cache_dir): shutil.rmtree(cache_dir) elif is_populated(cache_dir): return cache_dir parent = os.path.dirname(os.path.abspath(cache_dir)) or '.' os.makedirs(parent, exist_ok=True) if os.path.isdir(cache_dir): shutil.rmtree(cache_dir) # --filter=blob:none avoids the full history; --sparse plus the final # checkout limits us to the files we name. _run([ 'git', 'clone', '--filter=blob:none', '--no-checkout', '--depth', '1', UPSTREAM_URL, cache_dir, ]) _run(['git', 'sparse-checkout', 'set', '--no-cone', *SPARSE_PATTERNS], cwd=cache_dir) _run(['git', 'checkout', PINNED_COMMIT], cwd=cache_dir) missing = [name for name in REQUIRED_FILES if not os.path.exists(os.path.join(cache_dir, name))] if missing: raise SystemExit('fetch incomplete, missing: {}'.format(missing)) return cache_dir def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--cache', default=DEFAULT_CACHE, help='cache directory (default: %(default)s)') parser.add_argument('--force', action='store_true', help='re-fetch even if the cache is already populated') parser.add_argument('--check', action='store_true', help='verify the cache matches the pinned commit, ' 'without fetching') args = parser.parse_args(argv) if args.check: if not is_populated(args.cache): print('cache is not populated: {}'.format(args.cache)) return 1 head = _run(['git', 'rev-parse', 'HEAD'], cwd=args.cache).strip() if head != PINNED_COMMIT: print('cache at {}, expected {}'.format(head, PINNED_COMMIT)) return 1 print('cache OK at {}'.format(PINNED_COMMIT)) print(json.dumps(_checksums(args.cache), indent=2, sort_keys=True)) return 0 fetch(args.cache, force=args.force) print('game data ready in {}'.format(args.cache)) return 0 if __name__ == '__main__': raise SystemExit(main())