7279b3c207
Build / Build source distribution (push) Failing after 13s
Build / Execute unit tests (push) Failing after 8s
Build / Verify generated game data is current (push) Failing after 7s
Build / Upload Release Asset (push) Has been skipped
Build / upload_pypi (push) Has been skipped
142 lines
4.6 KiB
Python
142 lines
4.6 KiB
Python
"""Fetch the large game data dumps that generated tables are built from.
|
|
|
|
The app ships small, generated lookup tables (see ``src/static/vendor``). The
|
|
dumps they are derived from are far too large to version, so they are fetched
|
|
on demand into a gitignored cache.
|
|
|
|
This is a shallow, blobless, sparse clone, so it transfers only the handful of
|
|
files listed below rather than the repository's full history.
|
|
|
|
Usage:
|
|
python scripts/fetch_gamedata.py [--cache DIR] [--force] [--check]
|
|
"""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
|
|
UPSTREAM_URL = 'https://github.com/ao-data/ao-bin-dumps.git'
|
|
PINNED_COMMIT = '47e4f5aca4d30b7495afa5a625ce0d5795e32050'
|
|
|
|
# Only what the generators actually read. `localization.json` is deliberately
|
|
# absent: it is 90 MB, and it cannot supply a display name for every mob
|
|
# anyway (883 of 5479 mob definitions have no namelocatag). Nothing in
|
|
# attribution depends on it. See src/static/vendor/PROVENANCE.md.
|
|
REQUIRED_FILES = (
|
|
'items.json',
|
|
'spells.json',
|
|
'mobs.json',
|
|
'gamedata.json',
|
|
# Source for the vendored src/static/vendor/item_ids.txt.
|
|
'formatted/items.txt',
|
|
)
|
|
|
|
# Anchored to the repo root. Without the leading slash gitcone matches the
|
|
# basename anywhere in the tree, which drags in the redundant 24 MB
|
|
# formatted/items.json alongside the root one.
|
|
SPARSE_PATTERNS = tuple('/' + name for name in REQUIRED_FILES)
|
|
|
|
DEFAULT_CACHE = os.path.join('.cache', 'gamedata')
|
|
|
|
|
|
def _run(args, cwd=None):
|
|
result = subprocess.run(
|
|
args,
|
|
cwd=cwd,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
)
|
|
if result.returncode != 0:
|
|
sys.stderr.write(result.stdout.decode('utf-8', 'replace'))
|
|
raise SystemExit('command failed: {}'.format(' '.join(args)))
|
|
return result.stdout.decode('utf-8', 'replace')
|
|
|
|
|
|
def _checksums(directory):
|
|
"""Map of filename -> sha256 for the required files."""
|
|
digests = {}
|
|
for name in REQUIRED_FILES:
|
|
path = os.path.join(directory, name)
|
|
digest = hashlib.sha256()
|
|
with open(path, 'rb') as handle:
|
|
for chunk in iter(lambda: handle.read(1 << 20), b''):
|
|
digest.update(chunk)
|
|
digests[name] = digest.hexdigest()
|
|
return digests
|
|
|
|
|
|
def is_populated(directory):
|
|
return all(os.path.exists(os.path.join(directory, name))
|
|
for name in REQUIRED_FILES)
|
|
|
|
|
|
def fetch(cache_dir, force=False):
|
|
"""Ensure the dumps are present in ``cache_dir`` and return that path."""
|
|
if force and os.path.isdir(cache_dir):
|
|
shutil.rmtree(cache_dir)
|
|
elif is_populated(cache_dir):
|
|
return cache_dir
|
|
|
|
parent = os.path.dirname(os.path.abspath(cache_dir)) or '.'
|
|
os.makedirs(parent, exist_ok=True)
|
|
|
|
if os.path.isdir(cache_dir):
|
|
shutil.rmtree(cache_dir)
|
|
|
|
# --filter=blob:none avoids the full history; --sparse plus the final
|
|
# checkout limits us to the files we name.
|
|
_run([
|
|
'git', 'clone',
|
|
'--filter=blob:none',
|
|
'--no-checkout',
|
|
'--depth', '1',
|
|
UPSTREAM_URL,
|
|
cache_dir,
|
|
])
|
|
_run(['git', 'sparse-checkout', 'set', '--no-cone', *SPARSE_PATTERNS],
|
|
cwd=cache_dir)
|
|
_run(['git', 'checkout', PINNED_COMMIT], cwd=cache_dir)
|
|
|
|
missing = [name for name in REQUIRED_FILES
|
|
if not os.path.exists(os.path.join(cache_dir, name))]
|
|
if missing:
|
|
raise SystemExit('fetch incomplete, missing: {}'.format(missing))
|
|
|
|
return cache_dir
|
|
|
|
|
|
def main(argv=None):
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--cache', default=DEFAULT_CACHE,
|
|
help='cache directory (default: %(default)s)')
|
|
parser.add_argument('--force', action='store_true',
|
|
help='re-fetch even if the cache is already populated')
|
|
parser.add_argument('--check', action='store_true',
|
|
help='verify the cache matches the pinned commit, '
|
|
'without fetching')
|
|
args = parser.parse_args(argv)
|
|
|
|
if args.check:
|
|
if not is_populated(args.cache):
|
|
print('cache is not populated: {}'.format(args.cache))
|
|
return 1
|
|
head = _run(['git', 'rev-parse', 'HEAD'], cwd=args.cache).strip()
|
|
if head != PINNED_COMMIT:
|
|
print('cache at {}, expected {}'.format(head, PINNED_COMMIT))
|
|
return 1
|
|
print('cache OK at {}'.format(PINNED_COMMIT))
|
|
print(json.dumps(_checksums(args.cache), indent=2, sort_keys=True))
|
|
return 0
|
|
|
|
fetch(args.cache, force=args.force)
|
|
print('game data ready in {}'.format(args.cache))
|
|
return 0
|
|
|
|
|
|
if __name__ == '__main__':
|
|
raise SystemExit(main())
|