Working advanced tracking code
Build / Build source distribution (push) Failing after 13s
Build / Execute unit tests (push) Failing after 8s
Build / Verify generated game data is current (push) Failing after 7s
Build / Upload Release Asset (push) Has been skipped
Build / upload_pypi (push) Has been skipped
Build / Build source distribution (push) Failing after 13s
Build / Execute unit tests (push) Failing after 8s
Build / Verify generated game data is current (push) Failing after 7s
Build / Upload Release Asset (push) Has been skipped
Build / upload_pypi (push) Has been skipped
This commit is contained in:
@@ -0,0 +1,141 @@
|
||||
"""Fetch the large game data dumps that generated tables are built from.
|
||||
|
||||
The app ships small, generated lookup tables (see ``src/static/vendor``). The
|
||||
dumps they are derived from are far too large to version, so they are fetched
|
||||
on demand into a gitignored cache.
|
||||
|
||||
This is a shallow, blobless, sparse clone, so it transfers only the handful of
|
||||
files listed below rather than the repository's full history.
|
||||
|
||||
Usage:
|
||||
python scripts/fetch_gamedata.py [--cache DIR] [--force] [--check]
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
UPSTREAM_URL = 'https://github.com/ao-data/ao-bin-dumps.git'
|
||||
PINNED_COMMIT = '47e4f5aca4d30b7495afa5a625ce0d5795e32050'
|
||||
|
||||
# Only what the generators actually read. `localization.json` is deliberately
|
||||
# absent: it is 90 MB, and it cannot supply a display name for every mob
|
||||
# anyway (883 of 5479 mob definitions have no namelocatag). Nothing in
|
||||
# attribution depends on it. See src/static/vendor/PROVENANCE.md.
|
||||
REQUIRED_FILES = (
|
||||
'items.json',
|
||||
'spells.json',
|
||||
'mobs.json',
|
||||
'gamedata.json',
|
||||
# Source for the vendored src/static/vendor/item_ids.txt.
|
||||
'formatted/items.txt',
|
||||
)
|
||||
|
||||
# Anchored to the repo root. Without the leading slash gitcone matches the
|
||||
# basename anywhere in the tree, which drags in the redundant 24 MB
|
||||
# formatted/items.json alongside the root one.
|
||||
SPARSE_PATTERNS = tuple('/' + name for name in REQUIRED_FILES)
|
||||
|
||||
DEFAULT_CACHE = os.path.join('.cache', 'gamedata')
|
||||
|
||||
|
||||
def _run(args, cwd=None):
|
||||
result = subprocess.run(
|
||||
args,
|
||||
cwd=cwd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
sys.stderr.write(result.stdout.decode('utf-8', 'replace'))
|
||||
raise SystemExit('command failed: {}'.format(' '.join(args)))
|
||||
return result.stdout.decode('utf-8', 'replace')
|
||||
|
||||
|
||||
def _checksums(directory):
|
||||
"""Map of filename -> sha256 for the required files."""
|
||||
digests = {}
|
||||
for name in REQUIRED_FILES:
|
||||
path = os.path.join(directory, name)
|
||||
digest = hashlib.sha256()
|
||||
with open(path, 'rb') as handle:
|
||||
for chunk in iter(lambda: handle.read(1 << 20), b''):
|
||||
digest.update(chunk)
|
||||
digests[name] = digest.hexdigest()
|
||||
return digests
|
||||
|
||||
|
||||
def is_populated(directory):
|
||||
return all(os.path.exists(os.path.join(directory, name))
|
||||
for name in REQUIRED_FILES)
|
||||
|
||||
|
||||
def fetch(cache_dir, force=False):
|
||||
"""Ensure the dumps are present in ``cache_dir`` and return that path."""
|
||||
if force and os.path.isdir(cache_dir):
|
||||
shutil.rmtree(cache_dir)
|
||||
elif is_populated(cache_dir):
|
||||
return cache_dir
|
||||
|
||||
parent = os.path.dirname(os.path.abspath(cache_dir)) or '.'
|
||||
os.makedirs(parent, exist_ok=True)
|
||||
|
||||
if os.path.isdir(cache_dir):
|
||||
shutil.rmtree(cache_dir)
|
||||
|
||||
# --filter=blob:none avoids the full history; --sparse plus the final
|
||||
# checkout limits us to the files we name.
|
||||
_run([
|
||||
'git', 'clone',
|
||||
'--filter=blob:none',
|
||||
'--no-checkout',
|
||||
'--depth', '1',
|
||||
UPSTREAM_URL,
|
||||
cache_dir,
|
||||
])
|
||||
_run(['git', 'sparse-checkout', 'set', '--no-cone', *SPARSE_PATTERNS],
|
||||
cwd=cache_dir)
|
||||
_run(['git', 'checkout', PINNED_COMMIT], cwd=cache_dir)
|
||||
|
||||
missing = [name for name in REQUIRED_FILES
|
||||
if not os.path.exists(os.path.join(cache_dir, name))]
|
||||
if missing:
|
||||
raise SystemExit('fetch incomplete, missing: {}'.format(missing))
|
||||
|
||||
return cache_dir
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--cache', default=DEFAULT_CACHE,
|
||||
help='cache directory (default: %(default)s)')
|
||||
parser.add_argument('--force', action='store_true',
|
||||
help='re-fetch even if the cache is already populated')
|
||||
parser.add_argument('--check', action='store_true',
|
||||
help='verify the cache matches the pinned commit, '
|
||||
'without fetching')
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
if args.check:
|
||||
if not is_populated(args.cache):
|
||||
print('cache is not populated: {}'.format(args.cache))
|
||||
return 1
|
||||
head = _run(['git', 'rev-parse', 'HEAD'], cwd=args.cache).strip()
|
||||
if head != PINNED_COMMIT:
|
||||
print('cache at {}, expected {}'.format(head, PINNED_COMMIT))
|
||||
return 1
|
||||
print('cache OK at {}'.format(PINNED_COMMIT))
|
||||
print(json.dumps(_checksums(args.cache), indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
fetch(args.cache, force=args.force)
|
||||
print('game data ready in {}'.format(args.cache))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user