#!/usr/bin/env python3 """Reconciler traversal cost for several object counts; creates synthetic objects and removes them.""" import json import subprocess import sys from pathlib import Path ROOT = Path(__file__).resolve().parents[1] S3TOOL = ROOT / 'scripts/s3tool.sh' RECONCILE = ROOT / 'scripts/reconcile.sh' PREFIX = 'chats/bench' SIZES = (100, 1000, 5000) def call(arguments): return subprocess.run(arguments, capture_output=True, text=True, check=False) def reconcile(*arguments): result = call([str(RECONCILE), *arguments]) try: return json.loads(result.stdout) except json.JSONDecodeError: print(result.stdout[-500:], file=sys.stderr) print(result.stderr[-500:], file=sys.stderr) raise SystemExit('reconcile did not return JSON') def main(): measurements = [] for count in SIZES: seeded = call([str(S3TOOL), 'seed', PREFIX, str(count), '1024']) if seeded.returncode != 0: raise SystemExit('seed failed: ' + seeded.stderr[-400:]) scan = reconcile('-prefix', PREFIX + '/', '-grace', '0s') if scan['scanned_objects'] != count: raise SystemExit(f'expected {count} objects, scanned {scan["scanned_objects"]}') if len(scan['orphans']) != count: raise SystemExit(f'expected {count} orphans, got {len(scan["orphans"])}') repair = reconcile('-prefix', PREFIX + '/', '-grace', '0s', '-dry-run=false') if repair['deleted_objects'] != count: raise SystemExit(f'expected {count} deletions, got {repair["deleted_objects"]}') measurements.append({ 'objects': count, 'scan_ms': scan['duration_ms'], 'scan_s3_requests': scan['s3_requests'], 'scan_db_queries': scan['db_queries'], 'scan_objects_per_second': round(scan['objects_per_second'], 1), 'scan_ms_per_object': round(scan['duration_ms'] / count, 4), 'repair_ms': repair['duration_ms'], 'repair_s3_requests': repair['s3_requests'], 'repair_deleted': repair['deleted_objects'], }) print(json.dumps({'prefix': PREFIX, 'measurements': measurements}, indent=2)) if __name__ == '__main__': main()