forked from mtusi2026/mtusi2026lab4
62 lines
2.2 KiB
Python
Executable File
62 lines
2.2 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Reconciler traversal cost for several object counts; creates synthetic objects and removes them."""
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
S3TOOL = ROOT / 'scripts/s3tool.sh'
|
|
RECONCILE = ROOT / 'scripts/reconcile.sh'
|
|
PREFIX = 'chats/bench'
|
|
SIZES = (100, 1000, 5000)
|
|
|
|
|
|
def call(arguments):
|
|
return subprocess.run(arguments, capture_output=True, text=True, check=False)
|
|
|
|
|
|
def reconcile(*arguments):
|
|
result = call([str(RECONCILE), *arguments])
|
|
try:
|
|
return json.loads(result.stdout)
|
|
except json.JSONDecodeError:
|
|
print(result.stdout[-500:], file=sys.stderr)
|
|
print(result.stderr[-500:], file=sys.stderr)
|
|
raise SystemExit('reconcile did not return JSON')
|
|
|
|
|
|
def main():
|
|
measurements = []
|
|
for count in SIZES:
|
|
seeded = call([str(S3TOOL), 'seed', PREFIX, str(count), '1024'])
|
|
if seeded.returncode != 0:
|
|
raise SystemExit('seed failed: ' + seeded.stderr[-400:])
|
|
|
|
scan = reconcile('-prefix', PREFIX + '/', '-grace', '0s')
|
|
if scan['scanned_objects'] != count:
|
|
raise SystemExit(f'expected {count} objects, scanned {scan["scanned_objects"]}')
|
|
if len(scan['orphans']) != count:
|
|
raise SystemExit(f'expected {count} orphans, got {len(scan["orphans"])}')
|
|
|
|
repair = reconcile('-prefix', PREFIX + '/', '-grace', '0s', '-dry-run=false')
|
|
if repair['deleted_objects'] != count:
|
|
raise SystemExit(f'expected {count} deletions, got {repair["deleted_objects"]}')
|
|
|
|
measurements.append({
|
|
'objects': count,
|
|
'scan_ms': scan['duration_ms'],
|
|
'scan_s3_requests': scan['s3_requests'],
|
|
'scan_db_queries': scan['db_queries'],
|
|
'scan_objects_per_second': round(scan['objects_per_second'], 1),
|
|
'scan_ms_per_object': round(scan['duration_ms'] / count, 4),
|
|
'repair_ms': repair['duration_ms'],
|
|
'repair_s3_requests': repair['s3_requests'],
|
|
'repair_deleted': repair['deleted_objects'],
|
|
})
|
|
|
|
print(json.dumps({'prefix': PREFIX, 'measurements': measurements}, indent=2))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main() |