forked from mtusi2026/mtusi2026lab4
feat: two-store reconciler with safe repair and cost measurement (grade 5, track 2)
This commit is contained in:
Executable
+62
@@ -0,0 +1,62 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Reconciler traversal cost for several object counts; creates synthetic objects and removes them."""
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
S3TOOL = ROOT / 'scripts/s3tool.sh'
|
||||
RECONCILE = ROOT / 'scripts/reconcile.sh'
|
||||
PREFIX = 'chats/bench'
|
||||
SIZES = (100, 1000, 5000)
|
||||
|
||||
|
||||
def call(arguments):
|
||||
return subprocess.run(arguments, capture_output=True, text=True, check=False)
|
||||
|
||||
|
||||
def reconcile(*arguments):
|
||||
result = call([str(RECONCILE), *arguments])
|
||||
try:
|
||||
return json.loads(result.stdout)
|
||||
except json.JSONDecodeError:
|
||||
print(result.stdout[-500:], file=sys.stderr)
|
||||
print(result.stderr[-500:], file=sys.stderr)
|
||||
raise SystemExit('reconcile did not return JSON')
|
||||
|
||||
|
||||
def main():
|
||||
measurements = []
|
||||
for count in SIZES:
|
||||
seeded = call([str(S3TOOL), 'seed', PREFIX, str(count), '1024'])
|
||||
if seeded.returncode != 0:
|
||||
raise SystemExit('seed failed: ' + seeded.stderr[-400:])
|
||||
|
||||
scan = reconcile('-prefix', PREFIX + '/', '-grace', '0s')
|
||||
if scan['scanned_objects'] != count:
|
||||
raise SystemExit(f'expected {count} objects, scanned {scan["scanned_objects"]}')
|
||||
if len(scan['orphans']) != count:
|
||||
raise SystemExit(f'expected {count} orphans, got {len(scan["orphans"])}')
|
||||
|
||||
repair = reconcile('-prefix', PREFIX + '/', '-grace', '0s', '-dry-run=false')
|
||||
if repair['deleted_objects'] != count:
|
||||
raise SystemExit(f'expected {count} deletions, got {repair["deleted_objects"]}')
|
||||
|
||||
measurements.append({
|
||||
'objects': count,
|
||||
'scan_ms': scan['duration_ms'],
|
||||
'scan_s3_requests': scan['s3_requests'],
|
||||
'scan_db_queries': scan['db_queries'],
|
||||
'scan_objects_per_second': round(scan['objects_per_second'], 1),
|
||||
'scan_ms_per_object': round(scan['duration_ms'] / count, 4),
|
||||
'repair_ms': repair['duration_ms'],
|
||||
'repair_s3_requests': repair['s3_requests'],
|
||||
'repair_deleted': repair['deleted_objects'],
|
||||
})
|
||||
|
||||
print(json.dumps({'prefix': PREFIX, 'measurements': measurements}, indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -14,6 +14,7 @@ def main():
|
||||
p.add_argument('--expect', type=int, default=201)
|
||||
p.add_argument('--file', default=str(ROOT / 'scripts/fixtures/sample.jpg'))
|
||||
p.add_argument('--content-type', default='image/jpeg')
|
||||
p.add_argument('--attach', action='store_true', help='also attach the upload to a new message')
|
||||
|
||||
args = arguments('Single upload used by compensation drills; creates synthetic data', configure)
|
||||
source = Path(args.file)
|
||||
@@ -29,6 +30,9 @@ def main():
|
||||
if response['status'] == 201:
|
||||
attachment = response['json']
|
||||
print(f"attachment={attachment['id']} size={attachment['size_bytes']}")
|
||||
if args.attach:
|
||||
message = c.message(alice, chat, '', attachments=[attachment['id']])['json']
|
||||
print(f"message={message['id']} attachment_ids={message['attachment_ids']}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
Reference in New Issue
Block a user