#!/usr/bin/env python3 """Track 2: DB<->S3 consistency. Fault injection on both sides + reconciler cost. python scripts/track2/run_track2.py faults --output evidence/track2/faults.json python scripts/track2/run_track2.py cost --sizes 1000 5000 20000 --repeat 3 --output evidence/track2/cost.json The periodic `reconciler` service is stopped for the duration (it would race the experiment) and started again at the end. Synthetic data only. """ import argparse import hashlib import json from pathlib import Path import subprocess import sys import time import uuid ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT / 'scripts')) from lab4ops import NETWORK, PNG, TOOL_IMAGE, psql, s3tool, upload, winpath # noqa: E402 from lablib import sh # noqa: E402 from client import Client, check # noqa: E402 def s3put_bytes(key, data): cmd = ['docker', 'run', '--rm', '-i', '--network', NETWORK, '-v', winpath(ROOT / 'secrets') + ':/secrets:ro', '-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro', TOOL_IMAGE, 'python', '-I', '/tool/s3tool.py', 'put', key, '--stdin'] r = subprocess.run(cmd, input=data, capture_output=True) check(r.returncode == 0, 's3 put failed') return json.loads(r.stdout.decode().strip().splitlines()[-1]) def reconcile(*args): """One reconciler pass in a throwaway container; also returns its cgroup memory peak.""" out = sh('docker', 'compose', 'run', '--rm', '--no-deps', '-T', '--entrypoint', 'sh', 'reconciler', '-c', '/app/server reconcile ' + ' '.join(args) + ' > /tmp/r.json && cat /tmp/r.json ' '&& echo && cat /sys/fs/cgroup/memory.peak') lines = [x for x in out.splitlines() if x.strip()] rep = json.loads(lines[-2]) rep['container_memory_peak_mib'] = round(int(lines[-1]) / 2**20, 1) return rep def counts(prefix='attachments/'): return {'s3_objects': s3tool('count', prefix)['count'], 'db_rows': int(psql('SELECT count(*) FROM attachments')), 'db_pending': int(psql("SELECT count(*) FROM attachments WHERE status = 'pending'")), 'db_missing_marked': int(psql('SELECT count(*) FROM attachments WHERE missing_at IS NOT NULL'))} def summary(rep): keys = ('mode', 'objects_scanned', 'rows_scanned', 'orphan_objects', 'fresh_skipped', 'stale_pending', 'missing_objects', 'size_mismatch', 'recovered', 'fixed', 'errors', 'total_ms') return {k: rep[k] for k in keys} def scoped(rep, prefix, invert=False): out = {} for i in rep['items']: if i.get('key', '').startswith(prefix) == invert: continue k = i['kind'] + ('/skip' if i['action'].startswith('skip') else '') out[k] = out.get(k, 0) + 1 return out def insert_pending(att_id, chat, owner, key, age): psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height," " object_key, status, created_at, updated_at) VALUES ('%s', '%s', '%s', 'crash.png', 'image/png', 100," " repeat('0', 64), 1, 1, '%s', 'pending', now() - interval '%s', now() - interval '%s')" % (att_id, chat, owner, key, age, age)) def exists(key): return s3tool('get', key)['status'] == 200 def cmd_faults(a): c = Client(a.base_url, timeout=30) alice, bob, eve, chat = c.bootstrap() cid, prefix = chat['id'], 'attachments/' + chat['id'] + '/' key_of = lambda att_id: psql("SELECT object_key FROM attachments WHERE id = '%s'" % att_id) # healthy, referenced attachment (must survive everything) u1 = upload(c, alice, chat)['json'] # referenced attachment whose object will be lost (S3-side fault) u2 = upload(c, alice, chat)['json'] msg = c.message(alice, chat, '', attachments=[u1['id'], u2['id']])['json'] # unreferenced ready attachment whose metadata row will be lost (DB-side fault) u3 = upload(c, alice, chat)['json'] u3_key = key_of(u3['id']) f = {} f['F1_orphan_object_no_row'] = prefix + str(uuid.uuid4()) s3tool('put', f['F1_orphan_object_no_row']) psql("DELETE FROM attachments WHERE id = '%s'" % u3['id']) f['F2_metadata_row_lost(U3)'] = u3_key s3tool('delete', key_of(u2['id'])) f['F3_object_lost_of_used(U2)'] = key_of(u2['id']) f4_id = str(uuid.uuid4()) f['F4_crash_after_put_before_ready'] = prefix + f4_id insert_pending(f4_id, cid, alice['id'], f['F4_crash_after_put_before_ready'], '1 hour') s3tool('put', f['F4_crash_after_put_before_ready']) f5_id = str(uuid.uuid4()) f['F5_crash_before_put'] = prefix + f5_id insert_pending(f5_id, cid, alice['id'], f['F5_crash_before_put'], '1 hour') print('waiting %ds so that injected objects are older than grace=%s' % (a.wait, a.grace), flush=True) time.sleep(a.wait) # fresh state that must NOT be touched: an object being uploaded right now and its pending row f7_id = str(uuid.uuid4()) f['F6_F7_fresh_upload_in_progress'] = prefix + f7_id insert_pending(f7_id, cid, alice['id'], f['F6_F7_fresh_upload_in_progress'], '0 seconds') s3tool('put', f['F6_F7_fresh_upload_in_progress']) f8 = prefix + str(uuid.uuid4()) f['F8_fresh_orphan_object'] = f8 s3tool('put', f8) steps = [] before = counts() dry = reconcile('--grace', a.grace) after_dry = counts() steps.append({'step': 'dry-run', 'report': summary(dry), 'items': dry['items'], 'state_before': before, 'state_after': after_dry, 'dry_run_changed_nothing': before == after_dry}) check(before == after_dry, 'dry-run modified state') mine = scoped(dry, prefix) steps[-1]['this_experiment'] = mine steps[-1]['other_findings_outside_experiment'] = scoped(dry, prefix, invert=True) check(mine == {'orphan_object': 2, 'stale_pending': 2, 'missing_object': 1, 'orphan_object/skip': 1}, 'detection mismatch: %s' % mine) ap = reconcile('--apply', '--grace', a.grace) steps.append({'step': 'apply #1', 'report': summary(ap), 'this_experiment': scoped(ap, prefix), 'items': [i for i in ap['items'] if i.get('key', '').startswith(prefix)], 'state_after': counts()}) again = reconcile('--apply', '--grace', a.grace) steps.append({'step': 'apply #2 (repeat)', 'report': summary(again), 'this_experiment': scoped(again, prefix), 'state_after': counts()}) check(again['fixed'] == 0 and again['errors'] == 0, 'second apply was not a no-op') d1 = c.call('GET', '/api/v1/attachments/' + u1['id'] + '/content', actor=bob) msgs = c.call('GET', '/api/v1/chats/' + cid + '/messages?limit=100', actor=bob, schema='MessagePage')['json']['items'] verify = { 'U1_used_healthy_download_sha_ok': hashlib.sha256(d1['bytes']).hexdigest() == u1['sha256'], 'U2_metadata_kept': c.call('GET', '/api/v1/attachments/' + u2['id'], actor=bob)['status'], 'U2_content_status': c.call('GET', '/api/v1/attachments/' + u2['id'] + '/content', 404, actor=bob)['status'], 'U2_marked_missing': psql("SELECT missing_at IS NOT NULL FROM attachments WHERE id = '%s'" % u2['id']) == 't', 'message_still_references_U1_U2': [m for m in msgs if m['id'] == msg['id']][0]['attachment_ids'] == [u1['id'], u2['id']], 'F1_orphan_deleted': not exists(f['F1_orphan_object_no_row']), 'F2_orphan_of_lost_row_deleted': not exists(u3_key), 'F4_row_and_object_deleted': not exists(f['F4_crash_after_put_before_ready']) and psql("SELECT count(*) FROM attachments WHERE id = '%s'" % f4_id) == '0', 'F5_row_deleted': psql("SELECT count(*) FROM attachments WHERE id = '%s'" % f5_id) == '0', 'F7_fresh_pending_row_kept': psql("SELECT status FROM attachments WHERE id = '%s'" % f7_id) == 'pending', 'F7_fresh_object_kept': exists(f['F6_F7_fresh_upload_in_progress']), 'F8_fresh_orphan_kept': exists(f8), } for k, v in verify.items(): check(v is True or v in (200, 404), k) # restore U2 from a "backup" (same bytes): the reconciler clears the missing mark s3put_bytes(key_of(u2['id']), PNG) rec = reconcile('--apply', '--grace', a.grace) d2 = c.call('GET', '/api/v1/attachments/' + u2['id'] + '/content', actor=bob) verify['U2_restored_and_unmarked'] = (rec['recovered'] == 1 and hashlib.sha256(d2['bytes']).hexdigest() == u2['sha256'] and psql("SELECT missing_at IS NULL FROM attachments WHERE id = '%s'" % u2['id']) == 't') steps.append({'step': 'apply after restoring U2 object', 'report': summary(rec)}) check(verify['U2_restored_and_unmarked'], 'restore not detected') # default production grace (15m): the same kind of fresh garbage is only reported as skipped prod = reconcile('--grace', '15m') steps.append({'step': 'dry-run with production grace 15m', 'report': summary(prod)}) # clean the fresh leftovers with the regular procedure so the next run starts clean return {'grace_used': a.grace, 'faults': f, 'steps': steps, 'verify': verify} def cmd_cost(a): c = Client(a.base_url, timeout=30) alice, bob, eve, chat = c.bootstrap() run = uuid.uuid4().hex[:8] keys_file = ROOT / 'evidence/track2/.bench-keys.txt' created = 0 results = [] try: for n in sorted(a.sizes): add = n - created keys = ['attachments/bench-%s/%07d' % (run, i) for i in range(created, n)] psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256," " width, height, object_key, status, created_at, updated_at)" " SELECT gen_random_uuid(), '%s', '%s', 'bench.png', 'image/png', 100, repeat('0', 64), 1, 1," " 'attachments/bench-%s/' || lpad(i::text, 7, '0'), 'ready', now() - interval '1 day', now()" " FROM generate_series(%d, %d) AS i" % (chat['id'], alice['id'], run, created, n - 1)) t0 = time.monotonic() put = json.loads(subprocess.run( ['docker', 'run', '--rm', '-i', '--network', NETWORK, '-v', winpath(ROOT / 'secrets') + ':/secrets:ro', '-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro', TOOL_IMAGE, 'python', '-I', '/tool/s3tool.py', 'bulk-put', '--size', '100', '--workers', '32'], input='\n'.join(keys), capture_output=True, text=True).stdout.strip().splitlines()[-1]) check(put['ok'] == add, 'bulk put incomplete: %s' % put) created = n print('populated %d (+%d in %.1fs)' % (n, add, time.monotonic() - t0), flush=True) for r in range(a.repeat): rep = reconcile('--grace', '15m') row = {'bench_objects': n, 'repeat': r + 1, 'objects_scanned': rep['objects_scanned'], 'rows_scanned': rep['rows_scanned'], 'list_ms': rep['list_ms'], 'db_ms': rep['db_ms'], 'total_ms': rep['total_ms'], 'per_object_us': round(rep['per_object_us'], 1), 'container_memory_peak_mib': rep['container_memory_peak_mib'], 'findings': rep['orphan_objects'] + rep['stale_pending'] + rep['missing_objects']} print(json.dumps(row), flush=True) results.append(row) finally: if created: keys = ['attachments/bench-%s/%07d' % (run, i) for i in range(created)] psql("DELETE FROM attachments WHERE object_key LIKE 'attachments/bench-%s/%%'" % run) subprocess.run(['docker', 'run', '--rm', '-i', '--network', NETWORK, '-v', winpath(ROOT / 'secrets') + ':/secrets:ro', '-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro', TOOL_IMAGE, 'python', '-I', '/tool/s3tool.py', 'bulk-delete', '--workers', '32'], input='\n'.join(keys), capture_output=True, text=True) if keys_file.exists(): keys_file.unlink() return {'object_size_bytes': 100, 'reconciler_limits': '0.5 CPU, 128M', 's3': 'SeaweedFS 3.80, 1 CPU, 1G', 'runs': results} def main(): p = argparse.ArgumentParser() p.add_argument('cmd', choices=['faults', 'cost']) p.add_argument('--base-url', default='http://localhost:8080') p.add_argument('--grace', default='1m') p.add_argument('--wait', type=int, default=65) p.add_argument('--sizes', type=int, nargs='+', default=[1000, 5000, 20000]) p.add_argument('--repeat', type=int, default=3) p.add_argument('--output') a = p.parse_args() sh('docker', 'compose', 'stop', 'reconciler') try: res = {'command': a.cmd, 'date_utc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), 'commit': sh('git', 'rev-parse', '--short', 'HEAD', check=False), 'result': (cmd_faults if a.cmd == 'faults' else cmd_cost)(a)} finally: sh('docker', 'compose', 'start', 'reconciler') text = json.dumps(res, ensure_ascii=False, indent=2) if a.output: Path(a.output).parent.mkdir(parents=True, exist_ok=True) Path(a.output).write_text(text + '\n', encoding='utf-8') print(text if a.cmd == 'faults' else json.dumps(res['result']['runs'][-1])) print('PASS: track2 ' + a.cmd) if __name__ == '__main__': main()