forked from mtusi2026/mtusi2026lab4
255 lines
13 KiB
Python
255 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
"""Track 2: DB<->S3 consistency. Fault injection on both sides + reconciler cost.
|
|
|
|
python scripts/track2/run_track2.py faults --output evidence/track2/faults.json
|
|
python scripts/track2/run_track2.py cost --sizes 1000 5000 20000 --repeat 3 --output evidence/track2/cost.json
|
|
|
|
The periodic `reconciler` service is stopped for the duration (it would race the
|
|
experiment) and started again at the end. Synthetic data only.
|
|
"""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(ROOT / 'scripts'))
|
|
from lab4ops import NETWORK, PNG, TOOL_IMAGE, psql, s3tool, upload, winpath # noqa: E402
|
|
from lablib import sh # noqa: E402
|
|
from client import Client, check # noqa: E402
|
|
|
|
|
|
def s3put_bytes(key, data):
|
|
cmd = ['docker', 'run', '--rm', '-i', '--network', NETWORK,
|
|
'-v', winpath(ROOT / 'secrets') + ':/secrets:ro', '-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro',
|
|
TOOL_IMAGE, 'python', '-I', '/tool/s3tool.py', 'put', key, '--stdin']
|
|
r = subprocess.run(cmd, input=data, capture_output=True)
|
|
check(r.returncode == 0, 's3 put failed')
|
|
return json.loads(r.stdout.decode().strip().splitlines()[-1])
|
|
|
|
|
|
def reconcile(*args):
|
|
"""One reconciler pass in a throwaway container; also returns its cgroup memory peak."""
|
|
out = sh('docker', 'compose', 'run', '--rm', '--no-deps', '-T', '--entrypoint', 'sh', 'reconciler', '-c',
|
|
'/app/server reconcile ' + ' '.join(args) + ' > /tmp/r.json && cat /tmp/r.json '
|
|
'&& echo && cat /sys/fs/cgroup/memory.peak')
|
|
lines = [x for x in out.splitlines() if x.strip()]
|
|
rep = json.loads(lines[-2])
|
|
rep['container_memory_peak_mib'] = round(int(lines[-1]) / 2**20, 1)
|
|
return rep
|
|
|
|
|
|
def counts(prefix='attachments/'):
|
|
return {'s3_objects': s3tool('count', prefix)['count'],
|
|
'db_rows': int(psql('SELECT count(*) FROM attachments')),
|
|
'db_pending': int(psql("SELECT count(*) FROM attachments WHERE status = 'pending'")),
|
|
'db_missing_marked': int(psql('SELECT count(*) FROM attachments WHERE missing_at IS NOT NULL'))}
|
|
|
|
|
|
def summary(rep):
|
|
keys = ('mode', 'objects_scanned', 'rows_scanned', 'orphan_objects', 'fresh_skipped', 'stale_pending',
|
|
'missing_objects', 'size_mismatch', 'recovered', 'fixed', 'errors', 'total_ms')
|
|
return {k: rep[k] for k in keys}
|
|
|
|
|
|
def scoped(rep, prefix, invert=False):
|
|
out = {}
|
|
for i in rep['items']:
|
|
if i.get('key', '').startswith(prefix) == invert:
|
|
continue
|
|
k = i['kind'] + ('/skip' if i['action'].startswith('skip') else '')
|
|
out[k] = out.get(k, 0) + 1
|
|
return out
|
|
|
|
|
|
def insert_pending(att_id, chat, owner, key, age):
|
|
psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height,"
|
|
" object_key, status, created_at, updated_at) VALUES ('%s', '%s', '%s', 'crash.png', 'image/png', 100,"
|
|
" repeat('0', 64), 1, 1, '%s', 'pending', now() - interval '%s', now() - interval '%s')"
|
|
% (att_id, chat, owner, key, age, age))
|
|
|
|
|
|
def exists(key):
|
|
return s3tool('get', key)['status'] == 200
|
|
|
|
|
|
def cmd_faults(a):
|
|
c = Client(a.base_url, timeout=30)
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
cid, prefix = chat['id'], 'attachments/' + chat['id'] + '/'
|
|
key_of = lambda att_id: psql("SELECT object_key FROM attachments WHERE id = '%s'" % att_id)
|
|
|
|
# healthy, referenced attachment (must survive everything)
|
|
u1 = upload(c, alice, chat)['json']
|
|
# referenced attachment whose object will be lost (S3-side fault)
|
|
u2 = upload(c, alice, chat)['json']
|
|
msg = c.message(alice, chat, '', attachments=[u1['id'], u2['id']])['json']
|
|
# unreferenced ready attachment whose metadata row will be lost (DB-side fault)
|
|
u3 = upload(c, alice, chat)['json']
|
|
u3_key = key_of(u3['id'])
|
|
|
|
f = {}
|
|
f['F1_orphan_object_no_row'] = prefix + str(uuid.uuid4())
|
|
s3tool('put', f['F1_orphan_object_no_row'])
|
|
psql("DELETE FROM attachments WHERE id = '%s'" % u3['id'])
|
|
f['F2_metadata_row_lost(U3)'] = u3_key
|
|
s3tool('delete', key_of(u2['id']))
|
|
f['F3_object_lost_of_used(U2)'] = key_of(u2['id'])
|
|
f4_id = str(uuid.uuid4())
|
|
f['F4_crash_after_put_before_ready'] = prefix + f4_id
|
|
insert_pending(f4_id, cid, alice['id'], f['F4_crash_after_put_before_ready'], '1 hour')
|
|
s3tool('put', f['F4_crash_after_put_before_ready'])
|
|
f5_id = str(uuid.uuid4())
|
|
f['F5_crash_before_put'] = prefix + f5_id
|
|
insert_pending(f5_id, cid, alice['id'], f['F5_crash_before_put'], '1 hour')
|
|
|
|
print('waiting %ds so that injected objects are older than grace=%s' % (a.wait, a.grace), flush=True)
|
|
time.sleep(a.wait)
|
|
# fresh state that must NOT be touched: an object being uploaded right now and its pending row
|
|
f7_id = str(uuid.uuid4())
|
|
f['F6_F7_fresh_upload_in_progress'] = prefix + f7_id
|
|
insert_pending(f7_id, cid, alice['id'], f['F6_F7_fresh_upload_in_progress'], '0 seconds')
|
|
s3tool('put', f['F6_F7_fresh_upload_in_progress'])
|
|
f8 = prefix + str(uuid.uuid4())
|
|
f['F8_fresh_orphan_object'] = f8
|
|
s3tool('put', f8)
|
|
|
|
steps = []
|
|
before = counts()
|
|
dry = reconcile('--grace', a.grace)
|
|
after_dry = counts()
|
|
steps.append({'step': 'dry-run', 'report': summary(dry), 'items': dry['items'], 'state_before': before,
|
|
'state_after': after_dry, 'dry_run_changed_nothing': before == after_dry})
|
|
check(before == after_dry, 'dry-run modified state')
|
|
mine = scoped(dry, prefix)
|
|
steps[-1]['this_experiment'] = mine
|
|
steps[-1]['other_findings_outside_experiment'] = scoped(dry, prefix, invert=True)
|
|
check(mine == {'orphan_object': 2, 'stale_pending': 2, 'missing_object': 1, 'orphan_object/skip': 1},
|
|
'detection mismatch: %s' % mine)
|
|
|
|
ap = reconcile('--apply', '--grace', a.grace)
|
|
steps.append({'step': 'apply #1', 'report': summary(ap), 'this_experiment': scoped(ap, prefix),
|
|
'items': [i for i in ap['items'] if i.get('key', '').startswith(prefix)], 'state_after': counts()})
|
|
again = reconcile('--apply', '--grace', a.grace)
|
|
steps.append({'step': 'apply #2 (repeat)', 'report': summary(again), 'this_experiment': scoped(again, prefix),
|
|
'state_after': counts()})
|
|
check(again['fixed'] == 0 and again['errors'] == 0, 'second apply was not a no-op')
|
|
|
|
d1 = c.call('GET', '/api/v1/attachments/' + u1['id'] + '/content', actor=bob)
|
|
msgs = c.call('GET', '/api/v1/chats/' + cid + '/messages?limit=100', actor=bob, schema='MessagePage')['json']['items']
|
|
verify = {
|
|
'U1_used_healthy_download_sha_ok': hashlib.sha256(d1['bytes']).hexdigest() == u1['sha256'],
|
|
'U2_metadata_kept': c.call('GET', '/api/v1/attachments/' + u2['id'], actor=bob)['status'],
|
|
'U2_content_status': c.call('GET', '/api/v1/attachments/' + u2['id'] + '/content', 404, actor=bob)['status'],
|
|
'U2_marked_missing': psql("SELECT missing_at IS NOT NULL FROM attachments WHERE id = '%s'" % u2['id']) == 't',
|
|
'message_still_references_U1_U2': [m for m in msgs if m['id'] == msg['id']][0]['attachment_ids'] == [u1['id'], u2['id']],
|
|
'F1_orphan_deleted': not exists(f['F1_orphan_object_no_row']),
|
|
'F2_orphan_of_lost_row_deleted': not exists(u3_key),
|
|
'F4_row_and_object_deleted': not exists(f['F4_crash_after_put_before_ready'])
|
|
and psql("SELECT count(*) FROM attachments WHERE id = '%s'" % f4_id) == '0',
|
|
'F5_row_deleted': psql("SELECT count(*) FROM attachments WHERE id = '%s'" % f5_id) == '0',
|
|
'F7_fresh_pending_row_kept': psql("SELECT status FROM attachments WHERE id = '%s'" % f7_id) == 'pending',
|
|
'F7_fresh_object_kept': exists(f['F6_F7_fresh_upload_in_progress']),
|
|
'F8_fresh_orphan_kept': exists(f8),
|
|
}
|
|
for k, v in verify.items():
|
|
check(v is True or v in (200, 404), k)
|
|
|
|
# restore U2 from a "backup" (same bytes): the reconciler clears the missing mark
|
|
s3put_bytes(key_of(u2['id']), PNG)
|
|
rec = reconcile('--apply', '--grace', a.grace)
|
|
d2 = c.call('GET', '/api/v1/attachments/' + u2['id'] + '/content', actor=bob)
|
|
verify['U2_restored_and_unmarked'] = (rec['recovered'] == 1 and
|
|
hashlib.sha256(d2['bytes']).hexdigest() == u2['sha256'] and
|
|
psql("SELECT missing_at IS NULL FROM attachments WHERE id = '%s'" % u2['id']) == 't')
|
|
steps.append({'step': 'apply after restoring U2 object', 'report': summary(rec)})
|
|
check(verify['U2_restored_and_unmarked'], 'restore not detected')
|
|
|
|
# default production grace (15m): the same kind of fresh garbage is only reported as skipped
|
|
prod = reconcile('--grace', '15m')
|
|
steps.append({'step': 'dry-run with production grace 15m', 'report': summary(prod)})
|
|
# clean the fresh leftovers with the regular procedure so the next run starts clean
|
|
return {'grace_used': a.grace, 'faults': f, 'steps': steps, 'verify': verify}
|
|
|
|
|
|
def cmd_cost(a):
|
|
c = Client(a.base_url, timeout=30)
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
run = uuid.uuid4().hex[:8]
|
|
keys_file = ROOT / 'evidence/track2/.bench-keys.txt'
|
|
created = 0
|
|
results = []
|
|
try:
|
|
for n in sorted(a.sizes):
|
|
add = n - created
|
|
keys = ['attachments/bench-%s/%07d' % (run, i) for i in range(created, n)]
|
|
psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256,"
|
|
" width, height, object_key, status, created_at, updated_at)"
|
|
" SELECT gen_random_uuid(), '%s', '%s', 'bench.png', 'image/png', 100, repeat('0', 64), 1, 1,"
|
|
" 'attachments/bench-%s/' || lpad(i::text, 7, '0'), 'ready', now() - interval '1 day', now()"
|
|
" FROM generate_series(%d, %d) AS i" % (chat['id'], alice['id'], run, created, n - 1))
|
|
t0 = time.monotonic()
|
|
put = json.loads(subprocess.run(
|
|
['docker', 'run', '--rm', '-i', '--network', NETWORK, '-v', winpath(ROOT / 'secrets') + ':/secrets:ro',
|
|
'-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro', TOOL_IMAGE, 'python', '-I',
|
|
'/tool/s3tool.py', 'bulk-put', '--size', '100', '--workers', '32'],
|
|
input='\n'.join(keys), capture_output=True, text=True).stdout.strip().splitlines()[-1])
|
|
check(put['ok'] == add, 'bulk put incomplete: %s' % put)
|
|
created = n
|
|
print('populated %d (+%d in %.1fs)' % (n, add, time.monotonic() - t0), flush=True)
|
|
for r in range(a.repeat):
|
|
rep = reconcile('--grace', '15m')
|
|
row = {'bench_objects': n, 'repeat': r + 1, 'objects_scanned': rep['objects_scanned'],
|
|
'rows_scanned': rep['rows_scanned'], 'list_ms': rep['list_ms'], 'db_ms': rep['db_ms'],
|
|
'total_ms': rep['total_ms'], 'per_object_us': round(rep['per_object_us'], 1),
|
|
'container_memory_peak_mib': rep['container_memory_peak_mib'],
|
|
'findings': rep['orphan_objects'] + rep['stale_pending'] + rep['missing_objects']}
|
|
print(json.dumps(row), flush=True)
|
|
results.append(row)
|
|
finally:
|
|
if created:
|
|
keys = ['attachments/bench-%s/%07d' % (run, i) for i in range(created)]
|
|
psql("DELETE FROM attachments WHERE object_key LIKE 'attachments/bench-%s/%%'" % run)
|
|
subprocess.run(['docker', 'run', '--rm', '-i', '--network', NETWORK,
|
|
'-v', winpath(ROOT / 'secrets') + ':/secrets:ro',
|
|
'-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro', TOOL_IMAGE, 'python', '-I',
|
|
'/tool/s3tool.py', 'bulk-delete', '--workers', '32'],
|
|
input='\n'.join(keys), capture_output=True, text=True)
|
|
if keys_file.exists():
|
|
keys_file.unlink()
|
|
return {'object_size_bytes': 100, 'reconciler_limits': '0.5 CPU, 128M', 's3': 'SeaweedFS 3.80, 1 CPU, 1G',
|
|
'runs': results}
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument('cmd', choices=['faults', 'cost'])
|
|
p.add_argument('--base-url', default='http://localhost:8080')
|
|
p.add_argument('--grace', default='1m')
|
|
p.add_argument('--wait', type=int, default=65)
|
|
p.add_argument('--sizes', type=int, nargs='+', default=[1000, 5000, 20000])
|
|
p.add_argument('--repeat', type=int, default=3)
|
|
p.add_argument('--output')
|
|
a = p.parse_args()
|
|
sh('docker', 'compose', 'stop', 'reconciler')
|
|
try:
|
|
res = {'command': a.cmd, 'date_utc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
|
|
'commit': sh('git', 'rev-parse', '--short', 'HEAD', check=False),
|
|
'result': (cmd_faults if a.cmd == 'faults' else cmd_cost)(a)}
|
|
finally:
|
|
sh('docker', 'compose', 'start', 'reconciler')
|
|
text = json.dumps(res, ensure_ascii=False, indent=2)
|
|
if a.output:
|
|
Path(a.output).parent.mkdir(parents=True, exist_ok=True)
|
|
Path(a.output).write_text(text + '\n', encoding='utf-8')
|
|
print(text if a.cmd == 'faults' else json.dumps(res['result']['runs'][-1]))
|
|
print('PASS: track2 ' + a.cmd)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|