Files
mtusi2026lab4/scripts/track2/run_track2.py
T
2026-10-29 15:00:36 +03:00

255 lines
13 KiB
Python

#!/usr/bin/env python3
"""Track 2: DB<->S3 consistency. Fault injection on both sides + reconciler cost.
python scripts/track2/run_track2.py faults --output evidence/track2/faults.json
python scripts/track2/run_track2.py cost --sizes 1000 5000 20000 --repeat 3 --output evidence/track2/cost.json
The periodic `reconciler` service is stopped for the duration (it would race the
experiment) and started again at the end. Synthetic data only.
"""
import argparse
import hashlib
import json
from pathlib import Path
import subprocess
import sys
import time
import uuid
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / 'scripts'))
from lab4ops import NETWORK, PNG, TOOL_IMAGE, psql, s3tool, upload, winpath # noqa: E402
from lablib import sh # noqa: E402
from client import Client, check # noqa: E402
def s3put_bytes(key, data):
cmd = ['docker', 'run', '--rm', '-i', '--network', NETWORK,
'-v', winpath(ROOT / 'secrets') + ':/secrets:ro', '-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro',
TOOL_IMAGE, 'python', '-I', '/tool/s3tool.py', 'put', key, '--stdin']
r = subprocess.run(cmd, input=data, capture_output=True)
check(r.returncode == 0, 's3 put failed')
return json.loads(r.stdout.decode().strip().splitlines()[-1])
def reconcile(*args):
"""One reconciler pass in a throwaway container; also returns its cgroup memory peak."""
out = sh('docker', 'compose', 'run', '--rm', '--no-deps', '-T', '--entrypoint', 'sh', 'reconciler', '-c',
'/app/server reconcile ' + ' '.join(args) + ' > /tmp/r.json && cat /tmp/r.json '
'&& echo && cat /sys/fs/cgroup/memory.peak')
lines = [x for x in out.splitlines() if x.strip()]
rep = json.loads(lines[-2])
rep['container_memory_peak_mib'] = round(int(lines[-1]) / 2**20, 1)
return rep
def counts(prefix='attachments/'):
return {'s3_objects': s3tool('count', prefix)['count'],
'db_rows': int(psql('SELECT count(*) FROM attachments')),
'db_pending': int(psql("SELECT count(*) FROM attachments WHERE status = 'pending'")),
'db_missing_marked': int(psql('SELECT count(*) FROM attachments WHERE missing_at IS NOT NULL'))}
def summary(rep):
keys = ('mode', 'objects_scanned', 'rows_scanned', 'orphan_objects', 'fresh_skipped', 'stale_pending',
'missing_objects', 'size_mismatch', 'recovered', 'fixed', 'errors', 'total_ms')
return {k: rep[k] for k in keys}
def scoped(rep, prefix, invert=False):
out = {}
for i in rep['items']:
if i.get('key', '').startswith(prefix) == invert:
continue
k = i['kind'] + ('/skip' if i['action'].startswith('skip') else '')
out[k] = out.get(k, 0) + 1
return out
def insert_pending(att_id, chat, owner, key, age):
psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height,"
" object_key, status, created_at, updated_at) VALUES ('%s', '%s', '%s', 'crash.png', 'image/png', 100,"
" repeat('0', 64), 1, 1, '%s', 'pending', now() - interval '%s', now() - interval '%s')"
% (att_id, chat, owner, key, age, age))
def exists(key):
return s3tool('get', key)['status'] == 200
def cmd_faults(a):
c = Client(a.base_url, timeout=30)
alice, bob, eve, chat = c.bootstrap()
cid, prefix = chat['id'], 'attachments/' + chat['id'] + '/'
key_of = lambda att_id: psql("SELECT object_key FROM attachments WHERE id = '%s'" % att_id)
# healthy, referenced attachment (must survive everything)
u1 = upload(c, alice, chat)['json']
# referenced attachment whose object will be lost (S3-side fault)
u2 = upload(c, alice, chat)['json']
msg = c.message(alice, chat, '', attachments=[u1['id'], u2['id']])['json']
# unreferenced ready attachment whose metadata row will be lost (DB-side fault)
u3 = upload(c, alice, chat)['json']
u3_key = key_of(u3['id'])
f = {}
f['F1_orphan_object_no_row'] = prefix + str(uuid.uuid4())
s3tool('put', f['F1_orphan_object_no_row'])
psql("DELETE FROM attachments WHERE id = '%s'" % u3['id'])
f['F2_metadata_row_lost(U3)'] = u3_key
s3tool('delete', key_of(u2['id']))
f['F3_object_lost_of_used(U2)'] = key_of(u2['id'])
f4_id = str(uuid.uuid4())
f['F4_crash_after_put_before_ready'] = prefix + f4_id
insert_pending(f4_id, cid, alice['id'], f['F4_crash_after_put_before_ready'], '1 hour')
s3tool('put', f['F4_crash_after_put_before_ready'])
f5_id = str(uuid.uuid4())
f['F5_crash_before_put'] = prefix + f5_id
insert_pending(f5_id, cid, alice['id'], f['F5_crash_before_put'], '1 hour')
print('waiting %ds so that injected objects are older than grace=%s' % (a.wait, a.grace), flush=True)
time.sleep(a.wait)
# fresh state that must NOT be touched: an object being uploaded right now and its pending row
f7_id = str(uuid.uuid4())
f['F6_F7_fresh_upload_in_progress'] = prefix + f7_id
insert_pending(f7_id, cid, alice['id'], f['F6_F7_fresh_upload_in_progress'], '0 seconds')
s3tool('put', f['F6_F7_fresh_upload_in_progress'])
f8 = prefix + str(uuid.uuid4())
f['F8_fresh_orphan_object'] = f8
s3tool('put', f8)
steps = []
before = counts()
dry = reconcile('--grace', a.grace)
after_dry = counts()
steps.append({'step': 'dry-run', 'report': summary(dry), 'items': dry['items'], 'state_before': before,
'state_after': after_dry, 'dry_run_changed_nothing': before == after_dry})
check(before == after_dry, 'dry-run modified state')
mine = scoped(dry, prefix)
steps[-1]['this_experiment'] = mine
steps[-1]['other_findings_outside_experiment'] = scoped(dry, prefix, invert=True)
check(mine == {'orphan_object': 2, 'stale_pending': 2, 'missing_object': 1, 'orphan_object/skip': 1},
'detection mismatch: %s' % mine)
ap = reconcile('--apply', '--grace', a.grace)
steps.append({'step': 'apply #1', 'report': summary(ap), 'this_experiment': scoped(ap, prefix),
'items': [i for i in ap['items'] if i.get('key', '').startswith(prefix)], 'state_after': counts()})
again = reconcile('--apply', '--grace', a.grace)
steps.append({'step': 'apply #2 (repeat)', 'report': summary(again), 'this_experiment': scoped(again, prefix),
'state_after': counts()})
check(again['fixed'] == 0 and again['errors'] == 0, 'second apply was not a no-op')
d1 = c.call('GET', '/api/v1/attachments/' + u1['id'] + '/content', actor=bob)
msgs = c.call('GET', '/api/v1/chats/' + cid + '/messages?limit=100', actor=bob, schema='MessagePage')['json']['items']
verify = {
'U1_used_healthy_download_sha_ok': hashlib.sha256(d1['bytes']).hexdigest() == u1['sha256'],
'U2_metadata_kept': c.call('GET', '/api/v1/attachments/' + u2['id'], actor=bob)['status'],
'U2_content_status': c.call('GET', '/api/v1/attachments/' + u2['id'] + '/content', 404, actor=bob)['status'],
'U2_marked_missing': psql("SELECT missing_at IS NOT NULL FROM attachments WHERE id = '%s'" % u2['id']) == 't',
'message_still_references_U1_U2': [m for m in msgs if m['id'] == msg['id']][0]['attachment_ids'] == [u1['id'], u2['id']],
'F1_orphan_deleted': not exists(f['F1_orphan_object_no_row']),
'F2_orphan_of_lost_row_deleted': not exists(u3_key),
'F4_row_and_object_deleted': not exists(f['F4_crash_after_put_before_ready'])
and psql("SELECT count(*) FROM attachments WHERE id = '%s'" % f4_id) == '0',
'F5_row_deleted': psql("SELECT count(*) FROM attachments WHERE id = '%s'" % f5_id) == '0',
'F7_fresh_pending_row_kept': psql("SELECT status FROM attachments WHERE id = '%s'" % f7_id) == 'pending',
'F7_fresh_object_kept': exists(f['F6_F7_fresh_upload_in_progress']),
'F8_fresh_orphan_kept': exists(f8),
}
for k, v in verify.items():
check(v is True or v in (200, 404), k)
# restore U2 from a "backup" (same bytes): the reconciler clears the missing mark
s3put_bytes(key_of(u2['id']), PNG)
rec = reconcile('--apply', '--grace', a.grace)
d2 = c.call('GET', '/api/v1/attachments/' + u2['id'] + '/content', actor=bob)
verify['U2_restored_and_unmarked'] = (rec['recovered'] == 1 and
hashlib.sha256(d2['bytes']).hexdigest() == u2['sha256'] and
psql("SELECT missing_at IS NULL FROM attachments WHERE id = '%s'" % u2['id']) == 't')
steps.append({'step': 'apply after restoring U2 object', 'report': summary(rec)})
check(verify['U2_restored_and_unmarked'], 'restore not detected')
# default production grace (15m): the same kind of fresh garbage is only reported as skipped
prod = reconcile('--grace', '15m')
steps.append({'step': 'dry-run with production grace 15m', 'report': summary(prod)})
# clean the fresh leftovers with the regular procedure so the next run starts clean
return {'grace_used': a.grace, 'faults': f, 'steps': steps, 'verify': verify}
def cmd_cost(a):
c = Client(a.base_url, timeout=30)
alice, bob, eve, chat = c.bootstrap()
run = uuid.uuid4().hex[:8]
keys_file = ROOT / 'evidence/track2/.bench-keys.txt'
created = 0
results = []
try:
for n in sorted(a.sizes):
add = n - created
keys = ['attachments/bench-%s/%07d' % (run, i) for i in range(created, n)]
psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256,"
" width, height, object_key, status, created_at, updated_at)"
" SELECT gen_random_uuid(), '%s', '%s', 'bench.png', 'image/png', 100, repeat('0', 64), 1, 1,"
" 'attachments/bench-%s/' || lpad(i::text, 7, '0'), 'ready', now() - interval '1 day', now()"
" FROM generate_series(%d, %d) AS i" % (chat['id'], alice['id'], run, created, n - 1))
t0 = time.monotonic()
put = json.loads(subprocess.run(
['docker', 'run', '--rm', '-i', '--network', NETWORK, '-v', winpath(ROOT / 'secrets') + ':/secrets:ro',
'-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro', TOOL_IMAGE, 'python', '-I',
'/tool/s3tool.py', 'bulk-put', '--size', '100', '--workers', '32'],
input='\n'.join(keys), capture_output=True, text=True).stdout.strip().splitlines()[-1])
check(put['ok'] == add, 'bulk put incomplete: %s' % put)
created = n
print('populated %d (+%d in %.1fs)' % (n, add, time.monotonic() - t0), flush=True)
for r in range(a.repeat):
rep = reconcile('--grace', '15m')
row = {'bench_objects': n, 'repeat': r + 1, 'objects_scanned': rep['objects_scanned'],
'rows_scanned': rep['rows_scanned'], 'list_ms': rep['list_ms'], 'db_ms': rep['db_ms'],
'total_ms': rep['total_ms'], 'per_object_us': round(rep['per_object_us'], 1),
'container_memory_peak_mib': rep['container_memory_peak_mib'],
'findings': rep['orphan_objects'] + rep['stale_pending'] + rep['missing_objects']}
print(json.dumps(row), flush=True)
results.append(row)
finally:
if created:
keys = ['attachments/bench-%s/%07d' % (run, i) for i in range(created)]
psql("DELETE FROM attachments WHERE object_key LIKE 'attachments/bench-%s/%%'" % run)
subprocess.run(['docker', 'run', '--rm', '-i', '--network', NETWORK,
'-v', winpath(ROOT / 'secrets') + ':/secrets:ro',
'-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro', TOOL_IMAGE, 'python', '-I',
'/tool/s3tool.py', 'bulk-delete', '--workers', '32'],
input='\n'.join(keys), capture_output=True, text=True)
if keys_file.exists():
keys_file.unlink()
return {'object_size_bytes': 100, 'reconciler_limits': '0.5 CPU, 128M', 's3': 'SeaweedFS 3.80, 1 CPU, 1G',
'runs': results}
def main():
p = argparse.ArgumentParser()
p.add_argument('cmd', choices=['faults', 'cost'])
p.add_argument('--base-url', default='http://localhost:8080')
p.add_argument('--grace', default='1m')
p.add_argument('--wait', type=int, default=65)
p.add_argument('--sizes', type=int, nargs='+', default=[1000, 5000, 20000])
p.add_argument('--repeat', type=int, default=3)
p.add_argument('--output')
a = p.parse_args()
sh('docker', 'compose', 'stop', 'reconciler')
try:
res = {'command': a.cmd, 'date_utc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
'commit': sh('git', 'rev-parse', '--short', 'HEAD', check=False),
'result': (cmd_faults if a.cmd == 'faults' else cmd_cost)(a)}
finally:
sh('docker', 'compose', 'start', 'reconciler')
text = json.dumps(res, ensure_ascii=False, indent=2)
if a.output:
Path(a.output).parent.mkdir(parents=True, exist_ok=True)
Path(a.output).write_text(text + '\n', encoding='utf-8')
print(text if a.cmd == 'faults' else json.dumps(res['result']['runs'][-1]))
print('PASS: track2 ' + a.cmd)
if __name__ == '__main__':
main()