Files
mtusi2026lab6/student-tests/reconcile_faults.py
T

176 lines
11 KiB
Python

#!/usr/bin/env python3
"""Трек 2: внедрение расхождений PostgreSQL↔S3 и проверка reconciler.
Сценарий (Docker-стенд, reconciler запускается одноразово через docker compose run):
1. Через API: чат, 6 вложений, два из них прикреплены к сообщениям.
2. Внедряем ошибки с обеих сторон:
S3: сирота (объект без строки, старый), сирота свежая (моложе grace),
удалённый объект используемого и неиспользуемого вложения,
объект с подменённым размером;
БД: брошенное намерение с объектом и без, свежее намерение (загрузка «идёт»).
3. dry-run: всё найдено, ничего не изменено (снимки БД и S3 совпадают).
4. apply: исправлено; свежие и используемые не тронуты; API: здоровые вложения
скачиваются байт-в-байт, потерянные отвечают 503, сообщение сохраняет attachment_ids.
5. Повторный apply — ничего не исправляет (идемпотентность).
6. Объект «восстановлен из бэкапа» → apply снимает карантин, скачивание снова работает.
7. После grace: неиспользуемая запись карантина удаляется, используемая остаётся;
свежая сирота, ставшая старой, удаляется.
Grace в тесте — 8 s (--unsafe-short-grace); «старые» строки сдвинуты на 30 минут
назад, поэтому фоновый reconciler стенда (grace 1h) их не трогает.
"""
import json
import time
import uuid
from labutil import BUCKET, compose, fixture, mc, noise_png, object_key, psql, wait_status
from client import Client, arguments, check, run
GRACE = '8s'
def reconcile(*flags):
for _ in range(10):
r = compose('run', '--rm', '--no-deps', '-T', '--entrypoint', '/app/reconcile', 'reconciler',
'--grace', GRACE, '--unsafe-short-grace', *flags, check=False)
if r.returncode == 3: # фоновый reconciler держит advisory lock
time.sleep(2)
continue
check(r.returncode in (0, 4), 'reconcile failed: ' + r.stderr[-400:])
return json.loads(r.stdout.strip().splitlines()[-1])
raise RuntimeError('reconciler lock busy')
def snapshot(chat_id):
rows = psql("SELECT id || ':' || status || ':' || COALESCE(size_bytes, 0) FROM attachments "
"WHERE chat_id = '%s' ORDER BY id" % chat_id)
objs = mc('mc ls --recursive --json s3/%s/a/%s/' % (BUCKET, chat_id)).stdout
# mc ls --json отдаёт key относительно листинга — дополняем до полного ключа.
keys = sorted(('a/%s/%s' % (chat_id, json.loads(l)['key']), json.loads(l)['size'])
for l in objs.splitlines() if l.strip() and 'key' in json.loads(l))
return rows, keys
def main():
def configure(p):
p.add_argument('--output', default=None)
args = arguments('Reconciler fault injection (lab4 track 2)', configure)
c = Client(args.base_url, args.ca_file, timeout=30)
alice, bob, _, chat = c.bootstrap()
cid = chat['id']
def up(data):
a = c.call('POST', '/api/v1/chats/%s/attachments?filename=r.png' % cid, 202, actor=alice,
raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment')['json']
return wait_status(c, alice, a['id'], ('ready',))
datas = [noise_png(20_000 + i * 1000) for i in range(6)]
ok, lost_used, lost_free, resized, keep_used, restore_me = [up(d) for d in datas]
for att in (lost_used, keep_used, restore_me):
c.message(alice, chat, 'с вложением', attachments=[att['id']])
# --- внедрение ошибок ---
tmp = '/tmp/f.bin'
put = 'head -c %d /dev/urandom > ' + tmp + ' && mc cp -q ' + tmp + ' s3/' + BUCKET + '/%s >/dev/null'
orphan_old = 'a/%s/%s' % (cid, uuid.uuid4())
orphan_fresh = 'a/%s/%s' % (cid, uuid.uuid4())
stale_with = str(uuid.uuid4())
stale_without = str(uuid.uuid4())
inflight = str(uuid.uuid4())
mc(' && '.join([
put % (5000, orphan_old),
'mc rm -q s3/%s/%s' % (BUCKET, object_key(lost_used['id'])),
'mc rm -q s3/%s/%s' % (BUCKET, object_key(lost_free['id'])),
'mc rm -q s3/%s/%s' % (BUCKET, object_key(restore_me['id'])),
put % (777, object_key(resized['id'])),
put % (4000, 'a/%s/%s' % (cid, stale_with)),
put % (4000, 'a/%s/%s' % (cid, inflight)),
]))
for aid, age in ((stale_with, "now() - interval '30 minutes'"), (stale_without, "now() - interval '30 minutes'"),
(inflight, 'now()')):
psql("INSERT INTO attachments (id, chat_id, owner_id, filename, object_key, status, created_at, updated_at) "
"VALUES ('%s', '%s', '%s', 'x.png', 'a/%s/%s', 'uploading', %s, %s)" % (aid, cid, alice['id'], cid, aid, age, age))
time.sleep(9) # «старые» объекты стали старше grace
mc(put % (3000, orphan_fresh)) # а эта сирота — свежая
psql("UPDATE attachments SET updated_at = now() WHERE id = '%s'" % inflight) # загрузка «продолжается»
# --- dry-run ---
before = snapshot(cid)
dry = reconcile()
check(snapshot(cid) == before, 'dry-run changed state')
mine = lambda rep, kind: [i for i in rep['items'] if i['kind'] == kind and cid in i['key']]
expect = {'orphan_object': 1, 'stale_intent': 2, 'missing_object': 3, 'size_mismatch': 1}
for kind, n in expect.items():
check(len(mine(dry, kind)) == n, 'dry-run %s: %d, want %d' % (kind, len(mine(dry, kind)), n))
check(all(i['action'].startswith('would_') for i in dry['items'] if cid in i['key']), 'dry-run planned real actions')
# --- apply ---
applied = reconcile('--apply')
rows, keys = snapshot(cid)
status = {r.split(':')[0]: r.split(':')[1] for r in rows.splitlines()}
key_set = {k for k, _ in keys}
check(stale_with not in status and stale_without not in status, 'stale intents not removed')
check('a/%s/%s' % (cid, stale_with) not in key_set, 'stale intent object not removed')
check(orphan_old not in key_set, 'old orphan not removed')
check(orphan_fresh in key_set, 'FRESH orphan was deleted')
check(status.get(inflight) == 'uploading' and 'a/%s/%s' % (cid, inflight) in key_set, 'IN-FLIGHT upload touched')
for att in (lost_used, lost_free, restore_me, resized):
check(status.get(att['id']) == 'missing', '%s not quarantined' % att['id'])
check(status.get(ok['id']) == 'ready' and status.get(keep_used['id']) == 'ready', 'healthy rows touched')
# API после исправления
for att, data in ((ok, datas[0]), (keep_used, datas[4])):
d = c.call('GET', '/api/v1/attachments/%s/content' % att['id'], actor=bob)
check(d['bytes'] == data, 'healthy attachment changed')
c.call('GET', '/api/v1/attachments/%s' % lost_used['id'], 503, actor=bob)
c.call('GET', '/api/v1/attachments/%s/content' % lost_used['id'], 503, actor=bob)
hist = c.call('GET', '/api/v1/chats/%s/messages?limit=100' % cid, actor=bob, schema='MessagePage')['json']['items']
check(any(lost_used['id'] in m['attachment_ids'] for m in hist), 'message lost its attachment reference')
c.call('POST', '/api/v1/chats/%s/messages' % cid, 409, actor=alice, body={'text': '', 'attachment_ids': [lost_free['id']]},
headers={'Idempotency-Key': uuid.uuid4().hex})
# --- идемпотентность ---
again = reconcile('--apply')
check(not [i for i in again['items'] if cid in i['key']], 'second apply changed something: %s' % [i for i in again['items'] if cid in i['key']])
# --- восстановление объекта ---
with_data = datas[5]
open_key = object_key(restore_me['id'])
# Восстанавливаем исходные байты («из бэкапа») через stdin контейнера mc.
import subprocess
p = subprocess.run(['docker', 'compose', 'run', '--rm', '--no-deps', '-T', '--entrypoint', 'sh', 's3-init', '-c',
'mc alias set s3 http://s3:9000 "$(cat /run/secrets/s3_root_user)" "$(cat /run/secrets/s3_root_password)" >/dev/null; '
'cat > /tmp/r.png && mc cp -q /tmp/r.png s3/%s/%s >/dev/null' % (BUCKET, open_key)],
input=with_data, capture_output=True)
check(p.returncode == 0, 'restore put failed')
restored = reconcile('--apply')
check(len(mine(restored, 'restored')) == 1, 'restore not detected')
d = c.call('GET', '/api/v1/attachments/%s/content' % restore_me['id'], actor=bob)
check(d['bytes'] == with_data, 'restored attachment bytes differ')
# --- после grace ---
time.sleep(9)
late = reconcile('--apply')
rows, keys = snapshot(cid)
status = {r.split(':')[0]: r.split(':')[1] for r in rows.splitlines()}
check(lost_free['id'] not in status, 'unreferenced quarantined row not purged after grace')
check(resized['id'] not in status or status[resized['id']] == 'missing', 'resized row state')
check(status.get(lost_used['id']) == 'missing', 'REFERENCED quarantined row deleted')
check(orphan_fresh not in {k for k, _ in keys}, 'orphan not removed after grace')
summary = {
'injected': {'orphan_old': 1, 'orphan_fresh': 1, 'deleted_objects': 3, 'resized_object': 1,
'stale_intents': 2, 'inflight_intent': 1},
'dry_run_found': {k: len(mine(dry, k)) for k in expect}, 'dry_run_state_changed': False,
'apply_found': {k: len(mine(applied, k)) for k in expect},
'second_apply_actions': 0, 'restored': 1,
'after_grace': {k: len(mine(late, k)) for k in ('purge_missing', 'orphan_object')},
'never_deleted': ['fresh orphan (until grace)', 'in-flight intent', 'referenced quarantined row', 'healthy rows'],
'reports': {'dry_run': dry, 'apply': applied, 'second_apply': again, 'restore': restored, 'after_grace': late},
}
text = json.dumps(summary, ensure_ascii=False, indent=2)
if args.output:
open(args.output, 'w', encoding='utf-8').write(text + '\n')
print(json.dumps({k: v for k, v in summary.items() if k != 'reports'}, ensure_ascii=False, indent=2))
print('PASS: reconciler detected and safely fixed faults on both sides; dry-run and repeat are safe')
if __name__ == '__main__':
run(main)