176 lines
11 KiB
Python
176 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Трек 2: внедрение расхождений PostgreSQL↔S3 и проверка reconciler.
|
|
|
|
Сценарий (Docker-стенд, reconciler запускается одноразово через docker compose run):
|
|
1. Через API: чат, 6 вложений, два из них прикреплены к сообщениям.
|
|
2. Внедряем ошибки с обеих сторон:
|
|
S3: сирота (объект без строки, старый), сирота свежая (моложе grace),
|
|
удалённый объект используемого и неиспользуемого вложения,
|
|
объект с подменённым размером;
|
|
БД: брошенное намерение с объектом и без, свежее намерение (загрузка «идёт»).
|
|
3. dry-run: всё найдено, ничего не изменено (снимки БД и S3 совпадают).
|
|
4. apply: исправлено; свежие и используемые не тронуты; API: здоровые вложения
|
|
скачиваются байт-в-байт, потерянные отвечают 503, сообщение сохраняет attachment_ids.
|
|
5. Повторный apply — ничего не исправляет (идемпотентность).
|
|
6. Объект «восстановлен из бэкапа» → apply снимает карантин, скачивание снова работает.
|
|
7. После grace: неиспользуемая запись карантина удаляется, используемая остаётся;
|
|
свежая сирота, ставшая старой, удаляется.
|
|
|
|
Grace в тесте — 8 s (--unsafe-short-grace); «старые» строки сдвинуты на 30 минут
|
|
назад, поэтому фоновый reconciler стенда (grace 1h) их не трогает.
|
|
"""
|
|
import json
|
|
import time
|
|
import uuid
|
|
from labutil import BUCKET, compose, fixture, mc, noise_png, object_key, psql, wait_status
|
|
from client import Client, arguments, check, run
|
|
|
|
GRACE = '8s'
|
|
|
|
|
|
def reconcile(*flags):
|
|
for _ in range(10):
|
|
r = compose('run', '--rm', '--no-deps', '-T', '--entrypoint', '/app/reconcile', 'reconciler',
|
|
'--grace', GRACE, '--unsafe-short-grace', *flags, check=False)
|
|
if r.returncode == 3: # фоновый reconciler держит advisory lock
|
|
time.sleep(2)
|
|
continue
|
|
check(r.returncode in (0, 4), 'reconcile failed: ' + r.stderr[-400:])
|
|
return json.loads(r.stdout.strip().splitlines()[-1])
|
|
raise RuntimeError('reconciler lock busy')
|
|
|
|
|
|
def snapshot(chat_id):
|
|
rows = psql("SELECT id || ':' || status || ':' || COALESCE(size_bytes, 0) FROM attachments "
|
|
"WHERE chat_id = '%s' ORDER BY id" % chat_id)
|
|
objs = mc('mc ls --recursive --json s3/%s/a/%s/' % (BUCKET, chat_id)).stdout
|
|
# mc ls --json отдаёт key относительно листинга — дополняем до полного ключа.
|
|
keys = sorted(('a/%s/%s' % (chat_id, json.loads(l)['key']), json.loads(l)['size'])
|
|
for l in objs.splitlines() if l.strip() and 'key' in json.loads(l))
|
|
return rows, keys
|
|
|
|
|
|
def main():
|
|
def configure(p):
|
|
p.add_argument('--output', default=None)
|
|
args = arguments('Reconciler fault injection (lab4 track 2)', configure)
|
|
c = Client(args.base_url, args.ca_file, timeout=30)
|
|
alice, bob, _, chat = c.bootstrap()
|
|
cid = chat['id']
|
|
def up(data):
|
|
a = c.call('POST', '/api/v1/chats/%s/attachments?filename=r.png' % cid, 202, actor=alice,
|
|
raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment')['json']
|
|
return wait_status(c, alice, a['id'], ('ready',))
|
|
datas = [noise_png(20_000 + i * 1000) for i in range(6)]
|
|
ok, lost_used, lost_free, resized, keep_used, restore_me = [up(d) for d in datas]
|
|
for att in (lost_used, keep_used, restore_me):
|
|
c.message(alice, chat, 'с вложением', attachments=[att['id']])
|
|
|
|
# --- внедрение ошибок ---
|
|
tmp = '/tmp/f.bin'
|
|
put = 'head -c %d /dev/urandom > ' + tmp + ' && mc cp -q ' + tmp + ' s3/' + BUCKET + '/%s >/dev/null'
|
|
orphan_old = 'a/%s/%s' % (cid, uuid.uuid4())
|
|
orphan_fresh = 'a/%s/%s' % (cid, uuid.uuid4())
|
|
stale_with = str(uuid.uuid4())
|
|
stale_without = str(uuid.uuid4())
|
|
inflight = str(uuid.uuid4())
|
|
mc(' && '.join([
|
|
put % (5000, orphan_old),
|
|
'mc rm -q s3/%s/%s' % (BUCKET, object_key(lost_used['id'])),
|
|
'mc rm -q s3/%s/%s' % (BUCKET, object_key(lost_free['id'])),
|
|
'mc rm -q s3/%s/%s' % (BUCKET, object_key(restore_me['id'])),
|
|
put % (777, object_key(resized['id'])),
|
|
put % (4000, 'a/%s/%s' % (cid, stale_with)),
|
|
put % (4000, 'a/%s/%s' % (cid, inflight)),
|
|
]))
|
|
for aid, age in ((stale_with, "now() - interval '30 minutes'"), (stale_without, "now() - interval '30 minutes'"),
|
|
(inflight, 'now()')):
|
|
psql("INSERT INTO attachments (id, chat_id, owner_id, filename, object_key, status, created_at, updated_at) "
|
|
"VALUES ('%s', '%s', '%s', 'x.png', 'a/%s/%s', 'uploading', %s, %s)" % (aid, cid, alice['id'], cid, aid, age, age))
|
|
time.sleep(9) # «старые» объекты стали старше grace
|
|
mc(put % (3000, orphan_fresh)) # а эта сирота — свежая
|
|
psql("UPDATE attachments SET updated_at = now() WHERE id = '%s'" % inflight) # загрузка «продолжается»
|
|
|
|
# --- dry-run ---
|
|
before = snapshot(cid)
|
|
dry = reconcile()
|
|
check(snapshot(cid) == before, 'dry-run changed state')
|
|
mine = lambda rep, kind: [i for i in rep['items'] if i['kind'] == kind and cid in i['key']]
|
|
expect = {'orphan_object': 1, 'stale_intent': 2, 'missing_object': 3, 'size_mismatch': 1}
|
|
for kind, n in expect.items():
|
|
check(len(mine(dry, kind)) == n, 'dry-run %s: %d, want %d' % (kind, len(mine(dry, kind)), n))
|
|
check(all(i['action'].startswith('would_') for i in dry['items'] if cid in i['key']), 'dry-run planned real actions')
|
|
|
|
# --- apply ---
|
|
applied = reconcile('--apply')
|
|
rows, keys = snapshot(cid)
|
|
status = {r.split(':')[0]: r.split(':')[1] for r in rows.splitlines()}
|
|
key_set = {k for k, _ in keys}
|
|
check(stale_with not in status and stale_without not in status, 'stale intents not removed')
|
|
check('a/%s/%s' % (cid, stale_with) not in key_set, 'stale intent object not removed')
|
|
check(orphan_old not in key_set, 'old orphan not removed')
|
|
check(orphan_fresh in key_set, 'FRESH orphan was deleted')
|
|
check(status.get(inflight) == 'uploading' and 'a/%s/%s' % (cid, inflight) in key_set, 'IN-FLIGHT upload touched')
|
|
for att in (lost_used, lost_free, restore_me, resized):
|
|
check(status.get(att['id']) == 'missing', '%s not quarantined' % att['id'])
|
|
check(status.get(ok['id']) == 'ready' and status.get(keep_used['id']) == 'ready', 'healthy rows touched')
|
|
# API после исправления
|
|
for att, data in ((ok, datas[0]), (keep_used, datas[4])):
|
|
d = c.call('GET', '/api/v1/attachments/%s/content' % att['id'], actor=bob)
|
|
check(d['bytes'] == data, 'healthy attachment changed')
|
|
c.call('GET', '/api/v1/attachments/%s' % lost_used['id'], 503, actor=bob)
|
|
c.call('GET', '/api/v1/attachments/%s/content' % lost_used['id'], 503, actor=bob)
|
|
hist = c.call('GET', '/api/v1/chats/%s/messages?limit=100' % cid, actor=bob, schema='MessagePage')['json']['items']
|
|
check(any(lost_used['id'] in m['attachment_ids'] for m in hist), 'message lost its attachment reference')
|
|
c.call('POST', '/api/v1/chats/%s/messages' % cid, 409, actor=alice, body={'text': '', 'attachment_ids': [lost_free['id']]},
|
|
headers={'Idempotency-Key': uuid.uuid4().hex})
|
|
|
|
# --- идемпотентность ---
|
|
again = reconcile('--apply')
|
|
check(not [i for i in again['items'] if cid in i['key']], 'second apply changed something: %s' % [i for i in again['items'] if cid in i['key']])
|
|
|
|
# --- восстановление объекта ---
|
|
with_data = datas[5]
|
|
open_key = object_key(restore_me['id'])
|
|
# Восстанавливаем исходные байты («из бэкапа») через stdin контейнера mc.
|
|
import subprocess
|
|
p = subprocess.run(['docker', 'compose', 'run', '--rm', '--no-deps', '-T', '--entrypoint', 'sh', 's3-init', '-c',
|
|
'mc alias set s3 http://s3:9000 "$(cat /run/secrets/s3_root_user)" "$(cat /run/secrets/s3_root_password)" >/dev/null; '
|
|
'cat > /tmp/r.png && mc cp -q /tmp/r.png s3/%s/%s >/dev/null' % (BUCKET, open_key)],
|
|
input=with_data, capture_output=True)
|
|
check(p.returncode == 0, 'restore put failed')
|
|
restored = reconcile('--apply')
|
|
check(len(mine(restored, 'restored')) == 1, 'restore not detected')
|
|
d = c.call('GET', '/api/v1/attachments/%s/content' % restore_me['id'], actor=bob)
|
|
check(d['bytes'] == with_data, 'restored attachment bytes differ')
|
|
|
|
# --- после grace ---
|
|
time.sleep(9)
|
|
late = reconcile('--apply')
|
|
rows, keys = snapshot(cid)
|
|
status = {r.split(':')[0]: r.split(':')[1] for r in rows.splitlines()}
|
|
check(lost_free['id'] not in status, 'unreferenced quarantined row not purged after grace')
|
|
check(resized['id'] not in status or status[resized['id']] == 'missing', 'resized row state')
|
|
check(status.get(lost_used['id']) == 'missing', 'REFERENCED quarantined row deleted')
|
|
check(orphan_fresh not in {k for k, _ in keys}, 'orphan not removed after grace')
|
|
|
|
summary = {
|
|
'injected': {'orphan_old': 1, 'orphan_fresh': 1, 'deleted_objects': 3, 'resized_object': 1,
|
|
'stale_intents': 2, 'inflight_intent': 1},
|
|
'dry_run_found': {k: len(mine(dry, k)) for k in expect}, 'dry_run_state_changed': False,
|
|
'apply_found': {k: len(mine(applied, k)) for k in expect},
|
|
'second_apply_actions': 0, 'restored': 1,
|
|
'after_grace': {k: len(mine(late, k)) for k in ('purge_missing', 'orphan_object')},
|
|
'never_deleted': ['fresh orphan (until grace)', 'in-flight intent', 'referenced quarantined row', 'healthy rows'],
|
|
'reports': {'dry_run': dry, 'apply': applied, 'second_apply': again, 'restore': restored, 'after_grace': late},
|
|
}
|
|
text = json.dumps(summary, ensure_ascii=False, indent=2)
|
|
if args.output:
|
|
open(args.output, 'w', encoding='utf-8').write(text + '\n')
|
|
print(json.dumps({k: v for k, v in summary.items() if k != 'reports'}, ensure_ascii=False, indent=2))
|
|
print('PASS: reconciler detected and safely fixed faults on both sides; dry-run and repeat are safe')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
run(main)
|