Files
mtusi2026lab6/student-tests/attachments_ops.py
T

285 lines
14 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Операционные сценарии вложений на Docker-стенде (уровни 3–4).
failover upload → остановить обслужившую реплику → download через другую
s3-outage остановить S3: загрузка/скачивание 503, текстовый чат и readiness работают
interrupt обрыв клиента посреди тела: ни ready-вложения, ни намерения, ни объекта
db-gap PUT в S3 прошёл, PostgreSQL недоступен на MarkReady: объект удалён
компенсацией, намерение осталось → убирает reconciler
security bucket приватен (анонимно 403), у API-ключа нет прав вне bucket,
bootstrap идемпотентен, консоль/порты S3 не опубликованы
memory N одновременных загрузок ~9.8 MiB: пик памяти (cgroup memory.peak) реплик
Каждый сценарий печатает JSON-итог и PASS/FAIL.
"""
import hashlib
import json
import threading
import time
import uuid
from labutil import (BUCKET, api_container, bucket_count, compose, fixture, mc, noise_png, object_key, psql,
raw_post, sh, wait_status)
from client import Client, Failure, arguments, check, run
def upload(c, actor, chat, data, name='f.png', ctype='image/png', expected=202):
return c.call('POST', '/api/v1/chats/%s/attachments?filename=%s' % (chat['id'], name), expected,
actor=actor, raw=data, headers={'Content-Type': ctype},
schema='Attachment' if expected == 202 else None)
def wait_ready(c, timeout=90):
end = time.time() + timeout
while time.time() < end:
try:
c.call('GET', '/health/ready')
return
except Failure:
time.sleep(1)
raise Failure('stack not ready')
def scenario_failover(c):
alice, bob, _, chat = c.bootstrap()
data = noise_png(2 * 1024 * 1024)
r = upload(c, alice, chat, data)
att, served_by = r['json'], r['headers']['X-Instance-Id']
wait_status(c, bob, att['id'], ('ready',))
cid = api_container(served_by)
sh('docker', 'stop', '-t', '15', cid)
try:
seen = set()
for _ in range(6):
d = c.call('GET', '/api/v1/attachments/%s/content' % att['id'], actor=bob)
check(d['bytes'] == data, 'downloaded bytes differ after failover')
seen.add(d['headers']['X-Instance-Id'])
check(served_by not in seen, 'stopped replica still serving')
finally:
sh('docker', 'start', cid)
sh('sh', 'scripts/restore.sh')
return {'uploaded_by': served_by, 'downloaded_by': sorted(seen), 'sha256': hashlib.sha256(data).hexdigest(),
'stopped_container': cid[:12]}
def scenario_s3_outage(c):
alice, bob, _, chat = c.bootstrap()
small = fixture('small.png')
att = upload(c, alice, chat, small)['json']
wait_status(c, bob, att['id'], ('ready',))
compose('stop', 's3')
try:
t0 = time.time()
up = upload(c, alice, chat, small, expected=503)
up_s = time.time() - t0
c.call('GET', '/api/v1/attachments/%s/content' % att['id'], 503, actor=bob)
meta = c.call('GET', '/api/v1/attachments/%s' % att['id'], actor=bob, schema='Attachment')['json']
msg = c.message(alice, chat, 'S3 лежит, а чат работает')['json']
c.call('GET', '/health/ready', schema='Health')
leftovers = psql("SELECT count(*) FROM attachments WHERE chat_id = '%s' AND status = 'uploading'" % chat['id'])
finally:
compose('start', 's3')
compose('run', '--rm', 's3-init')
for _ in range(30):
try:
again = upload(c, alice, chat, small)['json']
break
except Failure:
time.sleep(1)
else:
raise Failure('upload did not recover after S3 start')
d = c.call('GET', '/api/v1/attachments/%s/content' % att['id'], actor=bob)
check(d['bytes'] == small, 'old attachment changed after S3 restart')
check(leftovers == '0', 'intent rows left after failed upload: ' + leftovers)
return {'upload_status_during_outage': up['status'], 'upload_error_seconds': round(up_s, 2),
'download_status_during_outage': 503, 'metadata_during_outage': meta['status'],
'text_message_during_outage': 201, 'readiness_during_outage': 200,
'intents_left': int(leftovers), 'upload_after_recovery': again['status']}
def scenario_interrupt(c):
alice, _, _, chat = c.bootstrap()
data = noise_png(4 * 1024 * 1024)
hdr = {'Authorization': 'Bearer ' + alice['token'], 'Content-Type': 'image/png',
'Content-Length': str(len(data))}
# Отправляем 1 MiB из 4 и обрываем соединение.
import socket
from urllib.parse import urlsplit
u = urlsplit(c.base)
s = socket.create_connection((u.hostname, u.port or 80))
head = 'POST /api/v1/chats/%s/attachments?filename=cut.png HTTP/1.1\r\nHost: %s\r\n' % (chat['id'], u.netloc)
s.sendall((head + ''.join('%s: %s\r\n' % kv for kv in hdr.items()) + '\r\n').encode())
s.sendall(data[:1024 * 1024])
time.sleep(1)
s.close()
time.sleep(3)
rows = psql("SELECT status FROM attachments WHERE chat_id = '%s'" % chat['id'])
objs = str(sum(bucket_count(p + chat['id'] + '/') for p in ('s/', 'o/', 'v/')))
check(rows == '', 'rows left after interrupted upload: %r' % rows)
check(objs == '0', 'objects left after interrupted upload: ' + objs)
return {'sent_bytes': 1024 * 1024, 'declared_bytes': len(data), 'rows_after': 0, 'objects_after': 0}
def scenario_db_gap(c):
"""PUT прошёл, а PostgreSQL «пропал» до MarkReady (docker pause)."""
alice, _, _, chat = c.bootstrap()
data = noise_png(3 * 1024 * 1024)
hdr = {'Authorization': 'Bearer ' + alice['token'], 'Content-Type': 'image/png', 'Content-Length': str(len(data))}
state = {}
def body():
yield data[:512 * 1024] # намерение уже записано, PUT идёт
time.sleep(2)
compose('pause', 'postgres') # БД «пропала» до MarkReady
state['paused_at'] = time.time()
yield data[512 * 1024:]
t0 = time.time()
try:
st, _ = raw_post(c.base, '/api/v1/chats/%s/attachments?filename=gap.png' % chat['id'], hdr, body(), read_timeout=60)
finally:
# Дольше таймаута запроса (30 s) и компенсации (10 s): DeleteIntent тоже не пройдёт.
time.sleep(max(0, 45 - (time.time() - state.get('paused_at', time.time()))))
compose('unpause', 'postgres')
wait_ready(c)
time.sleep(3) # отложенный UPDATE (если был) успевает закоммититься после unpause
rows = psql("SELECT id || ' ' || status FROM attachments WHERE chat_id = '%s'" % chat['id'])
staged = bucket_count('s/' + chat['id'] + '/')
status_after = rows.split()[-1] if rows else 'none'
check(st != 202, 'upload must not report success when the job was not committed (got %s)' % st)
# Инвариант: нет строки queued/ready без исходника.
check(not (status_after in ('queued', 'processing') and staged == 0 and bucket_count('o/' + chat['id'] + '/') == 0),
'job without source: %r' % rows)
if status_after in ('queued', 'processing', 'ready'):
# COMMIT всё-таки прошёл: pipeline доводит работу до ready.
aid = rows.split()[0]
for _ in range(120):
if psql("SELECT status FROM attachments WHERE id='%s'" % aid) == 'ready':
break
time.sleep(1)
else:
psql("UPDATE attachments SET updated_at = now() - interval '30 minutes' WHERE chat_id = '%s'" % chat['id'])
for _ in range(10):
r = compose('run', '--rm', '--no-deps', '-T', '--entrypoint', '/app/reconcile', 'reconciler',
'--apply', '--grace', '10m', check=False)
if r.returncode != 3:
break
time.sleep(2)
check(r.returncode in (0, 4), 'reconcile failed: ' + r.stderr[-300:])
final = psql("SELECT status FROM attachments WHERE chat_id = '%s'" % chat['id'])
check(final in ('ready', ''), 'inconsistent final state: %r' % final)
return {'client_status': st, 'seconds': round(time.time() - t0, 1),
'after_gap': {'row_status': status_after, 'staging_objects': staged},
'final': {'row_status': final or 'none'}}
def scenario_security(c):
alice, _, _, chat = c.bootstrap()
att = upload(c, alice, chat, fixture('small.png'))['json']
# С lab5 worker переносит оригинал staging s/ → o/: ключ берём после ready.
wait_status(c, alice, att['id'], ('ready',))
key = object_key(att['id'])
url = 'http://s3:9000/%s/%s' % (BUCKET, key)
# Анонимно изнутри сети стенда (из контейнера API): объект и листинг закрыты.
anon_get = compose('exec', '-T', 'api', 'wget', '-q', '-S', '-O', '/dev/null', url, check=False)
anon_list = compose('exec', '-T', 'api', 'wget', '-q', '-S', '-O', '/dev/null',
'http://s3:9000/%s?list-type=2' % BUCKET, check=False)
check(anon_get.returncode != 0 and '403' in anon_get.stderr, 'anonymous GET must be 403')
check(anon_list.returncode != 0 and '403' in anon_list.stderr, 'anonymous LIST must be 403')
# Ключ API: свой bucket — да, остальное — нет.
own = mc('mc stat s3/%s/%s >/dev/null && echo ok' % (BUCKET, key), as_user='api').stdout.strip()
mb = mc('mc mb s3/evil-bucket-%s' % uuid.uuid4().hex[:6], check=False, as_user='api')
admin = mc('mc admin user list s3', check=False, as_user='api')
buckets = mc('mc ls s3', check=False, as_user='api')
check(own == 'ok', 'api key cannot read own bucket')
check(mb.returncode != 0, 'api key must not create buckets')
check(admin.returncode != 0, 'api key must not use admin API')
check(buckets.returncode != 0 or 'evil' not in buckets.stdout, 'api key sees a bucket it created?!')
# Bootstrap идемпотентен: повторный запуск успешен, объект на месте.
before = mc('mc ls --recursive s3/%s | wc -l' % BUCKET).stdout.strip()
r1 = compose('run', '--rm', 's3-init')
r2 = compose('run', '--rm', 's3-init')
after = mc('mc ls --recursive s3/%s | wc -l' % BUCKET).stdout.strip()
check(before == after, 'bootstrap changed objects')
ports = compose('ps', 's3', '--format', '{{.Ports}}').stdout.strip()
check('0.0.0.0' not in ports and '127.0.0.1' not in ports, 's3 port is published: ' + ports)
return {'anonymous_get': '403', 'anonymous_list': '403', 'api_key_own_bucket': 'allowed',
'api_key_make_bucket': 'denied', 'api_key_admin': 'denied',
'api_key_visible_buckets': sorted(l.split()[-1].rstrip('/') for l in buckets.stdout.splitlines() if l.strip())
if buckets.returncode == 0 else 'denied',
'bootstrap_runs': [r1.returncode, r2.returncode], 'objects_before_after': [int(before), int(after)],
's3_published_ports': ports or 'none'}
def read_peak(cid):
r = sh('docker', 'exec', cid, 'cat', '/sys/fs/cgroup/memory.peak', check=False)
return int(r.stdout.strip()) if r.returncode == 0 and r.stdout.strip().isdigit() else None
def read_current(cid):
r = sh('docker', 'exec', cid, 'cat', '/sys/fs/cgroup/memory.current', check=False)
return int(r.stdout.strip()) if r.returncode == 0 and r.stdout.strip().isdigit() else None
def scenario_memory(c, parallel_levels=(10, 20)):
alice, _, _, chat = c.bootstrap()
data = noise_png(10 * 1024 * 1024 - 300_000)
out = {'file_bytes': len(data), 'runs': []}
for n in parallel_levels:
compose('restart', 'api') # новый cgroup → memory.peak с нуля
sh('sh', 'scripts/restore.sh')
wait_ready(c)
cids = compose('ps', '-q', 'api').stdout.split()
upload(c, alice, chat, fixture('small.png')) # прогрев пулов/соединений
base = {cid[:12]: read_peak(cid) for cid in cids}
barrier = threading.Barrier(n)
res = []
def worker():
local = Client(c.base, timeout=60)
barrier.wait()
t0 = time.time()
try:
upload(local, alice, chat, data, name='mem.png')
res.append(('ok', time.time() - t0))
except Failure as e:
res.append(('fail:' + str(e)[:60], time.time() - t0))
ths = [threading.Thread(target=worker) for _ in range(n)]
for t in ths:
t.start()
for t in ths:
t.join()
peak = {cid[:12]: read_peak(cid) for cid in cids}
ok = sum(1 for r in res if r[0] == 'ok')
mib = lambda v: round(v / 1048576, 1) if v else None
run = {'parallel': n, 'ok': ok, 'failed': [r[0] for r in res if r[0] != 'ok'],
'max_seconds': round(max(r[1] for r in res), 2),
'baseline_peak_mib': {k: mib(v) for k, v in base.items()},
'peak_mib': {k: mib(v) for k, v in peak.items()},
'payload_total_mib': round(n * len(data) / 1048576, 1)}
growth = [peak[k] - base[k] for k in peak if peak[k] and base[k]]
run['max_peak_growth_mib'] = mib(max(growth)) if growth else None
out['runs'].append(run)
check(ok == n, '%d of %d concurrent uploads failed: %s' % (n - ok, n, run['failed'][:3]))
return out
SCENARIOS = {'failover': scenario_failover, 's3-outage': scenario_s3_outage, 'interrupt': scenario_interrupt,
'db-gap': scenario_db_gap, 'security': scenario_security, 'memory': scenario_memory}
def main():
def configure(p):
p.add_argument('scenario', choices=sorted(SCENARIOS))
p.add_argument('--output', default=None)
args = arguments('Docker-level attachment scenarios (lab4)', configure)
c = Client(args.base_url, args.ca_file, timeout=60)
result = SCENARIOS[args.scenario](c)
text = json.dumps({'scenario': args.scenario, 'result': result}, ensure_ascii=False, indent=2)
if args.output:
open(args.output, 'w', encoding='utf-8').write(text + '\n')
print(text)
print('PASS: ' + args.scenario)
if __name__ == '__main__':
run(main)