285 lines
14 KiB
Python
285 lines
14 KiB
Python
#!/usr/bin/env python3
|
||
"""Операционные сценарии вложений на Docker-стенде (уровни 3–4).
|
||
|
||
failover upload → остановить обслужившую реплику → download через другую
|
||
s3-outage остановить S3: загрузка/скачивание 503, текстовый чат и readiness работают
|
||
interrupt обрыв клиента посреди тела: ни ready-вложения, ни намерения, ни объекта
|
||
db-gap PUT в S3 прошёл, PostgreSQL недоступен на MarkReady: объект удалён
|
||
компенсацией, намерение осталось → убирает reconciler
|
||
security bucket приватен (анонимно 403), у API-ключа нет прав вне bucket,
|
||
bootstrap идемпотентен, консоль/порты S3 не опубликованы
|
||
memory N одновременных загрузок ~9.8 MiB: пик памяти (cgroup memory.peak) реплик
|
||
|
||
Каждый сценарий печатает JSON-итог и PASS/FAIL.
|
||
"""
|
||
import hashlib
|
||
import json
|
||
import threading
|
||
import time
|
||
import uuid
|
||
from labutil import (BUCKET, api_container, bucket_count, compose, fixture, mc, noise_png, object_key, psql,
|
||
raw_post, sh, wait_status)
|
||
from client import Client, Failure, arguments, check, run
|
||
|
||
|
||
def upload(c, actor, chat, data, name='f.png', ctype='image/png', expected=202):
|
||
return c.call('POST', '/api/v1/chats/%s/attachments?filename=%s' % (chat['id'], name), expected,
|
||
actor=actor, raw=data, headers={'Content-Type': ctype},
|
||
schema='Attachment' if expected == 202 else None)
|
||
|
||
|
||
def wait_ready(c, timeout=90):
|
||
end = time.time() + timeout
|
||
while time.time() < end:
|
||
try:
|
||
c.call('GET', '/health/ready')
|
||
return
|
||
except Failure:
|
||
time.sleep(1)
|
||
raise Failure('stack not ready')
|
||
|
||
|
||
def scenario_failover(c):
|
||
alice, bob, _, chat = c.bootstrap()
|
||
data = noise_png(2 * 1024 * 1024)
|
||
r = upload(c, alice, chat, data)
|
||
att, served_by = r['json'], r['headers']['X-Instance-Id']
|
||
wait_status(c, bob, att['id'], ('ready',))
|
||
cid = api_container(served_by)
|
||
sh('docker', 'stop', '-t', '15', cid)
|
||
try:
|
||
seen = set()
|
||
for _ in range(6):
|
||
d = c.call('GET', '/api/v1/attachments/%s/content' % att['id'], actor=bob)
|
||
check(d['bytes'] == data, 'downloaded bytes differ after failover')
|
||
seen.add(d['headers']['X-Instance-Id'])
|
||
check(served_by not in seen, 'stopped replica still serving')
|
||
finally:
|
||
sh('docker', 'start', cid)
|
||
sh('sh', 'scripts/restore.sh')
|
||
return {'uploaded_by': served_by, 'downloaded_by': sorted(seen), 'sha256': hashlib.sha256(data).hexdigest(),
|
||
'stopped_container': cid[:12]}
|
||
|
||
|
||
def scenario_s3_outage(c):
|
||
alice, bob, _, chat = c.bootstrap()
|
||
small = fixture('small.png')
|
||
att = upload(c, alice, chat, small)['json']
|
||
wait_status(c, bob, att['id'], ('ready',))
|
||
compose('stop', 's3')
|
||
try:
|
||
t0 = time.time()
|
||
up = upload(c, alice, chat, small, expected=503)
|
||
up_s = time.time() - t0
|
||
c.call('GET', '/api/v1/attachments/%s/content' % att['id'], 503, actor=bob)
|
||
meta = c.call('GET', '/api/v1/attachments/%s' % att['id'], actor=bob, schema='Attachment')['json']
|
||
msg = c.message(alice, chat, 'S3 лежит, а чат работает')['json']
|
||
c.call('GET', '/health/ready', schema='Health')
|
||
leftovers = psql("SELECT count(*) FROM attachments WHERE chat_id = '%s' AND status = 'uploading'" % chat['id'])
|
||
finally:
|
||
compose('start', 's3')
|
||
compose('run', '--rm', 's3-init')
|
||
for _ in range(30):
|
||
try:
|
||
again = upload(c, alice, chat, small)['json']
|
||
break
|
||
except Failure:
|
||
time.sleep(1)
|
||
else:
|
||
raise Failure('upload did not recover after S3 start')
|
||
d = c.call('GET', '/api/v1/attachments/%s/content' % att['id'], actor=bob)
|
||
check(d['bytes'] == small, 'old attachment changed after S3 restart')
|
||
check(leftovers == '0', 'intent rows left after failed upload: ' + leftovers)
|
||
return {'upload_status_during_outage': up['status'], 'upload_error_seconds': round(up_s, 2),
|
||
'download_status_during_outage': 503, 'metadata_during_outage': meta['status'],
|
||
'text_message_during_outage': 201, 'readiness_during_outage': 200,
|
||
'intents_left': int(leftovers), 'upload_after_recovery': again['status']}
|
||
|
||
|
||
def scenario_interrupt(c):
|
||
alice, _, _, chat = c.bootstrap()
|
||
data = noise_png(4 * 1024 * 1024)
|
||
hdr = {'Authorization': 'Bearer ' + alice['token'], 'Content-Type': 'image/png',
|
||
'Content-Length': str(len(data))}
|
||
# Отправляем 1 MiB из 4 и обрываем соединение.
|
||
import socket
|
||
from urllib.parse import urlsplit
|
||
u = urlsplit(c.base)
|
||
s = socket.create_connection((u.hostname, u.port or 80))
|
||
head = 'POST /api/v1/chats/%s/attachments?filename=cut.png HTTP/1.1\r\nHost: %s\r\n' % (chat['id'], u.netloc)
|
||
s.sendall((head + ''.join('%s: %s\r\n' % kv for kv in hdr.items()) + '\r\n').encode())
|
||
s.sendall(data[:1024 * 1024])
|
||
time.sleep(1)
|
||
s.close()
|
||
time.sleep(3)
|
||
rows = psql("SELECT status FROM attachments WHERE chat_id = '%s'" % chat['id'])
|
||
objs = str(sum(bucket_count(p + chat['id'] + '/') for p in ('s/', 'o/', 'v/')))
|
||
check(rows == '', 'rows left after interrupted upload: %r' % rows)
|
||
check(objs == '0', 'objects left after interrupted upload: ' + objs)
|
||
return {'sent_bytes': 1024 * 1024, 'declared_bytes': len(data), 'rows_after': 0, 'objects_after': 0}
|
||
|
||
|
||
def scenario_db_gap(c):
|
||
"""PUT прошёл, а PostgreSQL «пропал» до MarkReady (docker pause)."""
|
||
alice, _, _, chat = c.bootstrap()
|
||
data = noise_png(3 * 1024 * 1024)
|
||
hdr = {'Authorization': 'Bearer ' + alice['token'], 'Content-Type': 'image/png', 'Content-Length': str(len(data))}
|
||
state = {}
|
||
|
||
def body():
|
||
yield data[:512 * 1024] # намерение уже записано, PUT идёт
|
||
time.sleep(2)
|
||
compose('pause', 'postgres') # БД «пропала» до MarkReady
|
||
state['paused_at'] = time.time()
|
||
yield data[512 * 1024:]
|
||
t0 = time.time()
|
||
try:
|
||
st, _ = raw_post(c.base, '/api/v1/chats/%s/attachments?filename=gap.png' % chat['id'], hdr, body(), read_timeout=60)
|
||
finally:
|
||
# Дольше таймаута запроса (30 s) и компенсации (10 s): DeleteIntent тоже не пройдёт.
|
||
time.sleep(max(0, 45 - (time.time() - state.get('paused_at', time.time()))))
|
||
compose('unpause', 'postgres')
|
||
wait_ready(c)
|
||
time.sleep(3) # отложенный UPDATE (если был) успевает закоммититься после unpause
|
||
rows = psql("SELECT id || ' ' || status FROM attachments WHERE chat_id = '%s'" % chat['id'])
|
||
staged = bucket_count('s/' + chat['id'] + '/')
|
||
status_after = rows.split()[-1] if rows else 'none'
|
||
check(st != 202, 'upload must not report success when the job was not committed (got %s)' % st)
|
||
# Инвариант: нет строки queued/ready без исходника.
|
||
check(not (status_after in ('queued', 'processing') and staged == 0 and bucket_count('o/' + chat['id'] + '/') == 0),
|
||
'job without source: %r' % rows)
|
||
if status_after in ('queued', 'processing', 'ready'):
|
||
# COMMIT всё-таки прошёл: pipeline доводит работу до ready.
|
||
aid = rows.split()[0]
|
||
for _ in range(120):
|
||
if psql("SELECT status FROM attachments WHERE id='%s'" % aid) == 'ready':
|
||
break
|
||
time.sleep(1)
|
||
else:
|
||
psql("UPDATE attachments SET updated_at = now() - interval '30 minutes' WHERE chat_id = '%s'" % chat['id'])
|
||
for _ in range(10):
|
||
r = compose('run', '--rm', '--no-deps', '-T', '--entrypoint', '/app/reconcile', 'reconciler',
|
||
'--apply', '--grace', '10m', check=False)
|
||
if r.returncode != 3:
|
||
break
|
||
time.sleep(2)
|
||
check(r.returncode in (0, 4), 'reconcile failed: ' + r.stderr[-300:])
|
||
final = psql("SELECT status FROM attachments WHERE chat_id = '%s'" % chat['id'])
|
||
check(final in ('ready', ''), 'inconsistent final state: %r' % final)
|
||
return {'client_status': st, 'seconds': round(time.time() - t0, 1),
|
||
'after_gap': {'row_status': status_after, 'staging_objects': staged},
|
||
'final': {'row_status': final or 'none'}}
|
||
|
||
|
||
def scenario_security(c):
|
||
alice, _, _, chat = c.bootstrap()
|
||
att = upload(c, alice, chat, fixture('small.png'))['json']
|
||
# С lab5 worker переносит оригинал staging s/ → o/: ключ берём после ready.
|
||
wait_status(c, alice, att['id'], ('ready',))
|
||
key = object_key(att['id'])
|
||
url = 'http://s3:9000/%s/%s' % (BUCKET, key)
|
||
# Анонимно изнутри сети стенда (из контейнера API): объект и листинг закрыты.
|
||
anon_get = compose('exec', '-T', 'api', 'wget', '-q', '-S', '-O', '/dev/null', url, check=False)
|
||
anon_list = compose('exec', '-T', 'api', 'wget', '-q', '-S', '-O', '/dev/null',
|
||
'http://s3:9000/%s?list-type=2' % BUCKET, check=False)
|
||
check(anon_get.returncode != 0 and '403' in anon_get.stderr, 'anonymous GET must be 403')
|
||
check(anon_list.returncode != 0 and '403' in anon_list.stderr, 'anonymous LIST must be 403')
|
||
# Ключ API: свой bucket — да, остальное — нет.
|
||
own = mc('mc stat s3/%s/%s >/dev/null && echo ok' % (BUCKET, key), as_user='api').stdout.strip()
|
||
mb = mc('mc mb s3/evil-bucket-%s' % uuid.uuid4().hex[:6], check=False, as_user='api')
|
||
admin = mc('mc admin user list s3', check=False, as_user='api')
|
||
buckets = mc('mc ls s3', check=False, as_user='api')
|
||
check(own == 'ok', 'api key cannot read own bucket')
|
||
check(mb.returncode != 0, 'api key must not create buckets')
|
||
check(admin.returncode != 0, 'api key must not use admin API')
|
||
check(buckets.returncode != 0 or 'evil' not in buckets.stdout, 'api key sees a bucket it created?!')
|
||
# Bootstrap идемпотентен: повторный запуск успешен, объект на месте.
|
||
before = mc('mc ls --recursive s3/%s | wc -l' % BUCKET).stdout.strip()
|
||
r1 = compose('run', '--rm', 's3-init')
|
||
r2 = compose('run', '--rm', 's3-init')
|
||
after = mc('mc ls --recursive s3/%s | wc -l' % BUCKET).stdout.strip()
|
||
check(before == after, 'bootstrap changed objects')
|
||
ports = compose('ps', 's3', '--format', '{{.Ports}}').stdout.strip()
|
||
check('0.0.0.0' not in ports and '127.0.0.1' not in ports, 's3 port is published: ' + ports)
|
||
return {'anonymous_get': '403', 'anonymous_list': '403', 'api_key_own_bucket': 'allowed',
|
||
'api_key_make_bucket': 'denied', 'api_key_admin': 'denied',
|
||
'api_key_visible_buckets': sorted(l.split()[-1].rstrip('/') for l in buckets.stdout.splitlines() if l.strip())
|
||
if buckets.returncode == 0 else 'denied',
|
||
'bootstrap_runs': [r1.returncode, r2.returncode], 'objects_before_after': [int(before), int(after)],
|
||
's3_published_ports': ports or 'none'}
|
||
|
||
|
||
def read_peak(cid):
|
||
r = sh('docker', 'exec', cid, 'cat', '/sys/fs/cgroup/memory.peak', check=False)
|
||
return int(r.stdout.strip()) if r.returncode == 0 and r.stdout.strip().isdigit() else None
|
||
|
||
|
||
def read_current(cid):
|
||
r = sh('docker', 'exec', cid, 'cat', '/sys/fs/cgroup/memory.current', check=False)
|
||
return int(r.stdout.strip()) if r.returncode == 0 and r.stdout.strip().isdigit() else None
|
||
|
||
|
||
def scenario_memory(c, parallel_levels=(10, 20)):
|
||
alice, _, _, chat = c.bootstrap()
|
||
data = noise_png(10 * 1024 * 1024 - 300_000)
|
||
out = {'file_bytes': len(data), 'runs': []}
|
||
for n in parallel_levels:
|
||
compose('restart', 'api') # новый cgroup → memory.peak с нуля
|
||
sh('sh', 'scripts/restore.sh')
|
||
wait_ready(c)
|
||
cids = compose('ps', '-q', 'api').stdout.split()
|
||
upload(c, alice, chat, fixture('small.png')) # прогрев пулов/соединений
|
||
base = {cid[:12]: read_peak(cid) for cid in cids}
|
||
barrier = threading.Barrier(n)
|
||
res = []
|
||
|
||
def worker():
|
||
local = Client(c.base, timeout=60)
|
||
barrier.wait()
|
||
t0 = time.time()
|
||
try:
|
||
upload(local, alice, chat, data, name='mem.png')
|
||
res.append(('ok', time.time() - t0))
|
||
except Failure as e:
|
||
res.append(('fail:' + str(e)[:60], time.time() - t0))
|
||
ths = [threading.Thread(target=worker) for _ in range(n)]
|
||
for t in ths:
|
||
t.start()
|
||
for t in ths:
|
||
t.join()
|
||
peak = {cid[:12]: read_peak(cid) for cid in cids}
|
||
ok = sum(1 for r in res if r[0] == 'ok')
|
||
mib = lambda v: round(v / 1048576, 1) if v else None
|
||
run = {'parallel': n, 'ok': ok, 'failed': [r[0] for r in res if r[0] != 'ok'],
|
||
'max_seconds': round(max(r[1] for r in res), 2),
|
||
'baseline_peak_mib': {k: mib(v) for k, v in base.items()},
|
||
'peak_mib': {k: mib(v) for k, v in peak.items()},
|
||
'payload_total_mib': round(n * len(data) / 1048576, 1)}
|
||
growth = [peak[k] - base[k] for k in peak if peak[k] and base[k]]
|
||
run['max_peak_growth_mib'] = mib(max(growth)) if growth else None
|
||
out['runs'].append(run)
|
||
check(ok == n, '%d of %d concurrent uploads failed: %s' % (n - ok, n, run['failed'][:3]))
|
||
return out
|
||
|
||
|
||
SCENARIOS = {'failover': scenario_failover, 's3-outage': scenario_s3_outage, 'interrupt': scenario_interrupt,
|
||
'db-gap': scenario_db_gap, 'security': scenario_security, 'memory': scenario_memory}
|
||
|
||
|
||
def main():
|
||
def configure(p):
|
||
p.add_argument('scenario', choices=sorted(SCENARIOS))
|
||
p.add_argument('--output', default=None)
|
||
args = arguments('Docker-level attachment scenarios (lab4)', configure)
|
||
c = Client(args.base_url, args.ca_file, timeout=60)
|
||
result = SCENARIOS[args.scenario](c)
|
||
text = json.dumps({'scenario': args.scenario, 'result': result}, ensure_ascii=False, indent=2)
|
||
if args.output:
|
||
open(args.output, 'w', encoding='utf-8').write(text + '\n')
|
||
print(text)
|
||
print('PASS: ' + args.scenario)
|
||
|
||
|
||
if __name__ == '__main__':
|
||
run(main)
|