#!/usr/bin/env python3 """Операционные сценарии вложений на Docker-стенде (уровни 3–4). failover upload → остановить обслужившую реплику → download через другую s3-outage остановить S3: загрузка/скачивание 503, текстовый чат и readiness работают interrupt обрыв клиента посреди тела: ни ready-вложения, ни намерения, ни объекта db-gap PUT в S3 прошёл, PostgreSQL недоступен на MarkReady: объект удалён компенсацией, намерение осталось → убирает reconciler security bucket приватен (анонимно 403), у API-ключа нет прав вне bucket, bootstrap идемпотентен, консоль/порты S3 не опубликованы memory N одновременных загрузок ~9.8 MiB: пик памяти (cgroup memory.peak) реплик Каждый сценарий печатает JSON-итог и PASS/FAIL. """ import hashlib import json import threading import time import uuid from labutil import (BUCKET, api_container, bucket_count, compose, fixture, mc, noise_png, object_key, psql, raw_post, sh, wait_status) from client import Client, Failure, arguments, check, run def upload(c, actor, chat, data, name='f.png', ctype='image/png', expected=202): return c.call('POST', '/api/v1/chats/%s/attachments?filename=%s' % (chat['id'], name), expected, actor=actor, raw=data, headers={'Content-Type': ctype}, schema='Attachment' if expected == 202 else None) def wait_ready(c, timeout=90): end = time.time() + timeout while time.time() < end: try: c.call('GET', '/health/ready') return except Failure: time.sleep(1) raise Failure('stack not ready') def scenario_failover(c): alice, bob, _, chat = c.bootstrap() data = noise_png(2 * 1024 * 1024) r = upload(c, alice, chat, data) att, served_by = r['json'], r['headers']['X-Instance-Id'] wait_status(c, bob, att['id'], ('ready',)) cid = api_container(served_by) sh('docker', 'stop', '-t', '15', cid) try: seen = set() for _ in range(6): d = c.call('GET', '/api/v1/attachments/%s/content' % att['id'], actor=bob) check(d['bytes'] == data, 'downloaded bytes differ after failover') seen.add(d['headers']['X-Instance-Id']) check(served_by not in seen, 'stopped replica still serving') finally: sh('docker', 'start', cid) sh('sh', 'scripts/restore.sh') return {'uploaded_by': served_by, 'downloaded_by': sorted(seen), 'sha256': hashlib.sha256(data).hexdigest(), 'stopped_container': cid[:12]} def scenario_s3_outage(c): alice, bob, _, chat = c.bootstrap() small = fixture('small.png') att = upload(c, alice, chat, small)['json'] wait_status(c, bob, att['id'], ('ready',)) compose('stop', 's3') try: t0 = time.time() up = upload(c, alice, chat, small, expected=503) up_s = time.time() - t0 c.call('GET', '/api/v1/attachments/%s/content' % att['id'], 503, actor=bob) meta = c.call('GET', '/api/v1/attachments/%s' % att['id'], actor=bob, schema='Attachment')['json'] msg = c.message(alice, chat, 'S3 лежит, а чат работает')['json'] c.call('GET', '/health/ready', schema='Health') leftovers = psql("SELECT count(*) FROM attachments WHERE chat_id = '%s' AND status = 'uploading'" % chat['id']) finally: compose('start', 's3') compose('run', '--rm', 's3-init') for _ in range(30): try: again = upload(c, alice, chat, small)['json'] break except Failure: time.sleep(1) else: raise Failure('upload did not recover after S3 start') d = c.call('GET', '/api/v1/attachments/%s/content' % att['id'], actor=bob) check(d['bytes'] == small, 'old attachment changed after S3 restart') check(leftovers == '0', 'intent rows left after failed upload: ' + leftovers) return {'upload_status_during_outage': up['status'], 'upload_error_seconds': round(up_s, 2), 'download_status_during_outage': 503, 'metadata_during_outage': meta['status'], 'text_message_during_outage': 201, 'readiness_during_outage': 200, 'intents_left': int(leftovers), 'upload_after_recovery': again['status']} def scenario_interrupt(c): alice, _, _, chat = c.bootstrap() data = noise_png(4 * 1024 * 1024) hdr = {'Authorization': 'Bearer ' + alice['token'], 'Content-Type': 'image/png', 'Content-Length': str(len(data))} # Отправляем 1 MiB из 4 и обрываем соединение. import socket from urllib.parse import urlsplit u = urlsplit(c.base) s = socket.create_connection((u.hostname, u.port or 80)) head = 'POST /api/v1/chats/%s/attachments?filename=cut.png HTTP/1.1\r\nHost: %s\r\n' % (chat['id'], u.netloc) s.sendall((head + ''.join('%s: %s\r\n' % kv for kv in hdr.items()) + '\r\n').encode()) s.sendall(data[:1024 * 1024]) time.sleep(1) s.close() time.sleep(3) rows = psql("SELECT status FROM attachments WHERE chat_id = '%s'" % chat['id']) objs = str(sum(bucket_count(p + chat['id'] + '/') for p in ('s/', 'o/', 'v/'))) check(rows == '', 'rows left after interrupted upload: %r' % rows) check(objs == '0', 'objects left after interrupted upload: ' + objs) return {'sent_bytes': 1024 * 1024, 'declared_bytes': len(data), 'rows_after': 0, 'objects_after': 0} def scenario_db_gap(c): """PUT прошёл, а PostgreSQL «пропал» до MarkReady (docker pause).""" alice, _, _, chat = c.bootstrap() data = noise_png(3 * 1024 * 1024) hdr = {'Authorization': 'Bearer ' + alice['token'], 'Content-Type': 'image/png', 'Content-Length': str(len(data))} state = {} def body(): yield data[:512 * 1024] # намерение уже записано, PUT идёт time.sleep(2) compose('pause', 'postgres') # БД «пропала» до MarkReady state['paused_at'] = time.time() yield data[512 * 1024:] t0 = time.time() try: st, _ = raw_post(c.base, '/api/v1/chats/%s/attachments?filename=gap.png' % chat['id'], hdr, body(), read_timeout=60) finally: # Дольше таймаута запроса (30 s) и компенсации (10 s): DeleteIntent тоже не пройдёт. time.sleep(max(0, 45 - (time.time() - state.get('paused_at', time.time())))) compose('unpause', 'postgres') wait_ready(c) time.sleep(3) # отложенный UPDATE (если был) успевает закоммититься после unpause rows = psql("SELECT id || ' ' || status FROM attachments WHERE chat_id = '%s'" % chat['id']) staged = bucket_count('s/' + chat['id'] + '/') status_after = rows.split()[-1] if rows else 'none' check(st != 202, 'upload must not report success when the job was not committed (got %s)' % st) # Инвариант: нет строки queued/ready без исходника. check(not (status_after in ('queued', 'processing') and staged == 0 and bucket_count('o/' + chat['id'] + '/') == 0), 'job without source: %r' % rows) if status_after in ('queued', 'processing', 'ready'): # COMMIT всё-таки прошёл: pipeline доводит работу до ready. aid = rows.split()[0] for _ in range(120): if psql("SELECT status FROM attachments WHERE id='%s'" % aid) == 'ready': break time.sleep(1) else: psql("UPDATE attachments SET updated_at = now() - interval '30 minutes' WHERE chat_id = '%s'" % chat['id']) for _ in range(10): r = compose('run', '--rm', '--no-deps', '-T', '--entrypoint', '/app/reconcile', 'reconciler', '--apply', '--grace', '10m', check=False) if r.returncode != 3: break time.sleep(2) check(r.returncode in (0, 4), 'reconcile failed: ' + r.stderr[-300:]) final = psql("SELECT status FROM attachments WHERE chat_id = '%s'" % chat['id']) check(final in ('ready', ''), 'inconsistent final state: %r' % final) return {'client_status': st, 'seconds': round(time.time() - t0, 1), 'after_gap': {'row_status': status_after, 'staging_objects': staged}, 'final': {'row_status': final or 'none'}} def scenario_security(c): alice, _, _, chat = c.bootstrap() att = upload(c, alice, chat, fixture('small.png'))['json'] # С lab5 worker переносит оригинал staging s/ → o/: ключ берём после ready. wait_status(c, alice, att['id'], ('ready',)) key = object_key(att['id']) url = 'http://s3:9000/%s/%s' % (BUCKET, key) # Анонимно изнутри сети стенда (из контейнера API): объект и листинг закрыты. anon_get = compose('exec', '-T', 'api', 'wget', '-q', '-S', '-O', '/dev/null', url, check=False) anon_list = compose('exec', '-T', 'api', 'wget', '-q', '-S', '-O', '/dev/null', 'http://s3:9000/%s?list-type=2' % BUCKET, check=False) check(anon_get.returncode != 0 and '403' in anon_get.stderr, 'anonymous GET must be 403') check(anon_list.returncode != 0 and '403' in anon_list.stderr, 'anonymous LIST must be 403') # Ключ API: свой bucket — да, остальное — нет. own = mc('mc stat s3/%s/%s >/dev/null && echo ok' % (BUCKET, key), as_user='api').stdout.strip() mb = mc('mc mb s3/evil-bucket-%s' % uuid.uuid4().hex[:6], check=False, as_user='api') admin = mc('mc admin user list s3', check=False, as_user='api') buckets = mc('mc ls s3', check=False, as_user='api') check(own == 'ok', 'api key cannot read own bucket') check(mb.returncode != 0, 'api key must not create buckets') check(admin.returncode != 0, 'api key must not use admin API') check(buckets.returncode != 0 or 'evil' not in buckets.stdout, 'api key sees a bucket it created?!') # Bootstrap идемпотентен: повторный запуск успешен, объект на месте. before = mc('mc ls --recursive s3/%s | wc -l' % BUCKET).stdout.strip() r1 = compose('run', '--rm', 's3-init') r2 = compose('run', '--rm', 's3-init') after = mc('mc ls --recursive s3/%s | wc -l' % BUCKET).stdout.strip() check(before == after, 'bootstrap changed objects') ports = compose('ps', 's3', '--format', '{{.Ports}}').stdout.strip() check('0.0.0.0' not in ports and '127.0.0.1' not in ports, 's3 port is published: ' + ports) return {'anonymous_get': '403', 'anonymous_list': '403', 'api_key_own_bucket': 'allowed', 'api_key_make_bucket': 'denied', 'api_key_admin': 'denied', 'api_key_visible_buckets': sorted(l.split()[-1].rstrip('/') for l in buckets.stdout.splitlines() if l.strip()) if buckets.returncode == 0 else 'denied', 'bootstrap_runs': [r1.returncode, r2.returncode], 'objects_before_after': [int(before), int(after)], 's3_published_ports': ports or 'none'} def read_peak(cid): r = sh('docker', 'exec', cid, 'cat', '/sys/fs/cgroup/memory.peak', check=False) return int(r.stdout.strip()) if r.returncode == 0 and r.stdout.strip().isdigit() else None def read_current(cid): r = sh('docker', 'exec', cid, 'cat', '/sys/fs/cgroup/memory.current', check=False) return int(r.stdout.strip()) if r.returncode == 0 and r.stdout.strip().isdigit() else None def scenario_memory(c, parallel_levels=(10, 20)): alice, _, _, chat = c.bootstrap() data = noise_png(10 * 1024 * 1024 - 300_000) out = {'file_bytes': len(data), 'runs': []} for n in parallel_levels: compose('restart', 'api') # новый cgroup → memory.peak с нуля sh('sh', 'scripts/restore.sh') wait_ready(c) cids = compose('ps', '-q', 'api').stdout.split() upload(c, alice, chat, fixture('small.png')) # прогрев пулов/соединений base = {cid[:12]: read_peak(cid) for cid in cids} barrier = threading.Barrier(n) res = [] def worker(): local = Client(c.base, timeout=60) barrier.wait() t0 = time.time() try: upload(local, alice, chat, data, name='mem.png') res.append(('ok', time.time() - t0)) except Failure as e: res.append(('fail:' + str(e)[:60], time.time() - t0)) ths = [threading.Thread(target=worker) for _ in range(n)] for t in ths: t.start() for t in ths: t.join() peak = {cid[:12]: read_peak(cid) for cid in cids} ok = sum(1 for r in res if r[0] == 'ok') mib = lambda v: round(v / 1048576, 1) if v else None run = {'parallel': n, 'ok': ok, 'failed': [r[0] for r in res if r[0] != 'ok'], 'max_seconds': round(max(r[1] for r in res), 2), 'baseline_peak_mib': {k: mib(v) for k, v in base.items()}, 'peak_mib': {k: mib(v) for k, v in peak.items()}, 'payload_total_mib': round(n * len(data) / 1048576, 1)} growth = [peak[k] - base[k] for k in peak if peak[k] and base[k]] run['max_peak_growth_mib'] = mib(max(growth)) if growth else None out['runs'].append(run) check(ok == n, '%d of %d concurrent uploads failed: %s' % (n - ok, n, run['failed'][:3])) return out SCENARIOS = {'failover': scenario_failover, 's3-outage': scenario_s3_outage, 'interrupt': scenario_interrupt, 'db-gap': scenario_db_gap, 'security': scenario_security, 'memory': scenario_memory} def main(): def configure(p): p.add_argument('scenario', choices=sorted(SCENARIOS)) p.add_argument('--output', default=None) args = arguments('Docker-level attachment scenarios (lab4)', configure) c = Client(args.base_url, args.ca_file, timeout=60) result = SCENARIOS[args.scenario](c) text = json.dumps({'scenario': args.scenario, 'result': result}, ensure_ascii=False, indent=2) if args.output: open(args.output, 'w', encoding='utf-8').write(text + '\n') print(text) print('PASS: ' + args.scenario) if __name__ == '__main__': run(main)