#!/usr/bin/env python3 """Operator checks for lab4 that need Docker access (run on the host, explicitly). python scripts/lab4ops.py private # bucket privacy outside the API python scripts/lab4ops.py failover-upload # upload -> stop that replica -> download via the other python scripts/lab4ops.py s3-outage # stop S3: uploads/downloads 503, text path keeps working python scripts/lab4ops.py no-local-file # API containers have no volume and no stored original python scripts/lab4ops.py bootstrap # migrate + bucket bootstrap are idempotent python scripts/lab4ops.py memory # peak RSS under concurrent uploads Each command prints JSON and, with --output, stores it in evidence/. Synthetic data only; tokens/credentials are never printed. """ import argparse from concurrent.futures import ThreadPoolExecutor import hashlib import json from pathlib import Path import subprocess import sys import threading import time ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / 'tests')) sys.path.insert(0, str(ROOT / 'student-tests')) sys.path.insert(0, str(ROOT / 'scripts')) from client import Client, Failure, check # noqa: E402 import imagegen # noqa: E402 from lablib import api_containers, sh, wait_healthy # noqa: E402 PNG = (ROOT / 'tests/fixtures/sample.png').read_bytes() NETWORK = 'mtusi-lab4_data' TOOL_IMAGE = 'python:3.12.8-alpine3.21' def winpath(p): s = str(p) return s.replace('\\', '/') def s3tool(*args, stdin=None): """Run s3tool.py in a throwaway container on the data network (S3 is not published).""" cmd = ['docker', 'run', '--rm', '-i', '--network', NETWORK, '-v', winpath(ROOT / 'secrets') + ':/secrets:ro', '-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro', TOOL_IMAGE, 'python', '-I', '/tool/s3tool.py', *args] r = subprocess.run(cmd, input=stdin, capture_output=True, text=True, encoding='utf-8') if r.returncode != 0: raise RuntimeError('s3tool %s failed: %s' % (args[0], (r.stdout + r.stderr).strip()[-300:])) return json.loads(r.stdout.strip().splitlines()[-1]) def psql(sql): return sh('docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-At', '-v', 'ON_ERROR_STOP=1', '-c', sql) def upload(c, actor, chat, data=PNG, name='sample.png', expected=201): return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + name, expected, actor=actor, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment' if expected == 201 else None) def cmd_private(a): c = Client(a.base_url, timeout=30) alice, bob, eve, chat = c.bootstrap() att = upload(c, alice, chat)['json'] key = psql("SELECT object_key FROM attachments WHERE id = '%s'" % att['id']) res = { 'object_key_shape': 'attachments// (server generated)', 'key_contains_filename': 'sample' in key, 'anonymous_get': s3tool('get', key, '--as', 'anonymous')['status'], 'anonymous_list': None, 'api_identity_get': s3tool('get', key)['status'], 'api_identity_get_sha256_matches': s3tool('get', key)['sha256'] == att['sha256'], 'api_identity_create_bucket': s3tool('create-bucket', 'lab4-forbidden-bucket')['status'], 'anonymous_create_bucket': s3tool('create-bucket', 'lab4-forbidden-bucket', '--as', 'anonymous')['status'], 'published_s3_ports': sh('docker', 'compose', 'port', 's3', '8333', check=False) or None, 'outsider_via_api_metadata': c.call('GET', '/api/v1/attachments/' + att['id'], 404, actor=eve)['status'], 'outsider_via_api_content': c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 404, actor=eve)['status'], } try: s3tool('list', 'attachments/', '--as', 'anonymous') res['anonymous_list'] = 200 except RuntimeError as exc: res['anonymous_list'] = 'denied (%s)' % str(exc).split(':')[-1].strip() # the API identity is bound to ONE bucket: the admin creates another one, the API cannot write there s3tool('create-bucket', 'lab4-other-bucket', '--as', 'admin') res['api_identity_put_other_bucket'] = subprocess.run( ['docker', 'run', '--rm', '-e', 'S3_BUCKET=lab4-other-bucket', '--network', NETWORK, '-v', winpath(ROOT / 'secrets') + ':/secrets:ro', '-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro', TOOL_IMAGE, 'python', '-I', '/tool/s3tool.py', 'put', 'probe'], capture_output=True, text=True).stdout.strip() check(res['anonymous_get'] == 403 and res['api_identity_get'] == 200, 'unexpected bucket privacy') check(res['api_identity_create_bucket'] == 403 and '"status": 403' in res['api_identity_put_other_bucket'], 'api identity has more rights than expected') return res def cmd_failover_upload(a): c = Client(a.base_url, timeout=30) alice, bob, eve, chat = c.bootstrap() r = upload(c, alice, chat) att, served_by = r['json'], r['headers']['X-Instance-Id'] target = [x for x in api_containers() if x['id'].startswith(served_by[:12])] check(len(target) == 1, 'cannot map X-Instance-Id to one api container') before = sh('docker', 'compose', 'ps', '-a', '--format', '{{.Name}} {{.State}}') sh('docker', 'stop', '-t', '15', target[0]['id']) t0 = time.monotonic() reads = [] try: for _ in range(10): try: d = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob) reads.append({'instance': d['headers']['X-Instance-Id'], 'sha256_ok': hashlib.sha256(d['bytes']).hexdigest() == att['sha256']}) except Failure as exc: reads.append({'error': str(exc)[-60:]}) time.sleep(0.3) during = sh('docker', 'compose', 'ps', '-a', '--format', '{{.Name}} {{.State}}') finally: sh('docker', 'start', target[0]['id']) wait_healthy(target[0]['id']) ok = [x for x in reads if x.get('sha256_ok')] check(len(ok) >= 8 and all(x['instance'] != served_by for x in ok), 'download did not move to the other replica') return {'uploaded_via': served_by, 'stopped_container': target[0]['name'], 'reads_after_stop': reads, 'seconds_observed': round(time.monotonic() - t0, 1), 'ps_before': before, 'ps_during': during} def cmd_s3_outage(a): c = Client(a.base_url, timeout=30) alice, bob, eve, chat = c.bootstrap() att = upload(c, alice, chat)['json'] pending_before = psql("SELECT count(*) FROM attachments WHERE status = 'pending'") sh('docker', 'compose', 'stop', 's3') res = {} try: u = upload(c, alice, chat, expected=503) res['upload_status'], res['upload_ms'] = u['status'], round(u['elapsed'] * 1000) d = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 503, actor=bob) res['download_status'], res['download_ms'] = d['status'], round(d['elapsed'] * 1000) res['metadata_status'] = c.call('GET', '/api/v1/attachments/' + att['id'], actor=bob)['status'] res['text_message_status'] = c.message(alice, chat, 'S3 лежит, текст работает')['status'] res['ready_status'] = c.call('GET', '/health/ready')['status'] # expected 0: compensation deletes the pending row; a possible object is an orphan for the reconciler res['pending_rows_left'] = int(psql( "SELECT count(*) FROM attachments WHERE status = 'pending'")) - int(pending_before) finally: sh('docker', 'compose', 'up', '-d', '--no-recreate', '--wait', 's3') t0 = time.monotonic() while True: try: upload(c, alice, chat) break except Failure: check(time.monotonic() - t0 < 60, 'uploads did not recover after S3 start') time.sleep(0.5) res['upload_recovered_after_s3_healthy_s'] = round(time.monotonic() - t0, 1) d = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob) res['old_file_intact'] = hashlib.sha256(d['bytes']).hexdigest() == att['sha256'] check(res['upload_status'] == 503 and res['text_message_status'] == 201 and res['ready_status'] == 200, 'S3 outage is not isolated to attachment endpoints') return res def cmd_no_local_file(a): out = {} for r in api_containers(): mounts = json.loads(sh('docker', 'inspect', '-f', '{{json .Mounts}}', r['id'])) ro = sh('docker', 'inspect', '-f', '{{.HostConfig.ReadonlyRootfs}}', r['id']) probe = sh('docker', 'exec', r['id'], 'sh', '-c', 'touch /app/probe 2>&1 || true') # Docker itself bind-mounts /etc/hosts, /etc/hostname, /etc/resolv.conf files = sh('docker', 'exec', r['id'], 'sh', '-c', 'find / -xdev -type f -newer /app/server ! -path "/proc/*" ! -path "/sys/*" ' '! -path "/run/secrets/*" ! -name hosts ! -name hostname ! -name resolv.conf ' '! -name .dockerenv 2>/dev/null | head -20') out[r['name']] = {'readonly_rootfs': ro == 'true', 'write_probe': probe, 'mounts': [m.get('Destination') for m in mounts], 'files_written_since_start': files.splitlines()} check(ro == 'true' and not files.strip(), 'API container can store files locally') return out def cmd_bootstrap(a): runs = [] for _ in range(2): for svc in ('migrate', 's3-init'): log = sh('docker', 'compose', 'run', '--rm', '--no-deps', '-T', svc) runs.append({'service': svc, 'last_log': json.loads(log.strip().splitlines()[-1])}) return runs def peak(cid): v = sh('docker', 'exec', cid, 'cat', '/sys/fs/cgroup/memory.peak', '/sys/fs/cgroup/memory.current').split() return int(v[0]) / 2**20, int(v[1]) / 2**20 def cmd_memory(a): big = imagegen.near_limit_png() c = Client(a.base_url, timeout=120) alice, bob, eve, chat = c.bootstrap() results = [] for conc in a.levels: for rep in range(a.repeat): sh('docker', 'compose', 'restart', 'api') for r in api_containers(): check(wait_healthy(r['id'], 60), 'api not healthy') time.sleep(1) s3_id = sh('docker', 'compose', 'ps', '-q', 's3') s3_restarts = sh('docker', 'inspect', '-f', '{{.RestartCount}}', s3_id) upload(c, alice, chat, big) # warm-up allocations of one replica base = {r['name']: peak(r['id']) for r in api_containers()} barrier = threading.Barrier(conc) def one(i): local = Client(a.base_url, timeout=120) barrier.wait() try: x = local.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=big.png', [201, 503], actor=alice, raw=big, headers={'Content-Type': 'image/png'}) return x['status'], x['elapsed'] except Failure as exc: return str(exc)[-40:], 0 t0 = time.monotonic() with ThreadPoolExecutor(conc) as pool: res = list(pool.map(one, range(conc))) wall = time.monotonic() - t0 after = {r['name']: peak(r['id']) for r in api_containers()} lat = sorted(e for s, e in res if s == 201) row = {'concurrency': conc, 'repeat': rep + 1, 'file_bytes': len(big), 'ok': sum(1 for s, _ in res if s == 201), 'status_503': sum(1 for s, _ in res if s == 503), 'other': [s for s, _ in res if s not in (201, 503)], 'wall_s': round(wall, 2), 'p50_ms': round(1000 * lat[len(lat) // 2]) if lat else None, 'max_ms': round(1000 * lat[-1]) if lat else None, 'peak_mib_per_replica': {k: round(v[0], 1) for k, v in after.items()}, 'baseline_current_mib': {k: round(v[1], 1) for k, v in base.items()}} row['max_peak_mib'] = max(row['peak_mib_per_replica'].values()) row['s3_peak_mib_since_start'] = round(peak(s3_id)[0], 1) row['s3_restarted_during_run'] = sh('docker', 'inspect', '-f', '{{.RestartCount}}', s3_id) != s3_restarts print(json.dumps(row, ensure_ascii=False), flush=True) results.append(row) return {'api_limit': '512M x 2 replicas, 1 CPU each', 'gomemlimit': '400MiB', 'upload_concurrency_per_replica': 8, 'decode_budget_mib': 128, 's3_limit': sh('docker', 'inspect', '-f', '{{.HostConfig.Memory}}', sh('docker', 'compose', 'ps', '-q', 's3')), 'runs': results} def main(): p = argparse.ArgumentParser() p.add_argument('cmd', choices=['private', 'failover-upload', 's3-outage', 'no-local-file', 'bootstrap', 'memory']) p.add_argument('--base-url', default='http://localhost:8080') p.add_argument('--output') p.add_argument('--levels', type=int, nargs='+', default=[1, 10, 20, 40]) p.add_argument('--repeat', type=int, default=2) a = p.parse_args() fn = globals()['cmd_' + a.cmd.replace('-', '_')] result = {'command': a.cmd, 'date_utc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), 'commit': sh('git', 'rev-parse', '--short', 'HEAD', check=False), 'result': fn(a)} text = json.dumps(result, ensure_ascii=False, indent=2) print(text) if a.output: Path(a.output).write_text(text + '\n', encoding='utf-8') print('PASS: ' + a.cmd) if __name__ == '__main__': main()