Files
mtusi2026lab4/scripts/lab4ops.py
2026-10-29 15:00:36 +03:00

270 lines
13 KiB
Python

#!/usr/bin/env python3
"""Operator checks for lab4 that need Docker access (run on the host, explicitly).
python scripts/lab4ops.py private # bucket privacy outside the API
python scripts/lab4ops.py failover-upload # upload -> stop that replica -> download via the other
python scripts/lab4ops.py s3-outage # stop S3: uploads/downloads 503, text path keeps working
python scripts/lab4ops.py no-local-file # API containers have no volume and no stored original
python scripts/lab4ops.py bootstrap # migrate + bucket bootstrap are idempotent
python scripts/lab4ops.py memory # peak RSS under concurrent uploads
Each command prints JSON and, with --output, stores it in evidence/.
Synthetic data only; tokens/credentials are never printed.
"""
import argparse
from concurrent.futures import ThreadPoolExecutor
import hashlib
import json
from pathlib import Path
import subprocess
import sys
import threading
import time
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
sys.path.insert(0, str(ROOT / 'student-tests'))
sys.path.insert(0, str(ROOT / 'scripts'))
from client import Client, Failure, check # noqa: E402
import imagegen # noqa: E402
from lablib import api_containers, sh, wait_healthy # noqa: E402
PNG = (ROOT / 'tests/fixtures/sample.png').read_bytes()
NETWORK = 'mtusi-lab4_data'
TOOL_IMAGE = 'python:3.12.8-alpine3.21'
def winpath(p):
s = str(p)
return s.replace('\\', '/')
def s3tool(*args, stdin=None):
"""Run s3tool.py in a throwaway container on the data network (S3 is not published)."""
cmd = ['docker', 'run', '--rm', '-i', '--network', NETWORK,
'-v', winpath(ROOT / 'secrets') + ':/secrets:ro',
'-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro',
TOOL_IMAGE, 'python', '-I', '/tool/s3tool.py', *args]
r = subprocess.run(cmd, input=stdin, capture_output=True, text=True, encoding='utf-8')
if r.returncode != 0:
raise RuntimeError('s3tool %s failed: %s' % (args[0], (r.stdout + r.stderr).strip()[-300:]))
return json.loads(r.stdout.strip().splitlines()[-1])
def psql(sql):
return sh('docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger',
'-At', '-v', 'ON_ERROR_STOP=1', '-c', sql)
def upload(c, actor, chat, data=PNG, name='sample.png', expected=201):
return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + name, expected,
actor=actor, raw=data, headers={'Content-Type': 'image/png'},
schema='Attachment' if expected == 201 else None)
def cmd_private(a):
c = Client(a.base_url, timeout=30)
alice, bob, eve, chat = c.bootstrap()
att = upload(c, alice, chat)['json']
key = psql("SELECT object_key FROM attachments WHERE id = '%s'" % att['id'])
res = {
'object_key_shape': 'attachments/<chat_id>/<attachment_id> (server generated)',
'key_contains_filename': 'sample' in key,
'anonymous_get': s3tool('get', key, '--as', 'anonymous')['status'],
'anonymous_list': None,
'api_identity_get': s3tool('get', key)['status'],
'api_identity_get_sha256_matches': s3tool('get', key)['sha256'] == att['sha256'],
'api_identity_create_bucket': s3tool('create-bucket', 'lab4-forbidden-bucket')['status'],
'anonymous_create_bucket': s3tool('create-bucket', 'lab4-forbidden-bucket', '--as', 'anonymous')['status'],
'published_s3_ports': sh('docker', 'compose', 'port', 's3', '8333', check=False) or None,
'outsider_via_api_metadata': c.call('GET', '/api/v1/attachments/' + att['id'], 404, actor=eve)['status'],
'outsider_via_api_content': c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 404, actor=eve)['status'],
}
try:
s3tool('list', 'attachments/', '--as', 'anonymous')
res['anonymous_list'] = 200
except RuntimeError as exc:
res['anonymous_list'] = 'denied (%s)' % str(exc).split(':')[-1].strip()
# the API identity is bound to ONE bucket: the admin creates another one, the API cannot write there
s3tool('create-bucket', 'lab4-other-bucket', '--as', 'admin')
res['api_identity_put_other_bucket'] = subprocess.run(
['docker', 'run', '--rm', '-e', 'S3_BUCKET=lab4-other-bucket', '--network', NETWORK,
'-v', winpath(ROOT / 'secrets') + ':/secrets:ro', '-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro',
TOOL_IMAGE, 'python', '-I', '/tool/s3tool.py', 'put', 'probe'],
capture_output=True, text=True).stdout.strip()
check(res['anonymous_get'] == 403 and res['api_identity_get'] == 200, 'unexpected bucket privacy')
check(res['api_identity_create_bucket'] == 403 and '"status": 403' in res['api_identity_put_other_bucket'],
'api identity has more rights than expected')
return res
def cmd_failover_upload(a):
c = Client(a.base_url, timeout=30)
alice, bob, eve, chat = c.bootstrap()
r = upload(c, alice, chat)
att, served_by = r['json'], r['headers']['X-Instance-Id']
target = [x for x in api_containers() if x['id'].startswith(served_by[:12])]
check(len(target) == 1, 'cannot map X-Instance-Id to one api container')
before = sh('docker', 'compose', 'ps', '-a', '--format', '{{.Name}} {{.State}}')
sh('docker', 'stop', '-t', '15', target[0]['id'])
t0 = time.monotonic()
reads = []
try:
for _ in range(10):
try:
d = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)
reads.append({'instance': d['headers']['X-Instance-Id'],
'sha256_ok': hashlib.sha256(d['bytes']).hexdigest() == att['sha256']})
except Failure as exc:
reads.append({'error': str(exc)[-60:]})
time.sleep(0.3)
during = sh('docker', 'compose', 'ps', '-a', '--format', '{{.Name}} {{.State}}')
finally:
sh('docker', 'start', target[0]['id'])
wait_healthy(target[0]['id'])
ok = [x for x in reads if x.get('sha256_ok')]
check(len(ok) >= 8 and all(x['instance'] != served_by for x in ok), 'download did not move to the other replica')
return {'uploaded_via': served_by, 'stopped_container': target[0]['name'], 'reads_after_stop': reads,
'seconds_observed': round(time.monotonic() - t0, 1), 'ps_before': before, 'ps_during': during}
def cmd_s3_outage(a):
c = Client(a.base_url, timeout=30)
alice, bob, eve, chat = c.bootstrap()
att = upload(c, alice, chat)['json']
pending_before = psql("SELECT count(*) FROM attachments WHERE status = 'pending'")
sh('docker', 'compose', 'stop', 's3')
res = {}
try:
u = upload(c, alice, chat, expected=503)
res['upload_status'], res['upload_ms'] = u['status'], round(u['elapsed'] * 1000)
d = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 503, actor=bob)
res['download_status'], res['download_ms'] = d['status'], round(d['elapsed'] * 1000)
res['metadata_status'] = c.call('GET', '/api/v1/attachments/' + att['id'], actor=bob)['status']
res['text_message_status'] = c.message(alice, chat, 'S3 лежит, текст работает')['status']
res['ready_status'] = c.call('GET', '/health/ready')['status']
# expected 0: compensation deletes the pending row; a possible object is an orphan for the reconciler
res['pending_rows_left'] = int(psql(
"SELECT count(*) FROM attachments WHERE status = 'pending'")) - int(pending_before)
finally:
sh('docker', 'compose', 'up', '-d', '--no-recreate', '--wait', 's3')
t0 = time.monotonic()
while True:
try:
upload(c, alice, chat)
break
except Failure:
check(time.monotonic() - t0 < 60, 'uploads did not recover after S3 start')
time.sleep(0.5)
res['upload_recovered_after_s3_healthy_s'] = round(time.monotonic() - t0, 1)
d = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)
res['old_file_intact'] = hashlib.sha256(d['bytes']).hexdigest() == att['sha256']
check(res['upload_status'] == 503 and res['text_message_status'] == 201 and res['ready_status'] == 200,
'S3 outage is not isolated to attachment endpoints')
return res
def cmd_no_local_file(a):
out = {}
for r in api_containers():
mounts = json.loads(sh('docker', 'inspect', '-f', '{{json .Mounts}}', r['id']))
ro = sh('docker', 'inspect', '-f', '{{.HostConfig.ReadonlyRootfs}}', r['id'])
probe = sh('docker', 'exec', r['id'], 'sh', '-c', 'touch /app/probe 2>&1 || true')
# Docker itself bind-mounts /etc/hosts, /etc/hostname, /etc/resolv.conf
files = sh('docker', 'exec', r['id'], 'sh', '-c',
'find / -xdev -type f -newer /app/server ! -path "/proc/*" ! -path "/sys/*" '
'! -path "/run/secrets/*" ! -name hosts ! -name hostname ! -name resolv.conf '
'! -name .dockerenv 2>/dev/null | head -20')
out[r['name']] = {'readonly_rootfs': ro == 'true', 'write_probe': probe,
'mounts': [m.get('Destination') for m in mounts],
'files_written_since_start': files.splitlines()}
check(ro == 'true' and not files.strip(), 'API container can store files locally')
return out
def cmd_bootstrap(a):
runs = []
for _ in range(2):
for svc in ('migrate', 's3-init'):
log = sh('docker', 'compose', 'run', '--rm', '--no-deps', '-T', svc)
runs.append({'service': svc, 'last_log': json.loads(log.strip().splitlines()[-1])})
return runs
def peak(cid):
v = sh('docker', 'exec', cid, 'cat', '/sys/fs/cgroup/memory.peak', '/sys/fs/cgroup/memory.current').split()
return int(v[0]) / 2**20, int(v[1]) / 2**20
def cmd_memory(a):
big = imagegen.near_limit_png()
c = Client(a.base_url, timeout=120)
alice, bob, eve, chat = c.bootstrap()
results = []
for conc in a.levels:
for rep in range(a.repeat):
sh('docker', 'compose', 'restart', 'api')
for r in api_containers():
check(wait_healthy(r['id'], 60), 'api not healthy')
time.sleep(1)
s3_id = sh('docker', 'compose', 'ps', '-q', 's3')
s3_restarts = sh('docker', 'inspect', '-f', '{{.RestartCount}}', s3_id)
upload(c, alice, chat, big) # warm-up allocations of one replica
base = {r['name']: peak(r['id']) for r in api_containers()}
barrier = threading.Barrier(conc)
def one(i):
local = Client(a.base_url, timeout=120)
barrier.wait()
try:
x = local.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=big.png',
[201, 503], actor=alice, raw=big, headers={'Content-Type': 'image/png'})
return x['status'], x['elapsed']
except Failure as exc:
return str(exc)[-40:], 0
t0 = time.monotonic()
with ThreadPoolExecutor(conc) as pool:
res = list(pool.map(one, range(conc)))
wall = time.monotonic() - t0
after = {r['name']: peak(r['id']) for r in api_containers()}
lat = sorted(e for s, e in res if s == 201)
row = {'concurrency': conc, 'repeat': rep + 1, 'file_bytes': len(big),
'ok': sum(1 for s, _ in res if s == 201), 'status_503': sum(1 for s, _ in res if s == 503),
'other': [s for s, _ in res if s not in (201, 503)], 'wall_s': round(wall, 2),
'p50_ms': round(1000 * lat[len(lat) // 2]) if lat else None,
'max_ms': round(1000 * lat[-1]) if lat else None,
'peak_mib_per_replica': {k: round(v[0], 1) for k, v in after.items()},
'baseline_current_mib': {k: round(v[1], 1) for k, v in base.items()}}
row['max_peak_mib'] = max(row['peak_mib_per_replica'].values())
row['s3_peak_mib_since_start'] = round(peak(s3_id)[0], 1)
row['s3_restarted_during_run'] = sh('docker', 'inspect', '-f', '{{.RestartCount}}', s3_id) != s3_restarts
print(json.dumps(row, ensure_ascii=False), flush=True)
results.append(row)
return {'api_limit': '512M x 2 replicas, 1 CPU each', 'gomemlimit': '400MiB',
'upload_concurrency_per_replica': 8, 'decode_budget_mib': 128,
's3_limit': sh('docker', 'inspect', '-f', '{{.HostConfig.Memory}}', sh('docker', 'compose', 'ps', '-q', 's3')),
'runs': results}
def main():
p = argparse.ArgumentParser()
p.add_argument('cmd', choices=['private', 'failover-upload', 's3-outage', 'no-local-file', 'bootstrap', 'memory'])
p.add_argument('--base-url', default='http://localhost:8080')
p.add_argument('--output')
p.add_argument('--levels', type=int, nargs='+', default=[1, 10, 20, 40])
p.add_argument('--repeat', type=int, default=2)
a = p.parse_args()
fn = globals()['cmd_' + a.cmd.replace('-', '_')]
result = {'command': a.cmd, 'date_utc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
'commit': sh('git', 'rev-parse', '--short', 'HEAD', check=False), 'result': fn(a)}
text = json.dumps(result, ensure_ascii=False, indent=2)
print(text)
if a.output:
Path(a.output).write_text(text + '\n', encoding='utf-8')
print('PASS: ' + a.cmd)
if __name__ == '__main__':
main()