270 lines
13 KiB
Python
270 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
"""Operator checks for lab4 that need Docker access (run on the host, explicitly).
|
|
|
|
python scripts/lab4ops.py private # bucket privacy outside the API
|
|
python scripts/lab4ops.py failover-upload # upload -> stop that replica -> download via the other
|
|
python scripts/lab4ops.py s3-outage # stop S3: uploads/downloads 503, text path keeps working
|
|
python scripts/lab4ops.py no-local-file # API containers have no volume and no stored original
|
|
python scripts/lab4ops.py bootstrap # migrate + bucket bootstrap are idempotent
|
|
python scripts/lab4ops.py memory # peak RSS under concurrent uploads
|
|
|
|
Each command prints JSON and, with --output, stores it in evidence/.
|
|
Synthetic data only; tokens/credentials are never printed.
|
|
"""
|
|
import argparse
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
sys.path.insert(0, str(ROOT / 'student-tests'))
|
|
sys.path.insert(0, str(ROOT / 'scripts'))
|
|
from client import Client, Failure, check # noqa: E402
|
|
import imagegen # noqa: E402
|
|
from lablib import api_containers, sh, wait_healthy # noqa: E402
|
|
|
|
PNG = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
|
NETWORK = 'mtusi-lab4_data'
|
|
TOOL_IMAGE = 'python:3.12.8-alpine3.21'
|
|
|
|
|
|
def winpath(p):
|
|
s = str(p)
|
|
return s.replace('\\', '/')
|
|
|
|
|
|
def s3tool(*args, stdin=None):
|
|
"""Run s3tool.py in a throwaway container on the data network (S3 is not published)."""
|
|
cmd = ['docker', 'run', '--rm', '-i', '--network', NETWORK,
|
|
'-v', winpath(ROOT / 'secrets') + ':/secrets:ro',
|
|
'-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro',
|
|
TOOL_IMAGE, 'python', '-I', '/tool/s3tool.py', *args]
|
|
r = subprocess.run(cmd, input=stdin, capture_output=True, text=True, encoding='utf-8')
|
|
if r.returncode != 0:
|
|
raise RuntimeError('s3tool %s failed: %s' % (args[0], (r.stdout + r.stderr).strip()[-300:]))
|
|
return json.loads(r.stdout.strip().splitlines()[-1])
|
|
|
|
|
|
def psql(sql):
|
|
return sh('docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger',
|
|
'-At', '-v', 'ON_ERROR_STOP=1', '-c', sql)
|
|
|
|
|
|
def upload(c, actor, chat, data=PNG, name='sample.png', expected=201):
|
|
return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + name, expected,
|
|
actor=actor, raw=data, headers={'Content-Type': 'image/png'},
|
|
schema='Attachment' if expected == 201 else None)
|
|
|
|
|
|
def cmd_private(a):
|
|
c = Client(a.base_url, timeout=30)
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
att = upload(c, alice, chat)['json']
|
|
key = psql("SELECT object_key FROM attachments WHERE id = '%s'" % att['id'])
|
|
res = {
|
|
'object_key_shape': 'attachments/<chat_id>/<attachment_id> (server generated)',
|
|
'key_contains_filename': 'sample' in key,
|
|
'anonymous_get': s3tool('get', key, '--as', 'anonymous')['status'],
|
|
'anonymous_list': None,
|
|
'api_identity_get': s3tool('get', key)['status'],
|
|
'api_identity_get_sha256_matches': s3tool('get', key)['sha256'] == att['sha256'],
|
|
'api_identity_create_bucket': s3tool('create-bucket', 'lab4-forbidden-bucket')['status'],
|
|
'anonymous_create_bucket': s3tool('create-bucket', 'lab4-forbidden-bucket', '--as', 'anonymous')['status'],
|
|
'published_s3_ports': sh('docker', 'compose', 'port', 's3', '8333', check=False) or None,
|
|
'outsider_via_api_metadata': c.call('GET', '/api/v1/attachments/' + att['id'], 404, actor=eve)['status'],
|
|
'outsider_via_api_content': c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 404, actor=eve)['status'],
|
|
}
|
|
try:
|
|
s3tool('list', 'attachments/', '--as', 'anonymous')
|
|
res['anonymous_list'] = 200
|
|
except RuntimeError as exc:
|
|
res['anonymous_list'] = 'denied (%s)' % str(exc).split(':')[-1].strip()
|
|
# the API identity is bound to ONE bucket: the admin creates another one, the API cannot write there
|
|
s3tool('create-bucket', 'lab4-other-bucket', '--as', 'admin')
|
|
res['api_identity_put_other_bucket'] = subprocess.run(
|
|
['docker', 'run', '--rm', '-e', 'S3_BUCKET=lab4-other-bucket', '--network', NETWORK,
|
|
'-v', winpath(ROOT / 'secrets') + ':/secrets:ro', '-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro',
|
|
TOOL_IMAGE, 'python', '-I', '/tool/s3tool.py', 'put', 'probe'],
|
|
capture_output=True, text=True).stdout.strip()
|
|
check(res['anonymous_get'] == 403 and res['api_identity_get'] == 200, 'unexpected bucket privacy')
|
|
check(res['api_identity_create_bucket'] == 403 and '"status": 403' in res['api_identity_put_other_bucket'],
|
|
'api identity has more rights than expected')
|
|
return res
|
|
|
|
|
|
def cmd_failover_upload(a):
|
|
c = Client(a.base_url, timeout=30)
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
r = upload(c, alice, chat)
|
|
att, served_by = r['json'], r['headers']['X-Instance-Id']
|
|
target = [x for x in api_containers() if x['id'].startswith(served_by[:12])]
|
|
check(len(target) == 1, 'cannot map X-Instance-Id to one api container')
|
|
before = sh('docker', 'compose', 'ps', '-a', '--format', '{{.Name}} {{.State}}')
|
|
sh('docker', 'stop', '-t', '15', target[0]['id'])
|
|
t0 = time.monotonic()
|
|
reads = []
|
|
try:
|
|
for _ in range(10):
|
|
try:
|
|
d = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)
|
|
reads.append({'instance': d['headers']['X-Instance-Id'],
|
|
'sha256_ok': hashlib.sha256(d['bytes']).hexdigest() == att['sha256']})
|
|
except Failure as exc:
|
|
reads.append({'error': str(exc)[-60:]})
|
|
time.sleep(0.3)
|
|
during = sh('docker', 'compose', 'ps', '-a', '--format', '{{.Name}} {{.State}}')
|
|
finally:
|
|
sh('docker', 'start', target[0]['id'])
|
|
wait_healthy(target[0]['id'])
|
|
ok = [x for x in reads if x.get('sha256_ok')]
|
|
check(len(ok) >= 8 and all(x['instance'] != served_by for x in ok), 'download did not move to the other replica')
|
|
return {'uploaded_via': served_by, 'stopped_container': target[0]['name'], 'reads_after_stop': reads,
|
|
'seconds_observed': round(time.monotonic() - t0, 1), 'ps_before': before, 'ps_during': during}
|
|
|
|
|
|
def cmd_s3_outage(a):
|
|
c = Client(a.base_url, timeout=30)
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
att = upload(c, alice, chat)['json']
|
|
pending_before = psql("SELECT count(*) FROM attachments WHERE status = 'pending'")
|
|
sh('docker', 'compose', 'stop', 's3')
|
|
res = {}
|
|
try:
|
|
u = upload(c, alice, chat, expected=503)
|
|
res['upload_status'], res['upload_ms'] = u['status'], round(u['elapsed'] * 1000)
|
|
d = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 503, actor=bob)
|
|
res['download_status'], res['download_ms'] = d['status'], round(d['elapsed'] * 1000)
|
|
res['metadata_status'] = c.call('GET', '/api/v1/attachments/' + att['id'], actor=bob)['status']
|
|
res['text_message_status'] = c.message(alice, chat, 'S3 лежит, текст работает')['status']
|
|
res['ready_status'] = c.call('GET', '/health/ready')['status']
|
|
# expected 0: compensation deletes the pending row; a possible object is an orphan for the reconciler
|
|
res['pending_rows_left'] = int(psql(
|
|
"SELECT count(*) FROM attachments WHERE status = 'pending'")) - int(pending_before)
|
|
finally:
|
|
sh('docker', 'compose', 'up', '-d', '--no-recreate', '--wait', 's3')
|
|
t0 = time.monotonic()
|
|
while True:
|
|
try:
|
|
upload(c, alice, chat)
|
|
break
|
|
except Failure:
|
|
check(time.monotonic() - t0 < 60, 'uploads did not recover after S3 start')
|
|
time.sleep(0.5)
|
|
res['upload_recovered_after_s3_healthy_s'] = round(time.monotonic() - t0, 1)
|
|
d = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)
|
|
res['old_file_intact'] = hashlib.sha256(d['bytes']).hexdigest() == att['sha256']
|
|
check(res['upload_status'] == 503 and res['text_message_status'] == 201 and res['ready_status'] == 200,
|
|
'S3 outage is not isolated to attachment endpoints')
|
|
return res
|
|
|
|
|
|
def cmd_no_local_file(a):
|
|
out = {}
|
|
for r in api_containers():
|
|
mounts = json.loads(sh('docker', 'inspect', '-f', '{{json .Mounts}}', r['id']))
|
|
ro = sh('docker', 'inspect', '-f', '{{.HostConfig.ReadonlyRootfs}}', r['id'])
|
|
probe = sh('docker', 'exec', r['id'], 'sh', '-c', 'touch /app/probe 2>&1 || true')
|
|
# Docker itself bind-mounts /etc/hosts, /etc/hostname, /etc/resolv.conf
|
|
files = sh('docker', 'exec', r['id'], 'sh', '-c',
|
|
'find / -xdev -type f -newer /app/server ! -path "/proc/*" ! -path "/sys/*" '
|
|
'! -path "/run/secrets/*" ! -name hosts ! -name hostname ! -name resolv.conf '
|
|
'! -name .dockerenv 2>/dev/null | head -20')
|
|
out[r['name']] = {'readonly_rootfs': ro == 'true', 'write_probe': probe,
|
|
'mounts': [m.get('Destination') for m in mounts],
|
|
'files_written_since_start': files.splitlines()}
|
|
check(ro == 'true' and not files.strip(), 'API container can store files locally')
|
|
return out
|
|
|
|
|
|
def cmd_bootstrap(a):
|
|
runs = []
|
|
for _ in range(2):
|
|
for svc in ('migrate', 's3-init'):
|
|
log = sh('docker', 'compose', 'run', '--rm', '--no-deps', '-T', svc)
|
|
runs.append({'service': svc, 'last_log': json.loads(log.strip().splitlines()[-1])})
|
|
return runs
|
|
|
|
|
|
def peak(cid):
|
|
v = sh('docker', 'exec', cid, 'cat', '/sys/fs/cgroup/memory.peak', '/sys/fs/cgroup/memory.current').split()
|
|
return int(v[0]) / 2**20, int(v[1]) / 2**20
|
|
|
|
|
|
def cmd_memory(a):
|
|
big = imagegen.near_limit_png()
|
|
c = Client(a.base_url, timeout=120)
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
results = []
|
|
for conc in a.levels:
|
|
for rep in range(a.repeat):
|
|
sh('docker', 'compose', 'restart', 'api')
|
|
for r in api_containers():
|
|
check(wait_healthy(r['id'], 60), 'api not healthy')
|
|
time.sleep(1)
|
|
s3_id = sh('docker', 'compose', 'ps', '-q', 's3')
|
|
s3_restarts = sh('docker', 'inspect', '-f', '{{.RestartCount}}', s3_id)
|
|
upload(c, alice, chat, big) # warm-up allocations of one replica
|
|
base = {r['name']: peak(r['id']) for r in api_containers()}
|
|
barrier = threading.Barrier(conc)
|
|
|
|
def one(i):
|
|
local = Client(a.base_url, timeout=120)
|
|
barrier.wait()
|
|
try:
|
|
x = local.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=big.png',
|
|
[201, 503], actor=alice, raw=big, headers={'Content-Type': 'image/png'})
|
|
return x['status'], x['elapsed']
|
|
except Failure as exc:
|
|
return str(exc)[-40:], 0
|
|
|
|
t0 = time.monotonic()
|
|
with ThreadPoolExecutor(conc) as pool:
|
|
res = list(pool.map(one, range(conc)))
|
|
wall = time.monotonic() - t0
|
|
after = {r['name']: peak(r['id']) for r in api_containers()}
|
|
lat = sorted(e for s, e in res if s == 201)
|
|
row = {'concurrency': conc, 'repeat': rep + 1, 'file_bytes': len(big),
|
|
'ok': sum(1 for s, _ in res if s == 201), 'status_503': sum(1 for s, _ in res if s == 503),
|
|
'other': [s for s, _ in res if s not in (201, 503)], 'wall_s': round(wall, 2),
|
|
'p50_ms': round(1000 * lat[len(lat) // 2]) if lat else None,
|
|
'max_ms': round(1000 * lat[-1]) if lat else None,
|
|
'peak_mib_per_replica': {k: round(v[0], 1) for k, v in after.items()},
|
|
'baseline_current_mib': {k: round(v[1], 1) for k, v in base.items()}}
|
|
row['max_peak_mib'] = max(row['peak_mib_per_replica'].values())
|
|
row['s3_peak_mib_since_start'] = round(peak(s3_id)[0], 1)
|
|
row['s3_restarted_during_run'] = sh('docker', 'inspect', '-f', '{{.RestartCount}}', s3_id) != s3_restarts
|
|
print(json.dumps(row, ensure_ascii=False), flush=True)
|
|
results.append(row)
|
|
return {'api_limit': '512M x 2 replicas, 1 CPU each', 'gomemlimit': '400MiB',
|
|
'upload_concurrency_per_replica': 8, 'decode_budget_mib': 128,
|
|
's3_limit': sh('docker', 'inspect', '-f', '{{.HostConfig.Memory}}', sh('docker', 'compose', 'ps', '-q', 's3')),
|
|
'runs': results}
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument('cmd', choices=['private', 'failover-upload', 's3-outage', 'no-local-file', 'bootstrap', 'memory'])
|
|
p.add_argument('--base-url', default='http://localhost:8080')
|
|
p.add_argument('--output')
|
|
p.add_argument('--levels', type=int, nargs='+', default=[1, 10, 20, 40])
|
|
p.add_argument('--repeat', type=int, default=2)
|
|
a = p.parse_args()
|
|
fn = globals()['cmd_' + a.cmd.replace('-', '_')]
|
|
result = {'command': a.cmd, 'date_utc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
|
|
'commit': sh('git', 'rev-parse', '--short', 'HEAD', check=False), 'result': fn(a)}
|
|
text = json.dumps(result, ensure_ascii=False, indent=2)
|
|
print(text)
|
|
if a.output:
|
|
Path(a.output).write_text(text + '\n', encoding='utf-8')
|
|
print('PASS: ' + a.cmd)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|