Files
mtusi2026lab6/student-tests/attachment_resilience.py
T
2026-11-05 21:17:50 +03:00

121 lines
5.5 KiB
Python

#!/usr/bin/env python3
"""Explicit local Docker tests: private bucket, uploader failover, S3 outage."""
import json
from pathlib import Path
import subprocess
import sys
import time
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, arguments, check, run
def compose(*args, input=None):
return subprocess.check_output(['docker', 'compose', *args], cwd=ROOT, input=input,
text=True, stderr=subprocess.DEVNULL)
def wait_ready(c):
deadline = time.monotonic() + 90
while True:
try:
c.call('GET', '/health/ready', schema='Health')
return
except Failure:
check(time.monotonic() < deadline, 'stack did not recover')
time.sleep(1)
def main():
def configure(parser):
parser.add_argument('--mode', required=True, choices=['private', 'failover', 's3-outage'])
args = arguments(__doc__, configure)
c = Client(args.base_url, args.ca_file)
alice, bob, eve, chat = c.bootstrap()
data = (ROOT / 'tests/fixtures/sample.png').read_bytes()
uploaded = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png',
201, actor=alice, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment')
att = uploaded['json']
content_path = '/api/v1/attachments/' + att['id'] + '/content'
if args.mode == 'private':
script = '''import sys, urllib.request, urllib.error
url = "http://s3:8333/messenger-attachments/originals/" + sys.stdin.read().strip()
try:
urllib.request.urlopen(url, timeout=5)
except urllib.error.HTTPError as error:
assert error.code == 403, error.code
print("PASS: anonymous S3 GET returns 403 for an existing object")
else:
raise AssertionError("private object leaked")
'''
print(compose('exec', '-T', 'api-a', '/app/.venv/bin/python', '-c', script, input=att['id']).strip())
elif args.mode == 'failover':
target = uploaded['headers']['X-Instance-Id']
service = None
for candidate in ('api-a', 'api-b'):
container = compose('ps', '-q', candidate).strip()
hostname = subprocess.check_output(['docker', 'inspect', '--format', '{{.Config.Hostname}}', container], text=True).strip()
if hostname == target:
service = candidate
check(service is not None, 'uploading replica not found')
try:
compose('stop', service)
deadline = time.monotonic() + 15
while True:
try:
download = c.call('GET', content_path, actor=bob)
break
except Failure:
check(time.monotonic() < deadline, 'download did not recover')
time.sleep(0.5)
check(download['headers']['X-Instance-Id'] != target, 'download used stopped uploader')
check(download['bytes'] == data, 'original changed after stopping uploader')
print('PASS: uploaded via ' + service + '; stopped that container; original downloaded through another replica')
finally:
compose('start', service)
wait_ready(c)
else:
try:
compose('stop', 's3')
# Direct API request inside the network: LB excludes unready backends.
script = '''import json, sys, urllib.request, urllib.error
payload = json.load(sys.stdin)
for method, path, body, headers in [
("GET", "/health/ready", None, {}),
("GET", payload["path"], None, {"Authorization": "Bearer " + payload["token"]}),
("POST", payload["upload_path"], bytes.fromhex(payload["bytes"]),
{"Authorization": "Bearer " + payload["token"], "Content-Type": "image/png"}),
]:
req = urllib.request.Request("http://localhost:8080" + path, data=body, headers=headers, method=method)
try:
urllib.request.urlopen(req, timeout=15)
except urllib.error.HTTPError as error:
assert error.code == 503, error.code
document = json.load(error)
assert document["error"]["code"] == "unavailable"
assert document["error"]["request_id"] == error.headers["X-Request-Id"]
else:
raise AssertionError("S3 outage was not reported")
assert urllib.request.urlopen("http://localhost:8080/health/live").status == 200
print("PASS: S3 outage gives 503 for readiness, download and upload; liveness stays 200")
'''
payload = {'path': content_path, 'token': bob['token'], 'bytes': data.hex(),
'upload_path': '/api/v1/chats/' + chat['id'] + '/attachments?filename=outage.png'}
print(compose('exec', '-T', 'api-a', '/app/.venv/bin/python', '-c', script,
input=json.dumps(payload)).strip())
finally:
compose('start', 's3')
# HeadBucket/read recovery alone does not prove the write path is ready.
# Use the same write-probing bootstrap as the documented restart hook.
compose('run', '--rm', '--no-deps', 's3-init')
wait_ready(c)
check(c.call('GET', content_path, actor=bob)['bytes'] == data, 'original lost after S3 outage')
c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=after-recovery.png',
201, actor=alice, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment')
print('PASS: original available and new upload accepted after S3 restart/bootstrap')
if __name__ == '__main__':
run(main)