Files
mtusi2026lab4/student-tests/faults.py
T

135 lines
5.8 KiB
Python

"""Actual replica stop after upload, interrupted request, and dependency outages."""
import hashlib
import json
import os
from pathlib import Path
import socket
import subprocess
import sys
import time
import urllib.error
import urllib.request
import uuid
from urllib.parse import urlsplit
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure
def docker(*args, code=None):
result = subprocess.run(['docker', *args], input=code, text=True, capture_output=True, check=True)
return result.stdout.strip()
def instance(container):
return docker('exec', container, 'python', '-c', "import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080/health/live').headers['X-Instance-Id'])")
def wait(c):
deadline = time.monotonic() + 60
while True:
try:
c.call('GET', '/health/ready', schema='Health')
return
except Failure:
if time.monotonic() > deadline:
raise
time.sleep(.2)
def main():
c = Client()
alice, bob, eve, chat = c.bootstrap()
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
containers = docker('compose', 'ps', '-q', 'api').splitlines()
assert len(containers) == 2
report = {'replicas': []}
for container in containers:
alias = instance(container)
for attempt in range(20):
response = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=failover.png', 201, actor=alice, raw=content, headers={'Content-Type': 'image/png'}, schema='Attachment')
if response['headers']['X-Instance-Id'] == alias:
break
else:
raise AssertionError('Target uploader never observed')
att = response['json']
message = c.message(alice, chat, 'file before stop', attachments=[att['id']])['json']
started = time.monotonic()
try:
docker('stop', '--time', '8', container)
state = json.loads(docker('inspect', '--format', '{{json .State}}', container))
assert not state['Running']
while True:
try:
data = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)
assert data['headers']['X-Instance-Id'] != alias and data['bytes'] == content
break
except Failure:
assert time.monotonic() - started < 10
time.sleep(.1)
key = uuid.uuid4().hex
write_failures = 0
while True:
try:
c.message(alice, chat, 'write survives stop', key=key)
break
except Failure:
write_failures += 1
assert time.monotonic() - started < 10
time.sleep(.1)
report['replicas'].append({'container': container, 'uploader': alias, 'downloader': data['headers']['X-Instance-Id'], 'confirmed_stopped': True, 'transient_write_failures': write_failures, 'recovery_seconds': round(time.monotonic() - started, 3), 'sha256': hashlib.sha256(data['bytes']).hexdigest(), 'exit_code': state['ExitCode'], 'oom_killed': state['OOMKilled']})
finally:
docker('compose', 'up', '-d', '--wait', '--no-deps', 'api')
wait(c)
# Send only part of a declared upload and terminate the TCP connection.
name = 'interrupted_' + uuid.uuid4().hex + '.png'
target = urlsplit(c.base)
connection = socket.create_connection((target.hostname, target.port), timeout=3)
headers = 'POST /api/v1/chats/' + chat['id'] + '/attachments?filename=' + name + ' HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer ' + alice['token'] + '\r\nContent-Type: image/png\r\nContent-Length: 10485760\r\nConnection: close\r\n\r\n'
connection.sendall(headers.encode() + content[:30])
connection.shutdown(socket.SHUT_WR)
connection.close()
time.sleep(.5)
rows = docker('compose', 'exec', '-T', 'api', 'python', '-c', "import sys; sys.path.insert(0,'src'); from postgres import setting; import psycopg; c=psycopg.connect(setting('DATABASE_URL')); print(c.execute('SELECT count(*) FROM attachments WHERE 1=1 AND filename = %s', ('" + name + "',)).fetchone()[0])")
assert rows == '0'
report['interrupted_before_validation_leaves_no_intent'] = True
# Observe API directly inside its container after the LB removes unready backends.
probe_code = """import sys, json, urllib.request, urllib.error
sys.path.insert(0,'src')
from postgres import Store
from common import APIError
s=Store()
try:
s.ready(); ready=200
except APIError as e: ready=e.status
try:
s.objects.get(%r); content_status=200
except APIError as e: content_status=e.status
live=urllib.request.urlopen('http://127.0.0.1:8080/health/live').status
print(json.dumps({'ready':ready,'content':content_status,'live':live}))
s.close()
""" % ('originals/' + att['id'])
try:
docker('compose', 'stop', 's3')
result = json.loads(docker('compose', 'exec', '-T', 'api', 'python', '-c', probe_code))
assert result == {'ready': 503, 'content': 503, 'live': 200}
report['s3_outage'] = result
finally:
docker('compose', 'up', '-d', '--wait', '--no-deps', 's3')
wait(c)
assert c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes'] == content
try:
docker('compose', 'stop', 'redis')
assert c.call('GET', '/api/v1/users/me', actor=alice, schema='User')['json'] == alice['user']
c.message(alice, chat, 'Redis fallback')
report['redis_outage_fallback'] = True
finally:
docker('compose', 'up', '-d', '--wait', '--no-deps', 'redis')
wait(c)
print(json.dumps(report, indent=2))
if __name__ == '__main__':
main()