135 lines
5.8 KiB
Python
135 lines
5.8 KiB
Python
"""Actual replica stop after upload, interrupted request, and dependency outages."""
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
import uuid
|
|
from urllib.parse import urlsplit
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client, Failure
|
|
|
|
|
|
def docker(*args, code=None):
|
|
result = subprocess.run(['docker', *args], input=code, text=True, capture_output=True, check=True)
|
|
return result.stdout.strip()
|
|
|
|
|
|
def instance(container):
|
|
return docker('exec', container, 'python', '-c', "import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080/health/live').headers['X-Instance-Id'])")
|
|
|
|
|
|
def wait(c):
|
|
deadline = time.monotonic() + 60
|
|
while True:
|
|
try:
|
|
c.call('GET', '/health/ready', schema='Health')
|
|
return
|
|
except Failure:
|
|
if time.monotonic() > deadline:
|
|
raise
|
|
time.sleep(.2)
|
|
|
|
|
|
def main():
|
|
c = Client()
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
|
containers = docker('compose', 'ps', '-q', 'api').splitlines()
|
|
assert len(containers) == 2
|
|
report = {'replicas': []}
|
|
for container in containers:
|
|
alias = instance(container)
|
|
for attempt in range(20):
|
|
response = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=failover.png', 201, actor=alice, raw=content, headers={'Content-Type': 'image/png'}, schema='Attachment')
|
|
if response['headers']['X-Instance-Id'] == alias:
|
|
break
|
|
else:
|
|
raise AssertionError('Target uploader never observed')
|
|
att = response['json']
|
|
message = c.message(alice, chat, 'file before stop', attachments=[att['id']])['json']
|
|
started = time.monotonic()
|
|
try:
|
|
docker('stop', '--time', '8', container)
|
|
state = json.loads(docker('inspect', '--format', '{{json .State}}', container))
|
|
assert not state['Running']
|
|
while True:
|
|
try:
|
|
data = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)
|
|
assert data['headers']['X-Instance-Id'] != alias and data['bytes'] == content
|
|
break
|
|
except Failure:
|
|
assert time.monotonic() - started < 10
|
|
time.sleep(.1)
|
|
key = uuid.uuid4().hex
|
|
write_failures = 0
|
|
while True:
|
|
try:
|
|
c.message(alice, chat, 'write survives stop', key=key)
|
|
break
|
|
except Failure:
|
|
write_failures += 1
|
|
assert time.monotonic() - started < 10
|
|
time.sleep(.1)
|
|
report['replicas'].append({'container': container, 'uploader': alias, 'downloader': data['headers']['X-Instance-Id'], 'confirmed_stopped': True, 'transient_write_failures': write_failures, 'recovery_seconds': round(time.monotonic() - started, 3), 'sha256': hashlib.sha256(data['bytes']).hexdigest(), 'exit_code': state['ExitCode'], 'oom_killed': state['OOMKilled']})
|
|
finally:
|
|
docker('compose', 'up', '-d', '--wait', '--no-deps', 'api')
|
|
wait(c)
|
|
# Send only part of a declared upload and terminate the TCP connection.
|
|
name = 'interrupted_' + uuid.uuid4().hex + '.png'
|
|
target = urlsplit(c.base)
|
|
connection = socket.create_connection((target.hostname, target.port), timeout=3)
|
|
headers = 'POST /api/v1/chats/' + chat['id'] + '/attachments?filename=' + name + ' HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer ' + alice['token'] + '\r\nContent-Type: image/png\r\nContent-Length: 10485760\r\nConnection: close\r\n\r\n'
|
|
connection.sendall(headers.encode() + content[:30])
|
|
connection.shutdown(socket.SHUT_WR)
|
|
connection.close()
|
|
time.sleep(.5)
|
|
rows = docker('compose', 'exec', '-T', 'api', 'python', '-c', "import sys; sys.path.insert(0,'src'); from postgres import setting; import psycopg; c=psycopg.connect(setting('DATABASE_URL')); print(c.execute('SELECT count(*) FROM attachments WHERE 1=1 AND filename = %s', ('" + name + "',)).fetchone()[0])")
|
|
assert rows == '0'
|
|
report['interrupted_before_validation_leaves_no_intent'] = True
|
|
# Observe API directly inside its container after the LB removes unready backends.
|
|
probe_code = """import sys, json, urllib.request, urllib.error
|
|
sys.path.insert(0,'src')
|
|
from postgres import Store
|
|
from common import APIError
|
|
s=Store()
|
|
try:
|
|
s.ready(); ready=200
|
|
except APIError as e: ready=e.status
|
|
try:
|
|
s.objects.get(%r); content_status=200
|
|
except APIError as e: content_status=e.status
|
|
live=urllib.request.urlopen('http://127.0.0.1:8080/health/live').status
|
|
print(json.dumps({'ready':ready,'content':content_status,'live':live}))
|
|
s.close()
|
|
""" % ('originals/' + att['id'])
|
|
try:
|
|
docker('compose', 'stop', 's3')
|
|
result = json.loads(docker('compose', 'exec', '-T', 'api', 'python', '-c', probe_code))
|
|
assert result == {'ready': 503, 'content': 503, 'live': 200}
|
|
report['s3_outage'] = result
|
|
finally:
|
|
docker('compose', 'up', '-d', '--wait', '--no-deps', 's3')
|
|
wait(c)
|
|
assert c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes'] == content
|
|
try:
|
|
docker('compose', 'stop', 'redis')
|
|
assert c.call('GET', '/api/v1/users/me', actor=alice, schema='User')['json'] == alice['user']
|
|
c.message(alice, chat, 'Redis fallback')
|
|
report['redis_outage_fallback'] = True
|
|
finally:
|
|
docker('compose', 'up', '-d', '--wait', '--no-deps', 'redis')
|
|
wait(c)
|
|
print(json.dumps(report, indent=2))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|