[+] lab3
This commit is contained in:
Executable
+28
@@ -0,0 +1,28 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
umask 077
|
||||
mkdir -p secrets
|
||||
python3 - <<'PY'
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
root = Path('secrets')
|
||||
p = root / 'postgres_password'
|
||||
if not p.exists():
|
||||
p.write_text(secrets.token_hex(24))
|
||||
pw = p.read_text().strip()
|
||||
for name, value in {
|
||||
'database_url': f'postgresql://messenger:{pw}@postgres:5432/messenger',
|
||||
'cache_url': 'redis://redis:6379/0',
|
||||
}.items():
|
||||
path = root / name
|
||||
if not path.exists():
|
||||
path.write_text(value)
|
||||
# Bind-mounted Docker secrets must be readable by container uid 10001.
|
||||
# Parent directory remains 0700 on the host.
|
||||
for path in root.iterdir():
|
||||
path.chmod(0o444)
|
||||
if not Path('.env').exists():
|
||||
Path('.env').write_text('SESSION_TTL_SECONDS=3600\n')
|
||||
print('Local secrets ready (values hidden).')
|
||||
PY
|
||||
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Capture repeatable short load run and sanitized environment/resource evidence."""
|
||||
import datetime
|
||||
import json
|
||||
import platform
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import cluster
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def output(args):
|
||||
return subprocess.check_output(args, cwd=ROOT, text=True).strip()
|
||||
|
||||
|
||||
def counts():
|
||||
code = "import json,psycopg; from messenger.db import setting; c=psycopg.connect(setting('DATABASE_URL')); print(json.dumps({t:c.execute('SELECT count(*) FROM '+t).fetchone()[0] for t in ['users','chats','messages','sessions','idempotency_records']})); c.close()"
|
||||
return json.loads(output(['docker','exec',cluster.container_ids('api')[0],'python','-c',code]))
|
||||
|
||||
|
||||
def resource_summary(samples):
|
||||
peaks = {}
|
||||
for sample in samples:
|
||||
for container in sample['containers']:
|
||||
peak = peaks.setdefault(container['Name'], {'cpu_percent_peak':0, 'memory_mib_peak':0})
|
||||
peak['cpu_percent_peak'] = max(peak['cpu_percent_peak'],float(container['CPUPerc'].rstrip('%')))
|
||||
value,unit = re.match(r'([0-9.]+)([A-Za-z]+)',container['MemUsage']).groups()
|
||||
multiplier = {'GiB':1024,'MiB':1,'KiB':1/1024,'B':1/1048576}[unit]
|
||||
peak['memory_mib_peak'] = round(max(peak['memory_mib_peak'],float(value)*multiplier),3)
|
||||
return {'samples':len(samples),'per_container_peaks':peaks}
|
||||
|
||||
|
||||
def main():
|
||||
names = cluster.container_ids()
|
||||
environment = {
|
||||
'captured_at_utc': datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
'host_os': platform.platform(), 'host_architecture': platform.machine(),
|
||||
'docker': json.loads(output(['docker','info','--format','{"server_version":"{{.ServerVersion}}","cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}},"architecture":"{{.Architecture}}"}'])),
|
||||
'python_in_image': output(['docker','exec',cluster.container_ids('api')[0],'python','--version']),
|
||||
'dataset_before': counts(),
|
||||
'limits': {'api_each':'0.75 CPU / 384 MiB','postgres':'1 CPU / 512 MiB','redis':'0.25 CPU / 128 MiB','lb':'0.5 CPU / 128 MiB'},
|
||||
'warmup': 'API already healthy; acceptance, persistence and dependency checks performed before benchmark. No separate timed load warmup.',
|
||||
'replicas':len(cluster.replicas()), 'repetitions': 1, 'duration_seconds':30, 'concurrency':4,
|
||||
}
|
||||
if sys.platform == 'darwin':
|
||||
environment['host_cpu'] = output(['sysctl','-n','machdep.cpu.brand_string'])
|
||||
environment['host_memory_bytes'] = int(output(['sysctl','-n','hw.memsize']))
|
||||
samples = []
|
||||
with (ROOT/'evidence/load-output.txt').open('w') as log:
|
||||
proc = subprocess.Popen([sys.executable,'tests/load.py','--duration','30','--concurrency','4','--output','evidence/load.json'],cwd=ROOT,stdout=log,stderr=subprocess.STDOUT)
|
||||
while proc.poll() is None:
|
||||
rows = output(['docker','stats','--no-stream','--format','{{json .}}',*names])
|
||||
samples.append({'at_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
'containers':[{k:v for k,v in json.loads(row).items() if k in ('Name','CPUPerc','MemUsage','MemPerc','PIDs')}
|
||||
for row in rows.splitlines()]})
|
||||
time.sleep(1)
|
||||
if proc.returncode:
|
||||
raise SystemExit(proc.returncode)
|
||||
environment['dataset_after'] = counts()
|
||||
(ROOT/'evidence/environment.json').write_text(json.dumps(environment,ensure_ascii=False,indent=2)+'\n')
|
||||
(ROOT/'evidence/resources.json').write_text(json.dumps(samples,indent=2)+'\n')
|
||||
(ROOT/'evidence/resource-summary.json').write_text(json.dumps(resource_summary(samples),indent=2)+'\n')
|
||||
print((ROOT/'evidence/load.json').read_text())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+16
@@ -0,0 +1,16 @@
|
||||
#!/bin/sh
|
||||
# A unique disposable Compose project; never touches normal project volumes.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
project="mtusi-lab3-migration-check-$$"
|
||||
if [ -n "$(docker compose -p "$project" ps -aq)" ]; then
|
||||
echo 'Refusing to reuse a pre-existing scratch project' >&2
|
||||
exit 1
|
||||
fi
|
||||
cleanup() {
|
||||
docker compose -p "$project" down --volumes >/dev/null 2>&1
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
docker compose -p "$project" up -d --wait --wait-timeout 90 postgres redis
|
||||
docker compose -p "$project" run --rm --no-deps api sh -c 'python -m messenger.migrate && python -m messenger.migrate && python -m pytest student-tests -q -o cache_dir=/tmp/pytest-cache'
|
||||
echo 'PASS: migrations and application tests on fresh disposable PostgreSQL and Redis volumes.'
|
||||
@@ -0,0 +1,196 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Local Compose orchestration. No credentials, tokens or environment dumps in evidence."""
|
||||
import argparse
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BASE_URL = os.environ.get('BASE_URL', 'http://localhost:8080').rstrip('/')
|
||||
|
||||
|
||||
def command(args, timeout=120):
|
||||
return subprocess.run(args, cwd=ROOT, check=True, capture_output=True, text=True, timeout=timeout).stdout.strip()
|
||||
|
||||
|
||||
def compose(*args, timeout=120):
|
||||
return command(['docker', 'compose', *args], timeout=timeout)
|
||||
|
||||
|
||||
def inspect(containers):
|
||||
if not containers:
|
||||
return []
|
||||
return json.loads(command(['docker','inspect',*containers]))
|
||||
|
||||
|
||||
def container_ids(service=None, all_containers=False):
|
||||
args = ['ps'] + (['--all'] if all_containers else []) + ['-q'] + ([service] if service else [])
|
||||
return compose(*args).split()
|
||||
|
||||
|
||||
def probe(container, host='127.0.0.1', path='/health/live'):
|
||||
code = '''import json,sys,urllib.request,urllib.error
|
||||
try:
|
||||
r=urllib.request.urlopen('http://'+sys.argv[1]+':8080'+sys.argv[2],timeout=4)
|
||||
except urllib.error.HTTPError as exc:
|
||||
r=exc
|
||||
print(json.dumps({'status':r.code,'instance':r.headers.get('X-Instance-Id'),'release':r.headers.get('X-App-Release')}))
|
||||
'''
|
||||
return json.loads(command(['docker','exec',container,'python','-c',code,host,path], timeout=8))
|
||||
|
||||
|
||||
def replicas():
|
||||
items = []
|
||||
for data in inspect(container_ids('api')):
|
||||
if data['State']['Running']:
|
||||
result = probe(data['Id'])
|
||||
items.append({'container':data['Id'], 'name':data['Name'].lstrip('/'),
|
||||
'instance':result['instance'], 'release':result['release'], 'image_id':data['Image']})
|
||||
return items
|
||||
|
||||
|
||||
def snapshot():
|
||||
items = []
|
||||
for data in inspect(container_ids(all_containers=True)):
|
||||
service = data['Config']['Labels']['com.docker.compose.service']
|
||||
item = {
|
||||
'service':service, 'container':data['Id'], 'name':data['Name'].lstrip('/'),
|
||||
'status':data['State']['Status'], 'health':data['State'].get('Health',{}).get('Status'),
|
||||
'started_at':data['State']['StartedAt'], 'image':data['Config']['Image'], 'image_id':data['Image'],
|
||||
'memory_bytes':data['HostConfig']['Memory'], 'nano_cpus':data['HostConfig']['NanoCpus'],
|
||||
'published_ports':data['HostConfig']['PortBindings'],
|
||||
'volumes':sorted(m['Name'] for m in data['Mounts'] if m['Type']=='volume'),
|
||||
'networks':sorted(data['NetworkSettings']['Networks']),
|
||||
}
|
||||
if service == 'api' and data['State']['Running']:
|
||||
result = probe(data['Id'])
|
||||
item.update({'http_status':result['status'], 'instance':result['instance'], 'release':result['release']})
|
||||
items.append(item)
|
||||
return items
|
||||
|
||||
|
||||
def wait_healthy(container, timeout=60):
|
||||
deadline = time.monotonic()+timeout
|
||||
while time.monotonic() < deadline:
|
||||
data = inspect([container])[0]
|
||||
if data['State'].get('Health',{}).get('Status') == 'healthy':
|
||||
return
|
||||
time.sleep(0.3)
|
||||
raise RuntimeError('Container did not become healthy')
|
||||
|
||||
|
||||
def wait_cluster(count=2, timeout=60):
|
||||
deadline = time.monotonic()+timeout
|
||||
seen = set()
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(BASE_URL+'/health/ready',timeout=5) as response:
|
||||
alias = response.headers.get('X-Instance-Id')
|
||||
if alias:
|
||||
seen.add(alias)
|
||||
if len(seen) >= count:
|
||||
return sorted(seen)
|
||||
except (OSError, urllib.error.URLError):
|
||||
pass
|
||||
time.sleep(0.1)
|
||||
raise RuntimeError('Load balancer did not discover required API instances')
|
||||
|
||||
|
||||
def stop_one(target):
|
||||
if not re.fullmatch(r'[a-f0-9]{32}', target):
|
||||
raise ValueError('Invalid target instance alias')
|
||||
matching = [item for item in replicas() if item['instance']==target]
|
||||
if len(matching) != 1:
|
||||
raise RuntimeError('Target must match exactly one running API process')
|
||||
item = matching[0]
|
||||
started = time.monotonic()
|
||||
command(['docker','stop','--time','7',item['container']],timeout=9)
|
||||
state = inspect([item['container']])[0]['State']
|
||||
if state['Running'] or state['ExitCode'] != 0:
|
||||
raise RuntimeError('API did not stop gracefully')
|
||||
result = {**item, 'stopped_at_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
'stop_seconds':round(time.monotonic()-started,4), 'exit_code':state['ExitCode'], 'left_stopped':True}
|
||||
(ROOT/'evidence/last-stop.json').write_text(json.dumps(result,indent=2)+'\n')
|
||||
return result
|
||||
|
||||
|
||||
def restart():
|
||||
compose('restart','api','postgres','redis','lb',timeout=30)
|
||||
compose('up','-d','--no-recreate','--wait','--wait-timeout','35',timeout=40)
|
||||
return wait_cluster(2,timeout=15)
|
||||
|
||||
|
||||
def front_address(container):
|
||||
networks = inspect([container])[0]['NetworkSettings']['Networks']
|
||||
return next(value['IPAddress'] for name,value in networks.items() if name.endswith('_front'))
|
||||
|
||||
|
||||
def replace_one(container):
|
||||
before = set(container_ids('api'))
|
||||
if container not in before or len(before) != 2:
|
||||
raise RuntimeError('Replacement expects exactly two running API containers')
|
||||
old_address = front_address(container)
|
||||
# Start-first ensures a genuinely different IP while the old address is still occupied.
|
||||
compose('up','-d','--no-deps','--no-recreate','--scale','api=3','api')
|
||||
expanded = set(container_ids('api'))
|
||||
added = expanded-before
|
||||
if len(added) != 1 or not before.issubset(expanded):
|
||||
raise RuntimeError('Start-first step unexpectedly changed an existing container')
|
||||
replacement = added.pop()
|
||||
wait_healthy(replacement)
|
||||
new_address = front_address(replacement)
|
||||
if new_address == old_address:
|
||||
raise RuntimeError('Replacement did not get a different IP')
|
||||
expected_three = {item['instance'] for item in replicas()}
|
||||
if set(wait_cluster(3)) != expected_three:
|
||||
raise RuntimeError('Load balancer did not discover all three live addresses')
|
||||
command(['docker','stop','--time','7',container],timeout=10)
|
||||
state = inspect([container])[0]['State']
|
||||
if state['ExitCode'] != 0:
|
||||
raise RuntimeError('Old release did not terminate gracefully')
|
||||
command(['docker','rm',container])
|
||||
compose('up','-d','--no-deps','--no-recreate','--scale','api=2','api')
|
||||
after = set(container_ids('api'))
|
||||
if after != (before-{container}) | {replacement}:
|
||||
raise RuntimeError('Sequential replacement changed the wrong containers')
|
||||
expected = {item['instance'] for item in replicas()}
|
||||
if set(wait_cluster(2)) != expected:
|
||||
raise RuntimeError('Unexpected final backend aliases')
|
||||
return {'old_container':container,'new_container':replacement,
|
||||
'old_ip':old_address,'new_ip':new_address,'dns_discovered_new_address':True,
|
||||
'instances_during_start_first':sorted(expected_three),
|
||||
'observed_instances':sorted(expected),'old_exit_code':state['ExitCode']}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('action',choices=['wait','snapshot','stop-one','restart','rolling','exec'])
|
||||
parser.add_argument('arguments',nargs=argparse.REMAINDER)
|
||||
args = parser.parse_args()
|
||||
if args.action == 'wait':
|
||||
print(json.dumps({'instances':wait_cluster()}))
|
||||
elif args.action == 'snapshot':
|
||||
print(json.dumps(snapshot(),indent=2))
|
||||
elif args.action == 'stop-one':
|
||||
print(json.dumps(stop_one(os.environ['TARGET_INSTANCE_ID'])))
|
||||
elif args.action == 'exec':
|
||||
ids = container_ids('api')
|
||||
if not ids or not args.arguments:
|
||||
raise RuntimeError('A live API container and command are required')
|
||||
raise SystemExit(subprocess.call(['docker','exec',ids[0],*args.arguments],cwd=ROOT))
|
||||
elif args.action == 'restart':
|
||||
print(json.dumps({'instances':restart()}))
|
||||
else:
|
||||
compose('run','--rm','--no-deps','migrate')
|
||||
print(json.dumps([replace_one(item['container']) for item in replicas()],indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
exec python3 scripts/cluster.py restart
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
exec python3 scripts/cluster.py stop-one
|
||||
@@ -0,0 +1,443 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Grade4 distributed acceptance through the only public entry point; no tokens in artifacts."""
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import uuid
|
||||
|
||||
import cluster
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0,str(ROOT/'tests'))
|
||||
from client import Client, Failure, check
|
||||
from resilience import history
|
||||
|
||||
|
||||
def save(name, value):
|
||||
(ROOT/'evidence'/name).write_text(json.dumps(value,ensure_ascii=False,indent=2)+'\n')
|
||||
print('PASS:', name)
|
||||
|
||||
|
||||
def percentiles(values):
|
||||
values = sorted(values)
|
||||
return {name:round(1000*values[max(0,math.ceil(p*len(values))-1)],3) if values else None
|
||||
for name,p in [('p50',.50),('p95',.95),('p99',.99)]}
|
||||
|
||||
|
||||
class Traffic:
|
||||
def __init__(self, concurrency=4):
|
||||
self.c = Client(timeout=6)
|
||||
self.alice,self.bob,_,self.chat = self.c.bootstrap()
|
||||
self.concurrency = concurrency
|
||||
self.done = threading.Event()
|
||||
self.lock = threading.Lock()
|
||||
self.attempts = []
|
||||
self.confirmed = {}
|
||||
self.prefix = uuid.uuid4().hex
|
||||
|
||||
def worker(self, number):
|
||||
client = Client(timeout=6)
|
||||
index = 0
|
||||
while not self.done.is_set():
|
||||
text = self.prefix+'-'+str(number)+'-'+str(index)
|
||||
key = self.prefix+'-'+str(number)+'-'+str(index)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
response = client.message(self.alice,self.chat,text,key=key)
|
||||
row = response['json']
|
||||
attempt = {'started':started,'ended':time.monotonic(),'ok':True,
|
||||
'latency':response['elapsed'],'instance':response['headers']['X-Instance-Id']}
|
||||
with self.lock:
|
||||
self.confirmed[row['id']] = row
|
||||
index += 1
|
||||
except Failure:
|
||||
# Only idempotent message POST is retried, with its original body and key.
|
||||
attempt = {'started':started,'ended':time.monotonic(),'ok':False,'instance':None}
|
||||
time.sleep(0.02)
|
||||
with self.lock:
|
||||
self.attempts.append(attempt)
|
||||
|
||||
def start(self):
|
||||
self.started = time.monotonic()
|
||||
self.pool = ThreadPoolExecutor(max_workers=self.concurrency)
|
||||
self.futures = [self.pool.submit(self.worker,n) for n in range(self.concurrency)]
|
||||
return self
|
||||
|
||||
def stop(self):
|
||||
self.done.set()
|
||||
for future in self.futures:
|
||||
future.result(timeout=10)
|
||||
self.pool.shutdown()
|
||||
self.ended = time.monotonic()
|
||||
|
||||
def summary(self, start=None, end=None):
|
||||
start = self.started if start is None else start
|
||||
end = getattr(self,'ended',time.monotonic()) if end is None else end
|
||||
with self.lock:
|
||||
attempts = [a for a in self.attempts if start <= a['started'] < end]
|
||||
success = [a for a in attempts if a['ok']]
|
||||
duration = end-start
|
||||
return {'seconds':round(duration,4),'concurrency':self.concurrency,
|
||||
'successful_requests':len(success),'failed_attempts':len(attempts)-len(success),
|
||||
'successful_rps':round(len(success)/duration,3),
|
||||
'error_ratio':round((len(attempts)-len(success))/len(attempts),6) if attempts else 0,
|
||||
'successful_latency_ms':percentiles([a['latency'] for a in success]),
|
||||
'instances':sorted({a['instance'] for a in success})}
|
||||
|
||||
def verify_data(self):
|
||||
rows = history(self.c,self.bob,self.chat)
|
||||
actual = {row['id']:row for row in rows}
|
||||
check(len(actual)==len(rows),'duplicated message ids')
|
||||
check(len({row['text'] for row in rows})==len(rows),'retry duplicated a message')
|
||||
check(all(actual.get(mid)==row for mid,row in self.confirmed.items()),'confirmed message lost or changed')
|
||||
return {'confirmed_unique_messages':len(self.confirmed), 'history_messages':len(rows),
|
||||
'confirmed_preserved':True, 'duplicate_ids':0, 'duplicate_texts':0}
|
||||
|
||||
|
||||
def concurrent_replay():
|
||||
cluster.wait_cluster()
|
||||
c = Client()
|
||||
alice,bob,_,chat = c.bootstrap()
|
||||
key = uuid.uuid4().hex
|
||||
barrier = threading.Barrier(8)
|
||||
def send(_):
|
||||
local = Client()
|
||||
barrier.wait()
|
||||
return local.message(alice,chat,'Один результат через разные процессы',key=key)
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
results = list(pool.map(send,range(8)))
|
||||
instances = {r['headers']['X-Instance-Id'] for r in results}
|
||||
check(len(instances)>=2,'concurrent retry did not reach different API processes')
|
||||
check(all(r['json']==results[0]['json'] for r in results),'concurrent key returned different messages')
|
||||
c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,
|
||||
body={'text':'Другой body'},headers={'Idempotency-Key':key})
|
||||
check(history(c,bob,chat)==[results[0]['json']],'repeated key created duplicates')
|
||||
save('concurrent-idempotency.json',{'result':'PASS','parallel_requests':8,'observed_instances':sorted(instances),
|
||||
'responses_201':8,'messages_created':1,'different_body_status':409})
|
||||
|
||||
|
||||
def verify_persistence():
|
||||
before = cluster.snapshot()
|
||||
result = subprocess.run([sys.executable,'tests/resilience.py','--mode','persistence','--action-script','scripts/restart.sh'],
|
||||
cwd=ROOT,check=True,capture_output=True,text=True,timeout=190)
|
||||
after = cluster.snapshot()
|
||||
roles = ['api','postgres','redis','lb']
|
||||
previous = {item['container']:item for item in before}
|
||||
for item in after:
|
||||
if item['service'] in roles:
|
||||
old = previous[item['container']]
|
||||
check(old['started_at']!=item['started_at'],'service did not restart: '+item['service'])
|
||||
check(old['volumes']==item['volumes'],'volume identities changed')
|
||||
(ROOT/'evidence/persistence.txt').write_text(result.stdout)
|
||||
save('persistence.json',{'result':'PASS','before':before,'after':after,'public_test':result.stdout.strip()})
|
||||
|
||||
|
||||
def public_failover():
|
||||
cluster.wait_cluster()
|
||||
try:
|
||||
result = subprocess.run([sys.executable,'tests/resilience.py','--mode','failover','--action-script','scripts/stop-one.sh'],
|
||||
cwd=ROOT,check=True,capture_output=True,text=True,timeout=100)
|
||||
(ROOT/'evidence/public-failover.txt').write_text(result.stdout)
|
||||
save('public-failover.json',{'result':'PASS','public_test':result.stdout.strip(),
|
||||
'stopped':json.loads((ROOT/'evidence/last-stop.json').read_text())})
|
||||
finally:
|
||||
cluster.compose('up','-d','--no-recreate','--wait','--wait-timeout','90')
|
||||
cluster.wait_cluster()
|
||||
|
||||
|
||||
def reject_new_connection(survivor, address):
|
||||
code = '''import json,socket,sys
|
||||
try:
|
||||
s=socket.create_connection((sys.argv[1],8080),timeout=1)
|
||||
except OSError as exc:
|
||||
print(json.dumps({'accepted':False,'error_type':type(exc).__name__}))
|
||||
else:
|
||||
s.close()
|
||||
print(json.dumps({'accepted':True}))
|
||||
'''
|
||||
return json.loads(cluster.command(['docker','exec',survivor,'python','-c',code,address]))
|
||||
|
||||
|
||||
def failover_each():
|
||||
cluster.wait_cluster()
|
||||
originals = cluster.replicas()
|
||||
check(len(originals)==2,'two running replicas required')
|
||||
traffic = Traffic().start()
|
||||
phases = []
|
||||
try:
|
||||
time.sleep(3)
|
||||
baseline_end = time.monotonic()
|
||||
baseline = traffic.summary(end=baseline_end)
|
||||
for original in originals:
|
||||
# Alias changes on restart, so resolve the current process of this exact container.
|
||||
current = next(r for r in cluster.replicas() if r['container']==original['container'])
|
||||
all_before = cluster.inspect([current['container']])[0]
|
||||
address = next(v['IPAddress'] for k,v in all_before['NetworkSettings']['Networks'].items() if k.endswith('_front'))
|
||||
survivor = next(r['container'] for r in cluster.replicas() if r['container']!=current['container'])
|
||||
read_path = '/api/v1/chats/'+traffic.chat['id']+'/messages?limit=1'
|
||||
marker = traffic.c.call('GET',read_path,actor=traffic.bob,schema='MessagePage')['json']['items'][0]
|
||||
start = time.monotonic()
|
||||
try:
|
||||
env = {**os.environ,'TARGET_INSTANCE_ID':current['instance']}
|
||||
subprocess.run(['scripts/stop-one.sh'],cwd=ROOT,env=env,check=True,capture_output=True,text=True,timeout=10)
|
||||
stop_returned = time.monotonic()
|
||||
stopped = json.loads((ROOT/'evidence/last-stop.json').read_text())
|
||||
read_deadline = start+10
|
||||
while True:
|
||||
try:
|
||||
read = traffic.c.call('GET',read_path,actor=traffic.bob,schema='MessagePage')
|
||||
check(read['json']['items'][0]==marker,'pre-failure message unavailable during outage')
|
||||
check(read['headers']['X-Instance-Id']!=current['instance'],'read served by stopped process')
|
||||
read_done = time.monotonic()
|
||||
break
|
||||
except Failure:
|
||||
check(time.monotonic()<read_deadline,'reads did not recover within10s')
|
||||
time.sleep(0.1)
|
||||
check(read_done-start<=10,'read recovery exceeded10s')
|
||||
rejected = reject_new_connection(survivor,address)
|
||||
check(not rejected['accepted'],'new connection accepted by stopped API')
|
||||
time.sleep(max(0,8-(time.monotonic()-start)))
|
||||
end = time.monotonic()
|
||||
summary = traffic.summary(start,end)
|
||||
with traffic.lock:
|
||||
successful = [a for a in traffic.attempts if a['ok'] and a['started']>=start]
|
||||
recent = [a for a in successful if a['started']>stop_returned]
|
||||
recovery = min(a['ended'] for a in recent)-start if recent else None
|
||||
after_stop = min(a['ended'] for a in recent)-stop_returned if recent else None
|
||||
check(recovery is not None and recovery<=10,'post-stop writes did not recover within10s of action start')
|
||||
check(len(recent)>=5 and all(a['instance']!=current['instance'] for a in recent[-5:]),'stopped alias still served requests')
|
||||
phases.append({'stopped':stopped, 'traffic':summary,'post_stop_write_seconds_from_action_start':round(recovery,4),
|
||||
'post_stop_write_seconds_from_stop_completion':round(after_stop,4),
|
||||
'read_during_outage':{'status':200,'confirmed_message_preserved':True,
|
||||
'seconds_from_action_start':round(read_done-start,4),
|
||||
'seconds_from_stop_completion':round(read_done-stop_returned,4)},
|
||||
'new_connection_after_stop':rejected,'stopped_remains_stopped':not cluster.inspect([current['container']])[0]['State']['Running']})
|
||||
finally:
|
||||
cluster.command(['docker','start',current['container']])
|
||||
cluster.wait_healthy(current['container'])
|
||||
cluster.wait_cluster()
|
||||
time.sleep(2)
|
||||
finally:
|
||||
traffic.stop()
|
||||
save('sequential-failover.json',{'result':'PASS','baseline':baseline,'phases':phases,
|
||||
'total':traffic.summary(),'data':traffic.verify_data(),
|
||||
'each_original_container_tested':len({p['stopped']['container'] for p in phases})==2})
|
||||
|
||||
|
||||
def rolling_update():
|
||||
cluster.wait_cluster()
|
||||
before = cluster.replicas()
|
||||
check(len(before)==2,'rolling demonstration expects two replicas')
|
||||
lb_before = next(item for item in cluster.snapshot() if item['service']=='lb')
|
||||
target_release = 'lab3-'+uuid.uuid4().hex[:8]
|
||||
traffic = Traffic().start()
|
||||
replacements = []
|
||||
try:
|
||||
time.sleep(3)
|
||||
build_start = time.monotonic()
|
||||
cluster.compose('build','--build-arg','APP_RELEASE='+target_release,'api',timeout=180)
|
||||
build_end = time.monotonic()
|
||||
migration_output = cluster.compose('run','--rm','--no-deps','migrate')
|
||||
swap_start = time.monotonic()
|
||||
for old in before:
|
||||
step_start = time.monotonic()
|
||||
result = cluster.replace_one(old['container'])
|
||||
result['seconds'] = round(time.monotonic()-step_start,4)
|
||||
replacements.append(result)
|
||||
swap_end = time.monotonic()
|
||||
time.sleep(3)
|
||||
finally:
|
||||
traffic.stop()
|
||||
after = cluster.replicas()
|
||||
lb_after = next(item for item in cluster.snapshot() if item['service']=='lb')
|
||||
check(lb_before['container']==lb_after['container'] and lb_before['started_at']==lb_after['started_at'],'LB restarted during DNS test')
|
||||
check(all(r['release']==target_release for r in after),'new release not observed on all replicas')
|
||||
check(not {r['container'] for r in before}&{r['container'] for r in after},'old containers remain after replacement')
|
||||
check(not {r['image_id'] for r in before}&{r['image_id'] for r in after},'image did not actually change')
|
||||
save('rolling-update.json',{'result':'PASS','before':before,'after':after,'target_release':target_release,
|
||||
'lb_before':lb_before,'lb_after':lb_after,'lb_not_restarted':True,
|
||||
'build_seconds':round(build_end-build_start,4),'migration_output':migration_output,
|
||||
'replacements':replacements,'during_replacement':traffic.summary(swap_start,swap_end),
|
||||
'total':traffic.summary(),'data':traffic.verify_data(),
|
||||
'change':'New image with a distinct release header; API code and schema remain compatible. Start-first: scale to3, prove new IP in DNS/LB, stop/remove one old container, return to2 with --no-recreate.'})
|
||||
|
||||
|
||||
def database_fixture(container, data):
|
||||
code = '''import json,sys,psycopg
|
||||
from psycopg import sql
|
||||
from messenger.db import setting
|
||||
p=json.load(sys.stdin)
|
||||
with psycopg.connect(setting('DATABASE_URL'),connect_timeout=3) as c:
|
||||
if p['action']=='install':
|
||||
c.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.text = {} THEN PERFORM pg_sleep(2); END IF; RETURN NEW; END $$").format(sql.Identifier(p['function']),sql.Literal(p['text'])))
|
||||
c.execute(sql.SQL('CREATE TRIGGER {} BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(p['trigger']),sql.Identifier(p['function'])))
|
||||
result={'installed':True}
|
||||
elif p['action']=='remove':
|
||||
c.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON messages').format(sql.Identifier(p['trigger'])))
|
||||
c.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(p['function'])))
|
||||
result={'removed':True}
|
||||
else:
|
||||
result={'waiting':c.execute("SELECT count(*) FROM pg_stat_activity WHERE datname=current_database() AND wait_event='PgSleep' AND query LIKE 'INSERT INTO messages%%'").fetchone()[0]}
|
||||
print(json.dumps(result))
|
||||
'''
|
||||
response = subprocess.run(['docker','exec','-i',container,'python','-c',code],cwd=ROOT,
|
||||
input=json.dumps(data),capture_output=True,text=True,check=True,timeout=8)
|
||||
return json.loads(response.stdout)
|
||||
|
||||
|
||||
def graceful_shutdown():
|
||||
cluster.wait_cluster()
|
||||
nodes = cluster.replicas()
|
||||
target,survivor = nodes[0],nodes[1]
|
||||
address = cluster.front_address(target['container'])
|
||||
c = Client()
|
||||
alice,bob,_,chat = c.bootstrap()
|
||||
path = '/api/v1/chats/'+chat['id']+'/messages'
|
||||
marker = 'drain_'+uuid.uuid4().hex
|
||||
fixture = {'function':'drain_'+uuid.uuid4().hex,'trigger':'drain_'+uuid.uuid4().hex,'text':marker}
|
||||
database_fixture(survivor['container'],{**fixture,'action':'install'})
|
||||
code = '''import json,sys,urllib.request,urllib.error
|
||||
p=json.load(sys.stdin)
|
||||
r=urllib.request.Request('http://'+p['address']+':8080'+p['path'],data=json.dumps({'text':p['text']}).encode(),method='POST',headers={'Content-Type':'application/json','Authorization':'Bearer '+p['token'],'Idempotency-Key':p['key']})
|
||||
try:
|
||||
response=urllib.request.urlopen(r,timeout=6)
|
||||
except urllib.error.HTTPError as exc:
|
||||
response=exc
|
||||
print(json.dumps({'status':response.code,'instance':response.headers.get('X-Instance-Id'),'message':json.load(response)}))
|
||||
'''
|
||||
request_data = {'address':address,'path':path,'text':marker,'token':alice['token'],'key':uuid.uuid4().hex}
|
||||
def in_flight():
|
||||
result = subprocess.run(['docker','exec','-i',survivor['container'],'python','-c',code],cwd=ROOT,
|
||||
input=json.dumps(request_data),capture_output=True,text=True,check=True,timeout=8)
|
||||
return json.loads(result.stdout)
|
||||
pool = ThreadPoolExecutor(max_workers=1)
|
||||
future = pool.submit(in_flight)
|
||||
try:
|
||||
deadline = time.monotonic()+3
|
||||
while not database_fixture(survivor['container'],{'action':'waiting'})['waiting']:
|
||||
check(time.monotonic()<deadline and not future.done(),'did not observe an in-flight SQL request')
|
||||
time.sleep(0.05)
|
||||
signaled = time.monotonic()
|
||||
cluster.command(['docker','kill','--signal','SIGTERM',target['container']])
|
||||
deadline = signaled+1.5
|
||||
while True:
|
||||
rejected = reject_new_connection(survivor['container'],address)
|
||||
still_running = cluster.inspect([target['container']])[0]['State']['Running']
|
||||
if not rejected['accepted']:
|
||||
break
|
||||
check(time.monotonic()<deadline,'new TCP connections still accepted after SIGTERM')
|
||||
time.sleep(0.05)
|
||||
check(still_running,'connection refusal observed only after process exit, not during drain')
|
||||
drained = future.result(timeout=8)
|
||||
check(drained['status']==201 and drained['instance']==target['instance'],'in-flight request did not finish on signaled process')
|
||||
check(drained['message']['text']==marker,'in-flight body changed')
|
||||
exit_code = cluster.command(['docker','wait',target['container']],timeout=8)
|
||||
elapsed = time.monotonic()-signaled
|
||||
check(exit_code=='0' and elapsed<5,'SIGTERM exceeded graceful budget or exited with error')
|
||||
check(history(c,bob,chat)==[drained['message']],'drained message missing or duplicated')
|
||||
result = {'result':'PASS','target_container':target['container'],'target_instance':target['instance'],
|
||||
'fixture':'temporary BEFORE INSERT trigger sleeps2s for one synthetic text; removed afterwards',
|
||||
'in_flight_observed_in_pg_stat_activity':True,'in_flight_response':201,
|
||||
'new_tcp_after_sigterm':rejected,'target_still_running_when_new_tcp_rejected':still_running,
|
||||
'exit_code':int(exit_code),'shutdown_seconds_from_sigterm':round(elapsed,4),
|
||||
'confirmed_message_preserved':True}
|
||||
finally:
|
||||
pool.shutdown(wait=True)
|
||||
database_fixture(survivor['container'],{**fixture,'action':'remove'})
|
||||
cluster.command(['docker','start',target['container']])
|
||||
cluster.wait_healthy(target['container'])
|
||||
cluster.wait_cluster()
|
||||
save('graceful-shutdown.json',result)
|
||||
|
||||
|
||||
def infrastructure():
|
||||
state = cluster.snapshot()
|
||||
for item in state:
|
||||
check(item['memory_bytes']>0 and item['nano_cpus']>0,'resource limit missing')
|
||||
if item['service']!='lb':
|
||||
check(not item['published_ports'],'service exposed a port bypassing LB')
|
||||
check(len([r for r in state if r['service']=='api' and r['status']=='running'])==2,'expected2live replicas')
|
||||
check(all(r['service']!='migrate' or r['status']=='exited' for r in state),'migration job still running')
|
||||
save('infrastructure.json',{'result':'PASS','containers':state})
|
||||
|
||||
|
||||
def dependency_failures():
|
||||
c = Client(timeout=6)
|
||||
actor = c.user('dependencies')
|
||||
c.call('GET','/api/v1/users/me',actor=actor,schema='User')
|
||||
result = {'result':'PASS'}
|
||||
try:
|
||||
cluster.compose('stop','-t','2','redis')
|
||||
response = c.call('GET','/api/v1/users/me',actor=actor,schema='User')
|
||||
check(response['json']==actor['user'] and response['elapsed']<3,'Redis fallback failed')
|
||||
c.call('GET','/api/v1/users/me',401)
|
||||
c.call('GET','/health/ready',schema='Health')
|
||||
result['redis'] = {'profile_status':200,'fallback_seconds':round(response['elapsed'],4),'anonymous':401,'readiness':200}
|
||||
finally:
|
||||
cluster.compose('start','redis')
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
def status(path, token=None):
|
||||
headers = {'Authorization':'Bearer '+token} if token else {}
|
||||
request = urllib.request.Request(c.base+path,headers=headers)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
response = urllib.request.urlopen(request,timeout=6)
|
||||
except urllib.error.HTTPError as error:
|
||||
response = error
|
||||
with response:
|
||||
return response.code, round(time.monotonic()-started,4)
|
||||
try:
|
||||
cluster.compose('stop','-t','2','postgres')
|
||||
code,elapsed = status('/api/v1/users/me',actor['token'])
|
||||
check(code in (502,503,504) and elapsed<6,'PostgreSQL outage did not safely reject authenticated request')
|
||||
anonymous,_ = status('/api/v1/users/me')
|
||||
check(anonymous in (401,502,503,504),'anonymous user trusted during DB outage')
|
||||
live = [cluster.probe(node['container'])['status'] for node in cluster.replicas()]
|
||||
check(all(code==200 for code in live),'API process liveness depends on DB')
|
||||
result['postgres'] = {'authenticated_status':code,'response_seconds':elapsed,'anonymous_status':anonymous,
|
||||
'internal_api_liveness':live,'note':'Nginx-generated502/503/504 may have no API JSON/instance headers when all backends become unavailable.'}
|
||||
finally:
|
||||
cluster.compose('start','postgres')
|
||||
cluster.wait_cluster()
|
||||
c.call('GET','/api/v1/users/me',actor=actor,schema='User')
|
||||
result['same_session_after_recovery'] = True
|
||||
save('dependency-failures.json',result)
|
||||
|
||||
|
||||
def verify_nginx():
|
||||
state = cluster.snapshot()
|
||||
lb = next(item for item in state if item['service']=='lb')
|
||||
check(lb['image'].startswith('nginx:'),'load balancer must use Nginx')
|
||||
cluster.command(['docker','exec',lb['container'],'nginx','-t'])
|
||||
version = cluster.command(['docker','exec',lb['container'],'sh','-c','nginx -v 2>&1'])
|
||||
c = Client()
|
||||
response = c.call('GET','/health/ready',schema='Health')
|
||||
check(response['headers'].get('Server')=='nginx','public entry point is not Nginx')
|
||||
check(len(cluster.wait_cluster())==2,'Nginx must discover two replicas')
|
||||
save('nginx.json',{'result':'PASS','version':version,'image':lb['image'],'public_server_header':'nginx',
|
||||
'config_syntax':'PASS','observed_instances':cluster.wait_cluster()})
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('mode',choices=['idempotency','persistence','public-failover','failover','rolling','sigterm','dependencies','infra','nginx','all'],default='all',nargs='?')
|
||||
args = parser.parse_args()
|
||||
modes = {'nginx':verify_nginx,'idempotency':concurrent_replay,'persistence':verify_persistence,'public-failover':public_failover,
|
||||
'failover':failover_each,'sigterm':graceful_shutdown,'rolling':rolling_update,'dependencies':dependency_failures,'infra':infrastructure}
|
||||
if args.mode=='all':
|
||||
for function in modes.values():
|
||||
function()
|
||||
else:
|
||||
modes[args.mode]()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user