This commit is contained in:
Xabenskie
2026-11-02 11:42:24 +03:00
parent 0ab3a341aa
commit 3ba9310ded
53 changed files with 3293 additions and 25 deletions
+28
View File
@@ -0,0 +1,28 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
umask 077
mkdir -p secrets
python3 - <<'PY'
from pathlib import Path
import secrets
root = Path('secrets')
p = root / 'postgres_password'
if not p.exists():
p.write_text(secrets.token_hex(24))
pw = p.read_text().strip()
for name, value in {
'database_url': f'postgresql://messenger:{pw}@postgres:5432/messenger',
'cache_url': 'redis://redis:6379/0',
}.items():
path = root / name
if not path.exists():
path.write_text(value)
# Bind-mounted Docker secrets must be readable by container uid 10001.
# Parent directory remains 0700 on the host.
for path in root.iterdir():
path.chmod(0o444)
if not Path('.env').exists():
Path('.env').write_text('SESSION_TTL_SECONDS=3600\n')
print('Local secrets ready (values hidden).')
PY
+71
View File
@@ -0,0 +1,71 @@
#!/usr/bin/env python3
"""Capture repeatable short load run and sanitized environment/resource evidence."""
import datetime
import json
import platform
import re
import subprocess
import sys
import time
import cluster
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def output(args):
return subprocess.check_output(args, cwd=ROOT, text=True).strip()
def counts():
code = "import json,psycopg; from messenger.db import setting; c=psycopg.connect(setting('DATABASE_URL')); print(json.dumps({t:c.execute('SELECT count(*) FROM '+t).fetchone()[0] for t in ['users','chats','messages','sessions','idempotency_records']})); c.close()"
return json.loads(output(['docker','exec',cluster.container_ids('api')[0],'python','-c',code]))
def resource_summary(samples):
peaks = {}
for sample in samples:
for container in sample['containers']:
peak = peaks.setdefault(container['Name'], {'cpu_percent_peak':0, 'memory_mib_peak':0})
peak['cpu_percent_peak'] = max(peak['cpu_percent_peak'],float(container['CPUPerc'].rstrip('%')))
value,unit = re.match(r'([0-9.]+)([A-Za-z]+)',container['MemUsage']).groups()
multiplier = {'GiB':1024,'MiB':1,'KiB':1/1024,'B':1/1048576}[unit]
peak['memory_mib_peak'] = round(max(peak['memory_mib_peak'],float(value)*multiplier),3)
return {'samples':len(samples),'per_container_peaks':peaks}
def main():
names = cluster.container_ids()
environment = {
'captured_at_utc': datetime.datetime.now(datetime.timezone.utc).isoformat(),
'host_os': platform.platform(), 'host_architecture': platform.machine(),
'docker': json.loads(output(['docker','info','--format','{"server_version":"{{.ServerVersion}}","cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}},"architecture":"{{.Architecture}}"}'])),
'python_in_image': output(['docker','exec',cluster.container_ids('api')[0],'python','--version']),
'dataset_before': counts(),
'limits': {'api_each':'0.75 CPU / 384 MiB','postgres':'1 CPU / 512 MiB','redis':'0.25 CPU / 128 MiB','lb':'0.5 CPU / 128 MiB'},
'warmup': 'API already healthy; acceptance, persistence and dependency checks performed before benchmark. No separate timed load warmup.',
'replicas':len(cluster.replicas()), 'repetitions': 1, 'duration_seconds':30, 'concurrency':4,
}
if sys.platform == 'darwin':
environment['host_cpu'] = output(['sysctl','-n','machdep.cpu.brand_string'])
environment['host_memory_bytes'] = int(output(['sysctl','-n','hw.memsize']))
samples = []
with (ROOT/'evidence/load-output.txt').open('w') as log:
proc = subprocess.Popen([sys.executable,'tests/load.py','--duration','30','--concurrency','4','--output','evidence/load.json'],cwd=ROOT,stdout=log,stderr=subprocess.STDOUT)
while proc.poll() is None:
rows = output(['docker','stats','--no-stream','--format','{{json .}}',*names])
samples.append({'at_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),
'containers':[{k:v for k,v in json.loads(row).items() if k in ('Name','CPUPerc','MemUsage','MemPerc','PIDs')}
for row in rows.splitlines()]})
time.sleep(1)
if proc.returncode:
raise SystemExit(proc.returncode)
environment['dataset_after'] = counts()
(ROOT/'evidence/environment.json').write_text(json.dumps(environment,ensure_ascii=False,indent=2)+'\n')
(ROOT/'evidence/resources.json').write_text(json.dumps(samples,indent=2)+'\n')
(ROOT/'evidence/resource-summary.json').write_text(json.dumps(resource_summary(samples),indent=2)+'\n')
print((ROOT/'evidence/load.json').read_text())
if __name__ == '__main__':
main()
+16
View File
@@ -0,0 +1,16 @@
#!/bin/sh
# A unique disposable Compose project; never touches normal project volumes.
set -eu
cd "$(dirname "$0")/.."
project="mtusi-lab3-migration-check-$$"
if [ -n "$(docker compose -p "$project" ps -aq)" ]; then
echo 'Refusing to reuse a pre-existing scratch project' >&2
exit 1
fi
cleanup() {
docker compose -p "$project" down --volumes >/dev/null 2>&1
}
trap cleanup EXIT INT TERM
docker compose -p "$project" up -d --wait --wait-timeout 90 postgres redis
docker compose -p "$project" run --rm --no-deps api sh -c 'python -m messenger.migrate && python -m messenger.migrate && python -m pytest student-tests -q -o cache_dir=/tmp/pytest-cache'
echo 'PASS: migrations and application tests on fresh disposable PostgreSQL and Redis volumes.'
+196
View File
@@ -0,0 +1,196 @@
#!/usr/bin/env python3
"""Local Compose orchestration. No credentials, tokens or environment dumps in evidence."""
import argparse
import datetime
import json
import os
import re
import subprocess
import sys
import time
import urllib.error
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
BASE_URL = os.environ.get('BASE_URL', 'http://localhost:8080').rstrip('/')
def command(args, timeout=120):
return subprocess.run(args, cwd=ROOT, check=True, capture_output=True, text=True, timeout=timeout).stdout.strip()
def compose(*args, timeout=120):
return command(['docker', 'compose', *args], timeout=timeout)
def inspect(containers):
if not containers:
return []
return json.loads(command(['docker','inspect',*containers]))
def container_ids(service=None, all_containers=False):
args = ['ps'] + (['--all'] if all_containers else []) + ['-q'] + ([service] if service else [])
return compose(*args).split()
def probe(container, host='127.0.0.1', path='/health/live'):
code = '''import json,sys,urllib.request,urllib.error
try:
r=urllib.request.urlopen('http://'+sys.argv[1]+':8080'+sys.argv[2],timeout=4)
except urllib.error.HTTPError as exc:
r=exc
print(json.dumps({'status':r.code,'instance':r.headers.get('X-Instance-Id'),'release':r.headers.get('X-App-Release')}))
'''
return json.loads(command(['docker','exec',container,'python','-c',code,host,path], timeout=8))
def replicas():
items = []
for data in inspect(container_ids('api')):
if data['State']['Running']:
result = probe(data['Id'])
items.append({'container':data['Id'], 'name':data['Name'].lstrip('/'),
'instance':result['instance'], 'release':result['release'], 'image_id':data['Image']})
return items
def snapshot():
items = []
for data in inspect(container_ids(all_containers=True)):
service = data['Config']['Labels']['com.docker.compose.service']
item = {
'service':service, 'container':data['Id'], 'name':data['Name'].lstrip('/'),
'status':data['State']['Status'], 'health':data['State'].get('Health',{}).get('Status'),
'started_at':data['State']['StartedAt'], 'image':data['Config']['Image'], 'image_id':data['Image'],
'memory_bytes':data['HostConfig']['Memory'], 'nano_cpus':data['HostConfig']['NanoCpus'],
'published_ports':data['HostConfig']['PortBindings'],
'volumes':sorted(m['Name'] for m in data['Mounts'] if m['Type']=='volume'),
'networks':sorted(data['NetworkSettings']['Networks']),
}
if service == 'api' and data['State']['Running']:
result = probe(data['Id'])
item.update({'http_status':result['status'], 'instance':result['instance'], 'release':result['release']})
items.append(item)
return items
def wait_healthy(container, timeout=60):
deadline = time.monotonic()+timeout
while time.monotonic() < deadline:
data = inspect([container])[0]
if data['State'].get('Health',{}).get('Status') == 'healthy':
return
time.sleep(0.3)
raise RuntimeError('Container did not become healthy')
def wait_cluster(count=2, timeout=60):
deadline = time.monotonic()+timeout
seen = set()
while time.monotonic() < deadline:
try:
with urllib.request.urlopen(BASE_URL+'/health/ready',timeout=5) as response:
alias = response.headers.get('X-Instance-Id')
if alias:
seen.add(alias)
if len(seen) >= count:
return sorted(seen)
except (OSError, urllib.error.URLError):
pass
time.sleep(0.1)
raise RuntimeError('Load balancer did not discover required API instances')
def stop_one(target):
if not re.fullmatch(r'[a-f0-9]{32}', target):
raise ValueError('Invalid target instance alias')
matching = [item for item in replicas() if item['instance']==target]
if len(matching) != 1:
raise RuntimeError('Target must match exactly one running API process')
item = matching[0]
started = time.monotonic()
command(['docker','stop','--time','7',item['container']],timeout=9)
state = inspect([item['container']])[0]['State']
if state['Running'] or state['ExitCode'] != 0:
raise RuntimeError('API did not stop gracefully')
result = {**item, 'stopped_at_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),
'stop_seconds':round(time.monotonic()-started,4), 'exit_code':state['ExitCode'], 'left_stopped':True}
(ROOT/'evidence/last-stop.json').write_text(json.dumps(result,indent=2)+'\n')
return result
def restart():
compose('restart','api','postgres','redis','lb',timeout=30)
compose('up','-d','--no-recreate','--wait','--wait-timeout','35',timeout=40)
return wait_cluster(2,timeout=15)
def front_address(container):
networks = inspect([container])[0]['NetworkSettings']['Networks']
return next(value['IPAddress'] for name,value in networks.items() if name.endswith('_front'))
def replace_one(container):
before = set(container_ids('api'))
if container not in before or len(before) != 2:
raise RuntimeError('Replacement expects exactly two running API containers')
old_address = front_address(container)
# Start-first ensures a genuinely different IP while the old address is still occupied.
compose('up','-d','--no-deps','--no-recreate','--scale','api=3','api')
expanded = set(container_ids('api'))
added = expanded-before
if len(added) != 1 or not before.issubset(expanded):
raise RuntimeError('Start-first step unexpectedly changed an existing container')
replacement = added.pop()
wait_healthy(replacement)
new_address = front_address(replacement)
if new_address == old_address:
raise RuntimeError('Replacement did not get a different IP')
expected_three = {item['instance'] for item in replicas()}
if set(wait_cluster(3)) != expected_three:
raise RuntimeError('Load balancer did not discover all three live addresses')
command(['docker','stop','--time','7',container],timeout=10)
state = inspect([container])[0]['State']
if state['ExitCode'] != 0:
raise RuntimeError('Old release did not terminate gracefully')
command(['docker','rm',container])
compose('up','-d','--no-deps','--no-recreate','--scale','api=2','api')
after = set(container_ids('api'))
if after != (before-{container}) | {replacement}:
raise RuntimeError('Sequential replacement changed the wrong containers')
expected = {item['instance'] for item in replicas()}
if set(wait_cluster(2)) != expected:
raise RuntimeError('Unexpected final backend aliases')
return {'old_container':container,'new_container':replacement,
'old_ip':old_address,'new_ip':new_address,'dns_discovered_new_address':True,
'instances_during_start_first':sorted(expected_three),
'observed_instances':sorted(expected),'old_exit_code':state['ExitCode']}
def main():
parser = argparse.ArgumentParser()
parser.add_argument('action',choices=['wait','snapshot','stop-one','restart','rolling','exec'])
parser.add_argument('arguments',nargs=argparse.REMAINDER)
args = parser.parse_args()
if args.action == 'wait':
print(json.dumps({'instances':wait_cluster()}))
elif args.action == 'snapshot':
print(json.dumps(snapshot(),indent=2))
elif args.action == 'stop-one':
print(json.dumps(stop_one(os.environ['TARGET_INSTANCE_ID'])))
elif args.action == 'exec':
ids = container_ids('api')
if not ids or not args.arguments:
raise RuntimeError('A live API container and command are required')
raise SystemExit(subprocess.call(['docker','exec',ids[0],*args.arguments],cwd=ROOT))
elif args.action == 'restart':
print(json.dumps({'instances':restart()}))
else:
compose('run','--rm','--no-deps','migrate')
print(json.dumps([replace_one(item['container']) for item in replicas()],indent=2))
if __name__ == '__main__':
main()
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
exec python3 scripts/cluster.py restart
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
exec python3 scripts/cluster.py stop-one
+443
View File
@@ -0,0 +1,443 @@
#!/usr/bin/env python3
"""Grade4 distributed acceptance through the only public entry point; no tokens in artifacts."""
import argparse
import json
import math
import os
from pathlib import Path
import subprocess
import sys
import threading
import time
from concurrent.futures import ThreadPoolExecutor
import uuid
import cluster
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0,str(ROOT/'tests'))
from client import Client, Failure, check
from resilience import history
def save(name, value):
(ROOT/'evidence'/name).write_text(json.dumps(value,ensure_ascii=False,indent=2)+'\n')
print('PASS:', name)
def percentiles(values):
values = sorted(values)
return {name:round(1000*values[max(0,math.ceil(p*len(values))-1)],3) if values else None
for name,p in [('p50',.50),('p95',.95),('p99',.99)]}
class Traffic:
def __init__(self, concurrency=4):
self.c = Client(timeout=6)
self.alice,self.bob,_,self.chat = self.c.bootstrap()
self.concurrency = concurrency
self.done = threading.Event()
self.lock = threading.Lock()
self.attempts = []
self.confirmed = {}
self.prefix = uuid.uuid4().hex
def worker(self, number):
client = Client(timeout=6)
index = 0
while not self.done.is_set():
text = self.prefix+'-'+str(number)+'-'+str(index)
key = self.prefix+'-'+str(number)+'-'+str(index)
started = time.monotonic()
try:
response = client.message(self.alice,self.chat,text,key=key)
row = response['json']
attempt = {'started':started,'ended':time.monotonic(),'ok':True,
'latency':response['elapsed'],'instance':response['headers']['X-Instance-Id']}
with self.lock:
self.confirmed[row['id']] = row
index += 1
except Failure:
# Only idempotent message POST is retried, with its original body and key.
attempt = {'started':started,'ended':time.monotonic(),'ok':False,'instance':None}
time.sleep(0.02)
with self.lock:
self.attempts.append(attempt)
def start(self):
self.started = time.monotonic()
self.pool = ThreadPoolExecutor(max_workers=self.concurrency)
self.futures = [self.pool.submit(self.worker,n) for n in range(self.concurrency)]
return self
def stop(self):
self.done.set()
for future in self.futures:
future.result(timeout=10)
self.pool.shutdown()
self.ended = time.monotonic()
def summary(self, start=None, end=None):
start = self.started if start is None else start
end = getattr(self,'ended',time.monotonic()) if end is None else end
with self.lock:
attempts = [a for a in self.attempts if start <= a['started'] < end]
success = [a for a in attempts if a['ok']]
duration = end-start
return {'seconds':round(duration,4),'concurrency':self.concurrency,
'successful_requests':len(success),'failed_attempts':len(attempts)-len(success),
'successful_rps':round(len(success)/duration,3),
'error_ratio':round((len(attempts)-len(success))/len(attempts),6) if attempts else 0,
'successful_latency_ms':percentiles([a['latency'] for a in success]),
'instances':sorted({a['instance'] for a in success})}
def verify_data(self):
rows = history(self.c,self.bob,self.chat)
actual = {row['id']:row for row in rows}
check(len(actual)==len(rows),'duplicated message ids')
check(len({row['text'] for row in rows})==len(rows),'retry duplicated a message')
check(all(actual.get(mid)==row for mid,row in self.confirmed.items()),'confirmed message lost or changed')
return {'confirmed_unique_messages':len(self.confirmed), 'history_messages':len(rows),
'confirmed_preserved':True, 'duplicate_ids':0, 'duplicate_texts':0}
def concurrent_replay():
cluster.wait_cluster()
c = Client()
alice,bob,_,chat = c.bootstrap()
key = uuid.uuid4().hex
barrier = threading.Barrier(8)
def send(_):
local = Client()
barrier.wait()
return local.message(alice,chat,'Один результат через разные процессы',key=key)
with ThreadPoolExecutor(max_workers=8) as pool:
results = list(pool.map(send,range(8)))
instances = {r['headers']['X-Instance-Id'] for r in results}
check(len(instances)>=2,'concurrent retry did not reach different API processes')
check(all(r['json']==results[0]['json'] for r in results),'concurrent key returned different messages')
c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,
body={'text':'Другой body'},headers={'Idempotency-Key':key})
check(history(c,bob,chat)==[results[0]['json']],'repeated key created duplicates')
save('concurrent-idempotency.json',{'result':'PASS','parallel_requests':8,'observed_instances':sorted(instances),
'responses_201':8,'messages_created':1,'different_body_status':409})
def verify_persistence():
before = cluster.snapshot()
result = subprocess.run([sys.executable,'tests/resilience.py','--mode','persistence','--action-script','scripts/restart.sh'],
cwd=ROOT,check=True,capture_output=True,text=True,timeout=190)
after = cluster.snapshot()
roles = ['api','postgres','redis','lb']
previous = {item['container']:item for item in before}
for item in after:
if item['service'] in roles:
old = previous[item['container']]
check(old['started_at']!=item['started_at'],'service did not restart: '+item['service'])
check(old['volumes']==item['volumes'],'volume identities changed')
(ROOT/'evidence/persistence.txt').write_text(result.stdout)
save('persistence.json',{'result':'PASS','before':before,'after':after,'public_test':result.stdout.strip()})
def public_failover():
cluster.wait_cluster()
try:
result = subprocess.run([sys.executable,'tests/resilience.py','--mode','failover','--action-script','scripts/stop-one.sh'],
cwd=ROOT,check=True,capture_output=True,text=True,timeout=100)
(ROOT/'evidence/public-failover.txt').write_text(result.stdout)
save('public-failover.json',{'result':'PASS','public_test':result.stdout.strip(),
'stopped':json.loads((ROOT/'evidence/last-stop.json').read_text())})
finally:
cluster.compose('up','-d','--no-recreate','--wait','--wait-timeout','90')
cluster.wait_cluster()
def reject_new_connection(survivor, address):
code = '''import json,socket,sys
try:
s=socket.create_connection((sys.argv[1],8080),timeout=1)
except OSError as exc:
print(json.dumps({'accepted':False,'error_type':type(exc).__name__}))
else:
s.close()
print(json.dumps({'accepted':True}))
'''
return json.loads(cluster.command(['docker','exec',survivor,'python','-c',code,address]))
def failover_each():
cluster.wait_cluster()
originals = cluster.replicas()
check(len(originals)==2,'two running replicas required')
traffic = Traffic().start()
phases = []
try:
time.sleep(3)
baseline_end = time.monotonic()
baseline = traffic.summary(end=baseline_end)
for original in originals:
# Alias changes on restart, so resolve the current process of this exact container.
current = next(r for r in cluster.replicas() if r['container']==original['container'])
all_before = cluster.inspect([current['container']])[0]
address = next(v['IPAddress'] for k,v in all_before['NetworkSettings']['Networks'].items() if k.endswith('_front'))
survivor = next(r['container'] for r in cluster.replicas() if r['container']!=current['container'])
read_path = '/api/v1/chats/'+traffic.chat['id']+'/messages?limit=1'
marker = traffic.c.call('GET',read_path,actor=traffic.bob,schema='MessagePage')['json']['items'][0]
start = time.monotonic()
try:
env = {**os.environ,'TARGET_INSTANCE_ID':current['instance']}
subprocess.run(['scripts/stop-one.sh'],cwd=ROOT,env=env,check=True,capture_output=True,text=True,timeout=10)
stop_returned = time.monotonic()
stopped = json.loads((ROOT/'evidence/last-stop.json').read_text())
read_deadline = start+10
while True:
try:
read = traffic.c.call('GET',read_path,actor=traffic.bob,schema='MessagePage')
check(read['json']['items'][0]==marker,'pre-failure message unavailable during outage')
check(read['headers']['X-Instance-Id']!=current['instance'],'read served by stopped process')
read_done = time.monotonic()
break
except Failure:
check(time.monotonic()<read_deadline,'reads did not recover within10s')
time.sleep(0.1)
check(read_done-start<=10,'read recovery exceeded10s')
rejected = reject_new_connection(survivor,address)
check(not rejected['accepted'],'new connection accepted by stopped API')
time.sleep(max(0,8-(time.monotonic()-start)))
end = time.monotonic()
summary = traffic.summary(start,end)
with traffic.lock:
successful = [a for a in traffic.attempts if a['ok'] and a['started']>=start]
recent = [a for a in successful if a['started']>stop_returned]
recovery = min(a['ended'] for a in recent)-start if recent else None
after_stop = min(a['ended'] for a in recent)-stop_returned if recent else None
check(recovery is not None and recovery<=10,'post-stop writes did not recover within10s of action start')
check(len(recent)>=5 and all(a['instance']!=current['instance'] for a in recent[-5:]),'stopped alias still served requests')
phases.append({'stopped':stopped, 'traffic':summary,'post_stop_write_seconds_from_action_start':round(recovery,4),
'post_stop_write_seconds_from_stop_completion':round(after_stop,4),
'read_during_outage':{'status':200,'confirmed_message_preserved':True,
'seconds_from_action_start':round(read_done-start,4),
'seconds_from_stop_completion':round(read_done-stop_returned,4)},
'new_connection_after_stop':rejected,'stopped_remains_stopped':not cluster.inspect([current['container']])[0]['State']['Running']})
finally:
cluster.command(['docker','start',current['container']])
cluster.wait_healthy(current['container'])
cluster.wait_cluster()
time.sleep(2)
finally:
traffic.stop()
save('sequential-failover.json',{'result':'PASS','baseline':baseline,'phases':phases,
'total':traffic.summary(),'data':traffic.verify_data(),
'each_original_container_tested':len({p['stopped']['container'] for p in phases})==2})
def rolling_update():
cluster.wait_cluster()
before = cluster.replicas()
check(len(before)==2,'rolling demonstration expects two replicas')
lb_before = next(item for item in cluster.snapshot() if item['service']=='lb')
target_release = 'lab3-'+uuid.uuid4().hex[:8]
traffic = Traffic().start()
replacements = []
try:
time.sleep(3)
build_start = time.monotonic()
cluster.compose('build','--build-arg','APP_RELEASE='+target_release,'api',timeout=180)
build_end = time.monotonic()
migration_output = cluster.compose('run','--rm','--no-deps','migrate')
swap_start = time.monotonic()
for old in before:
step_start = time.monotonic()
result = cluster.replace_one(old['container'])
result['seconds'] = round(time.monotonic()-step_start,4)
replacements.append(result)
swap_end = time.monotonic()
time.sleep(3)
finally:
traffic.stop()
after = cluster.replicas()
lb_after = next(item for item in cluster.snapshot() if item['service']=='lb')
check(lb_before['container']==lb_after['container'] and lb_before['started_at']==lb_after['started_at'],'LB restarted during DNS test')
check(all(r['release']==target_release for r in after),'new release not observed on all replicas')
check(not {r['container'] for r in before}&{r['container'] for r in after},'old containers remain after replacement')
check(not {r['image_id'] for r in before}&{r['image_id'] for r in after},'image did not actually change')
save('rolling-update.json',{'result':'PASS','before':before,'after':after,'target_release':target_release,
'lb_before':lb_before,'lb_after':lb_after,'lb_not_restarted':True,
'build_seconds':round(build_end-build_start,4),'migration_output':migration_output,
'replacements':replacements,'during_replacement':traffic.summary(swap_start,swap_end),
'total':traffic.summary(),'data':traffic.verify_data(),
'change':'New image with a distinct release header; API code and schema remain compatible. Start-first: scale to3, prove new IP in DNS/LB, stop/remove one old container, return to2 with --no-recreate.'})
def database_fixture(container, data):
code = '''import json,sys,psycopg
from psycopg import sql
from messenger.db import setting
p=json.load(sys.stdin)
with psycopg.connect(setting('DATABASE_URL'),connect_timeout=3) as c:
if p['action']=='install':
c.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.text = {} THEN PERFORM pg_sleep(2); END IF; RETURN NEW; END $$").format(sql.Identifier(p['function']),sql.Literal(p['text'])))
c.execute(sql.SQL('CREATE TRIGGER {} BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(p['trigger']),sql.Identifier(p['function'])))
result={'installed':True}
elif p['action']=='remove':
c.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON messages').format(sql.Identifier(p['trigger'])))
c.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(p['function'])))
result={'removed':True}
else:
result={'waiting':c.execute("SELECT count(*) FROM pg_stat_activity WHERE datname=current_database() AND wait_event='PgSleep' AND query LIKE 'INSERT INTO messages%%'").fetchone()[0]}
print(json.dumps(result))
'''
response = subprocess.run(['docker','exec','-i',container,'python','-c',code],cwd=ROOT,
input=json.dumps(data),capture_output=True,text=True,check=True,timeout=8)
return json.loads(response.stdout)
def graceful_shutdown():
cluster.wait_cluster()
nodes = cluster.replicas()
target,survivor = nodes[0],nodes[1]
address = cluster.front_address(target['container'])
c = Client()
alice,bob,_,chat = c.bootstrap()
path = '/api/v1/chats/'+chat['id']+'/messages'
marker = 'drain_'+uuid.uuid4().hex
fixture = {'function':'drain_'+uuid.uuid4().hex,'trigger':'drain_'+uuid.uuid4().hex,'text':marker}
database_fixture(survivor['container'],{**fixture,'action':'install'})
code = '''import json,sys,urllib.request,urllib.error
p=json.load(sys.stdin)
r=urllib.request.Request('http://'+p['address']+':8080'+p['path'],data=json.dumps({'text':p['text']}).encode(),method='POST',headers={'Content-Type':'application/json','Authorization':'Bearer '+p['token'],'Idempotency-Key':p['key']})
try:
response=urllib.request.urlopen(r,timeout=6)
except urllib.error.HTTPError as exc:
response=exc
print(json.dumps({'status':response.code,'instance':response.headers.get('X-Instance-Id'),'message':json.load(response)}))
'''
request_data = {'address':address,'path':path,'text':marker,'token':alice['token'],'key':uuid.uuid4().hex}
def in_flight():
result = subprocess.run(['docker','exec','-i',survivor['container'],'python','-c',code],cwd=ROOT,
input=json.dumps(request_data),capture_output=True,text=True,check=True,timeout=8)
return json.loads(result.stdout)
pool = ThreadPoolExecutor(max_workers=1)
future = pool.submit(in_flight)
try:
deadline = time.monotonic()+3
while not database_fixture(survivor['container'],{'action':'waiting'})['waiting']:
check(time.monotonic()<deadline and not future.done(),'did not observe an in-flight SQL request')
time.sleep(0.05)
signaled = time.monotonic()
cluster.command(['docker','kill','--signal','SIGTERM',target['container']])
deadline = signaled+1.5
while True:
rejected = reject_new_connection(survivor['container'],address)
still_running = cluster.inspect([target['container']])[0]['State']['Running']
if not rejected['accepted']:
break
check(time.monotonic()<deadline,'new TCP connections still accepted after SIGTERM')
time.sleep(0.05)
check(still_running,'connection refusal observed only after process exit, not during drain')
drained = future.result(timeout=8)
check(drained['status']==201 and drained['instance']==target['instance'],'in-flight request did not finish on signaled process')
check(drained['message']['text']==marker,'in-flight body changed')
exit_code = cluster.command(['docker','wait',target['container']],timeout=8)
elapsed = time.monotonic()-signaled
check(exit_code=='0' and elapsed<5,'SIGTERM exceeded graceful budget or exited with error')
check(history(c,bob,chat)==[drained['message']],'drained message missing or duplicated')
result = {'result':'PASS','target_container':target['container'],'target_instance':target['instance'],
'fixture':'temporary BEFORE INSERT trigger sleeps2s for one synthetic text; removed afterwards',
'in_flight_observed_in_pg_stat_activity':True,'in_flight_response':201,
'new_tcp_after_sigterm':rejected,'target_still_running_when_new_tcp_rejected':still_running,
'exit_code':int(exit_code),'shutdown_seconds_from_sigterm':round(elapsed,4),
'confirmed_message_preserved':True}
finally:
pool.shutdown(wait=True)
database_fixture(survivor['container'],{**fixture,'action':'remove'})
cluster.command(['docker','start',target['container']])
cluster.wait_healthy(target['container'])
cluster.wait_cluster()
save('graceful-shutdown.json',result)
def infrastructure():
state = cluster.snapshot()
for item in state:
check(item['memory_bytes']>0 and item['nano_cpus']>0,'resource limit missing')
if item['service']!='lb':
check(not item['published_ports'],'service exposed a port bypassing LB')
check(len([r for r in state if r['service']=='api' and r['status']=='running'])==2,'expected2live replicas')
check(all(r['service']!='migrate' or r['status']=='exited' for r in state),'migration job still running')
save('infrastructure.json',{'result':'PASS','containers':state})
def dependency_failures():
c = Client(timeout=6)
actor = c.user('dependencies')
c.call('GET','/api/v1/users/me',actor=actor,schema='User')
result = {'result':'PASS'}
try:
cluster.compose('stop','-t','2','redis')
response = c.call('GET','/api/v1/users/me',actor=actor,schema='User')
check(response['json']==actor['user'] and response['elapsed']<3,'Redis fallback failed')
c.call('GET','/api/v1/users/me',401)
c.call('GET','/health/ready',schema='Health')
result['redis'] = {'profile_status':200,'fallback_seconds':round(response['elapsed'],4),'anonymous':401,'readiness':200}
finally:
cluster.compose('start','redis')
import urllib.request
import urllib.error
def status(path, token=None):
headers = {'Authorization':'Bearer '+token} if token else {}
request = urllib.request.Request(c.base+path,headers=headers)
started = time.monotonic()
try:
response = urllib.request.urlopen(request,timeout=6)
except urllib.error.HTTPError as error:
response = error
with response:
return response.code, round(time.monotonic()-started,4)
try:
cluster.compose('stop','-t','2','postgres')
code,elapsed = status('/api/v1/users/me',actor['token'])
check(code in (502,503,504) and elapsed<6,'PostgreSQL outage did not safely reject authenticated request')
anonymous,_ = status('/api/v1/users/me')
check(anonymous in (401,502,503,504),'anonymous user trusted during DB outage')
live = [cluster.probe(node['container'])['status'] for node in cluster.replicas()]
check(all(code==200 for code in live),'API process liveness depends on DB')
result['postgres'] = {'authenticated_status':code,'response_seconds':elapsed,'anonymous_status':anonymous,
'internal_api_liveness':live,'note':'Nginx-generated502/503/504 may have no API JSON/instance headers when all backends become unavailable.'}
finally:
cluster.compose('start','postgres')
cluster.wait_cluster()
c.call('GET','/api/v1/users/me',actor=actor,schema='User')
result['same_session_after_recovery'] = True
save('dependency-failures.json',result)
def verify_nginx():
state = cluster.snapshot()
lb = next(item for item in state if item['service']=='lb')
check(lb['image'].startswith('nginx:'),'load balancer must use Nginx')
cluster.command(['docker','exec',lb['container'],'nginx','-t'])
version = cluster.command(['docker','exec',lb['container'],'sh','-c','nginx -v 2>&1'])
c = Client()
response = c.call('GET','/health/ready',schema='Health')
check(response['headers'].get('Server')=='nginx','public entry point is not Nginx')
check(len(cluster.wait_cluster())==2,'Nginx must discover two replicas')
save('nginx.json',{'result':'PASS','version':version,'image':lb['image'],'public_server_header':'nginx',
'config_syntax':'PASS','observed_instances':cluster.wait_cluster()})
def main():
parser = argparse.ArgumentParser()
parser.add_argument('mode',choices=['idempotency','persistence','public-failover','failover','rolling','sigterm','dependencies','infra','nginx','all'],default='all',nargs='?')
args = parser.parse_args()
modes = {'nginx':verify_nginx,'idempotency':concurrent_replay,'persistence':verify_persistence,'public-failover':public_failover,
'failover':failover_each,'sigterm':graceful_shutdown,'rolling':rolling_update,'dependencies':dependency_failures,'infra':infrastructure}
if args.mode=='all':
for function in modes.values():
function()
else:
modes[args.mode]()
if __name__ == '__main__':
main()