Files
mtusi2026lab3/scripts/verify_cluster.py
T

444 lines
23 KiB
Python

#!/usr/bin/env python3
"""Grade4 distributed acceptance through the only public entry point; no tokens in artifacts."""
import argparse
import json
import math
import os
from pathlib import Path
import subprocess
import sys
import threading
import time
from concurrent.futures import ThreadPoolExecutor
import uuid
import cluster
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0,str(ROOT/'tests'))
from client import Client, Failure, check
from resilience import history
def save(name, value):
(ROOT/'evidence'/name).write_text(json.dumps(value,ensure_ascii=False,indent=2)+'\n')
print('PASS:', name)
def percentiles(values):
values = sorted(values)
return {name:round(1000*values[max(0,math.ceil(p*len(values))-1)],3) if values else None
for name,p in [('p50',.50),('p95',.95),('p99',.99)]}
class Traffic:
def __init__(self, concurrency=4):
self.c = Client(timeout=6)
self.alice,self.bob,_,self.chat = self.c.bootstrap()
self.concurrency = concurrency
self.done = threading.Event()
self.lock = threading.Lock()
self.attempts = []
self.confirmed = {}
self.prefix = uuid.uuid4().hex
def worker(self, number):
client = Client(timeout=6)
index = 0
while not self.done.is_set():
text = self.prefix+'-'+str(number)+'-'+str(index)
key = self.prefix+'-'+str(number)+'-'+str(index)
started = time.monotonic()
try:
response = client.message(self.alice,self.chat,text,key=key)
row = response['json']
attempt = {'started':started,'ended':time.monotonic(),'ok':True,
'latency':response['elapsed'],'instance':response['headers']['X-Instance-Id']}
with self.lock:
self.confirmed[row['id']] = row
index += 1
except Failure:
# Only idempotent message POST is retried, with its original body and key.
attempt = {'started':started,'ended':time.monotonic(),'ok':False,'instance':None}
time.sleep(0.02)
with self.lock:
self.attempts.append(attempt)
def start(self):
self.started = time.monotonic()
self.pool = ThreadPoolExecutor(max_workers=self.concurrency)
self.futures = [self.pool.submit(self.worker,n) for n in range(self.concurrency)]
return self
def stop(self):
self.done.set()
for future in self.futures:
future.result(timeout=10)
self.pool.shutdown()
self.ended = time.monotonic()
def summary(self, start=None, end=None):
start = self.started if start is None else start
end = getattr(self,'ended',time.monotonic()) if end is None else end
with self.lock:
attempts = [a for a in self.attempts if start <= a['started'] < end]
success = [a for a in attempts if a['ok']]
duration = end-start
return {'seconds':round(duration,4),'concurrency':self.concurrency,
'successful_requests':len(success),'failed_attempts':len(attempts)-len(success),
'successful_rps':round(len(success)/duration,3),
'error_ratio':round((len(attempts)-len(success))/len(attempts),6) if attempts else 0,
'successful_latency_ms':percentiles([a['latency'] for a in success]),
'instances':sorted({a['instance'] for a in success})}
def verify_data(self):
rows = history(self.c,self.bob,self.chat)
actual = {row['id']:row for row in rows}
check(len(actual)==len(rows),'duplicated message ids')
check(len({row['text'] for row in rows})==len(rows),'retry duplicated a message')
check(all(actual.get(mid)==row for mid,row in self.confirmed.items()),'confirmed message lost or changed')
return {'confirmed_unique_messages':len(self.confirmed), 'history_messages':len(rows),
'confirmed_preserved':True, 'duplicate_ids':0, 'duplicate_texts':0}
def concurrent_replay():
cluster.wait_cluster()
c = Client()
alice,bob,_,chat = c.bootstrap()
key = uuid.uuid4().hex
barrier = threading.Barrier(8)
def send(_):
local = Client()
barrier.wait()
return local.message(alice,chat,'Один результат через разные процессы',key=key)
with ThreadPoolExecutor(max_workers=8) as pool:
results = list(pool.map(send,range(8)))
instances = {r['headers']['X-Instance-Id'] for r in results}
check(len(instances)>=2,'concurrent retry did not reach different API processes')
check(all(r['json']==results[0]['json'] for r in results),'concurrent key returned different messages')
c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,
body={'text':'Другой body'},headers={'Idempotency-Key':key})
check(history(c,bob,chat)==[results[0]['json']],'repeated key created duplicates')
save('concurrent-idempotency.json',{'result':'PASS','parallel_requests':8,'observed_instances':sorted(instances),
'responses_201':8,'messages_created':1,'different_body_status':409})
def verify_persistence():
before = cluster.snapshot()
result = subprocess.run([sys.executable,'tests/resilience.py','--mode','persistence','--action-script','scripts/restart.sh'],
cwd=ROOT,check=True,capture_output=True,text=True,timeout=190)
after = cluster.snapshot()
roles = ['api','postgres','redis','lb']
previous = {item['container']:item for item in before}
for item in after:
if item['service'] in roles:
old = previous[item['container']]
check(old['started_at']!=item['started_at'],'service did not restart: '+item['service'])
check(old['volumes']==item['volumes'],'volume identities changed')
(ROOT/'evidence/persistence.txt').write_text(result.stdout)
save('persistence.json',{'result':'PASS','before':before,'after':after,'public_test':result.stdout.strip()})
def public_failover():
cluster.wait_cluster()
try:
result = subprocess.run([sys.executable,'tests/resilience.py','--mode','failover','--action-script','scripts/stop-one.sh'],
cwd=ROOT,check=True,capture_output=True,text=True,timeout=100)
(ROOT/'evidence/public-failover.txt').write_text(result.stdout)
save('public-failover.json',{'result':'PASS','public_test':result.stdout.strip(),
'stopped':json.loads((ROOT/'evidence/last-stop.json').read_text())})
finally:
cluster.compose('up','-d','--no-recreate','--wait','--wait-timeout','90')
cluster.wait_cluster()
def reject_new_connection(survivor, address):
code = '''import json,socket,sys
try:
s=socket.create_connection((sys.argv[1],8080),timeout=1)
except OSError as exc:
print(json.dumps({'accepted':False,'error_type':type(exc).__name__}))
else:
s.close()
print(json.dumps({'accepted':True}))
'''
return json.loads(cluster.command(['docker','exec',survivor,'python','-c',code,address]))
def failover_each():
cluster.wait_cluster()
originals = cluster.replicas()
check(len(originals)==2,'two running replicas required')
traffic = Traffic().start()
phases = []
try:
time.sleep(3)
baseline_end = time.monotonic()
baseline = traffic.summary(end=baseline_end)
for original in originals:
# Alias changes on restart, so resolve the current process of this exact container.
current = next(r for r in cluster.replicas() if r['container']==original['container'])
all_before = cluster.inspect([current['container']])[0]
address = next(v['IPAddress'] for k,v in all_before['NetworkSettings']['Networks'].items() if k.endswith('_front'))
survivor = next(r['container'] for r in cluster.replicas() if r['container']!=current['container'])
read_path = '/api/v1/chats/'+traffic.chat['id']+'/messages?limit=1'
marker = traffic.c.call('GET',read_path,actor=traffic.bob,schema='MessagePage')['json']['items'][0]
start = time.monotonic()
try:
env = {**os.environ,'TARGET_INSTANCE_ID':current['instance']}
subprocess.run(['scripts/stop-one.sh'],cwd=ROOT,env=env,check=True,capture_output=True,text=True,timeout=10)
stop_returned = time.monotonic()
stopped = json.loads((ROOT/'evidence/last-stop.json').read_text())
read_deadline = start+10
while True:
try:
read = traffic.c.call('GET',read_path,actor=traffic.bob,schema='MessagePage')
check(read['json']['items'][0]==marker,'pre-failure message unavailable during outage')
check(read['headers']['X-Instance-Id']!=current['instance'],'read served by stopped process')
read_done = time.monotonic()
break
except Failure:
check(time.monotonic()<read_deadline,'reads did not recover within10s')
time.sleep(0.1)
check(read_done-start<=10,'read recovery exceeded10s')
rejected = reject_new_connection(survivor,address)
check(not rejected['accepted'],'new connection accepted by stopped API')
time.sleep(max(0,8-(time.monotonic()-start)))
end = time.monotonic()
summary = traffic.summary(start,end)
with traffic.lock:
successful = [a for a in traffic.attempts if a['ok'] and a['started']>=start]
recent = [a for a in successful if a['started']>stop_returned]
recovery = min(a['ended'] for a in recent)-start if recent else None
after_stop = min(a['ended'] for a in recent)-stop_returned if recent else None
check(recovery is not None and recovery<=10,'post-stop writes did not recover within10s of action start')
check(len(recent)>=5 and all(a['instance']!=current['instance'] for a in recent[-5:]),'stopped alias still served requests')
phases.append({'stopped':stopped, 'traffic':summary,'post_stop_write_seconds_from_action_start':round(recovery,4),
'post_stop_write_seconds_from_stop_completion':round(after_stop,4),
'read_during_outage':{'status':200,'confirmed_message_preserved':True,
'seconds_from_action_start':round(read_done-start,4),
'seconds_from_stop_completion':round(read_done-stop_returned,4)},
'new_connection_after_stop':rejected,'stopped_remains_stopped':not cluster.inspect([current['container']])[0]['State']['Running']})
finally:
cluster.command(['docker','start',current['container']])
cluster.wait_healthy(current['container'])
cluster.wait_cluster()
time.sleep(2)
finally:
traffic.stop()
save('sequential-failover.json',{'result':'PASS','baseline':baseline,'phases':phases,
'total':traffic.summary(),'data':traffic.verify_data(),
'each_original_container_tested':len({p['stopped']['container'] for p in phases})==2})
def rolling_update():
cluster.wait_cluster()
before = cluster.replicas()
check(len(before)==2,'rolling demonstration expects two replicas')
lb_before = next(item for item in cluster.snapshot() if item['service']=='lb')
target_release = 'lab3-'+uuid.uuid4().hex[:8]
traffic = Traffic().start()
replacements = []
try:
time.sleep(3)
build_start = time.monotonic()
cluster.compose('build','--build-arg','APP_RELEASE='+target_release,'api',timeout=180)
build_end = time.monotonic()
migration_output = cluster.compose('run','--rm','--no-deps','migrate')
swap_start = time.monotonic()
for old in before:
step_start = time.monotonic()
result = cluster.replace_one(old['container'])
result['seconds'] = round(time.monotonic()-step_start,4)
replacements.append(result)
swap_end = time.monotonic()
time.sleep(3)
finally:
traffic.stop()
after = cluster.replicas()
lb_after = next(item for item in cluster.snapshot() if item['service']=='lb')
check(lb_before['container']==lb_after['container'] and lb_before['started_at']==lb_after['started_at'],'LB restarted during DNS test')
check(all(r['release']==target_release for r in after),'new release not observed on all replicas')
check(not {r['container'] for r in before}&{r['container'] for r in after},'old containers remain after replacement')
check(not {r['image_id'] for r in before}&{r['image_id'] for r in after},'image did not actually change')
save('rolling-update.json',{'result':'PASS','before':before,'after':after,'target_release':target_release,
'lb_before':lb_before,'lb_after':lb_after,'lb_not_restarted':True,
'build_seconds':round(build_end-build_start,4),'migration_output':migration_output,
'replacements':replacements,'during_replacement':traffic.summary(swap_start,swap_end),
'total':traffic.summary(),'data':traffic.verify_data(),
'change':'New image with a distinct release header; API code and schema remain compatible. Start-first: scale to3, prove new IP in DNS/LB, stop/remove one old container, return to2 with --no-recreate.'})
def database_fixture(container, data):
code = '''import json,sys,psycopg
from psycopg import sql
from messenger.db import setting
p=json.load(sys.stdin)
with psycopg.connect(setting('DATABASE_URL'),connect_timeout=3) as c:
if p['action']=='install':
c.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.text = {} THEN PERFORM pg_sleep(2); END IF; RETURN NEW; END $$").format(sql.Identifier(p['function']),sql.Literal(p['text'])))
c.execute(sql.SQL('CREATE TRIGGER {} BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(p['trigger']),sql.Identifier(p['function'])))
result={'installed':True}
elif p['action']=='remove':
c.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON messages').format(sql.Identifier(p['trigger'])))
c.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(p['function'])))
result={'removed':True}
else:
result={'waiting':c.execute("SELECT count(*) FROM pg_stat_activity WHERE datname=current_database() AND wait_event='PgSleep' AND query LIKE 'INSERT INTO messages%%'").fetchone()[0]}
print(json.dumps(result))
'''
response = subprocess.run(['docker','exec','-i',container,'python','-c',code],cwd=ROOT,
input=json.dumps(data),capture_output=True,text=True,check=True,timeout=8)
return json.loads(response.stdout)
def graceful_shutdown():
cluster.wait_cluster()
nodes = cluster.replicas()
target,survivor = nodes[0],nodes[1]
address = cluster.front_address(target['container'])
c = Client()
alice,bob,_,chat = c.bootstrap()
path = '/api/v1/chats/'+chat['id']+'/messages'
marker = 'drain_'+uuid.uuid4().hex
fixture = {'function':'drain_'+uuid.uuid4().hex,'trigger':'drain_'+uuid.uuid4().hex,'text':marker}
database_fixture(survivor['container'],{**fixture,'action':'install'})
code = '''import json,sys,urllib.request,urllib.error
p=json.load(sys.stdin)
r=urllib.request.Request('http://'+p['address']+':8080'+p['path'],data=json.dumps({'text':p['text']}).encode(),method='POST',headers={'Content-Type':'application/json','Authorization':'Bearer '+p['token'],'Idempotency-Key':p['key']})
try:
response=urllib.request.urlopen(r,timeout=6)
except urllib.error.HTTPError as exc:
response=exc
print(json.dumps({'status':response.code,'instance':response.headers.get('X-Instance-Id'),'message':json.load(response)}))
'''
request_data = {'address':address,'path':path,'text':marker,'token':alice['token'],'key':uuid.uuid4().hex}
def in_flight():
result = subprocess.run(['docker','exec','-i',survivor['container'],'python','-c',code],cwd=ROOT,
input=json.dumps(request_data),capture_output=True,text=True,check=True,timeout=8)
return json.loads(result.stdout)
pool = ThreadPoolExecutor(max_workers=1)
future = pool.submit(in_flight)
try:
deadline = time.monotonic()+3
while not database_fixture(survivor['container'],{'action':'waiting'})['waiting']:
check(time.monotonic()<deadline and not future.done(),'did not observe an in-flight SQL request')
time.sleep(0.05)
signaled = time.monotonic()
cluster.command(['docker','kill','--signal','SIGTERM',target['container']])
deadline = signaled+1.5
while True:
rejected = reject_new_connection(survivor['container'],address)
still_running = cluster.inspect([target['container']])[0]['State']['Running']
if not rejected['accepted']:
break
check(time.monotonic()<deadline,'new TCP connections still accepted after SIGTERM')
time.sleep(0.05)
check(still_running,'connection refusal observed only after process exit, not during drain')
drained = future.result(timeout=8)
check(drained['status']==201 and drained['instance']==target['instance'],'in-flight request did not finish on signaled process')
check(drained['message']['text']==marker,'in-flight body changed')
exit_code = cluster.command(['docker','wait',target['container']],timeout=8)
elapsed = time.monotonic()-signaled
check(exit_code=='0' and elapsed<5,'SIGTERM exceeded graceful budget or exited with error')
check(history(c,bob,chat)==[drained['message']],'drained message missing or duplicated')
result = {'result':'PASS','target_container':target['container'],'target_instance':target['instance'],
'fixture':'temporary BEFORE INSERT trigger sleeps2s for one synthetic text; removed afterwards',
'in_flight_observed_in_pg_stat_activity':True,'in_flight_response':201,
'new_tcp_after_sigterm':rejected,'target_still_running_when_new_tcp_rejected':still_running,
'exit_code':int(exit_code),'shutdown_seconds_from_sigterm':round(elapsed,4),
'confirmed_message_preserved':True}
finally:
pool.shutdown(wait=True)
database_fixture(survivor['container'],{**fixture,'action':'remove'})
cluster.command(['docker','start',target['container']])
cluster.wait_healthy(target['container'])
cluster.wait_cluster()
save('graceful-shutdown.json',result)
def infrastructure():
state = cluster.snapshot()
for item in state:
check(item['memory_bytes']>0 and item['nano_cpus']>0,'resource limit missing')
if item['service']!='lb':
check(not item['published_ports'],'service exposed a port bypassing LB')
check(len([r for r in state if r['service']=='api' and r['status']=='running'])==2,'expected2live replicas')
check(all(r['service']!='migrate' or r['status']=='exited' for r in state),'migration job still running')
save('infrastructure.json',{'result':'PASS','containers':state})
def dependency_failures():
c = Client(timeout=6)
actor = c.user('dependencies')
c.call('GET','/api/v1/users/me',actor=actor,schema='User')
result = {'result':'PASS'}
try:
cluster.compose('stop','-t','2','redis')
response = c.call('GET','/api/v1/users/me',actor=actor,schema='User')
check(response['json']==actor['user'] and response['elapsed']<3,'Redis fallback failed')
c.call('GET','/api/v1/users/me',401)
c.call('GET','/health/ready',schema='Health')
result['redis'] = {'profile_status':200,'fallback_seconds':round(response['elapsed'],4),'anonymous':401,'readiness':200}
finally:
cluster.compose('start','redis')
import urllib.request
import urllib.error
def status(path, token=None):
headers = {'Authorization':'Bearer '+token} if token else {}
request = urllib.request.Request(c.base+path,headers=headers)
started = time.monotonic()
try:
response = urllib.request.urlopen(request,timeout=6)
except urllib.error.HTTPError as error:
response = error
with response:
return response.code, round(time.monotonic()-started,4)
try:
cluster.compose('stop','-t','2','postgres')
code,elapsed = status('/api/v1/users/me',actor['token'])
check(code in (502,503,504) and elapsed<6,'PostgreSQL outage did not safely reject authenticated request')
anonymous,_ = status('/api/v1/users/me')
check(anonymous in (401,502,503,504),'anonymous user trusted during DB outage')
live = [cluster.probe(node['container'])['status'] for node in cluster.replicas()]
check(all(code==200 for code in live),'API process liveness depends on DB')
result['postgres'] = {'authenticated_status':code,'response_seconds':elapsed,'anonymous_status':anonymous,
'internal_api_liveness':live,'note':'Nginx-generated502/503/504 may have no API JSON/instance headers when all backends become unavailable.'}
finally:
cluster.compose('start','postgres')
cluster.wait_cluster()
c.call('GET','/api/v1/users/me',actor=actor,schema='User')
result['same_session_after_recovery'] = True
save('dependency-failures.json',result)
def verify_nginx():
state = cluster.snapshot()
lb = next(item for item in state if item['service']=='lb')
check(lb['image'].startswith('nginx:'),'load balancer must use Nginx')
cluster.command(['docker','exec',lb['container'],'nginx','-t'])
version = cluster.command(['docker','exec',lb['container'],'sh','-c','nginx -v 2>&1'])
c = Client()
response = c.call('GET','/health/ready',schema='Health')
check(response['headers'].get('Server')=='nginx','public entry point is not Nginx')
check(len(cluster.wait_cluster())==2,'Nginx must discover two replicas')
save('nginx.json',{'result':'PASS','version':version,'image':lb['image'],'public_server_header':'nginx',
'config_syntax':'PASS','observed_instances':cluster.wait_cluster()})
def main():
parser = argparse.ArgumentParser()
parser.add_argument('mode',choices=['idempotency','persistence','public-failover','failover','rolling','sigterm','dependencies','infra','nginx','all'],default='all',nargs='?')
args = parser.parse_args()
modes = {'nginx':verify_nginx,'idempotency':concurrent_replay,'persistence':verify_persistence,'public-failover':public_failover,
'failover':failover_each,'sigterm':graceful_shutdown,'rolling':rolling_update,'dependencies':dependency_failures,'infra':infrastructure}
if args.mode=='all':
for function in modes.values():
function()
else:
modes[args.mode]()
if __name__ == '__main__':
main()