444 lines
23 KiB
Python
444 lines
23 KiB
Python
#!/usr/bin/env python3
|
|
"""Grade4 distributed acceptance through the only public entry point; no tokens in artifacts."""
|
|
import argparse
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import uuid
|
|
|
|
import cluster
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0,str(ROOT/'tests'))
|
|
from client import Client, Failure, check
|
|
from resilience import history
|
|
|
|
|
|
def save(name, value):
|
|
(ROOT/'evidence'/name).write_text(json.dumps(value,ensure_ascii=False,indent=2)+'\n')
|
|
print('PASS:', name)
|
|
|
|
|
|
def percentiles(values):
|
|
values = sorted(values)
|
|
return {name:round(1000*values[max(0,math.ceil(p*len(values))-1)],3) if values else None
|
|
for name,p in [('p50',.50),('p95',.95),('p99',.99)]}
|
|
|
|
|
|
class Traffic:
|
|
def __init__(self, concurrency=4):
|
|
self.c = Client(timeout=6)
|
|
self.alice,self.bob,_,self.chat = self.c.bootstrap()
|
|
self.concurrency = concurrency
|
|
self.done = threading.Event()
|
|
self.lock = threading.Lock()
|
|
self.attempts = []
|
|
self.confirmed = {}
|
|
self.prefix = uuid.uuid4().hex
|
|
|
|
def worker(self, number):
|
|
client = Client(timeout=6)
|
|
index = 0
|
|
while not self.done.is_set():
|
|
text = self.prefix+'-'+str(number)+'-'+str(index)
|
|
key = self.prefix+'-'+str(number)+'-'+str(index)
|
|
started = time.monotonic()
|
|
try:
|
|
response = client.message(self.alice,self.chat,text,key=key)
|
|
row = response['json']
|
|
attempt = {'started':started,'ended':time.monotonic(),'ok':True,
|
|
'latency':response['elapsed'],'instance':response['headers']['X-Instance-Id']}
|
|
with self.lock:
|
|
self.confirmed[row['id']] = row
|
|
index += 1
|
|
except Failure:
|
|
# Only idempotent message POST is retried, with its original body and key.
|
|
attempt = {'started':started,'ended':time.monotonic(),'ok':False,'instance':None}
|
|
time.sleep(0.02)
|
|
with self.lock:
|
|
self.attempts.append(attempt)
|
|
|
|
def start(self):
|
|
self.started = time.monotonic()
|
|
self.pool = ThreadPoolExecutor(max_workers=self.concurrency)
|
|
self.futures = [self.pool.submit(self.worker,n) for n in range(self.concurrency)]
|
|
return self
|
|
|
|
def stop(self):
|
|
self.done.set()
|
|
for future in self.futures:
|
|
future.result(timeout=10)
|
|
self.pool.shutdown()
|
|
self.ended = time.monotonic()
|
|
|
|
def summary(self, start=None, end=None):
|
|
start = self.started if start is None else start
|
|
end = getattr(self,'ended',time.monotonic()) if end is None else end
|
|
with self.lock:
|
|
attempts = [a for a in self.attempts if start <= a['started'] < end]
|
|
success = [a for a in attempts if a['ok']]
|
|
duration = end-start
|
|
return {'seconds':round(duration,4),'concurrency':self.concurrency,
|
|
'successful_requests':len(success),'failed_attempts':len(attempts)-len(success),
|
|
'successful_rps':round(len(success)/duration,3),
|
|
'error_ratio':round((len(attempts)-len(success))/len(attempts),6) if attempts else 0,
|
|
'successful_latency_ms':percentiles([a['latency'] for a in success]),
|
|
'instances':sorted({a['instance'] for a in success})}
|
|
|
|
def verify_data(self):
|
|
rows = history(self.c,self.bob,self.chat)
|
|
actual = {row['id']:row for row in rows}
|
|
check(len(actual)==len(rows),'duplicated message ids')
|
|
check(len({row['text'] for row in rows})==len(rows),'retry duplicated a message')
|
|
check(all(actual.get(mid)==row for mid,row in self.confirmed.items()),'confirmed message lost or changed')
|
|
return {'confirmed_unique_messages':len(self.confirmed), 'history_messages':len(rows),
|
|
'confirmed_preserved':True, 'duplicate_ids':0, 'duplicate_texts':0}
|
|
|
|
|
|
def concurrent_replay():
|
|
cluster.wait_cluster()
|
|
c = Client()
|
|
alice,bob,_,chat = c.bootstrap()
|
|
key = uuid.uuid4().hex
|
|
barrier = threading.Barrier(8)
|
|
def send(_):
|
|
local = Client()
|
|
barrier.wait()
|
|
return local.message(alice,chat,'Один результат через разные процессы',key=key)
|
|
with ThreadPoolExecutor(max_workers=8) as pool:
|
|
results = list(pool.map(send,range(8)))
|
|
instances = {r['headers']['X-Instance-Id'] for r in results}
|
|
check(len(instances)>=2,'concurrent retry did not reach different API processes')
|
|
check(all(r['json']==results[0]['json'] for r in results),'concurrent key returned different messages')
|
|
c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,
|
|
body={'text':'Другой body'},headers={'Idempotency-Key':key})
|
|
check(history(c,bob,chat)==[results[0]['json']],'repeated key created duplicates')
|
|
save('concurrent-idempotency.json',{'result':'PASS','parallel_requests':8,'observed_instances':sorted(instances),
|
|
'responses_201':8,'messages_created':1,'different_body_status':409})
|
|
|
|
|
|
def verify_persistence():
|
|
before = cluster.snapshot()
|
|
result = subprocess.run([sys.executable,'tests/resilience.py','--mode','persistence','--action-script','scripts/restart.sh'],
|
|
cwd=ROOT,check=True,capture_output=True,text=True,timeout=190)
|
|
after = cluster.snapshot()
|
|
roles = ['api','postgres','redis','lb']
|
|
previous = {item['container']:item for item in before}
|
|
for item in after:
|
|
if item['service'] in roles:
|
|
old = previous[item['container']]
|
|
check(old['started_at']!=item['started_at'],'service did not restart: '+item['service'])
|
|
check(old['volumes']==item['volumes'],'volume identities changed')
|
|
(ROOT/'evidence/persistence.txt').write_text(result.stdout)
|
|
save('persistence.json',{'result':'PASS','before':before,'after':after,'public_test':result.stdout.strip()})
|
|
|
|
|
|
def public_failover():
|
|
cluster.wait_cluster()
|
|
try:
|
|
result = subprocess.run([sys.executable,'tests/resilience.py','--mode','failover','--action-script','scripts/stop-one.sh'],
|
|
cwd=ROOT,check=True,capture_output=True,text=True,timeout=100)
|
|
(ROOT/'evidence/public-failover.txt').write_text(result.stdout)
|
|
save('public-failover.json',{'result':'PASS','public_test':result.stdout.strip(),
|
|
'stopped':json.loads((ROOT/'evidence/last-stop.json').read_text())})
|
|
finally:
|
|
cluster.compose('up','-d','--no-recreate','--wait','--wait-timeout','90')
|
|
cluster.wait_cluster()
|
|
|
|
|
|
def reject_new_connection(survivor, address):
|
|
code = '''import json,socket,sys
|
|
try:
|
|
s=socket.create_connection((sys.argv[1],8080),timeout=1)
|
|
except OSError as exc:
|
|
print(json.dumps({'accepted':False,'error_type':type(exc).__name__}))
|
|
else:
|
|
s.close()
|
|
print(json.dumps({'accepted':True}))
|
|
'''
|
|
return json.loads(cluster.command(['docker','exec',survivor,'python','-c',code,address]))
|
|
|
|
|
|
def failover_each():
|
|
cluster.wait_cluster()
|
|
originals = cluster.replicas()
|
|
check(len(originals)==2,'two running replicas required')
|
|
traffic = Traffic().start()
|
|
phases = []
|
|
try:
|
|
time.sleep(3)
|
|
baseline_end = time.monotonic()
|
|
baseline = traffic.summary(end=baseline_end)
|
|
for original in originals:
|
|
# Alias changes on restart, so resolve the current process of this exact container.
|
|
current = next(r for r in cluster.replicas() if r['container']==original['container'])
|
|
all_before = cluster.inspect([current['container']])[0]
|
|
address = next(v['IPAddress'] for k,v in all_before['NetworkSettings']['Networks'].items() if k.endswith('_front'))
|
|
survivor = next(r['container'] for r in cluster.replicas() if r['container']!=current['container'])
|
|
read_path = '/api/v1/chats/'+traffic.chat['id']+'/messages?limit=1'
|
|
marker = traffic.c.call('GET',read_path,actor=traffic.bob,schema='MessagePage')['json']['items'][0]
|
|
start = time.monotonic()
|
|
try:
|
|
env = {**os.environ,'TARGET_INSTANCE_ID':current['instance']}
|
|
subprocess.run(['scripts/stop-one.sh'],cwd=ROOT,env=env,check=True,capture_output=True,text=True,timeout=10)
|
|
stop_returned = time.monotonic()
|
|
stopped = json.loads((ROOT/'evidence/last-stop.json').read_text())
|
|
read_deadline = start+10
|
|
while True:
|
|
try:
|
|
read = traffic.c.call('GET',read_path,actor=traffic.bob,schema='MessagePage')
|
|
check(read['json']['items'][0]==marker,'pre-failure message unavailable during outage')
|
|
check(read['headers']['X-Instance-Id']!=current['instance'],'read served by stopped process')
|
|
read_done = time.monotonic()
|
|
break
|
|
except Failure:
|
|
check(time.monotonic()<read_deadline,'reads did not recover within10s')
|
|
time.sleep(0.1)
|
|
check(read_done-start<=10,'read recovery exceeded10s')
|
|
rejected = reject_new_connection(survivor,address)
|
|
check(not rejected['accepted'],'new connection accepted by stopped API')
|
|
time.sleep(max(0,8-(time.monotonic()-start)))
|
|
end = time.monotonic()
|
|
summary = traffic.summary(start,end)
|
|
with traffic.lock:
|
|
successful = [a for a in traffic.attempts if a['ok'] and a['started']>=start]
|
|
recent = [a for a in successful if a['started']>stop_returned]
|
|
recovery = min(a['ended'] for a in recent)-start if recent else None
|
|
after_stop = min(a['ended'] for a in recent)-stop_returned if recent else None
|
|
check(recovery is not None and recovery<=10,'post-stop writes did not recover within10s of action start')
|
|
check(len(recent)>=5 and all(a['instance']!=current['instance'] for a in recent[-5:]),'stopped alias still served requests')
|
|
phases.append({'stopped':stopped, 'traffic':summary,'post_stop_write_seconds_from_action_start':round(recovery,4),
|
|
'post_stop_write_seconds_from_stop_completion':round(after_stop,4),
|
|
'read_during_outage':{'status':200,'confirmed_message_preserved':True,
|
|
'seconds_from_action_start':round(read_done-start,4),
|
|
'seconds_from_stop_completion':round(read_done-stop_returned,4)},
|
|
'new_connection_after_stop':rejected,'stopped_remains_stopped':not cluster.inspect([current['container']])[0]['State']['Running']})
|
|
finally:
|
|
cluster.command(['docker','start',current['container']])
|
|
cluster.wait_healthy(current['container'])
|
|
cluster.wait_cluster()
|
|
time.sleep(2)
|
|
finally:
|
|
traffic.stop()
|
|
save('sequential-failover.json',{'result':'PASS','baseline':baseline,'phases':phases,
|
|
'total':traffic.summary(),'data':traffic.verify_data(),
|
|
'each_original_container_tested':len({p['stopped']['container'] for p in phases})==2})
|
|
|
|
|
|
def rolling_update():
|
|
cluster.wait_cluster()
|
|
before = cluster.replicas()
|
|
check(len(before)==2,'rolling demonstration expects two replicas')
|
|
lb_before = next(item for item in cluster.snapshot() if item['service']=='lb')
|
|
target_release = 'lab3-'+uuid.uuid4().hex[:8]
|
|
traffic = Traffic().start()
|
|
replacements = []
|
|
try:
|
|
time.sleep(3)
|
|
build_start = time.monotonic()
|
|
cluster.compose('build','--build-arg','APP_RELEASE='+target_release,'api',timeout=180)
|
|
build_end = time.monotonic()
|
|
migration_output = cluster.compose('run','--rm','--no-deps','migrate')
|
|
swap_start = time.monotonic()
|
|
for old in before:
|
|
step_start = time.monotonic()
|
|
result = cluster.replace_one(old['container'])
|
|
result['seconds'] = round(time.monotonic()-step_start,4)
|
|
replacements.append(result)
|
|
swap_end = time.monotonic()
|
|
time.sleep(3)
|
|
finally:
|
|
traffic.stop()
|
|
after = cluster.replicas()
|
|
lb_after = next(item for item in cluster.snapshot() if item['service']=='lb')
|
|
check(lb_before['container']==lb_after['container'] and lb_before['started_at']==lb_after['started_at'],'LB restarted during DNS test')
|
|
check(all(r['release']==target_release for r in after),'new release not observed on all replicas')
|
|
check(not {r['container'] for r in before}&{r['container'] for r in after},'old containers remain after replacement')
|
|
check(not {r['image_id'] for r in before}&{r['image_id'] for r in after},'image did not actually change')
|
|
save('rolling-update.json',{'result':'PASS','before':before,'after':after,'target_release':target_release,
|
|
'lb_before':lb_before,'lb_after':lb_after,'lb_not_restarted':True,
|
|
'build_seconds':round(build_end-build_start,4),'migration_output':migration_output,
|
|
'replacements':replacements,'during_replacement':traffic.summary(swap_start,swap_end),
|
|
'total':traffic.summary(),'data':traffic.verify_data(),
|
|
'change':'New image with a distinct release header; API code and schema remain compatible. Start-first: scale to3, prove new IP in DNS/LB, stop/remove one old container, return to2 with --no-recreate.'})
|
|
|
|
|
|
def database_fixture(container, data):
|
|
code = '''import json,sys,psycopg
|
|
from psycopg import sql
|
|
from messenger.db import setting
|
|
p=json.load(sys.stdin)
|
|
with psycopg.connect(setting('DATABASE_URL'),connect_timeout=3) as c:
|
|
if p['action']=='install':
|
|
c.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.text = {} THEN PERFORM pg_sleep(2); END IF; RETURN NEW; END $$").format(sql.Identifier(p['function']),sql.Literal(p['text'])))
|
|
c.execute(sql.SQL('CREATE TRIGGER {} BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(p['trigger']),sql.Identifier(p['function'])))
|
|
result={'installed':True}
|
|
elif p['action']=='remove':
|
|
c.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON messages').format(sql.Identifier(p['trigger'])))
|
|
c.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(p['function'])))
|
|
result={'removed':True}
|
|
else:
|
|
result={'waiting':c.execute("SELECT count(*) FROM pg_stat_activity WHERE datname=current_database() AND wait_event='PgSleep' AND query LIKE 'INSERT INTO messages%%'").fetchone()[0]}
|
|
print(json.dumps(result))
|
|
'''
|
|
response = subprocess.run(['docker','exec','-i',container,'python','-c',code],cwd=ROOT,
|
|
input=json.dumps(data),capture_output=True,text=True,check=True,timeout=8)
|
|
return json.loads(response.stdout)
|
|
|
|
|
|
def graceful_shutdown():
|
|
cluster.wait_cluster()
|
|
nodes = cluster.replicas()
|
|
target,survivor = nodes[0],nodes[1]
|
|
address = cluster.front_address(target['container'])
|
|
c = Client()
|
|
alice,bob,_,chat = c.bootstrap()
|
|
path = '/api/v1/chats/'+chat['id']+'/messages'
|
|
marker = 'drain_'+uuid.uuid4().hex
|
|
fixture = {'function':'drain_'+uuid.uuid4().hex,'trigger':'drain_'+uuid.uuid4().hex,'text':marker}
|
|
database_fixture(survivor['container'],{**fixture,'action':'install'})
|
|
code = '''import json,sys,urllib.request,urllib.error
|
|
p=json.load(sys.stdin)
|
|
r=urllib.request.Request('http://'+p['address']+':8080'+p['path'],data=json.dumps({'text':p['text']}).encode(),method='POST',headers={'Content-Type':'application/json','Authorization':'Bearer '+p['token'],'Idempotency-Key':p['key']})
|
|
try:
|
|
response=urllib.request.urlopen(r,timeout=6)
|
|
except urllib.error.HTTPError as exc:
|
|
response=exc
|
|
print(json.dumps({'status':response.code,'instance':response.headers.get('X-Instance-Id'),'message':json.load(response)}))
|
|
'''
|
|
request_data = {'address':address,'path':path,'text':marker,'token':alice['token'],'key':uuid.uuid4().hex}
|
|
def in_flight():
|
|
result = subprocess.run(['docker','exec','-i',survivor['container'],'python','-c',code],cwd=ROOT,
|
|
input=json.dumps(request_data),capture_output=True,text=True,check=True,timeout=8)
|
|
return json.loads(result.stdout)
|
|
pool = ThreadPoolExecutor(max_workers=1)
|
|
future = pool.submit(in_flight)
|
|
try:
|
|
deadline = time.monotonic()+3
|
|
while not database_fixture(survivor['container'],{'action':'waiting'})['waiting']:
|
|
check(time.monotonic()<deadline and not future.done(),'did not observe an in-flight SQL request')
|
|
time.sleep(0.05)
|
|
signaled = time.monotonic()
|
|
cluster.command(['docker','kill','--signal','SIGTERM',target['container']])
|
|
deadline = signaled+1.5
|
|
while True:
|
|
rejected = reject_new_connection(survivor['container'],address)
|
|
still_running = cluster.inspect([target['container']])[0]['State']['Running']
|
|
if not rejected['accepted']:
|
|
break
|
|
check(time.monotonic()<deadline,'new TCP connections still accepted after SIGTERM')
|
|
time.sleep(0.05)
|
|
check(still_running,'connection refusal observed only after process exit, not during drain')
|
|
drained = future.result(timeout=8)
|
|
check(drained['status']==201 and drained['instance']==target['instance'],'in-flight request did not finish on signaled process')
|
|
check(drained['message']['text']==marker,'in-flight body changed')
|
|
exit_code = cluster.command(['docker','wait',target['container']],timeout=8)
|
|
elapsed = time.monotonic()-signaled
|
|
check(exit_code=='0' and elapsed<5,'SIGTERM exceeded graceful budget or exited with error')
|
|
check(history(c,bob,chat)==[drained['message']],'drained message missing or duplicated')
|
|
result = {'result':'PASS','target_container':target['container'],'target_instance':target['instance'],
|
|
'fixture':'temporary BEFORE INSERT trigger sleeps2s for one synthetic text; removed afterwards',
|
|
'in_flight_observed_in_pg_stat_activity':True,'in_flight_response':201,
|
|
'new_tcp_after_sigterm':rejected,'target_still_running_when_new_tcp_rejected':still_running,
|
|
'exit_code':int(exit_code),'shutdown_seconds_from_sigterm':round(elapsed,4),
|
|
'confirmed_message_preserved':True}
|
|
finally:
|
|
pool.shutdown(wait=True)
|
|
database_fixture(survivor['container'],{**fixture,'action':'remove'})
|
|
cluster.command(['docker','start',target['container']])
|
|
cluster.wait_healthy(target['container'])
|
|
cluster.wait_cluster()
|
|
save('graceful-shutdown.json',result)
|
|
|
|
|
|
def infrastructure():
|
|
state = cluster.snapshot()
|
|
for item in state:
|
|
check(item['memory_bytes']>0 and item['nano_cpus']>0,'resource limit missing')
|
|
if item['service']!='lb':
|
|
check(not item['published_ports'],'service exposed a port bypassing LB')
|
|
check(len([r for r in state if r['service']=='api' and r['status']=='running'])==2,'expected2live replicas')
|
|
check(all(r['service']!='migrate' or r['status']=='exited' for r in state),'migration job still running')
|
|
save('infrastructure.json',{'result':'PASS','containers':state})
|
|
|
|
|
|
def dependency_failures():
|
|
c = Client(timeout=6)
|
|
actor = c.user('dependencies')
|
|
c.call('GET','/api/v1/users/me',actor=actor,schema='User')
|
|
result = {'result':'PASS'}
|
|
try:
|
|
cluster.compose('stop','-t','2','redis')
|
|
response = c.call('GET','/api/v1/users/me',actor=actor,schema='User')
|
|
check(response['json']==actor['user'] and response['elapsed']<3,'Redis fallback failed')
|
|
c.call('GET','/api/v1/users/me',401)
|
|
c.call('GET','/health/ready',schema='Health')
|
|
result['redis'] = {'profile_status':200,'fallback_seconds':round(response['elapsed'],4),'anonymous':401,'readiness':200}
|
|
finally:
|
|
cluster.compose('start','redis')
|
|
import urllib.request
|
|
import urllib.error
|
|
def status(path, token=None):
|
|
headers = {'Authorization':'Bearer '+token} if token else {}
|
|
request = urllib.request.Request(c.base+path,headers=headers)
|
|
started = time.monotonic()
|
|
try:
|
|
response = urllib.request.urlopen(request,timeout=6)
|
|
except urllib.error.HTTPError as error:
|
|
response = error
|
|
with response:
|
|
return response.code, round(time.monotonic()-started,4)
|
|
try:
|
|
cluster.compose('stop','-t','2','postgres')
|
|
code,elapsed = status('/api/v1/users/me',actor['token'])
|
|
check(code in (502,503,504) and elapsed<6,'PostgreSQL outage did not safely reject authenticated request')
|
|
anonymous,_ = status('/api/v1/users/me')
|
|
check(anonymous in (401,502,503,504),'anonymous user trusted during DB outage')
|
|
live = [cluster.probe(node['container'])['status'] for node in cluster.replicas()]
|
|
check(all(code==200 for code in live),'API process liveness depends on DB')
|
|
result['postgres'] = {'authenticated_status':code,'response_seconds':elapsed,'anonymous_status':anonymous,
|
|
'internal_api_liveness':live,'note':'Nginx-generated502/503/504 may have no API JSON/instance headers when all backends become unavailable.'}
|
|
finally:
|
|
cluster.compose('start','postgres')
|
|
cluster.wait_cluster()
|
|
c.call('GET','/api/v1/users/me',actor=actor,schema='User')
|
|
result['same_session_after_recovery'] = True
|
|
save('dependency-failures.json',result)
|
|
|
|
|
|
def verify_nginx():
|
|
state = cluster.snapshot()
|
|
lb = next(item for item in state if item['service']=='lb')
|
|
check(lb['image'].startswith('nginx:'),'load balancer must use Nginx')
|
|
cluster.command(['docker','exec',lb['container'],'nginx','-t'])
|
|
version = cluster.command(['docker','exec',lb['container'],'sh','-c','nginx -v 2>&1'])
|
|
c = Client()
|
|
response = c.call('GET','/health/ready',schema='Health')
|
|
check(response['headers'].get('Server')=='nginx','public entry point is not Nginx')
|
|
check(len(cluster.wait_cluster())==2,'Nginx must discover two replicas')
|
|
save('nginx.json',{'result':'PASS','version':version,'image':lb['image'],'public_server_header':'nginx',
|
|
'config_syntax':'PASS','observed_instances':cluster.wait_cluster()})
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('mode',choices=['idempotency','persistence','public-failover','failover','rolling','sigterm','dependencies','infra','nginx','all'],default='all',nargs='?')
|
|
args = parser.parse_args()
|
|
modes = {'nginx':verify_nginx,'idempotency':concurrent_replay,'persistence':verify_persistence,'public-failover':public_failover,
|
|
'failover':failover_each,'sigterm':graceful_shutdown,'rolling':rolling_update,'dependencies':dependency_failures,'infra':infrastructure}
|
|
if args.mode=='all':
|
|
for function in modes.values():
|
|
function()
|
|
else:
|
|
modes[args.mode]()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|