#!/usr/bin/env python3 """Grade4 distributed acceptance through the only public entry point; no tokens in artifacts.""" import argparse import json import math import os from pathlib import Path import subprocess import sys import threading import time from concurrent.futures import ThreadPoolExecutor import uuid import cluster ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0,str(ROOT/'tests')) from client import Client, Failure, check from resilience import history def save(name, value): (ROOT/'evidence'/name).write_text(json.dumps(value,ensure_ascii=False,indent=2)+'\n') print('PASS:', name) def percentiles(values): values = sorted(values) return {name:round(1000*values[max(0,math.ceil(p*len(values))-1)],3) if values else None for name,p in [('p50',.50),('p95',.95),('p99',.99)]} class Traffic: def __init__(self, concurrency=4): self.c = Client(timeout=6) self.alice,self.bob,_,self.chat = self.c.bootstrap() self.concurrency = concurrency self.done = threading.Event() self.lock = threading.Lock() self.attempts = [] self.confirmed = {} self.prefix = uuid.uuid4().hex def worker(self, number): client = Client(timeout=6) index = 0 while not self.done.is_set(): text = self.prefix+'-'+str(number)+'-'+str(index) key = self.prefix+'-'+str(number)+'-'+str(index) started = time.monotonic() try: response = client.message(self.alice,self.chat,text,key=key) row = response['json'] attempt = {'started':started,'ended':time.monotonic(),'ok':True, 'latency':response['elapsed'],'instance':response['headers']['X-Instance-Id']} with self.lock: self.confirmed[row['id']] = row index += 1 except Failure: # Only idempotent message POST is retried, with its original body and key. attempt = {'started':started,'ended':time.monotonic(),'ok':False,'instance':None} time.sleep(0.02) with self.lock: self.attempts.append(attempt) def start(self): self.started = time.monotonic() self.pool = ThreadPoolExecutor(max_workers=self.concurrency) self.futures = [self.pool.submit(self.worker,n) for n in range(self.concurrency)] return self def stop(self): self.done.set() for future in self.futures: future.result(timeout=10) self.pool.shutdown() self.ended = time.monotonic() def summary(self, start=None, end=None): start = self.started if start is None else start end = getattr(self,'ended',time.monotonic()) if end is None else end with self.lock: attempts = [a for a in self.attempts if start <= a['started'] < end] success = [a for a in attempts if a['ok']] duration = end-start return {'seconds':round(duration,4),'concurrency':self.concurrency, 'successful_requests':len(success),'failed_attempts':len(attempts)-len(success), 'successful_rps':round(len(success)/duration,3), 'error_ratio':round((len(attempts)-len(success))/len(attempts),6) if attempts else 0, 'successful_latency_ms':percentiles([a['latency'] for a in success]), 'instances':sorted({a['instance'] for a in success})} def verify_data(self): rows = history(self.c,self.bob,self.chat) actual = {row['id']:row for row in rows} check(len(actual)==len(rows),'duplicated message ids') check(len({row['text'] for row in rows})==len(rows),'retry duplicated a message') check(all(actual.get(mid)==row for mid,row in self.confirmed.items()),'confirmed message lost or changed') return {'confirmed_unique_messages':len(self.confirmed), 'history_messages':len(rows), 'confirmed_preserved':True, 'duplicate_ids':0, 'duplicate_texts':0} def concurrent_replay(): cluster.wait_cluster() c = Client() alice,bob,_,chat = c.bootstrap() key = uuid.uuid4().hex barrier = threading.Barrier(8) def send(_): local = Client() barrier.wait() return local.message(alice,chat,'Один результат через разные процессы',key=key) with ThreadPoolExecutor(max_workers=8) as pool: results = list(pool.map(send,range(8))) instances = {r['headers']['X-Instance-Id'] for r in results} check(len(instances)>=2,'concurrent retry did not reach different API processes') check(all(r['json']==results[0]['json'] for r in results),'concurrent key returned different messages') c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice, body={'text':'Другой body'},headers={'Idempotency-Key':key}) check(history(c,bob,chat)==[results[0]['json']],'repeated key created duplicates') save('concurrent-idempotency.json',{'result':'PASS','parallel_requests':8,'observed_instances':sorted(instances), 'responses_201':8,'messages_created':1,'different_body_status':409}) def verify_persistence(): before = cluster.snapshot() result = subprocess.run([sys.executable,'tests/resilience.py','--mode','persistence','--action-script','scripts/restart.sh'], cwd=ROOT,check=True,capture_output=True,text=True,timeout=190) after = cluster.snapshot() roles = ['api','postgres','redis','minio','cleanup','lb'] previous = {item['container']:item for item in before} for item in after: if item['service'] in roles: old = previous[item['container']] check(old['started_at']!=item['started_at'],'service did not restart: '+item['service']) check(old['volumes']==item['volumes'],'volume identities changed') (ROOT/'evidence/persistence.txt').write_text(result.stdout) save('persistence.json',{'result':'PASS','before':before,'after':after,'public_test':result.stdout.strip()}) def public_failover(): cluster.wait_cluster() try: result = subprocess.run([sys.executable,'tests/resilience.py','--mode','failover','--action-script','scripts/stop-one.sh'], cwd=ROOT,check=True,capture_output=True,text=True,timeout=100) (ROOT/'evidence/public-failover.txt').write_text(result.stdout) save('public-failover.json',{'result':'PASS','public_test':result.stdout.strip(), 'stopped':json.loads((ROOT/'evidence/last-stop.json').read_text())}) finally: cluster.compose('up','-d','--no-recreate','--wait','--wait-timeout','90') cluster.wait_cluster() def reject_new_connection(survivor, address): code = '''import json,socket,sys try: s=socket.create_connection((sys.argv[1],8080),timeout=1) except OSError as exc: print(json.dumps({'accepted':False,'error_type':type(exc).__name__})) else: s.close() print(json.dumps({'accepted':True})) ''' return json.loads(cluster.command(['docker','exec',survivor,'python','-c',code,address])) def failover_each(): cluster.wait_cluster() originals = cluster.replicas() check(len(originals)==2,'two running replicas required') traffic = Traffic().start() phases = [] try: time.sleep(3) baseline_end = time.monotonic() baseline = traffic.summary(end=baseline_end) for original in originals: # Alias changes on restart, so resolve the current process of this exact container. current = next(r for r in cluster.replicas() if r['container']==original['container']) all_before = cluster.inspect([current['container']])[0] address = next(v['IPAddress'] for k,v in all_before['NetworkSettings']['Networks'].items() if k.endswith('_front')) survivor = next(r['container'] for r in cluster.replicas() if r['container']!=current['container']) read_path = '/api/v1/chats/'+traffic.chat['id']+'/messages?limit=1' marker = traffic.c.call('GET',read_path,actor=traffic.bob,schema='MessagePage')['json']['items'][0] start = time.monotonic() try: env = {**os.environ,'TARGET_INSTANCE_ID':current['instance']} subprocess.run(['scripts/stop-one.sh'],cwd=ROOT,env=env,check=True,capture_output=True,text=True,timeout=10) stop_returned = time.monotonic() stopped = json.loads((ROOT/'evidence/last-stop.json').read_text()) read_deadline = start+10 while True: try: read = traffic.c.call('GET',read_path,actor=traffic.bob,schema='MessagePage') check(read['json']['items'][0]==marker,'pre-failure message unavailable during outage') check(read['headers']['X-Instance-Id']!=current['instance'],'read served by stopped process') read_done = time.monotonic() break except Failure: check(time.monotonic()=start] recent = [a for a in successful if a['started']>stop_returned] recovery = min(a['ended'] for a in recent)-start if recent else None after_stop = min(a['ended'] for a in recent)-stop_returned if recent else None check(recovery is not None and recovery<=10,'post-stop writes did not recover within10s of action start') check(len(recent)>=5 and all(a['instance']!=current['instance'] for a in recent[-5:]),'stopped alias still served requests') phases.append({'stopped':stopped, 'traffic':summary,'post_stop_write_seconds_from_action_start':round(recovery,4), 'post_stop_write_seconds_from_stop_completion':round(after_stop,4), 'read_during_outage':{'status':200,'confirmed_message_preserved':True, 'seconds_from_action_start':round(read_done-start,4), 'seconds_from_stop_completion':round(read_done-stop_returned,4)}, 'new_connection_after_stop':rejected,'stopped_remains_stopped':not cluster.inspect([current['container']])[0]['State']['Running']}) finally: cluster.command(['docker','start',current['container']]) cluster.wait_healthy(current['container']) cluster.wait_cluster() time.sleep(2) finally: traffic.stop() save('sequential-failover.json',{'result':'PASS','baseline':baseline,'phases':phases, 'total':traffic.summary(),'data':traffic.verify_data(), 'each_original_container_tested':len({p['stopped']['container'] for p in phases})==2}) def rolling_update(): cluster.wait_cluster() before = cluster.replicas() check(len(before)==2,'rolling demonstration expects two replicas') lb_before = next(item for item in cluster.snapshot() if item['service']=='lb') target_release = 'lab4-'+uuid.uuid4().hex[:8] traffic = Traffic().start() replacements = [] try: time.sleep(3) build_start = time.monotonic() cluster.compose('build','--build-arg','APP_RELEASE='+target_release,'api',timeout=180) build_end = time.monotonic() migration_output = cluster.compose('run','--rm','--no-deps','migrate') swap_start = time.monotonic() for old in before: step_start = time.monotonic() result = cluster.replace_one(old['container']) result['seconds'] = round(time.monotonic()-step_start,4) replacements.append(result) swap_end = time.monotonic() time.sleep(3) finally: traffic.stop() after = cluster.replicas() lb_after = next(item for item in cluster.snapshot() if item['service']=='lb') check(lb_before['container']==lb_after['container'] and lb_before['started_at']==lb_after['started_at'],'LB restarted during DNS test') check(all(r['release']==target_release for r in after),'new release not observed on all replicas') check(not {r['container'] for r in before}&{r['container'] for r in after},'old containers remain after replacement') check(not {r['image_id'] for r in before}&{r['image_id'] for r in after},'image did not actually change') save('rolling-update.json',{'result':'PASS','before':before,'after':after,'target_release':target_release, 'lb_before':lb_before,'lb_after':lb_after,'lb_not_restarted':True, 'build_seconds':round(build_end-build_start,4),'migration_output':migration_output, 'replacements':replacements,'during_replacement':traffic.summary(swap_start,swap_end), 'total':traffic.summary(),'data':traffic.verify_data(), 'change':'New image with a distinct release header; API code and schema remain compatible. Start-first: scale to3, prove new IP in DNS/LB, stop/remove one old container, return to2 with --no-recreate.'}) def database_fixture(container, data): code = '''import json,sys,psycopg from psycopg import sql from messenger.db import setting p=json.load(sys.stdin) with psycopg.connect(setting('DATABASE_URL'),connect_timeout=3) as c: if p['action']=='install': c.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.text = {} THEN PERFORM pg_sleep(2); END IF; RETURN NEW; END $$").format(sql.Identifier(p['function']),sql.Literal(p['text']))) c.execute(sql.SQL('CREATE TRIGGER {} BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(p['trigger']),sql.Identifier(p['function']))) result={'installed':True} elif p['action']=='remove': c.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON messages').format(sql.Identifier(p['trigger']))) c.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(p['function']))) result={'removed':True} else: result={'waiting':c.execute("SELECT count(*) FROM pg_stat_activity WHERE datname=current_database() AND wait_event='PgSleep' AND query LIKE 'INSERT INTO messages%%'").fetchone()[0]} print(json.dumps(result)) ''' response = subprocess.run(['docker','exec','-i',container,'python','-c',code],cwd=ROOT, input=json.dumps(data),capture_output=True,text=True,check=True,timeout=8) return json.loads(response.stdout) def graceful_shutdown(): cluster.wait_cluster() nodes = cluster.replicas() target,survivor = nodes[0],nodes[1] address = cluster.front_address(target['container']) c = Client() alice,bob,_,chat = c.bootstrap() path = '/api/v1/chats/'+chat['id']+'/messages' marker = 'drain_'+uuid.uuid4().hex fixture = {'function':'drain_'+uuid.uuid4().hex,'trigger':'drain_'+uuid.uuid4().hex,'text':marker} database_fixture(survivor['container'],{**fixture,'action':'install'}) code = '''import json,sys,urllib.request,urllib.error p=json.load(sys.stdin) r=urllib.request.Request('http://'+p['address']+':8080'+p['path'],data=json.dumps({'text':p['text']}).encode(),method='POST',headers={'Content-Type':'application/json','Authorization':'Bearer '+p['token'],'Idempotency-Key':p['key']}) try: response=urllib.request.urlopen(r,timeout=6) except urllib.error.HTTPError as exc: response=exc print(json.dumps({'status':response.code,'instance':response.headers.get('X-Instance-Id'),'message':json.load(response)})) ''' request_data = {'address':address,'path':path,'text':marker,'token':alice['token'],'key':uuid.uuid4().hex} def in_flight(): result = subprocess.run(['docker','exec','-i',survivor['container'],'python','-c',code],cwd=ROOT, input=json.dumps(request_data),capture_output=True,text=True,check=True,timeout=8) return json.loads(result.stdout) pool = ThreadPoolExecutor(max_workers=1) future = pool.submit(in_flight) try: deadline = time.monotonic()+3 while not database_fixture(survivor['container'],{'action':'waiting'})['waiting']: check(time.monotonic()0 and item['nano_cpus']>0,'resource limit missing') if item['service']!='lb': check(not item['published_ports'],'service exposed a port bypassing LB') check(len([r for r in state if r['service']=='api' and r['status']=='running'])==2,'expected2live replicas') check(all(r['service']!='migrate' or r['status']=='exited' for r in state),'migration job still running') save('infrastructure.json',{'result':'PASS','containers':state}) def dependency_failures(): c = Client(timeout=6) actor = c.user('dependencies') c.call('GET','/api/v1/users/me',actor=actor,schema='User') result = {'result':'PASS'} try: cluster.compose('stop','-t','2','redis') response = c.call('GET','/api/v1/users/me',actor=actor,schema='User') check(response['json']==actor['user'] and response['elapsed']<3,'Redis fallback failed') c.call('GET','/api/v1/users/me',401) c.call('GET','/health/ready',schema='Health') result['redis'] = {'profile_status':200,'fallback_seconds':round(response['elapsed'],4),'anonymous':401,'readiness':200} finally: cluster.compose('start','redis') import urllib.request import urllib.error def status(path, token=None): headers = {'Authorization':'Bearer '+token} if token else {} request = urllib.request.Request(c.base+path,headers=headers) started = time.monotonic() try: response = urllib.request.urlopen(request,timeout=6) except urllib.error.HTTPError as error: response = error with response: return response.code, round(time.monotonic()-started,4) try: cluster.compose('stop','-t','2','postgres') code,elapsed = status('/api/v1/users/me',actor['token']) check(code in (502,503,504) and elapsed<6,'PostgreSQL outage did not safely reject authenticated request') anonymous,_ = status('/api/v1/users/me') check(anonymous in (401,502,503,504),'anonymous user trusted during DB outage') live = [cluster.probe(node['container'])['status'] for node in cluster.replicas()] check(all(code==200 for code in live),'API process liveness depends on DB') result['postgres'] = {'authenticated_status':code,'response_seconds':elapsed,'anonymous_status':anonymous, 'internal_api_liveness':live,'note':'Nginx-generated502/503/504 may have no API JSON/instance headers when all backends become unavailable.'} finally: cluster.compose('start','postgres') cluster.wait_cluster() c.call('GET','/api/v1/users/me',actor=actor,schema='User') result['same_session_after_recovery'] = True save('dependency-failures.json',result) def verify_nginx(): state = cluster.snapshot() lb = next(item for item in state if item['service']=='lb') check(lb['image'].startswith('nginx:'),'load balancer must use Nginx') cluster.command(['docker','exec',lb['container'],'nginx','-t']) version = cluster.command(['docker','exec',lb['container'],'sh','-c','nginx -v 2>&1']) c = Client() response = c.call('GET','/health/ready',schema='Health') check(response['headers'].get('Server')=='nginx','public entry point is not Nginx') check(len(cluster.wait_cluster())==2,'Nginx must discover two replicas') save('nginx.json',{'result':'PASS','version':version,'image':lb['image'],'public_server_header':'nginx', 'config_syntax':'PASS','observed_instances':cluster.wait_cluster()}) def main(): parser = argparse.ArgumentParser() parser.add_argument('mode',choices=['idempotency','persistence','public-failover','failover','rolling','sigterm','dependencies','infra','nginx','all'],default='all',nargs='?') args = parser.parse_args() modes = {'nginx':verify_nginx,'idempotency':concurrent_replay,'persistence':verify_persistence,'public-failover':public_failover, 'failover':failover_each,'sigterm':graceful_shutdown,'rolling':rolling_update,'dependencies':dependency_failures,'infra':infrastructure} if args.mode=='all': for function in modes.values(): function() else: modes[args.mode]() if __name__ == '__main__': main()