"""Трек 2: повторяемые реальные crashes, дубли, reorder, версии и восстановление.""" import argparse import json import os from pathlib import Path import statistics import sys import time ROOT=Path(__file__).resolve().parents[1] sys.path.insert(0,str(ROOT)) from scripts.stack import command,containers,wait,save class Runner: def __init__(self): self.env=dict(os.environ,ALLOW_FAULT_INJECTION='1',WORKER_CONCURRENCY='1') self.base=os.getenv('BASE_URL','http://localhost:8080') def call(self,mode,ctx=None,**options): arguments=['docker','compose','run','--rm','--no-deps','-T','-e','BASE_URL=http://lb:8080', 'dispatcher','python','student-tests/pipeline_scenario.py',mode] for key,value in options.items():arguments.extend(['--'+key.replace('_','-'),str(value)]) # Контекст с Bearer остаётся в памяти клиента и stdin, не в argv/evidence. return json.loads(command(arguments,env=self.env,input=json.dumps(ctx or {}),timeout=150)) def service(self,action,name): if action=='stop':command(['docker','compose','stop',name],env=self.env) elif action=='restart': command(['docker','compose','stop',name],env=self.env) command(['docker','compose','up','-d','--no-deps','--wait',name],env=self.env) else: args=['docker','compose','up','-d','--no-deps'] if action=='recreate':args.append('--force-recreate') command([*args,name],env=self.env) if name in {'api','s3'} and action!='stop':wait(2,self.base) def infrastructure(self):return containers() def finish(self): normal=dict(os.environ,ALLOW_FAULT_INJECTION='0') command(['docker','compose','up','-d','--no-deps','--force-recreate','worker','dispatcher'],env=normal) command(['docker','compose','up','-d','--no-deps','reconciler'],env=normal) wait(2,self.base) def same_effect(before,after): assert before['attachment']==after['attachment'],'Duplicate altered public metadata' assert before['current_keys']==after['current_keys'],'Duplicate multiplied variants' assert before['variants_sha256']==after['variants_sha256'],'Duplicate changed bytes' assert before['original_sha256']==after['original_sha256'] def run_suite(runner,repeats,on_result): results=[] def ready(ctx): runner.call('wait',ctx) return runner.call('snapshot',ctx) def record(name,repeat,started,**values): row=dict(case=name,repetition=repeat,status='PASS',elapsed_seconds=round(time.monotonic()-started,3),**values) results.append(row);on_result(results) print(f'PASS: {name}, repeat {repeat}/{repeats}',flush=True) runner.service('stop','reconciler') for repeat in range(1,repeats+1): started=time.monotonic();runner.service('stop','worker');runner.service('start','dispatcher') ctx=runner.call('create');queued=runner.call('queued',ctx) published=runner.call('published',ctx) runner.service('stop','dispatcher');runner.service('recreate','api');runner.service('restart','broker') restored=runner.call('published',ctx);assert restored['stream_ids']==published['stream_ids'] runner.service('start','dispatcher');runner.service('recreate','worker') record('accepted_restart_broker_api',repeat,started,queued=queued,broker_preserved=restored,result=ready(ctx),infrastructure=runner.infrastructure()) for phase in ['before_publish','after_publish']: started=time.monotonic();runner.service('stop','worker');runner.service('stop','dispatcher') ctx=runner.call('create');runner.call('arm',ctx,phase=phase) runner.service('recreate','dispatcher');fired=runner.call('fault-fired',ctx,phase=phase) runner.service('recreate','worker');result=ready(ctx) record(phase,repeat,started,fault=fired,result=result,infrastructure=runner.infrastructure()) for phase in ['before_effect','after_original','after_effects','after_commit']: started=time.monotonic();runner.service('stop','worker') ctx=runner.call('create');runner.call('arm',ctx,phase=phase) runner.service('recreate','worker');fired=runner.call('fault-fired',ctx,phase=phase) result=ready(ctx) if phase=='after_commit':assert result['attempts']==1,'Committed result was processed again' else:assert result['attempts']>=2,'Abandoned lease was not recovered' record(phase,repeat,started,fault=fired,result=result,infrastructure=runner.infrastructure()) started=time.monotonic();ctx=runner.call('create');before=ready(ctx) copies=runner.call('duplicate',dict(ctx,copies=16)) acked=runner.call('drained',dict(ctx,stream_ids=copies['stream_ids'])) after=runner.call('snapshot',ctx);same_effect(before,after) rejected=runner.call('invalid-job',ctx) assert len([r for r in after['events'] if r['event']=='duplicate_ready'])>=16 record('duplicates_and_invalid_carrier',repeat,started,duplicates=acked,invalid_carrier=rejected,result=after) # Новая job приходит первой, старая после неё: контролируемый reorder. started=time.monotonic();runner.service('stop','worker');runner.service('stop','dispatcher') old=runner.call('create');new=runner.call('version',old) first=runner.call('duplicate',new);second=runner.call('duplicate',old) runner.service('recreate','worker') runner.call('drained',dict(new,stream_ids=first['stream_ids']+second['stream_ids'])) result=ready(new);old_audit=runner.call('audit',old) assert any(r['event']=='stale_ignored' for r in old_audit['events']) assert result['pipeline_version']==2 record('reorder_new_before_old',repeat,started,result=result,old_job=old_audit) runner.service('start','dispatcher') # Теперь старая версия уже имела ready; её поздний дубль не меняет v2. started=time.monotonic();old=runner.call('create');v1=ready(old) new=runner.call('version',old);before=ready(new) assert before['variants_sha256']!=v1['variants_sha256'],'Pipeline version did not change actual output' copies=runner.call('duplicate',dict(old,copies=3));runner.call('drained',dict(old,stream_ids=copies['stream_ids'])) after=runner.call('snapshot',new);same_effect(before,after) record('stale_after_new_ready',repeat,started,result=after,old_job=runner.call('audit',old)) started=time.monotonic();runner.service('stop','worker');ctx=runner.call('create') runner.call('missing-stage',ctx);runner.service('recreate','worker');runner.call('wait',ctx,state='failed') failed=runner.call('failed',ctx);assert failed['error_code']=='staging_missing' runner.call('restore-stage',ctx);runner.call('replay',ctx);result=ready(ctx) noop=runner.call('replay',ctx);assert noop['status']=='noop' record('permanent_failure_replay',repeat,started,failed=failed,ready_replay=noop,result=result) # Физический stop S3, а не подмена ответа или бесконечная retry-петля. started=time.monotonic();runner.service('stop','worker');ctx=runner.call('create') runner.service('stop','s3') try: time.sleep(4);unready=runner.call('unready',ctx) runner.service('recreate','worker');runner.call('wait',ctx,state='failed',timeout=120) finally:runner.service('start','s3') failed=runner.call('failed',ctx);assert failed['error_code']=='retry_exhausted' assert failed['attempts']==5 runner.call('replay',ctx);result=ready(ctx) record('bounded_retry_and_replay',repeat,started,unready=unready,failed=failed,result=result) started=time.monotonic();runner.service('stop','worker');ctx=runner.call('create') cleaned=runner.call('cleanup',ctx);runner.service('recreate','worker');result=ready(ctx) privacy=runner.call('privacy',ctx) record('staging_retention_and_privacy',repeat,started,cleanup=cleaned,privacy=privacy,result=result) return results def main(): p=argparse.ArgumentParser();p.add_argument('--repeats',type=int,default=3);args=p.parse_args() if not 1<=args.repeats<=5:p.error('repeats must be 1..5') runner=Runner() save('pipeline_environment.json',dict(docker=command(['docker','version','--format','{{.Server.Version}}']), compose=command(['docker','compose','version','--short']),containers=containers(),client_cpus=os.cpu_count(), pipeline=runner.call('config'))) def persist(rows):save('pipeline_raw.json',dict(status='RUNNING',track=2,repeats=args.repeats,cases=rows)) persist([]);save('pipeline_summary.json',dict(status='RUNNING',expected_checks=13*args.repeats)) try: try:rows=run_suite(runner,args.repeats,persist) finally:runner.finish() except Exception as exc: save('pipeline_summary.json',dict(status='FAILED',reason=type(exc).__name__)) raise save('pipeline_raw.json',dict(status='PASS',track=2,repeats=args.repeats,cases=rows)) summary=[] for case in dict.fromkeys(r['case'] for r in rows): subset=[r for r in rows if r['case']==case] summary.append(dict(case=case,repetitions=len(subset),status='PASS', median_seconds=statistics.median(r['elapsed_seconds'] for r in subset))) save('pipeline_summary.json',dict(status='PASS',track=2,cases=summary,checks=len(rows), invariants=['accepted work preserved','one ready commit per job','exactly three named variants', 'original checksum preserved','stale version cannot replace current metadata/bytes', 'bounded retries and explicit replay','staging cleanup protects active jobs'])) print('Experiment completed; results saved in evidence/',flush=True) if __name__=='__main__':main()