Files
mtusi2026lab5/scripts/experiment.py
T
2026-10-26 18:05:06 +03:00

164 lines
10 KiB
Python

"""Трек 2: повторяемые реальные crashes, дубли, reorder, версии и восстановление."""
import argparse
import json
import os
from pathlib import Path
import statistics
import sys
import time
ROOT=Path(__file__).resolve().parents[1]
sys.path.insert(0,str(ROOT))
from scripts.stack import command,containers,wait,save
class Runner:
def __init__(self):
self.env=dict(os.environ,ALLOW_FAULT_INJECTION='1',WORKER_CONCURRENCY='1')
self.base=os.getenv('BASE_URL','http://localhost:8080')
def call(self,mode,ctx=None,**options):
arguments=['docker','compose','run','--rm','--no-deps','-T','-e','BASE_URL=http://lb:8080',
'dispatcher','python','student-tests/pipeline_scenario.py',mode]
for key,value in options.items():arguments.extend(['--'+key.replace('_','-'),str(value)])
# Контекст с Bearer остаётся в памяти клиента и stdin, не в argv/evidence.
return json.loads(command(arguments,env=self.env,input=json.dumps(ctx or {}),timeout=150))
def service(self,action,name):
if action=='stop':command(['docker','compose','stop',name],env=self.env)
elif action=='restart':
command(['docker','compose','stop',name],env=self.env)
command(['docker','compose','up','-d','--no-deps','--wait',name],env=self.env)
else:
args=['docker','compose','up','-d','--no-deps']
if action=='recreate':args.append('--force-recreate')
command([*args,name],env=self.env)
if name in {'api','s3'} and action!='stop':wait(2,self.base)
def infrastructure(self):return containers()
def finish(self):
normal=dict(os.environ,ALLOW_FAULT_INJECTION='0')
command(['docker','compose','up','-d','--no-deps','--force-recreate','worker','dispatcher'],env=normal)
command(['docker','compose','up','-d','--no-deps','reconciler'],env=normal)
wait(2,self.base)
def same_effect(before,after):
assert before['attachment']==after['attachment'],'Duplicate altered public metadata'
assert before['current_keys']==after['current_keys'],'Duplicate multiplied variants'
assert before['variants_sha256']==after['variants_sha256'],'Duplicate changed bytes'
assert before['original_sha256']==after['original_sha256']
def run_suite(runner,repeats,on_result):
results=[]
def ready(ctx):
runner.call('wait',ctx)
return runner.call('snapshot',ctx)
def record(name,repeat,started,**values):
row=dict(case=name,repetition=repeat,status='PASS',elapsed_seconds=round(time.monotonic()-started,3),**values)
results.append(row);on_result(results)
print(f'PASS: {name}, repeat {repeat}/{repeats}',flush=True)
runner.service('stop','reconciler')
for repeat in range(1,repeats+1):
started=time.monotonic();runner.service('stop','worker');runner.service('start','dispatcher')
ctx=runner.call('create');queued=runner.call('queued',ctx)
published=runner.call('published',ctx)
runner.service('stop','dispatcher');runner.service('recreate','api');runner.service('restart','broker')
restored=runner.call('published',ctx);assert restored['stream_ids']==published['stream_ids']
runner.service('start','dispatcher');runner.service('recreate','worker')
record('accepted_restart_broker_api',repeat,started,queued=queued,broker_preserved=restored,result=ready(ctx),infrastructure=runner.infrastructure())
for phase in ['before_publish','after_publish']:
started=time.monotonic();runner.service('stop','worker');runner.service('stop','dispatcher')
ctx=runner.call('create');runner.call('arm',ctx,phase=phase)
runner.service('recreate','dispatcher');fired=runner.call('fault-fired',ctx,phase=phase)
runner.service('recreate','worker');result=ready(ctx)
record(phase,repeat,started,fault=fired,result=result,infrastructure=runner.infrastructure())
for phase in ['before_effect','after_original','after_effects','after_commit']:
started=time.monotonic();runner.service('stop','worker')
ctx=runner.call('create');runner.call('arm',ctx,phase=phase)
runner.service('recreate','worker');fired=runner.call('fault-fired',ctx,phase=phase)
result=ready(ctx)
if phase=='after_commit':assert result['attempts']==1,'Committed result was processed again'
else:assert result['attempts']>=2,'Abandoned lease was not recovered'
record(phase,repeat,started,fault=fired,result=result,infrastructure=runner.infrastructure())
started=time.monotonic();ctx=runner.call('create');before=ready(ctx)
copies=runner.call('duplicate',dict(ctx,copies=16))
acked=runner.call('drained',dict(ctx,stream_ids=copies['stream_ids']))
after=runner.call('snapshot',ctx);same_effect(before,after)
rejected=runner.call('invalid-job',ctx)
assert len([r for r in after['events'] if r['event']=='duplicate_ready'])>=16
record('duplicates_and_invalid_carrier',repeat,started,duplicates=acked,invalid_carrier=rejected,result=after)
# Новая job приходит первой, старая после неё: контролируемый reorder.
started=time.monotonic();runner.service('stop','worker');runner.service('stop','dispatcher')
old=runner.call('create');new=runner.call('version',old)
first=runner.call('duplicate',new);second=runner.call('duplicate',old)
runner.service('recreate','worker')
runner.call('drained',dict(new,stream_ids=first['stream_ids']+second['stream_ids']))
result=ready(new);old_audit=runner.call('audit',old)
assert any(r['event']=='stale_ignored' for r in old_audit['events'])
assert result['pipeline_version']==2
record('reorder_new_before_old',repeat,started,result=result,old_job=old_audit)
runner.service('start','dispatcher')
# Теперь старая версия уже имела ready; её поздний дубль не меняет v2.
started=time.monotonic();old=runner.call('create');v1=ready(old)
new=runner.call('version',old);before=ready(new)
assert before['variants_sha256']!=v1['variants_sha256'],'Pipeline version did not change actual output'
copies=runner.call('duplicate',dict(old,copies=3));runner.call('drained',dict(old,stream_ids=copies['stream_ids']))
after=runner.call('snapshot',new);same_effect(before,after)
record('stale_after_new_ready',repeat,started,result=after,old_job=runner.call('audit',old))
started=time.monotonic();runner.service('stop','worker');ctx=runner.call('create')
runner.call('missing-stage',ctx);runner.service('recreate','worker');runner.call('wait',ctx,state='failed')
failed=runner.call('failed',ctx);assert failed['error_code']=='staging_missing'
runner.call('restore-stage',ctx);runner.call('replay',ctx);result=ready(ctx)
noop=runner.call('replay',ctx);assert noop['status']=='noop'
record('permanent_failure_replay',repeat,started,failed=failed,ready_replay=noop,result=result)
# Физический stop S3, а не подмена ответа или бесконечная retry-петля.
started=time.monotonic();runner.service('stop','worker');ctx=runner.call('create')
runner.service('stop','s3')
try:
time.sleep(4);unready=runner.call('unready',ctx)
runner.service('recreate','worker');runner.call('wait',ctx,state='failed',timeout=120)
finally:runner.service('start','s3')
failed=runner.call('failed',ctx);assert failed['error_code']=='retry_exhausted'
assert failed['attempts']==5
runner.call('replay',ctx);result=ready(ctx)
record('bounded_retry_and_replay',repeat,started,unready=unready,failed=failed,result=result)
started=time.monotonic();runner.service('stop','worker');ctx=runner.call('create')
cleaned=runner.call('cleanup',ctx);runner.service('recreate','worker');result=ready(ctx)
privacy=runner.call('privacy',ctx)
record('staging_retention_and_privacy',repeat,started,cleanup=cleaned,privacy=privacy,result=result)
return results
def main():
p=argparse.ArgumentParser();p.add_argument('--repeats',type=int,default=3);args=p.parse_args()
if not 1<=args.repeats<=5:p.error('repeats must be 1..5')
runner=Runner()
save('pipeline_environment.json',dict(docker=command(['docker','version','--format','{{.Server.Version}}']),
compose=command(['docker','compose','version','--short']),containers=containers(),client_cpus=os.cpu_count(),
pipeline=runner.call('config')))
def persist(rows):save('pipeline_raw.json',dict(status='RUNNING',track=2,repeats=args.repeats,cases=rows))
persist([]);save('pipeline_summary.json',dict(status='RUNNING',expected_checks=13*args.repeats))
try:
try:rows=run_suite(runner,args.repeats,persist)
finally:runner.finish()
except Exception as exc:
save('pipeline_summary.json',dict(status='FAILED',reason=type(exc).__name__))
raise
save('pipeline_raw.json',dict(status='PASS',track=2,repeats=args.repeats,cases=rows))
summary=[]
for case in dict.fromkeys(r['case'] for r in rows):
subset=[r for r in rows if r['case']==case]
summary.append(dict(case=case,repetitions=len(subset),status='PASS',
median_seconds=statistics.median(r['elapsed_seconds'] for r in subset)))
save('pipeline_summary.json',dict(status='PASS',track=2,cases=summary,checks=len(rows),
invariants=['accepted work preserved','one ready commit per job','exactly three named variants',
'original checksum preserved','stale version cannot replace current metadata/bytes',
'bounded retries and explicit replay','staging cleanup protects active jobs']))
print('Experiment completed; results saved in evidence/',flush=True)
if __name__=='__main__':main()