forked from mtusi2026/mtusi2026lab5
164 lines
10 KiB
Python
164 lines
10 KiB
Python
"""Трек 2: повторяемые реальные crashes, дубли, reorder, версии и восстановление."""
|
|
import argparse
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import statistics
|
|
import sys
|
|
import time
|
|
|
|
ROOT=Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0,str(ROOT))
|
|
from scripts.stack import command,containers,wait,save
|
|
|
|
|
|
class Runner:
|
|
def __init__(self):
|
|
self.env=dict(os.environ,ALLOW_FAULT_INJECTION='1',WORKER_CONCURRENCY='1')
|
|
self.base=os.getenv('BASE_URL','http://localhost:8080')
|
|
|
|
def call(self,mode,ctx=None,**options):
|
|
arguments=['docker','compose','run','--rm','--no-deps','-T','-e','BASE_URL=http://lb:8080',
|
|
'dispatcher','python','student-tests/pipeline_scenario.py',mode]
|
|
for key,value in options.items():arguments.extend(['--'+key.replace('_','-'),str(value)])
|
|
# Контекст с Bearer остаётся в памяти клиента и stdin, не в argv/evidence.
|
|
return json.loads(command(arguments,env=self.env,input=json.dumps(ctx or {}),timeout=150))
|
|
|
|
def service(self,action,name):
|
|
if action=='stop':command(['docker','compose','stop',name],env=self.env)
|
|
elif action=='restart':
|
|
command(['docker','compose','stop',name],env=self.env)
|
|
command(['docker','compose','up','-d','--no-deps','--wait',name],env=self.env)
|
|
else:
|
|
args=['docker','compose','up','-d','--no-deps']
|
|
if action=='recreate':args.append('--force-recreate')
|
|
command([*args,name],env=self.env)
|
|
if name in {'api','s3'} and action!='stop':wait(2,self.base)
|
|
|
|
def infrastructure(self):return containers()
|
|
|
|
def finish(self):
|
|
normal=dict(os.environ,ALLOW_FAULT_INJECTION='0')
|
|
command(['docker','compose','up','-d','--no-deps','--force-recreate','worker','dispatcher'],env=normal)
|
|
command(['docker','compose','up','-d','--no-deps','reconciler'],env=normal)
|
|
wait(2,self.base)
|
|
|
|
|
|
def same_effect(before,after):
|
|
assert before['attachment']==after['attachment'],'Duplicate altered public metadata'
|
|
assert before['current_keys']==after['current_keys'],'Duplicate multiplied variants'
|
|
assert before['variants_sha256']==after['variants_sha256'],'Duplicate changed bytes'
|
|
assert before['original_sha256']==after['original_sha256']
|
|
|
|
|
|
def run_suite(runner,repeats,on_result):
|
|
results=[]
|
|
def ready(ctx):
|
|
runner.call('wait',ctx)
|
|
return runner.call('snapshot',ctx)
|
|
def record(name,repeat,started,**values):
|
|
row=dict(case=name,repetition=repeat,status='PASS',elapsed_seconds=round(time.monotonic()-started,3),**values)
|
|
results.append(row);on_result(results)
|
|
print(f'PASS: {name}, repeat {repeat}/{repeats}',flush=True)
|
|
runner.service('stop','reconciler')
|
|
for repeat in range(1,repeats+1):
|
|
started=time.monotonic();runner.service('stop','worker');runner.service('start','dispatcher')
|
|
ctx=runner.call('create');queued=runner.call('queued',ctx)
|
|
published=runner.call('published',ctx)
|
|
runner.service('stop','dispatcher');runner.service('recreate','api');runner.service('restart','broker')
|
|
restored=runner.call('published',ctx);assert restored['stream_ids']==published['stream_ids']
|
|
runner.service('start','dispatcher');runner.service('recreate','worker')
|
|
record('accepted_restart_broker_api',repeat,started,queued=queued,broker_preserved=restored,result=ready(ctx),infrastructure=runner.infrastructure())
|
|
for phase in ['before_publish','after_publish']:
|
|
started=time.monotonic();runner.service('stop','worker');runner.service('stop','dispatcher')
|
|
ctx=runner.call('create');runner.call('arm',ctx,phase=phase)
|
|
runner.service('recreate','dispatcher');fired=runner.call('fault-fired',ctx,phase=phase)
|
|
runner.service('recreate','worker');result=ready(ctx)
|
|
record(phase,repeat,started,fault=fired,result=result,infrastructure=runner.infrastructure())
|
|
for phase in ['before_effect','after_original','after_effects','after_commit']:
|
|
started=time.monotonic();runner.service('stop','worker')
|
|
ctx=runner.call('create');runner.call('arm',ctx,phase=phase)
|
|
runner.service('recreate','worker');fired=runner.call('fault-fired',ctx,phase=phase)
|
|
result=ready(ctx)
|
|
if phase=='after_commit':assert result['attempts']==1,'Committed result was processed again'
|
|
else:assert result['attempts']>=2,'Abandoned lease was not recovered'
|
|
record(phase,repeat,started,fault=fired,result=result,infrastructure=runner.infrastructure())
|
|
started=time.monotonic();ctx=runner.call('create');before=ready(ctx)
|
|
copies=runner.call('duplicate',dict(ctx,copies=16))
|
|
acked=runner.call('drained',dict(ctx,stream_ids=copies['stream_ids']))
|
|
after=runner.call('snapshot',ctx);same_effect(before,after)
|
|
rejected=runner.call('invalid-job',ctx)
|
|
assert len([r for r in after['events'] if r['event']=='duplicate_ready'])>=16
|
|
record('duplicates_and_invalid_carrier',repeat,started,duplicates=acked,invalid_carrier=rejected,result=after)
|
|
# Новая job приходит первой, старая после неё: контролируемый reorder.
|
|
started=time.monotonic();runner.service('stop','worker');runner.service('stop','dispatcher')
|
|
old=runner.call('create');new=runner.call('version',old)
|
|
first=runner.call('duplicate',new);second=runner.call('duplicate',old)
|
|
runner.service('recreate','worker')
|
|
runner.call('drained',dict(new,stream_ids=first['stream_ids']+second['stream_ids']))
|
|
result=ready(new);old_audit=runner.call('audit',old)
|
|
assert any(r['event']=='stale_ignored' for r in old_audit['events'])
|
|
assert result['pipeline_version']==2
|
|
record('reorder_new_before_old',repeat,started,result=result,old_job=old_audit)
|
|
runner.service('start','dispatcher')
|
|
# Теперь старая версия уже имела ready; её поздний дубль не меняет v2.
|
|
started=time.monotonic();old=runner.call('create');v1=ready(old)
|
|
new=runner.call('version',old);before=ready(new)
|
|
assert before['variants_sha256']!=v1['variants_sha256'],'Pipeline version did not change actual output'
|
|
copies=runner.call('duplicate',dict(old,copies=3));runner.call('drained',dict(old,stream_ids=copies['stream_ids']))
|
|
after=runner.call('snapshot',new);same_effect(before,after)
|
|
record('stale_after_new_ready',repeat,started,result=after,old_job=runner.call('audit',old))
|
|
started=time.monotonic();runner.service('stop','worker');ctx=runner.call('create')
|
|
runner.call('missing-stage',ctx);runner.service('recreate','worker');runner.call('wait',ctx,state='failed')
|
|
failed=runner.call('failed',ctx);assert failed['error_code']=='staging_missing'
|
|
runner.call('restore-stage',ctx);runner.call('replay',ctx);result=ready(ctx)
|
|
noop=runner.call('replay',ctx);assert noop['status']=='noop'
|
|
record('permanent_failure_replay',repeat,started,failed=failed,ready_replay=noop,result=result)
|
|
# Физический stop S3, а не подмена ответа или бесконечная retry-петля.
|
|
started=time.monotonic();runner.service('stop','worker');ctx=runner.call('create')
|
|
runner.service('stop','s3')
|
|
try:
|
|
time.sleep(4);unready=runner.call('unready',ctx)
|
|
runner.service('recreate','worker');runner.call('wait',ctx,state='failed',timeout=120)
|
|
finally:runner.service('start','s3')
|
|
failed=runner.call('failed',ctx);assert failed['error_code']=='retry_exhausted'
|
|
assert failed['attempts']==5
|
|
runner.call('replay',ctx);result=ready(ctx)
|
|
record('bounded_retry_and_replay',repeat,started,unready=unready,failed=failed,result=result)
|
|
started=time.monotonic();runner.service('stop','worker');ctx=runner.call('create')
|
|
cleaned=runner.call('cleanup',ctx);runner.service('recreate','worker');result=ready(ctx)
|
|
privacy=runner.call('privacy',ctx)
|
|
record('staging_retention_and_privacy',repeat,started,cleanup=cleaned,privacy=privacy,result=result)
|
|
return results
|
|
|
|
|
|
def main():
|
|
p=argparse.ArgumentParser();p.add_argument('--repeats',type=int,default=3);args=p.parse_args()
|
|
if not 1<=args.repeats<=5:p.error('repeats must be 1..5')
|
|
runner=Runner()
|
|
save('pipeline_environment.json',dict(docker=command(['docker','version','--format','{{.Server.Version}}']),
|
|
compose=command(['docker','compose','version','--short']),containers=containers(),client_cpus=os.cpu_count(),
|
|
pipeline=runner.call('config')))
|
|
def persist(rows):save('pipeline_raw.json',dict(status='RUNNING',track=2,repeats=args.repeats,cases=rows))
|
|
persist([]);save('pipeline_summary.json',dict(status='RUNNING',expected_checks=13*args.repeats))
|
|
try:
|
|
try:rows=run_suite(runner,args.repeats,persist)
|
|
finally:runner.finish()
|
|
except Exception as exc:
|
|
save('pipeline_summary.json',dict(status='FAILED',reason=type(exc).__name__))
|
|
raise
|
|
save('pipeline_raw.json',dict(status='PASS',track=2,repeats=args.repeats,cases=rows))
|
|
summary=[]
|
|
for case in dict.fromkeys(r['case'] for r in rows):
|
|
subset=[r for r in rows if r['case']==case]
|
|
summary.append(dict(case=case,repetitions=len(subset),status='PASS',
|
|
median_seconds=statistics.median(r['elapsed_seconds'] for r in subset)))
|
|
save('pipeline_summary.json',dict(status='PASS',track=2,cases=summary,checks=len(rows),
|
|
invariants=['accepted work preserved','one ready commit per job','exactly three named variants',
|
|
'original checksum preserved','stale version cannot replace current metadata/bytes',
|
|
'bounded retries and explicit replay','staging cleanup protects active jobs']))
|
|
print('Experiment completed; results saved in evidence/',flush=True)
|
|
|
|
|
|
if __name__=='__main__':main()
|