211 lines
12 KiB
Python
211 lines
12 KiB
Python
"""Fault/recovery lab5 на явно выбранном Compose-стенде; только синтетические данные."""
|
|
|
|
import argparse
|
|
from datetime import datetime
|
|
import hashlib
|
|
from io import BytesIO
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import time
|
|
from uuid import UUID, uuid4
|
|
|
|
from PIL import Image
|
|
|
|
from scripts.check_outbox_recovery import COMPOSE, ROOT, compose, require_compose
|
|
from scripts.deployment import ComposeProject
|
|
from tests.client import Client, check
|
|
|
|
PROBE = '''
|
|
import asyncio, hashlib, json
|
|
from uuid import UUID
|
|
from sqlalchemy import select
|
|
from app.core.config import Settings
|
|
from app.core.database import create_db_engine, create_session_factory
|
|
from app.models.job import AttachmentJob
|
|
from app.models.attachment import Attachment
|
|
from app.storage.s3 import create_s3_storage
|
|
async def main():
|
|
settings=Settings(); engine=create_db_engine(settings); storage=create_s3_storage(settings.s3)
|
|
try:
|
|
async with create_session_factory(engine)() as session:
|
|
job, att=(await session.execute(select(AttachmentJob,Attachment).join(Attachment,Attachment.id==AttachmentJob.attachment_id).where(Attachment.id==UUID(ATTACHMENT_ID)))).one()
|
|
result=dict(job_id=str(job.id),status=att.status,attempt=job.attempt,published_attempt=job.published_attempt,limit=job.attempt_limit,updated_at=att.updated_at,next_attempt_at=job.next_attempt_at,last_error_code=job.last_error_code,permanent=job.last_error_permanent)
|
|
objects=[]
|
|
for prefix in ('originals/'+UUID(ATTACHMENT_ID).hex+'/', 'variants/'+UUID(ATTACHMENT_ID).hex+'/'):
|
|
entries=await asyncio.to_thread(lambda: list(storage._client.list_objects(storage.settings.bucket,prefix=prefix,recursive=True)))
|
|
for obj in entries:
|
|
stream=await storage.open_object(obj.object_name); checksum=hashlib.sha256()
|
|
try:
|
|
async for chunk in stream: checksum.update(chunk)
|
|
finally: await stream.aclose()
|
|
objects.append(dict(name=obj.object_name.split('/')[-1],size=obj.size,sha256=checksum.hexdigest(),last_modified=obj.last_modified))
|
|
result['objects']=sorted(objects,key=lambda x:x['name']);print(json.dumps(result,default=str))
|
|
finally: await storage.aclose();await engine.dispose()
|
|
asyncio.run(main())
|
|
'''
|
|
|
|
|
|
def probe(attachment_id):
|
|
code = 'ATTACHMENT_ID = ' + repr(str(UUID(attachment_id))) + '\n' + PROBE
|
|
result = subprocess.run(COMPOSE + ['exec', '-T', 'api', 'python', '-'], input=code, text=True, capture_output=True, cwd=ROOT, timeout=30)
|
|
check(result.returncode == 0, 'Probe failed; diagnostic output hidden')
|
|
return json.loads(result.stdout)
|
|
|
|
|
|
def admin(*arguments, expected=0):
|
|
result = compose('run', '--rm', '--no-deps', 'worker', 'python', '-m', 'app.core.job_admin', *arguments)
|
|
check(result.returncode == expected, 'Unexpected operator command exit status')
|
|
return json.loads(result.stdout)
|
|
|
|
|
|
def verify_result(client, actor, attachment, original):
|
|
ready = client.wait_ready(actor, attachment, wait_seconds=180)
|
|
check(ready['id'] == attachment['id'], 'Attachment identity changed')
|
|
path = '/api/v1/attachments/' + ready['id'] + '/content'
|
|
check(client.call('GET', path, actor=actor)['bytes'] == original, 'Original changed')
|
|
check({v['name'] for v in ready['variants']} == {'thumbnail','crop','optimized'}, 'Incomplete pipeline')
|
|
for variant in ready['variants']:
|
|
data = client.call('GET', path+'?variant='+variant['name'], actor=actor)['bytes']
|
|
check(hashlib.sha256(data).hexdigest() == variant['sha256'], 'Checksum mismatch')
|
|
with Image.open(BytesIO(data)) as image:
|
|
check(image.size == (variant['width'],variant['height']), 'Metadata dimensions mismatch')
|
|
check(image.format == ('WEBP' if variant['name']=='optimized' else 'JPEG'), 'Wrong format')
|
|
check(not image.getexif() and not image.info.get('icc_profile') and not image.info.get('xmp'), 'Metadata leaked')
|
|
if variant['name']=='crop':check(image.size==(128,128),'Wrong crop size')
|
|
snapshot=probe(ready['id'])
|
|
check(len(snapshot['objects']) == 4, 'Expected exactly one original and three variants')
|
|
return snapshot
|
|
|
|
|
|
def crash(client, actor, chat, mode):
|
|
name='lab5-worker-fault-'+uuid4().hex
|
|
try:
|
|
require_compose('stop','worker')
|
|
attachment,original=client.attachment(actor,chat)
|
|
result=compose('run','--rm','--no-deps','--name',name,'worker','python','-m','app.core.job_worker','--'+mode,attachment['id'])
|
|
phase='after_object:optimized' if mode=='crash-after-object' else 'before_ack'
|
|
marker=json.dumps({'fault':phase,'attachment_id':attachment['id']})
|
|
check(result.returncode==137 and marker in result.stdout,'Targeted crash not observed')
|
|
before=probe(attachment['id'])
|
|
check(before['status']==('processing' if mode=='crash-after-object' else 'ready'),'Unexpected crash state')
|
|
check(len(before['objects'])==4,'Crash must occur after all four PUTs')
|
|
finally:
|
|
subprocess.run(['docker','rm','-f',name],capture_output=True,timeout=30)
|
|
require_compose('start','worker')
|
|
after=verify_result(client,actor,attachment,original)
|
|
if mode=='crash-before-ack':
|
|
# Wait for RabbitMQ to deliver and acknowledge the ready duplicate.
|
|
wait_queue_empty()
|
|
after=probe(attachment['id'])
|
|
check(after==before,'Ready duplicate rewrote result')
|
|
else:
|
|
check(after['attempt']==before['attempt']+1,'Lease did not recover the interrupted attempt')
|
|
check([(x['name'],x['sha256']) for x in before['objects']]==[(x['name'],x['sha256']) for x in after['objects']],'Replay changed object content')
|
|
return dict(scenario=mode,attachment_id=attachment['id'],exit_code=137,before=before,after=after)
|
|
|
|
|
|
def retry_replay(client,actor,chat,permanent):
|
|
name='lab5-retry-fault-'+uuid4().hex
|
|
samples=[]
|
|
configuration=ComposeProject().config['services']['worker']['environment']
|
|
retry_base=float(configuration['JOB_RETRY_SECONDS'])
|
|
retry_max=float(configuration['JOB_RETRY_MAX_SECONDS'])
|
|
try:
|
|
require_compose('stop','worker')
|
|
attachment,original=client.attachment(actor,chat)
|
|
mode='fail-permanent' if permanent else 'fail-transient'
|
|
require_compose('run','-d','--no-deps','--name',name,'worker','python','-m','app.core.job_worker','--'+mode,attachment['id'])
|
|
deadline=time.monotonic()+180
|
|
while time.monotonic()<deadline:
|
|
sample=probe(attachment['id'])
|
|
if sample['attempt'] and sample['status'] in {'queued', 'failed'} and (not samples or samples[-1]['attempt']!=sample['attempt']):samples.append(sample)
|
|
if sample['status']=='failed':break
|
|
time.sleep(.5)
|
|
check(sample['status']=='failed','Retry never reached failed')
|
|
check(sample['attempt']==(1 if permanent else sample['limit']),'Incorrect attempt budget')
|
|
check(sample['permanent']==permanent,'Incorrect error classification')
|
|
if not permanent:
|
|
check(len(samples)==sample['limit'],'Missed retry samples')
|
|
for entry in samples:
|
|
delay=(datetime.fromisoformat(entry['next_attempt_at'])-datetime.fromisoformat(entry['updated_at'])).total_seconds()
|
|
ceiling=min(retry_max,retry_base*2**(entry['attempt']-1))
|
|
check(ceiling/2<=delay<=ceiling,'Retry delay outside configured jitter bounds')
|
|
client.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=actor,expected=409)
|
|
job_id=sample['job_id'];request_id=str(uuid4())
|
|
listing=admin('list-failed')
|
|
check(any(row['job_id']==job_id for row in listing),'Job missing from failed registry')
|
|
subprocess.run(['docker','rm','-f',name],capture_output=True,timeout=30,check=True)
|
|
first=admin('replay',job_id,'--request-id',request_id,'--actor','verification','--reason','dependency_restored')
|
|
check(first['status']=='queued','Replay did not queue job')
|
|
finally:
|
|
subprocess.run(['docker','rm','-f',name],capture_output=True,timeout=30)
|
|
require_compose('start','worker')
|
|
after=verify_result(client,actor,attachment,original)
|
|
repeat=admin('replay',job_id,'--request-id',request_id,'--actor','verification','--reason','dependency_restored')
|
|
check(repeat['already_applied'],'Replay request id not idempotent')
|
|
refused=admin('replay',job_id,'--request-id',str(uuid4()),'--actor','verification','--reason','manual_review',expected=2)
|
|
check(refused['error_code']=='job_not_failed','Successful job was replayed')
|
|
check(probe(attachment['id'])==after,'Operator replay altered ready job')
|
|
check(len(admin('history',job_id))==1,'Incorrect audit count')
|
|
return dict(scenario=mode,attachment_id=attachment['id'],samples=samples,after=after,replay_idempotent=True)
|
|
|
|
|
|
def queue_state():
|
|
output=require_compose('exec','-T','broker','rabbitmqctl','list_queues','--formatter=json','name','messages_ready','messages_unacknowledged','consumers','durable')
|
|
return next(row for row in json.loads(output) if row['name']=='attachment-jobs')
|
|
|
|
|
|
def wait_queue_empty():
|
|
deadline=time.monotonic()+20
|
|
while time.monotonic()<deadline:
|
|
state=queue_state()
|
|
if state['messages_ready']==0 and state['messages_unacknowledged']==0:
|
|
return state
|
|
time.sleep(.5)
|
|
check(False,'RabbitMQ did not acknowledge all completed deliveries')
|
|
|
|
|
|
def persistence(client,actor,chat):
|
|
try:
|
|
require_compose('stop','worker')
|
|
attachment,original=client.attachment(actor,chat)
|
|
client.message(actor,chat,'',attachments=[attachment['id']])
|
|
client.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=actor,expected=409)
|
|
time.sleep(2)
|
|
queued_before=queue_state()
|
|
check(queued_before['messages_ready']>=1 and queued_before['durable'], 'No durable queued message before restart')
|
|
result=subprocess.run([str(ROOT/'scripts/restart.sh')],cwd=ROOT,capture_output=True,timeout=180)
|
|
check(result.returncode==0,'Restart hook failed')
|
|
check('worker' not in require_compose('ps','--status','running','--services').splitlines(),'Restart started deliberately stopped worker')
|
|
queued_after=queue_state()
|
|
check(queued_after['messages_ready']>=1 and queued_after['durable'], 'Broker lost queued delivery across restart')
|
|
client.message(actor,chat,'session and chat survived broker/API restart')
|
|
finally:require_compose('start','worker')
|
|
after=verify_result(client,actor,attachment,original)
|
|
return dict(scenario='worker_stopped_restart_all',attachment_id=attachment['id'],queue_before=queued_before,queue_after=queued_after,after=after)
|
|
|
|
|
|
def main():
|
|
parser=argparse.ArgumentParser()
|
|
parser.add_argument('--base-url',required=True)
|
|
parser.add_argument('--scenario',choices=['all','persistence','crash-after-object','crash-before-ack','fail-transient','fail-permanent'],default='all')
|
|
parser.add_argument('--output',default='evidence/lab5-jobs.json')
|
|
args=parser.parse_args()
|
|
client=Client(args.base_url,timeout=40)
|
|
aliases={r.instance_id for r in ComposeProject().replicas() if r.running}
|
|
check(client.call('GET','/health/live',schema='Health')['headers'].get('X-Instance-Id') in aliases,'Wrong Compose project for BASE_URL')
|
|
actor,_,_,chat=client.bootstrap()
|
|
results=[]
|
|
operations={'persistence':persistence,'crash-after-object':lambda *x:crash(*x,'crash-after-object'),'crash-before-ack':lambda *x:crash(*x,'crash-before-ack'),'fail-transient':lambda *x:retry_replay(*x,False),'fail-permanent':lambda *x:retry_replay(*x,True)}
|
|
selected=operations.values() if args.scenario=='all' else [operations[args.scenario]]
|
|
for operation in selected:
|
|
result=operation(client,actor,chat);results.append(result)
|
|
Path(args.output).parent.mkdir(parents=True,exist_ok=True)
|
|
Path(args.output).write_text(json.dumps(dict(status='in_progress',results=results),indent=2)+'\n')
|
|
print('PASS: '+result['scenario'],flush=True)
|
|
Path(args.output).write_text(json.dumps(dict(status='passed',results=results),indent=2)+'\n')
|
|
|
|
|
|
if __name__=='__main__':main()
|