included rabbitmq as broker
This commit is contained in:
@@ -0,0 +1,210 @@
|
||||
"""Fault/recovery lab5 на явно выбранном Compose-стенде; только синтетические данные."""
|
||||
|
||||
import argparse
|
||||
from datetime import datetime
|
||||
import hashlib
|
||||
from io import BytesIO
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from scripts.check_outbox_recovery import COMPOSE, ROOT, compose, require_compose
|
||||
from scripts.deployment import ComposeProject
|
||||
from tests.client import Client, check
|
||||
|
||||
PROBE = '''
|
||||
import asyncio, hashlib, json
|
||||
from uuid import UUID
|
||||
from sqlalchemy import select
|
||||
from app.core.config import Settings
|
||||
from app.core.database import create_db_engine, create_session_factory
|
||||
from app.models.job import AttachmentJob
|
||||
from app.models.attachment import Attachment
|
||||
from app.storage.s3 import create_s3_storage
|
||||
async def main():
|
||||
settings=Settings(); engine=create_db_engine(settings); storage=create_s3_storage(settings.s3)
|
||||
try:
|
||||
async with create_session_factory(engine)() as session:
|
||||
job, att=(await session.execute(select(AttachmentJob,Attachment).join(Attachment,Attachment.id==AttachmentJob.attachment_id).where(Attachment.id==UUID(ATTACHMENT_ID)))).one()
|
||||
result=dict(job_id=str(job.id),status=att.status,attempt=job.attempt,published_attempt=job.published_attempt,limit=job.attempt_limit,updated_at=att.updated_at,next_attempt_at=job.next_attempt_at,last_error_code=job.last_error_code,permanent=job.last_error_permanent)
|
||||
objects=[]
|
||||
for prefix in ('originals/'+UUID(ATTACHMENT_ID).hex+'/', 'variants/'+UUID(ATTACHMENT_ID).hex+'/'):
|
||||
entries=await asyncio.to_thread(lambda: list(storage._client.list_objects(storage.settings.bucket,prefix=prefix,recursive=True)))
|
||||
for obj in entries:
|
||||
stream=await storage.open_object(obj.object_name); checksum=hashlib.sha256()
|
||||
try:
|
||||
async for chunk in stream: checksum.update(chunk)
|
||||
finally: await stream.aclose()
|
||||
objects.append(dict(name=obj.object_name.split('/')[-1],size=obj.size,sha256=checksum.hexdigest(),last_modified=obj.last_modified))
|
||||
result['objects']=sorted(objects,key=lambda x:x['name']);print(json.dumps(result,default=str))
|
||||
finally: await storage.aclose();await engine.dispose()
|
||||
asyncio.run(main())
|
||||
'''
|
||||
|
||||
|
||||
def probe(attachment_id):
|
||||
code = 'ATTACHMENT_ID = ' + repr(str(UUID(attachment_id))) + '\n' + PROBE
|
||||
result = subprocess.run(COMPOSE + ['exec', '-T', 'api', 'python', '-'], input=code, text=True, capture_output=True, cwd=ROOT, timeout=30)
|
||||
check(result.returncode == 0, 'Probe failed; diagnostic output hidden')
|
||||
return json.loads(result.stdout)
|
||||
|
||||
|
||||
def admin(*arguments, expected=0):
|
||||
result = compose('run', '--rm', '--no-deps', 'worker', 'python', '-m', 'app.core.job_admin', *arguments)
|
||||
check(result.returncode == expected, 'Unexpected operator command exit status')
|
||||
return json.loads(result.stdout)
|
||||
|
||||
|
||||
def verify_result(client, actor, attachment, original):
|
||||
ready = client.wait_ready(actor, attachment, wait_seconds=180)
|
||||
check(ready['id'] == attachment['id'], 'Attachment identity changed')
|
||||
path = '/api/v1/attachments/' + ready['id'] + '/content'
|
||||
check(client.call('GET', path, actor=actor)['bytes'] == original, 'Original changed')
|
||||
check({v['name'] for v in ready['variants']} == {'thumbnail','crop','optimized'}, 'Incomplete pipeline')
|
||||
for variant in ready['variants']:
|
||||
data = client.call('GET', path+'?variant='+variant['name'], actor=actor)['bytes']
|
||||
check(hashlib.sha256(data).hexdigest() == variant['sha256'], 'Checksum mismatch')
|
||||
with Image.open(BytesIO(data)) as image:
|
||||
check(image.size == (variant['width'],variant['height']), 'Metadata dimensions mismatch')
|
||||
check(image.format == ('WEBP' if variant['name']=='optimized' else 'JPEG'), 'Wrong format')
|
||||
check(not image.getexif() and not image.info.get('icc_profile') and not image.info.get('xmp'), 'Metadata leaked')
|
||||
if variant['name']=='crop':check(image.size==(128,128),'Wrong crop size')
|
||||
snapshot=probe(ready['id'])
|
||||
check(len(snapshot['objects']) == 4, 'Expected exactly one original and three variants')
|
||||
return snapshot
|
||||
|
||||
|
||||
def crash(client, actor, chat, mode):
|
||||
name='lab5-worker-fault-'+uuid4().hex
|
||||
try:
|
||||
require_compose('stop','worker')
|
||||
attachment,original=client.attachment(actor,chat)
|
||||
result=compose('run','--rm','--no-deps','--name',name,'worker','python','-m','app.core.job_worker','--'+mode,attachment['id'])
|
||||
phase='after_object:optimized' if mode=='crash-after-object' else 'before_ack'
|
||||
marker=json.dumps({'fault':phase,'attachment_id':attachment['id']})
|
||||
check(result.returncode==137 and marker in result.stdout,'Targeted crash not observed')
|
||||
before=probe(attachment['id'])
|
||||
check(before['status']==('processing' if mode=='crash-after-object' else 'ready'),'Unexpected crash state')
|
||||
check(len(before['objects'])==4,'Crash must occur after all four PUTs')
|
||||
finally:
|
||||
subprocess.run(['docker','rm','-f',name],capture_output=True,timeout=30)
|
||||
require_compose('start','worker')
|
||||
after=verify_result(client,actor,attachment,original)
|
||||
if mode=='crash-before-ack':
|
||||
# Wait for RabbitMQ to deliver and acknowledge the ready duplicate.
|
||||
wait_queue_empty()
|
||||
after=probe(attachment['id'])
|
||||
check(after==before,'Ready duplicate rewrote result')
|
||||
else:
|
||||
check(after['attempt']==before['attempt']+1,'Lease did not recover the interrupted attempt')
|
||||
check([(x['name'],x['sha256']) for x in before['objects']]==[(x['name'],x['sha256']) for x in after['objects']],'Replay changed object content')
|
||||
return dict(scenario=mode,attachment_id=attachment['id'],exit_code=137,before=before,after=after)
|
||||
|
||||
|
||||
def retry_replay(client,actor,chat,permanent):
|
||||
name='lab5-retry-fault-'+uuid4().hex
|
||||
samples=[]
|
||||
configuration=ComposeProject().config['services']['worker']['environment']
|
||||
retry_base=float(configuration['JOB_RETRY_SECONDS'])
|
||||
retry_max=float(configuration['JOB_RETRY_MAX_SECONDS'])
|
||||
try:
|
||||
require_compose('stop','worker')
|
||||
attachment,original=client.attachment(actor,chat)
|
||||
mode='fail-permanent' if permanent else 'fail-transient'
|
||||
require_compose('run','-d','--no-deps','--name',name,'worker','python','-m','app.core.job_worker','--'+mode,attachment['id'])
|
||||
deadline=time.monotonic()+180
|
||||
while time.monotonic()<deadline:
|
||||
sample=probe(attachment['id'])
|
||||
if sample['attempt'] and sample['status'] in {'queued', 'failed'} and (not samples or samples[-1]['attempt']!=sample['attempt']):samples.append(sample)
|
||||
if sample['status']=='failed':break
|
||||
time.sleep(.5)
|
||||
check(sample['status']=='failed','Retry never reached failed')
|
||||
check(sample['attempt']==(1 if permanent else sample['limit']),'Incorrect attempt budget')
|
||||
check(sample['permanent']==permanent,'Incorrect error classification')
|
||||
if not permanent:
|
||||
check(len(samples)==sample['limit'],'Missed retry samples')
|
||||
for entry in samples:
|
||||
delay=(datetime.fromisoformat(entry['next_attempt_at'])-datetime.fromisoformat(entry['updated_at'])).total_seconds()
|
||||
ceiling=min(retry_max,retry_base*2**(entry['attempt']-1))
|
||||
check(ceiling/2<=delay<=ceiling,'Retry delay outside configured jitter bounds')
|
||||
client.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=actor,expected=409)
|
||||
job_id=sample['job_id'];request_id=str(uuid4())
|
||||
listing=admin('list-failed')
|
||||
check(any(row['job_id']==job_id for row in listing),'Job missing from failed registry')
|
||||
subprocess.run(['docker','rm','-f',name],capture_output=True,timeout=30,check=True)
|
||||
first=admin('replay',job_id,'--request-id',request_id,'--actor','verification','--reason','dependency_restored')
|
||||
check(first['status']=='queued','Replay did not queue job')
|
||||
finally:
|
||||
subprocess.run(['docker','rm','-f',name],capture_output=True,timeout=30)
|
||||
require_compose('start','worker')
|
||||
after=verify_result(client,actor,attachment,original)
|
||||
repeat=admin('replay',job_id,'--request-id',request_id,'--actor','verification','--reason','dependency_restored')
|
||||
check(repeat['already_applied'],'Replay request id not idempotent')
|
||||
refused=admin('replay',job_id,'--request-id',str(uuid4()),'--actor','verification','--reason','manual_review',expected=2)
|
||||
check(refused['error_code']=='job_not_failed','Successful job was replayed')
|
||||
check(probe(attachment['id'])==after,'Operator replay altered ready job')
|
||||
check(len(admin('history',job_id))==1,'Incorrect audit count')
|
||||
return dict(scenario=mode,attachment_id=attachment['id'],samples=samples,after=after,replay_idempotent=True)
|
||||
|
||||
|
||||
def queue_state():
|
||||
output=require_compose('exec','-T','broker','rabbitmqctl','list_queues','--formatter=json','name','messages_ready','messages_unacknowledged','consumers','durable')
|
||||
return next(row for row in json.loads(output) if row['name']=='attachment-jobs')
|
||||
|
||||
|
||||
def wait_queue_empty():
|
||||
deadline=time.monotonic()+20
|
||||
while time.monotonic()<deadline:
|
||||
state=queue_state()
|
||||
if state['messages_ready']==0 and state['messages_unacknowledged']==0:
|
||||
return state
|
||||
time.sleep(.5)
|
||||
check(False,'RabbitMQ did not acknowledge all completed deliveries')
|
||||
|
||||
|
||||
def persistence(client,actor,chat):
|
||||
try:
|
||||
require_compose('stop','worker')
|
||||
attachment,original=client.attachment(actor,chat)
|
||||
client.message(actor,chat,'',attachments=[attachment['id']])
|
||||
client.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=actor,expected=409)
|
||||
time.sleep(2)
|
||||
queued_before=queue_state()
|
||||
check(queued_before['messages_ready']>=1 and queued_before['durable'], 'No durable queued message before restart')
|
||||
result=subprocess.run([str(ROOT/'scripts/restart.sh')],cwd=ROOT,capture_output=True,timeout=180)
|
||||
check(result.returncode==0,'Restart hook failed')
|
||||
check('worker' not in require_compose('ps','--status','running','--services').splitlines(),'Restart started deliberately stopped worker')
|
||||
queued_after=queue_state()
|
||||
check(queued_after['messages_ready']>=1 and queued_after['durable'], 'Broker lost queued delivery across restart')
|
||||
client.message(actor,chat,'session and chat survived broker/API restart')
|
||||
finally:require_compose('start','worker')
|
||||
after=verify_result(client,actor,attachment,original)
|
||||
return dict(scenario='worker_stopped_restart_all',attachment_id=attachment['id'],queue_before=queued_before,queue_after=queued_after,after=after)
|
||||
|
||||
|
||||
def main():
|
||||
parser=argparse.ArgumentParser()
|
||||
parser.add_argument('--base-url',required=True)
|
||||
parser.add_argument('--scenario',choices=['all','persistence','crash-after-object','crash-before-ack','fail-transient','fail-permanent'],default='all')
|
||||
parser.add_argument('--output',default='evidence/lab5-jobs.json')
|
||||
args=parser.parse_args()
|
||||
client=Client(args.base_url,timeout=40)
|
||||
aliases={r.instance_id for r in ComposeProject().replicas() if r.running}
|
||||
check(client.call('GET','/health/live',schema='Health')['headers'].get('X-Instance-Id') in aliases,'Wrong Compose project for BASE_URL')
|
||||
actor,_,_,chat=client.bootstrap()
|
||||
results=[]
|
||||
operations={'persistence':persistence,'crash-after-object':lambda *x:crash(*x,'crash-after-object'),'crash-before-ack':lambda *x:crash(*x,'crash-before-ack'),'fail-transient':lambda *x:retry_replay(*x,False),'fail-permanent':lambda *x:retry_replay(*x,True)}
|
||||
selected=operations.values() if args.scenario=='all' else [operations[args.scenario]]
|
||||
for operation in selected:
|
||||
result=operation(client,actor,chat);results.append(result)
|
||||
Path(args.output).parent.mkdir(parents=True,exist_ok=True)
|
||||
Path(args.output).write_text(json.dumps(dict(status='in_progress',results=results),indent=2)+'\n')
|
||||
print('PASS: '+result['scenario'],flush=True)
|
||||
Path(args.output).write_text(json.dumps(dict(status='passed',results=results),indent=2)+'\n')
|
||||
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Сценарий COMMIT → SIGKILL до publish → ready без повторного upload.
|
||||
|
||||
Требуется уже работающий отдельный стенд. Скрипт не выполняется при запуске API.
|
||||
Публичные tests не изменяет, secrets/tokens не печатает и evidence сам не сохраняет.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
from uuid import uuid4
|
||||
|
||||
from scripts.deployment import ComposeProject
|
||||
from tests.client import Client, check
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
COMPOSE = ["docker", "compose", "--env-file", ".env"]
|
||||
|
||||
|
||||
def compose(*arguments: str, timeout: int = 90) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(COMPOSE + list(arguments), cwd=ROOT, text=True, capture_output=True, timeout=timeout)
|
||||
|
||||
|
||||
def require_compose(*arguments: str) -> str:
|
||||
result = compose(*arguments)
|
||||
check(result.returncode == 0, "Compose command failed; output hidden")
|
||||
return result.stdout
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Отказ publisher после COMMIT перед RabbitMQ publish")
|
||||
parser.add_argument("--base-url", required=True)
|
||||
parser.add_argument("--ca-file")
|
||||
args = parser.parse_args()
|
||||
running = set(require_compose("ps", "--status", "running", "--services").splitlines())
|
||||
check({"api", "postgres", "minio", "broker", "publisher", "worker"} <= running, "Start the lab5 stack before this scenario")
|
||||
client = Client(args.base_url, args.ca_file, timeout=40)
|
||||
project = ComposeProject()
|
||||
aliases = {replica.instance_id for replica in project.replicas() if replica.running}
|
||||
live = client.call("GET", "/health/live", schema="Health")
|
||||
check(live["headers"].get("X-Instance-Id") in aliases, "BASE_URL does not belong to the selected Compose project")
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
attachment = None
|
||||
fault_container = "lab5-outbox-fault-" + uuid4().hex
|
||||
try:
|
||||
require_compose("stop", "publisher")
|
||||
attachment, original = client.attachment(alice, chat)
|
||||
message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"]
|
||||
client.call("GET", f"/api/v1/attachments/{attachment['id']}/content", expected=409, actor=alice)
|
||||
killed = compose(
|
||||
"run", "--rm", "--no-deps", "--name", fault_container, "publisher", "python", "-m",
|
||||
"app.core.outbox_publisher", "--crash-before-publish", attachment["id"],
|
||||
)
|
||||
marker = json.dumps({"fault": "before_publish", "attachment_id": attachment["id"]})
|
||||
check(killed.returncode == 137 and marker in killed.stdout, "Expected targeted SIGKILL before publish was not observed")
|
||||
pending = client.call("GET", f"/api/v1/attachments/{attachment['id']}", actor=alice, schema="Attachment")["json"]
|
||||
check(pending["status"] == "queued", "Crash must leave accepted job queued")
|
||||
finally:
|
||||
# Если docker compose run оборвался по timeout, одноразовый контейнер тоже удаляется.
|
||||
try:
|
||||
subprocess.run(["docker", "rm", "-f", fault_container], capture_output=True, timeout=30)
|
||||
finally:
|
||||
require_compose("start", "publisher")
|
||||
ready = client.wait_ready(alice, attachment, wait_seconds=60)
|
||||
check(ready["id"] == attachment["id"], "Recovered a different attachment")
|
||||
check({variant["name"] for variant in ready["variants"]} == {"thumbnail", "crop", "optimized"}, "Missing required variants")
|
||||
path = f"/api/v1/attachments/{attachment['id']}/content"
|
||||
check(client.call("GET", path, actor=alice)["bytes"] == original, "Original changed after recovery")
|
||||
for variant in ready["variants"]:
|
||||
downloaded = client.call("GET", path + "?variant=" + variant["name"], actor=alice)["bytes"]
|
||||
check(len(downloaded) == variant["size_bytes"] and hashlib.sha256(downloaded).hexdigest() == variant["sha256"], "Recovered variant checksum mismatch")
|
||||
print(json.dumps({
|
||||
"scenario": "commit_before_publish", "attachment_id": ready["id"],
|
||||
"message_id": message["id"], "publisher_exit_code": killed.returncode,
|
||||
"status_after_crash": pending["status"], "status_after_restart": ready["status"],
|
||||
"variants": [variant["name"] for variant in ready["variants"]],
|
||||
"reuploaded": False,
|
||||
}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -14,7 +14,7 @@ def main() -> None:
|
||||
destination.chmod(0o600)
|
||||
directory = root / "secrets"
|
||||
directory.mkdir(mode=0o700, exist_ok=True)
|
||||
for name in ("db_password", "redis_password", "minio_root_user", "minio_root_password", "s3_access_key_id", "s3_secret_access_key"):
|
||||
for name in ("db_password", "redis_password", "broker_password", "minio_root_user", "minio_root_password", "s3_access_key_id", "s3_secret_access_key"):
|
||||
path = directory / name
|
||||
try:
|
||||
descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o444)
|
||||
|
||||
+9
-4
@@ -22,11 +22,16 @@ wait_healthy() {
|
||||
done
|
||||
}
|
||||
|
||||
compose stop cleanup
|
||||
compose restart postgres redis minio
|
||||
wait_healthy postgres redis minio
|
||||
worker_running=$(compose ps --status running --quiet worker)
|
||||
compose stop worker publisher cleanup
|
||||
compose restart postgres redis minio broker
|
||||
wait_healthy postgres redis minio broker
|
||||
compose restart api
|
||||
wait_healthy api
|
||||
compose restart nginx
|
||||
wait_healthy nginx
|
||||
compose start cleanup
|
||||
compose start publisher cleanup
|
||||
# Сценарий с заранее остановленным worker сохраняет его остановленным.
|
||||
if [ -n "$worker_running" ]; then
|
||||
compose start worker
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user