included rabbitmq as broker

This commit is contained in:
2026-11-10 07:01:17 +03:00
parent 0a70358829
commit 2b9fb3f375
49 changed files with 3072 additions and 287 deletions
+210
View File
@@ -0,0 +1,210 @@
"""Fault/recovery lab5 на явно выбранном Compose-стенде; только синтетические данные."""
import argparse
from datetime import datetime
import hashlib
from io import BytesIO
import json
from pathlib import Path
import subprocess
import time
from uuid import UUID, uuid4
from PIL import Image
from scripts.check_outbox_recovery import COMPOSE, ROOT, compose, require_compose
from scripts.deployment import ComposeProject
from tests.client import Client, check
PROBE = '''
import asyncio, hashlib, json
from uuid import UUID
from sqlalchemy import select
from app.core.config import Settings
from app.core.database import create_db_engine, create_session_factory
from app.models.job import AttachmentJob
from app.models.attachment import Attachment
from app.storage.s3 import create_s3_storage
async def main():
settings=Settings(); engine=create_db_engine(settings); storage=create_s3_storage(settings.s3)
try:
async with create_session_factory(engine)() as session:
job, att=(await session.execute(select(AttachmentJob,Attachment).join(Attachment,Attachment.id==AttachmentJob.attachment_id).where(Attachment.id==UUID(ATTACHMENT_ID)))).one()
result=dict(job_id=str(job.id),status=att.status,attempt=job.attempt,published_attempt=job.published_attempt,limit=job.attempt_limit,updated_at=att.updated_at,next_attempt_at=job.next_attempt_at,last_error_code=job.last_error_code,permanent=job.last_error_permanent)
objects=[]
for prefix in ('originals/'+UUID(ATTACHMENT_ID).hex+'/', 'variants/'+UUID(ATTACHMENT_ID).hex+'/'):
entries=await asyncio.to_thread(lambda: list(storage._client.list_objects(storage.settings.bucket,prefix=prefix,recursive=True)))
for obj in entries:
stream=await storage.open_object(obj.object_name); checksum=hashlib.sha256()
try:
async for chunk in stream: checksum.update(chunk)
finally: await stream.aclose()
objects.append(dict(name=obj.object_name.split('/')[-1],size=obj.size,sha256=checksum.hexdigest(),last_modified=obj.last_modified))
result['objects']=sorted(objects,key=lambda x:x['name']);print(json.dumps(result,default=str))
finally: await storage.aclose();await engine.dispose()
asyncio.run(main())
'''
def probe(attachment_id):
code = 'ATTACHMENT_ID = ' + repr(str(UUID(attachment_id))) + '\n' + PROBE
result = subprocess.run(COMPOSE + ['exec', '-T', 'api', 'python', '-'], input=code, text=True, capture_output=True, cwd=ROOT, timeout=30)
check(result.returncode == 0, 'Probe failed; diagnostic output hidden')
return json.loads(result.stdout)
def admin(*arguments, expected=0):
result = compose('run', '--rm', '--no-deps', 'worker', 'python', '-m', 'app.core.job_admin', *arguments)
check(result.returncode == expected, 'Unexpected operator command exit status')
return json.loads(result.stdout)
def verify_result(client, actor, attachment, original):
ready = client.wait_ready(actor, attachment, wait_seconds=180)
check(ready['id'] == attachment['id'], 'Attachment identity changed')
path = '/api/v1/attachments/' + ready['id'] + '/content'
check(client.call('GET', path, actor=actor)['bytes'] == original, 'Original changed')
check({v['name'] for v in ready['variants']} == {'thumbnail','crop','optimized'}, 'Incomplete pipeline')
for variant in ready['variants']:
data = client.call('GET', path+'?variant='+variant['name'], actor=actor)['bytes']
check(hashlib.sha256(data).hexdigest() == variant['sha256'], 'Checksum mismatch')
with Image.open(BytesIO(data)) as image:
check(image.size == (variant['width'],variant['height']), 'Metadata dimensions mismatch')
check(image.format == ('WEBP' if variant['name']=='optimized' else 'JPEG'), 'Wrong format')
check(not image.getexif() and not image.info.get('icc_profile') and not image.info.get('xmp'), 'Metadata leaked')
if variant['name']=='crop':check(image.size==(128,128),'Wrong crop size')
snapshot=probe(ready['id'])
check(len(snapshot['objects']) == 4, 'Expected exactly one original and three variants')
return snapshot
def crash(client, actor, chat, mode):
name='lab5-worker-fault-'+uuid4().hex
try:
require_compose('stop','worker')
attachment,original=client.attachment(actor,chat)
result=compose('run','--rm','--no-deps','--name',name,'worker','python','-m','app.core.job_worker','--'+mode,attachment['id'])
phase='after_object:optimized' if mode=='crash-after-object' else 'before_ack'
marker=json.dumps({'fault':phase,'attachment_id':attachment['id']})
check(result.returncode==137 and marker in result.stdout,'Targeted crash not observed')
before=probe(attachment['id'])
check(before['status']==('processing' if mode=='crash-after-object' else 'ready'),'Unexpected crash state')
check(len(before['objects'])==4,'Crash must occur after all four PUTs')
finally:
subprocess.run(['docker','rm','-f',name],capture_output=True,timeout=30)
require_compose('start','worker')
after=verify_result(client,actor,attachment,original)
if mode=='crash-before-ack':
# Wait for RabbitMQ to deliver and acknowledge the ready duplicate.
wait_queue_empty()
after=probe(attachment['id'])
check(after==before,'Ready duplicate rewrote result')
else:
check(after['attempt']==before['attempt']+1,'Lease did not recover the interrupted attempt')
check([(x['name'],x['sha256']) for x in before['objects']]==[(x['name'],x['sha256']) for x in after['objects']],'Replay changed object content')
return dict(scenario=mode,attachment_id=attachment['id'],exit_code=137,before=before,after=after)
def retry_replay(client,actor,chat,permanent):
name='lab5-retry-fault-'+uuid4().hex
samples=[]
configuration=ComposeProject().config['services']['worker']['environment']
retry_base=float(configuration['JOB_RETRY_SECONDS'])
retry_max=float(configuration['JOB_RETRY_MAX_SECONDS'])
try:
require_compose('stop','worker')
attachment,original=client.attachment(actor,chat)
mode='fail-permanent' if permanent else 'fail-transient'
require_compose('run','-d','--no-deps','--name',name,'worker','python','-m','app.core.job_worker','--'+mode,attachment['id'])
deadline=time.monotonic()+180
while time.monotonic()<deadline:
sample=probe(attachment['id'])
if sample['attempt'] and sample['status'] in {'queued', 'failed'} and (not samples or samples[-1]['attempt']!=sample['attempt']):samples.append(sample)
if sample['status']=='failed':break
time.sleep(.5)
check(sample['status']=='failed','Retry never reached failed')
check(sample['attempt']==(1 if permanent else sample['limit']),'Incorrect attempt budget')
check(sample['permanent']==permanent,'Incorrect error classification')
if not permanent:
check(len(samples)==sample['limit'],'Missed retry samples')
for entry in samples:
delay=(datetime.fromisoformat(entry['next_attempt_at'])-datetime.fromisoformat(entry['updated_at'])).total_seconds()
ceiling=min(retry_max,retry_base*2**(entry['attempt']-1))
check(ceiling/2<=delay<=ceiling,'Retry delay outside configured jitter bounds')
client.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=actor,expected=409)
job_id=sample['job_id'];request_id=str(uuid4())
listing=admin('list-failed')
check(any(row['job_id']==job_id for row in listing),'Job missing from failed registry')
subprocess.run(['docker','rm','-f',name],capture_output=True,timeout=30,check=True)
first=admin('replay',job_id,'--request-id',request_id,'--actor','verification','--reason','dependency_restored')
check(first['status']=='queued','Replay did not queue job')
finally:
subprocess.run(['docker','rm','-f',name],capture_output=True,timeout=30)
require_compose('start','worker')
after=verify_result(client,actor,attachment,original)
repeat=admin('replay',job_id,'--request-id',request_id,'--actor','verification','--reason','dependency_restored')
check(repeat['already_applied'],'Replay request id not idempotent')
refused=admin('replay',job_id,'--request-id',str(uuid4()),'--actor','verification','--reason','manual_review',expected=2)
check(refused['error_code']=='job_not_failed','Successful job was replayed')
check(probe(attachment['id'])==after,'Operator replay altered ready job')
check(len(admin('history',job_id))==1,'Incorrect audit count')
return dict(scenario=mode,attachment_id=attachment['id'],samples=samples,after=after,replay_idempotent=True)
def queue_state():
output=require_compose('exec','-T','broker','rabbitmqctl','list_queues','--formatter=json','name','messages_ready','messages_unacknowledged','consumers','durable')
return next(row for row in json.loads(output) if row['name']=='attachment-jobs')
def wait_queue_empty():
deadline=time.monotonic()+20
while time.monotonic()<deadline:
state=queue_state()
if state['messages_ready']==0 and state['messages_unacknowledged']==0:
return state
time.sleep(.5)
check(False,'RabbitMQ did not acknowledge all completed deliveries')
def persistence(client,actor,chat):
try:
require_compose('stop','worker')
attachment,original=client.attachment(actor,chat)
client.message(actor,chat,'',attachments=[attachment['id']])
client.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=actor,expected=409)
time.sleep(2)
queued_before=queue_state()
check(queued_before['messages_ready']>=1 and queued_before['durable'], 'No durable queued message before restart')
result=subprocess.run([str(ROOT/'scripts/restart.sh')],cwd=ROOT,capture_output=True,timeout=180)
check(result.returncode==0,'Restart hook failed')
check('worker' not in require_compose('ps','--status','running','--services').splitlines(),'Restart started deliberately stopped worker')
queued_after=queue_state()
check(queued_after['messages_ready']>=1 and queued_after['durable'], 'Broker lost queued delivery across restart')
client.message(actor,chat,'session and chat survived broker/API restart')
finally:require_compose('start','worker')
after=verify_result(client,actor,attachment,original)
return dict(scenario='worker_stopped_restart_all',attachment_id=attachment['id'],queue_before=queued_before,queue_after=queued_after,after=after)
def main():
parser=argparse.ArgumentParser()
parser.add_argument('--base-url',required=True)
parser.add_argument('--scenario',choices=['all','persistence','crash-after-object','crash-before-ack','fail-transient','fail-permanent'],default='all')
parser.add_argument('--output',default='evidence/lab5-jobs.json')
args=parser.parse_args()
client=Client(args.base_url,timeout=40)
aliases={r.instance_id for r in ComposeProject().replicas() if r.running}
check(client.call('GET','/health/live',schema='Health')['headers'].get('X-Instance-Id') in aliases,'Wrong Compose project for BASE_URL')
actor,_,_,chat=client.bootstrap()
results=[]
operations={'persistence':persistence,'crash-after-object':lambda *x:crash(*x,'crash-after-object'),'crash-before-ack':lambda *x:crash(*x,'crash-before-ack'),'fail-transient':lambda *x:retry_replay(*x,False),'fail-permanent':lambda *x:retry_replay(*x,True)}
selected=operations.values() if args.scenario=='all' else [operations[args.scenario]]
for operation in selected:
result=operation(client,actor,chat);results.append(result)
Path(args.output).parent.mkdir(parents=True,exist_ok=True)
Path(args.output).write_text(json.dumps(dict(status='in_progress',results=results),indent=2)+'\n')
print('PASS: '+result['scenario'],flush=True)
Path(args.output).write_text(json.dumps(dict(status='passed',results=results),indent=2)+'\n')
if __name__=='__main__':main()
+83
View File
@@ -0,0 +1,83 @@
"""Сценарий COMMIT → SIGKILL до publish → ready без повторного upload.
Требуется уже работающий отдельный стенд. Скрипт не выполняется при запуске API.
Публичные tests не изменяет, secrets/tokens не печатает и evidence сам не сохраняет.
"""
import argparse
import hashlib
import json
from pathlib import Path
import subprocess
from uuid import uuid4
from scripts.deployment import ComposeProject
from tests.client import Client, check
ROOT = Path(__file__).resolve().parents[1]
COMPOSE = ["docker", "compose", "--env-file", ".env"]
def compose(*arguments: str, timeout: int = 90) -> subprocess.CompletedProcess:
return subprocess.run(COMPOSE + list(arguments), cwd=ROOT, text=True, capture_output=True, timeout=timeout)
def require_compose(*arguments: str) -> str:
result = compose(*arguments)
check(result.returncode == 0, "Compose command failed; output hidden")
return result.stdout
def main() -> None:
parser = argparse.ArgumentParser(description="Отказ publisher после COMMIT перед RabbitMQ publish")
parser.add_argument("--base-url", required=True)
parser.add_argument("--ca-file")
args = parser.parse_args()
running = set(require_compose("ps", "--status", "running", "--services").splitlines())
check({"api", "postgres", "minio", "broker", "publisher", "worker"} <= running, "Start the lab5 stack before this scenario")
client = Client(args.base_url, args.ca_file, timeout=40)
project = ComposeProject()
aliases = {replica.instance_id for replica in project.replicas() if replica.running}
live = client.call("GET", "/health/live", schema="Health")
check(live["headers"].get("X-Instance-Id") in aliases, "BASE_URL does not belong to the selected Compose project")
alice, _, _, chat = client.bootstrap()
attachment = None
fault_container = "lab5-outbox-fault-" + uuid4().hex
try:
require_compose("stop", "publisher")
attachment, original = client.attachment(alice, chat)
message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"]
client.call("GET", f"/api/v1/attachments/{attachment['id']}/content", expected=409, actor=alice)
killed = compose(
"run", "--rm", "--no-deps", "--name", fault_container, "publisher", "python", "-m",
"app.core.outbox_publisher", "--crash-before-publish", attachment["id"],
)
marker = json.dumps({"fault": "before_publish", "attachment_id": attachment["id"]})
check(killed.returncode == 137 and marker in killed.stdout, "Expected targeted SIGKILL before publish was not observed")
pending = client.call("GET", f"/api/v1/attachments/{attachment['id']}", actor=alice, schema="Attachment")["json"]
check(pending["status"] == "queued", "Crash must leave accepted job queued")
finally:
# Если docker compose run оборвался по timeout, одноразовый контейнер тоже удаляется.
try:
subprocess.run(["docker", "rm", "-f", fault_container], capture_output=True, timeout=30)
finally:
require_compose("start", "publisher")
ready = client.wait_ready(alice, attachment, wait_seconds=60)
check(ready["id"] == attachment["id"], "Recovered a different attachment")
check({variant["name"] for variant in ready["variants"]} == {"thumbnail", "crop", "optimized"}, "Missing required variants")
path = f"/api/v1/attachments/{attachment['id']}/content"
check(client.call("GET", path, actor=alice)["bytes"] == original, "Original changed after recovery")
for variant in ready["variants"]:
downloaded = client.call("GET", path + "?variant=" + variant["name"], actor=alice)["bytes"]
check(len(downloaded) == variant["size_bytes"] and hashlib.sha256(downloaded).hexdigest() == variant["sha256"], "Recovered variant checksum mismatch")
print(json.dumps({
"scenario": "commit_before_publish", "attachment_id": ready["id"],
"message_id": message["id"], "publisher_exit_code": killed.returncode,
"status_after_crash": pending["status"], "status_after_restart": ready["status"],
"variants": [variant["name"] for variant in ready["variants"]],
"reuploaded": False,
}))
if __name__ == "__main__":
main()
+1 -1
View File
@@ -14,7 +14,7 @@ def main() -> None:
destination.chmod(0o600)
directory = root / "secrets"
directory.mkdir(mode=0o700, exist_ok=True)
for name in ("db_password", "redis_password", "minio_root_user", "minio_root_password", "s3_access_key_id", "s3_secret_access_key"):
for name in ("db_password", "redis_password", "broker_password", "minio_root_user", "minio_root_password", "s3_access_key_id", "s3_secret_access_key"):
path = directory / name
try:
descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o444)
+9 -4
View File
@@ -22,11 +22,16 @@ wait_healthy() {
done
}
compose stop cleanup
compose restart postgres redis minio
wait_healthy postgres redis minio
worker_running=$(compose ps --status running --quiet worker)
compose stop worker publisher cleanup
compose restart postgres redis minio broker
wait_healthy postgres redis minio broker
compose restart api
wait_healthy api
compose restart nginx
wait_healthy nginx
compose start cleanup
compose start publisher cleanup
# Сценарий с заранее остановленным worker сохраняет его остановленным.
if [ -n "$worker_running" ]; then
compose start worker
fi