"""Fault/recovery очереди на явно выбранном Compose-стенде; только синтетические данные.""" import argparse from datetime import datetime import hashlib from io import BytesIO import json from pathlib import Path import subprocess import time from uuid import UUID, uuid4 from PIL import Image from scripts.check_outbox_recovery import COMPOSE, ROOT, compose, require_compose from scripts.deployment import ComposeProject from tests.client import Client, check PROBE = """ import asyncio, hashlib, json from uuid import UUID from sqlalchemy import select from app.core.config import Settings from app.core.database import create_db_engine, create_session_factory from app.models.job import AttachmentJob from app.models.attachment import Attachment from app.storage.s3 import create_s3_storage async def main(): settings=Settings(); engine=create_db_engine(settings); storage=create_s3_storage(settings.s3) try: async with create_session_factory(engine)() as session: job, att=(await session.execute(select(AttachmentJob,Attachment).join(Attachment,Attachment.id==AttachmentJob.attachment_id).where(Attachment.id==UUID(ATTACHMENT_ID)))).one() result=dict(job_id=str(job.id),status=att.status,attempt=job.attempt,published_attempt=job.published_attempt,limit=job.attempt_limit,updated_at=att.updated_at,next_attempt_at=job.next_attempt_at,last_error_code=job.last_error_code,permanent=job.last_error_permanent) objects=[] for prefix in ('originals/'+UUID(ATTACHMENT_ID).hex+'/', 'variants/'+UUID(ATTACHMENT_ID).hex+'/'): entries=await asyncio.to_thread(lambda: list(storage._client.list_objects(storage.settings.bucket,prefix=prefix,recursive=True))) for obj in entries: stream=await storage.open_object(obj.object_name); checksum=hashlib.sha256() try: async for chunk in stream: checksum.update(chunk) finally: await stream.aclose() objects.append(dict(name=obj.object_name.split('/')[-1],size=obj.size,sha256=checksum.hexdigest(),last_modified=obj.last_modified)) result['objects']=sorted(objects,key=lambda x:x['name']);print(json.dumps(result,default=str)) finally: await storage.aclose();await engine.dispose() asyncio.run(main()) """ def probe(attachment_id): code = "ATTACHMENT_ID = " + repr(str(UUID(attachment_id))) + "\n" + PROBE result = subprocess.run( COMPOSE + ["exec", "-T", "api", "python", "-"], input=code, text=True, capture_output=True, cwd=ROOT, timeout=30, ) check(result.returncode == 0, "Probe failed; diagnostic output hidden") return json.loads(result.stdout) def admin(*arguments, expected=0): result = compose( "run", "--rm", "--no-deps", "worker", "python", "-m", "app.core.job_admin", *arguments ) check(result.returncode == expected, "Unexpected operator command exit status") return json.loads(result.stdout) def verify_result(client, actor, attachment, original): ready = client.wait_ready(actor, attachment, wait_seconds=180) check(ready["id"] == attachment["id"], "Attachment identity changed") path = "/api/v1/attachments/" + ready["id"] + "/content" check( client.call("GET", path, actor=actor)["bytes"] == original, "Original changed" ) check( {v["name"] for v in ready["variants"]} == {"thumbnail", "crop", "optimized"}, "Incomplete pipeline", ) for variant in ready["variants"]: data = client.call("GET", path + "?variant=" + variant["name"], actor=actor)[ "bytes" ] check( hashlib.sha256(data).hexdigest() == variant["sha256"], "Checksum mismatch" ) with Image.open(BytesIO(data)) as image: check( image.size == (variant["width"], variant["height"]), "Metadata dimensions mismatch", ) check( image.format == ("WEBP" if variant["name"] == "optimized" else "JPEG"), "Wrong format", ) check( not image.getexif() and not image.info.get("icc_profile") and not image.info.get("xmp"), "Metadata leaked", ) if variant["name"] == "crop": check(image.size == (128, 128), "Wrong crop size") snapshot = probe(ready["id"]) check( len(snapshot["objects"]) == 4, "Expected exactly one original and three variants", ) return snapshot def crash(client, actor, chat, mode): name = "messenger-worker-fault-" + uuid4().hex try: require_compose("stop", "worker") attachment, original = client.attachment(actor, chat) result = compose( "run", "--rm", "--no-deps", "--name", name, "worker", "python", "-m", "app.core.job_worker", "--" + mode, attachment["id"], ) phase = ( "after_object:optimized" if mode == "crash-after-object" else "before_ack" ) marker = json.dumps({"fault": phase, "attachment_id": attachment["id"]}) check( result.returncode == 137 and marker in result.stdout, "Targeted crash not observed", ) before = probe(attachment["id"]) check( before["status"] == ("processing" if mode == "crash-after-object" else "ready"), "Unexpected crash state", ) check(len(before["objects"]) == 4, "Crash must occur after all four PUTs") finally: subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=30) require_compose("start", "worker") after = verify_result(client, actor, attachment, original) if mode == "crash-before-ack": # Wait for RabbitMQ to deliver and acknowledge the ready duplicate. wait_queue_empty() after = probe(attachment["id"]) check(after == before, "Ready duplicate rewrote result") else: check( after["attempt"] == before["attempt"] + 1, "Lease did not recover the interrupted attempt", ) check( [(x["name"], x["sha256"]) for x in before["objects"]] == [(x["name"], x["sha256"]) for x in after["objects"]], "Replay changed object content", ) return dict( scenario=mode, attachment_id=attachment["id"], exit_code=137, before=before, after=after, ) def retry_replay(client, actor, chat, permanent): name = "messenger-retry-fault-" + uuid4().hex samples = [] configuration = ComposeProject().config["services"]["worker"]["environment"] retry_base = float(configuration["JOB_RETRY_SECONDS"]) retry_max = float(configuration["JOB_RETRY_MAX_SECONDS"]) try: require_compose("stop", "worker") attachment, original = client.attachment(actor, chat) mode = "fail-permanent" if permanent else "fail-transient" require_compose( "run", "-d", "--no-deps", "--name", name, "worker", "python", "-m", "app.core.job_worker", "--" + mode, attachment["id"], ) deadline = time.monotonic() + 180 while time.monotonic() < deadline: sample = probe(attachment["id"]) if ( sample["attempt"] and sample["status"] in {"queued", "failed"} and (not samples or samples[-1]["attempt"] != sample["attempt"]) ): samples.append(sample) if sample["status"] == "failed": break time.sleep(0.5) check(sample["status"] == "failed", "Retry never reached failed") check( sample["attempt"] == (1 if permanent else sample["limit"]), "Incorrect attempt budget", ) check(sample["permanent"] == permanent, "Incorrect error classification") if not permanent: check(len(samples) == sample["limit"], "Missed retry samples") for entry in samples: delay = ( datetime.fromisoformat(entry["next_attempt_at"]) - datetime.fromisoformat(entry["updated_at"]) ).total_seconds() ceiling = min(retry_max, retry_base * 2 ** (entry["attempt"] - 1)) check( ceiling / 2 <= delay <= ceiling, "Retry delay outside configured jitter bounds", ) client.call( "GET", "/api/v1/attachments/" + attachment["id"] + "/content", actor=actor, expected=409, ) job_id = sample["job_id"] request_id = str(uuid4()) listing = admin("list-failed") check( any(row["job_id"] == job_id for row in listing), "Job missing from failed registry", ) subprocess.run( ["docker", "rm", "-f", name], capture_output=True, timeout=30, check=True ) first = admin( "replay", job_id, "--request-id", request_id, "--actor", "verification", "--reason", "dependency_restored", ) check(first["status"] == "queued", "Replay did not queue job") finally: subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=30) require_compose("start", "worker") after = verify_result(client, actor, attachment, original) repeat = admin( "replay", job_id, "--request-id", request_id, "--actor", "verification", "--reason", "dependency_restored", ) check(repeat["already_applied"], "Replay request id not idempotent") refused = admin( "replay", job_id, "--request-id", str(uuid4()), "--actor", "verification", "--reason", "manual_review", expected=2, ) check(refused["error_code"] == "job_not_failed", "Successful job was replayed") check(probe(attachment["id"]) == after, "Operator replay altered ready job") check(len(admin("history", job_id)) == 1, "Incorrect audit count") return dict( scenario=mode, attachment_id=attachment["id"], samples=samples, after=after, replay_idempotent=True, ) def queue_state(): output = require_compose( "exec", "-T", "broker", "rabbitmqctl", "list_queues", "--formatter=json", "name", "messages_ready", "messages_unacknowledged", "consumers", "durable", ) return next(row for row in json.loads(output) if row["name"] == "attachment-jobs") def wait_queue_empty(): deadline = time.monotonic() + 20 while time.monotonic() < deadline: state = queue_state() if state["messages_ready"] == 0 and state["messages_unacknowledged"] == 0: return state time.sleep(0.5) check(False, "RabbitMQ did not acknowledge all completed deliveries") def persistence(client, actor, chat): try: require_compose("stop", "worker") attachment, original = client.attachment(actor, chat) client.message(actor, chat, "", attachments=[attachment["id"]]) client.call( "GET", "/api/v1/attachments/" + attachment["id"] + "/content", actor=actor, expected=409, ) time.sleep(2) queued_before = queue_state() check( queued_before["messages_ready"] >= 1 and queued_before["durable"], "No durable queued message before restart", ) result = subprocess.run( [str(ROOT / "scripts/restart.sh")], cwd=ROOT, capture_output=True, timeout=180, ) check(result.returncode == 0, "Restart hook failed") check( "worker" not in require_compose( "ps", "--status", "running", "--services" ).splitlines(), "Restart started deliberately stopped worker", ) queued_after = queue_state() check( queued_after["messages_ready"] >= 1 and queued_after["durable"], "Broker lost queued delivery across restart", ) client.message(actor, chat, "session and chat survived broker/API restart") finally: require_compose("start", "worker") after = verify_result(client, actor, attachment, original) return dict( scenario="worker_stopped_restart_all", attachment_id=attachment["id"], queue_before=queued_before, queue_after=queued_after, after=after, ) def main(): parser = argparse.ArgumentParser() parser.add_argument("--base-url", required=True) parser.add_argument("--ca-file") parser.add_argument( "--scenario", choices=[ "all", "persistence", "crash-after-object", "crash-before-ack", "fail-transient", "fail-permanent", ], default="all", ) parser.add_argument("--output", default="evidence/jobs.json") args = parser.parse_args() client = Client(args.base_url, args.ca_file, timeout=40) aliases = {r.instance_id for r in ComposeProject().replicas() if r.running} check( client.call("GET", "/health/live", schema="Health")["headers"].get( "X-Instance-Id" ) in aliases, "Wrong Compose project for BASE_URL", ) actor, _, _, chat = client.bootstrap() results = [] operations = { "persistence": persistence, "crash-after-object": lambda *x: crash(*x, "crash-after-object"), "crash-before-ack": lambda *x: crash(*x, "crash-before-ack"), "fail-transient": lambda *x: retry_replay(*x, False), "fail-permanent": lambda *x: retry_replay(*x, True), } selected = ( operations.values() if args.scenario == "all" else [operations[args.scenario]] ) for operation in selected: result = operation(client, actor, chat) results.append(result) Path(args.output).parent.mkdir(parents=True, exist_ok=True) Path(args.output).write_text( json.dumps(dict(status="in_progress", results=results), indent=2) + "\n" ) print("PASS: " + result["scenario"], flush=True) Path(args.output).write_text( json.dumps(dict(status="passed", results=results), indent=2) + "\n" ) if __name__ == "__main__": main()