430 lines
15 KiB
Python
430 lines
15 KiB
Python
"""Fault/recovery очереди на явно выбранном Compose-стенде; только синтетические данные."""
|
|
|
|
import argparse
|
|
from datetime import datetime
|
|
import hashlib
|
|
from io import BytesIO
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import time
|
|
from uuid import UUID, uuid4
|
|
|
|
from PIL import Image
|
|
|
|
from scripts.check_outbox_recovery import COMPOSE, ROOT, compose, require_compose
|
|
from scripts.deployment import ComposeProject
|
|
from tests.client import Client, check
|
|
|
|
PROBE = """
|
|
import asyncio, hashlib, json
|
|
from uuid import UUID
|
|
from sqlalchemy import select
|
|
from app.core.config import Settings
|
|
from app.core.database import create_db_engine, create_session_factory
|
|
from app.models.job import AttachmentJob
|
|
from app.models.attachment import Attachment
|
|
from app.storage.s3 import create_s3_storage
|
|
async def main():
|
|
settings=Settings(); engine=create_db_engine(settings); storage=create_s3_storage(settings.s3)
|
|
try:
|
|
async with create_session_factory(engine)() as session:
|
|
job, att=(await session.execute(select(AttachmentJob,Attachment).join(Attachment,Attachment.id==AttachmentJob.attachment_id).where(Attachment.id==UUID(ATTACHMENT_ID)))).one()
|
|
result=dict(job_id=str(job.id),status=att.status,attempt=job.attempt,published_attempt=job.published_attempt,limit=job.attempt_limit,updated_at=att.updated_at,next_attempt_at=job.next_attempt_at,last_error_code=job.last_error_code,permanent=job.last_error_permanent)
|
|
objects=[]
|
|
for prefix in ('originals/'+UUID(ATTACHMENT_ID).hex+'/', 'variants/'+UUID(ATTACHMENT_ID).hex+'/'):
|
|
entries=await asyncio.to_thread(lambda: list(storage._client.list_objects(storage.settings.bucket,prefix=prefix,recursive=True)))
|
|
for obj in entries:
|
|
stream=await storage.open_object(obj.object_name); checksum=hashlib.sha256()
|
|
try:
|
|
async for chunk in stream: checksum.update(chunk)
|
|
finally: await stream.aclose()
|
|
objects.append(dict(name=obj.object_name.split('/')[-1],size=obj.size,sha256=checksum.hexdigest(),last_modified=obj.last_modified))
|
|
result['objects']=sorted(objects,key=lambda x:x['name']);print(json.dumps(result,default=str))
|
|
finally: await storage.aclose();await engine.dispose()
|
|
asyncio.run(main())
|
|
"""
|
|
|
|
|
|
def probe(attachment_id):
|
|
code = "ATTACHMENT_ID = " + repr(str(UUID(attachment_id))) + "\n" + PROBE
|
|
result = subprocess.run(
|
|
COMPOSE + ["exec", "-T", "api", "python", "-"],
|
|
input=code,
|
|
text=True,
|
|
capture_output=True,
|
|
cwd=ROOT,
|
|
timeout=30,
|
|
)
|
|
check(result.returncode == 0, "Probe failed; diagnostic output hidden")
|
|
return json.loads(result.stdout)
|
|
|
|
|
|
def admin(*arguments, expected=0):
|
|
result = compose(
|
|
"run",
|
|
"--rm",
|
|
"--no-deps",
|
|
"worker",
|
|
"python",
|
|
"-m",
|
|
"app.core.job_admin",
|
|
*arguments
|
|
)
|
|
check(result.returncode == expected, "Unexpected operator command exit status")
|
|
return json.loads(result.stdout)
|
|
|
|
|
|
def verify_result(client, actor, attachment, original):
|
|
ready = client.wait_ready(actor, attachment, wait_seconds=180)
|
|
check(ready["id"] == attachment["id"], "Attachment identity changed")
|
|
path = "/api/v1/attachments/" + ready["id"] + "/content"
|
|
check(
|
|
client.call("GET", path, actor=actor)["bytes"] == original, "Original changed"
|
|
)
|
|
check(
|
|
{v["name"] for v in ready["variants"]} == {"thumbnail", "crop", "optimized"},
|
|
"Incomplete pipeline",
|
|
)
|
|
for variant in ready["variants"]:
|
|
data = client.call("GET", path + "?variant=" + variant["name"], actor=actor)[
|
|
"bytes"
|
|
]
|
|
check(
|
|
hashlib.sha256(data).hexdigest() == variant["sha256"], "Checksum mismatch"
|
|
)
|
|
with Image.open(BytesIO(data)) as image:
|
|
check(
|
|
image.size == (variant["width"], variant["height"]),
|
|
"Metadata dimensions mismatch",
|
|
)
|
|
check(
|
|
image.format == ("WEBP" if variant["name"] == "optimized" else "JPEG"),
|
|
"Wrong format",
|
|
)
|
|
check(
|
|
not image.getexif()
|
|
and not image.info.get("icc_profile")
|
|
and not image.info.get("xmp"),
|
|
"Metadata leaked",
|
|
)
|
|
if variant["name"] == "crop":
|
|
check(image.size == (128, 128), "Wrong crop size")
|
|
snapshot = probe(ready["id"])
|
|
check(
|
|
len(snapshot["objects"]) == 4,
|
|
"Expected exactly one original and three variants",
|
|
)
|
|
return snapshot
|
|
|
|
|
|
def crash(client, actor, chat, mode):
|
|
name = "messenger-worker-fault-" + uuid4().hex
|
|
try:
|
|
require_compose("stop", "worker")
|
|
attachment, original = client.attachment(actor, chat)
|
|
result = compose(
|
|
"run",
|
|
"--rm",
|
|
"--no-deps",
|
|
"--name",
|
|
name,
|
|
"worker",
|
|
"python",
|
|
"-m",
|
|
"app.core.job_worker",
|
|
"--" + mode,
|
|
attachment["id"],
|
|
)
|
|
phase = (
|
|
"after_object:optimized" if mode == "crash-after-object" else "before_ack"
|
|
)
|
|
marker = json.dumps({"fault": phase, "attachment_id": attachment["id"]})
|
|
check(
|
|
result.returncode == 137 and marker in result.stdout,
|
|
"Targeted crash not observed",
|
|
)
|
|
before = probe(attachment["id"])
|
|
check(
|
|
before["status"]
|
|
== ("processing" if mode == "crash-after-object" else "ready"),
|
|
"Unexpected crash state",
|
|
)
|
|
check(len(before["objects"]) == 4, "Crash must occur after all four PUTs")
|
|
finally:
|
|
subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=30)
|
|
require_compose("start", "worker")
|
|
after = verify_result(client, actor, attachment, original)
|
|
if mode == "crash-before-ack":
|
|
# Wait for RabbitMQ to deliver and acknowledge the ready duplicate.
|
|
wait_queue_empty()
|
|
after = probe(attachment["id"])
|
|
check(after == before, "Ready duplicate rewrote result")
|
|
else:
|
|
check(
|
|
after["attempt"] == before["attempt"] + 1,
|
|
"Lease did not recover the interrupted attempt",
|
|
)
|
|
check(
|
|
[(x["name"], x["sha256"]) for x in before["objects"]]
|
|
== [(x["name"], x["sha256"]) for x in after["objects"]],
|
|
"Replay changed object content",
|
|
)
|
|
return dict(
|
|
scenario=mode,
|
|
attachment_id=attachment["id"],
|
|
exit_code=137,
|
|
before=before,
|
|
after=after,
|
|
)
|
|
|
|
|
|
def retry_replay(client, actor, chat, permanent):
|
|
name = "messenger-retry-fault-" + uuid4().hex
|
|
samples = []
|
|
configuration = ComposeProject().config["services"]["worker"]["environment"]
|
|
retry_base = float(configuration["JOB_RETRY_SECONDS"])
|
|
retry_max = float(configuration["JOB_RETRY_MAX_SECONDS"])
|
|
try:
|
|
require_compose("stop", "worker")
|
|
attachment, original = client.attachment(actor, chat)
|
|
mode = "fail-permanent" if permanent else "fail-transient"
|
|
require_compose(
|
|
"run",
|
|
"-d",
|
|
"--no-deps",
|
|
"--name",
|
|
name,
|
|
"worker",
|
|
"python",
|
|
"-m",
|
|
"app.core.job_worker",
|
|
"--" + mode,
|
|
attachment["id"],
|
|
)
|
|
deadline = time.monotonic() + 180
|
|
while time.monotonic() < deadline:
|
|
sample = probe(attachment["id"])
|
|
if (
|
|
sample["attempt"]
|
|
and sample["status"] in {"queued", "failed"}
|
|
and (not samples or samples[-1]["attempt"] != sample["attempt"])
|
|
):
|
|
samples.append(sample)
|
|
if sample["status"] == "failed":
|
|
break
|
|
time.sleep(0.5)
|
|
check(sample["status"] == "failed", "Retry never reached failed")
|
|
check(
|
|
sample["attempt"] == (1 if permanent else sample["limit"]),
|
|
"Incorrect attempt budget",
|
|
)
|
|
check(sample["permanent"] == permanent, "Incorrect error classification")
|
|
if not permanent:
|
|
check(len(samples) == sample["limit"], "Missed retry samples")
|
|
for entry in samples:
|
|
delay = (
|
|
datetime.fromisoformat(entry["next_attempt_at"])
|
|
- datetime.fromisoformat(entry["updated_at"])
|
|
).total_seconds()
|
|
ceiling = min(retry_max, retry_base * 2 ** (entry["attempt"] - 1))
|
|
check(
|
|
ceiling / 2 <= delay <= ceiling,
|
|
"Retry delay outside configured jitter bounds",
|
|
)
|
|
client.call(
|
|
"GET",
|
|
"/api/v1/attachments/" + attachment["id"] + "/content",
|
|
actor=actor,
|
|
expected=409,
|
|
)
|
|
job_id = sample["job_id"]
|
|
request_id = str(uuid4())
|
|
listing = admin("list-failed")
|
|
check(
|
|
any(row["job_id"] == job_id for row in listing),
|
|
"Job missing from failed registry",
|
|
)
|
|
subprocess.run(
|
|
["docker", "rm", "-f", name], capture_output=True, timeout=30, check=True
|
|
)
|
|
first = admin(
|
|
"replay",
|
|
job_id,
|
|
"--request-id",
|
|
request_id,
|
|
"--actor",
|
|
"verification",
|
|
"--reason",
|
|
"dependency_restored",
|
|
)
|
|
check(first["status"] == "queued", "Replay did not queue job")
|
|
finally:
|
|
subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=30)
|
|
require_compose("start", "worker")
|
|
after = verify_result(client, actor, attachment, original)
|
|
repeat = admin(
|
|
"replay",
|
|
job_id,
|
|
"--request-id",
|
|
request_id,
|
|
"--actor",
|
|
"verification",
|
|
"--reason",
|
|
"dependency_restored",
|
|
)
|
|
check(repeat["already_applied"], "Replay request id not idempotent")
|
|
refused = admin(
|
|
"replay",
|
|
job_id,
|
|
"--request-id",
|
|
str(uuid4()),
|
|
"--actor",
|
|
"verification",
|
|
"--reason",
|
|
"manual_review",
|
|
expected=2,
|
|
)
|
|
check(refused["error_code"] == "job_not_failed", "Successful job was replayed")
|
|
check(probe(attachment["id"]) == after, "Operator replay altered ready job")
|
|
check(len(admin("history", job_id)) == 1, "Incorrect audit count")
|
|
return dict(
|
|
scenario=mode,
|
|
attachment_id=attachment["id"],
|
|
samples=samples,
|
|
after=after,
|
|
replay_idempotent=True,
|
|
)
|
|
|
|
|
|
def queue_state():
|
|
output = require_compose(
|
|
"exec",
|
|
"-T",
|
|
"broker",
|
|
"rabbitmqctl",
|
|
"list_queues",
|
|
"--formatter=json",
|
|
"name",
|
|
"messages_ready",
|
|
"messages_unacknowledged",
|
|
"consumers",
|
|
"durable",
|
|
)
|
|
return next(row for row in json.loads(output) if row["name"] == "attachment-jobs")
|
|
|
|
|
|
def wait_queue_empty():
|
|
deadline = time.monotonic() + 20
|
|
while time.monotonic() < deadline:
|
|
state = queue_state()
|
|
if state["messages_ready"] == 0 and state["messages_unacknowledged"] == 0:
|
|
return state
|
|
time.sleep(0.5)
|
|
check(False, "RabbitMQ did not acknowledge all completed deliveries")
|
|
|
|
|
|
def persistence(client, actor, chat):
|
|
try:
|
|
require_compose("stop", "worker")
|
|
attachment, original = client.attachment(actor, chat)
|
|
client.message(actor, chat, "", attachments=[attachment["id"]])
|
|
client.call(
|
|
"GET",
|
|
"/api/v1/attachments/" + attachment["id"] + "/content",
|
|
actor=actor,
|
|
expected=409,
|
|
)
|
|
time.sleep(2)
|
|
queued_before = queue_state()
|
|
check(
|
|
queued_before["messages_ready"] >= 1 and queued_before["durable"],
|
|
"No durable queued message before restart",
|
|
)
|
|
result = subprocess.run(
|
|
[str(ROOT / "scripts/restart.sh")],
|
|
cwd=ROOT,
|
|
capture_output=True,
|
|
timeout=180,
|
|
)
|
|
check(result.returncode == 0, "Restart hook failed")
|
|
check(
|
|
"worker"
|
|
not in require_compose(
|
|
"ps", "--status", "running", "--services"
|
|
).splitlines(),
|
|
"Restart started deliberately stopped worker",
|
|
)
|
|
queued_after = queue_state()
|
|
check(
|
|
queued_after["messages_ready"] >= 1 and queued_after["durable"],
|
|
"Broker lost queued delivery across restart",
|
|
)
|
|
client.message(actor, chat, "session and chat survived broker/API restart")
|
|
finally:
|
|
require_compose("start", "worker")
|
|
after = verify_result(client, actor, attachment, original)
|
|
return dict(
|
|
scenario="worker_stopped_restart_all",
|
|
attachment_id=attachment["id"],
|
|
queue_before=queued_before,
|
|
queue_after=queued_after,
|
|
after=after,
|
|
)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--base-url", required=True)
|
|
parser.add_argument("--ca-file")
|
|
parser.add_argument(
|
|
"--scenario",
|
|
choices=[
|
|
"all",
|
|
"persistence",
|
|
"crash-after-object",
|
|
"crash-before-ack",
|
|
"fail-transient",
|
|
"fail-permanent",
|
|
],
|
|
default="all",
|
|
)
|
|
parser.add_argument("--output", default="evidence/jobs.json")
|
|
args = parser.parse_args()
|
|
client = Client(args.base_url, args.ca_file, timeout=40)
|
|
aliases = {r.instance_id for r in ComposeProject().replicas() if r.running}
|
|
check(
|
|
client.call("GET", "/health/live", schema="Health")["headers"].get(
|
|
"X-Instance-Id"
|
|
)
|
|
in aliases,
|
|
"Wrong Compose project for BASE_URL",
|
|
)
|
|
actor, _, _, chat = client.bootstrap()
|
|
results = []
|
|
operations = {
|
|
"persistence": persistence,
|
|
"crash-after-object": lambda *x: crash(*x, "crash-after-object"),
|
|
"crash-before-ack": lambda *x: crash(*x, "crash-before-ack"),
|
|
"fail-transient": lambda *x: retry_replay(*x, False),
|
|
"fail-permanent": lambda *x: retry_replay(*x, True),
|
|
}
|
|
selected = (
|
|
operations.values() if args.scenario == "all" else [operations[args.scenario]]
|
|
)
|
|
for operation in selected:
|
|
result = operation(client, actor, chat)
|
|
results.append(result)
|
|
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
|
|
Path(args.output).write_text(
|
|
json.dumps(dict(status="in_progress", results=results), indent=2) + "\n"
|
|
)
|
|
print("PASS: " + result["scenario"], flush=True)
|
|
Path(args.output).write_text(
|
|
json.dumps(dict(status="passed", results=results), indent=2) + "\n"
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|