Files
mtusi2026labs/evidence/level3.json
T
ArtyomandClaude Opus 5.5 a55bc716c4 Record level 3 evidence and fill in the report
Every grade 3 item of lab 5 is proven on a stand raised from empty
volumes at fe26938: the public smoke passes with 202 queued and a real
256x160 JPEG thumbnail; the suite runs its six grade 3 scenarios and
eight invariants green, including worker down with both API replicas,
the relay and the broker restarted while the job stays in the queue,
a redelivered finished job that changes nothing, an S3 outage that
recovers on attempt 2 and five transient failures that end in
retry_exhausted; the stand evidence shows the queued job with
delivery mode 2, the job id as message id and exactly the contract
fields, 409 before ready and a clean SIGTERM of the relay and the
worker; persistence restarts all eight services with their volumes,
failover keeps 134 writes without errors, and the lab 4 scenarios,
S3 checks, upload faults and container policy still pass.

The report describes the clean clone start, the processes and the
source of truth, what is stored at the moment of 202, the outbox and
relay, the claim, fencing and ack order of the worker, the retry
policy, the failure windows, the broker topology and permissions, the
roles table and the mapping of .env.example.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-29 01:31:07 +03:00

255 lines
6.2 KiB
JSON

{
"suite": "lab5-pipeline-idempotency",
"started_at": "2026-10-28T22:20:16.079560Z",
"config": {
"base_url": "http://localhost:18085",
"level": 3,
"only": "",
"expect_variants": "thumbnail",
"max_attempts": 5,
"stall_seconds": 15.0,
"compose": "docker compose",
"compose_faults": "docker compose -f compose.yaml -f infra/compose.faults.yaml"
},
"invariants": [
{
"id": "terminal",
"statement": "every 202 reaches its expected terminal state",
"result": "PASS",
"details": {
"accepted": 5
}
},
{
"id": "single_effect",
"statement": "one variant set per (attachment_id, pipeline_version, variant)",
"result": "PASS",
"details": {
"rows": 0
}
},
{
"id": "no_ready_rollback",
"statement": "ready never returns to queued or processing",
"result": "PASS",
"details": {
"traced": 5
}
},
{
"id": "monotonic_publication",
"statement": "published_version only moves forward",
"result": "PASS",
"details": {
"sampled": 2
}
},
{
"id": "bytes_match_metadata",
"statement": "sha256, size and real dimensions match served bytes",
"result": "PASS",
"details": {
"verified": 4
}
},
{
"id": "bounded_attempts",
"statement": "attempt <= JOB_MAX_ATTEMPTS",
"result": "PASS",
"details": {
"rows": 0
}
},
{
"id": "no_orphans",
"statement": "no pending outbox, messages, dead letters, staging, objects",
"result": "PASS",
"details": {
"queues": {
"attachments.process": [
0,
0
],
"attachments.process.dlq": [
0,
0
]
},
"s3": "SKIP: --s3-list is not set"
}
},
{
"id": "no_lost_202",
"statement": "every 202 is present in PostgreSQL and the API",
"result": "PASS",
"details": {
"accepted": 5
}
}
],
"scenarios": [
{
"name": "smoke_pipeline",
"level": 3,
"window": "upload, ready, variants match metadata",
"seconds": 0.3,
"result": "PASS",
"observed": {
"attachment_id": "01a18499-a3b4-71f7-9116-e01ab2e3927b",
"variants": [
"thumbnail"
]
}
},
{
"name": "worker_down_broker_restart",
"level": 3,
"window": "worker down, API and broker restart",
"seconds": 25.2,
"result": "PASS",
"observed": {
"attachment_id": "01a18499-a689-742f-9c5c-bc6196e95b17",
"timeline": [
{
"at": "2026-10-28T22:20:24.579204Z",
"step": "worker stopped"
},
{
"at": "2026-10-28T22:20:24.595477Z",
"step": "upload 202 queued",
"attachment_id": "01a18499-a689-742f-9c5c-bc6196e95b17"
},
{
"at": "2026-10-28T22:20:24.599071Z",
"step": "content 409 while queued"
},
{
"at": "2026-10-28T22:20:24.619147Z",
"step": "message with attachment 201, other messages work",
"message_id": "01a18499-a698-7776-9a20-eacea53325fe"
},
{
"at": "2026-10-28T22:20:31.645448Z",
"step": "job is in the broker queue",
"queues": {
"attachments.process": [
1,
0
],
"attachments.process.dlq": [
0,
0
]
}
},
{
"at": "2026-10-28T22:20:37.799903Z",
"step": "API replicas and relay restarted"
},
{
"at": "2026-10-28T22:20:46.499917Z",
"step": "broker restarted with its volume, job survived",
"queues": {
"attachments.process": [
1,
0
],
"attachments.process.dlq": [
0,
0
]
}
},
{
"at": "2026-10-28T22:20:49.240721Z",
"step": "worker started"
},
{
"at": "2026-10-28T22:20:49.423263Z",
"step": "ready",
"variants": [
"thumbnail"
]
}
]
}
},
{
"name": "duplicate_after_ready",
"level": 3,
"window": "redelivery of a finished job",
"seconds": 3.9,
"result": "PASS",
"observed": {
"attachment_id": "01a1849a-0796-74ec-9ed8-4fdce40fa325",
"events": [
{
"kind": "enqueued",
"attempt": 1
},
{
"kind": "claimed",
"attempt": 1
},
{
"kind": "done",
"attempt": 1
},
{
"kind": "duplicate_noop",
"attempt": 1
}
]
}
},
{
"name": "poison_image",
"level": 3,
"window": "permanent error without retries",
"seconds": 0.1,
"result": "PASS",
"observed": {
"rejected_by_api": 415
}
},
{
"name": "s3_outage_recovers",
"level": 3,
"window": "transient error with retry",
"seconds": 13.8,
"result": "PASS",
"observed": {
"attachment_id": "01a1849a-18c4-7330-b1d6-a596a0e473c1",
"jobs": [
{
"job_id": "01a1849a-18cf-7463-b86e-697e4a60189d",
"pipeline_version": 1,
"status": "done",
"attempt": 2
}
]
}
},
{
"name": "retry_exhausted",
"level": 3,
"window": "transient error until JOB_MAX_ATTEMPTS",
"seconds": 22.9,
"result": "PASS",
"observed": {
"attachment_id": "01a1849a-5ba2-759c-8166-8c6061753c4b",
"jobs": [
{
"job_id": "01a1849a-5bba-75f8-99bc-cdcfe0dc84d8",
"pipeline_version": 1,
"status": "failed",
"attempt": 5
}
]
}
}
],
"result": "PASS",
"finished_at": "2026-10-28T22:21:35.933583Z"
}