forked from mtusi2026/mtusi2026lab5
evidence/level4.json: all 17 scenarios of levels 3 and 4 and the eight invariants pass on 0b85edf; evidence/operator.txt shows the failed registry, a replay to ready, a noop replay, an expired replay and the DLQ. Smoke, topology, level3, persistence, failover, lab 4 scenarios, limits, student tests and contract checks are refreshed on the same commit. REPORT.md describes the variants, worker limits, jitter, lease renewal, crash points, replay and retention, adds rows 4.1-4.4, the level 4 failure experiments and the updated limitations. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
259 lines
6.2 KiB
JSON
259 lines
6.2 KiB
JSON
{
|
|
"suite": "lab5-pipeline-idempotency",
|
|
"started_at": "2026-10-29T00:21:40.224842Z",
|
|
"config": {
|
|
"base_url": "http://localhost:18085",
|
|
"level": 3,
|
|
"only": "",
|
|
"expect_variants": "thumbnail",
|
|
"max_attempts": 5,
|
|
"stall_seconds": 15.0,
|
|
"compose": "docker compose",
|
|
"compose_faults": "docker compose -f compose.yaml -f infra/compose.faults.yaml"
|
|
},
|
|
"invariants": [
|
|
{
|
|
"id": "terminal",
|
|
"statement": "every 202 reaches its expected terminal state",
|
|
"result": "PASS",
|
|
"details": {
|
|
"accepted": 5
|
|
}
|
|
},
|
|
{
|
|
"id": "single_effect",
|
|
"statement": "one variant set per (attachment_id, pipeline_version, variant)",
|
|
"result": "PASS",
|
|
"details": {
|
|
"rows": 0
|
|
}
|
|
},
|
|
{
|
|
"id": "no_ready_rollback",
|
|
"statement": "ready never returns to queued or processing",
|
|
"result": "PASS",
|
|
"details": {
|
|
"traced": 5
|
|
}
|
|
},
|
|
{
|
|
"id": "monotonic_publication",
|
|
"statement": "published_version only moves forward",
|
|
"result": "PASS",
|
|
"details": {
|
|
"sampled": 2
|
|
}
|
|
},
|
|
{
|
|
"id": "bytes_match_metadata",
|
|
"statement": "sha256, size and real dimensions match served bytes",
|
|
"result": "PASS",
|
|
"details": {
|
|
"verified": 4
|
|
}
|
|
},
|
|
{
|
|
"id": "bounded_attempts",
|
|
"statement": "attempt <= JOB_MAX_ATTEMPTS",
|
|
"result": "PASS",
|
|
"details": {
|
|
"rows": 0
|
|
}
|
|
},
|
|
{
|
|
"id": "no_orphans",
|
|
"statement": "no pending outbox, messages, dead letters, staging, objects",
|
|
"result": "PASS",
|
|
"details": {
|
|
"queues": {
|
|
"attachments.process": [
|
|
0,
|
|
0
|
|
],
|
|
"attachments.process.dlq": [
|
|
1,
|
|
0
|
|
]
|
|
},
|
|
"s3": "SKIP: --s3-list is not set"
|
|
}
|
|
},
|
|
{
|
|
"id": "no_lost_202",
|
|
"statement": "every 202 is present in PostgreSQL and the API",
|
|
"result": "PASS",
|
|
"details": {
|
|
"accepted": 5
|
|
}
|
|
}
|
|
],
|
|
"scenarios": [
|
|
{
|
|
"name": "smoke_pipeline",
|
|
"level": 3,
|
|
"window": "upload, ready, variants match metadata",
|
|
"seconds": 1.3,
|
|
"result": "PASS",
|
|
"observed": {
|
|
"attachment_id": "01a18508-c6f6-774b-ae60-24aa728ad62c",
|
|
"variants": [
|
|
"crop",
|
|
"optimized",
|
|
"thumbnail"
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"name": "worker_down_broker_restart",
|
|
"level": 3,
|
|
"window": "worker down, API and broker restart",
|
|
"seconds": 24.0,
|
|
"result": "PASS",
|
|
"observed": {
|
|
"attachment_id": "01a18508-cd97-737c-bb03-5409de83ff8e",
|
|
"timeline": [
|
|
{
|
|
"at": "2026-10-29T00:21:49.075959Z",
|
|
"step": "worker stopped"
|
|
},
|
|
{
|
|
"at": "2026-10-29T00:21:49.095967Z",
|
|
"step": "upload 202 queued",
|
|
"attachment_id": "01a18508-cd97-737c-bb03-5409de83ff8e"
|
|
},
|
|
{
|
|
"at": "2026-10-29T00:21:49.099982Z",
|
|
"step": "content 409 while queued"
|
|
},
|
|
{
|
|
"at": "2026-10-29T00:21:49.138263Z",
|
|
"step": "message with attachment 201, other messages work",
|
|
"message_id": "01a18508-cdad-708f-8324-18123e9c7758"
|
|
},
|
|
{
|
|
"at": "2026-10-29T00:21:54.782528Z",
|
|
"step": "job is in the broker queue",
|
|
"queues": {
|
|
"attachments.process": [
|
|
1,
|
|
0
|
|
],
|
|
"attachments.process.dlq": [
|
|
1,
|
|
0
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"at": "2026-10-29T00:22:00.869187Z",
|
|
"step": "API replicas and relay restarted"
|
|
},
|
|
{
|
|
"at": "2026-10-29T00:22:09.698869Z",
|
|
"step": "broker restarted with its volume, job survived",
|
|
"queues": {
|
|
"attachments.process": [
|
|
1,
|
|
0
|
|
],
|
|
"attachments.process.dlq": [
|
|
1,
|
|
0
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"at": "2026-10-29T00:22:12.427256Z",
|
|
"step": "worker started"
|
|
},
|
|
{
|
|
"at": "2026-10-29T00:22:12.707560Z",
|
|
"step": "ready",
|
|
"variants": [
|
|
"crop",
|
|
"optimized",
|
|
"thumbnail"
|
|
]
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"name": "duplicate_after_ready",
|
|
"level": 3,
|
|
"window": "redelivery of a finished job",
|
|
"seconds": 7.4,
|
|
"result": "PASS",
|
|
"observed": {
|
|
"attachment_id": "01a18509-29ee-7116-b3df-b04caf1c48e7",
|
|
"events": [
|
|
{
|
|
"kind": "enqueued",
|
|
"attempt": 1
|
|
},
|
|
{
|
|
"kind": "claimed",
|
|
"attempt": 1
|
|
},
|
|
{
|
|
"kind": "done",
|
|
"attempt": 1
|
|
},
|
|
{
|
|
"kind": "duplicate_noop",
|
|
"attempt": 1
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"name": "poison_image",
|
|
"level": 3,
|
|
"window": "permanent error without retries",
|
|
"seconds": 0.0,
|
|
"result": "PASS",
|
|
"observed": {
|
|
"rejected_by_api": 415
|
|
}
|
|
},
|
|
{
|
|
"name": "s3_outage_recovers",
|
|
"level": 3,
|
|
"window": "transient error with retry",
|
|
"seconds": 11.6,
|
|
"result": "PASS",
|
|
"observed": {
|
|
"attachment_id": "01a18509-4892-72ec-8f31-ea07c55239c7",
|
|
"jobs": [
|
|
{
|
|
"job_id": "01a18509-4898-70c0-9198-8af96f5da78b",
|
|
"pipeline_version": 1,
|
|
"status": "done",
|
|
"attempt": 2
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"name": "retry_exhausted",
|
|
"level": 3,
|
|
"window": "transient error until JOB_MAX_ATTEMPTS",
|
|
"seconds": 23.4,
|
|
"result": "PASS",
|
|
"observed": {
|
|
"attachment_id": "01a18509-8268-718e-912d-c6718d5492e0",
|
|
"jobs": [
|
|
{
|
|
"job_id": "01a18509-826e-7736-8d1e-ba66a970dc8e",
|
|
"pipeline_version": 1,
|
|
"status": "failed",
|
|
"attempt": 5
|
|
}
|
|
]
|
|
}
|
|
}
|
|
],
|
|
"result": "PASS",
|
|
"finished_at": "2026-10-29T00:23:06.519678Z"
|
|
}
|