Files
mtusi2026lab2/evidence/level3.json
T
ArtyomandClaude Opus 5.5 573220e12a Record level 4 evidence and fill in the report
evidence/level4.json: all 17 scenarios of levels 3 and 4 and the eight
invariants pass on 0b85edf; evidence/operator.txt shows the failed
registry, a replay to ready, a noop replay, an expired replay and the
DLQ. Smoke, topology, level3, persistence, failover, lab 4 scenarios,
limits, student tests and contract checks are refreshed on the same
commit. REPORT.md describes the variants, worker limits, jitter, lease
renewal, crash points, replay and retention, adds rows 4.1-4.4, the
level 4 failure experiments and the updated limitations.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-29 03:39:20 +03:00

259 lines
6.2 KiB
JSON

{
"suite": "lab5-pipeline-idempotency",
"started_at": "2026-10-29T00:21:40.224842Z",
"config": {
"base_url": "http://localhost:18085",
"level": 3,
"only": "",
"expect_variants": "thumbnail",
"max_attempts": 5,
"stall_seconds": 15.0,
"compose": "docker compose",
"compose_faults": "docker compose -f compose.yaml -f infra/compose.faults.yaml"
},
"invariants": [
{
"id": "terminal",
"statement": "every 202 reaches its expected terminal state",
"result": "PASS",
"details": {
"accepted": 5
}
},
{
"id": "single_effect",
"statement": "one variant set per (attachment_id, pipeline_version, variant)",
"result": "PASS",
"details": {
"rows": 0
}
},
{
"id": "no_ready_rollback",
"statement": "ready never returns to queued or processing",
"result": "PASS",
"details": {
"traced": 5
}
},
{
"id": "monotonic_publication",
"statement": "published_version only moves forward",
"result": "PASS",
"details": {
"sampled": 2
}
},
{
"id": "bytes_match_metadata",
"statement": "sha256, size and real dimensions match served bytes",
"result": "PASS",
"details": {
"verified": 4
}
},
{
"id": "bounded_attempts",
"statement": "attempt <= JOB_MAX_ATTEMPTS",
"result": "PASS",
"details": {
"rows": 0
}
},
{
"id": "no_orphans",
"statement": "no pending outbox, messages, dead letters, staging, objects",
"result": "PASS",
"details": {
"queues": {
"attachments.process": [
0,
0
],
"attachments.process.dlq": [
1,
0
]
},
"s3": "SKIP: --s3-list is not set"
}
},
{
"id": "no_lost_202",
"statement": "every 202 is present in PostgreSQL and the API",
"result": "PASS",
"details": {
"accepted": 5
}
}
],
"scenarios": [
{
"name": "smoke_pipeline",
"level": 3,
"window": "upload, ready, variants match metadata",
"seconds": 1.3,
"result": "PASS",
"observed": {
"attachment_id": "01a18508-c6f6-774b-ae60-24aa728ad62c",
"variants": [
"crop",
"optimized",
"thumbnail"
]
}
},
{
"name": "worker_down_broker_restart",
"level": 3,
"window": "worker down, API and broker restart",
"seconds": 24.0,
"result": "PASS",
"observed": {
"attachment_id": "01a18508-cd97-737c-bb03-5409de83ff8e",
"timeline": [
{
"at": "2026-10-29T00:21:49.075959Z",
"step": "worker stopped"
},
{
"at": "2026-10-29T00:21:49.095967Z",
"step": "upload 202 queued",
"attachment_id": "01a18508-cd97-737c-bb03-5409de83ff8e"
},
{
"at": "2026-10-29T00:21:49.099982Z",
"step": "content 409 while queued"
},
{
"at": "2026-10-29T00:21:49.138263Z",
"step": "message with attachment 201, other messages work",
"message_id": "01a18508-cdad-708f-8324-18123e9c7758"
},
{
"at": "2026-10-29T00:21:54.782528Z",
"step": "job is in the broker queue",
"queues": {
"attachments.process": [
1,
0
],
"attachments.process.dlq": [
1,
0
]
}
},
{
"at": "2026-10-29T00:22:00.869187Z",
"step": "API replicas and relay restarted"
},
{
"at": "2026-10-29T00:22:09.698869Z",
"step": "broker restarted with its volume, job survived",
"queues": {
"attachments.process": [
1,
0
],
"attachments.process.dlq": [
1,
0
]
}
},
{
"at": "2026-10-29T00:22:12.427256Z",
"step": "worker started"
},
{
"at": "2026-10-29T00:22:12.707560Z",
"step": "ready",
"variants": [
"crop",
"optimized",
"thumbnail"
]
}
]
}
},
{
"name": "duplicate_after_ready",
"level": 3,
"window": "redelivery of a finished job",
"seconds": 7.4,
"result": "PASS",
"observed": {
"attachment_id": "01a18509-29ee-7116-b3df-b04caf1c48e7",
"events": [
{
"kind": "enqueued",
"attempt": 1
},
{
"kind": "claimed",
"attempt": 1
},
{
"kind": "done",
"attempt": 1
},
{
"kind": "duplicate_noop",
"attempt": 1
}
]
}
},
{
"name": "poison_image",
"level": 3,
"window": "permanent error without retries",
"seconds": 0.0,
"result": "PASS",
"observed": {
"rejected_by_api": 415
}
},
{
"name": "s3_outage_recovers",
"level": 3,
"window": "transient error with retry",
"seconds": 11.6,
"result": "PASS",
"observed": {
"attachment_id": "01a18509-4892-72ec-8f31-ea07c55239c7",
"jobs": [
{
"job_id": "01a18509-4898-70c0-9198-8af96f5da78b",
"pipeline_version": 1,
"status": "done",
"attempt": 2
}
]
}
},
{
"name": "retry_exhausted",
"level": 3,
"window": "transient error until JOB_MAX_ATTEMPTS",
"seconds": 23.4,
"result": "PASS",
"observed": {
"attachment_id": "01a18509-8268-718e-912d-c6718d5492e0",
"jobs": [
{
"job_id": "01a18509-826e-7736-8d1e-ba66a970dc8e",
"pipeline_version": 1,
"status": "failed",
"attempt": 5
}
]
}
}
],
"result": "PASS",
"finished_at": "2026-10-29T00:23:06.519678Z"
}