Files
mtusi2026lab2/evidence/level3.json
T

271 lines
6.7 KiB
JSON

{
"suite": "lab5-pipeline-idempotency",
"started_at": "2026-11-06T09:35:15.796014Z",
"source_sha": "95511002b12510178a3cc2a22566b43e2b2b6c81",
"config": {
"base_url": "http://127.0.0.1:18086",
"level": 3,
"only": "",
"expect_variants": "thumbnail",
"max_attempts": 5,
"stall_seconds": 15.0,
"compose": "docker compose",
"compose_faults": "docker compose -f compose.yaml -f infra/compose.faults.yaml"
},
"invariants": [
{
"id": "terminal",
"statement": "every 202 reaches its expected terminal state",
"result": "PASS",
"details": {
"accepted": 5
}
},
{
"id": "single_effect",
"statement": "one variant set per (attachment_id, pipeline_version, variant)",
"result": "PASS",
"details": {
"rows": 0
}
},
{
"id": "no_ready_rollback",
"statement": "ready never returns to queued or processing",
"result": "PASS",
"details": {
"traced": 5
}
},
{
"id": "monotonic_publication",
"statement": "published_version only moves forward",
"result": "PASS",
"details": {
"sampled": 2
}
},
{
"id": "bytes_match_metadata",
"statement": "sha256, size and real dimensions match served bytes",
"result": "PASS",
"details": {
"verified": 4
}
},
{
"id": "bounded_attempts",
"statement": "attempt <= JOB_MAX_ATTEMPTS",
"result": "PASS",
"details": {
"rows": 0
}
},
{
"id": "no_orphans",
"statement": "no pending outbox, messages, dead letters, staging, objects",
"result": "PASS",
"details": {
"queues": {
"attachments.process": [
0,
0
],
"attachments.process.dlq": [
0,
0
]
},
"s3_objects": 17,
"scoped_attachments": 5
}
},
{
"id": "no_lost_202",
"statement": "every 202 is present in PostgreSQL and the API",
"result": "PASS",
"details": {
"accepted": 5
}
}
],
"accepted_ids": [
"01a1b036-7bdb-72a8-9a20-d784124b6f09",
"01a1b036-838f-7073-a603-58ef2ffca05d",
"01a1b036-d38a-7375-ba48-c82f1e984740",
"01a1b036-e669-7726-8ed5-eb948e03323e",
"01a1b037-1c4a-72b4-8027-95bdd74c185a"
],
"scenarios": [
{
"name": "smoke_pipeline",
"level": 3,
"window": "upload, ready, variants match metadata",
"seconds": 1.4,
"result": "PASS",
"observed": {
"attachment_id": "01a1b036-7bdb-72a8-9a20-d784124b6f09",
"variants": [
"crop",
"optimized",
"thumbnail"
]
}
},
{
"name": "worker_down_broker_restart",
"level": 3,
"window": "worker down, API and broker restart",
"seconds": 21.1,
"result": "PASS",
"observed": {
"attachment_id": "01a1b036-838f-7073-a603-58ef2ffca05d",
"timeline": [
{
"at": "2026-11-06T09:35:25.067956Z",
"step": "worker stopped",
"containers": [
"mtusi2026lab6-worker-1 cd5b9bcf05a3 exited Exited (0) Less than a second ago"
]
},
{
"at": "2026-11-06T09:35:25.081364Z",
"step": "upload 202 queued",
"attachment_id": "01a1b036-838f-7073-a603-58ef2ffca05d"
},
{
"at": "2026-11-06T09:35:25.086007Z",
"step": "content 409 while queued"
},
{
"at": "2026-11-06T09:35:25.109208Z",
"step": "message with attachment 201, other messages work",
"message_id": "01a1b036-839e-71c7-9abc-6d747695610e"
},
{
"at": "2026-11-06T09:35:30.399187Z",
"step": "job is in the broker queue",
"queues": {
"attachments.process": [
1,
0
],
"attachments.process.dlq": [
0,
0
]
}
},
{
"at": "2026-11-06T09:35:36.592722Z",
"step": "API replicas and relay restarted"
},
{
"at": "2026-11-06T09:35:42.601391Z",
"step": "broker restarted with its volume, job survived",
"queues": {
"attachments.process": [
1,
0
],
"attachments.process.dlq": [
0,
0
]
}
},
{
"at": "2026-11-06T09:35:45.362335Z",
"step": "worker started"
},
{
"at": "2026-11-06T09:35:45.539899Z",
"step": "ready",
"variants": [
"crop",
"optimized",
"thumbnail"
]
}
]
}
},
{
"name": "duplicate_after_ready",
"level": 3,
"window": "redelivery of a finished job",
"seconds": 4.4,
"result": "PASS",
"observed": {
"attachment_id": "01a1b036-d38a-7375-ba48-c82f1e984740",
"events": [
{
"kind": "enqueued",
"attempt": 1
},
{
"kind": "claimed",
"attempt": 1
},
{
"kind": "done",
"attempt": 1
},
{
"kind": "duplicate_noop",
"attempt": 1
}
]
}
},
{
"name": "poison_image",
"level": 3,
"window": "permanent error without retries",
"seconds": 0.0,
"result": "PASS",
"observed": {
"rejected_by_api": 415
}
},
{
"name": "s3_outage_recovers",
"level": 3,
"window": "transient error with retry",
"seconds": 10.3,
"result": "PASS",
"observed": {
"attachment_id": "01a1b036-e669-7726-8ed5-eb948e03323e",
"jobs": [
{
"job_id": "01a1b036-e671-7294-8d09-37bc4619030b",
"pipeline_version": 1,
"status": "done",
"attempt": 2
}
]
}
},
{
"name": "retry_exhausted",
"level": 3,
"window": "transient error until JOB_MAX_ATTEMPTS",
"seconds": 20.3,
"result": "PASS",
"observed": {
"attachment_id": "01a1b037-1c4a-72b4-8027-95bdd74c185a",
"jobs": [
{
"job_id": "01a1b037-1c57-7496-8cc0-61aa229fd7a2",
"pipeline_version": 1,
"status": "failed",
"attempt": 5
}
]
}
}
],
"result": "PASS",
"finished_at": "2026-11-06T09:36:32.454476Z"
}