271 lines
6.7 KiB
JSON
271 lines
6.7 KiB
JSON
{
|
|
"suite": "lab5-pipeline-idempotency",
|
|
"started_at": "2026-11-06T09:35:15.796014Z",
|
|
"source_sha": "95511002b12510178a3cc2a22566b43e2b2b6c81",
|
|
"config": {
|
|
"base_url": "http://127.0.0.1:18086",
|
|
"level": 3,
|
|
"only": "",
|
|
"expect_variants": "thumbnail",
|
|
"max_attempts": 5,
|
|
"stall_seconds": 15.0,
|
|
"compose": "docker compose",
|
|
"compose_faults": "docker compose -f compose.yaml -f infra/compose.faults.yaml"
|
|
},
|
|
"invariants": [
|
|
{
|
|
"id": "terminal",
|
|
"statement": "every 202 reaches its expected terminal state",
|
|
"result": "PASS",
|
|
"details": {
|
|
"accepted": 5
|
|
}
|
|
},
|
|
{
|
|
"id": "single_effect",
|
|
"statement": "one variant set per (attachment_id, pipeline_version, variant)",
|
|
"result": "PASS",
|
|
"details": {
|
|
"rows": 0
|
|
}
|
|
},
|
|
{
|
|
"id": "no_ready_rollback",
|
|
"statement": "ready never returns to queued or processing",
|
|
"result": "PASS",
|
|
"details": {
|
|
"traced": 5
|
|
}
|
|
},
|
|
{
|
|
"id": "monotonic_publication",
|
|
"statement": "published_version only moves forward",
|
|
"result": "PASS",
|
|
"details": {
|
|
"sampled": 2
|
|
}
|
|
},
|
|
{
|
|
"id": "bytes_match_metadata",
|
|
"statement": "sha256, size and real dimensions match served bytes",
|
|
"result": "PASS",
|
|
"details": {
|
|
"verified": 4
|
|
}
|
|
},
|
|
{
|
|
"id": "bounded_attempts",
|
|
"statement": "attempt <= JOB_MAX_ATTEMPTS",
|
|
"result": "PASS",
|
|
"details": {
|
|
"rows": 0
|
|
}
|
|
},
|
|
{
|
|
"id": "no_orphans",
|
|
"statement": "no pending outbox, messages, dead letters, staging, objects",
|
|
"result": "PASS",
|
|
"details": {
|
|
"queues": {
|
|
"attachments.process": [
|
|
0,
|
|
0
|
|
],
|
|
"attachments.process.dlq": [
|
|
0,
|
|
0
|
|
]
|
|
},
|
|
"s3_objects": 17,
|
|
"scoped_attachments": 5
|
|
}
|
|
},
|
|
{
|
|
"id": "no_lost_202",
|
|
"statement": "every 202 is present in PostgreSQL and the API",
|
|
"result": "PASS",
|
|
"details": {
|
|
"accepted": 5
|
|
}
|
|
}
|
|
],
|
|
"accepted_ids": [
|
|
"01a1b036-7bdb-72a8-9a20-d784124b6f09",
|
|
"01a1b036-838f-7073-a603-58ef2ffca05d",
|
|
"01a1b036-d38a-7375-ba48-c82f1e984740",
|
|
"01a1b036-e669-7726-8ed5-eb948e03323e",
|
|
"01a1b037-1c4a-72b4-8027-95bdd74c185a"
|
|
],
|
|
"scenarios": [
|
|
{
|
|
"name": "smoke_pipeline",
|
|
"level": 3,
|
|
"window": "upload, ready, variants match metadata",
|
|
"seconds": 1.4,
|
|
"result": "PASS",
|
|
"observed": {
|
|
"attachment_id": "01a1b036-7bdb-72a8-9a20-d784124b6f09",
|
|
"variants": [
|
|
"crop",
|
|
"optimized",
|
|
"thumbnail"
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"name": "worker_down_broker_restart",
|
|
"level": 3,
|
|
"window": "worker down, API and broker restart",
|
|
"seconds": 21.1,
|
|
"result": "PASS",
|
|
"observed": {
|
|
"attachment_id": "01a1b036-838f-7073-a603-58ef2ffca05d",
|
|
"timeline": [
|
|
{
|
|
"at": "2026-11-06T09:35:25.067956Z",
|
|
"step": "worker stopped",
|
|
"containers": [
|
|
"mtusi2026lab6-worker-1 cd5b9bcf05a3 exited Exited (0) Less than a second ago"
|
|
]
|
|
},
|
|
{
|
|
"at": "2026-11-06T09:35:25.081364Z",
|
|
"step": "upload 202 queued",
|
|
"attachment_id": "01a1b036-838f-7073-a603-58ef2ffca05d"
|
|
},
|
|
{
|
|
"at": "2026-11-06T09:35:25.086007Z",
|
|
"step": "content 409 while queued"
|
|
},
|
|
{
|
|
"at": "2026-11-06T09:35:25.109208Z",
|
|
"step": "message with attachment 201, other messages work",
|
|
"message_id": "01a1b036-839e-71c7-9abc-6d747695610e"
|
|
},
|
|
{
|
|
"at": "2026-11-06T09:35:30.399187Z",
|
|
"step": "job is in the broker queue",
|
|
"queues": {
|
|
"attachments.process": [
|
|
1,
|
|
0
|
|
],
|
|
"attachments.process.dlq": [
|
|
0,
|
|
0
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"at": "2026-11-06T09:35:36.592722Z",
|
|
"step": "API replicas and relay restarted"
|
|
},
|
|
{
|
|
"at": "2026-11-06T09:35:42.601391Z",
|
|
"step": "broker restarted with its volume, job survived",
|
|
"queues": {
|
|
"attachments.process": [
|
|
1,
|
|
0
|
|
],
|
|
"attachments.process.dlq": [
|
|
0,
|
|
0
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"at": "2026-11-06T09:35:45.362335Z",
|
|
"step": "worker started"
|
|
},
|
|
{
|
|
"at": "2026-11-06T09:35:45.539899Z",
|
|
"step": "ready",
|
|
"variants": [
|
|
"crop",
|
|
"optimized",
|
|
"thumbnail"
|
|
]
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"name": "duplicate_after_ready",
|
|
"level": 3,
|
|
"window": "redelivery of a finished job",
|
|
"seconds": 4.4,
|
|
"result": "PASS",
|
|
"observed": {
|
|
"attachment_id": "01a1b036-d38a-7375-ba48-c82f1e984740",
|
|
"events": [
|
|
{
|
|
"kind": "enqueued",
|
|
"attempt": 1
|
|
},
|
|
{
|
|
"kind": "claimed",
|
|
"attempt": 1
|
|
},
|
|
{
|
|
"kind": "done",
|
|
"attempt": 1
|
|
},
|
|
{
|
|
"kind": "duplicate_noop",
|
|
"attempt": 1
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"name": "poison_image",
|
|
"level": 3,
|
|
"window": "permanent error without retries",
|
|
"seconds": 0.0,
|
|
"result": "PASS",
|
|
"observed": {
|
|
"rejected_by_api": 415
|
|
}
|
|
},
|
|
{
|
|
"name": "s3_outage_recovers",
|
|
"level": 3,
|
|
"window": "transient error with retry",
|
|
"seconds": 10.3,
|
|
"result": "PASS",
|
|
"observed": {
|
|
"attachment_id": "01a1b036-e669-7726-8ed5-eb948e03323e",
|
|
"jobs": [
|
|
{
|
|
"job_id": "01a1b036-e671-7294-8d09-37bc4619030b",
|
|
"pipeline_version": 1,
|
|
"status": "done",
|
|
"attempt": 2
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"name": "retry_exhausted",
|
|
"level": 3,
|
|
"window": "transient error until JOB_MAX_ATTEMPTS",
|
|
"seconds": 20.3,
|
|
"result": "PASS",
|
|
"observed": {
|
|
"attachment_id": "01a1b037-1c4a-72b4-8027-95bdd74c185a",
|
|
"jobs": [
|
|
{
|
|
"job_id": "01a1b037-1c57-7496-8cc0-61aa229fd7a2",
|
|
"pipeline_version": 1,
|
|
"status": "failed",
|
|
"attempt": 5
|
|
}
|
|
]
|
|
}
|
|
}
|
|
],
|
|
"result": "PASS",
|
|
"finished_at": "2026-11-06T09:36:32.454476Z"
|
|
}
|