Files
mtusi2026lab5/evidence/operator.txt
T
ArtyomandClaude Opus 5.5 573220e12a Record level 4 evidence and fill in the report
evidence/level4.json: all 17 scenarios of levels 3 and 4 and the eight
invariants pass on 0b85edf; evidence/operator.txt shows the failed
registry, a replay to ready, a noop replay, an expired replay and the
DLQ. Smoke, topology, level3, persistence, failover, lab 4 scenarios,
limits, student tests and contract checks are refreshed on the same
commit. REPORT.md describes the variants, worker limits, jitter, lease
renewal, crash points, replay and retention, adds rows 4.1-4.4, the
level 4 failure experiments and the updated limitations.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-29 03:39:20 +03:00

69 lines
4.8 KiB
Plaintext

date: 2026-10-29T00:32:35Z commit: 0b85edf
# ID — вложение, доведённое до failed retry_exhausted: pipeline_suite.py --only retry_exhausted
$ make failed # реестр failed-заданий (последние 20), последняя строка — JSON; первые три записи
[
{
"job_id": "01a18512-1b5f-75e8-afc5-4d8de5a1fd37",
"attachment_id": "01a18512-1b35-7151-b14a-b65248339365",
"pipeline_version": 1,
"attempt": 5,
"error_code": "retry_exhausted",
"last_error": "Unavailable",
"failed_at": "2026-10-29T00:32:12.404490+00:00"
},
{
"job_id": "01a1850e-f5c8-74cf-bb11-17f5660ff7cb",
"attachment_id": "01a1850e-f5b0-731a-b6a5-e50c724f4f69",
"pipeline_version": 1,
"attempt": 5,
"error_code": "retry_exhausted",
"last_error": "Unavailable",
"failed_at": "2026-10-29T00:28:43.569198+00:00"
},
{
"job_id": "01a1850e-8623-7161-af00-316420cb53db",
"attachment_id": "01a1850e-85dc-7779-9d26-df88e9a5fe8b",
"pipeline_version": 1,
"attempt": 5,
"error_code": "retry_exhausted",
"last_error": "Unavailable",
"failed_at": "2026-10-29T00:28:18.113299+00:00"
}
]
... всего 13
$ make replay ID=01a18512-1b35-7151-b14a-b65248339365
{"result": "replayed", "attachment_id": "01a18512-1b35-7151-b14a-b65248339365", "operator": "cli"}
$ psql: состояние вложения и задания через 6 с
status | error_code | job_id | job_status | attempt | staging_removed
--------+------------+--------------------------------------+------------+---------+-----------------
ready | | 01a18512-1b5f-75e8-afc5-4d8de5a1fd37 | done | 1 | t
(1 row)
$ make replay ID=01a18512-1b35-7151-b14a-b65248339365 # повтор для ready
{"result": "noop", "attachment_id": "01a18512-1b35-7151-b14a-b65248339365", "operator": "cli"}
$ psql: SELECT kind, attempt, detail FROM job_events WHERE attachment_id = ID ORDER BY id
kind | attempt | detail
-----------------+---------+------------------------------------------------------------------------------------------------------------
enqueued | 1 | {}
claimed | 1 | {}
retry_scheduled | 2 | {"error": "Unavailable", "delay_seconds": 0.554707, "next_attempt_at": "2026-10-29T00:32:00.190332+00:00"}
claimed | 2 | {}
retry_scheduled | 3 | {"error": "Unavailable", "delay_seconds": 1.70871, "next_attempt_at": "2026-10-29T00:32:05.029872+00:00"}
claimed | 3 | {}
retry_scheduled | 4 | {"error": "Unavailable", "delay_seconds": 2.197943, "next_attempt_at": "2026-10-29T00:32:07.534942+00:00"}
claimed | 4 | {}
retry_scheduled | 5 | {"error": "Unavailable", "delay_seconds": 4.2547, "next_attempt_at": "2026-10-29T00:32:12.125315+00:00"}
claimed | 5 | {}
failed | 5 | {"error": "Unavailable", "error_code": "retry_exhausted"}
replayed | 1 | {"operator": "cli", "error_code": "retry_exhausted"}
claimed | 1 | {}
done | 1 | {}
replay_noop | 1 | {"result": "noop", "operator": "cli"}
(15 rows)
$ make replay ID=01a1850a-e8d8-7122-ab63-13e85b328efe # failed, staging удалён по retention (сценарий failed_retention)
{"result": "expired", "attachment_id": "01a1850a-e8d8-7122-ab63-13e85b328efe", "operator": "cli"}
$ make dlq # DLQ без извлечения (ack_requeue_true)
71 true attachments.dlx attachment.process.v1 1 content_type: "application/json",delivery_mode: 2,headers: {"x-death":[{"count":1,"exchange":"attachments","queue":"attachments.process","reason":"rejected","routing-keys":["attachment.process.v1"],"time":1793233047}],"x-delivery-count":2,"x-first-death-exchange":"attachments","x-first-death-queue":"attachments.process","x-first-death-reason":"rejected","x-last-death-exchange":"attachments","x-last-death-queue":"attachments.process","x-last-death-reason":"rejected"},message_id: "63d240e5-f57d-485b-9cc7-32b477a90248",type: "attachment.process.v1", {"schema_version": 2, "job_id": "63d240e5-f57d-485b-9cc7-32b477a90248"} string
71 true attachments.dlx attachment.process.v1 0 content_type: "application/json",delivery_mode: 2,headers: {"x-death":[{"count":1,"exchange":"attachments","queue":"attachments.process","reason":"rejected","routing-keys":["attachment.process.v1"],"time":1793233636}],"x-delivery-count":1,"x-first-death-exchange":"attachments","x-first-death-queue":"attachments.process","x-first-death-reason":"rejected","x-last-death-exchange":"attachments","x-last-death-queue":"attachments.process","x-last-death-reason":"rejected"},message_id: "c9c23d22-98a8-40ab-a151-279a330433c9",type: "attachment.process.v1", {"schema_version": 2, "job_id": "c9c23d22-98a8-40ab-a151-279a330433c9"} string