Record clean deployment, JPEG checks and bounded container memory

This commit is contained in:
2026-10-16 22:07:58 +03:00
parent 4cfaab25b4
commit 47e6737654
3 changed files with 806 additions and 0 deletions
+17
View File
@@ -149,6 +149,23 @@ orphan от прерванного предварительного запуск
проходов с разными действиями, а не статистически независимый benchmark. На большом
bucket понадобится разбиение по prefix и контроль частоты; grace и locks сохраняются.
## Чистая копия и исправления первого запуска
Повторный `git clone --local` в отдельный каталог с новыми PG/S3 volumes и secrets:
build/bootstrap/migrations/ready и полный `verify_stack.py` прошли. Доказательства:
`evidence/clean-clone-integration.json`, `evidence/clean-clone-restart.json`.
Дополнительно проверены валидный JPEG побайтно и обрезанный JPEG → 415.
Failover: 115 успешных записей, 2 временные ошибки, восстановление 4.328 с.
Все 10 загрузок завершились; cgroup v1 peak **151740416 / 162308096 байт**
(144.71 / 154.79 MiB), `failcnt=0`, `oom_kill=0` на обеих репликах.
На пустом S3 первый PUT может ждать создания volumes: bootstrap теперь ограниченно
повторяет операцию с фиксированным служебным ключом и проверяет чтение bytes.
Первый запуск выявил также необходимость штатного nonroot uid 1000 для S3 при
drop ALL caps и проблему параллельной сборки одинакового тега у нескольких сервисов:
`build` объявлен только у API, остальные используют общий готовый образ.
Сбор памяти поддерживает cgroup v1 и v2, поскольку этот Docker/WSL использует v1.
## Ограничения и источники
Локальный компьютер: Windows 10, i5-11400H, 8 GB RAM; WSL2 3 GB RAM/4 CPU/4 GB swap.
+198
View File
@@ -0,0 +1,198 @@
{
"public_smoke": "PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.",
"negative_uploads": {
"oversize": 413,
"corrupt": 415,
"truncated": 415,
"mismatch": 415,
"unsupported": 415,
"path": 400,
"backslash": 400,
"control": 400,
"pixels": 413
},
"jpeg_validation": "valid JPEG returned byte-for-byte; truncated entropy stream rejected with 415",
"attachment_acl_and_canonical_body": "foreign/unknown attachments 404; absent and [] attachment_ids replay same message",
"private_bucket": {
"anonymous_get": 403,
"api_create_other_bucket": 403,
"api_write_outside_prefix": 403
},
"upload_then_stop_download": {
"stopped_container": "b0a8d4d6c153c378aadf6303f33846b7fab572564fc36920eb6a1c3e1bed62aa",
"uploader_instance": "d1629e5d4ce04df4bd4ea3d1ae198213",
"downloader_instance": "da42b3ba785949008f795e135670b728",
"sha256_matches": true
},
"interrupted_upload": "partial body closed: no attachment metadata",
"s3_down_status": 503,
"bootstrap_repeat": "Private bucket ready; bootstrap is repeatable",
"public_persistence": "Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.",
"public_failover": "Executing explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 115, \"failed_attempts\": 2, \"first_recovery_seconds\": 4.328}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.",
"concurrent_uploads": {
"clients": 10,
"bytes_each": 8104028,
"before_memory_kib": {
"b0a8d4d6c153c378aadf6303f33846b7fab572564fc36920eb6a1c3e1bed62aa": {
"VmRSS": 94248,
"VmHWM": 119968,
"cgroup": {
"version": 1,
"peak_bytes": 105979904,
"failcnt": 0,
"oom_control": "oom_kill_disable 0\nunder_oom 0\noom_kill 0\n"
}
},
"395705f22be6d83f694cc98e96fe5bfe628151af93fd5f1e939704ff21fc5dc8": {
"VmRSS": 98060,
"VmHWM": 129428,
"cgroup": {
"version": 1,
"peak_bytes": 108998656,
"failcnt": 0,
"oom_control": "oom_kill_disable 0\nunder_oom 0\noom_kill 0\n"
}
}
},
"after_memory_kib": {
"b0a8d4d6c153c378aadf6303f33846b7fab572564fc36920eb6a1c3e1bed62aa": {
"VmRSS": 134000,
"VmHWM": 134192,
"cgroup": {
"version": 1,
"peak_bytes": 151740416,
"failcnt": 0,
"oom_control": "oom_kill_disable 0\nunder_oom 0\noom_kill 0\n"
}
},
"395705f22be6d83f694cc98e96fe5bfe628151af93fd5f1e939704ff21fc5dc8": {
"VmRSS": 144808,
"VmHWM": 145008,
"cgroup": {
"version": 1,
"peak_bytes": 162308096,
"failcnt": 0,
"oom_control": "oom_kill_disable 0\nunder_oom 0\noom_kill 0\n"
}
}
},
"uploads": [
{
"i": 0,
"seconds": 4.813,
"statuses": [
201
]
},
{
"i": 1,
"seconds": 6.912,
"statuses": [
503,
503,
503,
503,
503,
503,
503,
503,
503,
503,
503,
201
]
},
{
"i": 2,
"seconds": 6.376,
"statuses": [
503,
503,
503,
503,
503,
201
]
},
{
"i": 3,
"seconds": 7.03,
"statuses": [
503,
503,
503,
503,
503,
503,
503,
503,
503,
503,
503,
201
]
},
{
"i": 4,
"seconds": 6.32,
"statuses": [
503,
503,
503,
503,
503,
503,
201
]
},
{
"i": 5,
"seconds": 4.5,
"statuses": [
201
]
},
{
"i": 6,
"seconds": 6.258,
"statuses": [
503,
503,
503,
503,
503,
503,
201
]
},
{
"i": 7,
"seconds": 4.847,
"statuses": [
201
]
},
{
"i": 8,
"seconds": 4.534,
"statuses": [
201
]
},
{
"i": 9,
"seconds": 6.628,
"statuses": [
503,
503,
503,
503,
503,
503,
201
]
}
]
},
"passed": true
}
+591
View File
@@ -0,0 +1,591 @@
{
"before": {
"b0a8d4d6c153c378aadf6303f33846b7fab572564fc36920eb6a1c3e1bed62aa": {
"service": "api",
"state": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 89629,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-10-16T19:04:51.580983558Z",
"FinishedAt": "2026-10-16T19:04:50.691195534Z",
"Health": {
"Status": "healthy",
"FailingStreak": 0,
"Log": [
{
"Start": "2026-10-16T19:05:20.211235074Z",
"End": "2026-10-16T19:05:20.652084583Z",
"ExitCode": 1,
"Output": "Traceback (most recent call last):\n File \"<string>\", line 1, in <module>\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n"
},
{
"Start": "2026-10-16T19:05:25.652679198Z",
"End": "2026-10-16T19:05:26.123505815Z",
"ExitCode": 1,
"Output": "Traceback (most recent call last):\n File \"<string>\", line 1, in <module>\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n"
},
{
"Start": "2026-10-16T19:05:31.126069514Z",
"End": "2026-10-16T19:05:31.609616392Z",
"ExitCode": 1,
"Output": "Traceback (most recent call last):\n File \"<string>\", line 1, in <module>\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n"
},
{
"Start": "2026-10-16T19:05:37.925954313Z",
"End": "2026-10-16T19:05:38.361362305Z",
"ExitCode": 1,
"Output": "Traceback (most recent call last):\n File \"<string>\", line 1, in <module>\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n"
},
{
"Start": "2026-10-16T19:05:43.362382377Z",
"End": "2026-10-16T19:05:43.839170887Z",
"ExitCode": 0,
"Output": ""
}
]
}
}
},
"395705f22be6d83f694cc98e96fe5bfe628151af93fd5f1e939704ff21fc5dc8": {
"service": "api",
"state": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 57715,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-10-16T14:21:40.202839878Z",
"FinishedAt": "2026-10-16T14:21:39.839302297Z",
"Health": {
"Status": "healthy",
"FailingStreak": 0,
"Log": [
{
"Start": "2026-10-16T19:05:21.557583747Z",
"End": "2026-10-16T19:05:22.142090738Z",
"ExitCode": 1,
"Output": "Traceback (most recent call last):\n File \"<string>\", line 1, in <module>\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n"
},
{
"Start": "2026-10-16T19:05:27.143338283Z",
"End": "2026-10-16T19:05:27.622167618Z",
"ExitCode": 1,
"Output": "Traceback (most recent call last):\n File \"<string>\", line 1, in <module>\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n"
},
{
"Start": "2026-10-16T19:05:32.623688976Z",
"End": "2026-10-16T19:05:33.046515405Z",
"ExitCode": 1,
"Output": "Traceback (most recent call last):\n File \"<string>\", line 1, in <module>\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n"
},
{
"Start": "2026-10-16T19:05:39.363083138Z",
"End": "2026-10-16T19:05:39.842740067Z",
"ExitCode": 1,
"Output": "Traceback (most recent call last):\n File \"<string>\", line 1, in <module>\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n"
},
{
"Start": "2026-10-16T19:05:44.84414073Z",
"End": "2026-10-16T19:05:45.339352229Z",
"ExitCode": 0,
"Output": ""
}
]
}
}
},
"2441f99f958bbd516d0ab7b93eb96e09b7f875a4164f9ff9d645a9aede2dc6c5": {
"service": "db",
"state": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 57338,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-10-16T14:21:37.770464288Z",
"FinishedAt": "2026-10-16T14:21:37.196628506Z",
"Health": {
"Status": "healthy",
"FailingStreak": 0,
"Log": [
{
"Start": "2026-10-16T19:05:38.745087502Z",
"End": "2026-10-16T19:05:38.789808433Z",
"ExitCode": 0,
"Output": "/var/run/postgresql:5432 - accepting connections\n"
},
{
"Start": "2026-10-16T19:05:40.790459849Z",
"End": "2026-10-16T19:05:40.835150776Z",
"ExitCode": 0,
"Output": "/var/run/postgresql:5432 - accepting connections\n"
},
{
"Start": "2026-10-16T19:05:42.836652693Z",
"End": "2026-10-16T19:05:42.881695222Z",
"ExitCode": 0,
"Output": "/var/run/postgresql:5432 - accepting connections\n"
},
{
"Start": "2026-10-16T19:05:44.882830897Z",
"End": "2026-10-16T19:05:44.932299003Z",
"ExitCode": 0,
"Output": "/var/run/postgresql:5432 - accepting connections\n"
},
{
"Start": "2026-10-16T19:05:46.933638229Z",
"End": "2026-10-16T19:05:46.99111665Z",
"ExitCode": 0,
"Output": "/var/run/postgresql:5432 - accepting connections\n"
}
]
}
}
},
"e4fb86e6ec85e6c96ae794ba39e8fc93d179bada127bfb8cebe4f5a5361877d1": {
"service": "cache",
"state": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 57302,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-10-16T14:21:37.642224221Z",
"FinishedAt": "2026-10-16T14:21:37.158949295Z",
"Health": {
"Status": "healthy",
"FailingStreak": 0,
"Log": [
{
"Start": "2026-10-16T19:05:39.218402845Z",
"End": "2026-10-16T19:05:39.23869686Z",
"ExitCode": 0,
"Output": "PONG\n"
},
{
"Start": "2026-10-16T19:05:41.239357605Z",
"End": "2026-10-16T19:05:41.259293587Z",
"ExitCode": 0,
"Output": "PONG\n"
},
{
"Start": "2026-10-16T19:05:43.261038537Z",
"End": "2026-10-16T19:05:43.287970539Z",
"ExitCode": 0,
"Output": "PONG\n"
},
{
"Start": "2026-10-16T19:05:45.28872912Z",
"End": "2026-10-16T19:05:45.310829129Z",
"ExitCode": 0,
"Output": "PONG\n"
},
{
"Start": "2026-10-16T19:05:47.311317596Z",
"End": "2026-10-16T19:05:47.331220589Z",
"ExitCode": 0,
"Output": "PONG\n"
}
]
}
}
},
"dad1c78ccfbd99a5f2e5ee467c3216b81bb70dd4a9d9bfa957a1514050e762bc": {
"service": "lb",
"state": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 57396,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-10-16T14:21:37.890968447Z",
"FinishedAt": "2026-10-16T14:21:37.097695286Z"
}
},
"6e8f2c3c72100a9e08ad6bd157697931fcb23b7846c5929a86033ee1e5f5a978": {
"service": "s3",
"state": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 90601,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-10-16T19:05:21.285471656Z",
"FinishedAt": "2026-10-16T19:05:20.730580171Z"
}
},
"572e906935d7ec6f3dd28d51caaee7d25696558344f288631ba92977b5f3bc45": {
"service": "reconciler",
"state": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 57604,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-10-16T14:21:39.595976344Z",
"FinishedAt": "2026-10-16T14:21:39.251848171Z",
"Health": {
"Status": "healthy",
"FailingStreak": 0,
"Log": [
{
"Start": "2026-10-16T19:04:58.650013908Z",
"End": "2026-10-16T19:04:58.687567121Z",
"ExitCode": 0,
"Output": ""
},
{
"Start": "2026-10-16T19:05:10.130199277Z",
"End": "2026-10-16T19:05:10.161910968Z",
"ExitCode": 0,
"Output": ""
},
{
"Start": "2026-10-16T19:05:20.162424877Z",
"End": "2026-10-16T19:05:20.192425192Z",
"ExitCode": 0,
"Output": ""
},
{
"Start": "2026-10-16T19:05:30.193378293Z",
"End": "2026-10-16T19:05:30.223733363Z",
"ExitCode": 0,
"Output": ""
},
{
"Start": "2026-10-16T19:05:41.540431923Z",
"End": "2026-10-16T19:05:41.575560046Z",
"ExitCode": 0,
"Output": ""
}
]
}
}
}
},
"after": {
"b0a8d4d6c153c378aadf6303f33846b7fab572564fc36920eb6a1c3e1bed62aa": {
"service": "api",
"state": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 91908,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-10-16T19:05:51.503236355Z",
"FinishedAt": "2026-10-16T19:05:51.051892302Z",
"Health": {
"Status": "starting",
"FailingStreak": 0,
"Log": [
{
"Start": "2026-10-16T19:05:43.362382377Z",
"End": "2026-10-16T19:05:43.839170887Z",
"ExitCode": 0,
"Output": ""
},
{
"Start": "2026-10-16T19:05:48.839798292Z",
"End": "2026-10-16T19:05:49.453801773Z",
"ExitCode": 0,
"Output": ""
},
{
"Start": "2026-10-16T19:05:56.761820425Z",
"End": "2026-10-16T19:05:57.223778281Z",
"ExitCode": 1,
"Output": "Traceback (most recent call last):\n File \"<string>\", line 1, in <module>\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n"
},
{
"Start": "2026-10-16T19:06:02.22498739Z",
"End": "2026-10-16T19:06:02.671587739Z",
"ExitCode": 1,
"Output": "Traceback (most recent call last):\n File \"<string>\", line 1, in <module>\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n"
},
{
"Start": "2026-10-16T19:06:09.099465314Z",
"End": "2026-10-16T19:06:09.563981298Z",
"ExitCode": 1,
"Output": "Traceback (most recent call last):\n File \"<string>\", line 1, in <module>\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n"
}
]
}
}
},
"395705f22be6d83f694cc98e96fe5bfe628151af93fd5f1e939704ff21fc5dc8": {
"service": "api",
"state": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 92149,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-10-16T19:05:53.745039633Z",
"FinishedAt": "2026-10-16T19:05:53.471949599Z",
"Health": {
"Status": "starting",
"FailingStreak": 0,
"Log": [
{
"Start": "2026-10-16T19:05:39.363083138Z",
"End": "2026-10-16T19:05:39.842740067Z",
"ExitCode": 1,
"Output": "Traceback (most recent call last):\n File \"<string>\", line 1, in <module>\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n"
},
{
"Start": "2026-10-16T19:05:44.84414073Z",
"End": "2026-10-16T19:05:45.339352229Z",
"ExitCode": 0,
"Output": ""
},
{
"Start": "2026-10-16T19:05:50.339801368Z",
"End": "2026-10-16T19:05:52.041595819Z",
"ExitCode": 1,
"Output": "Traceback (most recent call last):\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 1357, in do_open\n h.request(req.get_method(), req.selector, req.data, headers,\n File \"/usr/local/lib/python3.12/http/client.py\", line 1386, in request\n self._send_request(method, url, body, headers, encode_chunked)\n File \"/usr/local/lib/python3.12/http/client.py\", line 1432, in _send_request\n self.endheaders(body, encode_chunked=encode_chunked)\n File \"/usr/local/lib/python3.12/http/client.py\", line 1381, in endheaders\n self._send_output(message_body, encode_chunked=encode_chunked)\n File \"/usr/local/lib/python3.12/http/client.py\", line 1141, in _send_output\n self.send(msg)\n File \"/usr/local/lib/python3.12/http/client.py\", line 1085, in send\n self.connect()\n File \"/usr/local/lib/python3.12/http/client.py\", line 1051, in connect\n self.sock = self._create_connection(\n ^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/socket.py\", line 865, in create_connection\n raise exceptions[0]\n File \"/usr/local/lib/python3.12/socket.py\", line 850, in create_connection\n sock.connect(sa)\nConnectionRefusedError: [Errno 111] Connection refused\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File \"<string>\", line 1, in <module>\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 515, in open\n response = self._open(req, data)\n ^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 532, in _open\n result = self._call_chain(self.handle_open, protocol, protocol +\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 1386, in http_open\n return self.do_open(http.client.HTTPConnection, req)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 1360, in do_open\n raise URLError(err)\nurllib.error.URLError: <urlopen error [Errno 111] Connection refused>\n"
},
{
"Start": "2026-10-16T19:05:58.87057203Z",
"End": "2026-10-16T19:05:59.550549126Z",
"ExitCode": 1,
"Output": "Traceback (most recent call last):\n File \"<string>\", line 1, in <module>\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n"
},
{
"Start": "2026-10-16T19:06:05.214084373Z",
"End": "2026-10-16T19:06:06.387122841Z",
"ExitCode": 1,
"Output": "Traceback (most recent call last):\n File \"<string>\", line 1, in <module>\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n"
}
]
}
}
},
"2441f99f958bbd516d0ab7b93eb96e09b7f875a4164f9ff9d645a9aede2dc6c5": {
"service": "db",
"state": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 91717,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-10-16T19:05:50.585367281Z",
"FinishedAt": "2026-10-16T19:05:50.114533944Z",
"Health": {
"Status": "healthy",
"FailingStreak": 0,
"Log": [
{
"Start": "2026-10-16T19:06:01.764202467Z",
"End": "2026-10-16T19:06:01.803981152Z",
"ExitCode": 0,
"Output": "/var/run/postgresql:5432 - accepting connections\n"
},
{
"Start": "2026-10-16T19:06:03.805033747Z",
"End": "2026-10-16T19:06:03.844587109Z",
"ExitCode": 0,
"Output": "/var/run/postgresql:5432 - accepting connections\n"
},
{
"Start": "2026-10-16T19:06:07.273151038Z",
"End": "2026-10-16T19:06:07.316174892Z",
"ExitCode": 0,
"Output": "/var/run/postgresql:5432 - accepting connections\n"
},
{
"Start": "2026-10-16T19:06:09.316906087Z",
"End": "2026-10-16T19:06:09.357124824Z",
"ExitCode": 0,
"Output": "/var/run/postgresql:5432 - accepting connections\n"
},
{
"Start": "2026-10-16T19:06:11.358455466Z",
"End": "2026-10-16T19:06:11.402615578Z",
"ExitCode": 0,
"Output": "/var/run/postgresql:5432 - accepting connections\n"
}
]
}
}
},
"e4fb86e6ec85e6c96ae794ba39e8fc93d179bada127bfb8cebe4f5a5361877d1": {
"service": "cache",
"state": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 91752,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-10-16T19:05:50.786619159Z",
"FinishedAt": "2026-10-16T19:05:50.084558012Z",
"Health": {
"Status": "healthy",
"FailingStreak": 0,
"Log": [
{
"Start": "2026-10-16T19:06:01.623734818Z",
"End": "2026-10-16T19:06:01.646386929Z",
"ExitCode": 0,
"Output": "PONG\n"
},
{
"Start": "2026-10-16T19:06:03.646921834Z",
"End": "2026-10-16T19:06:03.668289714Z",
"ExitCode": 0,
"Output": "PONG\n"
},
{
"Start": "2026-10-16T19:06:07.09579857Z",
"End": "2026-10-16T19:06:07.114392094Z",
"ExitCode": 0,
"Output": "PONG\n"
},
{
"Start": "2026-10-16T19:06:09.115447984Z",
"End": "2026-10-16T19:06:09.135323666Z",
"ExitCode": 0,
"Output": "PONG\n"
},
{
"Start": "2026-10-16T19:06:11.13641835Z",
"End": "2026-10-16T19:06:11.15433417Z",
"ExitCode": 0,
"Output": "PONG\n"
}
]
}
}
},
"dad1c78ccfbd99a5f2e5ee467c3216b81bb70dd4a9d9bfa957a1514050e762bc": {
"service": "lb",
"state": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 91827,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-10-16T19:05:51.121747424Z",
"FinishedAt": "2026-10-16T19:05:50.233763703Z"
}
},
"6e8f2c3c72100a9e08ad6bd157697931fcb23b7846c5929a86033ee1e5f5a978": {
"service": "s3",
"state": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 92676,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-10-16T19:06:11.831976196Z",
"FinishedAt": "2026-10-16T19:06:11.505939192Z"
}
},
"572e906935d7ec6f3dd28d51caaee7d25696558344f288631ba92977b5f3bc45": {
"service": "reconciler",
"state": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 92038,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-10-16T19:05:52.824086086Z",
"FinishedAt": "2026-10-16T19:05:52.546565655Z",
"Health": {
"Status": "healthy",
"FailingStreak": 0,
"Log": [
{
"Start": "2026-10-16T19:05:30.193378293Z",
"End": "2026-10-16T19:05:30.223733363Z",
"ExitCode": 0,
"Output": ""
},
{
"Start": "2026-10-16T19:05:41.540431923Z",
"End": "2026-10-16T19:05:41.575560046Z",
"ExitCode": 0,
"Output": ""
},
{
"Start": "2026-10-16T19:05:51.576781033Z",
"End": "2026-10-16T19:05:51.823968089Z",
"ExitCode": 0,
"Output": ""
},
{
"Start": "2026-10-16T19:05:57.973400351Z",
"End": "2026-10-16T19:05:58.003734353Z",
"ExitCode": 0,
"Output": ""
},
{
"Start": "2026-10-16T19:06:09.432278398Z",
"End": "2026-10-16T19:06:09.46125484Z",
"ExitCode": 0,
"Output": ""
}
]
}
}
}
},
"volumes_removed": false
}