From 47e67376549329ae4634f3452e776ade2238bdb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=93=D0=BB=D0=B5=D0=B1=20=D0=9F=D0=B0=D1=80=D0=B0=D0=BC?= =?UTF-8?q?=D0=BE=D0=BD=D0=BE=D0=B2?= Date: Fri, 16 Oct 2026 22:07:58 +0300 Subject: [PATCH] Record clean deployment, JPEG checks and bounded container memory --- REPORT.md | 17 + evidence/clean-clone-integration.json | 198 +++++++++ evidence/clean-clone-restart.json | 591 ++++++++++++++++++++++++++ 3 files changed, 806 insertions(+) create mode 100644 evidence/clean-clone-integration.json create mode 100644 evidence/clean-clone-restart.json diff --git a/REPORT.md b/REPORT.md index ccb2445e..7356ef2a 100644 --- a/REPORT.md +++ b/REPORT.md @@ -149,6 +149,23 @@ orphan от прерванного предварительного запуск проходов с разными действиями, а не статистически независимый benchmark. На большом bucket понадобится разбиение по prefix и контроль частоты; grace и locks сохраняются. +## Чистая копия и исправления первого запуска + +Повторный `git clone --local` в отдельный каталог с новыми PG/S3 volumes и secrets: +build/bootstrap/migrations/ready и полный `verify_stack.py` прошли. Доказательства: +`evidence/clean-clone-integration.json`, `evidence/clean-clone-restart.json`. +Дополнительно проверены валидный JPEG побайтно и обрезанный JPEG → 415. +Failover: 115 успешных записей, 2 временные ошибки, восстановление 4.328 с. +Все 10 загрузок завершились; cgroup v1 peak **151740416 / 162308096 байт** +(144.71 / 154.79 MiB), `failcnt=0`, `oom_kill=0` на обеих репликах. + +На пустом S3 первый PUT может ждать создания volumes: bootstrap теперь ограниченно +повторяет операцию с фиксированным служебным ключом и проверяет чтение bytes. +Первый запуск выявил также необходимость штатного nonroot uid 1000 для S3 при +drop ALL caps и проблему параллельной сборки одинакового тега у нескольких сервисов: +`build` объявлен только у API, остальные используют общий готовый образ. +Сбор памяти поддерживает cgroup v1 и v2, поскольку этот Docker/WSL использует v1. + ## Ограничения и источники Локальный компьютер: Windows 10, i5-11400H, 8 GB RAM; WSL2 3 GB RAM/4 CPU/4 GB swap. diff --git a/evidence/clean-clone-integration.json b/evidence/clean-clone-integration.json new file mode 100644 index 00000000..8eeed9e3 --- /dev/null +++ b/evidence/clean-clone-integration.json @@ -0,0 +1,198 @@ +{ + "public_smoke": "PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.", + "negative_uploads": { + "oversize": 413, + "corrupt": 415, + "truncated": 415, + "mismatch": 415, + "unsupported": 415, + "path": 400, + "backslash": 400, + "control": 400, + "pixels": 413 + }, + "jpeg_validation": "valid JPEG returned byte-for-byte; truncated entropy stream rejected with 415", + "attachment_acl_and_canonical_body": "foreign/unknown attachments 404; absent and [] attachment_ids replay same message", + "private_bucket": { + "anonymous_get": 403, + "api_create_other_bucket": 403, + "api_write_outside_prefix": 403 + }, + "upload_then_stop_download": { + "stopped_container": "b0a8d4d6c153c378aadf6303f33846b7fab572564fc36920eb6a1c3e1bed62aa", + "uploader_instance": "d1629e5d4ce04df4bd4ea3d1ae198213", + "downloader_instance": "da42b3ba785949008f795e135670b728", + "sha256_matches": true + }, + "interrupted_upload": "partial body closed: no attachment metadata", + "s3_down_status": 503, + "bootstrap_repeat": "Private bucket ready; bootstrap is repeatable", + "public_persistence": "Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.", + "public_failover": "Executing explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 115, \"failed_attempts\": 2, \"first_recovery_seconds\": 4.328}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.", + "concurrent_uploads": { + "clients": 10, + "bytes_each": 8104028, + "before_memory_kib": { + "b0a8d4d6c153c378aadf6303f33846b7fab572564fc36920eb6a1c3e1bed62aa": { + "VmRSS": 94248, + "VmHWM": 119968, + "cgroup": { + "version": 1, + "peak_bytes": 105979904, + "failcnt": 0, + "oom_control": "oom_kill_disable 0\nunder_oom 0\noom_kill 0\n" + } + }, + "395705f22be6d83f694cc98e96fe5bfe628151af93fd5f1e939704ff21fc5dc8": { + "VmRSS": 98060, + "VmHWM": 129428, + "cgroup": { + "version": 1, + "peak_bytes": 108998656, + "failcnt": 0, + "oom_control": "oom_kill_disable 0\nunder_oom 0\noom_kill 0\n" + } + } + }, + "after_memory_kib": { + "b0a8d4d6c153c378aadf6303f33846b7fab572564fc36920eb6a1c3e1bed62aa": { + "VmRSS": 134000, + "VmHWM": 134192, + "cgroup": { + "version": 1, + "peak_bytes": 151740416, + "failcnt": 0, + "oom_control": "oom_kill_disable 0\nunder_oom 0\noom_kill 0\n" + } + }, + "395705f22be6d83f694cc98e96fe5bfe628151af93fd5f1e939704ff21fc5dc8": { + "VmRSS": 144808, + "VmHWM": 145008, + "cgroup": { + "version": 1, + "peak_bytes": 162308096, + "failcnt": 0, + "oom_control": "oom_kill_disable 0\nunder_oom 0\noom_kill 0\n" + } + } + }, + "uploads": [ + { + "i": 0, + "seconds": 4.813, + "statuses": [ + 201 + ] + }, + { + "i": 1, + "seconds": 6.912, + "statuses": [ + 503, + 503, + 503, + 503, + 503, + 503, + 503, + 503, + 503, + 503, + 503, + 201 + ] + }, + { + "i": 2, + "seconds": 6.376, + "statuses": [ + 503, + 503, + 503, + 503, + 503, + 201 + ] + }, + { + "i": 3, + "seconds": 7.03, + "statuses": [ + 503, + 503, + 503, + 503, + 503, + 503, + 503, + 503, + 503, + 503, + 503, + 201 + ] + }, + { + "i": 4, + "seconds": 6.32, + "statuses": [ + 503, + 503, + 503, + 503, + 503, + 503, + 201 + ] + }, + { + "i": 5, + "seconds": 4.5, + "statuses": [ + 201 + ] + }, + { + "i": 6, + "seconds": 6.258, + "statuses": [ + 503, + 503, + 503, + 503, + 503, + 503, + 201 + ] + }, + { + "i": 7, + "seconds": 4.847, + "statuses": [ + 201 + ] + }, + { + "i": 8, + "seconds": 4.534, + "statuses": [ + 201 + ] + }, + { + "i": 9, + "seconds": 6.628, + "statuses": [ + 503, + 503, + 503, + 503, + 503, + 503, + 201 + ] + } + ] + }, + "passed": true +} diff --git a/evidence/clean-clone-restart.json b/evidence/clean-clone-restart.json new file mode 100644 index 00000000..27b8d0a1 --- /dev/null +++ b/evidence/clean-clone-restart.json @@ -0,0 +1,591 @@ +{ + "before": { + "b0a8d4d6c153c378aadf6303f33846b7fab572564fc36920eb6a1c3e1bed62aa": { + "service": "api", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 89629, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:04:51.580983558Z", + "FinishedAt": "2026-10-16T19:04:50.691195534Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T19:05:20.211235074Z", + "End": "2026-10-16T19:05:20.652084583Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T19:05:25.652679198Z", + "End": "2026-10-16T19:05:26.123505815Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T19:05:31.126069514Z", + "End": "2026-10-16T19:05:31.609616392Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T19:05:37.925954313Z", + "End": "2026-10-16T19:05:38.361362305Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T19:05:43.362382377Z", + "End": "2026-10-16T19:05:43.839170887Z", + "ExitCode": 0, + "Output": "" + } + ] + } + } + }, + "395705f22be6d83f694cc98e96fe5bfe628151af93fd5f1e939704ff21fc5dc8": { + "service": "api", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 57715, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T14:21:40.202839878Z", + "FinishedAt": "2026-10-16T14:21:39.839302297Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T19:05:21.557583747Z", + "End": "2026-10-16T19:05:22.142090738Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T19:05:27.143338283Z", + "End": "2026-10-16T19:05:27.622167618Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T19:05:32.623688976Z", + "End": "2026-10-16T19:05:33.046515405Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T19:05:39.363083138Z", + "End": "2026-10-16T19:05:39.842740067Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T19:05:44.84414073Z", + "End": "2026-10-16T19:05:45.339352229Z", + "ExitCode": 0, + "Output": "" + } + ] + } + } + }, + "2441f99f958bbd516d0ab7b93eb96e09b7f875a4164f9ff9d645a9aede2dc6c5": { + "service": "db", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 57338, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T14:21:37.770464288Z", + "FinishedAt": "2026-10-16T14:21:37.196628506Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T19:05:38.745087502Z", + "End": "2026-10-16T19:05:38.789808433Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-16T19:05:40.790459849Z", + "End": "2026-10-16T19:05:40.835150776Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-16T19:05:42.836652693Z", + "End": "2026-10-16T19:05:42.881695222Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-16T19:05:44.882830897Z", + "End": "2026-10-16T19:05:44.932299003Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-16T19:05:46.933638229Z", + "End": "2026-10-16T19:05:46.99111665Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + } + ] + } + } + }, + "e4fb86e6ec85e6c96ae794ba39e8fc93d179bada127bfb8cebe4f5a5361877d1": { + "service": "cache", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 57302, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T14:21:37.642224221Z", + "FinishedAt": "2026-10-16T14:21:37.158949295Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T19:05:39.218402845Z", + "End": "2026-10-16T19:05:39.23869686Z", + "ExitCode": 0, + "Output": "PONG\n" + }, + { + "Start": "2026-10-16T19:05:41.239357605Z", + "End": "2026-10-16T19:05:41.259293587Z", + "ExitCode": 0, + "Output": "PONG\n" + }, + { + "Start": "2026-10-16T19:05:43.261038537Z", + "End": "2026-10-16T19:05:43.287970539Z", + "ExitCode": 0, + "Output": "PONG\n" + }, + { + "Start": "2026-10-16T19:05:45.28872912Z", + "End": "2026-10-16T19:05:45.310829129Z", + "ExitCode": 0, + "Output": "PONG\n" + }, + { + "Start": "2026-10-16T19:05:47.311317596Z", + "End": "2026-10-16T19:05:47.331220589Z", + "ExitCode": 0, + "Output": "PONG\n" + } + ] + } + } + }, + "dad1c78ccfbd99a5f2e5ee467c3216b81bb70dd4a9d9bfa957a1514050e762bc": { + "service": "lb", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 57396, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T14:21:37.890968447Z", + "FinishedAt": "2026-10-16T14:21:37.097695286Z" + } + }, + "6e8f2c3c72100a9e08ad6bd157697931fcb23b7846c5929a86033ee1e5f5a978": { + "service": "s3", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 90601, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:05:21.285471656Z", + "FinishedAt": "2026-10-16T19:05:20.730580171Z" + } + }, + "572e906935d7ec6f3dd28d51caaee7d25696558344f288631ba92977b5f3bc45": { + "service": "reconciler", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 57604, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T14:21:39.595976344Z", + "FinishedAt": "2026-10-16T14:21:39.251848171Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T19:04:58.650013908Z", + "End": "2026-10-16T19:04:58.687567121Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:05:10.130199277Z", + "End": "2026-10-16T19:05:10.161910968Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:05:20.162424877Z", + "End": "2026-10-16T19:05:20.192425192Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:05:30.193378293Z", + "End": "2026-10-16T19:05:30.223733363Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:05:41.540431923Z", + "End": "2026-10-16T19:05:41.575560046Z", + "ExitCode": 0, + "Output": "" + } + ] + } + } + } + }, + "after": { + "b0a8d4d6c153c378aadf6303f33846b7fab572564fc36920eb6a1c3e1bed62aa": { + "service": "api", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 91908, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:05:51.503236355Z", + "FinishedAt": "2026-10-16T19:05:51.051892302Z", + "Health": { + "Status": "starting", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T19:05:43.362382377Z", + "End": "2026-10-16T19:05:43.839170887Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:05:48.839798292Z", + "End": "2026-10-16T19:05:49.453801773Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:05:56.761820425Z", + "End": "2026-10-16T19:05:57.223778281Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T19:06:02.22498739Z", + "End": "2026-10-16T19:06:02.671587739Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T19:06:09.099465314Z", + "End": "2026-10-16T19:06:09.563981298Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + } + ] + } + } + }, + "395705f22be6d83f694cc98e96fe5bfe628151af93fd5f1e939704ff21fc5dc8": { + "service": "api", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 92149, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:05:53.745039633Z", + "FinishedAt": "2026-10-16T19:05:53.471949599Z", + "Health": { + "Status": "starting", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T19:05:39.363083138Z", + "End": "2026-10-16T19:05:39.842740067Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T19:05:44.84414073Z", + "End": "2026-10-16T19:05:45.339352229Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:05:50.339801368Z", + "End": "2026-10-16T19:05:52.041595819Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 1357, in do_open\n h.request(req.get_method(), req.selector, req.data, headers,\n File \"/usr/local/lib/python3.12/http/client.py\", line 1386, in request\n self._send_request(method, url, body, headers, encode_chunked)\n File \"/usr/local/lib/python3.12/http/client.py\", line 1432, in _send_request\n self.endheaders(body, encode_chunked=encode_chunked)\n File \"/usr/local/lib/python3.12/http/client.py\", line 1381, in endheaders\n self._send_output(message_body, encode_chunked=encode_chunked)\n File \"/usr/local/lib/python3.12/http/client.py\", line 1141, in _send_output\n self.send(msg)\n File \"/usr/local/lib/python3.12/http/client.py\", line 1085, in send\n self.connect()\n File \"/usr/local/lib/python3.12/http/client.py\", line 1051, in connect\n self.sock = self._create_connection(\n ^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/socket.py\", line 865, in create_connection\n raise exceptions[0]\n File \"/usr/local/lib/python3.12/socket.py\", line 850, in create_connection\n sock.connect(sa)\nConnectionRefusedError: [Errno 111] Connection refused\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 515, in open\n response = self._open(req, data)\n ^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 532, in _open\n result = self._call_chain(self.handle_open, protocol, protocol +\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 1386, in http_open\n return self.do_open(http.client.HTTPConnection, req)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 1360, in do_open\n raise URLError(err)\nurllib.error.URLError: \n" + }, + { + "Start": "2026-10-16T19:05:58.87057203Z", + "End": "2026-10-16T19:05:59.550549126Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T19:06:05.214084373Z", + "End": "2026-10-16T19:06:06.387122841Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + } + ] + } + } + }, + "2441f99f958bbd516d0ab7b93eb96e09b7f875a4164f9ff9d645a9aede2dc6c5": { + "service": "db", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 91717, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:05:50.585367281Z", + "FinishedAt": "2026-10-16T19:05:50.114533944Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T19:06:01.764202467Z", + "End": "2026-10-16T19:06:01.803981152Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-16T19:06:03.805033747Z", + "End": "2026-10-16T19:06:03.844587109Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-16T19:06:07.273151038Z", + "End": "2026-10-16T19:06:07.316174892Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-16T19:06:09.316906087Z", + "End": "2026-10-16T19:06:09.357124824Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-16T19:06:11.358455466Z", + "End": "2026-10-16T19:06:11.402615578Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + } + ] + } + } + }, + "e4fb86e6ec85e6c96ae794ba39e8fc93d179bada127bfb8cebe4f5a5361877d1": { + "service": "cache", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 91752, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:05:50.786619159Z", + "FinishedAt": "2026-10-16T19:05:50.084558012Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T19:06:01.623734818Z", + "End": "2026-10-16T19:06:01.646386929Z", + "ExitCode": 0, + "Output": "PONG\n" + }, + { + "Start": "2026-10-16T19:06:03.646921834Z", + "End": "2026-10-16T19:06:03.668289714Z", + "ExitCode": 0, + "Output": "PONG\n" + }, + { + "Start": "2026-10-16T19:06:07.09579857Z", + "End": "2026-10-16T19:06:07.114392094Z", + "ExitCode": 0, + "Output": "PONG\n" + }, + { + "Start": "2026-10-16T19:06:09.115447984Z", + "End": "2026-10-16T19:06:09.135323666Z", + "ExitCode": 0, + "Output": "PONG\n" + }, + { + "Start": "2026-10-16T19:06:11.13641835Z", + "End": "2026-10-16T19:06:11.15433417Z", + "ExitCode": 0, + "Output": "PONG\n" + } + ] + } + } + }, + "dad1c78ccfbd99a5f2e5ee467c3216b81bb70dd4a9d9bfa957a1514050e762bc": { + "service": "lb", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 91827, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:05:51.121747424Z", + "FinishedAt": "2026-10-16T19:05:50.233763703Z" + } + }, + "6e8f2c3c72100a9e08ad6bd157697931fcb23b7846c5929a86033ee1e5f5a978": { + "service": "s3", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 92676, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:06:11.831976196Z", + "FinishedAt": "2026-10-16T19:06:11.505939192Z" + } + }, + "572e906935d7ec6f3dd28d51caaee7d25696558344f288631ba92977b5f3bc45": { + "service": "reconciler", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 92038, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:05:52.824086086Z", + "FinishedAt": "2026-10-16T19:05:52.546565655Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T19:05:30.193378293Z", + "End": "2026-10-16T19:05:30.223733363Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:05:41.540431923Z", + "End": "2026-10-16T19:05:41.575560046Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:05:51.576781033Z", + "End": "2026-10-16T19:05:51.823968089Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:05:57.973400351Z", + "End": "2026-10-16T19:05:58.003734353Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:06:09.432278398Z", + "End": "2026-10-16T19:06:09.46125484Z", + "ExitCode": 0, + "Output": "" + } + ] + } + } + } + }, + "volumes_removed": false +}