From 9696ad71a46657112fde4156077364b231ee49e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D0=BD=D0=B5=D0=BA?= <88394552+Lunatuk@users.noreply.github.com> Date: Fri, 6 Nov 2026 10:47:51 +0300 Subject: [PATCH] =?UTF-8?q?=D0=B8=D1=81=D0=BF=D1=80=D0=B0=D0=B2=D0=B8?= =?UTF-8?q?=D0=BB=20=D0=BE=D1=88=D0=B8=D0=B1=D0=BA=D1=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- REPORT.md | 44 +- evidence/lab4-checks.json | 1435 ++++++++++++++++------------- evidence/lab4-checks.txt | 740 +++++++-------- evidence/postgres-pause.json | 76 ++ src/db.py | 17 + student-tests/docker_lifecycle.py | 3 + student-tests/postgres_pause.py | 145 +++ 7 files changed, 1419 insertions(+), 1041 deletions(-) create mode 100644 evidence/postgres-pause.json create mode 100644 student-tests/postgres_pause.py diff --git a/REPORT.md b/REPORT.md index 0fc22be..359640a 100644 --- a/REPORT.md +++ b/REPORT.md @@ -87,6 +87,23 @@ Pillow открывает только PNG/JPEG, проверяет реальн /health/ready проверяет PostgreSQL, Valkey и существование S3 bucket подписанным HEAD. /health/live проверяет процесс. При отказе S3 обязательные upload/download получают 503; LB исключает неготовые API. После отправки части download последующий сетевой обрыв означает неполный HTTP download: статус уже не заменяется, клиент может повторить безопасный GET. +### Пауза PostgreSQL и клиентский таймаут + +Помимо connect_timeout=2 s, ожидания пула 2 s и серверного statement_timeout=3 s, каждое ожидание PostgreSQL I/O ограничено на стороне приложения двумя секундами. TimedConnection использует timeout метода wait() закреплённой версии Psycopg 3.3.6. Это распространяется на проверку соединения из пула, выполнение SQL, COMMIT и ROLLBACK, даже когда процесс PostgreSQL приостановлен и серверный таймер не работает. + +При OperationalError соединение закрывается до выхода из контекста транзакции. API возвращает 503 unavailable в обычном формате, не ожидая отмены запроса или ROLLBACK от приостановленного сервера. Зависших потоков и выданных навсегда соединений не остаётся; пул создаёт замену. После docker compose unpause postgres readiness и запросы с прежними сессиями восстанавливаются без перезапуска API. + +Регрессионная проверка получает соединение до pause и отключает серверный statement_timeout, чтобы отдельно проверить клиентское ограничение SQL. Затем держит настоящую PostgreSQL на pause не менее 12 s, проверяет прямой /health/ready, 10 параллельных повторов и /health/live внутри API-контейнера, минуя LB. После unpause сверяются неизменные ID/StartedAt API, действующая сессия и сохранённое сообщение. Сценарий включён в check-docker.sh; отдельно запускается в своём временном стенде: + +```sh +./scripts/build.sh +.venv/bin/python student-tests/postgres_pause.py +``` + +Результат отдельного запуска: [evidence/postgres-pause.json](evidence/postgres-pause.json). Полный прогон сохраняет этот сценарий в поле postgres_pause файла lab4-checks.json. + +Фактическая проверка 06.11.2026: SQL без серверного таймера прерван за 2.001 s, соединение закрыто; прямой readiness вернул 503 за 2.012 s. Все 10 параллельных readiness-запросов вернули 503, максимум 2.299 s. PostgreSQL оставалась на pause 12.565 s; live отвечал 200. После unpause действующая сессия и сообщение доступны через LB за 2.874 s, без restart API. + ## Согласованность S3 и PostgreSQL на уровне 4 1. После приёма и проверки файла отдельный COMMIT сохраняет attachment_uploads с ID и object_key. @@ -134,6 +151,7 @@ check-docker.sh выбирает отдельное имя lab4-check- | Ограничены RAM/таймауты; ≥10 одновременных загрузок с замером памяти | Barrier на 10 клиентах, настоящие PNG 1600×1200, cgroup каждые 20 ms, SHA-256 всех downloads | attachments.parallel_uploads | | Идемпотентный bootstrap и ограниченные S3 credentials; закрытая администрация | Два bootstrap, запрещённые S3 операции, TCP-пробы из API | attachments.private_bucket | | Недоступность S3 и восстановление | Остановка настоящего s3-service, запросы внутри API в обход исключившего backend LB | attachments.s3_outage; dependency_readiness | +| PostgreSQL на pause: ограниченный прямой readiness и восстановление без restart API | postgres_pause.py: SQL без серверного таймера, pause ≥12 s, 10 параллельных запросов, unpause | postgres_pause; evidence/postgres-pause.json | Все актуальные результаты — [evidence/lab4-checks.txt](evidence/lab4-checks.txt) и [evidence/lab4-checks.json](evidence/lab4-checks.json). Токены, пароли и исходные тела запросов туда не сохраняются. PASS публичного smoke сам по себе не считается доказательством всех критериев. @@ -143,42 +161,43 @@ check-docker.sh выбирает отдельное имя lab4-check- Замер включает Python-процесс монитора и файловый cache cgroup; это консервативная оценка контейнерной памяти, а не чистый RSS приложения. Пик выборки может пропустить события короче 20 ms; memory.peak дополнительно показывает пик за жизнь контейнера. Это один ограниченный учебный прогон с 10 клиентами, без поиска предельной производительности. p50/p95 относятся к успешным HTTP upload; coordinated omission и влияние клиента остаются ограничениями. -### Результат фактического прогона 05.11.2026 +### Результат фактического прогона 06.11.2026 -Среда: Darwin arm64; Docker 29.0.1 aarch64 CPUs=10 RAM=8217448448; Compose 2.40.3-desktop.1. Две API-реплики, по 1 CPU и 384 MiB. Прогрев — полный smoke и 39 тестов до эксперимента; накопившаяся память отражена в baseline. Временный project: lab4-check-753ba80cb3. +Среда: Darwin arm64; Docker 29.0.1 aarch64 CPUs=10 RAM=8217448448; Compose 2.40.3-desktop.1. Две API-реплики, по 1 CPU и 384 MiB. Прогрев — полный smoke и 39 тестов до эксперимента; накопившаяся память отражена в baseline. Временный project: lab4-check-5aad8e0065. | Показатель | Наблюдение | | --- | --- | | Параллельные uploads | 10 из 10, ошибок 0 | -| Один PNG | 5765613 bytes, 1600×1200 RGB | -| Длительность / успешные upload/s | 0.805 s / 12.426 | -| p50 / p95 HTTP upload | 596.684 / 798.322 ms | +| Один PNG | 5769335 bytes, 1600×1200 RGB | +| Длительность / успешные upload/s | 0.914 s / 10.939 | +| p50 / p95 HTTP upload | 610.113 / 904.09 ms | | Оригиналы | 10 download, все SHA-256 совпали | | OOM | Нет | | API instance | Baseline cgroup, MiB | Пик выборки во время uploads, MiB | Пик cgroup за жизнь контейнера, MiB | Лимит, MiB | | --- | ---: | ---: | ---: | ---: | -| 887ffd1465a8 | 179.97 | 194.26 | 261.37 | 384 | -| 8f41208a6406 | 63.74 | 86.14 | 88.71 | 384 | +| 19a474c69210 | 190.10 | 205.22 | 261.53 | 384 | +| ca1f62dfc015 | 64.43 | 98.59 | 100.28 | 384 | Различие baseline между репликами возникло после предыдущих проверок и аллокаций; сравнение не трактуется как чистое измерение стоимости одного декодирования. Пик за жизнь контейнера включает более ранние тесты. Гипотеза о соблюдении лимита в этом профиле подтверждена; универсальная производительность или поведение при произвольном числе клиентов не выводятся. | Воздействие | Наблюдение | Восстановление / потеря подтверждённых данных | | --- | --- | --- | -| Остановлена загрузившая API-реплика 8f41208a6406 | Download обслужила 887ffd1465a8; исходный SHA-256 совпал | 3.589 s от начала stop; потерь нет | +| Остановлена загрузившая API-реплика ca1f62dfc015 | Download обслужила 19a474c69210; исходный SHA-256 совпал | 3.535 s от начала stop; потерь нет | | Прервано HTTP-тело | Metadata и журнал не созданы; S3 PUT не начинался | Очистка временного файла; ready-объектов не затрагивает | | БД отказала после S3 PUT | HTTP 503; объект существует вместе с журналом; ready-metadata отсутствует | Свежий/активный объект сохранён; после искусственного старения безопасно удалён; worker проверен отдельно | | S3 service остановлен | live=200, ready/upload/download=503 в едином формате API | После старта прежний токен и оригинал работают; downtime отдельно не измерялся | +| PostgreSQL на pause 14.572 s | Прямой ready=503 за 2.013 s; 10 повторов, максимум 2.107 s; live=200 | После unpause 1.975 s; API не перезапущены, сессия и сообщение сохранены | | Restart всего стенда с volumes | StartedAt всех постоянных процессов изменились; сессия, metadata, bytes и повтор сообщения сохранены | Уложилось в бюджет ожидания 90 s; потерь нет | -| SIGTERM с незавершённым POST | Новые соединения прекращены за 0.162 s; exit code 0, без OOM | Выход за 5.448 s, меньше 8 s | +| SIGTERM с незавершённым POST | Новые соединения прекращены за 0.185 s; exit code 0, без OOM | Выход за 5.523 s, меньше 8 s | Дополнительно сохранены результаты отказа каждой реплики и поочерёдной замены под нагрузкой: | Сценарий | Длительность, s | Успешные HTTP/s | Переходные ошибки | Макс. пауза POST / GET, s | Потери / дубли | | --- | ---: | ---: | ---: | --- | --- | -| stop 887ffd1465a8 | 16.012 | 92.43 | 4 | 2.042 / 1.984 | 0 / 0 | -| stop 8f41208a6406 | 16.026 | 91.85 | 4 | 1.986 / 1.99 | 0 / 0 | -| sequential API replacement | 22.153 | 88.47 | 8 | 1.048 / 1.031 | 0 / 0 | +| stop 19a474c69210 | 16.012 | 88.68 | 4 | 2.002 / 1.995 | 0 / 0 | +| stop ca1f62dfc015 | 16.008 | 91.58 | 4 | 2.002 / 1.999 | 0 / 0 | +| sequential API replacement | 24.869 | 90.63 | 8 | 1.04 / 1.04 | 0 / 0 | Этот профиль из lab3 использует четыре закрытых клиента: POST сообщения и GET истории, пауза 50 ms. При неопределённом POST повторяется тот же ключ/тело. Длительность включает 1 s прогрева и воздействие; исходные посекундные счётчики есть в JSON. Это проверка восстановления и отсутствия потерь, а не поиск предельного RPS. @@ -196,3 +215,4 @@ check-docker.sh выбирает отдельное имя lab4-check- - [SeaweedFS 3.85: IAM и ограничения действий](https://github.com/seaweedfs/seaweedfs/blob/3.85/weed/s3api/auth_credentials.go). - [MinIO Python SDK](https://minio-py.min.io/). - [Pillow Image: open, verify, load и лимиты](https://pillow.readthedocs.io/en/stable/reference/Image.html). +- [Psycopg 3.3.6: клиентский timeout ожидания I/O](https://github.com/psycopg/psycopg/blob/3.3.6/psycopg/psycopg/connection.py). diff --git a/evidence/lab4-checks.json b/evidence/lab4-checks.json index 9104484..f410993 100644 --- a/evidence/lab4-checks.json +++ b/evidence/lab4-checks.json @@ -1,7 +1,7 @@ { - "utc": "2026-11-05T16:13:18.569036+00:00", - "project": "lab4-check-753ba80cb3", - "base_url": "http://127.0.0.1:65032", + "utc": "2026-11-06T07:29:29.234494+00:00", + "project": "lab4-check-5aad8e0065", + "base_url": "http://127.0.0.1:57722", "host": "Darwin arm64", "docker": "29.0.1 aarch64 CPUs=10 RAM=8217448448", "profile": { @@ -10,15 +10,15 @@ "operation": "POST message then GET history, same-key retry after uncertain POST" }, "compose_version": "2.40.3-desktop.1", - "source_commit": "5c9b3e206a07c0346c9765923cd9560358c3a420", + "source_commit": "c43d3490a07ad939e4e4748863589b73c84ad05a", "working_tree": true, "containers": [ { - "id": "887ffd1465a8191971a9f04cb3deb79994889e288fbe4bc7224052fa3094a710", - "instance": "887ffd1465a8", + "id": "19a474c69210a7be1c6631830e8fa7f251cf6dd3890c7407843090383543ee75", + "instance": "19a474c69210", "service": "api", - "image": "sha256:cb6318e7f06fd815219d3324008ff58c6b6ebb44fb9ea9b76e43546a8b0f78d5", - "started_at": "2026-11-05T16:13:46.833090004Z", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:29:57.621726877Z", "running": true, "exit_code": 0, "memory_bytes": 402653184, @@ -48,17 +48,22 @@ ] }, { - "id": "8f41208a64062523e85a27c5d759029d6c9940df78417691c85900dcb8e6f158", - "instance": "8f41208a6406", + "id": "ca1f62dfc0156fc235dbcd2f52b58eec2282e6ba8d3e056de11ca92758c69df0", + "instance": "ca1f62dfc015", "service": "api", - "image": "sha256:cb6318e7f06fd815219d3324008ff58c6b6ebb44fb9ea9b76e43546a8b0f78d5", - "started_at": "2026-11-05T16:13:46.727185879Z", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:29:57.502184793Z", "running": true, "exit_code": 0, "memory_bytes": 402653184, "nano_cpus": 1000000000, "published_ports": {}, "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, { "type": "bind", "destination": "/run/secrets/s3_secret_access_key", @@ -73,31 +78,21 @@ "type": "bind", "destination": "/run/secrets/cache_url", "name": null - }, - { - "type": "bind", - "destination": "/run/secrets/s3_access_key_id", - "name": null } ] }, { - "id": "10143cbf3e42de96c7a3078ea5b42d24bbf9702a4f46e57d4a9aa2a7ffbbab61", - "instance": "10143cbf3e42", + "id": "bfaeabc178285f04b23c0578b9c2d22b8e54ab97c4cb0d70eb874c46dd92acdb", + "instance": "bfaeabc17828", "service": "cleanup", - "image": "sha256:cb6318e7f06fd815219d3324008ff58c6b6ebb44fb9ea9b76e43546a8b0f78d5", - "started_at": "2026-11-05T16:13:46.726778921Z", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:29:57.50177996Z", "running": true, "exit_code": 0, "memory_bytes": 402653184, "nano_cpus": 1000000000, "published_ports": {}, "mounts": [ - { - "type": "bind", - "destination": "/run/secrets/s3_secret_access_key", - "name": null - }, { "type": "bind", "destination": "/run/secrets/database_url", @@ -107,15 +102,20 @@ "type": "bind", "destination": "/run/secrets/s3_access_key_id", "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null } ] }, { - "id": "aecf8751126b66c807155750bf349fa6af90aa2a322bf078bd903bae7fba5b53", - "instance": "aecf8751126b", + "id": "3bc52373c3d129bac2f4765eaf221fbe4461ae859616920aba3f3cf952a9e449", + "instance": "3bc52373c3d1", "service": "lb", "image": "sha256:3a819392f5a6af2a9203d1c63ff0feba74f0c940623a558511be06d7d85e361c", - "started_at": "2026-11-05T16:13:52.440463757Z", + "started_at": "2026-11-06T07:30:03.264509213Z", "running": true, "exit_code": 0, "memory_bytes": 67108864, @@ -124,7 +124,7 @@ "8080/tcp": [ { "HostIp": "127.0.0.1", - "HostPort": "65032" + "HostPort": "57722" } ] }, @@ -137,11 +137,11 @@ ] }, { - "id": "5274aa99b987b7af4cc7680c634cb6a8d9c38d05f6ff0f49dac61c48d7844629", - "instance": "5274aa99b987", + "id": "533e4e511e0441fa577519bbca2a58bc4ceede3ab1660f4c8b026a54b69829f3", + "instance": "533e4e511e04", "service": "migrate", - "image": "sha256:cb6318e7f06fd815219d3324008ff58c6b6ebb44fb9ea9b76e43546a8b0f78d5", - "started_at": "2026-11-05T16:13:23.022573091Z", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:29:33.838693505Z", "running": false, "exit_code": 0, "memory_bytes": 402653184, @@ -156,11 +156,11 @@ ] }, { - "id": "1d892ce3140664f694767e9737f67ab691ca4e25b8129874d2e2fe1f20bbddda", - "instance": "1d892ce31406", + "id": "60274b0d5ab994316a8b828669ce937b08e758c5ce6dfc6d8d915d5cf4c46046", + "instance": "60274b0d5ab9", "service": "postgres", "image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", - "started_at": "2026-11-05T16:13:20.380429756Z", + "started_at": "2026-11-06T07:29:31.155947879Z", "running": true, "exit_code": 0, "memory_bytes": 536870912, @@ -170,7 +170,7 @@ { "type": "volume", "destination": "/var/lib/postgresql/data", - "name": "lab4-check-753ba80cb3_postgres_data" + "name": "lab4-check-5aad8e0065_postgres_data" }, { "type": "bind", @@ -180,11 +180,11 @@ ] }, { - "id": "3c9d7b1f26816060cb94a7bcf246d06c04c1fa3c6cf104f38e051eca1c11bfe1", - "instance": "3c9d7b1f2681", + "id": "0ba4ea69ac8dca26d0348fb351aaf286164ee5e72b2123ffdb89579469af53c9", + "instance": "0ba4ea69ac8d", "service": "s3", "image": "sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", - "started_at": "2026-11-05T16:13:43.008405419Z", + "started_at": "2026-11-06T07:29:53.783891208Z", "running": true, "exit_code": 0, "memory_bytes": 268435456, @@ -199,16 +199,16 @@ { "type": "volume", "destination": "/data", - "name": "5884fdd395ed5d5bb60b03bbb71fdd685825042d009cda0a137b77e42fef1ebf" + "name": "56e17d6e3f080fb7912c461068d6c41768244d709e8a55849e355a0895694340" } ] }, { - "id": "1615ae74805ae0ca3d9bd98fe25ea40ef8cd2db8fe7cb8c6938f8a427e3bc2db", - "instance": "1615ae74805a", + "id": "813a7d0f6af841b2ed44a684a33baddbbe41405b08d4bdd499ee7ed12b9eccc1", + "instance": "813a7d0f6af8", "service": "s3-bootstrap", - "image": "sha256:cb6318e7f06fd815219d3324008ff58c6b6ebb44fb9ea9b76e43546a8b0f78d5", - "started_at": "2026-11-05T16:13:45.622155504Z", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:29:56.406424376Z", "running": false, "exit_code": 0, "memory_bytes": 134217728, @@ -228,11 +228,11 @@ ] }, { - "id": "b403c79e680014fba72123d64d7742d70a57a4a08b4c7cd6f036eb752a63456e", - "instance": "b403c79e6800", + "id": "df540b4c43cfa85275ff158c300583671dbb3eaf12e75b9f7497f5414b3cb36b", + "instance": "df540b4c43cf", "service": "s3-storage", "image": "sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", - "started_at": "2026-11-05T16:13:20.382611881Z", + "started_at": "2026-11-06T07:29:31.154902962Z", "running": true, "exit_code": 0, "memory_bytes": 536870912, @@ -242,16 +242,16 @@ { "type": "volume", "destination": "/data", - "name": "lab4-check-753ba80cb3_s3_data" + "name": "lab4-check-5aad8e0065_s3_data" } ] }, { - "id": "030c8bacc201f114875911d3e3ddd3ee159a0a266d849104f775b2615b941937", - "instance": "030c8bacc201", + "id": "02dcfb1c815e68cbe7979e537c8d763076b7b59229c03dbfa850124eeef28a12", + "instance": "02dcfb1c815e", "service": "valkey", "image": "sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571", - "started_at": "2026-11-05T16:13:20.381988089Z", + "started_at": "2026-11-06T07:29:31.157226212Z", "running": true, "exit_code": 0, "memory_bytes": 134217728, @@ -261,7 +261,7 @@ { "type": "volume", "destination": "/data", - "name": "lab4-check-753ba80cb3_valkey_data" + "name": "lab4-check-5aad8e0065_valkey_data" } ] } @@ -281,11 +281,11 @@ "bootstrap_runs": 2 }, "upload_replica_failover": { - "uploaded_by": "8f41208a6406", - "stopped_container": "8f41208a64062523e85a27c5d759029d6c9940df78417691c85900dcb8e6f158", - "downloaded_by": "887ffd1465a8", + "uploaded_by": "ca1f62dfc015", + "stopped_container": "ca1f62dfc0156fc235dbcd2f52b58eec2282e6ba8d3e056de11ca92758c69df0", + "downloaded_by": "19a474c69210", "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", - "recovery_seconds": 3.589 + "recovery_seconds": 3.535 }, "interrupted_body": { "metadata_or_journal_created": false @@ -315,371 +315,403 @@ 1600, 1200 ], - "size_bytes": 5765613, - "sha256": "79199b764c56a75da074e2cc1157cc471d53d23dc1cb141edba352d25e376ac3", - "duration_seconds": 0.805, - "successful_rps": 12.426, - "p50_ms": 596.684, - "p95_ms": 798.322, + "size_bytes": 5769335, + "sha256": "b43f2e8293c15183ae478f3bc43cd262e1fa2c532e1c2cfbabceaf3566b4a0d1", + "duration_seconds": 0.914, + "successful_rps": 10.939, + "p50_ms": 610.113, + "p95_ms": 904.09, "memory_sample_interval_seconds": 0.02, "memory_includes_monitor_process": true, "memory": [ { - "baseline_bytes": 188715008, - "sample_peak_bytes": 203698176, - "lifetime_peak_bytes": 274067456, + "baseline_bytes": 199331840, + "sample_peak_bytes": 215191552, + "lifetime_peak_bytes": 274231296, "samples": [ [ 0.0, - 188715008 + 199098368 ], [ 0.021, - 188608512 + 201449472 ], [ - 0.042, - 188723200 + 0.043, + 201371648 ], [ - 0.063, - 188723200 + 0.064, + 201367552 ], [ - 0.084, - 188723200 + 0.085, + 201625600 ], [ - 0.105, - 188731392 + 0.108, + 202936320 ], [ - 0.127, - 188727296 - ], - [ - 0.149, - 188723200 - ], - [ - 0.17, - 188874752 - ], - [ - 0.191, - 190881792 - ], - [ - 0.212, - 192102400 - ], - [ - 0.233, - 193662976 - ], - [ - 0.254, - 193961984 - ], - [ - 0.275, - 194826240 - ], - [ - 0.296, - 195448832 - ], - [ - 0.317, - 196419584 - ], - [ - 0.338, - 197005312 - ], - [ - 0.359, - 198537216 - ], - [ - 0.38, - 198995968 - ], - [ - 0.401, - 201351168 - ], - [ - 0.422, + 0.13, 202309632 ], [ - 0.443, - 202698752 + 0.151, + 188710912 ], [ - 0.464, - 203296768 + 0.172, + 189378560 + ], + [ + 0.193, + 190976000 + ], + [ + 0.214, + 192249856 + ], + [ + 0.235, + 193933312 + ], + [ + 0.255, + 195235840 + ], + [ + 0.276, + 196837376 + ], + [ + 0.297, + 197255168 + ], + [ + 0.318, + 198422528 + ], + [ + 0.339, + 198475776 + ], + [ + 0.36, + 200818688 + ], + [ + 0.381, + 201170944 + ], + [ + 0.402, + 204890112 + ], + [ + 0.423, + 206053376 + ], + [ + 0.444, + 206901248 + ], + [ + 0.465, + 207024128 ], [ 0.486, - 203456512 + 207085568 ], [ 0.507, - 203698176 + 209682432 ], [ - 0.528, - 203694080 + 0.531, + 214925312 ], [ - 0.549, - 203694080 + 0.552, + 215191552 ], [ - 0.57, - 203694080 + 0.573, + 215191552 ], [ - 0.591, - 203689984 + 0.594, + 211496960 ], [ - 0.613, - 203694080 + 0.615, + 211415040 ], [ - 0.635, - 203411456 + 0.636, + 211611648 ], [ - 0.656, - 203218944 + 0.657, + 211410944 ], [ - 0.677, - 203214848 + 0.678, + 206876672 ], [ - 0.698, - 203214848 + 0.699, + 207892480 ], [ 0.72, - 202133504 + 207699968 ], [ - 0.744, - 201838592 + 0.742, + 208068608 ], [ - 0.764, - 201822208 + 0.763, + 202301440 ], [ - 0.785, - 195977216 + 0.784, + 207642624 ], [ - 0.806, - 198823936 + 0.805, + 207413248 ], [ - 0.827, - 198758400 + 0.826, + 207593472 ], [ - 0.848, - 198758400 + 0.847, + 207593472 ], [ - 0.869, - 198762496 + 0.868, + 204062720 ], [ - 0.89, - 196005888 + 0.889, + 203718656 ], [ - 0.911, - 195731456 + 0.91, + 205983744 ], [ - 0.932, - 195731456 + 0.931, + 203411456 + ], + [ + 0.952, + 203411456 + ], + [ + 0.973, + 200871936 + ], + [ + 0.996, + 197525504 + ], + [ + 1.019, + 197496832 + ], + [ + 1.04, + 197496832 ] ], - "instance": "887ffd1465a8", + "instance": "19a474c69210", "limit_bytes": 402653184 }, { - "baseline_bytes": 66834432, - "sample_peak_bytes": 90324992, - "lifetime_peak_bytes": 93024256, + "baseline_bytes": 67555328, + "sample_peak_bytes": 103378944, + "lifetime_peak_bytes": 105148416, "samples": [ [ 0.0, - 66834432 + 67555328 ], [ 0.02, - 67706880 + 68780032 ], [ 0.041, - 69509120 + 71090176 ], [ 0.062, - 70037504 + 71782400 ], [ 0.083, - 71856128 + 73428992 ], [ 0.104, - 72515584 + 74698752 ], [ 0.125, - 73310208 + 76619776 ], [ 0.146, - 73994240 + 77070336 ], [ 0.167, - 75051008 + 78118912 ], [ - 0.188, - 75055104 + 0.189, + 79024128 ], [ - 0.21, - 77008896 + 0.209, + 80576512 ], [ 0.23, - 77328384 + 81674240 ], [ 0.251, - 80379904 + 84914176 ], [ 0.272, - 82636800 + 87511040 ], [ 0.293, - 82620416 + 88616960 ], [ - 0.314, - 83005440 + 0.315, + 89174016 ], [ - 0.336, - 83689472 + 0.335, + 89919488 ], [ 0.356, - 87851008 + 89976832 ], [ - 0.378, - 89878528 + 0.381, + 96006144 ], [ - 0.398, - 89747456 + 0.402, + 96436224 ], [ - 0.419, - 89747456 + 0.422, + 95522816 ], [ - 0.44, - 87896064 + 0.444, + 93450240 ], [ - 0.462, - 88268800 + 0.465, + 98467840 ], [ - 0.483, - 90234880 + 0.485, + 103378944 ], [ - 0.503, - 90218496 + 0.507, + 102744064 ], [ - 0.524, - 89128960 + 0.527, + 102805504 ], [ - 0.545, - 90324992 + 0.549, + 100847616 ], [ - 0.567, - 90263552 + 0.571, + 102080512 ], [ - 0.588, - 90259456 + 0.592, + 102064128 ], [ - 0.609, - 85856256 + 0.614, + 102064128 ], [ - 0.63, - 86413312 + 0.635, + 102318080 ], [ - 0.65, - 86913024 + 0.655, + 103100416 ], [ - 0.671, - 86097920 + 0.676, + 100167680 ], [ - 0.693, - 86142976 + 0.722, + 100024320 ], [ - 0.713, - 83120128 + 0.743, + 100548608 ], [ - 0.734, - 83083264 + 0.764, + 101707776 ], [ - 0.756, - 83079168 + 0.785, + 87347200 ], [ - 0.776, - 77225984 + 0.822, + 81424384 ], [ - 0.798, - 77201408 + 0.843, + 83718144 ], [ - 0.818, - 77135872 + 0.866, + 83685376 + ], + [ + 0.887, + 83664896 + ], + [ + 0.909, + 83668992 + ], + [ + 0.93, + 78086144 ] ], - "instance": "8f41208a6406", + "instance": "ca1f62dfc015", "limit_bytes": 402653184 } ], @@ -690,11 +722,45 @@ "persistence": { "before": [ { - "id": "887ffd1465a8191971a9f04cb3deb79994889e288fbe4bc7224052fa3094a710", - "instance": "887ffd1465a8", + "id": "19a474c69210a7be1c6631830e8fa7f251cf6dd3890c7407843090383543ee75", + "instance": "19a474c69210", "service": "api", - "image": "sha256:cb6318e7f06fd815219d3324008ff58c6b6ebb44fb9ea9b76e43546a8b0f78d5", - "started_at": "2026-11-05T16:13:46.833090004Z", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:29:57.621726877Z", + "running": true, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + } + ] + }, + { + "id": "ca1f62dfc0156fc235dbcd2f52b58eec2282e6ba8d3e056de11ca92758c69df0", + "instance": "ca1f62dfc015", + "service": "api", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:30:34.948927714Z", "running": true, "exit_code": 0, "memory_bytes": 402653184, @@ -724,45 +790,11 @@ ] }, { - "id": "8f41208a64062523e85a27c5d759029d6c9940df78417691c85900dcb8e6f158", - "instance": "8f41208a6406", - "service": "api", - "image": "sha256:cb6318e7f06fd815219d3324008ff58c6b6ebb44fb9ea9b76e43546a8b0f78d5", - "started_at": "2026-11-05T16:14:24.166448674Z", - "running": true, - "exit_code": 0, - "memory_bytes": 402653184, - "nano_cpus": 1000000000, - "published_ports": {}, - "mounts": [ - { - "type": "bind", - "destination": "/run/secrets/s3_secret_access_key", - "name": null - }, - { - "type": "bind", - "destination": "/run/secrets/database_url", - "name": null - }, - { - "type": "bind", - "destination": "/run/secrets/cache_url", - "name": null - }, - { - "type": "bind", - "destination": "/run/secrets/s3_access_key_id", - "name": null - } - ] - }, - { - "id": "10143cbf3e42de96c7a3078ea5b42d24bbf9702a4f46e57d4a9aa2a7ffbbab61", - "instance": "10143cbf3e42", + "id": "bfaeabc178285f04b23c0578b9c2d22b8e54ab97c4cb0d70eb874c46dd92acdb", + "instance": "bfaeabc17828", "service": "cleanup", - "image": "sha256:cb6318e7f06fd815219d3324008ff58c6b6ebb44fb9ea9b76e43546a8b0f78d5", - "started_at": "2026-11-05T16:14:50.281562506Z", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:31:01.671594045Z", "running": true, "exit_code": 0, "memory_bytes": 402653184, @@ -787,11 +819,11 @@ ] }, { - "id": "aecf8751126b66c807155750bf349fa6af90aa2a322bf078bd903bae7fba5b53", - "instance": "aecf8751126b", + "id": "3bc52373c3d129bac2f4765eaf221fbe4461ae859616920aba3f3cf952a9e449", + "instance": "3bc52373c3d1", "service": "lb", "image": "sha256:3a819392f5a6af2a9203d1c63ff0feba74f0c940623a558511be06d7d85e361c", - "started_at": "2026-11-05T16:13:52.440463757Z", + "started_at": "2026-11-06T07:30:03.264509213Z", "running": true, "exit_code": 0, "memory_bytes": 67108864, @@ -800,7 +832,7 @@ "8080/tcp": [ { "HostIp": "127.0.0.1", - "HostPort": "65032" + "HostPort": "57722" } ] }, @@ -813,11 +845,11 @@ ] }, { - "id": "1d892ce3140664f694767e9737f67ab691ca4e25b8129874d2e2fe1f20bbddda", - "instance": "1d892ce31406", + "id": "60274b0d5ab994316a8b828669ce937b08e758c5ce6dfc6d8d915d5cf4c46046", + "instance": "60274b0d5ab9", "service": "postgres", "image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", - "started_at": "2026-11-05T16:13:20.380429756Z", + "started_at": "2026-11-06T07:29:31.155947879Z", "running": true, "exit_code": 0, "memory_bytes": 536870912, @@ -827,7 +859,7 @@ { "type": "volume", "destination": "/var/lib/postgresql/data", - "name": "lab4-check-753ba80cb3_postgres_data" + "name": "lab4-check-5aad8e0065_postgres_data" }, { "type": "bind", @@ -837,11 +869,11 @@ ] }, { - "id": "3c9d7b1f26816060cb94a7bcf246d06c04c1fa3c6cf104f38e051eca1c11bfe1", - "instance": "3c9d7b1f2681", + "id": "0ba4ea69ac8dca26d0348fb351aaf286164ee5e72b2123ffdb89579469af53c9", + "instance": "0ba4ea69ac8d", "service": "s3", "image": "sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", - "started_at": "2026-11-05T16:14:43.155925128Z", + "started_at": "2026-11-06T07:30:53.867775792Z", "running": true, "exit_code": 0, "memory_bytes": 268435456, @@ -856,16 +888,16 @@ { "type": "volume", "destination": "/data", - "name": "5884fdd395ed5d5bb60b03bbb71fdd685825042d009cda0a137b77e42fef1ebf" + "name": "56e17d6e3f080fb7912c461068d6c41768244d709e8a55849e355a0895694340" } ] }, { - "id": "b403c79e680014fba72123d64d7742d70a57a4a08b4c7cd6f036eb752a63456e", - "instance": "b403c79e6800", + "id": "df540b4c43cfa85275ff158c300583671dbb3eaf12e75b9f7497f5414b3cb36b", + "instance": "df540b4c43cf", "service": "s3-storage", "image": "sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", - "started_at": "2026-11-05T16:13:20.382611881Z", + "started_at": "2026-11-06T07:29:31.154902962Z", "running": true, "exit_code": 0, "memory_bytes": 536870912, @@ -875,16 +907,16 @@ { "type": "volume", "destination": "/data", - "name": "lab4-check-753ba80cb3_s3_data" + "name": "lab4-check-5aad8e0065_s3_data" } ] }, { - "id": "030c8bacc201f114875911d3e3ddd3ee159a0a266d849104f775b2615b941937", - "instance": "030c8bacc201", + "id": "02dcfb1c815e68cbe7979e537c8d763076b7b59229c03dbfa850124eeef28a12", + "instance": "02dcfb1c815e", "service": "valkey", "image": "sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571", - "started_at": "2026-11-05T16:13:20.381988089Z", + "started_at": "2026-11-06T07:29:31.157226212Z", "running": true, "exit_code": 0, "memory_bytes": 134217728, @@ -894,18 +926,18 @@ { "type": "volume", "destination": "/data", - "name": "lab4-check-753ba80cb3_valkey_data" + "name": "lab4-check-5aad8e0065_valkey_data" } ] } ], "after": [ { - "id": "887ffd1465a8191971a9f04cb3deb79994889e288fbe4bc7224052fa3094a710", - "instance": "887ffd1465a8", + "id": "19a474c69210a7be1c6631830e8fa7f251cf6dd3890c7407843090383543ee75", + "instance": "19a474c69210", "service": "api", - "image": "sha256:cb6318e7f06fd815219d3324008ff58c6b6ebb44fb9ea9b76e43546a8b0f78d5", - "started_at": "2026-11-05T16:14:58.905051635Z", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:31:09.786276466Z", "running": true, "exit_code": 0, "memory_bytes": 402653184, @@ -935,17 +967,22 @@ ] }, { - "id": "8f41208a64062523e85a27c5d759029d6c9940df78417691c85900dcb8e6f158", - "instance": "8f41208a6406", + "id": "ca1f62dfc0156fc235dbcd2f52b58eec2282e6ba8d3e056de11ca92758c69df0", + "instance": "ca1f62dfc015", "service": "api", - "image": "sha256:cb6318e7f06fd815219d3324008ff58c6b6ebb44fb9ea9b76e43546a8b0f78d5", - "started_at": "2026-11-05T16:14:59.190444177Z", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:31:10.013737549Z", "running": true, "exit_code": 0, "memory_bytes": 402653184, "nano_cpus": 1000000000, "published_ports": {}, "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, { "type": "bind", "destination": "/run/secrets/s3_secret_access_key", @@ -960,20 +997,15 @@ "type": "bind", "destination": "/run/secrets/cache_url", "name": null - }, - { - "type": "bind", - "destination": "/run/secrets/s3_access_key_id", - "name": null } ] }, { - "id": "10143cbf3e42de96c7a3078ea5b42d24bbf9702a4f46e57d4a9aa2a7ffbbab61", - "instance": "10143cbf3e42", + "id": "bfaeabc178285f04b23c0578b9c2d22b8e54ab97c4cb0d70eb874c46dd92acdb", + "instance": "bfaeabc17828", "service": "cleanup", - "image": "sha256:cb6318e7f06fd815219d3324008ff58c6b6ebb44fb9ea9b76e43546a8b0f78d5", - "started_at": "2026-11-05T16:15:06.604429847Z", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:31:17.411326011Z", "running": true, "exit_code": 0, "memory_bytes": 402653184, @@ -998,11 +1030,11 @@ ] }, { - "id": "aecf8751126b66c807155750bf349fa6af90aa2a322bf078bd903bae7fba5b53", - "instance": "aecf8751126b", + "id": "3bc52373c3d129bac2f4765eaf221fbe4461ae859616920aba3f3cf952a9e449", + "instance": "3bc52373c3d1", "service": "lb", "image": "sha256:3a819392f5a6af2a9203d1c63ff0feba74f0c940623a558511be06d7d85e361c", - "started_at": "2026-11-05T16:14:58.790557135Z", + "started_at": "2026-11-06T07:31:09.542736424Z", "running": true, "exit_code": 0, "memory_bytes": 67108864, @@ -1011,7 +1043,7 @@ "8080/tcp": [ { "HostIp": "127.0.0.1", - "HostPort": "65032" + "HostPort": "57722" } ] }, @@ -1024,11 +1056,11 @@ ] }, { - "id": "1d892ce3140664f694767e9737f67ab691ca4e25b8129874d2e2fe1f20bbddda", - "instance": "1d892ce31406", + "id": "60274b0d5ab994316a8b828669ce937b08e758c5ce6dfc6d8d915d5cf4c46046", + "instance": "60274b0d5ab9", "service": "postgres", "image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", - "started_at": "2026-11-05T16:14:58.812168052Z", + "started_at": "2026-11-06T07:31:09.573287299Z", "running": true, "exit_code": 0, "memory_bytes": 536870912, @@ -1038,7 +1070,7 @@ { "type": "volume", "destination": "/var/lib/postgresql/data", - "name": "lab4-check-753ba80cb3_postgres_data" + "name": "lab4-check-5aad8e0065_postgres_data" }, { "type": "bind", @@ -1048,11 +1080,11 @@ ] }, { - "id": "3c9d7b1f26816060cb94a7bcf246d06c04c1fa3c6cf104f38e051eca1c11bfe1", - "instance": "3c9d7b1f2681", + "id": "0ba4ea69ac8dca26d0348fb351aaf286164ee5e72b2123ffdb89579469af53c9", + "instance": "0ba4ea69ac8d", "service": "s3", "image": "sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", - "started_at": "2026-11-05T16:14:58.840776635Z", + "started_at": "2026-11-06T07:31:09.608616716Z", "running": true, "exit_code": 0, "memory_bytes": 268435456, @@ -1067,16 +1099,16 @@ { "type": "volume", "destination": "/data", - "name": "5884fdd395ed5d5bb60b03bbb71fdd685825042d009cda0a137b77e42fef1ebf" + "name": "56e17d6e3f080fb7912c461068d6c41768244d709e8a55849e355a0895694340" } ] }, { - "id": "b403c79e680014fba72123d64d7742d70a57a4a08b4c7cd6f036eb752a63456e", - "instance": "b403c79e6800", + "id": "df540b4c43cfa85275ff158c300583671dbb3eaf12e75b9f7497f5414b3cb36b", + "instance": "df540b4c43cf", "service": "s3-storage", "image": "sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", - "started_at": "2026-11-05T16:15:06.569589264Z", + "started_at": "2026-11-06T07:31:17.386629011Z", "running": true, "exit_code": 0, "memory_bytes": 536870912, @@ -1086,16 +1118,16 @@ { "type": "volume", "destination": "/data", - "name": "lab4-check-753ba80cb3_s3_data" + "name": "lab4-check-5aad8e0065_s3_data" } ] }, { - "id": "030c8bacc201f114875911d3e3ddd3ee159a0a266d849104f775b2615b941937", - "instance": "030c8bacc201", + "id": "02dcfb1c815e68cbe7979e537c8d763076b7b59229c03dbfa850124eeef28a12", + "instance": "02dcfb1c815e", "service": "valkey", "image": "sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571", - "started_at": "2026-11-05T16:14:58.861777302Z", + "started_at": "2026-11-06T07:31:09.641720258Z", "running": true, "exit_code": 0, "memory_bytes": 134217728, @@ -1105,7 +1137,7 @@ { "type": "volume", "destination": "/data", - "name": "lab4-check-753ba80cb3_valkey_data" + "name": "lab4-check-5aad8e0065_valkey_data" } ] } @@ -1114,10 +1146,10 @@ "idempotency_replay_after_restart": true }, "sigterm": { - "instance": "887ffd1465a8", + "instance": "19a474c69210", "unfinished_request": true, - "shutdown_seconds": 5.448, - "new_connections_refused_seconds": 0.162, + "shutdown_seconds": 5.523, + "new_connections_refused_seconds": 0.185, "exit_code": 0, "oom_killed": false }, @@ -1141,15 +1173,122 @@ "lb_ready": 503 } ], + "postgres_pause": { + "sql_without_server_timeout": { + "elapsed_seconds": 2.0, + "connection_closed": true + }, + "direct_ready": { + "status": 503, + "elapsed_seconds": 2.013 + }, + "direct_live": { + "status": 200, + "elapsed_seconds": 0.006 + }, + "repeated_ready": [ + { + "status": 503, + "elapsed_seconds": 2.107 + }, + { + "status": 503, + "elapsed_seconds": 2.097 + }, + { + "status": 503, + "elapsed_seconds": 2.094 + }, + { + "status": 503, + "elapsed_seconds": 2.012 + }, + { + "status": 503, + "elapsed_seconds": 2.009 + }, + { + "status": 503, + "elapsed_seconds": 2.095 + }, + { + "status": 503, + "elapsed_seconds": 2.094 + }, + { + "status": 503, + "elapsed_seconds": 2.099 + }, + { + "status": 503, + "elapsed_seconds": 2.007 + }, + { + "status": 503, + "elapsed_seconds": 2.102 + } + ], + "ready_after_12_seconds": { + "status": 503, + "elapsed_seconds": 2.008 + }, + "live_after_repeated_requests": { + "status": 200, + "elapsed_seconds": 0.006 + }, + "paused_seconds": 14.572, + "recovered_ready": { + "status": 200, + "elapsed_seconds": 0.009 + }, + "recovery_seconds": 1.975, + "api_not_restarted": true, + "existing_session_and_message_preserved": true, + "passed": true + }, "public_failover": { "passed": true, "containers_after": [ { - "id": "887ffd1465a8191971a9f04cb3deb79994889e288fbe4bc7224052fa3094a710", - "instance": "887ffd1465a8", + "id": "19a474c69210a7be1c6631830e8fa7f251cf6dd3890c7407843090383543ee75", + "instance": "19a474c69210", "service": "api", - "image": "sha256:cb6318e7f06fd815219d3324008ff58c6b6ebb44fb9ea9b76e43546a8b0f78d5", - "started_at": "2026-11-05T16:15:39.329511417Z", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:31:50.739701388Z", + "running": false, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + } + ] + }, + { + "id": "ca1f62dfc0156fc235dbcd2f52b58eec2282e6ba8d3e056de11ca92758c69df0", + "instance": "ca1f62dfc015", + "service": "api", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:31:10.013737549Z", "running": true, "exit_code": 0, "memory_bytes": 402653184, @@ -1177,78 +1316,44 @@ "name": null } ] - }, - { - "id": "8f41208a64062523e85a27c5d759029d6c9940df78417691c85900dcb8e6f158", - "instance": "8f41208a6406", - "service": "api", - "image": "sha256:cb6318e7f06fd815219d3324008ff58c6b6ebb44fb9ea9b76e43546a8b0f78d5", - "started_at": "2026-11-05T16:14:59.190444177Z", - "running": false, - "exit_code": 0, - "memory_bytes": 402653184, - "nano_cpus": 1000000000, - "published_ports": {}, - "mounts": [ - { - "type": "bind", - "destination": "/run/secrets/s3_secret_access_key", - "name": null - }, - { - "type": "bind", - "destination": "/run/secrets/database_url", - "name": null - }, - { - "type": "bind", - "destination": "/run/secrets/cache_url", - "name": null - }, - { - "type": "bind", - "destination": "/run/secrets/s3_access_key_id", - "name": null - } - ] } ] }, "failovers": [ { - "label": "stop 887ffd1465a8", + "label": "stop 19a474c69210", "duration_seconds": 16.012, - "successful_requests": 1480, - "successful_writes": 739, + "successful_requests": 1420, + "successful_writes": 709, "errors": 4, - "successful_rps": 92.43, + "successful_rps": 88.68, "first_success_after_action_seconds": { - "POST": 0.006, - "GET": 0.008 + "POST": 0.051, + "GET": 0.004 }, "maximum_success_gap_seconds": { - "POST": 2.042, - "GET": 1.984 + "POST": 2.002, + "GET": 1.995 }, - "action_start_seconds": 1.001, - "action_end_seconds": 1.708, - "p50_ms": 9.065, - "p95_ms": 18.196, + "action_start_seconds": 1.006, + "action_end_seconds": 1.77, + "p50_ms": 10.62, + "p95_ms": 24.67, "confirmed_data_preserved": true, "duplicate_messages": 0, "instances": [ - "887ffd1465a8", - "8f41208a6406" + "19a474c69210", + "ca1f62dfc015" ], "per_second": [ { "second": 0, - "successes": 117, + "successes": 111, "errors": 0 }, { "second": 1, - "successes": 38, + "successes": 28, "errors": 0 }, { @@ -1258,27 +1363,27 @@ }, { "second": 3, - "successes": 62, + "successes": 70, "errors": 4 }, { "second": 4, - "successes": 107, + "successes": 101, "errors": 0 }, { "second": 5, - "successes": 104, + "successes": 94, "errors": 0 }, { "second": 6, - "successes": 105, + "successes": 106, "errors": 0 }, { "second": 7, - "successes": 112, + "successes": 100, "errors": 0 }, { @@ -1288,27 +1393,27 @@ }, { "second": 9, - "successes": 111, + "successes": 107, "errors": 0 }, { "second": 10, - "successes": 105, + "successes": 93, "errors": 0 }, { "second": 11, - "successes": 109, + "successes": 97, "errors": 0 }, { "second": 12, - "successes": 99, + "successes": 101, "errors": 0 }, { "second": 13, - "successes": 104, + "successes": 107, "errors": 0 }, { @@ -1323,38 +1428,38 @@ }, { "second": 16, - "successes": 2, + "successes": 0, "errors": 0 } ], "error_samples": [ [ - 3.325078833993757, + 3.2868632079917006, "POST", - "POST /api/v1/chats/90558f6b-bac8-4d25-9200-0b6550fc7907/messages: connection/TLS failure (TimeoutError)" + "POST /api/v1/chats/ba1880c0-8f0e-4d7d-ac12-a00beb77fb97/messages: connection/TLS failure (TimeoutError)" ], [ - 3.3309984169900417, + 3.2872469169960823, + "POST", + "POST /api/v1/chats/ba1880c0-8f0e-4d7d-ac12-a00beb77fb97/messages: connection/TLS failure (TimeoutError)" + ], + [ + 3.2959944579924922, + "POST", + "POST /api/v1/chats/ba1880c0-8f0e-4d7d-ac12-a00beb77fb97/messages: connection/TLS failure (TimeoutError)" + ], + [ + 3.3688164169871015, "GET", - "GET /api/v1/chats/90558f6b-bac8-4d25-9200-0b6550fc7907/messages: connection/TLS failure (TimeoutError)" - ], - [ - 3.3486054169916315, - "POST", - "POST /api/v1/chats/90558f6b-bac8-4d25-9200-0b6550fc7907/messages: connection/TLS failure (TimeoutError)" - ], - [ - 3.414436291990569, - "POST", - "POST /api/v1/chats/90558f6b-bac8-4d25-9200-0b6550fc7907/messages: connection/TLS failure (TimeoutError)" + "GET /api/v1/chats/ba1880c0-8f0e-4d7d-ac12-a00beb77fb97/messages: connection/TLS failure (TimeoutError)" ] ], "action": { - "id": "887ffd1465a8191971a9f04cb3deb79994889e288fbe4bc7224052fa3094a710", - "instance": "887ffd1465a8", + "id": "19a474c69210a7be1c6631830e8fa7f251cf6dd3890c7407843090383543ee75", + "instance": "19a474c69210", "service": "api", - "image": "sha256:cb6318e7f06fd815219d3324008ff58c6b6ebb44fb9ea9b76e43546a8b0f78d5", - "started_at": "2026-11-05T16:15:39.329511417Z", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:32:49.121677512Z", "running": false, "exit_code": 0, "memory_bytes": 402653184, @@ -1385,29 +1490,29 @@ } }, { - "label": "stop 8f41208a6406", - "duration_seconds": 16.026, - "successful_requests": 1472, - "successful_writes": 735, + "label": "stop ca1f62dfc015", + "duration_seconds": 16.008, + "successful_requests": 1466, + "successful_writes": 732, "errors": 4, - "successful_rps": 91.85, + "successful_rps": 91.58, "first_success_after_action_seconds": { - "POST": 0.029, - "GET": 0.038 + "POST": 0.018, + "GET": 0.028 }, "maximum_success_gap_seconds": { - "POST": 1.986, - "GET": 1.99 + "POST": 2.002, + "GET": 1.999 }, - "action_start_seconds": 1.011, - "action_end_seconds": 1.806, - "p50_ms": 9.112, - "p95_ms": 20.767, + "action_start_seconds": 1.002, + "action_end_seconds": 1.832, + "p50_ms": 9.397, + "p95_ms": 18.616, "confirmed_data_preserved": true, "duplicate_messages": 0, "instances": [ - "887ffd1465a8", - "8f41208a6406" + "19a474c69210", + "ca1f62dfc015" ], "per_second": [ { @@ -1417,7 +1522,7 @@ }, { "second": 1, - "successes": 45, + "successes": 43, "errors": 0 }, { @@ -1427,22 +1532,22 @@ }, { "second": 3, - "successes": 53, + "successes": 52, "errors": 4 }, { "second": 4, - "successes": 107, + "successes": 108, "errors": 0 }, { "second": 5, - "successes": 107, + "successes": 103, "errors": 0 }, { "second": 6, - "successes": 107, + "successes": 106, "errors": 0 }, { @@ -1452,78 +1557,78 @@ }, { "second": 8, - "successes": 99, + "successes": 95, "errors": 0 }, { "second": 9, - "successes": 108, - "errors": 0 - }, - { - "second": 10, - "successes": 100, - "errors": 0 - }, - { - "second": 11, - "successes": 107, - "errors": 0 - }, - { - "second": 12, - "successes": 112, - "errors": 0 - }, - { - "second": 13, - "successes": 99, - "errors": 0 - }, - { - "second": 14, "successes": 103, "errors": 0 }, + { + "second": 10, + "successes": 103, + "errors": 0 + }, + { + "second": 11, + "successes": 106, + "errors": 0 + }, + { + "second": 12, + "successes": 102, + "errors": 0 + }, + { + "second": 13, + "successes": 109, + "errors": 0 + }, + { + "second": 14, + "successes": 109, + "errors": 0 + }, { "second": 15, - "successes": 98, + "successes": 103, "errors": 0 }, { "second": 16, - "successes": 4, + "successes": 1, "errors": 0 } ], "error_samples": [ [ - 3.379369415997644, + 3.398277083004359, "POST", - "POST /api/v1/chats/4f88aa82-8938-466a-95a0-d514336024ea/messages: connection/TLS failure (TimeoutError)" + "POST /api/v1/chats/a2185e1c-f008-4190-ae30-86115db5f6df/messages: connection/TLS failure (TimeoutError)" ], [ - 3.3918063749879366, + 3.405396166999708, "POST", - "POST /api/v1/chats/4f88aa82-8938-466a-95a0-d514336024ea/messages: connection/TLS failure (TimeoutError)" + "POST /api/v1/chats/a2185e1c-f008-4190-ae30-86115db5f6df/messages: connection/TLS failure (TimeoutError)" ], [ - 3.399761665990809, + 3.4118699169921456, "POST", - "POST /api/v1/chats/4f88aa82-8938-466a-95a0-d514336024ea/messages: connection/TLS failure (TimeoutError)" + "POST /api/v1/chats/a2185e1c-f008-4190-ae30-86115db5f6df/messages: connection/TLS failure (TimeoutError)" ], [ - 3.473328790991218, + 3.4863260830024956, "GET", - "GET /api/v1/chats/4f88aa82-8938-466a-95a0-d514336024ea/messages: connection/TLS failure (TimeoutError)" + "GET /api/v1/chats/a2185e1c-f008-4190-ae30-86115db5f6df/messages: connection/TLS failure (TimeoutError)" ] ], "action": { - "id": "8f41208a64062523e85a27c5d759029d6c9940df78417691c85900dcb8e6f158", - "instance": "8f41208a6406", + "id": "ca1f62dfc0156fc235dbcd2f52b58eec2282e6ba8d3e056de11ca92758c69df0", + "instance": "ca1f62dfc015", "service": "api", - "image": "sha256:cb6318e7f06fd815219d3324008ff58c6b6ebb44fb9ea9b76e43546a8b0f78d5", - "started_at": "2026-11-05T16:16:18.896292297Z", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:31:10.013737549Z", "running": false, "exit_code": 0, "memory_bytes": 402653184, @@ -1556,35 +1661,35 @@ ], "rolling_update": { "label": "sequential API replacement", - "duration_seconds": 22.153, - "successful_requests": 1960, - "successful_writes": 979, + "duration_seconds": 24.869, + "successful_requests": 2254, + "successful_writes": 1128, "errors": 8, - "successful_rps": 88.47, + "successful_rps": 90.63, "first_success_after_action_seconds": { - "POST": 0.035, - "GET": 0.041 + "POST": 0.01, + "GET": 0.021 }, "maximum_success_gap_seconds": { - "POST": 1.048, - "GET": 1.031 + "POST": 1.04, + "GET": 1.04 }, - "action_start_seconds": 1.004, - "action_end_seconds": 19.149, - "p50_ms": 8.859, - "p95_ms": 17.468, + "action_start_seconds": 1.009, + "action_end_seconds": 21.862, + "p50_ms": 8.907, + "p95_ms": 16.321, "confirmed_data_preserved": true, "duplicate_messages": 0, "instances": [ - "4479505e52f5", - "887ffd1465a8", - "8f41208a6406", - "9be50a7967ed" + "19a474c69210", + "63549c0d63ba", + "ca1f62dfc015", + "ed4f3c64a9f6" ], "per_second": [ { "second": 0, - "successes": 120, + "successes": 112, "errors": 0 }, { @@ -1594,37 +1699,37 @@ }, { "second": 2, - "successes": 110, + "successes": 104, "errors": 0 }, { "second": 3, - "successes": 102, + "successes": 101, "errors": 0 }, { "second": 4, - "successes": 111, + "successes": 112, "errors": 0 }, { "second": 5, - "successes": 52, + "successes": 46, "errors": 0 }, { "second": 6, - "successes": 3, + "successes": 44, "errors": 4 }, { "second": 7, - "successes": 51, + "successes": 12, "errors": 0 }, { "second": 8, - "successes": 102, + "successes": 112, "errors": 0 }, { @@ -1634,105 +1739,149 @@ }, { "second": 10, - "successes": 108, + "successes": 105, "errors": 0 }, { "second": 11, - "successes": 113, + "successes": 111, "errors": 0 }, { "second": 12, - "successes": 30, + "successes": 107, "errors": 0 }, { "second": 13, - "successes": 43, - "errors": 4 - }, - { - "second": 14, - "successes": 28, - "errors": 0 - }, - { - "second": 15, - "successes": 108, - "errors": 0 - }, - { - "second": 16, - "successes": 106, - "errors": 0 - }, - { - "second": 17, - "successes": 112, - "errors": 0 - }, - { - "second": 18, - "successes": 106, - "errors": 0 - }, - { - "second": 19, "successes": 109, "errors": 0 }, { - "second": 20, - "successes": 103, + "second": 14, + "successes": 51, "errors": 0 }, { - "second": 21, + "second": 15, + "successes": 43, + "errors": 4 + }, + { + "second": 16, + "successes": 12, + "errors": 0 + }, + { + "second": 17, + "successes": 107, + "errors": 0 + }, + { + "second": 18, + "successes": 101, + "errors": 0 + }, + { + "second": 19, "successes": 112, "errors": 0 }, + { + "second": 20, + "successes": 112, + "errors": 0 + }, + { + "second": 21, + "successes": 106, + "errors": 0 + }, { "second": 22, - "successes": 16, + "successes": 112, + "errors": 0 + }, + { + "second": 23, + "successes": 111, + "errors": 0 + }, + { + "second": 24, + "successes": 97, "errors": 0 } ], "error_samples": [ [ - 6.467671833001077, + 6.444048083998496, "POST", - "POST /api/v1/chats/07035b84-deb6-4ad8-ba1e-5b435fcf227e/messages: expected [201], received 503" + "POST /api/v1/chats/0fb92954-72c5-4454-bdb6-48e8b0f30ba9/messages: expected [201], received 503" ], [ - 6.489907541996217, + 6.461145833993214, "POST", - "POST /api/v1/chats/07035b84-deb6-4ad8-ba1e-5b435fcf227e/messages: expected [201], received 503" + "POST /api/v1/chats/0fb92954-72c5-4454-bdb6-48e8b0f30ba9/messages: expected [201], received 503" ], [ - 6.499776625001687, + 6.465980209002737, "GET", - "GET /api/v1/chats/07035b84-deb6-4ad8-ba1e-5b435fcf227e/messages: expected [200], received 503" + "GET /api/v1/chats/0fb92954-72c5-4454-bdb6-48e8b0f30ba9/messages: expected [200], received 503" ], [ - 6.534639249992324, + 6.52059274999192, "GET", - "GET /api/v1/chats/07035b84-deb6-4ad8-ba1e-5b435fcf227e/messages: expected [200], received 503" + "GET /api/v1/chats/0fb92954-72c5-4454-bdb6-48e8b0f30ba9/messages: expected [200], received 503" ], [ - 13.272986249998212, + 15.429505041989614, "POST", - "POST /api/v1/chats/07035b84-deb6-4ad8-ba1e-5b435fcf227e/messages: expected [201], received 503" + "POST /api/v1/chats/0fb92954-72c5-4454-bdb6-48e8b0f30ba9/messages: expected [201], received 503" ] ], "action": { "before": [ { - "id": "887ffd1465a8191971a9f04cb3deb79994889e288fbe4bc7224052fa3094a710", - "instance": "887ffd1465a8", + "id": "19a474c69210a7be1c6631830e8fa7f251cf6dd3890c7407843090383543ee75", + "instance": "19a474c69210", "service": "api", - "image": "sha256:cb6318e7f06fd815219d3324008ff58c6b6ebb44fb9ea9b76e43546a8b0f78d5", - "started_at": "2026-11-05T16:16:43.393904378Z", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:33:13.735740384Z", + "running": true, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + } + ] + }, + { + "id": "ca1f62dfc0156fc235dbcd2f52b58eec2282e6ba8d3e056de11ca92758c69df0", + "instance": "ca1f62dfc015", + "service": "api", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:33:38.250478257Z", "running": true, "exit_code": 0, "memory_bytes": 402653184, @@ -1762,45 +1911,11 @@ ] }, { - "id": "8f41208a64062523e85a27c5d759029d6c9940df78417691c85900dcb8e6f158", - "instance": "8f41208a6406", - "service": "api", - "image": "sha256:cb6318e7f06fd815219d3324008ff58c6b6ebb44fb9ea9b76e43546a8b0f78d5", - "started_at": "2026-11-05T16:17:07.789637667Z", - "running": true, - "exit_code": 0, - "memory_bytes": 402653184, - "nano_cpus": 1000000000, - "published_ports": {}, - "mounts": [ - { - "type": "bind", - "destination": "/run/secrets/s3_secret_access_key", - "name": null - }, - { - "type": "bind", - "destination": "/run/secrets/database_url", - "name": null - }, - { - "type": "bind", - "destination": "/run/secrets/cache_url", - "name": null - }, - { - "type": "bind", - "destination": "/run/secrets/s3_access_key_id", - "name": null - } - ] - }, - { - "id": "10143cbf3e42de96c7a3078ea5b42d24bbf9702a4f46e57d4a9aa2a7ffbbab61", - "instance": "10143cbf3e42", + "id": "bfaeabc178285f04b23c0578b9c2d22b8e54ab97c4cb0d70eb874c46dd92acdb", + "instance": "bfaeabc17828", "service": "cleanup", - "image": "sha256:cb6318e7f06fd815219d3324008ff58c6b6ebb44fb9ea9b76e43546a8b0f78d5", - "started_at": "2026-11-05T16:15:06.604429847Z", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:31:17.411326011Z", "running": true, "exit_code": 0, "memory_bytes": 402653184, @@ -1825,11 +1940,11 @@ ] }, { - "id": "aecf8751126b66c807155750bf349fa6af90aa2a322bf078bd903bae7fba5b53", - "instance": "aecf8751126b", + "id": "3bc52373c3d129bac2f4765eaf221fbe4461ae859616920aba3f3cf952a9e449", + "instance": "3bc52373c3d1", "service": "lb", "image": "sha256:3a819392f5a6af2a9203d1c63ff0feba74f0c940623a558511be06d7d85e361c", - "started_at": "2026-11-05T16:14:58.790557135Z", + "started_at": "2026-11-06T07:31:09.542736424Z", "running": true, "exit_code": 0, "memory_bytes": 67108864, @@ -1838,7 +1953,7 @@ "8080/tcp": [ { "HostIp": "127.0.0.1", - "HostPort": "65032" + "HostPort": "57722" } ] }, @@ -1851,11 +1966,11 @@ ] }, { - "id": "1d892ce3140664f694767e9737f67ab691ca4e25b8129874d2e2fe1f20bbddda", - "instance": "1d892ce31406", + "id": "60274b0d5ab994316a8b828669ce937b08e758c5ce6dfc6d8d915d5cf4c46046", + "instance": "60274b0d5ab9", "service": "postgres", "image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", - "started_at": "2026-11-05T16:15:52.946969591Z", + "started_at": "2026-11-06T07:32:04.849361089Z", "running": true, "exit_code": 0, "memory_bytes": 536870912, @@ -1865,7 +1980,7 @@ { "type": "volume", "destination": "/var/lib/postgresql/data", - "name": "lab4-check-753ba80cb3_postgres_data" + "name": "lab4-check-5aad8e0065_postgres_data" }, { "type": "bind", @@ -1875,11 +1990,11 @@ ] }, { - "id": "3c9d7b1f26816060cb94a7bcf246d06c04c1fa3c6cf104f38e051eca1c11bfe1", - "instance": "3c9d7b1f2681", + "id": "0ba4ea69ac8dca26d0348fb351aaf286164ee5e72b2123ffdb89579469af53c9", + "instance": "0ba4ea69ac8d", "service": "s3", "image": "sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", - "started_at": "2026-11-05T16:15:58.478104926Z", + "started_at": "2026-11-06T07:32:11.905651425Z", "running": true, "exit_code": 0, "memory_bytes": 268435456, @@ -1894,16 +2009,16 @@ { "type": "volume", "destination": "/data", - "name": "5884fdd395ed5d5bb60b03bbb71fdd685825042d009cda0a137b77e42fef1ebf" + "name": "56e17d6e3f080fb7912c461068d6c41768244d709e8a55849e355a0895694340" } ] }, { - "id": "b403c79e680014fba72123d64d7742d70a57a4a08b4c7cd6f036eb752a63456e", - "instance": "b403c79e6800", + "id": "df540b4c43cfa85275ff158c300583671dbb3eaf12e75b9f7497f5414b3cb36b", + "instance": "df540b4c43cf", "service": "s3-storage", "image": "sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", - "started_at": "2026-11-05T16:15:06.569589264Z", + "started_at": "2026-11-06T07:31:17.386629011Z", "running": true, "exit_code": 0, "memory_bytes": 536870912, @@ -1913,16 +2028,16 @@ { "type": "volume", "destination": "/data", - "name": "lab4-check-753ba80cb3_s3_data" + "name": "lab4-check-5aad8e0065_s3_data" } ] }, { - "id": "030c8bacc201f114875911d3e3ddd3ee159a0a266d849104f775b2615b941937", - "instance": "030c8bacc201", + "id": "02dcfb1c815e68cbe7979e537c8d763076b7b59229c03dbfa850124eeef28a12", + "instance": "02dcfb1c815e", "service": "valkey", "image": "sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571", - "started_at": "2026-11-05T16:15:46.860700088Z", + "started_at": "2026-11-06T07:31:58.693966586Z", "running": true, "exit_code": 0, "memory_bytes": 134217728, @@ -1932,18 +2047,18 @@ { "type": "volume", "destination": "/data", - "name": "lab4-check-753ba80cb3_valkey_data" + "name": "lab4-check-5aad8e0065_valkey_data" } ] } ], "after": [ { - "id": "9be50a7967ed59707098e73e42c6f0b054bf4d60ee4dad0a9f8f0ed6afc148e6", - "instance": "9be50a7967ed", + "id": "ed4f3c64a9f6b1e8c80f4e061c2017aae179b3feb768d662fd087f2cc96c52fd", + "instance": "ed4f3c64a9f6", "service": "api", - "image": "sha256:413691a40751a490bab701b9498431365ae10bfd081d55decba718e5e90a0f5d", - "started_at": "2026-11-05T16:17:20.086358506Z", + "image": "sha256:93c838d55d79a0001346ecab28f72621d9540ba5c52eaeb1ca6020286dd05fd6", + "started_at": "2026-11-06T07:33:50.659312471Z", "running": true, "exit_code": 0, "memory_bytes": 402653184, @@ -1973,11 +2088,11 @@ ] }, { - "id": "4479505e52f5bf17b2ceb2d0d799a4616bb846086cbbcc99d100aa1dc69571a7", - "instance": "4479505e52f5", + "id": "63549c0d63ba2fcee39baf26eb9ffa0da68e38d2d781a81e4a29f2eff6a0afb8", + "instance": "63549c0d63ba", "service": "api", - "image": "sha256:413691a40751a490bab701b9498431365ae10bfd081d55decba718e5e90a0f5d", - "started_at": "2026-11-05T16:17:26.864185551Z", + "image": "sha256:93c838d55d79a0001346ecab28f72621d9540ba5c52eaeb1ca6020286dd05fd6", + "started_at": "2026-11-06T07:33:59.668146169Z", "running": true, "exit_code": 0, "memory_bytes": 402653184, @@ -2007,11 +2122,11 @@ ] }, { - "id": "10143cbf3e42de96c7a3078ea5b42d24bbf9702a4f46e57d4a9aa2a7ffbbab61", - "instance": "10143cbf3e42", + "id": "bfaeabc178285f04b23c0578b9c2d22b8e54ab97c4cb0d70eb874c46dd92acdb", + "instance": "bfaeabc17828", "service": "cleanup", - "image": "sha256:cb6318e7f06fd815219d3324008ff58c6b6ebb44fb9ea9b76e43546a8b0f78d5", - "started_at": "2026-11-05T16:15:06.604429847Z", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:31:17.411326011Z", "running": true, "exit_code": 0, "memory_bytes": 402653184, @@ -2036,11 +2151,11 @@ ] }, { - "id": "aecf8751126b66c807155750bf349fa6af90aa2a322bf078bd903bae7fba5b53", - "instance": "aecf8751126b", + "id": "3bc52373c3d129bac2f4765eaf221fbe4461ae859616920aba3f3cf952a9e449", + "instance": "3bc52373c3d1", "service": "lb", "image": "sha256:3a819392f5a6af2a9203d1c63ff0feba74f0c940623a558511be06d7d85e361c", - "started_at": "2026-11-05T16:14:58.790557135Z", + "started_at": "2026-11-06T07:31:09.542736424Z", "running": true, "exit_code": 0, "memory_bytes": 67108864, @@ -2049,7 +2164,7 @@ "8080/tcp": [ { "HostIp": "127.0.0.1", - "HostPort": "65032" + "HostPort": "57722" } ] }, @@ -2062,11 +2177,11 @@ ] }, { - "id": "1d892ce3140664f694767e9737f67ab691ca4e25b8129874d2e2fe1f20bbddda", - "instance": "1d892ce31406", + "id": "60274b0d5ab994316a8b828669ce937b08e758c5ce6dfc6d8d915d5cf4c46046", + "instance": "60274b0d5ab9", "service": "postgres", "image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", - "started_at": "2026-11-05T16:15:52.946969591Z", + "started_at": "2026-11-06T07:32:04.849361089Z", "running": true, "exit_code": 0, "memory_bytes": 536870912, @@ -2076,7 +2191,7 @@ { "type": "volume", "destination": "/var/lib/postgresql/data", - "name": "lab4-check-753ba80cb3_postgres_data" + "name": "lab4-check-5aad8e0065_postgres_data" }, { "type": "bind", @@ -2086,11 +2201,11 @@ ] }, { - "id": "3c9d7b1f26816060cb94a7bcf246d06c04c1fa3c6cf104f38e051eca1c11bfe1", - "instance": "3c9d7b1f2681", + "id": "0ba4ea69ac8dca26d0348fb351aaf286164ee5e72b2123ffdb89579469af53c9", + "instance": "0ba4ea69ac8d", "service": "s3", "image": "sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", - "started_at": "2026-11-05T16:15:58.478104926Z", + "started_at": "2026-11-06T07:32:11.905651425Z", "running": true, "exit_code": 0, "memory_bytes": 268435456, @@ -2105,16 +2220,16 @@ { "type": "volume", "destination": "/data", - "name": "5884fdd395ed5d5bb60b03bbb71fdd685825042d009cda0a137b77e42fef1ebf" + "name": "56e17d6e3f080fb7912c461068d6c41768244d709e8a55849e355a0895694340" } ] }, { - "id": "b403c79e680014fba72123d64d7742d70a57a4a08b4c7cd6f036eb752a63456e", - "instance": "b403c79e6800", + "id": "df540b4c43cfa85275ff158c300583671dbb3eaf12e75b9f7497f5414b3cb36b", + "instance": "df540b4c43cf", "service": "s3-storage", "image": "sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", - "started_at": "2026-11-05T16:15:06.569589264Z", + "started_at": "2026-11-06T07:31:17.386629011Z", "running": true, "exit_code": 0, "memory_bytes": 536870912, @@ -2124,16 +2239,16 @@ { "type": "volume", "destination": "/data", - "name": "lab4-check-753ba80cb3_s3_data" + "name": "lab4-check-5aad8e0065_s3_data" } ] }, { - "id": "030c8bacc201f114875911d3e3ddd3ee159a0a266d849104f775b2615b941937", - "instance": "030c8bacc201", + "id": "02dcfb1c815e68cbe7979e537c8d763076b7b59229c03dbfa850124eeef28a12", + "instance": "02dcfb1c815e", "service": "valkey", "image": "sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571", - "started_at": "2026-11-05T16:15:46.860700088Z", + "started_at": "2026-11-06T07:31:58.693966586Z", "running": true, "exit_code": 0, "memory_bytes": 134217728, @@ -2143,7 +2258,7 @@ { "type": "volume", "destination": "/data", - "name": "lab4-check-753ba80cb3_valkey_data" + "name": "lab4-check-5aad8e0065_valkey_data" } ] } diff --git a/evidence/lab4-checks.txt b/evidence/lab4-checks.txt index 3eb74f0..991fcd1 100644 --- a/evidence/lab4-checks.txt +++ b/evidence/lab4-checks.txt @@ -1,4 +1,4 @@ -Command: ./scripts/check-docker.sh; project: lab4-check-753ba80cb3 +Command: ./scripts/check-docker.sh; project: lab4-check-5aad8e0065 Docker: 29.0.1 aarch64 CPUs=10 RAM=8217448448 Local .env and secrets are ready; values are not printed #1 [internal] load local bake definitions @@ -21,25 +21,25 @@ Local .env and secrets are ready; values are not printed #5 DONE 0.0s #6 [1/8] FROM docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 -#6 resolve docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 0.0s done +#6 resolve docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 done #6 DONE 0.0s #7 [4/8] RUN python -m pip install --no-cache-dir -r requirements.txt && python -m pip check #7 CACHED -#8 [5/8] RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app +#8 [7/8] COPY migrations/ ./migrations/ #8 CACHED #9 [2/8] WORKDIR /app #9 CACHED -#10 [7/8] COPY migrations/ ./migrations/ +#10 [6/8] COPY src/ ./src/ #10 CACHED #11 [3/8] COPY requirements.txt . #11 CACHED -#12 [6/8] COPY src/ ./src/ +#12 [5/8] RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app #12 CACHED #13 [8/8] COPY research/ ./research/ @@ -47,97 +47,97 @@ Local .env and secrets are ready; values are not printed #14 exporting to image #14 exporting layers done -#14 exporting manifest sha256:1c6cb19983cdfb308e1ec43f59338922d2fec797ff8fdc9636526f9de20a0693 done -#14 exporting config sha256:bc7c1361d0a40f2beb749094a98a091632f75a255c5841fcdf80a33b166720eb done -#14 exporting attestation manifest sha256:ee8ad230ca1748e0ae7559c1966adbe734edbc78280e59f8553b3f6ad1139713 done -#14 exporting manifest list sha256:cb6318e7f06fd815219d3324008ff58c6b6ebb44fb9ea9b76e43546a8b0f78d5 done +#14 exporting manifest sha256:644f5d408505f5982b75f18404860de692ee837b2b2e18f5c92b0b057081b7ec done +#14 exporting config sha256:726011343b32f5b6277348f2f8b38f95b8c31754ae0d21336375124a18aa2e0e done +#14 exporting attestation manifest sha256:1146b210a20471d58663fb48a3a24ba22986971d0cce8777f912886768f43306 done +#14 exporting manifest list sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d done #14 naming to docker.io/library/mtusi-messenger:lab4 done #14 unpacking to docker.io/library/mtusi-messenger:lab4 done -#14 DONE 0.0s +#14 DONE 0.1s #15 resolving provenance for metadata file #15 DONE 0.0s mtusi-messenger:lab4 Built - Network lab4-check-753ba80cb3_default Creating - Network lab4-check-753ba80cb3_default Created - Network lab4-check-753ba80cb3_s3-internal Creating - Network lab4-check-753ba80cb3_s3-internal Created - Volume lab4-check-753ba80cb3_s3_data Creating - Volume lab4-check-753ba80cb3_s3_data Created - Volume lab4-check-753ba80cb3_postgres_data Creating - Volume lab4-check-753ba80cb3_postgres_data Created - Volume lab4-check-753ba80cb3_valkey_data Creating - Volume lab4-check-753ba80cb3_valkey_data Created - Container lab4-check-753ba80cb3-postgres-1 Creating - Container lab4-check-753ba80cb3-valkey-1 Creating - Container lab4-check-753ba80cb3-s3-storage-1 Creating - Container lab4-check-753ba80cb3-valkey-1 Created - Container lab4-check-753ba80cb3-s3-storage-1 Created - Container lab4-check-753ba80cb3-s3-1 Creating - Container lab4-check-753ba80cb3-postgres-1 Created - Container lab4-check-753ba80cb3-migrate-1 Creating - Container lab4-check-753ba80cb3-s3-1 Created - Container lab4-check-753ba80cb3-s3-bootstrap-1 Creating - Container lab4-check-753ba80cb3-migrate-1 Created - Container lab4-check-753ba80cb3-s3-bootstrap-1 Created - Container lab4-check-753ba80cb3-api-2 Creating - Container lab4-check-753ba80cb3-api-1 Creating - Container lab4-check-753ba80cb3-cleanup-1 Creating - Container lab4-check-753ba80cb3-api-1 Created - Container lab4-check-753ba80cb3-api-2 Created - Container lab4-check-753ba80cb3-lb-1 Creating - Container lab4-check-753ba80cb3-cleanup-1 Created - Container lab4-check-753ba80cb3-lb-1 Created - Container lab4-check-753ba80cb3-s3-storage-1 Starting - Container lab4-check-753ba80cb3-valkey-1 Starting - Container lab4-check-753ba80cb3-postgres-1 Starting - Container lab4-check-753ba80cb3-s3-storage-1 Started - Container lab4-check-753ba80cb3-s3-storage-1 Waiting - Container lab4-check-753ba80cb3-valkey-1 Started - Container lab4-check-753ba80cb3-postgres-1 Started - Container lab4-check-753ba80cb3-postgres-1 Waiting - Container lab4-check-753ba80cb3-postgres-1 Healthy - Container lab4-check-753ba80cb3-migrate-1 Starting - Container lab4-check-753ba80cb3-migrate-1 Started - Container lab4-check-753ba80cb3-s3-storage-1 Healthy - Container lab4-check-753ba80cb3-s3-1 Starting - Container lab4-check-753ba80cb3-s3-1 Started - Container lab4-check-753ba80cb3-s3-1 Waiting - Container lab4-check-753ba80cb3-s3-1 Healthy - Container lab4-check-753ba80cb3-s3-bootstrap-1 Starting - Container lab4-check-753ba80cb3-s3-bootstrap-1 Started - Container lab4-check-753ba80cb3-s3-bootstrap-1 Waiting - Container lab4-check-753ba80cb3-migrate-1 Waiting - Container lab4-check-753ba80cb3-postgres-1 Waiting - Container lab4-check-753ba80cb3-valkey-1 Waiting - Container lab4-check-753ba80cb3-migrate-1 Waiting - Container lab4-check-753ba80cb3-s3-1 Waiting - Container lab4-check-753ba80cb3-s3-bootstrap-1 Waiting - Container lab4-check-753ba80cb3-migrate-1 Exited - Container lab4-check-753ba80cb3-postgres-1 Healthy - Container lab4-check-753ba80cb3-migrate-1 Exited - Container lab4-check-753ba80cb3-valkey-1 Healthy - Container lab4-check-753ba80cb3-s3-1 Healthy - Container lab4-check-753ba80cb3-s3-bootstrap-1 Exited - Container lab4-check-753ba80cb3-api-2 Starting - Container lab4-check-753ba80cb3-s3-bootstrap-1 Exited - Container lab4-check-753ba80cb3-cleanup-1 Starting - Container lab4-check-753ba80cb3-cleanup-1 Started - Container lab4-check-753ba80cb3-api-2 Started - Container lab4-check-753ba80cb3-api-1 Starting - Container lab4-check-753ba80cb3-api-1 Started - Container lab4-check-753ba80cb3-api-2 Waiting - Container lab4-check-753ba80cb3-api-1 Waiting - Container lab4-check-753ba80cb3-api-2 Healthy - Container lab4-check-753ba80cb3-api-1 Healthy - Container lab4-check-753ba80cb3-lb-1 Starting - Container lab4-check-753ba80cb3-lb-1 Started -API ready through LB: 887ffd1465a8, 8f41208a6406 + Network lab4-check-5aad8e0065_default Creating + Network lab4-check-5aad8e0065_default Created + Network lab4-check-5aad8e0065_s3-internal Creating + Network lab4-check-5aad8e0065_s3-internal Created + Volume lab4-check-5aad8e0065_valkey_data Creating + Volume lab4-check-5aad8e0065_valkey_data Created + Volume lab4-check-5aad8e0065_postgres_data Creating + Volume lab4-check-5aad8e0065_postgres_data Created + Volume lab4-check-5aad8e0065_s3_data Creating + Volume lab4-check-5aad8e0065_s3_data Created + Container lab4-check-5aad8e0065-s3-storage-1 Creating + Container lab4-check-5aad8e0065-valkey-1 Creating + Container lab4-check-5aad8e0065-postgres-1 Creating + Container lab4-check-5aad8e0065-valkey-1 Created + Container lab4-check-5aad8e0065-s3-storage-1 Created + Container lab4-check-5aad8e0065-s3-1 Creating + Container lab4-check-5aad8e0065-postgres-1 Created + Container lab4-check-5aad8e0065-migrate-1 Creating + Container lab4-check-5aad8e0065-s3-1 Created + Container lab4-check-5aad8e0065-s3-bootstrap-1 Creating + Container lab4-check-5aad8e0065-migrate-1 Created + Container lab4-check-5aad8e0065-s3-bootstrap-1 Created + Container lab4-check-5aad8e0065-api-2 Creating + Container lab4-check-5aad8e0065-api-1 Creating + Container lab4-check-5aad8e0065-cleanup-1 Creating + Container lab4-check-5aad8e0065-api-1 Created + Container lab4-check-5aad8e0065-cleanup-1 Created + Container lab4-check-5aad8e0065-api-2 Created + Container lab4-check-5aad8e0065-lb-1 Creating + Container lab4-check-5aad8e0065-lb-1 Created + Container lab4-check-5aad8e0065-postgres-1 Starting + Container lab4-check-5aad8e0065-valkey-1 Starting + Container lab4-check-5aad8e0065-s3-storage-1 Starting + Container lab4-check-5aad8e0065-s3-storage-1 Started + Container lab4-check-5aad8e0065-s3-storage-1 Waiting + Container lab4-check-5aad8e0065-valkey-1 Started + Container lab4-check-5aad8e0065-postgres-1 Started + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Starting + Container lab4-check-5aad8e0065-migrate-1 Started + Container lab4-check-5aad8e0065-s3-storage-1 Healthy + Container lab4-check-5aad8e0065-s3-1 Starting + Container lab4-check-5aad8e0065-s3-1 Started + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Starting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Started + Container lab4-check-5aad8e0065-migrate-1 Waiting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Waiting + Container lab4-check-5aad8e0065-valkey-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Waiting + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Exited + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Exited + Container lab4-check-5aad8e0065-valkey-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Exited + Container lab4-check-5aad8e0065-api-2 Starting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Exited + Container lab4-check-5aad8e0065-cleanup-1 Starting + Container lab4-check-5aad8e0065-cleanup-1 Started + Container lab4-check-5aad8e0065-api-2 Started + Container lab4-check-5aad8e0065-api-1 Starting + Container lab4-check-5aad8e0065-api-1 Started + Container lab4-check-5aad8e0065-api-2 Waiting + Container lab4-check-5aad8e0065-api-1 Waiting + Container lab4-check-5aad8e0065-api-2 Healthy + Container lab4-check-5aad8e0065-api-1 Healthy + Container lab4-check-5aad8e0065-lb-1 Starting + Container lab4-check-5aad8e0065-lb-1 Started +API ready through LB: 19a474c69210, ca1f62dfc015 PASS: 2 real API containers, CPU/RAM limits, API non-root and no public ports; only LB published -Проверяем сервер: http://127.0.0.1:65032 +Проверяем сервер: http://127.0.0.1:57722 /Users/vanek/Desktop/VNPlabs/VNPlab4/mtusi2026lab4/.venv/bin/python scripts/check_contract.py PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. -/Users/vanek/Desktop/VNPlabs/VNPlab4/mtusi2026lab4/.venv/bin/python tests/smoke.py --base-url "http://127.0.0.1:65032" +/Users/vanek/Desktop/VNPlabs/VNPlab4/mtusi2026lab4/.venv/bin/python tests/smoke.py --base-url "http://127.0.0.1:57722" PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. test_chat_access (test_api.ApiTests.test_chat_access) ... ok test_chat_pagination_and_cursor_owner (test_api.ApiTests.test_chat_pagination_and_cursor_owner) ... ok @@ -180,225 +180,228 @@ test_unicode_password_with_colon_and_private_response (test_sessions.SessionTest test_wrong_password_and_basic_only_for_login (test_sessions.SessionTests.test_wrong_password_and_basic_only_for_login) ... ok ---------------------------------------------------------------------- -Ran 39 tests in 9.473s +Ran 39 tests in 9.593s OK PASS: 10 clients sent 100 messages; history contains exactly 100 unique messages. - Container lab4-check-753ba80cb3-postgres-1 Running - Container lab4-check-753ba80cb3-postgres-1 Running + Container lab4-check-5aad8e0065-postgres-1 Running + Container lab4-check-5aad8e0065-postgres-1 Running PASS: migrations only in one-shot service; repeated execution preserves data - Container lab4-check-753ba80cb3-cleanup-1 Stopping - Container lab4-check-753ba80cb3-cleanup-1 Stopped - Container lab4-check-753ba80cb3-s3-storage-1 Running - Container lab4-check-753ba80cb3-s3-1 Running - Container lab4-check-753ba80cb3-s3-storage-1 Waiting - Container lab4-check-753ba80cb3-s3-storage-1 Healthy - Container lab4-check-753ba80cb3-s3-storage-1 Running - Container lab4-check-753ba80cb3-s3-1 Running - Container lab4-check-753ba80cb3-s3-storage-1 Waiting - Container lab4-check-753ba80cb3-s3-storage-1 Healthy + Container lab4-check-5aad8e0065-cleanup-1 Stopping + Container lab4-check-5aad8e0065-cleanup-1 Stopped + Container lab4-check-5aad8e0065-s3-storage-1 Running + Container lab4-check-5aad8e0065-s3-1 Running + Container lab4-check-5aad8e0065-s3-storage-1 Waiting + Container lab4-check-5aad8e0065-s3-storage-1 Healthy + Container lab4-check-5aad8e0065-s3-storage-1 Running + Container lab4-check-5aad8e0065-s3-1 Running + Container lab4-check-5aad8e0065-s3-storage-1 Waiting + Container lab4-check-5aad8e0065-s3-storage-1 Healthy PASS: private S3, limited credentials, isolated administration, repeated bootstrap -Stopped API: 8f41208a6406 container=8f41208a64062523e85a27c5d759029d6c9940df78417691c85900dcb8e6f158 -API ready through LB: 887ffd1465a8 - Container lab4-check-753ba80cb3-s3-storage-1 Waiting - Container lab4-check-753ba80cb3-postgres-1 Waiting - Container lab4-check-753ba80cb3-s3-storage-1 Healthy - Container lab4-check-753ba80cb3-s3-1 Waiting - Container lab4-check-753ba80cb3-postgres-1 Healthy - Container lab4-check-753ba80cb3-migrate-1 Starting - Container lab4-check-753ba80cb3-migrate-1 Started - Container lab4-check-753ba80cb3-s3-1 Healthy - Container lab4-check-753ba80cb3-s3-bootstrap-1 Starting - Container lab4-check-753ba80cb3-s3-bootstrap-1 Started - Container lab4-check-753ba80cb3-valkey-1 Waiting - Container lab4-check-753ba80cb3-migrate-1 Waiting - Container lab4-check-753ba80cb3-s3-1 Waiting - Container lab4-check-753ba80cb3-s3-bootstrap-1 Waiting - Container lab4-check-753ba80cb3-postgres-1 Waiting - Container lab4-check-753ba80cb3-valkey-1 Healthy - Container lab4-check-753ba80cb3-migrate-1 Exited - Container lab4-check-753ba80cb3-s3-1 Healthy - Container lab4-check-753ba80cb3-postgres-1 Healthy - Container lab4-check-753ba80cb3-s3-bootstrap-1 Exited - Container lab4-check-753ba80cb3-api-2 Starting - Container lab4-check-753ba80cb3-api-2 Started -API ready through LB: 887ffd1465a8, 8f41208a6406 +Stopped API: ca1f62dfc015 container=ca1f62dfc0156fc235dbcd2f52b58eec2282e6ba8d3e056de11ca92758c69df0 +API ready through LB: 19a474c69210 + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-s3-storage-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-s3-storage-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Starting + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Started + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Starting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Started + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-valkey-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Exited + Container lab4-check-5aad8e0065-valkey-1 Healthy + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Exited + Container lab4-check-5aad8e0065-api-2 Starting + Container lab4-check-5aad8e0065-api-2 Started +API ready through LB: 19a474c69210, ca1f62dfc015 PASS: original downloaded through another replica after stopping uploader PASS: post-PUT DB failure recorded; cleanup protects fresh, active and ready uploads - Container lab4-check-753ba80cb3-s3-1 Stopping - Container lab4-check-753ba80cb3-s3-1 Stopped - Container lab4-check-753ba80cb3-s3-storage-1 Waiting - Container lab4-check-753ba80cb3-s3-storage-1 Healthy - Container lab4-check-753ba80cb3-s3-1 Starting - Container lab4-check-753ba80cb3-s3-1 Started -API ready through LB: 887ffd1465a8, 8f41208a6406 + Container lab4-check-5aad8e0065-s3-1 Stopping + Container lab4-check-5aad8e0065-s3-1 Stopped + Container lab4-check-5aad8e0065-s3-storage-1 Waiting + Container lab4-check-5aad8e0065-s3-storage-1 Healthy + Container lab4-check-5aad8e0065-s3-1 Starting + Container lab4-check-5aad8e0065-s3-1 Started +API ready through LB: 19a474c69210, ca1f62dfc015 PASS: S3 outage gives API 503; recovery preserves original and session - Container lab4-check-753ba80cb3-postgres-1 Waiting - Container lab4-check-753ba80cb3-s3-storage-1 Waiting - Container lab4-check-753ba80cb3-s3-storage-1 Healthy - Container lab4-check-753ba80cb3-postgres-1 Healthy - Container lab4-check-753ba80cb3-migrate-1 Starting - Container lab4-check-753ba80cb3-s3-1 Waiting - Container lab4-check-753ba80cb3-migrate-1 Started - Container lab4-check-753ba80cb3-s3-1 Healthy - Container lab4-check-753ba80cb3-s3-bootstrap-1 Starting - Container lab4-check-753ba80cb3-s3-bootstrap-1 Started - Container lab4-check-753ba80cb3-s3-bootstrap-1 Waiting - Container lab4-check-753ba80cb3-migrate-1 Waiting - Container lab4-check-753ba80cb3-migrate-1 Exited - Container lab4-check-753ba80cb3-s3-bootstrap-1 Exited - Container lab4-check-753ba80cb3-cleanup-1 Starting - Container lab4-check-753ba80cb3-cleanup-1 Started -PASS: 10 concurrent real PNG uploads; all checksums match; memory peaks: [203698176, 90324992] - Container lab4-check-753ba80cb3-s3-storage-1 Waiting - Container lab4-check-753ba80cb3-postgres-1 Waiting - Container lab4-check-753ba80cb3-postgres-1 Healthy - Container lab4-check-753ba80cb3-s3-storage-1 Healthy - Container lab4-check-753ba80cb3-migrate-1 Starting - Container lab4-check-753ba80cb3-s3-1 Waiting - Container lab4-check-753ba80cb3-migrate-1 Started - Container lab4-check-753ba80cb3-s3-1 Healthy - Container lab4-check-753ba80cb3-s3-bootstrap-1 Starting - Container lab4-check-753ba80cb3-s3-bootstrap-1 Started - Container lab4-check-753ba80cb3-migrate-1 Waiting - Container lab4-check-753ba80cb3-s3-bootstrap-1 Waiting - Container lab4-check-753ba80cb3-postgres-1 Waiting - Container lab4-check-753ba80cb3-valkey-1 Waiting - Container lab4-check-753ba80cb3-migrate-1 Waiting - Container lab4-check-753ba80cb3-s3-1 Waiting - Container lab4-check-753ba80cb3-s3-bootstrap-1 Waiting - Container lab4-check-753ba80cb3-s3-1 Healthy - Container lab4-check-753ba80cb3-migrate-1 Exited - Container lab4-check-753ba80cb3-migrate-1 Exited - Container lab4-check-753ba80cb3-postgres-1 Healthy - Container lab4-check-753ba80cb3-valkey-1 Healthy - Container lab4-check-753ba80cb3-s3-bootstrap-1 Exited - Container lab4-check-753ba80cb3-s3-bootstrap-1 Exited -/Users/vanek/Desktop/VNPlabs/VNPlab4/mtusi2026lab4/.venv/bin/python tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:65032" + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-s3-storage-1 Waiting + Container lab4-check-5aad8e0065-s3-storage-1 Healthy + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Starting + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Started + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Starting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Started + Container lab4-check-5aad8e0065-migrate-1 Waiting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Exited + Container lab4-check-5aad8e0065-s3-bootstrap-1 Exited + Container lab4-check-5aad8e0065-cleanup-1 Starting + Container lab4-check-5aad8e0065-cleanup-1 Started +PASS: 10 concurrent real PNG uploads; all checksums match; memory peaks: [215191552, 103378944] + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-s3-storage-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-s3-storage-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Starting + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Started + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Starting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Started + Container lab4-check-5aad8e0065-valkey-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Waiting + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Waiting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Exited + Container lab4-check-5aad8e0065-valkey-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Exited + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Exited + Container lab4-check-5aad8e0065-s3-bootstrap-1 Exited +/Users/vanek/Desktop/VNPlabs/VNPlab4/mtusi2026lab4/.venv/bin/python tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:57722" Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. PASS: both API, LB, PostgreSQL, Valkey, S3 and cleaner actually restarted; data and session preserved -PASS: SIGTERM with unfinished request: {"instance": "887ffd1465a8", "unfinished_request": true, "shutdown_seconds": 5.448, "new_connections_refused_seconds": 0.162, "exit_code": 0, "oom_killed": false} - Container lab4-check-753ba80cb3-s3-storage-1 Waiting - Container lab4-check-753ba80cb3-postgres-1 Waiting - Container lab4-check-753ba80cb3-postgres-1 Healthy - Container lab4-check-753ba80cb3-migrate-1 Starting - Container lab4-check-753ba80cb3-s3-storage-1 Healthy - Container lab4-check-753ba80cb3-s3-1 Waiting - Container lab4-check-753ba80cb3-migrate-1 Started - Container lab4-check-753ba80cb3-s3-1 Healthy - Container lab4-check-753ba80cb3-s3-bootstrap-1 Starting - Container lab4-check-753ba80cb3-s3-bootstrap-1 Started - Container lab4-check-753ba80cb3-migrate-1 Waiting - Container lab4-check-753ba80cb3-s3-1 Waiting - Container lab4-check-753ba80cb3-s3-bootstrap-1 Waiting - Container lab4-check-753ba80cb3-postgres-1 Waiting - Container lab4-check-753ba80cb3-valkey-1 Waiting - Container lab4-check-753ba80cb3-postgres-1 Healthy - Container lab4-check-753ba80cb3-valkey-1 Healthy - Container lab4-check-753ba80cb3-s3-1 Healthy - Container lab4-check-753ba80cb3-migrate-1 Exited - Container lab4-check-753ba80cb3-s3-bootstrap-1 Exited - Container lab4-check-753ba80cb3-api-1 Starting - Container lab4-check-753ba80cb3-api-1 Started -API ready through LB: 887ffd1465a8, 8f41208a6406 - Container lab4-check-753ba80cb3-valkey-1 Stopping - Container lab4-check-753ba80cb3-valkey-1 Stopped +PASS: SIGTERM with unfinished request: {"instance": "19a474c69210", "unfinished_request": true, "shutdown_seconds": 5.523, "new_connections_refused_seconds": 0.185, "exit_code": 0, "oom_killed": false} + Container lab4-check-5aad8e0065-s3-storage-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-s3-storage-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Starting + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Started + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Starting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Started + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-valkey-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Waiting + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Waiting + Container lab4-check-5aad8e0065-valkey-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Exited + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Exited + Container lab4-check-5aad8e0065-api-1 Starting + Container lab4-check-5aad8e0065-api-1 Started +API ready through LB: 19a474c69210, ca1f62dfc015 + Container lab4-check-5aad8e0065-valkey-1 Stopping + Container lab4-check-5aad8e0065-valkey-1 Stopped PASS: valkey outage: API live=200, ready=503, LB ready=503 - Container lab4-check-753ba80cb3-valkey-1 Starting - Container lab4-check-753ba80cb3-valkey-1 Started -API ready through LB: 887ffd1465a8, 8f41208a6406 - Container lab4-check-753ba80cb3-postgres-1 Stopping - Container lab4-check-753ba80cb3-postgres-1 Stopped + Container lab4-check-5aad8e0065-valkey-1 Starting + Container lab4-check-5aad8e0065-valkey-1 Started +API ready through LB: 19a474c69210, ca1f62dfc015 + Container lab4-check-5aad8e0065-postgres-1 Stopping + Container lab4-check-5aad8e0065-postgres-1 Stopped PASS: postgres outage: API live=200, ready=503, LB ready=503 - Container lab4-check-753ba80cb3-postgres-1 Starting - Container lab4-check-753ba80cb3-postgres-1 Started -API ready through LB: 887ffd1465a8, 8f41208a6406 - Container lab4-check-753ba80cb3-s3-1 Stopping - Container lab4-check-753ba80cb3-s3-1 Stopped + Container lab4-check-5aad8e0065-postgres-1 Starting + Container lab4-check-5aad8e0065-postgres-1 Started +API ready through LB: 19a474c69210, ca1f62dfc015 + Container lab4-check-5aad8e0065-s3-1 Stopping + Container lab4-check-5aad8e0065-s3-1 Stopped PASS: s3 outage: API live=200, ready=503, LB ready=503 - Container lab4-check-753ba80cb3-s3-storage-1 Waiting - Container lab4-check-753ba80cb3-s3-storage-1 Healthy - Container lab4-check-753ba80cb3-s3-1 Starting - Container lab4-check-753ba80cb3-s3-1 Started -API ready through LB: 887ffd1465a8, 8f41208a6406 -/Users/vanek/Desktop/VNPlabs/VNPlab4/mtusi2026lab4/.venv/bin/python tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:65032" + Container lab4-check-5aad8e0065-s3-storage-1 Waiting + Container lab4-check-5aad8e0065-s3-storage-1 Healthy + Container lab4-check-5aad8e0065-s3-1 Starting + Container lab4-check-5aad8e0065-s3-1 Started +API ready through LB: 19a474c69210, ca1f62dfc015 + Container lab4-check-5aad8e0065-postgres-1 Paused + Container lab4-check-5aad8e0065-postgres-1 Unpaused +PASS: paused PostgreSQL: SQL and direct readiness bounded; live works; unpause restores API without restart +/Users/vanek/Desktop/VNPlabs/VNPlab4/mtusi2026lab4/.venv/bin/python tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:57722" Executing explicit failover hook; hook output suppressed to avoid leaking secrets. -{"observed_instances_before": 2, "successful_writes": 111, "failed_attempts": 1, "first_recovery_seconds": 2.469} +{"observed_instances_before": 2, "successful_writes": 110, "failed_attempts": 1, "first_recovery_seconds": 2.467} PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. - Container lab4-check-753ba80cb3-postgres-1 Waiting - Container lab4-check-753ba80cb3-s3-storage-1 Waiting - Container lab4-check-753ba80cb3-s3-storage-1 Healthy - Container lab4-check-753ba80cb3-postgres-1 Healthy - Container lab4-check-753ba80cb3-migrate-1 Starting - Container lab4-check-753ba80cb3-s3-1 Waiting - Container lab4-check-753ba80cb3-migrate-1 Started - Container lab4-check-753ba80cb3-s3-1 Healthy - Container lab4-check-753ba80cb3-s3-bootstrap-1 Starting - Container lab4-check-753ba80cb3-s3-bootstrap-1 Started - Container lab4-check-753ba80cb3-s3-bootstrap-1 Waiting - Container lab4-check-753ba80cb3-postgres-1 Waiting - Container lab4-check-753ba80cb3-valkey-1 Waiting - Container lab4-check-753ba80cb3-migrate-1 Waiting - Container lab4-check-753ba80cb3-s3-1 Waiting - Container lab4-check-753ba80cb3-migrate-1 Exited - Container lab4-check-753ba80cb3-valkey-1 Healthy - Container lab4-check-753ba80cb3-postgres-1 Healthy - Container lab4-check-753ba80cb3-s3-1 Healthy - Container lab4-check-753ba80cb3-s3-bootstrap-1 Exited - Container lab4-check-753ba80cb3-api-2 Starting - Container lab4-check-753ba80cb3-api-2 Started -API ready through LB: 887ffd1465a8, 8f41208a6406 -Stopped API: 887ffd1465a8 container=887ffd1465a8191971a9f04cb3deb79994889e288fbe4bc7224052fa3094a710 -{"label": "stop 887ffd1465a8", "duration_seconds": 16.012, "successful_requests": 1480, "successful_writes": 739, "errors": 4, "successful_rps": 92.43, "first_success_after_action_seconds": {"POST": 0.006, "GET": 0.008}, "maximum_success_gap_seconds": {"POST": 2.042, "GET": 1.984}, "action_start_seconds": 1.001, "action_end_seconds": 1.708, "p50_ms": 9.065, "p95_ms": 18.196, "confirmed_data_preserved": true, "duplicate_messages": 0, "instances": ["887ffd1465a8", "8f41208a6406"]} - Container lab4-check-753ba80cb3-s3-storage-1 Waiting - Container lab4-check-753ba80cb3-postgres-1 Waiting - Container lab4-check-753ba80cb3-postgres-1 Healthy - Container lab4-check-753ba80cb3-s3-storage-1 Healthy - Container lab4-check-753ba80cb3-migrate-1 Starting - Container lab4-check-753ba80cb3-s3-1 Waiting - Container lab4-check-753ba80cb3-migrate-1 Started - Container lab4-check-753ba80cb3-s3-1 Healthy - Container lab4-check-753ba80cb3-s3-bootstrap-1 Starting - Container lab4-check-753ba80cb3-s3-bootstrap-1 Started - Container lab4-check-753ba80cb3-valkey-1 Waiting - Container lab4-check-753ba80cb3-migrate-1 Waiting - Container lab4-check-753ba80cb3-s3-1 Waiting - Container lab4-check-753ba80cb3-s3-bootstrap-1 Waiting - Container lab4-check-753ba80cb3-postgres-1 Waiting - Container lab4-check-753ba80cb3-s3-1 Healthy - Container lab4-check-753ba80cb3-migrate-1 Exited - Container lab4-check-753ba80cb3-postgres-1 Healthy - Container lab4-check-753ba80cb3-valkey-1 Healthy - Container lab4-check-753ba80cb3-s3-bootstrap-1 Exited - Container lab4-check-753ba80cb3-api-1 Starting - Container lab4-check-753ba80cb3-api-1 Started -API ready through LB: 887ffd1465a8, 8f41208a6406 -Stopped API: 8f41208a6406 container=8f41208a64062523e85a27c5d759029d6c9940df78417691c85900dcb8e6f158 -{"label": "stop 8f41208a6406", "duration_seconds": 16.026, "successful_requests": 1472, "successful_writes": 735, "errors": 4, "successful_rps": 91.85, "first_success_after_action_seconds": {"POST": 0.029, "GET": 0.038}, "maximum_success_gap_seconds": {"POST": 1.986, "GET": 1.99}, "action_start_seconds": 1.011, "action_end_seconds": 1.806, "p50_ms": 9.112, "p95_ms": 20.767, "confirmed_data_preserved": true, "duplicate_messages": 0, "instances": ["887ffd1465a8", "8f41208a6406"]} - Container lab4-check-753ba80cb3-postgres-1 Waiting - Container lab4-check-753ba80cb3-s3-storage-1 Waiting - Container lab4-check-753ba80cb3-s3-storage-1 Healthy - Container lab4-check-753ba80cb3-postgres-1 Healthy - Container lab4-check-753ba80cb3-s3-1 Waiting - Container lab4-check-753ba80cb3-migrate-1 Starting - Container lab4-check-753ba80cb3-migrate-1 Started - Container lab4-check-753ba80cb3-s3-1 Healthy - Container lab4-check-753ba80cb3-s3-bootstrap-1 Starting - Container lab4-check-753ba80cb3-s3-bootstrap-1 Started - Container lab4-check-753ba80cb3-s3-1 Waiting - Container lab4-check-753ba80cb3-s3-bootstrap-1 Waiting - Container lab4-check-753ba80cb3-postgres-1 Waiting - Container lab4-check-753ba80cb3-valkey-1 Waiting - Container lab4-check-753ba80cb3-migrate-1 Waiting - Container lab4-check-753ba80cb3-postgres-1 Healthy - Container lab4-check-753ba80cb3-migrate-1 Exited - Container lab4-check-753ba80cb3-valkey-1 Healthy - Container lab4-check-753ba80cb3-s3-1 Healthy - Container lab4-check-753ba80cb3-s3-bootstrap-1 Exited - Container lab4-check-753ba80cb3-api-2 Starting - Container lab4-check-753ba80cb3-api-2 Started -API ready through LB: 887ffd1465a8, 8f41208a6406 + Container lab4-check-5aad8e0065-s3-storage-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-s3-storage-1 Healthy + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Starting + Container lab4-check-5aad8e0065-migrate-1 Started + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Starting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Started + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-valkey-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Waiting + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Waiting + Container lab4-check-5aad8e0065-valkey-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Exited + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Exited + Container lab4-check-5aad8e0065-api-1 Starting + Container lab4-check-5aad8e0065-api-1 Started +API ready through LB: 19a474c69210, ca1f62dfc015 +Stopped API: 19a474c69210 container=19a474c69210a7be1c6631830e8fa7f251cf6dd3890c7407843090383543ee75 +{"label": "stop 19a474c69210", "duration_seconds": 16.012, "successful_requests": 1420, "successful_writes": 709, "errors": 4, "successful_rps": 88.68, "first_success_after_action_seconds": {"POST": 0.051, "GET": 0.004}, "maximum_success_gap_seconds": {"POST": 2.002, "GET": 1.995}, "action_start_seconds": 1.006, "action_end_seconds": 1.77, "p50_ms": 10.62, "p95_ms": 24.67, "confirmed_data_preserved": true, "duplicate_messages": 0, "instances": ["19a474c69210", "ca1f62dfc015"]} + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-s3-storage-1 Waiting + Container lab4-check-5aad8e0065-s3-storage-1 Healthy + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Starting + Container lab4-check-5aad8e0065-migrate-1 Started + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Starting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Started + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-valkey-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Waiting + Container lab4-check-5aad8e0065-valkey-1 Healthy + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Exited + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Exited + Container lab4-check-5aad8e0065-api-1 Starting + Container lab4-check-5aad8e0065-api-1 Started +API ready through LB: 19a474c69210, ca1f62dfc015 +Stopped API: ca1f62dfc015 container=ca1f62dfc0156fc235dbcd2f52b58eec2282e6ba8d3e056de11ca92758c69df0 +{"label": "stop ca1f62dfc015", "duration_seconds": 16.008, "successful_requests": 1466, "successful_writes": 732, "errors": 4, "successful_rps": 91.58, "first_success_after_action_seconds": {"POST": 0.018, "GET": 0.028}, "maximum_success_gap_seconds": {"POST": 2.002, "GET": 1.999}, "action_start_seconds": 1.002, "action_end_seconds": 1.832, "p50_ms": 9.397, "p95_ms": 18.616, "confirmed_data_preserved": true, "duplicate_messages": 0, "instances": ["19a474c69210", "ca1f62dfc015"]} + Container lab4-check-5aad8e0065-s3-storage-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-s3-storage-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Starting + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Started + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Starting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Started + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-valkey-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Waiting + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Waiting + Container lab4-check-5aad8e0065-valkey-1 Healthy + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Exited + Container lab4-check-5aad8e0065-s3-bootstrap-1 Exited + Container lab4-check-5aad8e0065-api-2 Starting + Container lab4-check-5aad8e0065-api-2 Started +API ready through LB: 19a474c69210, ca1f62dfc015 Local .env and secrets are ready; values are not printed #1 [internal] load local bake definitions #1 reading from stdin 564B done @@ -423,22 +426,22 @@ Local .env and secrets are ready; values are not printed #6 resolve docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 0.0s done #6 DONE 0.0s -#7 [4/8] RUN python -m pip install --no-cache-dir -r requirements.txt && python -m pip check +#7 [3/8] COPY requirements.txt . #7 CACHED -#8 [7/8] COPY migrations/ ./migrations/ +#8 [6/8] COPY src/ ./src/ #8 CACHED -#9 [3/8] COPY requirements.txt . +#9 [2/8] WORKDIR /app #9 CACHED -#10 [5/8] RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app +#10 [7/8] COPY migrations/ ./migrations/ #10 CACHED -#11 [6/8] COPY src/ ./src/ +#11 [5/8] RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app #11 CACHED -#12 [2/8] WORKDIR /app +#12 [4/8] RUN python -m pip install --no-cache-dir -r requirements.txt && python -m pip check #12 CACHED #13 [8/8] COPY research/ ./research/ @@ -446,10 +449,10 @@ Local .env and secrets are ready; values are not printed #14 exporting to image #14 exporting layers done -#14 exporting manifest sha256:1c6cb19983cdfb308e1ec43f59338922d2fec797ff8fdc9636526f9de20a0693 done -#14 exporting config sha256:bc7c1361d0a40f2beb749094a98a091632f75a255c5841fcdf80a33b166720eb done -#14 exporting attestation manifest sha256:d2cddc9cbcfc350b5ccf03a93a9a2ea63dc1b623c099006f4187f3782a500abc 0.0s done -#14 exporting manifest list sha256:413691a40751a490bab701b9498431365ae10bfd081d55decba718e5e90a0f5d done +#14 exporting manifest sha256:644f5d408505f5982b75f18404860de692ee837b2b2e18f5c92b0b057081b7ec done +#14 exporting config sha256:726011343b32f5b6277348f2f8b38f95b8c31754ae0d21336375124a18aa2e0e done +#14 exporting attestation manifest sha256:f4a5559518ea1ef0e3747d6084c8ad8dbc8b16dc41cb00062303d0f895b9a235 0.0s done +#14 exporting manifest list sha256:93c838d55d79a0001346ecab28f72621d9540ba5c52eaeb1ca6020286dd05fd6 done #14 naming to docker.io/library/mtusi-messenger:lab4 done #14 unpacking to docker.io/library/mtusi-messenger:lab4 done #14 DONE 0.1s @@ -457,75 +460,74 @@ Local .env and secrets are ready; values are not printed #15 resolving provenance for metadata file #15 DONE 0.0s mtusi-messenger:lab4 Built -Local .env and secrets are ready; values are not printed - Container lab4-check-753ba80cb3-postgres-1 Running + Container lab4-check-5aad8e0065-postgres-1 Running Migrations: already applied -API ready through LB: 887ffd1465a8, 8f41208a6406 -Replacing API: 887ffd1465a8 - Container lab4-check-753ba80cb3-api-2 Running - Container lab4-check-753ba80cb3-api-3 Creating - Container lab4-check-753ba80cb3-api-3 Created - Container lab4-check-753ba80cb3-api-3 Starting - Container lab4-check-753ba80cb3-api-3 Started -API ready through LB: 8f41208a6406, 9be50a7967ed -Replacing API: 8f41208a6406 - Container lab4-check-753ba80cb3-api-3 Running - Container lab4-check-753ba80cb3-api-4 Creating - Container lab4-check-753ba80cb3-api-4 Created - Container lab4-check-753ba80cb3-api-4 Starting - Container lab4-check-753ba80cb3-api-4 Started -API ready through LB: 4479505e52f5, 9be50a7967ed +API ready through LB: 19a474c69210, ca1f62dfc015 +Replacing API: 19a474c69210 + Container lab4-check-5aad8e0065-api-2 Running + Container lab4-check-5aad8e0065-api-3 Creating + Container lab4-check-5aad8e0065-api-3 Created + Container lab4-check-5aad8e0065-api-3 Starting + Container lab4-check-5aad8e0065-api-3 Started +API ready through LB: ca1f62dfc015, ed4f3c64a9f6 +Replacing API: ca1f62dfc015 + Container lab4-check-5aad8e0065-api-3 Running + Container lab4-check-5aad8e0065-api-4 Creating + Container lab4-check-5aad8e0065-api-4 Created + Container lab4-check-5aad8e0065-api-4 Starting + Container lab4-check-5aad8e0065-api-4 Started +API ready through LB: 63549c0d63ba, ed4f3c64a9f6 PASS: all API containers replaced sequentially; stores and LB kept running -{"label": "sequential API replacement", "duration_seconds": 22.153, "successful_requests": 1960, "successful_writes": 979, "errors": 8, "successful_rps": 88.47, "first_success_after_action_seconds": {"POST": 0.035, "GET": 0.041}, "maximum_success_gap_seconds": {"POST": 1.048, "GET": 1.031}, "action_start_seconds": 1.004, "action_end_seconds": 19.149, "p50_ms": 8.859, "p95_ms": 17.468, "confirmed_data_preserved": true, "duplicate_messages": 0, "instances": ["4479505e52f5", "887ffd1465a8", "8f41208a6406", "9be50a7967ed"]} +{"label": "sequential API replacement", "duration_seconds": 24.869, "successful_requests": 2254, "successful_writes": 1128, "errors": 8, "successful_rps": 90.63, "first_success_after_action_seconds": {"POST": 0.01, "GET": 0.021}, "maximum_success_gap_seconds": {"POST": 1.04, "GET": 1.04}, "action_start_seconds": 1.009, "action_end_seconds": 21.862, "p50_ms": 8.907, "p95_ms": 16.321, "confirmed_data_preserved": true, "duplicate_messages": 0, "instances": ["19a474c69210", "63549c0d63ba", "ca1f62dfc015", "ed4f3c64a9f6"]} PASS: all lab4 grade-4 checks completed - Container lab4-check-753ba80cb3-lb-1 Stopping - Container lab4-check-753ba80cb3-cleanup-1 Stopping - Container lab4-check-753ba80cb3-lb-1 Stopped - Container lab4-check-753ba80cb3-lb-1 Removing - Container lab4-check-753ba80cb3-lb-1 Removed - Container lab4-check-753ba80cb3-api-3 Stopping - Container lab4-check-753ba80cb3-api-4 Stopping - Container lab4-check-753ba80cb3-api-4 Stopped - Container lab4-check-753ba80cb3-api-4 Removing - Container lab4-check-753ba80cb3-api-4 Removed - Container lab4-check-753ba80cb3-api-3 Stopped - Container lab4-check-753ba80cb3-api-3 Removing - Container lab4-check-753ba80cb3-api-3 Removed - Container lab4-check-753ba80cb3-valkey-1 Stopping - Container lab4-check-753ba80cb3-valkey-1 Stopped - Container lab4-check-753ba80cb3-valkey-1 Removing - Container lab4-check-753ba80cb3-valkey-1 Removed - Container lab4-check-753ba80cb3-cleanup-1 Stopped - Container lab4-check-753ba80cb3-cleanup-1 Removing - Container lab4-check-753ba80cb3-cleanup-1 Removed - Container lab4-check-753ba80cb3-s3-bootstrap-1 Stopping - Container lab4-check-753ba80cb3-migrate-1 Stopping - Container lab4-check-753ba80cb3-s3-bootstrap-1 Stopped - Container lab4-check-753ba80cb3-migrate-1 Stopped - Container lab4-check-753ba80cb3-migrate-1 Removing - Container lab4-check-753ba80cb3-s3-bootstrap-1 Removing - Container lab4-check-753ba80cb3-migrate-1 Removed - Container lab4-check-753ba80cb3-postgres-1 Stopping - Container lab4-check-753ba80cb3-s3-bootstrap-1 Removed - Container lab4-check-753ba80cb3-s3-1 Stopping - Container lab4-check-753ba80cb3-postgres-1 Stopped - Container lab4-check-753ba80cb3-postgres-1 Removing - Container lab4-check-753ba80cb3-postgres-1 Removed - Container lab4-check-753ba80cb3-s3-1 Stopped - Container lab4-check-753ba80cb3-s3-1 Removing - Container lab4-check-753ba80cb3-s3-1 Removed - Container lab4-check-753ba80cb3-s3-storage-1 Stopping - Container lab4-check-753ba80cb3-s3-storage-1 Stopped - Container lab4-check-753ba80cb3-s3-storage-1 Removing - Container lab4-check-753ba80cb3-s3-storage-1 Removed - Network lab4-check-753ba80cb3_s3-internal Removing - Volume lab4-check-753ba80cb3_postgres_data Removing - Volume lab4-check-753ba80cb3_s3_data Removing - Network lab4-check-753ba80cb3_default Removing - Volume lab4-check-753ba80cb3_valkey_data Removing - Volume lab4-check-753ba80cb3_s3_data Removed - Volume lab4-check-753ba80cb3_postgres_data Removed - Volume lab4-check-753ba80cb3_valkey_data Removed - Network lab4-check-753ba80cb3_s3-internal Removed - Network lab4-check-753ba80cb3_default Removed -Removed only temporary test containers and volumes: lab4-check-753ba80cb3 + Container lab4-check-5aad8e0065-lb-1 Stopping + Container lab4-check-5aad8e0065-cleanup-1 Stopping + Container lab4-check-5aad8e0065-lb-1 Stopped + Container lab4-check-5aad8e0065-lb-1 Removing + Container lab4-check-5aad8e0065-lb-1 Removed + Container lab4-check-5aad8e0065-api-3 Stopping + Container lab4-check-5aad8e0065-api-4 Stopping + Container lab4-check-5aad8e0065-api-3 Stopped + Container lab4-check-5aad8e0065-api-3 Removing + Container lab4-check-5aad8e0065-api-3 Removed + Container lab4-check-5aad8e0065-api-4 Stopped + Container lab4-check-5aad8e0065-api-4 Removing + Container lab4-check-5aad8e0065-api-4 Removed + Container lab4-check-5aad8e0065-valkey-1 Stopping + Container lab4-check-5aad8e0065-valkey-1 Stopped + Container lab4-check-5aad8e0065-valkey-1 Removing + Container lab4-check-5aad8e0065-valkey-1 Removed + Container lab4-check-5aad8e0065-cleanup-1 Stopped + Container lab4-check-5aad8e0065-cleanup-1 Removing + Container lab4-check-5aad8e0065-cleanup-1 Removed + Container lab4-check-5aad8e0065-s3-bootstrap-1 Stopping + Container lab4-check-5aad8e0065-migrate-1 Stopping + Container lab4-check-5aad8e0065-migrate-1 Stopped + Container lab4-check-5aad8e0065-migrate-1 Removing + Container lab4-check-5aad8e0065-s3-bootstrap-1 Stopped + Container lab4-check-5aad8e0065-s3-bootstrap-1 Removing + Container lab4-check-5aad8e0065-s3-bootstrap-1 Removed + Container lab4-check-5aad8e0065-s3-1 Stopping + Container lab4-check-5aad8e0065-migrate-1 Removed + Container lab4-check-5aad8e0065-postgres-1 Stopping + Container lab4-check-5aad8e0065-postgres-1 Stopped + Container lab4-check-5aad8e0065-postgres-1 Removing + Container lab4-check-5aad8e0065-postgres-1 Removed + Container lab4-check-5aad8e0065-s3-1 Stopped + Container lab4-check-5aad8e0065-s3-1 Removing + Container lab4-check-5aad8e0065-s3-1 Removed + Container lab4-check-5aad8e0065-s3-storage-1 Stopping + Container lab4-check-5aad8e0065-s3-storage-1 Stopped + Container lab4-check-5aad8e0065-s3-storage-1 Removing + Container lab4-check-5aad8e0065-s3-storage-1 Removed + Volume lab4-check-5aad8e0065_valkey_data Removing + Network lab4-check-5aad8e0065_s3-internal Removing + Volume lab4-check-5aad8e0065_postgres_data Removing + Volume lab4-check-5aad8e0065_s3_data Removing + Network lab4-check-5aad8e0065_default Removing + Volume lab4-check-5aad8e0065_valkey_data Removed + Volume lab4-check-5aad8e0065_s3_data Removed + Volume lab4-check-5aad8e0065_postgres_data Removed + Network lab4-check-5aad8e0065_s3-internal Removed + Network lab4-check-5aad8e0065_default Removed +Removed only temporary test containers and volumes: lab4-check-5aad8e0065 diff --git a/evidence/postgres-pause.json b/evidence/postgres-pause.json new file mode 100644 index 0000000..7338271 --- /dev/null +++ b/evidence/postgres-pause.json @@ -0,0 +1,76 @@ +{ + "utc": "2026-11-06T07:27:40.514878+00:00", + "project": "lab4-pg-pause-30f1d978ce", + "command": ".venv/bin/python student-tests/postgres_pause.py", + "passed": true, + "sql_without_server_timeout": { + "elapsed_seconds": 2.001, + "connection_closed": true + }, + "direct_ready": { + "status": 503, + "elapsed_seconds": 2.012 + }, + "direct_live": { + "status": 200, + "elapsed_seconds": 0.006 + }, + "repeated_ready": [ + { + "status": 503, + "elapsed_seconds": 2.11 + }, + { + "status": 503, + "elapsed_seconds": 2.011 + }, + { + "status": 503, + "elapsed_seconds": 2.014 + }, + { + "status": 503, + "elapsed_seconds": 2.097 + }, + { + "status": 503, + "elapsed_seconds": 2.194 + }, + { + "status": 503, + "elapsed_seconds": 2.013 + }, + { + "status": 503, + "elapsed_seconds": 2.299 + }, + { + "status": 503, + "elapsed_seconds": 2.203 + }, + { + "status": 503, + "elapsed_seconds": 2.013 + }, + { + "status": 503, + "elapsed_seconds": 2.094 + } + ], + "ready_after_12_seconds": { + "status": 503, + "elapsed_seconds": 0.009 + }, + "live_after_repeated_requests": { + "status": 200, + "elapsed_seconds": 0.006 + }, + "paused_seconds": 12.565, + "recovered_ready": { + "status": 200, + "elapsed_seconds": 0.009 + }, + "recovery_seconds": 2.874, + "api_not_restarted": true, + "existing_session_and_message_preserved": true +} diff --git a/src/db.py b/src/db.py index 8580eca..973bbc2 100644 --- a/src/db.py +++ b/src/db.py @@ -9,8 +9,25 @@ from src.config import secret from src.errors import ApiError +IO_TIMEOUT = 2.0 + + +class TimedConnection(psycopg.Connection): + def wait(self, gen, interval=0.1, timeout=None): + # statement_timeout работает на сервере и не помогает при docker pause. + # Ограничиваем само ожидание I/O: проверку пула, SQL, COMMIT и ROLLBACK. + timeout = IO_TIMEOUT if timeout is None else min(timeout, IO_TIMEOUT) + try: + return super().wait(gen, interval=interval, timeout=timeout) + except psycopg.OperationalError: + # Не ждём отмены/ROLLBACK от зависшего сервера. Пул заменит соединение. + self.close() + raise + + pool = ConnectionPool( conninfo=secret("DATABASE_URL"), open=False, + connection_class=TimedConnection, min_size=1, max_size=5, max_waiting=20, timeout=2, check=ConnectionPool.check_connection, kwargs={ diff --git a/student-tests/docker_lifecycle.py b/student-tests/docker_lifecycle.py index 75582ee..4800ec6 100644 --- a/student-tests/docker_lifecycle.py +++ b/student-tests/docker_lifecycle.py @@ -17,6 +17,7 @@ from helpers import Client, ROOT from client import Failure from resilience import history from attachment_checks import run_checks +from postgres_pause import check_postgres_pause def docker(*args): @@ -263,6 +264,8 @@ time.sleep(30) script("wait-ready.sh") dependency_client.call("GET", "/api/v1/users/me", actor=actor, schema="User") + result["postgres_pause"] = check_postgres_pause(compose, apis, base) + subprocess.run(["make", "test-failover", "PYTHON=" + sys.executable, "ACTION_SCRIPT=scripts/stop-one.sh"], cwd=ROOT, env=env, check=True, timeout=90) assert sum(i["State"]["Running"] for i in apis()) == 1 diff --git a/student-tests/postgres_pause.py b/student-tests/postgres_pause.py new file mode 100644 index 0000000..07bd2fb --- /dev/null +++ b/student-tests/postgres_pause.py @@ -0,0 +1,145 @@ +"""Пауза живой PostgreSQL: прямой readiness, ограниченное SQL I/O и восстановление.""" +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import socket +import subprocess +import sys +import time +from uuid import uuid4 + +from helpers import Client, ROOT +from client import Failure + + +def check_postgres_pause(compose, apis, base): + client = Client(base) + alice, bob, _, chat = client.bootstrap() + saved_message = client.message(alice, chat, "До паузы PostgreSQL")["json"] + before = {info["Id"]:info["State"]["StartedAt"] for info in apis() if info["State"]["Running"]} + probe_id = sorted(before)[0] + + def direct_probe(path, expected): + # HTTP напрямую в API, без HAProxy и его собственных таймаутов/503. + code = """ +import json,time +from urllib.error import HTTPError +from urllib.request import urlopen +started=time.monotonic() +try: response=urlopen('http://127.0.0.1:8080/PATH',timeout=5) +except HTTPError as error: response=error +with response: + body=json.load(response) + elapsed=time.monotonic()-started + assert response.status==EXPECTED, response.status + assert response.headers['X-Request-Id'] and response.headers['X-Instance-Id'] + if response.status==503: + assert body['error']['code']=='unavailable' + assert body['error']['request_id']==response.headers['X-Request-Id'] + assert elapsed<4.5, elapsed + print(json.dumps({'status':response.status,'elapsed_seconds':round(elapsed,3)})) +""".replace("PATH", path).replace("EXPECTED", str(expected)) + completed = subprocess.run(["docker", "exec", probe_id, "python", "-c", code], + capture_output=True, text=True, timeout=8, check=True) + return json.loads(completed.stdout) + + # Соединение получено ДО pause: проверяем именно выполняющийся SQL, а не + # только timeout получения нового соединения или check callback пула. + query_code = """ +import json,sys,time +import psycopg +from src import db +db.pool.open(wait=True,timeout=10) +conn=db.pool.getconn() +conn.execute('SET statement_timeout=0') +print('connected',flush=True) +sys.stdin.readline() +started=time.monotonic() +try: + try: conn.execute('SELECT 1') + except psycopg.OperationalError: + elapsed=time.monotonic()-started + assert elapsed<4.5, elapsed + assert conn.closed, 'timed out connection was not discarded' + print(json.dumps({'elapsed_seconds':round(elapsed,3),'connection_closed':conn.closed}),flush=True) + else: raise AssertionError('paused database answered a query') +finally: + db.pool.putconn(conn) + db.pool.close() +""" + query = subprocess.Popen(["docker", "exec", "-i", probe_id, "python", "-u", "-c", query_code], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + result = {} + try: + assert query.stdout.readline().strip() == "connected" + compose("pause", "postgres") + paused_at = time.monotonic() + output, error = query.communicate("\n", timeout=10) + assert query.returncode == 0, "SQL I/O deadline failed: " + error + result["sql_without_server_timeout"] = json.loads(output) + result["direct_ready"] = direct_probe("health/ready", 503) + result["direct_live"] = direct_probe("health/live", 200) + # Повторные запросы при длительной паузе не должны оставлять занятые потоки. + with ThreadPoolExecutor(max_workers=10) as executor: + result["repeated_ready"] = list(executor.map(lambda _: direct_probe("health/ready", 503), range(10))) + # Сервер остаётся на pause не менее 12 секунд; ограничиваем именно API. + remaining = 12 - (time.monotonic() - paused_at) + if remaining > 0: + time.sleep(remaining) + result["ready_after_12_seconds"] = direct_probe("health/ready", 503) + result["live_after_repeated_requests"] = direct_probe("health/live", 200) + result["paused_seconds"] = round(time.monotonic() - paused_at, 3) + finally: + compose("unpause", "postgres") + if query.poll() is None: + query.communicate(timeout=10) + + resumed_at = time.monotonic() + while True: + try: + result["recovered_ready"] = direct_probe("health/ready", 200) + client.call("GET", "/api/v1/users/me", actor=alice, schema="User") + messages = client.call("GET", "/api/v1/chats/" + chat["id"] + "/messages", actor=bob, schema="MessagePage")["json"]["items"] + assert saved_message in messages + break + except (subprocess.CalledProcessError, Failure): + assert time.monotonic() - resumed_at < 30, "readiness did not recover after unpause" + time.sleep(.2) + assert before == {info["Id"]:info["State"]["StartedAt"] for info in apis() if info["State"]["Running"]} + result.update({"recovery_seconds":round(time.monotonic()-resumed_at,3), "api_not_restarted":True, + "existing_session_and_message_preserved":True, "passed":True}) + print("PASS: paused PostgreSQL: SQL and direct readiness bounded; live works; unpause restores API without restart", flush=True) + return result + + +def main(): + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + port = listener.getsockname()[1] + project = "lab4-pg-pause-" + uuid4().hex[:10] + base = f"http://127.0.0.1:{port}" + env = dict(os.environ, COMPOSE_PROJECT_NAME=project, HOST_PORT=str(port), API_REPLICAS="2", BASE_URL=base) + + def compose(*args): + return subprocess.check_output(["docker", "compose", *args], cwd=ROOT, env=env, text=True, timeout=180).strip() + + def apis(): + ids = compose("ps", "-a", "-q", "api").split() + return json.loads(subprocess.check_output(["docker", "inspect", *ids], text=True)) + + result = {"utc":datetime.now(timezone.utc).isoformat(), "project":project, + "command":".venv/bin/python student-tests/postgres_pause.py", "passed":False} + try: + compose("up", "-d", "--no-build", "lb") + subprocess.run([str(ROOT / "scripts/wait-ready.sh")], cwd=ROOT, env=env, check=True, timeout=120) + result.update(check_postgres_pause(compose, apis, base)) + print(json.dumps(result, ensure_ascii=False), flush=True) + finally: + Path(ROOT / "evidence/postgres-pause.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n") + compose("down", "--volumes", "--timeout", "8") + + +if __name__ == "__main__": + main()