From d30f418ceccc08b13834fff3d4ecbfa9761905bd Mon Sep 17 00:00:00 2001 From: Sergey Lyakhonov Date: Fri, 23 Oct 2026 19:09:56 +0300 Subject: [PATCH] =?UTF-8?q?=D0=97=D0=B0=D0=B2=D0=B5=D1=80=D1=88=D0=B8?= =?UTF-8?q?=D1=82=D1=8C=20=D1=83=D1=80=D0=BE=D0=B2=D0=B5=D0=BD=D1=8C=204:?= =?UTF-8?q?=20=D0=BF=D1=80=D0=BE=D0=B2=D0=B5=D1=80=D0=B8=D1=82=D1=8C=20?= =?UTF-8?q?=D0=BE=D1=82=D0=BA=D0=B0=D0=B7=D1=8B,=20=D0=BF=D0=BE=D0=BE?= =?UTF-8?q?=D1=87=D0=B5=D1=80=D1=91=D0=B4=D0=BD=D1=83=D1=8E=20=D0=B7=D0=B0?= =?UTF-8?q?=D0=BC=D0=B5=D0=BD=D1=83=20=D0=B8=20=D0=BE=D0=B3=D1=80=D0=B0?= =?UTF-8?q?=D0=BD=D0=B8=D1=87=D0=B8=D1=82=D1=8C=20=D0=BE=D0=B6=D0=B8=D0=B4?= =?UTF-8?q?=D0=B0=D0=BD=D0=B8=D1=8F=20=D0=BA=D1=8D=D1=88=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 6 + .github/workflows/contracts.yml | 20 +- compose.yaml | 1 + evidence/cpu.json | 5 + evidence/dependencies.txt | 12 + evidence/docker-resources.txt | 1 + evidence/failover-1-containers.jsonl | 12 +- evidence/failover-1.txt | 6 +- evidence/failover-2-containers.jsonl | 12 +- evidence/failover-2.txt | 4 +- evidence/final-checks.txt | 4 + evidence/idempotency.txt | 2 +- evidence/level4.md | 15 + evidence/memory.json | 3 + evidence/migrations.txt | 2 + evidence/persistence-started-after.txt | 10 +- evidence/persistence-started-before.txt | 10 +- evidence/research/rolling.json | 1416 +++++++++++++++++++++++ evidence/transactions.txt | 2 + evidence/transfer.md | 3 +- infra/operations.md | 17 + scripts/experiments.py | 122 ++ scripts/verify_failover.py | 51 + scripts/wait_ready.py | 1 + src/app.py | 4 +- src/cache.py | 21 +- src/routes/messages.py | 9 +- src/storage.py | 13 +- student-tests/README.md | 19 + student-tests/benchmark.py | 104 ++ student-tests/test_dependencies.py | 43 + student-tests/test_lab3.py | 15 +- student-tests/test_transactions.py | 37 + 33 files changed, 1962 insertions(+), 40 deletions(-) create mode 100644 evidence/cpu.json create mode 100644 evidence/dependencies.txt create mode 100644 evidence/docker-resources.txt create mode 100644 evidence/final-checks.txt create mode 100644 evidence/level4.md create mode 100644 evidence/memory.json create mode 100644 evidence/migrations.txt create mode 100644 evidence/research/rolling.json create mode 100644 evidence/transactions.txt create mode 100644 infra/operations.md create mode 100644 scripts/experiments.py create mode 100644 scripts/verify_failover.py create mode 100644 student-tests/README.md create mode 100644 student-tests/benchmark.py create mode 100644 student-tests/test_dependencies.py create mode 100644 student-tests/test_transactions.py diff --git a/.env.example b/.env.example index d32b9be0..01c14fe2 100644 --- a/.env.example +++ b/.env.example @@ -11,3 +11,9 @@ SESSION_STORE=postgres CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 + +# Лимиты одной API-реплики и воспроизводимый переключатель исследования. +API_CPUS=0.5 +API_MEMORY=256m +POOL_SIZE=10 +OPTIMIZED_WRITES=1 diff --git a/.github/workflows/contracts.yml b/.github/workflows/contracts.yml index 11cf289f..aadf476b 100644 --- a/.github/workflows/contracts.yml +++ b/.github/workflows/contracts.yml @@ -12,5 +12,21 @@ jobs: python-version: '3.12' - run: python -m pip install -r requirements-dev.txt - run: make check validate PYTHON=python -# This job validates the skeleton, not a student solution. -# Add build, readiness and make test in a separate application job. + application: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python scripts/bootstrap.py + - run: docker compose up -d --build + - run: python scripts/wait_ready.py --replicas 2 + - run: make test PYTHON=python + - run: python student-tests/test_lab3.py + - run: make test-persistence PYTHON=python ACTION_SCRIPT=scripts/restart.sh + - run: python scripts/wait_ready.py --replicas 2 + - run: make test-failover PYTHON=python ACTION_SCRIPT=scripts/stop-one.sh + - if: always() + run: docker compose down diff --git a/compose.yaml b/compose.yaml index 26cb7462..80859499 100644 --- a/compose.yaml +++ b/compose.yaml @@ -7,6 +7,7 @@ x-api: &api CACHE_URL: redis://redis:6379/0 SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} POOL_SIZE: ${POOL_SIZE:-10} + OPTIMIZED_WRITES: ${OPTIMIZED_WRITES:-1} secrets: [database_url] depends_on: migrate: diff --git a/evidence/cpu.json b/evidence/cpu.json new file mode 100644 index 00000000..11aa23a8 --- /dev/null +++ b/evidence/cpu.json @@ -0,0 +1,5 @@ +{ + "Name": "13th Gen Intel(R) Core(TM) i5-1340P", + "NumberOfCores": 12, + "NumberOfLogicalProcessors": 16 +} diff --git a/evidence/dependencies.txt b/evidence/dependencies.txt new file mode 100644 index 00000000..51416198 --- /dev/null +++ b/evidence/dependencies.txt @@ -0,0 +1,12 @@ + Container lyakhonov-lab3-redis-1 Stopping + Container lyakhonov-lab3-redis-1 Stopped + Container lyakhonov-lab3-redis-1 Starting + Container lyakhonov-lab3-redis-1 Started + Container lyakhonov-lab3-db-1 Stopping + Container lyakhonov-lab3-db-1 Stopped + Container lyakhonov-lab3-db-1 Starting + Container lyakhonov-lab3-db-1 Started +PASS: API ready, replicas: 2 +PASS: Redis остановлен, действующая сессия и профиль доступны из PostgreSQL за 0.27 с +PASS: PostgreSQL остановлен; закрытый отказ 503 за 3.037 с +PASS: прежняя сессия сохранилась после восстановления БД diff --git a/evidence/docker-resources.txt b/evidence/docker-resources.txt new file mode 100644 index 00000000..cb91ee33 --- /dev/null +++ b/evidence/docker-resources.txt @@ -0,0 +1 @@ +16 8126664704 "Docker Desktop" diff --git a/evidence/failover-1-containers.jsonl b/evidence/failover-1-containers.jsonl index 5dcd3a34..2b36ae57 100644 --- a/evidence/failover-1-containers.jsonl +++ b/evidence/failover-1-containers.jsonl @@ -1,6 +1,6 @@ -{"Command":"\"python -m uvicorn s…\"","CreatedAt":"2026-10-23 18:31:24 +0300 MSK","ExitCode":0,"Health":"","ID":"c086e70205ab","Image":"lyakhonov-messenger:lab3","Labels":"com.docker.compose.project.working_dir=C:\\Users\\User\\Desktop\\vysoko-nagr 3,com.docker.compose.service=api,com.docker.compose.config-hash=93b46b7eddf2f406e22ff9739f28d29e78cf3fb8c27d6862629d64204aa83029,com.docker.compose.image=sha256:f65f532e3042a0f9f8fcd9ffbf1722aaf432e327ced9f7a9bf0562e21d4c2ac8,com.docker.compose.project=lyakhonov-lab3,desktop.docker.io/mounts/0/SourceKind=hostFile,desktop.docker.io/mounts/0/Target=/run/secrets/database_url,com.docker.compose.replace=api-1,desktop.docker.io/mounts/0/Source=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\secrets\\database_url,desktop.docker.io/ports.scheme=v2,com.docker.compose.version=2.40.3,com.docker.compose.container-number=1,com.docker.compose.depends_on=migrate:service_completed_successfully:false,com.docker.compose.oneoff=False,com.docker.compose.project.config_files=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\compose.yaml","LocalVolumes":"0","Mounts":"/run/desktop/m…","Name":"lyakhonov-lab3-api-1","Names":"lyakhonov-lab3-api-1","Networks":"lyakhonov-lab3_default","Ports":"","Project":"lyakhonov-lab3","Publishers":[],"RunningFor":"About a minute ago","Service":"api","Size":"0B","State":"exited","Status":"Exited (0) 13 seconds ago"} -{"Command":"\"python -m uvicorn s…\"","CreatedAt":"2026-10-23 18:31:24 +0300 MSK","ExitCode":0,"Health":"healthy","ID":"1522443443f2","Image":"lyakhonov-messenger:lab3","Labels":"com.docker.compose.version=2.40.3,desktop.docker.io/mounts/0/Source=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\secrets\\database_url,desktop.docker.io/mounts/0/SourceKind=hostFile,desktop.docker.io/mounts/0/Target=/run/secrets/database_url,desktop.docker.io/ports.scheme=v2,com.docker.compose.container-number=2,com.docker.compose.image=sha256:f65f532e3042a0f9f8fcd9ffbf1722aaf432e327ced9f7a9bf0562e21d4c2ac8,com.docker.compose.oneoff=False,com.docker.compose.project=lyakhonov-lab3,com.docker.compose.replace=api-2,com.docker.compose.config-hash=93b46b7eddf2f406e22ff9739f28d29e78cf3fb8c27d6862629d64204aa83029,com.docker.compose.service=api,com.docker.compose.depends_on=migrate:service_completed_successfully:false,com.docker.compose.project.config_files=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\compose.yaml,com.docker.compose.project.working_dir=C:\\Users\\User\\Desktop\\vysoko-nagr 3","LocalVolumes":"0","Mounts":"/run/desktop/m…","Name":"lyakhonov-lab3-api-2","Names":"lyakhonov-lab3-api-2","Networks":"lyakhonov-lab3_default","Ports":"8080/tcp","Project":"lyakhonov-lab3","Publishers":[{"URL":"","TargetPort":8080,"PublishedPort":0,"Protocol":"tcp"}],"RunningFor":"About a minute ago","Service":"api","Size":"0B","State":"running","Status":"Up 22 seconds (healthy)"} -{"Command":"\"docker-entrypoint.s…\"","CreatedAt":"2026-10-23 18:29:58 +0300 MSK","ExitCode":0,"Health":"healthy","ID":"f2cdcc232751","Image":"postgres:16.6-alpine","Labels":"com.docker.compose.version=2.40.3,desktop.docker.io/mounts/0/Source=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\secrets\\db_password,com.docker.compose.config-hash=fa2b9ebefd39686b3404aa264a806204d268522150797263203427d945e51cd6,com.docker.compose.container-number=1,com.docker.compose.project=lyakhonov-lab3,desktop.docker.io/mounts/0/SourceKind=hostFile,desktop.docker.io/mounts/0/Target=/run/secrets/db_password,desktop.docker.io/ports.scheme=v2,com.docker.compose.depends_on=,com.docker.compose.image=sha256:1d04b9ba1d4996401f2552b51beda8187f175c0645c091e4781134fc9c9a3eef,com.docker.compose.oneoff=False,com.docker.compose.project.config_files=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\compose.yaml,com.docker.compose.project.working_dir=C:\\Users\\User\\Desktop\\vysoko-nagr 3,com.docker.compose.service=db","LocalVolumes":"1","Mounts":"/run/desktop/m…,lyakhonov-lab3…","Name":"lyakhonov-lab3-db-1","Names":"lyakhonov-lab3-db-1","Networks":"lyakhonov-lab3_default","Ports":"5432/tcp","Project":"lyakhonov-lab3","Publishers":[{"URL":"","TargetPort":5432,"PublishedPort":0,"Protocol":"tcp"}],"RunningFor":"2 minutes ago","Service":"db","Size":"0B","State":"running","Status":"Up 22 seconds (healthy)"} -{"Command":"\"docker-entrypoint.s…\"","CreatedAt":"2026-10-23 18:29:58 +0300 MSK","ExitCode":0,"Health":"healthy","ID":"4577101bc498","Image":"haproxy:3.0.6-alpine","Labels":"desktop.docker.io/binds/0/Source=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\infra\\haproxy.cfg,com.docker.compose.service=lb,com.docker.compose.container-number=1,com.docker.compose.image=sha256:c22c8710886104a48b920306f063401f0d11811858e3c6b9d87d88a7556b2e61,com.docker.compose.oneoff=False,com.docker.compose.project=lyakhonov-lab3,com.docker.compose.project.working_dir=C:\\Users\\User\\Desktop\\vysoko-nagr 3,desktop.docker.io/ports.scheme=v2,desktop.docker.io/ports/8080/tcp=127.0.0.1:8080,com.docker.compose.config-hash=c1b282a081343dbd036539a8c3bd364189a89534f9629cd69621bf9b57b5c061,com.docker.compose.depends_on=api:service_started:false,com.docker.compose.project.config_files=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\compose.yaml,com.docker.compose.version=2.40.3,desktop.docker.io/binds/0/SourceKind=hostFile,desktop.docker.io/binds/0/Target=/usr/local/etc/haproxy/haproxy.cfg","LocalVolumes":"0","Mounts":"/run/desktop/m…","Name":"lyakhonov-lab3-lb-1","Names":"lyakhonov-lab3-lb-1","Networks":"lyakhonov-lab3_default","Ports":"127.0.0.1:8080-\u003e8080/tcp","Project":"lyakhonov-lab3","Publishers":[{"URL":"127.0.0.1","TargetPort":8080,"PublishedPort":8080,"Protocol":"tcp"}],"RunningFor":"2 minutes ago","Service":"lb","Size":"0B","State":"running","Status":"Up 22 seconds (healthy)"} -{"Command":"\"python -m src.migra…\"","CreatedAt":"2026-10-23 18:31:23 +0300 MSK","ExitCode":0,"Health":"","ID":"c52ff4396329","Image":"lyakhonov-messenger:lab3","Labels":"com.docker.compose.depends_on=db:service_healthy:false,com.docker.compose.project=lyakhonov-lab3,com.docker.compose.service=migrate,desktop.docker.io/mounts/0/SourceKind=hostFile,com.docker.compose.config-hash=da46b111c5f3fa5d397c163d5add9113f69561f1561daa6ab1fba452f763242b,com.docker.compose.container-number=1,com.docker.compose.version=2.40.3,desktop.docker.io/mounts/0/Target=/run/secrets/database_url,com.docker.compose.image=sha256:f65f532e3042a0f9f8fcd9ffbf1722aaf432e327ced9f7a9bf0562e21d4c2ac8,com.docker.compose.oneoff=False,com.docker.compose.project.working_dir=C:\\Users\\User\\Desktop\\vysoko-nagr 3,desktop.docker.io/ports.scheme=v2,com.docker.compose.project.config_files=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\compose.yaml,com.docker.compose.replace=migrate-1,desktop.docker.io/mounts/0/Source=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\secrets\\database_url","LocalVolumes":"0","Mounts":"/run/desktop/m…","Name":"lyakhonov-lab3-migrate-1","Names":"lyakhonov-lab3-migrate-1","Networks":"lyakhonov-lab3_default","Ports":"","Project":"lyakhonov-lab3","Publishers":[],"RunningFor":"About a minute ago","Service":"migrate","Size":"0B","State":"exited","Status":"Exited (0) About a minute ago"} -{"Command":"\"docker-entrypoint.s…\"","CreatedAt":"2026-10-23 18:29:58 +0300 MSK","ExitCode":0,"Health":"","ID":"889db8e55eb2","Image":"redis:7.4.1-alpine","Labels":"com.docker.compose.project.config_files=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\compose.yaml,com.docker.compose.project.working_dir=C:\\Users\\User\\Desktop\\vysoko-nagr 3,com.docker.compose.version=2.40.3,com.docker.compose.config-hash=8484e75b165c3444cbb27a6887af7c18d49a9970a0c13127eb76917ce9adf837,com.docker.compose.container-number=1,com.docker.compose.depends_on=,com.docker.compose.image=sha256:c1e88455c85225310bbea54816e9c3f4b5295815e6dbf80c34d40afc6df28275,com.docker.compose.oneoff=False,com.docker.compose.service=redis,desktop.docker.io/ports.scheme=v2,com.docker.compose.project=lyakhonov-lab3","LocalVolumes":"1","Mounts":"99efe97b370b00…","Name":"lyakhonov-lab3-redis-1","Names":"lyakhonov-lab3-redis-1","Networks":"lyakhonov-lab3_default","Ports":"6379/tcp","Project":"lyakhonov-lab3","Publishers":[{"URL":"","TargetPort":6379,"PublishedPort":0,"Protocol":"tcp"}],"RunningFor":"2 minutes ago","Service":"redis","Size":"0B","State":"running","Status":"Up 22 seconds"} +{"Command":"\"python -m uvicorn s…\"","CreatedAt":"2026-10-23 19:05:09 +0300 MSK","ExitCode":0,"Health":"","ID":"e37175f9a3fa","Image":"lyakhonov-messenger:lab3","Labels":"desktop.docker.io/mounts/0/SourceKind=hostFile,desktop.docker.io/mounts/0/Target=/run/secrets/database_url,com.docker.compose.image=sha256:639e8935bac8e884626a86dc4bd71c623e1b5495ee6376a7e9e90583329c7292,com.docker.compose.oneoff=False,com.docker.compose.project.working_dir=C:\\Users\\User\\Desktop\\vysoko-nagr 3,desktop.docker.io/mounts/0/Source=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\secrets\\database_url,com.docker.compose.project=lyakhonov-lab3,com.docker.compose.container-number=10,com.docker.compose.project.config_files=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\compose.yaml,com.docker.compose.service=api,desktop.docker.io/ports.scheme=v2,com.docker.compose.config-hash=266d51102adf95b26fad565654ce091b36e87c7e55ca953339603fcd72da0e48,com.docker.compose.depends_on=migrate:service_completed_successfully:false,com.docker.compose.replace=api-10,com.docker.compose.version=2.40.3","LocalVolumes":"0","Mounts":"/run/desktop/m…","Name":"lyakhonov-lab3-api-10","Names":"lyakhonov-lab3-api-10","Networks":"lyakhonov-lab3_default","Ports":"","Project":"lyakhonov-lab3","Publishers":[],"RunningFor":"About a minute ago","Service":"api","Size":"0B","State":"exited","Status":"Exited (0) 13 seconds ago"} +{"Command":"\"python -m uvicorn s…\"","CreatedAt":"2026-10-23 19:05:09 +0300 MSK","ExitCode":0,"Health":"healthy","ID":"612c05caacd4","Image":"lyakhonov-messenger:lab3","Labels":"com.docker.compose.project=lyakhonov-lab3,desktop.docker.io/mounts/0/SourceKind=hostFile,desktop.docker.io/ports.scheme=v2,com.docker.compose.oneoff=False,desktop.docker.io/mounts/0/Source=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\secrets\\database_url,com.docker.compose.service=api,com.docker.compose.config-hash=266d51102adf95b26fad565654ce091b36e87c7e55ca953339603fcd72da0e48,com.docker.compose.container-number=9,com.docker.compose.project.config_files=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\compose.yaml,com.docker.compose.project.working_dir=C:\\Users\\User\\Desktop\\vysoko-nagr 3,com.docker.compose.replace=api-9,com.docker.compose.version=2.40.3,desktop.docker.io/mounts/0/Target=/run/secrets/database_url,com.docker.compose.depends_on=migrate:service_completed_successfully:false,com.docker.compose.image=sha256:639e8935bac8e884626a86dc4bd71c623e1b5495ee6376a7e9e90583329c7292","LocalVolumes":"0","Mounts":"/run/desktop/m…","Name":"lyakhonov-lab3-api-9","Names":"lyakhonov-lab3-api-9","Networks":"lyakhonov-lab3_default","Ports":"8080/tcp","Project":"lyakhonov-lab3","Publishers":[{"URL":"","TargetPort":8080,"PublishedPort":0,"Protocol":"tcp"}],"RunningFor":"About a minute ago","Service":"api","Size":"0B","State":"running","Status":"Up About a minute (healthy)"} +{"Command":"\"docker-entrypoint.s…\"","CreatedAt":"2026-10-23 18:29:58 +0300 MSK","ExitCode":0,"Health":"healthy","ID":"f2cdcc232751","Image":"postgres:16.6-alpine","Labels":"com.docker.compose.project.working_dir=C:\\Users\\User\\Desktop\\vysoko-nagr 3,com.docker.compose.version=2.40.3,desktop.docker.io/mounts/0/Source=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\secrets\\db_password,desktop.docker.io/mounts/0/Target=/run/secrets/db_password,com.docker.compose.depends_on=,com.docker.compose.oneoff=False,com.docker.compose.project.config_files=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\compose.yaml,com.docker.compose.service=db,desktop.docker.io/mounts/0/SourceKind=hostFile,desktop.docker.io/ports.scheme=v2,com.docker.compose.config-hash=fa2b9ebefd39686b3404aa264a806204d268522150797263203427d945e51cd6,com.docker.compose.container-number=1,com.docker.compose.image=sha256:1d04b9ba1d4996401f2552b51beda8187f175c0645c091e4781134fc9c9a3eef,com.docker.compose.project=lyakhonov-lab3","LocalVolumes":"1","Mounts":"/run/desktop/m…,lyakhonov-lab3…","Name":"lyakhonov-lab3-db-1","Names":"lyakhonov-lab3-db-1","Networks":"lyakhonov-lab3_default","Ports":"5432/tcp","Project":"lyakhonov-lab3","Publishers":[{"URL":"","TargetPort":5432,"PublishedPort":0,"Protocol":"tcp"}],"RunningFor":"36 minutes ago","Service":"db","Size":"0B","State":"running","Status":"Up 21 seconds (healthy)"} +{"Command":"\"docker-entrypoint.s…\"","CreatedAt":"2026-10-23 18:29:58 +0300 MSK","ExitCode":0,"Health":"healthy","ID":"4577101bc498","Image":"haproxy:3.0.6-alpine","Labels":"com.docker.compose.config-hash=c1b282a081343dbd036539a8c3bd364189a89534f9629cd69621bf9b57b5c061,com.docker.compose.depends_on=api:service_started:false,com.docker.compose.image=sha256:c22c8710886104a48b920306f063401f0d11811858e3c6b9d87d88a7556b2e61,desktop.docker.io/binds/0/Target=/usr/local/etc/haproxy/haproxy.cfg,com.docker.compose.container-number=1,com.docker.compose.project.config_files=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\compose.yaml,com.docker.compose.project.working_dir=C:\\Users\\User\\Desktop\\vysoko-nagr 3,desktop.docker.io/binds/0/Source=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\infra\\haproxy.cfg,desktop.docker.io/binds/0/SourceKind=hostFile,com.docker.compose.version=2.40.3,desktop.docker.io/ports.scheme=v2,com.docker.compose.oneoff=False,com.docker.compose.project=lyakhonov-lab3,com.docker.compose.service=lb,desktop.docker.io/ports/8080/tcp=127.0.0.1:8080","LocalVolumes":"0","Mounts":"/run/desktop/m…","Name":"lyakhonov-lab3-lb-1","Names":"lyakhonov-lab3-lb-1","Networks":"lyakhonov-lab3_default","Ports":"127.0.0.1:8080-\u003e8080/tcp","Project":"lyakhonov-lab3","Publishers":[{"URL":"127.0.0.1","TargetPort":8080,"PublishedPort":8080,"Protocol":"tcp"}],"RunningFor":"36 minutes ago","Service":"lb","Size":"0B","State":"running","Status":"Up 2 minutes (healthy)"} +{"Command":"\"python -m src.migra…\"","CreatedAt":"2026-10-23 19:05:08 +0300 MSK","ExitCode":0,"Health":"","ID":"1031c94e3574","Image":"lyakhonov-messenger:lab3","Labels":"com.docker.compose.oneoff=False,com.docker.compose.project.working_dir=C:\\Users\\User\\Desktop\\vysoko-nagr 3,desktop.docker.io/mounts/0/Target=/run/secrets/database_url,desktop.docker.io/ports.scheme=v2,com.docker.compose.config-hash=8769eace7a6a1e1092198d406a7c10a8c417bc97737d952ee0fe97abf2fb98de,com.docker.compose.depends_on=db:service_healthy:false,com.docker.compose.image=sha256:639e8935bac8e884626a86dc4bd71c623e1b5495ee6376a7e9e90583329c7292,com.docker.compose.version=2.40.3,desktop.docker.io/mounts/0/Source=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\secrets\\database_url,com.docker.compose.container-number=1,com.docker.compose.project=lyakhonov-lab3,com.docker.compose.project.config_files=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\compose.yaml,com.docker.compose.replace=migrate-1,com.docker.compose.service=migrate,desktop.docker.io/mounts/0/SourceKind=hostFile","LocalVolumes":"0","Mounts":"/run/desktop/m…","Name":"lyakhonov-lab3-migrate-1","Names":"lyakhonov-lab3-migrate-1","Networks":"lyakhonov-lab3_default","Ports":"","Project":"lyakhonov-lab3","Publishers":[],"RunningFor":"About a minute ago","Service":"migrate","Size":"0B","State":"exited","Status":"Exited (0) 18 seconds ago"} +{"Command":"\"docker-entrypoint.s…\"","CreatedAt":"2026-10-23 18:29:58 +0300 MSK","ExitCode":0,"Health":"","ID":"889db8e55eb2","Image":"redis:7.4.1-alpine","Labels":"com.docker.compose.image=sha256:c1e88455c85225310bbea54816e9c3f4b5295815e6dbf80c34d40afc6df28275,com.docker.compose.project=lyakhonov-lab3,com.docker.compose.service=redis,com.docker.compose.version=2.40.3,desktop.docker.io/ports.scheme=v2,com.docker.compose.config-hash=8484e75b165c3444cbb27a6887af7c18d49a9970a0c13127eb76917ce9adf837,com.docker.compose.depends_on=,com.docker.compose.oneoff=False,com.docker.compose.project.config_files=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\compose.yaml,com.docker.compose.project.working_dir=C:\\Users\\User\\Desktop\\vysoko-nagr 3,com.docker.compose.container-number=1","LocalVolumes":"1","Mounts":"99efe97b370b00…","Name":"lyakhonov-lab3-redis-1","Names":"lyakhonov-lab3-redis-1","Networks":"lyakhonov-lab3_default","Ports":"6379/tcp","Project":"lyakhonov-lab3","Publishers":[{"URL":"","TargetPort":6379,"PublishedPort":0,"Protocol":"tcp"}],"RunningFor":"36 minutes ago","Service":"redis","Size":"0B","State":"running","Status":"Up 26 seconds"} diff --git a/evidence/failover-1.txt b/evidence/failover-1.txt index b001ae95..47544800 100644 --- a/evidence/failover-1.txt +++ b/evidence/failover-1.txt @@ -1,7 +1,3 @@ -'test' is not recognized as an internal or external command, -operable program or batch file. -"Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2 -python tests/resilience.py --mode failover --action-script "scripts/stop-one.cmd" --base-url "http://localhost:8080" Executing explicit failover hook; hook output suppressed to avoid leaking secrets. -{"observed_instances_before": 2, "successful_writes": 89, "failed_attempts": 4, "first_recovery_seconds": 3.114} +{"observed_instances_before": 2, "successful_writes": 93, "failed_attempts": 3, "first_recovery_seconds": 2.875} PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/failover-2-containers.jsonl b/evidence/failover-2-containers.jsonl index 11986bf5..927a4aaf 100644 --- a/evidence/failover-2-containers.jsonl +++ b/evidence/failover-2-containers.jsonl @@ -1,6 +1,6 @@ -{"Command":"\"python -m uvicorn s…\"","CreatedAt":"2026-10-23 18:31:24 +0300 MSK","ExitCode":0,"Health":"starting","ID":"c086e70205ab","Image":"lyakhonov-messenger:lab3","Labels":"com.docker.compose.project.config_files=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\compose.yaml,com.docker.compose.project.working_dir=C:\\Users\\User\\Desktop\\vysoko-nagr 3,desktop.docker.io/ports.scheme=v2,com.docker.compose.container-number=1,com.docker.compose.project=lyakhonov-lab3,desktop.docker.io/mounts/0/Source=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\secrets\\database_url,desktop.docker.io/mounts/0/Target=/run/secrets/database_url,com.docker.compose.image=sha256:f65f532e3042a0f9f8fcd9ffbf1722aaf432e327ced9f7a9bf0562e21d4c2ac8,desktop.docker.io/mounts/0/SourceKind=hostFile,com.docker.compose.config-hash=93b46b7eddf2f406e22ff9739f28d29e78cf3fb8c27d6862629d64204aa83029,com.docker.compose.replace=api-1,com.docker.compose.service=api,com.docker.compose.version=2.40.3,com.docker.compose.depends_on=migrate:service_completed_successfully:false,com.docker.compose.oneoff=False","LocalVolumes":"0","Mounts":"/run/desktop/m…","Name":"lyakhonov-lab3-api-1","Names":"lyakhonov-lab3-api-1","Networks":"lyakhonov-lab3_default","Ports":"8080/tcp","Project":"lyakhonov-lab3","Publishers":[{"URL":"","TargetPort":8080,"PublishedPort":0,"Protocol":"tcp"}],"RunningFor":"2 minutes ago","Service":"api","Size":"0B","State":"running","Status":"Up 3 seconds (health: starting)"} -{"Command":"\"python -m uvicorn s…\"","CreatedAt":"2026-10-23 18:31:24 +0300 MSK","ExitCode":0,"Health":"healthy","ID":"1522443443f2","Image":"lyakhonov-messenger:lab3","Labels":"com.docker.compose.image=sha256:f65f532e3042a0f9f8fcd9ffbf1722aaf432e327ced9f7a9bf0562e21d4c2ac8,com.docker.compose.project=lyakhonov-lab3,com.docker.compose.project.config_files=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\compose.yaml,desktop.docker.io/mounts/0/Source=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\secrets\\database_url,desktop.docker.io/mounts/0/Target=/run/secrets/database_url,com.docker.compose.config-hash=93b46b7eddf2f406e22ff9739f28d29e78cf3fb8c27d6862629d64204aa83029,com.docker.compose.depends_on=migrate:service_completed_successfully:false,com.docker.compose.replace=api-2,desktop.docker.io/mounts/0/SourceKind=hostFile,com.docker.compose.container-number=2,com.docker.compose.service=api,com.docker.compose.version=2.40.3,desktop.docker.io/ports.scheme=v2,com.docker.compose.oneoff=False,com.docker.compose.project.working_dir=C:\\Users\\User\\Desktop\\vysoko-nagr 3","LocalVolumes":"0","Mounts":"/run/desktop/m…","Name":"lyakhonov-lab3-api-2","Names":"lyakhonov-lab3-api-2","Networks":"lyakhonov-lab3_default","Ports":"8080/tcp","Project":"lyakhonov-lab3","Publishers":[{"URL":"","TargetPort":8080,"PublishedPort":0,"Protocol":"tcp"}],"RunningFor":"2 minutes ago","Service":"api","Size":"0B","State":"running","Status":"Up About a minute (healthy)"} -{"Command":"\"docker-entrypoint.s…\"","CreatedAt":"2026-10-23 18:29:58 +0300 MSK","ExitCode":0,"Health":"healthy","ID":"f2cdcc232751","Image":"postgres:16.6-alpine","Labels":"com.docker.compose.project=lyakhonov-lab3,com.docker.compose.project.config_files=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\compose.yaml,com.docker.compose.project.working_dir=C:\\Users\\User\\Desktop\\vysoko-nagr 3,desktop.docker.io/mounts/0/SourceKind=hostFile,com.docker.compose.container-number=1,com.docker.compose.depends_on=,com.docker.compose.oneoff=False,com.docker.compose.service=db,com.docker.compose.version=2.40.3,desktop.docker.io/mounts/0/Source=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\secrets\\db_password,desktop.docker.io/mounts/0/Target=/run/secrets/db_password,desktop.docker.io/ports.scheme=v2,com.docker.compose.config-hash=fa2b9ebefd39686b3404aa264a806204d268522150797263203427d945e51cd6,com.docker.compose.image=sha256:1d04b9ba1d4996401f2552b51beda8187f175c0645c091e4781134fc9c9a3eef","LocalVolumes":"1","Mounts":"/run/desktop/m…,lyakhonov-lab3…","Name":"lyakhonov-lab3-db-1","Names":"lyakhonov-lab3-db-1","Networks":"lyakhonov-lab3_default","Ports":"5432/tcp","Project":"lyakhonov-lab3","Publishers":[{"URL":"","TargetPort":5432,"PublishedPort":0,"Protocol":"tcp"}],"RunningFor":"3 minutes ago","Service":"db","Size":"0B","State":"running","Status":"Up About a minute (healthy)"} -{"Command":"\"docker-entrypoint.s…\"","CreatedAt":"2026-10-23 18:29:58 +0300 MSK","ExitCode":0,"Health":"healthy","ID":"4577101bc498","Image":"haproxy:3.0.6-alpine","Labels":"com.docker.compose.project.config_files=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\compose.yaml,desktop.docker.io/binds/0/SourceKind=hostFile,desktop.docker.io/ports.scheme=v2,desktop.docker.io/ports/8080/tcp=127.0.0.1:8080,com.docker.compose.oneoff=False,com.docker.compose.service=lb,com.docker.compose.depends_on=api:service_started:false,com.docker.compose.image=sha256:c22c8710886104a48b920306f063401f0d11811858e3c6b9d87d88a7556b2e61,desktop.docker.io/binds/0/Source=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\infra\\haproxy.cfg,com.docker.compose.config-hash=c1b282a081343dbd036539a8c3bd364189a89534f9629cd69621bf9b57b5c061,com.docker.compose.container-number=1,com.docker.compose.project=lyakhonov-lab3,com.docker.compose.project.working_dir=C:\\Users\\User\\Desktop\\vysoko-nagr 3,com.docker.compose.version=2.40.3,desktop.docker.io/binds/0/Target=/usr/local/etc/haproxy/haproxy.cfg","LocalVolumes":"0","Mounts":"/run/desktop/m…","Name":"lyakhonov-lab3-lb-1","Names":"lyakhonov-lab3-lb-1","Networks":"lyakhonov-lab3_default","Ports":"127.0.0.1:8080-\u003e8080/tcp","Project":"lyakhonov-lab3","Publishers":[{"URL":"127.0.0.1","TargetPort":8080,"PublishedPort":8080,"Protocol":"tcp"}],"RunningFor":"3 minutes ago","Service":"lb","Size":"0B","State":"running","Status":"Up About a minute (healthy)"} -{"Command":"\"python -m src.migra…\"","CreatedAt":"2026-10-23 18:31:23 +0300 MSK","ExitCode":0,"Health":"","ID":"c52ff4396329","Image":"lyakhonov-messenger:lab3","Labels":"desktop.docker.io/mounts/0/Source=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\secrets\\database_url,desktop.docker.io/mounts/0/SourceKind=hostFile,com.docker.compose.config-hash=da46b111c5f3fa5d397c163d5add9113f69561f1561daa6ab1fba452f763242b,com.docker.compose.project=lyakhonov-lab3,com.docker.compose.project.working_dir=C:\\Users\\User\\Desktop\\vysoko-nagr 3,desktop.docker.io/mounts/0/Target=/run/secrets/database_url,com.docker.compose.container-number=1,com.docker.compose.depends_on=db:service_healthy:false,com.docker.compose.image=sha256:f65f532e3042a0f9f8fcd9ffbf1722aaf432e327ced9f7a9bf0562e21d4c2ac8,com.docker.compose.oneoff=False,desktop.docker.io/ports.scheme=v2,com.docker.compose.project.config_files=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\compose.yaml,com.docker.compose.replace=migrate-1,com.docker.compose.service=migrate,com.docker.compose.version=2.40.3","LocalVolumes":"0","Mounts":"/run/desktop/m…","Name":"lyakhonov-lab3-migrate-1","Names":"lyakhonov-lab3-migrate-1","Networks":"lyakhonov-lab3_default","Ports":"","Project":"lyakhonov-lab3","Publishers":[],"RunningFor":"2 minutes ago","Service":"migrate","Size":"0B","State":"exited","Status":"Exited (0) 3 seconds ago"} -{"Command":"\"docker-entrypoint.s…\"","CreatedAt":"2026-10-23 18:29:58 +0300 MSK","ExitCode":0,"Health":"","ID":"889db8e55eb2","Image":"redis:7.4.1-alpine","Labels":"com.docker.compose.container-number=1,com.docker.compose.oneoff=False,com.docker.compose.project.working_dir=C:\\Users\\User\\Desktop\\vysoko-nagr 3,desktop.docker.io/ports.scheme=v2,com.docker.compose.config-hash=8484e75b165c3444cbb27a6887af7c18d49a9970a0c13127eb76917ce9adf837,com.docker.compose.depends_on=,com.docker.compose.image=sha256:c1e88455c85225310bbea54816e9c3f4b5295815e6dbf80c34d40afc6df28275,com.docker.compose.project=lyakhonov-lab3,com.docker.compose.project.config_files=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\compose.yaml,com.docker.compose.service=redis,com.docker.compose.version=2.40.3","LocalVolumes":"1","Mounts":"99efe97b370b00…","Name":"lyakhonov-lab3-redis-1","Names":"lyakhonov-lab3-redis-1","Networks":"lyakhonov-lab3_default","Ports":"6379/tcp","Project":"lyakhonov-lab3","Publishers":[{"URL":"","TargetPort":6379,"PublishedPort":0,"Protocol":"tcp"}],"RunningFor":"3 minutes ago","Service":"redis","Size":"0B","State":"running","Status":"Up About a minute"} +{"Command":"\"python -m uvicorn s…\"","CreatedAt":"2026-10-23 19:05:09 +0300 MSK","ExitCode":0,"Health":"healthy","ID":"e37175f9a3fa","Image":"lyakhonov-messenger:lab3","Labels":"com.docker.compose.container-number=10,com.docker.compose.depends_on=migrate:service_completed_successfully:false,desktop.docker.io/mounts/0/Source=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\secrets\\database_url,desktop.docker.io/mounts/0/SourceKind=hostFile,com.docker.compose.project.working_dir=C:\\Users\\User\\Desktop\\vysoko-nagr 3,com.docker.compose.service=api,desktop.docker.io/mounts/0/Target=/run/secrets/database_url,com.docker.compose.oneoff=False,com.docker.compose.project=lyakhonov-lab3,com.docker.compose.project.config_files=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\compose.yaml,com.docker.compose.version=2.40.3,com.docker.compose.config-hash=266d51102adf95b26fad565654ce091b36e87c7e55ca953339603fcd72da0e48,com.docker.compose.image=sha256:639e8935bac8e884626a86dc4bd71c623e1b5495ee6376a7e9e90583329c7292,com.docker.compose.replace=api-10,desktop.docker.io/ports.scheme=v2","LocalVolumes":"0","Mounts":"/run/desktop/m…","Name":"lyakhonov-lab3-api-10","Names":"lyakhonov-lab3-api-10","Networks":"lyakhonov-lab3_default","Ports":"8080/tcp","Project":"lyakhonov-lab3","Publishers":[{"URL":"","TargetPort":8080,"PublishedPort":0,"Protocol":"tcp"}],"RunningFor":"2 minutes ago","Service":"api","Size":"0B","State":"running","Status":"Up 21 seconds (healthy)"} +{"Command":"\"python -m uvicorn s…\"","CreatedAt":"2026-10-23 19:05:09 +0300 MSK","ExitCode":0,"Health":"","ID":"612c05caacd4","Image":"lyakhonov-messenger:lab3","Labels":"com.docker.compose.image=sha256:639e8935bac8e884626a86dc4bd71c623e1b5495ee6376a7e9e90583329c7292,com.docker.compose.project=lyakhonov-lab3,com.docker.compose.project.config_files=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\compose.yaml,com.docker.compose.project.working_dir=C:\\Users\\User\\Desktop\\vysoko-nagr 3,com.docker.compose.service=api,com.docker.compose.config-hash=266d51102adf95b26fad565654ce091b36e87c7e55ca953339603fcd72da0e48,com.docker.compose.depends_on=migrate:service_completed_successfully:false,com.docker.compose.version=2.40.3,desktop.docker.io/mounts/0/Target=/run/secrets/database_url,com.docker.compose.container-number=9,com.docker.compose.oneoff=False,com.docker.compose.replace=api-9,desktop.docker.io/mounts/0/Source=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\secrets\\database_url,desktop.docker.io/mounts/0/SourceKind=hostFile,desktop.docker.io/ports.scheme=v2","LocalVolumes":"0","Mounts":"/run/desktop/m…","Name":"lyakhonov-lab3-api-9","Names":"lyakhonov-lab3-api-9","Networks":"lyakhonov-lab3_default","Ports":"","Project":"lyakhonov-lab3","Publishers":[],"RunningFor":"2 minutes ago","Service":"api","Size":"0B","State":"exited","Status":"Exited (0) 13 seconds ago"} +{"Command":"\"docker-entrypoint.s…\"","CreatedAt":"2026-10-23 18:29:58 +0300 MSK","ExitCode":0,"Health":"healthy","ID":"f2cdcc232751","Image":"postgres:16.6-alpine","Labels":"com.docker.compose.project.working_dir=C:\\Users\\User\\Desktop\\vysoko-nagr 3,com.docker.compose.service=db,desktop.docker.io/mounts/0/SourceKind=hostFile,desktop.docker.io/mounts/0/Target=/run/secrets/db_password,desktop.docker.io/ports.scheme=v2,com.docker.compose.config-hash=fa2b9ebefd39686b3404aa264a806204d268522150797263203427d945e51cd6,com.docker.compose.container-number=1,com.docker.compose.depends_on=,com.docker.compose.project=lyakhonov-lab3,com.docker.compose.version=2.40.3,desktop.docker.io/mounts/0/Source=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\secrets\\db_password,com.docker.compose.image=sha256:1d04b9ba1d4996401f2552b51beda8187f175c0645c091e4781134fc9c9a3eef,com.docker.compose.oneoff=False,com.docker.compose.project.config_files=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\compose.yaml","LocalVolumes":"1","Mounts":"/run/desktop/m…,lyakhonov-lab3…","Name":"lyakhonov-lab3-db-1","Names":"lyakhonov-lab3-db-1","Networks":"lyakhonov-lab3_default","Ports":"5432/tcp","Project":"lyakhonov-lab3","Publishers":[{"URL":"","TargetPort":5432,"PublishedPort":0,"Protocol":"tcp"}],"RunningFor":"37 minutes ago","Service":"db","Size":"0B","State":"running","Status":"Up About a minute (healthy)"} +{"Command":"\"docker-entrypoint.s…\"","CreatedAt":"2026-10-23 18:29:58 +0300 MSK","ExitCode":0,"Health":"healthy","ID":"4577101bc498","Image":"haproxy:3.0.6-alpine","Labels":"com.docker.compose.config-hash=c1b282a081343dbd036539a8c3bd364189a89534f9629cd69621bf9b57b5c061,com.docker.compose.oneoff=False,com.docker.compose.project=lyakhonov-lab3,com.docker.compose.service=lb,com.docker.compose.depends_on=api:service_started:false,com.docker.compose.project.config_files=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\compose.yaml,desktop.docker.io/binds/0/SourceKind=hostFile,desktop.docker.io/binds/0/Target=/usr/local/etc/haproxy/haproxy.cfg,desktop.docker.io/ports.scheme=v2,com.docker.compose.container-number=1,desktop.docker.io/ports/8080/tcp=127.0.0.1:8080,com.docker.compose.image=sha256:c22c8710886104a48b920306f063401f0d11811858e3c6b9d87d88a7556b2e61,com.docker.compose.project.working_dir=C:\\Users\\User\\Desktop\\vysoko-nagr 3,com.docker.compose.version=2.40.3,desktop.docker.io/binds/0/Source=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\infra\\haproxy.cfg","LocalVolumes":"0","Mounts":"/run/desktop/m…","Name":"lyakhonov-lab3-lb-1","Names":"lyakhonov-lab3-lb-1","Networks":"lyakhonov-lab3_default","Ports":"127.0.0.1:8080-\u003e8080/tcp","Project":"lyakhonov-lab3","Publishers":[{"URL":"127.0.0.1","TargetPort":8080,"PublishedPort":8080,"Protocol":"tcp"}],"RunningFor":"37 minutes ago","Service":"lb","Size":"0B","State":"running","Status":"Up 4 minutes (healthy)"} +{"Command":"\"python -m src.migra…\"","CreatedAt":"2026-10-23 19:05:08 +0300 MSK","ExitCode":0,"Health":"","ID":"1031c94e3574","Image":"lyakhonov-messenger:lab3","Labels":"com.docker.compose.project.working_dir=C:\\Users\\User\\Desktop\\vysoko-nagr 3,com.docker.compose.version=2.40.3,com.docker.compose.oneoff=False,com.docker.compose.service=migrate,desktop.docker.io/mounts/0/Target=/run/secrets/database_url,com.docker.compose.container-number=1,com.docker.compose.depends_on=db:service_healthy:false,com.docker.compose.image=sha256:639e8935bac8e884626a86dc4bd71c623e1b5495ee6376a7e9e90583329c7292,desktop.docker.io/mounts/0/Source=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\secrets\\database_url,desktop.docker.io/mounts/0/SourceKind=hostFile,desktop.docker.io/ports.scheme=v2,com.docker.compose.config-hash=8769eace7a6a1e1092198d406a7c10a8c417bc97737d952ee0fe97abf2fb98de,com.docker.compose.project=lyakhonov-lab3,com.docker.compose.project.config_files=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\compose.yaml,com.docker.compose.replace=migrate-1","LocalVolumes":"0","Mounts":"/run/desktop/m…","Name":"lyakhonov-lab3-migrate-1","Names":"lyakhonov-lab3-migrate-1","Networks":"lyakhonov-lab3_default","Ports":"","Project":"lyakhonov-lab3","Publishers":[],"RunningFor":"2 minutes ago","Service":"migrate","Size":"0B","State":"exited","Status":"Exited (0) 21 seconds ago"} +{"Command":"\"docker-entrypoint.s…\"","CreatedAt":"2026-10-23 18:29:58 +0300 MSK","ExitCode":0,"Health":"","ID":"889db8e55eb2","Image":"redis:7.4.1-alpine","Labels":"com.docker.compose.config-hash=8484e75b165c3444cbb27a6887af7c18d49a9970a0c13127eb76917ce9adf837,com.docker.compose.depends_on=,com.docker.compose.oneoff=False,com.docker.compose.project=lyakhonov-lab3,com.docker.compose.service=redis,com.docker.compose.version=2.40.3,com.docker.compose.container-number=1,com.docker.compose.image=sha256:c1e88455c85225310bbea54816e9c3f4b5295815e6dbf80c34d40afc6df28275,com.docker.compose.project.config_files=C:\\Users\\User\\Desktop\\vysoko-nagr 3\\compose.yaml,com.docker.compose.project.working_dir=C:\\Users\\User\\Desktop\\vysoko-nagr 3,desktop.docker.io/ports.scheme=v2","LocalVolumes":"1","Mounts":"99efe97b370b00…","Name":"lyakhonov-lab3-redis-1","Names":"lyakhonov-lab3-redis-1","Networks":"lyakhonov-lab3_default","Ports":"6379/tcp","Project":"lyakhonov-lab3","Publishers":[{"URL":"","TargetPort":6379,"PublishedPort":0,"Protocol":"tcp"}],"RunningFor":"37 minutes ago","Service":"redis","Size":"0B","State":"running","Status":"Up About a minute"} diff --git a/evidence/failover-2.txt b/evidence/failover-2.txt index fd0e8fed..038e4f66 100644 --- a/evidence/failover-2.txt +++ b/evidence/failover-2.txt @@ -1 +1,3 @@ -python tests/resilience.py --mode failover --action-script "scripts/stop-one.cmd" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 97, "failed_attempts": 2, "first_recovery_seconds": 2.755} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/final-checks.txt b/evidence/final-checks.txt new file mode 100644 index 00000000..f717921f --- /dev/null +++ b/evidence/final-checks.txt @@ -0,0 +1,4 @@ +python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +python tests/smoke.py --base-url "http://localhost:8080" +PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/idempotency.txt b/evidence/idempotency.txt index 4f06fa33..62bc4a1f 100644 --- a/evidence/idempotency.txt +++ b/evidence/idempotency.txt @@ -1 +1 @@ -PASS: 32 параллельных запроса, 2 реплики; один результат, конфликт, валидация, ACL, области ключа и отзыв сессии. +PASS: 32 одинаковых и 16 конфликтующих параллельных запросов, 2 реплики; один результат, конфликт, валидация, ACL, области ключа и отзыв сессии. diff --git a/evidence/level4.md b/evidence/level4.md new file mode 100644 index 00000000..fcb90526 --- /dev/null +++ b/evidence/level4.md @@ -0,0 +1,15 @@ +# Завершение общего уровня 4 + +Автор: Ляхонов Сергей, БВТ2301. Дата: 23.10.2026. + +Остановлены и восстановлены обе разные реальные реплики api-10 и api-9. Публичные failover-сценарии прошли: восстановление записи 2.875 и 2.755 с, ошибки переходного периода 3 и 2; сохранность и отсутствие дублей подтверждены. См. failover-1/2.txt и снимки контейнеров. + +Полный рестарт API, БД, Redis и LB сохранил прежний токен и данные. StartedAt до/после сохранены. Повтор миграций не создал новых версий: 001 и 002. + +На окончательном образе выполнена поочерёдная замена обоих API под нагрузкой: 1977 успешных ответов, 83 ошибок, 43.73 RPS. Подтверждённых ID проверено 2265; все найдены в истории. Переходные ошибки не скрываются: 4.029%. Доказательство — research/rolling.json с ID старых/новых процессов и временами действий. + +Проверены остановка Redis, ограниченный fallback (0.270 с), остановка PostgreSQL (503 за 3.037 с), восстановление прежней сессии. Исправлено ожидание DNS необязательного кэша: максимум один фоновый вызов, ожидание 250 мс, без роста очереди. Отдельно проверены имитация истечения ключа, транзакционный откат при искусственном 500 и успешный повтор. Trigger теста удалён. + +Smoke, 32 одинаковых и 16 конфликтующих параллельных запросов прошли. Исходный путь записи оставлен воспроизводимым через OPTIMIZED_WRITES=0; улучшенный путь с шестью вместо восьми явных SQL включён по умолчанию. Парная оценка его производительности относится к следующему исследовательскому этапу. + +Настройки timeouts, readiness, SIGTERM, DNS, лимитов и миграций описаны в infra/operations.md. Публичные тесты и обязательный контракт сохранены. CI дополнен отдельным application job; удалённый запуск не заявляется. diff --git a/evidence/memory.json b/evidence/memory.json new file mode 100644 index 00000000..5db49758 --- /dev/null +++ b/evidence/memory.json @@ -0,0 +1,3 @@ +{ + "TotalPhysicalMemory": 16778182656 +} diff --git a/evidence/migrations.txt b/evidence/migrations.txt new file mode 100644 index 00000000..b5a7a63e --- /dev/null +++ b/evidence/migrations.txt @@ -0,0 +1,2 @@ +001_initial.sql +002_idempotency.sql diff --git a/evidence/persistence-started-after.txt b/evidence/persistence-started-after.txt index 77807528..7d4d5742 100644 --- a/evidence/persistence-started-after.txt +++ b/evidence/persistence-started-after.txt @@ -1,5 +1,5 @@ -c086e70205ab46e1d0578d0ddd7c8b81d92b6792b4e7489bca24f4a1ae7cedba 2026-10-23T15:34:55.775855887Z -1522443443f214f454783dde216b41c6415269adc0e46293db086093c80e64f8 2026-10-23T15:34:55.822169592Z -f2cdcc232751e2c2400763e935897851aca1399824ed751ec96a51180282a207 2026-10-23T15:34:55.199585718Z -4577101bc498d9f217ad682ab8af2a4688f7772287a1755eab603ecc946ce2b7 2026-10-23T15:34:55.133175966Z -889db8e55eb20c99fbe462f3d407763760cc40f0a053b144dcb9f6d13f6255c8 2026-10-23T15:34:55.264568727Z +e37175f9a3fa89d28d266869eb804e06e36502a4702a6661668caf684555bcef 2026-10-23T16:08:02.552700017Z +612c05caacd47e750f542e055ebaa578c0f6a221d9091e6ce5df2fa945c52b65 2026-10-23T16:08:02.47911506Z +f2cdcc232751e2c2400763e935897851aca1399824ed751ec96a51180282a207 2026-10-23T16:08:02.306518292Z +4577101bc498d9f217ad682ab8af2a4688f7772287a1755eab603ecc946ce2b7 2026-10-23T16:08:02.041383013Z +889db8e55eb20c99fbe462f3d407763760cc40f0a053b144dcb9f6d13f6255c8 2026-10-23T16:08:02.163804371Z diff --git a/evidence/persistence-started-before.txt b/evidence/persistence-started-before.txt index c96fd93e..5d5f3946 100644 --- a/evidence/persistence-started-before.txt +++ b/evidence/persistence-started-before.txt @@ -1,5 +1,5 @@ -c086e70205ab46e1d0578d0ddd7c8b81d92b6792b4e7489bca24f4a1ae7cedba 2026-10-23T15:33:57.680101854Z -1522443443f214f454783dde216b41c6415269adc0e46293db086093c80e64f8 2026-10-23T15:33:57.629513846Z -f2cdcc232751e2c2400763e935897851aca1399824ed751ec96a51180282a207 2026-10-23T15:33:57.51109515Z -4577101bc498d9f217ad682ab8af2a4688f7772287a1755eab603ecc946ce2b7 2026-10-23T15:33:57.292608383Z -889db8e55eb20c99fbe462f3d407763760cc40f0a053b144dcb9f6d13f6255c8 2026-10-23T15:33:57.389825012Z +e37175f9a3fa89d28d266869eb804e06e36502a4702a6661668caf684555bcef 2026-10-23T16:07:31.750783546Z +612c05caacd47e750f542e055ebaa578c0f6a221d9091e6ce5df2fa945c52b65 2026-10-23T16:07:55.423787453Z +f2cdcc232751e2c2400763e935897851aca1399824ed751ec96a51180282a207 2026-10-23T16:06:20.168424171Z +4577101bc498d9f217ad682ab8af2a4688f7772287a1755eab603ecc946ce2b7 2026-10-23T16:03:47.244254065Z +889db8e55eb20c99fbe462f3d407763760cc40f0a053b144dcb9f6d13f6255c8 2026-10-23T16:06:15.458166414Z diff --git a/evidence/research/rolling.json b/evidence/research/rolling.json new file mode 100644 index 00000000..16455e18 --- /dev/null +++ b/evidence/research/rolling.json @@ -0,0 +1,1416 @@ +{ + "duration": 45.0, + "warmup": 5, + "elapsed": 45.21266958399792, + "concurrency": 16, + "successful_requests": 1977, + "failed_requests": 83, + "errors": { + "503": 67, + "connection": 16 + }, + "error_ratio": 0.04029126213592233, + "successful_rps": 43.726681441073715, + "successful_latency_ms": { + "p50": 398.37, + "p95": 921.695, + "p99": 1155.103 + }, + "instances": { + "612c05caacd4-5f9dc50d": 364, + "e37175f9a3fa-ec119f6f": 87, + "125c1e66e5bf-51c52a85": 902, + "65d711abae2e-6a9bb556": 624 + }, + "initial_chat_messages": 100, + "confirmed_ids_checked": 2265, + "history_count_after": 2365, + "confirmed_data_preserved": true, + "warmup_messages": 288, + "final_chat_messages_expected_min": 2365, + "note": "Closed-loop; coordinated omission; percentiles cover successes; no retries; uncertain writes may exist.", + "label": "rolling", + "started_utc": "2026-10-23T16:08:13.637899+00:00", + "replicas": 2, + "cpu_per_replica": 0.5, + "memory_mib_per_replica": 256, + "pool_per_replica": 10, + "database_messages_before": 66033, + "database_messages_after": 68398, + "samples": [ + { + "seconds": 3.183, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "11.97%", + "Container": "11cd774f4e060ce533d47473491be4336a62e0b3d1954ecfdd6cd785f034e6ae", + "ID": "11cd774f4e06", + "MemPerc": "2.86%", + "MemUsage": "14.62MiB / 512MiB", + "Name": "lyakhonov-lab3-load", + "NetIO": "116kB / 164kB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "39.96%", + "Container": "e37175f9a3fa89d28d266869eb804e06e36502a4702a6661668caf684555bcef", + "ID": "e37175f9a3fa", + "MemPerc": "26.98%", + "MemUsage": "69.06MiB / 256MiB", + "Name": "lyakhonov-lab3-api-10", + "NetIO": "271kB / 279kB", + "PIDs": "18" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "37.01%", + "Container": "612c05caacd47e750f542e055ebaa578c0f6a221d9091e6ce5df2fa945c52b65", + "ID": "612c05caacd4", + "MemPerc": "34.56%", + "MemUsage": "88.47MiB / 256MiB", + "Name": "lyakhonov-lab3-api-9", + "NetIO": "263kB / 262kB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "11.81%", + "Container": "4577101bc498d9f217ad682ab8af2a4688f7772287a1755eab603ecc946ce2b7", + "ID": "4577101bc498", + "MemPerc": "9.77%", + "MemUsage": "12.51MiB / 128MiB", + "Name": "lyakhonov-lab3-lb-1", + "NetIO": "342kB / 341kB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 221kB", + "CPUPerc": "34.05%", + "Container": "f2cdcc232751e2c2400763e935897851aca1399824ed751ec96a51180282a207", + "ID": "f2cdcc232751", + "MemPerc": "11.32%", + "MemUsage": "57.94MiB / 512MiB", + "Name": "lyakhonov-lab3-db-1", + "NetIO": "393kB / 334kB", + "PIDs": "21" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.54%", + "Container": "889db8e55eb20c99fbe462f3d407763760cc40f0a053b144dcb9f6d13f6255c8", + "ID": "889db8e55eb2", + "MemPerc": "2.60%", + "MemUsage": "3.332MiB / 128MiB", + "Name": "lyakhonov-lab3-redis-1", + "NetIO": "4.29kB / 1.33kB", + "PIDs": "6" + } + ] + }, + { + "seconds": 6.493, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "11.84%", + "Container": "11cd774f4e060ce533d47473491be4336a62e0b3d1954ecfdd6cd785f034e6ae", + "ID": "11cd774f4e06", + "MemPerc": "2.89%", + "MemUsage": "14.8MiB / 512MiB", + "Name": "lyakhonov-lab3-load", + "NetIO": "304kB / 501kB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "23.43%", + "Container": "e37175f9a3fa89d28d266869eb804e06e36502a4702a6661668caf684555bcef", + "ID": "e37175f9a3fa", + "MemPerc": "27.09%", + "MemUsage": "69.36MiB / 256MiB", + "Name": "lyakhonov-lab3-api-10", + "NetIO": "669kB / 693kB", + "PIDs": "18" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "52.92%", + "Container": "612c05caacd47e750f542e055ebaa578c0f6a221d9091e6ce5df2fa945c52b65", + "ID": "612c05caacd4", + "MemPerc": "37.96%", + "MemUsage": "97.18MiB / 256MiB", + "Name": "lyakhonov-lab3-api-9", + "NetIO": "650kB / 689kB", + "PIDs": "29" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.42%", + "Container": "4577101bc498d9f217ad682ab8af2a4688f7772287a1755eab603ecc946ce2b7", + "ID": "4577101bc498", + "MemPerc": "9.81%", + "MemUsage": "12.55MiB / 128MiB", + "Name": "lyakhonov-lab3-lb-1", + "NetIO": "728kB / 726kB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 221kB", + "CPUPerc": "16.27%", + "Container": "f2cdcc232751e2c2400763e935897851aca1399824ed751ec96a51180282a207", + "ID": "f2cdcc232751", + "MemPerc": "12.88%", + "MemUsage": "65.94MiB / 512MiB", + "Name": "lyakhonov-lab3-db-1", + "NetIO": "1.03MB / 772kB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.62%", + "Container": "889db8e55eb20c99fbe462f3d407763760cc40f0a053b144dcb9f6d13f6255c8", + "ID": "889db8e55eb2", + "MemPerc": "2.61%", + "MemUsage": "3.336MiB / 128MiB", + "Name": "lyakhonov-lab3-redis-1", + "NetIO": "4.33kB / 1.33kB", + "PIDs": "6" + } + ] + }, + { + "seconds": 9.787, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.80%", + "Container": "11cd774f4e060ce533d47473491be4336a62e0b3d1954ecfdd6cd785f034e6ae", + "ID": "11cd774f4e06", + "MemPerc": "2.90%", + "MemUsage": "14.84MiB / 512MiB", + "Name": "lyakhonov-lab3-load", + "NetIO": "432kB / 730kB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "13.56%", + "Container": "e37175f9a3fa89d28d266869eb804e06e36502a4702a6661668caf684555bcef", + "ID": "e37175f9a3fa", + "MemPerc": "27.09%", + "MemUsage": "69.36MiB / 256MiB", + "Name": "lyakhonov-lab3-api-10", + "NetIO": "1.05MB / 1.08MB", + "PIDs": "18" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "50.84%", + "Container": "612c05caacd47e750f542e055ebaa578c0f6a221d9091e6ce5df2fa945c52b65", + "ID": "612c05caacd4", + "MemPerc": "34.84%", + "MemUsage": "89.2MiB / 256MiB", + "Name": "lyakhonov-lab3-api-9", + "NetIO": "1.03MB / 1.09MB", + "PIDs": "28" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.01%", + "Container": "4577101bc498d9f217ad682ab8af2a4688f7772287a1755eab603ecc946ce2b7", + "ID": "4577101bc498", + "MemPerc": "9.80%", + "MemUsage": "12.55MiB / 128MiB", + "Name": "lyakhonov-lab3-lb-1", + "NetIO": "1.41MB / 1.4MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 254kB", + "CPUPerc": "12.88%", + "Container": "f2cdcc232751e2c2400763e935897851aca1399824ed751ec96a51180282a207", + "ID": "f2cdcc232751", + "MemPerc": "12.93%", + "MemUsage": "66.19MiB / 512MiB", + "Name": "lyakhonov-lab3-db-1", + "NetIO": "1.62MB / 1.2MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.47%", + "Container": "889db8e55eb20c99fbe462f3d407763760cc40f0a053b144dcb9f6d13f6255c8", + "ID": "889db8e55eb2", + "MemPerc": "2.60%", + "MemUsage": "3.332MiB / 128MiB", + "Name": "lyakhonov-lab3-redis-1", + "NetIO": "4.33kB / 1.33kB", + "PIDs": "6" + } + ] + }, + { + "seconds": 18.837, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "27.75%", + "Container": "125c1e66e5bf9fb015328a33cd86802f86ef190c1d747ba6eec18a91ba511c56", + "ID": "125c1e66e5bf", + "MemPerc": "19.31%", + "MemUsage": "49.42MiB / 256MiB", + "Name": "lyakhonov-lab3-api-10", + "NetIO": "321kB / 313kB", + "PIDs": "12" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.54%", + "Container": "11cd774f4e060ce533d47473491be4336a62e0b3d1954ecfdd6cd785f034e6ae", + "ID": "11cd774f4e06", + "MemPerc": "2.92%", + "MemUsage": "14.96MiB / 512MiB", + "Name": "lyakhonov-lab3-load", + "NetIO": "858kB / 1.54MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "49.21%", + "Container": "612c05caacd47e750f542e055ebaa578c0f6a221d9091e6ce5df2fa945c52b65", + "ID": "612c05caacd4", + "MemPerc": "35.01%", + "MemUsage": "89.61MiB / 256MiB", + "Name": "lyakhonov-lab3-api-9", + "NetIO": "2.02MB / 2.13MB", + "PIDs": "28" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.47%", + "Container": "4577101bc498d9f217ad682ab8af2a4688f7772287a1755eab603ecc946ce2b7", + "ID": "4577101bc498", + "MemPerc": "10.16%", + "MemUsage": "13.01MiB / 128MiB", + "Name": "lyakhonov-lab3-lb-1", + "NetIO": "2.51MB / 2.43MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 414kB", + "CPUPerc": "9.97%", + "Container": "f2cdcc232751e2c2400763e935897851aca1399824ed751ec96a51180282a207", + "ID": "f2cdcc232751", + "MemPerc": "11.11%", + "MemUsage": "56.89MiB / 512MiB", + "Name": "lyakhonov-lab3-db-1", + "NetIO": "2.61MB / 1.91MB", + "PIDs": "18" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.66%", + "Container": "889db8e55eb20c99fbe462f3d407763760cc40f0a053b144dcb9f6d13f6255c8", + "ID": "889db8e55eb2", + "MemPerc": "2.60%", + "MemUsage": "3.332MiB / 128MiB", + "Name": "lyakhonov-lab3-redis-1", + "NetIO": "5.12kB / 1.4kB", + "PIDs": "6" + } + ] + }, + { + "seconds": 21.893, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "15.02%", + "Container": "125c1e66e5bf9fb015328a33cd86802f86ef190c1d747ba6eec18a91ba511c56", + "ID": "125c1e66e5bf", + "MemPerc": "19.51%", + "MemUsage": "49.95MiB / 256MiB", + "Name": "lyakhonov-lab3-api-10", + "NetIO": "633kB / 611kB", + "PIDs": "12" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.00%", + "Container": "11cd774f4e060ce533d47473491be4336a62e0b3d1954ecfdd6cd785f034e6ae", + "ID": "11cd774f4e06", + "MemPerc": "2.93%", + "MemUsage": "14.99MiB / 512MiB", + "Name": "lyakhonov-lab3-load", + "NetIO": "1.01MB / 1.81MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "52.10%", + "Container": "612c05caacd47e750f542e055ebaa578c0f6a221d9091e6ce5df2fa945c52b65", + "ID": "612c05caacd4", + "MemPerc": "34.90%", + "MemUsage": "89.36MiB / 256MiB", + "Name": "lyakhonov-lab3-api-9", + "NetIO": "2.33MB / 2.47MB", + "PIDs": "28" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.63%", + "Container": "4577101bc498d9f217ad682ab8af2a4688f7772287a1755eab603ecc946ce2b7", + "ID": "4577101bc498", + "MemPerc": "9.98%", + "MemUsage": "12.77MiB / 128MiB", + "Name": "lyakhonov-lab3-lb-1", + "NetIO": "2.94MB / 2.86MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 414kB", + "CPUPerc": "14.27%", + "Container": "f2cdcc232751e2c2400763e935897851aca1399824ed751ec96a51180282a207", + "ID": "f2cdcc232751", + "MemPerc": "11.24%", + "MemUsage": "57.53MiB / 512MiB", + "Name": "lyakhonov-lab3-db-1", + "NetIO": "3.23MB / 2.36MB", + "PIDs": "18" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.67%", + "Container": "889db8e55eb20c99fbe462f3d407763760cc40f0a053b144dcb9f6d13f6255c8", + "ID": "889db8e55eb2", + "MemPerc": "2.60%", + "MemUsage": "3.332MiB / 128MiB", + "Name": "lyakhonov-lab3-redis-1", + "NetIO": "5.12kB / 1.4kB", + "PIDs": "6" + } + ] + }, + { + "seconds": 29.984, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "18.13%", + "Container": "65d711abae2eb870c7763938ad28fa5c3c921b77c16b5168d7c2b1eef021ed1f", + "ID": "65d711abae2e", + "MemPerc": "19.42%", + "MemUsage": "49.71MiB / 256MiB", + "Name": "lyakhonov-lab3-api-11", + "NetIO": "184kB / 183kB", + "PIDs": "12" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "50.02%", + "Container": "125c1e66e5bf9fb015328a33cd86802f86ef190c1d747ba6eec18a91ba511c56", + "ID": "125c1e66e5bf", + "MemPerc": "20.33%", + "MemUsage": "52.04MiB / 256MiB", + "Name": "lyakhonov-lab3-api-10", + "NetIO": "1.4MB / 1.42MB", + "PIDs": "27" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.41%", + "Container": "11cd774f4e060ce533d47473491be4336a62e0b3d1954ecfdd6cd785f034e6ae", + "ID": "11cd774f4e06", + "MemPerc": "2.94%", + "MemUsage": "15.07MiB / 512MiB", + "Name": "lyakhonov-lab3-load", + "NetIO": "1.28MB / 2.37MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.61%", + "Container": "4577101bc498d9f217ad682ab8af2a4688f7772287a1755eab603ecc946ce2b7", + "ID": "4577101bc498", + "MemPerc": "10.08%", + "MemUsage": "12.9MiB / 128MiB", + "Name": "lyakhonov-lab3-lb-1", + "NetIO": "3.79MB / 3.62MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 446kB", + "CPUPerc": "17.47%", + "Container": "f2cdcc232751e2c2400763e935897851aca1399824ed751ec96a51180282a207", + "ID": "f2cdcc232751", + "MemPerc": "11.57%", + "MemUsage": "59.24MiB / 512MiB", + "Name": "lyakhonov-lab3-db-1", + "NetIO": "4.1MB / 2.96MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.74%", + "Container": "889db8e55eb20c99fbe462f3d407763760cc40f0a053b144dcb9f6d13f6255c8", + "ID": "889db8e55eb2", + "MemPerc": "2.60%", + "MemUsage": "3.328MiB / 128MiB", + "Name": "lyakhonov-lab3-redis-1", + "NetIO": "6.5kB / 1.47kB", + "PIDs": "6" + } + ] + }, + { + "seconds": 33.309, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "27.10%", + "Container": "65d711abae2eb870c7763938ad28fa5c3c921b77c16b5168d7c2b1eef021ed1f", + "ID": "65d711abae2e", + "MemPerc": "22.62%", + "MemUsage": "57.91MiB / 256MiB", + "Name": "lyakhonov-lab3-api-11", + "NetIO": "606kB / 592kB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "51.19%", + "Container": "125c1e66e5bf9fb015328a33cd86802f86ef190c1d747ba6eec18a91ba511c56", + "ID": "125c1e66e5bf", + "MemPerc": "20.46%", + "MemUsage": "52.39MiB / 256MiB", + "Name": "lyakhonov-lab3-api-10", + "NetIO": "1.83MB / 1.86MB", + "PIDs": "27" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.25%", + "Container": "11cd774f4e060ce533d47473491be4336a62e0b3d1954ecfdd6cd785f034e6ae", + "ID": "11cd774f4e06", + "MemPerc": "2.95%", + "MemUsage": "15.11MiB / 512MiB", + "Name": "lyakhonov-lab3-load", + "NetIO": "1.41MB / 2.6MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "9.71%", + "Container": "4577101bc498d9f217ad682ab8af2a4688f7772287a1755eab603ecc946ce2b7", + "ID": "4577101bc498", + "MemPerc": "10.41%", + "MemUsage": "13.33MiB / 128MiB", + "Name": "lyakhonov-lab3-lb-1", + "NetIO": "4.37MB / 4.2MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 446kB", + "CPUPerc": "16.92%", + "Container": "f2cdcc232751e2c2400763e935897851aca1399824ed751ec96a51180282a207", + "ID": "f2cdcc232751", + "MemPerc": "11.78%", + "MemUsage": "60.29MiB / 512MiB", + "Name": "lyakhonov-lab3-db-1", + "NetIO": "4.73MB / 3.43MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.57%", + "Container": "889db8e55eb20c99fbe462f3d407763760cc40f0a053b144dcb9f6d13f6255c8", + "ID": "889db8e55eb2", + "MemPerc": "2.60%", + "MemUsage": "3.328MiB / 128MiB", + "Name": "lyakhonov-lab3-redis-1", + "NetIO": "6.5kB / 1.47kB", + "PIDs": "6" + } + ] + }, + { + "seconds": 36.537, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "20.32%", + "Container": "65d711abae2eb870c7763938ad28fa5c3c921b77c16b5168d7c2b1eef021ed1f", + "ID": "65d711abae2e", + "MemPerc": "19.35%", + "MemUsage": "49.54MiB / 256MiB", + "Name": "lyakhonov-lab3-api-11", + "NetIO": "1.01MB / 981kB", + "PIDs": "12" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "49.59%", + "Container": "125c1e66e5bf9fb015328a33cd86802f86ef190c1d747ba6eec18a91ba511c56", + "ID": "125c1e66e5bf", + "MemPerc": "20.50%", + "MemUsage": "52.47MiB / 256MiB", + "Name": "lyakhonov-lab3-api-10", + "NetIO": "2.23MB / 2.28MB", + "PIDs": "27" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.23%", + "Container": "11cd774f4e060ce533d47473491be4336a62e0b3d1954ecfdd6cd785f034e6ae", + "ID": "11cd774f4e06", + "MemPerc": "2.97%", + "MemUsage": "15.2MiB / 512MiB", + "Name": "lyakhonov-lab3-load", + "NetIO": "1.68MB / 3.08MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.73%", + "Container": "4577101bc498d9f217ad682ab8af2a4688f7772287a1755eab603ecc946ce2b7", + "ID": "4577101bc498", + "MemPerc": "10.45%", + "MemUsage": "13.38MiB / 128MiB", + "Name": "lyakhonov-lab3-lb-1", + "NetIO": "4.93MB / 4.75MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 446kB", + "CPUPerc": "15.12%", + "Container": "f2cdcc232751e2c2400763e935897851aca1399824ed751ec96a51180282a207", + "ID": "f2cdcc232751", + "MemPerc": "11.89%", + "MemUsage": "60.86MiB / 512MiB", + "Name": "lyakhonov-lab3-db-1", + "NetIO": "5.33MB / 3.88MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.63%", + "Container": "889db8e55eb20c99fbe462f3d407763760cc40f0a053b144dcb9f6d13f6255c8", + "ID": "889db8e55eb2", + "MemPerc": "2.80%", + "MemUsage": "3.578MiB / 128MiB", + "Name": "lyakhonov-lab3-redis-1", + "NetIO": "6.5kB / 1.47kB", + "PIDs": "6" + } + ] + }, + { + "seconds": 39.905, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "14.59%", + "Container": "65d711abae2eb870c7763938ad28fa5c3c921b77c16b5168d7c2b1eef021ed1f", + "ID": "65d711abae2e", + "MemPerc": "19.36%", + "MemUsage": "49.55MiB / 256MiB", + "Name": "lyakhonov-lab3-api-11", + "NetIO": "1.36MB / 1.33MB", + "PIDs": "12" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "52.01%", + "Container": "125c1e66e5bf9fb015328a33cd86802f86ef190c1d747ba6eec18a91ba511c56", + "ID": "125c1e66e5bf", + "MemPerc": "24.33%", + "MemUsage": "62.27MiB / 256MiB", + "Name": "lyakhonov-lab3-api-10", + "NetIO": "2.58MB / 2.66MB", + "PIDs": "29" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "11.91%", + "Container": "11cd774f4e060ce533d47473491be4336a62e0b3d1954ecfdd6cd785f034e6ae", + "ID": "11cd774f4e06", + "MemPerc": "2.98%", + "MemUsage": "15.24MiB / 512MiB", + "Name": "lyakhonov-lab3-load", + "NetIO": "1.85MB / 3.39MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.16%", + "Container": "4577101bc498d9f217ad682ab8af2a4688f7772287a1755eab603ecc946ce2b7", + "ID": "4577101bc498", + "MemPerc": "10.08%", + "MemUsage": "12.9MiB / 128MiB", + "Name": "lyakhonov-lab3-lb-1", + "NetIO": "5.42MB / 5.24MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 446kB", + "CPUPerc": "16.46%", + "Container": "f2cdcc232751e2c2400763e935897851aca1399824ed751ec96a51180282a207", + "ID": "f2cdcc232751", + "MemPerc": "11.94%", + "MemUsage": "61.13MiB / 512MiB", + "Name": "lyakhonov-lab3-db-1", + "NetIO": "5.88MB / 4.27MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.71%", + "Container": "889db8e55eb20c99fbe462f3d407763760cc40f0a053b144dcb9f6d13f6255c8", + "ID": "889db8e55eb2", + "MemPerc": "2.60%", + "MemUsage": "3.328MiB / 128MiB", + "Name": "lyakhonov-lab3-redis-1", + "NetIO": "6.5kB / 1.47kB", + "PIDs": "6" + } + ] + }, + { + "seconds": 43.202, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "15.23%", + "Container": "65d711abae2eb870c7763938ad28fa5c3c921b77c16b5168d7c2b1eef021ed1f", + "ID": "65d711abae2e", + "MemPerc": "19.46%", + "MemUsage": "49.81MiB / 256MiB", + "Name": "lyakhonov-lab3-api-11", + "NetIO": "1.64MB / 1.6MB", + "PIDs": "12" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "49.25%", + "Container": "125c1e66e5bf9fb015328a33cd86802f86ef190c1d747ba6eec18a91ba511c56", + "ID": "125c1e66e5bf", + "MemPerc": "20.51%", + "MemUsage": "52.5MiB / 256MiB", + "Name": "lyakhonov-lab3-api-10", + "NetIO": "3.01MB / 3.1MB", + "PIDs": "27" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.11%", + "Container": "11cd774f4e060ce533d47473491be4336a62e0b3d1954ecfdd6cd785f034e6ae", + "ID": "11cd774f4e06", + "MemPerc": "2.98%", + "MemUsage": "15.25MiB / 512MiB", + "Name": "lyakhonov-lab3-load", + "NetIO": "2.06MB / 3.76MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.56%", + "Container": "4577101bc498d9f217ad682ab8af2a4688f7772287a1755eab603ecc946ce2b7", + "ID": "4577101bc498", + "MemPerc": "10.08%", + "MemUsage": "12.9MiB / 128MiB", + "Name": "lyakhonov-lab3-lb-1", + "NetIO": "6.01MB / 5.82MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 954kB", + "CPUPerc": "13.26%", + "Container": "f2cdcc232751e2c2400763e935897851aca1399824ed751ec96a51180282a207", + "ID": "f2cdcc232751", + "MemPerc": "12.05%", + "MemUsage": "61.69MiB / 512MiB", + "Name": "lyakhonov-lab3-db-1", + "NetIO": "6.5MB / 4.74MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.58%", + "Container": "889db8e55eb20c99fbe462f3d407763760cc40f0a053b144dcb9f6d13f6255c8", + "ID": "889db8e55eb2", + "MemPerc": "2.60%", + "MemUsage": "3.328MiB / 128MiB", + "Name": "lyakhonov-lab3-redis-1", + "NetIO": "6.5kB / 1.47kB", + "PIDs": "6" + } + ] + }, + { + "seconds": 46.49, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "15.95%", + "Container": "65d711abae2eb870c7763938ad28fa5c3c921b77c16b5168d7c2b1eef021ed1f", + "ID": "65d711abae2e", + "MemPerc": "19.39%", + "MemUsage": "49.64MiB / 256MiB", + "Name": "lyakhonov-lab3-api-11", + "NetIO": "2.16MB / 2.1MB", + "PIDs": "13" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "50.53%", + "Container": "125c1e66e5bf9fb015328a33cd86802f86ef190c1d747ba6eec18a91ba511c56", + "ID": "125c1e66e5bf", + "MemPerc": "24.12%", + "MemUsage": "61.74MiB / 256MiB", + "Name": "lyakhonov-lab3-api-10", + "NetIO": "3.25MB / 3.35MB", + "PIDs": "29" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "13.41%", + "Container": "11cd774f4e060ce533d47473491be4336a62e0b3d1954ecfdd6cd785f034e6ae", + "ID": "11cd774f4e06", + "MemPerc": "2.99%", + "MemUsage": "15.33MiB / 512MiB", + "Name": "lyakhonov-lab3-load", + "NetIO": "2.24MB / 4.09MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.57%", + "Container": "4577101bc498d9f217ad682ab8af2a4688f7772287a1755eab603ecc946ce2b7", + "ID": "4577101bc498", + "MemPerc": "10.08%", + "MemUsage": "12.91MiB / 128MiB", + "Name": "lyakhonov-lab3-lb-1", + "NetIO": "6.53MB / 6.34MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 954kB", + "CPUPerc": "14.06%", + "Container": "f2cdcc232751e2c2400763e935897851aca1399824ed751ec96a51180282a207", + "ID": "f2cdcc232751", + "MemPerc": "12.27%", + "MemUsage": "62.84MiB / 512MiB", + "Name": "lyakhonov-lab3-db-1", + "NetIO": "7.07MB / 5.15MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.70%", + "Container": "889db8e55eb20c99fbe462f3d407763760cc40f0a053b144dcb9f6d13f6255c8", + "ID": "889db8e55eb2", + "MemPerc": "2.60%", + "MemUsage": "3.328MiB / 128MiB", + "Name": "lyakhonov-lab3-redis-1", + "NetIO": "6.5kB / 1.47kB", + "PIDs": "6" + } + ] + }, + { + "seconds": 49.745, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "30.17%", + "Container": "65d711abae2eb870c7763938ad28fa5c3c921b77c16b5168d7c2b1eef021ed1f", + "ID": "65d711abae2e", + "MemPerc": "19.41%", + "MemUsage": "49.68MiB / 256MiB", + "Name": "lyakhonov-lab3-api-11", + "NetIO": "2.43MB / 2.36MB", + "PIDs": "13" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "49.40%", + "Container": "125c1e66e5bf9fb015328a33cd86802f86ef190c1d747ba6eec18a91ba511c56", + "ID": "125c1e66e5bf", + "MemPerc": "20.42%", + "MemUsage": "52.27MiB / 256MiB", + "Name": "lyakhonov-lab3-api-10", + "NetIO": "3.64MB / 3.78MB", + "PIDs": "27" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.48%", + "Container": "11cd774f4e060ce533d47473491be4336a62e0b3d1954ecfdd6cd785f034e6ae", + "ID": "11cd774f4e06", + "MemPerc": "2.99%", + "MemUsage": "15.3MiB / 512MiB", + "Name": "lyakhonov-lab3-load", + "NetIO": "2.37MB / 4.32MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "7.23%", + "Container": "4577101bc498d9f217ad682ab8af2a4688f7772287a1755eab603ecc946ce2b7", + "ID": "4577101bc498", + "MemPerc": "10.26%", + "MemUsage": "13.14MiB / 128MiB", + "Name": "lyakhonov-lab3-lb-1", + "NetIO": "6.9MB / 6.71MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 987kB", + "CPUPerc": "17.18%", + "Container": "f2cdcc232751e2c2400763e935897851aca1399824ed751ec96a51180282a207", + "ID": "f2cdcc232751", + "MemPerc": "12.23%", + "MemUsage": "62.6MiB / 512MiB", + "Name": "lyakhonov-lab3-db-1", + "NetIO": "7.48MB / 5.45MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.87%", + "Container": "889db8e55eb20c99fbe462f3d407763760cc40f0a053b144dcb9f6d13f6255c8", + "ID": "889db8e55eb2", + "MemPerc": "2.60%", + "MemUsage": "3.328MiB / 128MiB", + "Name": "lyakhonov-lab3-redis-1", + "NetIO": "6.5kB / 1.47kB", + "PIDs": "6" + } + ] + }, + { + "seconds": 53.01, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "11.22%", + "Container": "65d711abae2eb870c7763938ad28fa5c3c921b77c16b5168d7c2b1eef021ed1f", + "ID": "65d711abae2e", + "MemPerc": "19.48%", + "MemUsage": "49.87MiB / 256MiB", + "Name": "lyakhonov-lab3-api-11", + "NetIO": "2.79MB / 2.72MB", + "PIDs": "13" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "46.97%", + "Container": "125c1e66e5bf9fb015328a33cd86802f86ef190c1d747ba6eec18a91ba511c56", + "ID": "125c1e66e5bf", + "MemPerc": "20.77%", + "MemUsage": "53.16MiB / 256MiB", + "Name": "lyakhonov-lab3-api-10", + "NetIO": "4.08MB / 4.26MB", + "PIDs": "26" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.48%", + "Container": "4577101bc498d9f217ad682ab8af2a4688f7772287a1755eab603ecc946ce2b7", + "ID": "4577101bc498", + "MemPerc": "10.08%", + "MemUsage": "12.91MiB / 128MiB", + "Name": "lyakhonov-lab3-lb-1", + "NetIO": "7.49MB / 7.28MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 987kB", + "CPUPerc": "12.71%", + "Container": "f2cdcc232751e2c2400763e935897851aca1399824ed751ec96a51180282a207", + "ID": "f2cdcc232751", + "MemPerc": "12.21%", + "MemUsage": "62.5MiB / 512MiB", + "Name": "lyakhonov-lab3-db-1", + "NetIO": "8.03MB / 5.94MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.59%", + "Container": "889db8e55eb20c99fbe462f3d407763760cc40f0a053b144dcb9f6d13f6255c8", + "ID": "889db8e55eb2", + "MemPerc": "2.60%", + "MemUsage": "3.328MiB / 128MiB", + "Name": "lyakhonov-lab3-redis-1", + "NetIO": "6.5kB / 1.47kB", + "PIDs": "6" + } + ] + } + ], + "database_waits": [ + "Client:ClientRead:16\nCPU/active:running:1", + "Client:ClientRead:16\nCPU/active:running:1", + "Client:ClientRead:16\nCPU/active:running:1", + "Client:ClientRead:12\nCPU/active:running:1", + "Client:ClientRead:12\nCPU/active:running:1", + "Client:ClientRead:13\nCPU/active:running:1", + "Client:ClientRead:13\nCPU/active:running:1", + "Client:ClientRead:13\nCPU/active:running:1", + "Client:ClientRead:13\nCPU/active:running:1", + "Client:ClientRead:13\nCPU/active:running:1", + "Client:ClientRead:13\nCPU/active:running:1", + "Client:ClientRead:13\nCPU/active:running:1", + "Client:ClientRead:13\nCPU/active:running:1" + ], + "rolling_actions": [ + { + "old_id": "e37175f9a3fa89d28d266869eb804e06e36502a4702a6661668caf684555bcef", + "start_seconds": 10.324476799927652, + "finished_seconds": 16.1121479999274 + }, + { + "old_id": "612c05caacd47e750f542e055ebaa578c0f6a221d9091e6ce5df2fa945c52b65", + "start_seconds": 22.527321300003678, + "finished_seconds": 27.791109999874607 + } + ], + "before_state": [ + { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 97752, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-23T16:08:02.552700017Z", + "FinishedAt": "2026-10-23T16:08:02.083465687Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-23T16:08:01.779790532Z", + "End": "2026-10-23T16:08:02.081226356Z", + "ExitCode": 137, + "Output": "" + }, + { + "Start": "2026-10-23T16:08:08.007328589Z", + "End": "2026-10-23T16:08:08.494558026Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-23T16:08:10.496293318Z", + "End": "2026-10-23T16:08:10.624785692Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-23T16:08:12.626087517Z", + "End": "2026-10-23T16:08:12.801770889Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-23T16:08:14.802341943Z", + "End": "2026-10-23T16:08:15.022685561Z", + "ExitCode": 0, + "Output": "" + } + ] + }, + "id": "e37175f9a3fa89d28d266869eb804e06e36502a4702a6661668caf684555bcef" + }, + { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 97691, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-23T16:08:02.47911506Z", + "FinishedAt": "2026-10-23T16:08:01.911279848Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-23T16:08:00.633368914Z", + "End": "2026-10-23T16:08:01.032345004Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-23T16:08:07.856926767Z", + "End": "2026-10-23T16:08:08.338006416Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-23T16:08:10.339499908Z", + "End": "2026-10-23T16:08:10.520513261Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-23T16:08:12.521641067Z", + "End": "2026-10-23T16:08:12.651437439Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-23T16:08:14.653141088Z", + "End": "2026-10-23T16:08:14.854972266Z", + "ExitCode": 0, + "Output": "" + } + ] + }, + "id": "612c05caacd47e750f542e055ebaa578c0f6a221d9091e6ce5df2fa945c52b65" + }, + { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 97632, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-23T16:08:02.306518292Z", + "FinishedAt": "2026-10-23T16:08:01.486171752Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-23T16:08:06.852065153Z", + "End": "2026-10-23T16:08:06.936786202Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-23T16:08:08.937764148Z", + "End": "2026-10-23T16:08:09.03751949Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-23T16:08:11.038985963Z", + "End": "2026-10-23T16:08:11.122797018Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-23T16:08:13.123544594Z", + "End": "2026-10-23T16:08:13.196773925Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-23T16:08:15.198458612Z", + "End": "2026-10-23T16:08:15.264733796Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + } + ] + }, + "id": "f2cdcc232751e2c2400763e935897851aca1399824ed751ec96a51180282a207" + }, + { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 97555, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-23T16:08:02.041383013Z", + "FinishedAt": "2026-10-23T16:08:01.41859737Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-23T16:07:48.334361426Z", + "End": "2026-10-23T16:07:48.504037239Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-23T16:07:53.504629618Z", + "End": "2026-10-23T16:07:53.839034836Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-23T16:07:58.8392155Z", + "End": "2026-10-23T16:07:58.999561622Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-23T16:08:07.695874365Z", + "End": "2026-10-23T16:08:07.838345265Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-23T16:08:12.839983079Z", + "End": "2026-10-23T16:08:12.932356107Z", + "ExitCode": 0, + "Output": "" + } + ] + }, + "id": "4577101bc498d9f217ad682ab8af2a4688f7772287a1755eab603ecc946ce2b7" + }, + { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 97610, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-23T16:08:02.163804371Z", + "FinishedAt": "2026-10-23T16:08:01.435097119Z", + "id": "889db8e55eb20c99fbe462f3d407763760cc40f0a053b144dcb9f6d13f6255c8" + } + ], + "after_state": [ + { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 98894, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-23T16:08:28.437249772Z", + "FinishedAt": "0001-01-01T00:00:00Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-23T16:09:00.448489952Z", + "End": "2026-10-23T16:09:01.321415182Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-23T16:09:03.322192838Z", + "End": "2026-10-23T16:09:04.427527886Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-23T16:09:06.428300109Z", + "End": "2026-10-23T16:09:07.340122452Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-23T16:09:09.341095338Z", + "End": "2026-10-23T16:09:09.636684359Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-23T16:09:11.637839547Z", + "End": "2026-10-23T16:09:11.746679825Z", + "ExitCode": 0, + "Output": "" + } + ] + }, + "id": "125c1e66e5bf9fb015328a33cd86802f86ef190c1d747ba6eec18a91ba511c56" + }, + { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 99478, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-23T16:08:41.169842958Z", + "FinishedAt": "0001-01-01T00:00:00Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-23T16:09:02.765762166Z", + "End": "2026-10-23T16:09:03.044506638Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-23T16:09:05.046077244Z", + "End": "2026-10-23T16:09:05.444888867Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-23T16:09:07.445473336Z", + "End": "2026-10-23T16:09:07.652888398Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-23T16:09:09.65460039Z", + "End": "2026-10-23T16:09:10.031028475Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-23T16:09:12.031958749Z", + "End": "2026-10-23T16:09:12.162187836Z", + "ExitCode": 0, + "Output": "" + } + ] + }, + "id": "65d711abae2eb870c7763938ad28fa5c3c921b77c16b5168d7c2b1eef021ed1f" + }, + { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 97632, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-23T16:08:02.306518292Z", + "FinishedAt": "2026-10-23T16:08:01.486171752Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-23T16:09:03.223844225Z", + "End": "2026-10-23T16:09:03.323892667Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-23T16:09:05.325762438Z", + "End": "2026-10-23T16:09:05.460705863Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-23T16:09:07.462107334Z", + "End": "2026-10-23T16:09:07.568988924Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-23T16:09:09.569814927Z", + "End": "2026-10-23T16:09:09.711816743Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-23T16:09:11.713715433Z", + "End": "2026-10-23T16:09:11.785346552Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + } + ] + }, + "id": "f2cdcc232751e2c2400763e935897851aca1399824ed751ec96a51180282a207" + }, + { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 97555, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-23T16:08:02.041383013Z", + "FinishedAt": "2026-10-23T16:08:01.41859737Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-23T16:08:49.023074194Z", + "End": "2026-10-23T16:08:49.161933668Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-23T16:08:54.163260088Z", + "End": "2026-10-23T16:08:54.356935691Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-23T16:08:59.357215102Z", + "End": "2026-10-23T16:08:59.458788885Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-23T16:09:04.460254287Z", + "End": "2026-10-23T16:09:04.670157956Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-23T16:09:09.672312857Z", + "End": "2026-10-23T16:09:09.919302972Z", + "ExitCode": 0, + "Output": "" + } + ] + }, + "id": "4577101bc498d9f217ad682ab8af2a4688f7772287a1755eab603ecc946ce2b7" + }, + { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 97610, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-23T16:08:02.163804371Z", + "FinishedAt": "2026-10-23T16:08:01.435097119Z", + "id": "889db8e55eb20c99fbe462f3d407763760cc40f0a053b144dcb9f6d13f6255c8" + } + ], + "api_cgroup_cpu_delta": {}, + "optimized_writes_requested": 1, + "application_image_id": "sha256:639e8935bac8e884626a86dc4bd71c623e1b5495ee6376a7e9e90583329c7292" +} diff --git a/evidence/transactions.txt b/evidence/transactions.txt new file mode 100644 index 00000000..860ab218 --- /dev/null +++ b/evidence/transactions.txt @@ -0,0 +1,2 @@ +PASS: искусственно состаренный ключ заменён; исходное сообщение сохранено +PASS: искусственная ошибка 500 не закэширована; транзакция откатилась, повтор создал одно сообщение diff --git a/evidence/transfer.md b/evidence/transfer.md index 78907dc3..3cfa4f99 100644 --- a/evidence/transfer.md +++ b/evidence/transfer.md @@ -7,4 +7,5 @@ В исходном репозитории обнаружены пользовательские изменения scripts/research.py и пяти файлов student-tests; эти файлы не переносились и не изменялись. Секреты, Git, тома, результаты старых экспериментов и сгенерированные данные не переносились. Это завершённый этап переноса, а не реализация новых требований lab3. Проверка: make check PYTHON=python. -`nНа Windows требуется PYTHONUTF8=1: без него публичный скрипт читает UTF-8 как cp1251. + +На Windows требуется PYTHONUTF8=1: без него публичный скрипт читает UTF-8 как cp1251. diff --git a/infra/operations.md b/infra/operations.md new file mode 100644 index 00000000..0b935b03 --- /dev/null +++ b/infra/operations.md @@ -0,0 +1,17 @@ +# Эксплуатационные границы lab3 + +Два и более API обслуживают одну PostgreSQL и Redis через HAProxy. Sticky session нет. Внешний порт только у LB. Compose DNS возвращает адреса service api; HAProxy server-template имеет четыре слота, опрашивает DNS с hold valid/obsolete 1 с. Масштабирование этого стенда ограничено четырьмя репликами. + +Readiness API проверяет PostgreSQL; сессии и идемпотентность требуют её доступности. Redis — необязательный кэш профилей; при его отказе читается PostgreSQL. Кэш не хранит сессии или историю. Межрепличный read-after-write обеспечивается чтением основной БД после COMMIT, а не инвалидацией локальной истории. + +Socket timeout Redis не ограничивает системное DNS-разрешение. Поэтому обращения к кэшу выполняет отдельный executor с одним worker и максимум одним незавершённым вызовом. Ожидание результата ограничено 250 мс; занятый worker приводит к немедленному обходу кэша. Очередь DNS-задач не растёт. Неудавшийся необязательный put также не блокирует PostgreSQL. Это устранено после воспроизведённого 504 при остановке Redis; повторная проверка измеряет fallback. + +HAProxy проверяет /health/ready раз в 1 с с timeout check 1 с; fall=2, rise=1. Оценка обнаружения молча недоступного backend — до 4 с при стабильной работе DNS и планировщика; сетевые отказы наблюдаются раньше. Это конфигурационный бюджет, не гарантия времени на перегруженной машине. connect timeout 1 с, server 5 с, client 10 с, retries=0. Повторы POST решает клиент с исходным ключом. Ресурс очереди: 100 обслуживаемых запросов на API; пул 10, max_waiting=32, ожидание пула 3 с; statement timeout 3 с, lock timeout 2 с. Перегрузка и временная недоступность БД дают 503. + +Uvicorn при SIGTERM закрывает listener, дожидается активных запросов до 8 с, затем закрывает пул. Compose stop_grace_period=12 с. LB может увидеть отказ соединения до следующей проверки готовности; такие ошибки учитываются в экспериментах. Никаких автоматических повторов неидемпотентных POST у LB нет. + +Failover hook намеренно использует docker stop --time 1 как сценарий быстрого отказа, оставляет выбранную реплику остановленной и не удаляет её. Rolling-сценарий использует stop --time 10, затем удаление старого контейнера и создание нового; это отдельная проверка согласованного завершения. Нулевое число переходных ошибок не заявляется. + +Сервис migrate запускается до API, содержит advisory lock и версионирование. В Dockerfile API миграции отсутствуют. Повтор migrate безопасен. Миграция 002 добавляет отдельную таблицу и индекс, не изменяет старые столбцы; совместима с чтением работающей версией. При переходе с lab2 нельзя маршрутизировать записи к старому приложению, не понимающему обязательный ключ. Проверенная замена lab3→lab3 эквивалентна rolling deployment; несовместимые изменения API/DDL требуют отдельной стратегии expand/contract. + +Остановка и замена контейнера не доказывают переживание потери узла. Единственный LB, PostgreSQL, Docker-host и его диск остаются SPOF. Единственный Redis не критичен для корректности. Данные находятся в собственном именованном томе lyakhonov-lab3_postgres_data. diff --git a/scripts/experiments.py b/scripts/experiments.py new file mode 100644 index 00000000..0edc8780 --- /dev/null +++ b/scripts/experiments.py @@ -0,0 +1,122 @@ +"""Воспроизводимые серии lab3; контейнеры и тома предыдущей лабы не затрагиваются.""" +import argparse +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import random +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +OUT = ROOT / 'evidence' / 'research' +OUT.mkdir(parents=True, exist_ok=True) + + +def command(*args, env=None): + r = subprocess.run(args, cwd=ROOT, env=env, text=True, capture_output=True) + if r.returncode: + raise RuntimeError('Ошибка команды: ' + ' '.join(args[:4]) + '\n' + r.stderr[-2000:]) + return r.stdout.strip() + + +def configure(n, cpu, memory, pool, optimized=1): + env = dict(os.environ, API_REPLICAS=str(n), API_CPUS=str(cpu), API_MEMORY=f'{memory}m', POOL_SIZE=str(pool), OPTIMIZED_WRITES=str(optimized), PYTHONUTF8='1') + command('docker', 'compose', 'up', '-d', '--no-deps', '--scale', f'api={n}', 'api', env=env) + command(sys.executable, 'scripts/wait_ready.py', '--replicas', str(n), env=env) + return env + + +def state(): + ids = command('docker', 'compose', 'ps', '-q').splitlines() + return [json.loads(command('docker', 'inspect', '--format', '{{json .State}}', i)) | {'id': i} for i in ids] + + +def sql(query): + return command('docker', 'compose', 'exec', '-T', 'db', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', query) + + +def cpu_counters(): + result={} + for cid in command('docker','compose','ps','-q','api').splitlines(): + raw=command('docker','exec',cid,'cat','/sys/fs/cgroup/cpu.stat') + result[cid]={line.split()[0]:int(line.split()[1]) for line in raw.splitlines()} + return result + + +def benchmark(label, n, cpu, memory, pool, concurrency=32, duration=20, rolling=False, optimized=1): + configure(n, cpu, memory, pool, optimized) + started = datetime.now(timezone.utc).isoformat() + before = sql('SELECT count(*) FROM messages') + cpu_before=cpu_counters() + before_state = state() if rolling else None + args = ['docker', 'run', '--rm', '--name', 'lyakhonov-lab3-load', '--network', 'lyakhonov-lab3_default', '--cpus', '2', '--memory', '512m', + '-v', f'{ROOT / "student-tests"}:/work/student-tests:ro', '-v', f'{ROOT / "tests"}:/work/tests:ro', + '-v', f'{ROOT / "lab.json"}:/work/lab.json:ro', '-v', f'{ROOT / "contracts"}:/work/contracts:ro', + '-w', '/work', 'lyakhonov-messenger:lab3', 'python', 'student-tests/benchmark.py', '--concurrency', str(concurrency), '--duration', str(duration)] + proc = subprocess.Popen(args, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + samples, waits, actions = [], [], [] + timer = time.monotonic() + rolling_ids = command('docker', 'compose', 'ps', '-q', 'api').splitlines() if rolling else [] + while proc.poll() is None: + raw = command('docker', 'stats', '--no-stream', '--format', '{{json .}}') + samples.append({'seconds': round(time.monotonic()-timer, 3), 'containers': [json.loads(line) for line in raw.splitlines() if 'lyakhonov-lab3' in line]}) + waits.append(sql("SELECT coalesce(wait_event_type,'CPU/active')||':'||coalesce(wait_event,'running')||':'||count(*) FROM pg_stat_activity WHERE datname='messenger' AND backend_type='client backend' GROUP BY 1=0,wait_event_type,wait_event")) + # Поочерёдная замена каждого контейнера: старый останавливается, новый создаётся с тем же номером сервиса. + if rolling and rolling_ids and time.monotonic()-timer > 10 + len(actions)*10: + old = rolling_ids.pop(0) + step = time.monotonic() + command('docker', 'stop', '--time', '10', old) + command('docker', 'rm', old) + configure(n, cpu, memory, pool) + actions.append({'old_id': old, 'start_seconds': step-timer, 'finished_seconds': time.monotonic()-timer}) + stdout, stderr = proc.communicate() + if proc.returncode: + raise RuntimeError(stderr) + result = json.loads(stdout) + cpu_after=cpu_counters() + result.update({'label': label, 'started_utc': started, 'replicas': n, 'cpu_per_replica': cpu, 'memory_mib_per_replica': memory, 'pool_per_replica': pool, + 'database_messages_before': int(before), 'database_messages_after': int(sql('SELECT count(*) FROM messages')), + 'samples': samples, 'database_waits': waits, 'rolling_actions': actions, 'before_state': before_state, 'after_state': state() if rolling else None}) + result['api_cgroup_cpu_delta']={cid:{k:v-cpu_before[cid].get(k,0) for k,v in counters.items()} for cid,counters in cpu_after.items() if cid in cpu_before} + result['optimized_writes_requested']=optimized + result['application_image_id']=command('docker','image','inspect','lyakhonov-messenger:lab3','--format','{{.Id}}') + (OUT / (label+'.json')).write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n', encoding='utf-8') + print(label, 'RPS=', round(result['successful_rps'], 2), 'errors=', result['failed_requests'], flush=True) + return result + + +def main(): + p = argparse.ArgumentParser() + p.add_argument('--phase', choices=['pilot', 'matrix', 'saturation', 'pool', 'rolling', 'optimize'], required=True) + p.add_argument('--only', action='append', default=[], help='Повторить только названные точки выбранной серии') + args = p.parse_args() + if args.phase == 'pilot': + benchmark('pilot', 2, 1, 512, 10, duration=10) + elif args.phase == 'matrix': + jobs = [(mode, n, rep) for mode in ['fixed', 'growth'] for n in [1, 2, 4] for rep in range(1, 4)] + random.Random(2301).shuffle(jobs) + for mode, n, rep in jobs: + label=f'{mode}-{n}-{rep}' + if not args.only or label in args.only: + benchmark(label, n, 2/n if mode=='fixed' else .5, 1024//n if mode=='fixed' else 256, 10) + elif args.phase == 'saturation': + for concurrency in [4, 16, 64, 128]: + for rep in range(1, 3): + benchmark(f'saturation-{concurrency}-{rep}', 4, .5, 256, 10, concurrency) + elif args.phase == 'pool': + for rep in range(1, 4): + for pool in ([1,10] if rep%2 else [10,1]): + benchmark(f'pool-{pool}-{rep}', 2, 1, 512, pool, 64) + elif args.phase == 'rolling': + benchmark('rolling', 2, .5, 256, 10, 16, 45, True) + elif args.phase == 'optimize': + for rep in range(1,4): + for mode in ([0,1] if rep%2 else [1,0]): + benchmark(f'optimize-{mode}-{rep}',2,1,512,10,32,optimized=mode) + configure(2, .5, 256, 10) + + +if __name__ == '__main__': + main() diff --git a/scripts/verify_failover.py b/scripts/verify_failover.py new file mode 100644 index 00000000..c207dbb6 --- /dev/null +++ b/scripts/verify_failover.py @@ -0,0 +1,51 @@ +"""Проверить публичным сценарием остановку каждой из двух реальных API-реплик.""" +import json +from datetime import datetime, timezone +import os +from pathlib import Path +import subprocess +import sys +import urllib.request + +ROOT=Path(__file__).resolve().parents[1] +env=dict(os.environ,PYTHONUTF8='1') +seen=set() +stamp=datetime.now(timezone.utc).strftime('%H%M%S') + + +def run(*args): + return subprocess.check_output(args,cwd=ROOT,env=env,text=True,encoding='utf-8').strip() + + +for attempt in range(1,9): + run('docker','compose','start','api') + run(sys.executable,'scripts/wait_ready.py','--replicas','2') + # Обычные запросы меняют позицию round-robin; повтор нужен, если снова выбрана уже проверенная реплика. + for _ in range(attempt%3): + with urllib.request.urlopen('http://localhost:8080/health/ready',timeout=3) as r: + r.read() + hook='scripts/stop-one.cmd' if os.name=='nt' else 'scripts/stop-one.sh' + output=run(sys.executable,'tests/resilience.py','--mode','failover','--action-script',hook) + snapshot=run('docker','compose','ps','-a','--format','json') + rows=[json.loads(line) for line in snapshot.splitlines()] + stopped=[r for r in rows if r['Service']=='api' and r['State']=='exited'] + if len(stopped)!=1: + raise SystemExit('Нужна ровно одна остановленная API') + name=stopped[0]['Name'] + audit=ROOT/'evidence/research-audit' + audit.mkdir(exist_ok=True) + (audit/f'failover-{stamp}-attempt-{attempt}.txt').write_text(output+'\n',encoding='utf-8') + (audit/f'failover-{stamp}-attempt-{attempt}-containers.jsonl').write_text(snapshot+'\n',encoding='utf-8') + if name not in seen: + seen.add(name) + index=len(seen) + (ROOT/f'evidence/failover-{index}.txt').write_text(output+'\n',encoding='utf-8') + (ROOT/f'evidence/failover-{index}-containers.jsonl').write_text(snapshot+'\n',encoding='utf-8') + print('PASS:',name,'отказ проверен',flush=True) + if len(seen)==2: + break +else: + raise SystemExit('Не удалось наблюдать отказ двух разных реплик') +run('docker','compose','start','api') +run(sys.executable,'scripts/wait_ready.py','--replicas','2') +print('PASS: обе реальные реплики проверены по очереди и восстановлены') diff --git a/scripts/wait_ready.py b/scripts/wait_ready.py index 6a395346..b483a49b 100644 --- a/scripts/wait_ready.py +++ b/scripts/wait_ready.py @@ -15,6 +15,7 @@ for _ in range(120): if len(observed)>=args.replicas: print('PASS: API ready, replicas:', len(observed)) break + time.sleep(0.1) except Exception: time.sleep(0.5) else: diff --git a/src/app.py b/src/app.py index f2810ac0..1b02146e 100644 --- a/src/app.py +++ b/src/app.py @@ -3,7 +3,7 @@ import os import socket from contextlib import asynccontextmanager import psycopg -from psycopg_pool import PoolTimeout +from psycopg_pool import PoolTimeout, TooManyRequests from starlette.concurrency import run_in_threadpool from fastapi import FastAPI, Request @@ -32,7 +32,9 @@ def create_app() -> FastAPI: @app.exception_handler(psycopg.OperationalError) @app.exception_handler(psycopg.errors.QueryCanceled) + @app.exception_handler(psycopg.errors.LockNotAvailable) @app.exception_handler(PoolTimeout) + @app.exception_handler(TooManyRequests) async def dependency_error(request, exc): return JSONResponse(status_code=503, content=ErrorResponse(error={ "code": "unavailable", "message": "хранилище временно недоступно", diff --git a/src/cache.py b/src/cache.py index 1bbf01ca..edeb5357 100644 --- a/src/cache.py +++ b/src/cache.py @@ -1,6 +1,8 @@ """Необязательный кэш неизменяемого публичного профиля; ошибки обходятся через БД.""" import logging import os +from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutureTimeout +import threading from redis import Redis, RedisError from pydantic import ValidationError from src.config import setting @@ -14,12 +16,27 @@ class ProfileCache: socket_timeout=0.2, max_connections=16) if url else None self.ttl = int(os.getenv('PROFILE_CACHE_TTL_SECONDS','60')) self.hits = self.misses = self.errors = 0 + # DNS может ждать дольше socket timeout; один фоновый вызов ограничивает и время, и очередь. + self._executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix='profile-cache') + self._pending = None + self._guard = threading.Lock() + + def _bounded(self, action, *args, **kwargs): + with self._guard: + if self._pending is not None and not self._pending.done(): + raise RedisError('кэш занят') + self._pending = self._executor.submit(action, *args, **kwargs) + future = self._pending + try: + return future.result(timeout=0.25) + except FutureTimeout: + raise RedisError('истекло время ожидания кэша') from None def get(self, user_id): if not self.client: return None try: - value = self.client.get('profile:v1:'+user_id) + value = self._bounded(self.client.get, 'profile:v1:'+user_id) if value: user=User.model_validate_json(value) if user.id != user_id: @@ -35,7 +52,7 @@ class ProfileCache: def put(self, user): if self.client: try: - self.client.set('profile:v1:'+user.id,user.model_dump_json(),ex=self.ttl) + self._bounded(self.client.set, 'profile:v1:'+user.id,user.model_dump_json(),ex=self.ttl) except RedisError: self.errors += 1 diff --git a/src/routes/messages.py b/src/routes/messages.py index 267b03bd..1f8ea128 100644 --- a/src/routes/messages.py +++ b/src/routes/messages.py @@ -1,5 +1,6 @@ from uuid import UUID import re +import os from fastapi import APIRouter, Depends, HTTPException, Request @@ -21,8 +22,12 @@ def create_message( request_id = get_request_id(request) chat_id_str = str(chat_id) - chat = storage.get_chat(chat_id_str) - if not chat or x_user_id not in chat.member_ids: + if os.getenv('OPTIMIZED_WRITES','1') == '1': + allowed = storage.is_chat_member(chat_id_str, x_user_id) + else: + chat = storage.get_chat(chat_id_str) + allowed = bool(chat and x_user_id in chat.member_ids) + if not allowed: raise HTTPException( status_code=404, detail=make_error("not_found", "чат не найден", request_id).model_dump(), diff --git a/src/storage.py b/src/storage.py index 8d8a2572..358184ec 100644 --- a/src/storage.py +++ b/src/storage.py @@ -178,6 +178,11 @@ class PostgresStorage: members = conn.execute('SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id', (chat_id,)).fetchall() return Chat(**public_row(row), member_ids=[str(x['user_id']) for x in members]) + def is_chat_member(self, chat_id, user_id): + """Проверить доступ без загрузки всего чата и списка участников.""" + with self.pool.connection() as conn: + return conn.execute('SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s', (chat_id,user_id)).fetchone() is not None + def list_chats_for_user(self, user_id, cursor, limit): with self.pool.connection() as conn: params = [user_id] @@ -205,9 +210,13 @@ class PostgresStorage: raise ConflictError('ключ уже использован с другим сообщением') row = conn.execute('SELECT * FROM messages WHERE id=%s', (old['message_id'],)).fetchone() return Message(**public_row(row)) - conn.execute('DELETE FROM message_requests WHERE user_id=%s AND chat_id=%s AND key=%s', (sender_id,chat_id,key)) + optimized = os.getenv('OPTIMIZED_WRITES','1') == '1' + if not optimized: + conn.execute('DELETE FROM message_requests WHERE user_id=%s AND chat_id=%s AND key=%s', (sender_id,chat_id,key)) row = conn.execute('INSERT INTO messages (id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) RETURNING *', (uuid.uuid4(),chat_id,sender_id,text)).fetchone() - conn.execute("INSERT INTO message_requests VALUES (%s,%s,%s,%s,%s,clock_timestamp()+interval '24 hours')", (sender_id,chat_id,key,text,row['id'])) + # Под той же блокировкой UPSERT заменяет просроченный результат без отдельного DELETE. + suffix = ' ON CONFLICT (user_id,chat_id,key) DO UPDATE SET body=EXCLUDED.body,message_id=EXCLUDED.message_id,expires_at=EXCLUDED.expires_at' if optimized else '' + conn.execute("INSERT INTO message_requests VALUES (%s,%s,%s,%s,%s,clock_timestamp()+interval '24 hours')"+suffix, (sender_id,chat_id,key,text,row['id'])) return Message(**public_row(row)) def list_messages(self, chat_id, cursor, limit): diff --git a/student-tests/README.md b/student-tests/README.md new file mode 100644 index 00000000..4d671f6f --- /dev/null +++ b/student-tests/README.md @@ -0,0 +1,19 @@ +# Собственные проверки лабораторной 3 + +Все тесты создают только синтетические данные. Работать из корня проекта. На Windows установить `PYTHONUTF8=1`. + +- `python student-tests/test_lab3.py`: 32 конкурентных повтора через LB, наблюдение двух процессов, один результат, 409 для другого тела, неверные ключи, ACL, области пользователя/чата, отзыв сессии. +- `python scripts/experiments.py --phase rolling`: поочерёдное удаление и пересоздание каждой API-реплики под нагрузкой. Docker-доступ необходим. +- `python scripts/experiments.py --phase matrix`: три повтора 1/2/4 реплик при фиксированных суммарных CPU/RAM и при росте ресурсов. Порядок перемешан с seed=2301. +- `python scripts/experiments.py --phase saturation`: интенсивность 4/16/64/128 параллельных клиентов, два повтора каждой точки. +- `python scripts/experiments.py --phase pool`: три парных сравнения размера пула 1 и 10 на реплику при конкуренции 64. +- `python scripts/experiments.py --phase optimize`: три парных сравнения исходного и улучшенного пути записи в одном образе при одинаковых ресурсах. OPTIMIZED_WRITES=0/1, по умолчанию включено улучшение. +- `python student-tests/test_dependencies.py`: явная остановка Redis и PostgreSQL с восстановлением; fallback к БД, ограниченный отказ 503, прежняя сессия после восстановления. +- `python student-tests/test_transactions.py`: имитация expiry ключа изменением только собственной тестовой строки, затем искусственная ошибка 500 через временный trigger, снятие trigger и успешный retry того же ключа. Выполнять отдельно от нагрузки; trigger ограничен UUID нового синтетического чата и удаляется в finally. +- `python scripts/plot_results.py`: построение графиков Matplotlib из сохранённых JSON, без новых запросов к приложению. + +`benchmark.py` запускается скриптом эксперимента внутри отдельного контейнера (2 CPU, 512 MiB) и той же сети. Каждый поток держит своё HTTP-соединение; прозрачных повторов POST нет. Внутренняя нагрузка также идёт через HAProxy. Секреты и токены в результаты не записываются. + +Начальная история нового чата — 100 сообщений. Прогрев 5 с, замер 20 с (rolling 45 с). Сохраняются p50/p95/p99 успешных запросов, ошибки, ID процессов, размер набора, CPU/RAM и состояния ожидания БД. Это closed-loop эксперимент с coordinated omission, а не доказательство предельной производительности при произвольном входном потоке. + +Публичные тесты tests/ сохранены. Собственные тесты лабораторной 2, привязанные к одиночному API и не передающие обязательный ключ, не выдаются за совместимые тесты лабораторной 3. diff --git a/student-tests/benchmark.py b/student-tests/benchmark.py new file mode 100644 index 00000000..5da447d2 --- /dev/null +++ b/student-tests/benchmark.py @@ -0,0 +1,104 @@ +"""Внутрисетевой генератор: постоянное HTTP-соединение на поток, без повторов POST.""" +from concurrent.futures import ThreadPoolExecutor +from collections import Counter +import argparse +import http.client +import json +import math +from pathlib import Path +import sys +import threading +import time +import uuid +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client + +p = argparse.ArgumentParser() +p.add_argument('--duration', type=float, default=20) +p.add_argument('--concurrency', type=int, default=32) +p.add_argument('--warmup', type=float, default=5) +p.add_argument('--action', choices=['none', 'rolling'], default='none') +args = p.parse_args() +c = Client('http://lb:8080') +alice, bob, eve, chat = c.bootstrap() +path = '/api/v1/chats/' + chat['id'] + '/messages' +for i in range(100): + c.message(alice, chat, 'Начальная история %d' % i) +barrier = threading.Barrier(args.concurrency) + + +def worker(worker_id): + conn = http.client.HTTPConnection('lb', 8080, timeout=5) + headers = {'Content-Type': 'application/json', 'Authorization': 'Bearer ' + alice['token']} + barrier.wait() + start = time.monotonic() + measure_start = start + args.warmup + deadline = measure_start + args.duration + values, failures, instances = [], Counter(), Counter() + warm_count = 0 + confirmed_ids = set() + while time.monotonic() < deadline: + before = time.monotonic() + measured = before >= measure_start + headers['Idempotency-Key'] = uuid.uuid4().hex + try: + conn.request('POST', path, json.dumps({'text': 'Нагрузка ' + 'я' * 240}).encode(), headers) + response = conn.getresponse() + raw = response.read() + payload = json.loads(raw) if response.status == 201 else {} + if response.status == 201 and payload.get('chat_id') == chat['id']: + confirmed_ids.add(payload['id']) + if measured: + values.append(time.monotonic() - before) + instances[response.getheader('X-Instance-Id')] += 1 + else: + warm_count += 1 + elif measured: + failures[str(response.status)] += 1 + except (OSError, ValueError, http.client.HTTPException): + conn.close() + if measured: + failures['connection'] += 1 + conn.close() + return values, failures, instances, warm_count, time.monotonic()-measure_start, confirmed_ids + + +with ThreadPoolExecutor(max_workers=args.concurrency) as pool: + results = list(pool.map(worker, range(args.concurrency))) +values = sorted(v for r in results for v in r[0]) +errors, instances = Counter(), Counter() +for r in results: + errors.update(r[1]) + instances.update(r[2]) +elapsed = max(r[4] for r in results) +confirmed = set().union(*(r[5] for r in results)) +seen = set() +cursor = None +from urllib.parse import quote +for _ in range(10000): + page = c.call('GET', path+'?limit=100'+('&cursor='+quote(cursor, safe='') if cursor else ''), actor=bob, schema='MessagePage')['json'] + for item in page['items']: + if item['id'] in seen: + raise AssertionError('Дубль сообщения в истории') + seen.add(item['id']) + cursor = page['next_cursor'] + if cursor is None: + break +else: + raise AssertionError('Превышена граница обхода истории') +if not confirmed.issubset(seen): + raise AssertionError('Потеря подтверждённого сообщения') + + +def percentile(q): + return round(values[max(0, math.ceil(q*len(values))-1)]*1000, 3) if values else None + + +print(json.dumps({'duration': args.duration, 'warmup': args.warmup, 'elapsed': elapsed, + 'concurrency': args.concurrency, 'successful_requests': len(values), 'failed_requests': sum(errors.values()), + 'errors': dict(errors), 'error_ratio': sum(errors.values()) / max(1, len(values)+sum(errors.values())), + 'successful_rps': len(values)/elapsed, 'successful_latency_ms': {'p50': percentile(.5), 'p95': percentile(.95), 'p99': percentile(.99)}, + 'instances': dict(instances), 'initial_chat_messages': 100, + 'confirmed_ids_checked': len(confirmed), 'history_count_after': len(seen), 'confirmed_data_preserved': True, + 'warmup_messages': sum(r[3] for r in results), 'final_chat_messages_expected_min': 100+sum(r[3] for r in results)+len(values), + 'note': 'Closed-loop; coordinated omission; percentiles cover successes; no retries; uncertain writes may exist.'}, ensure_ascii=False)) diff --git a/student-tests/test_dependencies.py b/student-tests/test_dependencies.py new file mode 100644 index 00000000..aa7416b3 --- /dev/null +++ b/student-tests/test_dependencies.py @@ -0,0 +1,43 @@ +"""Проверка ограниченного отказа зависимостей без удаления учебного тома.""" +from pathlib import Path +import subprocess +import sys +import time +import urllib.error +import urllib.request +sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests')) +from client import Client, check + + +def compose(*args): + subprocess.run(['docker','compose',*args],check=True,stdout=subprocess.DEVNULL) + + +c=Client() +alice=c.user('dependency') +try: + compose('stop','redis') + cache_started=time.monotonic() + check(c.call('GET','/api/v1/users/me',actor=alice,schema='User')['json']['id']==alice['id'],'нет fallback к PostgreSQL') + check(time.monotonic()-cache_started<2,'кэш задержал fallback более чем на 2 секунды') + print('PASS: Redis остановлен, действующая сессия и профиль доступны из PostgreSQL за',round(time.monotonic()-cache_started,3),'с') +finally: + compose('start','redis') +try: + compose('stop','db') + started=time.monotonic() + req=urllib.request.Request('http://localhost:8080/api/v1/users/me',headers={'Authorization':'Bearer '+alice['token']}) + try: + urllib.request.urlopen(req,timeout=8) + except urllib.error.HTTPError as error: + check(error.code==503,'при недоступной БД должен быть ограниченный отказ 503') + error.close() + else: + raise AssertionError('запрос доверен при недоступной БД') + check(time.monotonic()-started<8,'отказ не ограничен таймаутом') + print('PASS: PostgreSQL остановлен; закрытый отказ 503 за',round(time.monotonic()-started,3),'с') +finally: + compose('start','db') + subprocess.run([sys.executable,'scripts/wait_ready.py','--replicas','2'],check=True) +check(c.call('GET','/api/v1/users/me',actor=alice,schema='User')['json']['id']==alice['id'],'прежняя сессия не восстановилась') +print('PASS: прежняя сессия сохранилась после восстановления БД') diff --git a/student-tests/test_lab3.py b/student-tests/test_lab3.py index 9149e0f1..fbd35886 100644 --- a/student-tests/test_lab3.py +++ b/student-tests/test_lab3.py @@ -32,6 +32,19 @@ check(other['id'] != results[0]['json']['id'], 'ключ не изолирова second = c.call('POST', '/api/v1/chats', 201, actor=alice, body={'title': 'Другой чат', 'member_ids': [alice['id']]}, schema='Chat')['json'] c.message(alice, second, 'Независимый чат', key=key) +race_key = uuid.uuid4().hex + + +def conflict_race(i): + return Client().call('POST', path, [201,409], actor=alice, + body={'text': 'Тело ' + str(i%2)}, headers={'Idempotency-Key': race_key}, schema='Message') + + +with ThreadPoolExecutor(max_workers=16) as pool: + race = list(pool.map(conflict_race, range(16))) +check(sum(r['status']==201 for r in race)==8, 'неверная сериализация разных тел') +check(sum(r['status']==409 for r in race)==8, 'не обнаружен конфликт параллельных тел') +check(len({r['json']['id'] for r in race if r['status']==201})==1, 'гонка разных тел создала дубли') c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=alice) c.call('POST', path, 401, actor=alice, body={'text': 'Общий конкурентный ключ'}, headers={'Idempotency-Key': key}) -print('PASS: 32 параллельных запроса,', len(instances), 'реплики; один результат, конфликт, валидация, ACL, области ключа и отзыв сессии.') +print('PASS: 32 одинаковых и 16 конфликтующих параллельных запросов,', len(instances), 'реплики; один результат, конфликт, валидация, ACL, области ключа и отзыв сессии.') diff --git a/student-tests/test_transactions.py b/student-tests/test_transactions.py new file mode 100644 index 00000000..0fd4c418 --- /dev/null +++ b/student-tests/test_transactions.py @@ -0,0 +1,37 @@ +"""Проверка expiry ключа и отката при искусственной транзиентной ошибке БД.""" +from pathlib import Path +import subprocess +import sys +import uuid +sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests')) +from client import Client, check + + +def sql(statement): + subprocess.run(['docker','compose','exec','-T','db','psql','-v','ON_ERROR_STOP=1','-U','messenger','-d','messenger','-c',statement],check=True,stdout=subprocess.DEVNULL) + + +c=Client() +alice,bob,eve,chat=c.bootstrap() +key=uuid.uuid4().hex +first=c.message(alice,chat,'До истечения',key=key)['json'] +check(c.message(alice,chat,'До истечения',key=key)['json']==first,'результат повтора изменился') +# ID и ключ созданы тестом, не берутся из пользовательского ввода. +sql(f"UPDATE message_requests SET expires_at=clock_timestamp()-interval '1 second' WHERE chat_id='{chat['id']}' AND key='{key}'") +second=c.message(alice,chat,'После истечения',key=key)['json'] +check(second['id']!=first['id'],'просроченный ключ не переиспользуется') +print('PASS: искусственно состаренный ключ заменён; исходное сообщение сохранено') +fault_key=uuid.uuid4().hex +path='/api/v1/chats/'+chat['id']+'/messages' +try: + sql(f"""CREATE FUNCTION lab3_inject_failure() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN IF NEW.chat_id='{chat['id']}'::uuid THEN RAISE EXCEPTION 'synthetic failure' USING ERRCODE='P0001'; END IF; RETURN NEW; END $$; + CREATE TRIGGER lab3_failure BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION lab3_inject_failure();""") + c.call('POST',path,500,actor=alice,body={'text':'После временной ошибки'},headers={'Idempotency-Key':fault_key}) +finally: + sql('DROP TRIGGER IF EXISTS lab3_failure ON messages; DROP FUNCTION IF EXISTS lab3_inject_failure();') +third=c.message(alice,chat,'После временной ошибки',key=fault_key)['json'] +check(c.message(alice,chat,'После временной ошибки',key=fault_key)['json']==third,'после отката повтор не стабилен') +history=c.call('GET',path,actor=bob,schema='MessagePage')['json']['items'] +check({m['id'] for m in history}=={first['id'],second['id'],third['id']},'потеря или дубли после expiry/отката') +print('PASS: искусственная ошибка 500 не закэширована; транзакция откатилась, повтор создал одно сообщение')