fix(lab6): preserve lab5 resilience and reproducible recovery

This commit is contained in:
kolya5544
2026-11-12 04:31:30 +03:00
parent 1d908578e3
commit 6d87f5426c
13 changed files with 430 additions and 14 deletions
+2
View File
@@ -0,0 +1,2 @@
# Keep shell hooks runnable after a Windows clone with core.autocrlf=true.
*.sh text eol=lf
+25
View File
@@ -0,0 +1,25 @@
name: Application smoke
on: [push, pull_request]
permissions:
contents: read
jobs:
application:
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Prepare isolated lab credentials
run: |
sh scripts/init-secrets.sh
cp .env.example .env
docker compose --env-file .env config --quiet
- name: Build and wait for application
run: docker compose --env-file .env up -d --build --wait --wait-timeout 120
- name: Check contract and public scenario
run: make check test PYTHON=python BASE_URL=http://127.0.0.1:8080
- name: Stop disposable CI stack
if: always()
run: docker compose --env-file .env down
+13 -1
View File
@@ -17,7 +17,8 @@ defaults
timeout queue 5s
retries 3
option redispatch 1
retry-on conn-failure empty-response response-timeout 0rtt-rejected
# Only retry before a connection is established: upload POST has no idempotency key.
retry-on conn-failure
# Docker DNS (127.0.0.11): адреса реплик API обновляются без перезапуска LB (scale up/down, пересоздание контейнеров).
resolvers docker
@@ -33,6 +34,9 @@ resolvers docker
frontend public
bind :8080
# If the shared DB is down, live API processes still produce contract JSON,
# validate missing credentials, and serve /health/live independently of DB.
use_backend api_live if { nbsrv(api) eq 0 }
default_backend api
# Внутренний healthcheck самого LB (контейнерный HEALTHCHECK); наружу не публикуется.
@@ -48,3 +52,11 @@ backend api
http-check expect status 200
default-server check inter 1s fall 2 rise 2 resolvers docker init-addr none
server-template api 1-8 api:8080
backend api_live
balance roundrobin
option httpchk
http-check send meth GET uri /health/live
http-check expect status 200
default-server check inter 1s fall 2 rise 2 resolvers docker init-addr none
server-template api 1-8 api:8080
+3
View File
@@ -0,0 +1,3 @@
aiohttp==3.14.3
Pillow==12.3.0
matplotlib==3.11.2
+2 -2
View File
@@ -5,5 +5,5 @@ docker compose --env-file .env restart postgres valkey seaweedfs rabbitmq worker
if errorlevel 1 exit /b 1
docker compose --env-file .env up -d --wait --no-recreate --no-deps postgres valkey seaweedfs rabbitmq worker api lb otel-collector prometheus loki tempo grafana >NUL 2>&1
if errorlevel 1 exit /b 1
ping -n 12 127.0.0.1 >NUL
exit /b 0
docker compose --env-file .env exec -T api python -m app.s3_recovery
exit /b %errorlevel%
+1 -3
View File
@@ -5,6 +5,4 @@ set -eu
cd "$(dirname "$0")/.."
docker compose --env-file .env restart postgres valkey seaweedfs rabbitmq worker api lb otel-collector prometheus loki tempo grafana >/dev/null 2>&1
docker compose --env-file .env up -d --wait --no-recreate --no-deps postgres valkey seaweedfs rabbitmq worker api lb otel-collector prometheus loki tempo grafana >/dev/null 2>&1
# стабилизация: API готов и отвечает через LB (повторные проверки до 60 с), затем короткая пауза для S3/брокера
i=0; until curl -fsS -o /dev/null http://127.0.0.1:8080/health/ready 2>/dev/null; do i=$((i+1)); [ "$i" -gt 60 ] && break; sleep 1; done
sleep 6
docker compose --env-file .env exec -T api python -m app.s3_recovery
+34 -7
View File
@@ -15,6 +15,7 @@ import argparse
import asyncio
import json
import time
import uuid
from datetime import datetime, timedelta, timezone
import asyncpg
@@ -23,6 +24,17 @@ from . import config
from .s3 import S3
def object_owner(key):
"""Recognize server-generated prefixes, including effects not committed yet."""
parts = key.split("/")
if (parts[0] == "att" and len(parts) >= 3) or (parts[0] == "staging" and len(parts) == 2):
try:
return uuid.UUID(parts[1])
except ValueError:
pass
return None
async def reconcile(apply, grace, prefix):
started = time.perf_counter()
s3 = S3()
@@ -30,9 +42,15 @@ async def reconcile(apply, grace, prefix):
conn = await asyncpg.connect(config.DATABASE_URL, timeout=5)
try:
t0 = time.perf_counter()
rows = await conn.fetch("SELECT id, s3_key, size_bytes, status, created_at FROM attachments")
rows = await conn.fetch("SELECT id, s3_key, staging_key, size_bytes, status, created_at FROM attachments")
variants = await conn.fetch(
"SELECT a.id, v.s3_key, v.size_bytes, a.status, a.created_at "
"FROM attachment_variants v JOIN attachments a ON a.id = v.attachment_id")
db_ms = (time.perf_counter() - t0) * 1000
by_key = {r["s3_key"]: r for r in rows}
by_key = {r["s3_key"]: dict(r, check=True) for r in [*rows, *variants]}
# Staging may already be removed after ready, or retained for failed/replay.
by_key.update({r["staging_key"]: dict(r, check=False) for r in rows if r["staging_key"]})
owners = {r["id"] for r in rows}
t0 = time.perf_counter()
objects = {}
async for key, size, modified in s3.list_keys(prefix):
@@ -41,23 +59,32 @@ async def reconcile(apply, grace, prefix):
now = datetime.now(timezone.utc)
cutoff = now - timedelta(seconds=grace)
report = {"mode": "apply" if apply else "dry-run", "grace_seconds": grace, "db_rows": len(rows), "s3_objects": len(objects),
"orphan_objects": [], "orphans_skipped_fresh": 0, "missing_objects": [], "size_mismatch": [], "fixed": {"deleted_orphans": 0, "marked_failed": 0}}
"orphan_objects": [], "orphans_skipped_fresh": 0, "owned_objects_skipped": 0,
"missing_objects": [], "size_mismatch": [], "fixed": {"deleted_orphans": 0, "marked_failed": 0}}
for key, (size, modified) in objects.items():
row = by_key.get(key)
if row is None:
if modified < cutoff:
if object_owner(key) in owners:
# Worker can write S3 variants before committing their metadata.
report["owned_objects_skipped"] += 1
elif modified < cutoff:
report["orphan_objects"].append(key)
else:
report["orphans_skipped_fresh"] += 1
elif row["status"] == "ready" and row["size_bytes"] != size:
elif row["check"] and row["status"] == "ready" and row["size_bytes"] != size:
report["size_mismatch"].append(str(row["id"]))
for key, row in by_key.items():
if key.startswith(prefix) and key not in objects and row["status"] == "ready" and row["created_at"] < cutoff:
if row["check"] and key.startswith(prefix) and key not in objects and row["status"] == "ready" and row["created_at"] < cutoff:
report["missing_objects"].append(str(row["id"]))
for field in ("missing_objects", "size_mismatch"):
report[field] = sorted(set(report[field]))
if apply:
for key in report["orphan_objects"]:
# повторная проверка перед удалением: за время обхода строка могла появиться
if await conn.fetchval("SELECT 1 FROM attachments WHERE s3_key = $1", key) is None:
referenced = await conn.fetchval(
"SELECT EXISTS (SELECT 1 FROM attachments WHERE s3_key = $1 OR staging_key = $1 OR id = $2) "
"OR EXISTS (SELECT 1 FROM attachment_variants WHERE s3_key = $1)", key, object_owner(key))
if not referenced:
await s3.delete(key)
report["fixed"]["deleted_orphans"] += 1
for ids, code in ((report["missing_objects"], "object_missing"), (report["size_mismatch"], "size_mismatch")):
+77
View File
@@ -0,0 +1,77 @@
"""Read-only readiness probe after restart: ranged GET of recent ready originals.
An open S3 socket / HEAD bucket does not prove that SeaweedFS volume locations
have been registered. Read one byte from up to 16 recent ready originals.
This is a bounded recovery hook, not an audit of every historical object.
"""
import asyncio
import sys
import asyncpg
from . import config
from .s3 import S3
async def probe(s3):
count = 0
batch = []
async def read(key, size):
params = {"Bucket": config.S3_BUCKET, "Key": key}
if size:
params["Range"] = "bytes=0-0"
response = await s3.client.get_object(**params)
async with response["Body"] as stream:
data = await stream.read()
if len(data) != min(size, 1):
raise ValueError("unexpected ranged read size")
async def flush():
results = await asyncio.gather(*(read(key, size) for key, size in batch), return_exceptions=True)
for result in results:
if isinstance(result, BaseException):
raise result
await s3.ping()
conn = await asyncpg.connect(config.DATABASE_URL, timeout=5)
try:
rows = await conn.fetch("SELECT s3_key, size_bytes FROM attachments "
"WHERE status = 'ready' ORDER BY accepted_at DESC LIMIT 16")
finally:
await conn.close()
for row in rows:
key, size = row["s3_key"], row["size_bytes"]
batch.append((key, size))
count += 1
if len(batch) == 8:
await flush()
batch.clear()
if batch:
await flush()
return count
async def main():
s3 = S3()
await s3.start()
try:
async with asyncio.timeout(45):
while True:
try:
count = await probe(s3)
print(f"PASS: S3 recovery, {count} recent ready originals readable")
return 0
except Exception as exc:
# Do not print SDK requests/credentials or object contents.
print(f"Waiting for S3 data volumes: {type(exc).__name__}", flush=True)
await asyncio.sleep(1)
except TimeoutError:
print("FAIL: S3 data recovery exceeded 45 seconds")
return 1
finally:
await s3.stop()
if __name__ == "__main__":
sys.exit(asyncio.run(main()))
+32
View File
@@ -0,0 +1,32 @@
# Собственные проверки ЛР6
Запустите стенд по REPORT.md. Python 3.11+ с `python -m pip install -r research/requirements.txt` включает Pillow для пиксельных проверок. Публичным tests/ достаточно Python 3.9+. В Windows используйте `PYTHONUTF8=1`.
Все сценарии выполняются **последовательно** на учебном стенде: часть тестов явно останавливает сервисы, внедряет отказы и восстанавливает настройки. Volumes не удаляются. Профиль для проверок наблюдаемости — OTEL_ENABLED=1, head sampling 1.0, обычный collector.yaml.
```sh
python scripts/check_contract.py
python scripts/validate_openapi.py # предварительно установить requirements-dev.txt
python tests/smoke.py --base-url http://127.0.0.1:8080
for suite in test_api test_idempotency test_attachments test_topology test_pipeline test_observability test_observability_retry test_reconcile5 test_resilience5 test_expiry test_dependencies; do
python -m unittest discover -s student-tests -p "$suite.py" -v || exit 1
done
python tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url http://127.0.0.1:8080
python tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url http://127.0.0.1:8080
sh scripts/restore-replicas.sh
```
В Windows публичным resilience передаются restart.cmd / stop-one.cmd; восстановление API — `docker compose up -d --wait --no-recreate --no-deps api`. Failover повторяется с восстановлением, чтобы проверить обе реплики. Фактические остановленные контейнеры и перезапуск всех сервисов сохранены в evidence/recheck_2026-11-12/.
Набор содержит 66 тестов: API 12, идемпотентность 9, вложения 11, топология 6, pipeline 9, наблюдаемость 5, retry trace 1, reconciler 2, отказы pipeline 7, TTL 2, зависимости 2. Проверки наблюдаемости ищут реальные metrics/logs/traces, связь API → producer → consumer → S3/DB, retry в той же трассе, низкую кардинальность и отсутствие синтетических секретов в сигналах.
Исследования для выбранного трека (запускать после тестов, не одновременно):
```sh
mkdir -p evidence/my-run
python research/incidents.py --out evidence/my-run
python research/obs_cost.py --out evidence/my-run --profiles off on100 head10 tail --repeat 2
python research/plot_obs_cost.py --out evidence/my-run
```
Четыре контролируемых инцидента проверяют firing/resolved; off/on100/head10/tail сравниваются при одинаковых ресурсах, 64 клиентах, 8 процессах, 5 с прогрева и 30 с измерения (2 повтора). Эти прогоны занимают около 20 минут. Секреты создаются scripts/init-secrets.sh и в Git не входят. Все команды и реальные результаты текущей проверки — evidence/checks_2026-11-12.txt и evidence/recheck_2026-11-12/.
+82
View File
@@ -0,0 +1,82 @@
"""Поведение при недоступности зависимостей: Valkey (кэш) и PostgreSQL (источник истины).
python student-tests/test_dependencies.py
Контейнеры останавливаются и запускаются обратно (volumes сохраняются). Ожидания:
* Valkey остановлен -> кэш обходится, API работает (чтение сессии из PostgreSQL), logout работает;
* PostgreSQL остановлен -> 503 в контрактном формате с Retry-After (не 500, не «анонимный» доступ),
/health/ready = 503, запрос без токена по-прежнему 401; после запуска БД API восстанавливается сам.
"""
import os
import subprocess
import sys
import time
import unittest
sys.path.insert(0, os.path.dirname(__file__))
from test_api import ROOT, basic, call, register # noqa: E402
def compose(*args):
return subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, check=True, capture_output=True, text=True)
def wait(predicate, seconds=60):
deadline = time.time() + seconds
while time.time() < deadline:
try:
if predicate():
return
except Exception:
pass
time.sleep(0.5)
raise RuntimeError("условие не выполнено за %d с" % seconds)
class DependencyTest(unittest.TestCase):
def tearDown(self):
compose("start", "postgres", "valkey")
wait(lambda: call("GET", "/health/ready")[0] == 200)
def test_valkey_down_is_bypassed(self):
actor = register("dep")
self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200)
compose("stop", "valkey")
started = time.time()
for _ in range(5):
self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200)
self.assertLess(time.time() - started, 3, "circuit breaker не должен давать каждому запросу ждать таймаут")
# вход и выход работают без кэша
s = call("POST", "/api/v1/auth/sessions", headers=basic(actor["user"]["username"], actor["password"]))
self.assertEqual(s[0], 201)
self.assertEqual(call("DELETE", "/api/v1/auth/sessions/current", token=s[2]["token"])[0], 204)
self.assertEqual(call("GET", "/api/v1/users/me", token=s[2]["token"])[0], 401) # отзыв виден без кэша
self.assertEqual(call("GET", "/health/ready")[0], 200) # кэш не критичен для readiness
def test_postgres_down_gives_bounded_503_never_anonymous_access(self):
actor = register("dep")
self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200) # токен в кэше
compose("stop", "postgres")
# Wait past HAProxy's fall=2 health checks: cover the all-unready state,
# not just the short interval while the load balancer still routes normally.
time.sleep(5)
for method, path, body, token in (("GET", "/api/v1/users/me", None, actor["token"]),
("GET", "/api/v1/chats", None, actor["token"]),
("POST", "/api/v1/chats", {"title": "x", "member_ids": [actor["user"]["id"]]}, actor["token"]),
("GET", "/api/v1/users/me", None, "Z" * 43)): # неизвестный токен: кэш пуст -> БД -> 503
r = call(method, path, body, token)
self.assertEqual(r[0], 503, (path, r))
self.assertEqual(r[2]["error"]["code"], "unavailable")
self.assertGreaterEqual(int(r[1]["Retry-After"]), 1)
self.assertEqual(r[2]["error"]["request_id"], r[1]["X-Request-Id"])
r = call("POST", "/api/v1/auth/sessions", headers=basic(actor["user"]["username"], actor["password"]))
self.assertEqual(r[0], 503)
self.assertEqual(call("GET", "/health/ready")[0], 503)
self.assertEqual(call("GET", "/health/live")[0], 200)
self.assertEqual(call("GET", "/api/v1/users/me")[0], 401) # без токена — по-прежнему 401
compose("start", "postgres")
wait(lambda: call("GET", "/health/ready")[0] == 200)
self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200) # сессия пережила простой БД
if __name__ == "__main__":
unittest.main()
+69
View File
@@ -0,0 +1,69 @@
"""Проверка срока жизни сессии при коротком SESSION_TTL_SECONDS (по умолчанию 2).
python student-tests/test_expiry.py
Скрипт пересоздаёт только контейнер api с SESSION_TTL_SECONDS=2, проверяет expiry и возвращает настройки
по умолчанию. Данные и volumes не затрагиваются.
"""
import os
import subprocess
import sys
import time
import unittest
sys.path.insert(0, os.path.dirname(__file__))
from test_api import ROOT, call, register # noqa: E402
TTL = 2
def compose(*args, env=None):
return subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, check=True,
capture_output=True, text=True, env=dict(os.environ, **(env or {})))
def wait_ready(seconds=60):
deadline = time.time() + seconds
while time.time() < deadline:
try:
if call("GET", "/health/ready")[0] == 200:
return
except Exception:
pass
time.sleep(0.5)
raise RuntimeError("api не стал ready")
class ExpiryTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
compose("up", "-d", "--no-deps", "--force-recreate", "api", env={"SESSION_TTL_SECONDS": str(TTL)})
wait_ready()
@classmethod
def tearDownClass(cls):
compose("up", "-d", "--no-deps", "--force-recreate", "api")
wait_ready()
def test_session_expires_after_ttl(self):
actor = register("exp")
session = actor["session"]
self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200) # валидна, кэш прогрет
time.sleep(TTL + 1.5)
r = call("GET", "/api/v1/users/me", token=actor["token"])
self.assertEqual(r[0], 401, "после TTL токен должен быть отвергнут (и кэшем, и БД)")
self.assertEqual(r[1]["WWW-Authenticate"], 'Bearer realm="messenger"')
self.assertEqual(r[2]["error"]["code"], "unauthorized")
self.assertEqual(call("DELETE", "/api/v1/auth/sessions/current", token=actor["token"])[0], 401)
self.assertIn("expires_at", session)
def test_expiry_is_not_extended_by_use(self):
actor = register("exp")
for _ in range(3):
call("GET", "/api/v1/users/me", token=actor["token"])
time.sleep(0.5)
time.sleep(1.5)
self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 401)
if __name__ == "__main__":
unittest.main()
+89
View File
@@ -0,0 +1,89 @@
"""Reconciliation must preserve variants, in-flight effects and durable staging."""
import hashlib
import json
import os
import subprocess
import sys
import unittest
import uuid
sys.path.insert(0, os.path.dirname(__file__))
from test_api import ROOT, call, register
from test_attachments import PNG, download, upload
from test_pipeline import wait_status
def compose(*args):
return subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT,
check=True, capture_output=True, text=True, encoding='utf-8').stdout
def reconcile(prefix, grace=0):
return json.loads(compose('run', '--rm', '-T', 'api', 'python', '-m', 'app.reconcile',
'--apply', '--grace', str(grace), '--prefix', prefix))
def s3_action(action, key):
code = ('import asyncio\nfrom app import config\nfrom app.s3 import S3\nasync def m():\n'
' s=S3(); await s.start()\n'
+ {'put': ' await s.put_bytes(%r, b"temporary", "application/octet-stream")\n',
'head': ' print((await s.head(%r)) is not None)\n',
'delete': ' await s.delete(%r)\n'}[action] % key
+ ' await s.stop()\nasyncio.run(m())')
return compose('exec', '-T', 'api', 'python', '-c', code).strip()
class ReconcilePipelineTest(unittest.TestCase):
def setUp(self):
self.actor = register('reconcile')
self.token = self.actor['token']
self.chat = call('POST', '/api/v1/chats', {'title': 'reconcile', 'member_ids': [self.actor['user']['id']]}, self.token)[2]
def accept(self):
status, _, att = upload(self.token, self.chat['id'], PNG)
self.assertEqual(status, 202)
return att
def test_ready_variants_and_uncommitted_side_effect_are_preserved(self):
att = wait_status(self.token, self.accept()['id'])
key = 'att/%s/v1/uncommitted' % att['id']
s3_action('put', key)
try:
report = reconcile('att/%s/' % att['id'])
self.assertEqual(report['fixed'], {'deleted_orphans': 0, 'marked_failed': 0})
self.assertEqual(report['orphan_objects_count'], 0)
self.assertEqual(report['owned_objects_skipped'], 1)
self.assertEqual(s3_action('head', key), 'True')
self.assertEqual(download(self.token, att['id'])[2], PNG)
for variant in att['variants']:
code, _, data = download(self.token, att['id'], '/content?variant=' + variant['name'])
self.assertEqual(code, 200)
self.assertEqual(hashlib.sha256(data).hexdigest(), variant['sha256'])
finally:
s3_action('delete', key)
def test_staging_is_protected_and_unowned_orphan_obeys_grace(self):
compose('stop', 'worker')
att = None
orphan = 'staging/' + str(uuid.uuid4())
try:
att = self.accept()
report = reconcile('staging/' + att['id'])
self.assertEqual(report['fixed'], {'deleted_orphans': 0, 'marked_failed': 0})
self.assertEqual(s3_action('head', 'staging/' + att['id']), 'True')
s3_action('put', orphan)
fresh = reconcile(orphan, grace=900)
self.assertEqual(fresh['orphans_skipped_fresh'], 1)
self.assertEqual(fresh['fixed']['deleted_orphans'], 0)
self.assertEqual(reconcile(orphan)['fixed']['deleted_orphans'], 1)
self.assertEqual(s3_action('head', orphan), 'False')
self.assertEqual(reconcile(orphan)['fixed']['deleted_orphans'], 0)
finally:
s3_action('delete', orphan)
compose('start', 'worker')
self.assertEqual(wait_status(self.token, att['id'])['status'], 'ready')
self.assertEqual(download(self.token, att['id'])[2], PNG)
if __name__ == '__main__':
unittest.main()
+1 -1
View File
@@ -56,7 +56,7 @@ class ResilienceTest(unittest.TestCase):
compose("up", "-d", "--no-deps", "rabbitmq", "api")
def setUp(self):
compose("start", "rabbitmq", "postgres", "minio", "valkey", check=False)
compose("start", "rabbitmq", "postgres", "seaweedfs", "valkey")
wait(lambda: healthy("rabbitmq") and healthy("postgres"), 90, "rabbitmq/postgres healthy")
wait(api_ready, 60, "api ready")
self.a = register("res")