diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..b759e5b --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Keep shell hooks runnable after a Windows clone with core.autocrlf=true. +*.sh text eol=lf diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml new file mode 100644 index 0000000..a6996f2 --- /dev/null +++ b/.github/workflows/application.yml @@ -0,0 +1,25 @@ +name: Application smoke +on: [push, pull_request] +permissions: + contents: read +jobs: + application: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Prepare isolated lab credentials + run: | + sh scripts/init-secrets.sh + cp .env.example .env + docker compose --env-file .env config --quiet + - name: Build and wait for application + run: docker compose --env-file .env up -d --build --wait --wait-timeout 120 + - name: Check contract and public scenario + run: make check test PYTHON=python BASE_URL=http://127.0.0.1:8080 + - name: Stop disposable CI stack + if: always() + run: docker compose --env-file .env down diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg index 4be6130..e446fe8 100644 --- a/infra/haproxy/haproxy.cfg +++ b/infra/haproxy/haproxy.cfg @@ -17,7 +17,8 @@ defaults timeout queue 5s retries 3 option redispatch 1 - retry-on conn-failure empty-response response-timeout 0rtt-rejected + # Only retry before a connection is established: upload POST has no idempotency key. + retry-on conn-failure # Docker DNS (127.0.0.11): адреса реплик API обновляются без перезапуска LB (scale up/down, пересоздание контейнеров). resolvers docker @@ -33,6 +34,9 @@ resolvers docker frontend public bind :8080 + # If the shared DB is down, live API processes still produce contract JSON, + # validate missing credentials, and serve /health/live independently of DB. + use_backend api_live if { nbsrv(api) eq 0 } default_backend api # Внутренний healthcheck самого LB (контейнерный HEALTHCHECK); наружу не публикуется. @@ -48,3 +52,11 @@ backend api http-check expect status 200 default-server check inter 1s fall 2 rise 2 resolvers docker init-addr none server-template api 1-8 api:8080 + +backend api_live + balance roundrobin + option httpchk + http-check send meth GET uri /health/live + http-check expect status 200 + default-server check inter 1s fall 2 rise 2 resolvers docker init-addr none + server-template api 1-8 api:8080 diff --git a/research/requirements.txt b/research/requirements.txt new file mode 100644 index 0000000..2ce1068 --- /dev/null +++ b/research/requirements.txt @@ -0,0 +1,3 @@ +aiohttp==3.14.3 +Pillow==12.3.0 +matplotlib==3.11.2 diff --git a/scripts/restart.cmd b/scripts/restart.cmd index 5881510..0239809 100644 --- a/scripts/restart.cmd +++ b/scripts/restart.cmd @@ -5,5 +5,5 @@ docker compose --env-file .env restart postgres valkey seaweedfs rabbitmq worker if errorlevel 1 exit /b 1 docker compose --env-file .env up -d --wait --no-recreate --no-deps postgres valkey seaweedfs rabbitmq worker api lb otel-collector prometheus loki tempo grafana >NUL 2>&1 if errorlevel 1 exit /b 1 -ping -n 12 127.0.0.1 >NUL -exit /b 0 +docker compose --env-file .env exec -T api python -m app.s3_recovery +exit /b %errorlevel% diff --git a/scripts/restart.sh b/scripts/restart.sh index e681222..41be5a6 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -5,6 +5,4 @@ set -eu cd "$(dirname "$0")/.." docker compose --env-file .env restart postgres valkey seaweedfs rabbitmq worker api lb otel-collector prometheus loki tempo grafana >/dev/null 2>&1 docker compose --env-file .env up -d --wait --no-recreate --no-deps postgres valkey seaweedfs rabbitmq worker api lb otel-collector prometheus loki tempo grafana >/dev/null 2>&1 -# стабилизация: API готов и отвечает через LB (повторные проверки до 60 с), затем короткая пауза для S3/брокера -i=0; until curl -fsS -o /dev/null http://127.0.0.1:8080/health/ready 2>/dev/null; do i=$((i+1)); [ "$i" -gt 60 ] && break; sleep 1; done -sleep 6 +docker compose --env-file .env exec -T api python -m app.s3_recovery diff --git a/src/app/reconcile.py b/src/app/reconcile.py index ea4538c..64038c9 100644 --- a/src/app/reconcile.py +++ b/src/app/reconcile.py @@ -15,6 +15,7 @@ import argparse import asyncio import json import time +import uuid from datetime import datetime, timedelta, timezone import asyncpg @@ -23,6 +24,17 @@ from . import config from .s3 import S3 +def object_owner(key): + """Recognize server-generated prefixes, including effects not committed yet.""" + parts = key.split("/") + if (parts[0] == "att" and len(parts) >= 3) or (parts[0] == "staging" and len(parts) == 2): + try: + return uuid.UUID(parts[1]) + except ValueError: + pass + return None + + async def reconcile(apply, grace, prefix): started = time.perf_counter() s3 = S3() @@ -30,9 +42,15 @@ async def reconcile(apply, grace, prefix): conn = await asyncpg.connect(config.DATABASE_URL, timeout=5) try: t0 = time.perf_counter() - rows = await conn.fetch("SELECT id, s3_key, size_bytes, status, created_at FROM attachments") + rows = await conn.fetch("SELECT id, s3_key, staging_key, size_bytes, status, created_at FROM attachments") + variants = await conn.fetch( + "SELECT a.id, v.s3_key, v.size_bytes, a.status, a.created_at " + "FROM attachment_variants v JOIN attachments a ON a.id = v.attachment_id") db_ms = (time.perf_counter() - t0) * 1000 - by_key = {r["s3_key"]: r for r in rows} + by_key = {r["s3_key"]: dict(r, check=True) for r in [*rows, *variants]} + # Staging may already be removed after ready, or retained for failed/replay. + by_key.update({r["staging_key"]: dict(r, check=False) for r in rows if r["staging_key"]}) + owners = {r["id"] for r in rows} t0 = time.perf_counter() objects = {} async for key, size, modified in s3.list_keys(prefix): @@ -41,23 +59,32 @@ async def reconcile(apply, grace, prefix): now = datetime.now(timezone.utc) cutoff = now - timedelta(seconds=grace) report = {"mode": "apply" if apply else "dry-run", "grace_seconds": grace, "db_rows": len(rows), "s3_objects": len(objects), - "orphan_objects": [], "orphans_skipped_fresh": 0, "missing_objects": [], "size_mismatch": [], "fixed": {"deleted_orphans": 0, "marked_failed": 0}} + "orphan_objects": [], "orphans_skipped_fresh": 0, "owned_objects_skipped": 0, + "missing_objects": [], "size_mismatch": [], "fixed": {"deleted_orphans": 0, "marked_failed": 0}} for key, (size, modified) in objects.items(): row = by_key.get(key) if row is None: - if modified < cutoff: + if object_owner(key) in owners: + # Worker can write S3 variants before committing their metadata. + report["owned_objects_skipped"] += 1 + elif modified < cutoff: report["orphan_objects"].append(key) else: report["orphans_skipped_fresh"] += 1 - elif row["status"] == "ready" and row["size_bytes"] != size: + elif row["check"] and row["status"] == "ready" and row["size_bytes"] != size: report["size_mismatch"].append(str(row["id"])) for key, row in by_key.items(): - if key.startswith(prefix) and key not in objects and row["status"] == "ready" and row["created_at"] < cutoff: + if row["check"] and key.startswith(prefix) and key not in objects and row["status"] == "ready" and row["created_at"] < cutoff: report["missing_objects"].append(str(row["id"])) + for field in ("missing_objects", "size_mismatch"): + report[field] = sorted(set(report[field])) if apply: for key in report["orphan_objects"]: # повторная проверка перед удалением: за время обхода строка могла появиться - if await conn.fetchval("SELECT 1 FROM attachments WHERE s3_key = $1", key) is None: + referenced = await conn.fetchval( + "SELECT EXISTS (SELECT 1 FROM attachments WHERE s3_key = $1 OR staging_key = $1 OR id = $2) " + "OR EXISTS (SELECT 1 FROM attachment_variants WHERE s3_key = $1)", key, object_owner(key)) + if not referenced: await s3.delete(key) report["fixed"]["deleted_orphans"] += 1 for ids, code in ((report["missing_objects"], "object_missing"), (report["size_mismatch"], "size_mismatch")): diff --git a/src/app/s3_recovery.py b/src/app/s3_recovery.py new file mode 100644 index 0000000..50191d9 --- /dev/null +++ b/src/app/s3_recovery.py @@ -0,0 +1,77 @@ +"""Read-only readiness probe after restart: ranged GET of recent ready originals. + +An open S3 socket / HEAD bucket does not prove that SeaweedFS volume locations +have been registered. Read one byte from up to 16 recent ready originals. +This is a bounded recovery hook, not an audit of every historical object. +""" +import asyncio +import sys + +import asyncpg + +from . import config +from .s3 import S3 + + +async def probe(s3): + count = 0 + batch = [] + + async def read(key, size): + params = {"Bucket": config.S3_BUCKET, "Key": key} + if size: + params["Range"] = "bytes=0-0" + response = await s3.client.get_object(**params) + async with response["Body"] as stream: + data = await stream.read() + if len(data) != min(size, 1): + raise ValueError("unexpected ranged read size") + + async def flush(): + results = await asyncio.gather(*(read(key, size) for key, size in batch), return_exceptions=True) + for result in results: + if isinstance(result, BaseException): + raise result + + await s3.ping() + conn = await asyncpg.connect(config.DATABASE_URL, timeout=5) + try: + rows = await conn.fetch("SELECT s3_key, size_bytes FROM attachments " + "WHERE status = 'ready' ORDER BY accepted_at DESC LIMIT 16") + finally: + await conn.close() + for row in rows: + key, size = row["s3_key"], row["size_bytes"] + batch.append((key, size)) + count += 1 + if len(batch) == 8: + await flush() + batch.clear() + if batch: + await flush() + return count + + +async def main(): + s3 = S3() + await s3.start() + try: + async with asyncio.timeout(45): + while True: + try: + count = await probe(s3) + print(f"PASS: S3 recovery, {count} recent ready originals readable") + return 0 + except Exception as exc: + # Do not print SDK requests/credentials or object contents. + print(f"Waiting for S3 data volumes: {type(exc).__name__}", flush=True) + await asyncio.sleep(1) + except TimeoutError: + print("FAIL: S3 data recovery exceeded 45 seconds") + return 1 + finally: + await s3.stop() + + +if __name__ == "__main__": + sys.exit(asyncio.run(main())) diff --git a/student-tests/README.md b/student-tests/README.md new file mode 100644 index 0000000..9e1d9e3 --- /dev/null +++ b/student-tests/README.md @@ -0,0 +1,32 @@ +# Собственные проверки ЛР6 + +Запустите стенд по REPORT.md. Python 3.11+ с `python -m pip install -r research/requirements.txt` включает Pillow для пиксельных проверок. Публичным tests/ достаточно Python 3.9+. В Windows используйте `PYTHONUTF8=1`. + +Все сценарии выполняются **последовательно** на учебном стенде: часть тестов явно останавливает сервисы, внедряет отказы и восстанавливает настройки. Volumes не удаляются. Профиль для проверок наблюдаемости — OTEL_ENABLED=1, head sampling 1.0, обычный collector.yaml. + +```sh +python scripts/check_contract.py +python scripts/validate_openapi.py # предварительно установить requirements-dev.txt +python tests/smoke.py --base-url http://127.0.0.1:8080 +for suite in test_api test_idempotency test_attachments test_topology test_pipeline test_observability test_observability_retry test_reconcile5 test_resilience5 test_expiry test_dependencies; do + python -m unittest discover -s student-tests -p "$suite.py" -v || exit 1 +done +python tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url http://127.0.0.1:8080 +python tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url http://127.0.0.1:8080 +sh scripts/restore-replicas.sh +``` + +В Windows публичным resilience передаются restart.cmd / stop-one.cmd; восстановление API — `docker compose up -d --wait --no-recreate --no-deps api`. Failover повторяется с восстановлением, чтобы проверить обе реплики. Фактические остановленные контейнеры и перезапуск всех сервисов сохранены в evidence/recheck_2026-11-12/. + +Набор содержит 66 тестов: API 12, идемпотентность 9, вложения 11, топология 6, pipeline 9, наблюдаемость 5, retry trace 1, reconciler 2, отказы pipeline 7, TTL 2, зависимости 2. Проверки наблюдаемости ищут реальные metrics/logs/traces, связь API → producer → consumer → S3/DB, retry в той же трассе, низкую кардинальность и отсутствие синтетических секретов в сигналах. + +Исследования для выбранного трека (запускать после тестов, не одновременно): + +```sh +mkdir -p evidence/my-run +python research/incidents.py --out evidence/my-run +python research/obs_cost.py --out evidence/my-run --profiles off on100 head10 tail --repeat 2 +python research/plot_obs_cost.py --out evidence/my-run +``` + +Четыре контролируемых инцидента проверяют firing/resolved; off/on100/head10/tail сравниваются при одинаковых ресурсах, 64 клиентах, 8 процессах, 5 с прогрева и 30 с измерения (2 повтора). Эти прогоны занимают около 20 минут. Секреты создаются scripts/init-secrets.sh и в Git не входят. Все команды и реальные результаты текущей проверки — evidence/checks_2026-11-12.txt и evidence/recheck_2026-11-12/. diff --git a/student-tests/test_dependencies.py b/student-tests/test_dependencies.py new file mode 100644 index 0000000..add0711 --- /dev/null +++ b/student-tests/test_dependencies.py @@ -0,0 +1,82 @@ +"""Поведение при недоступности зависимостей: Valkey (кэш) и PostgreSQL (источник истины). + + python student-tests/test_dependencies.py +Контейнеры останавливаются и запускаются обратно (volumes сохраняются). Ожидания: + * Valkey остановлен -> кэш обходится, API работает (чтение сессии из PostgreSQL), logout работает; + * PostgreSQL остановлен -> 503 в контрактном формате с Retry-After (не 500, не «анонимный» доступ), + /health/ready = 503, запрос без токена по-прежнему 401; после запуска БД API восстанавливается сам. +""" +import os +import subprocess +import sys +import time +import unittest + +sys.path.insert(0, os.path.dirname(__file__)) +from test_api import ROOT, basic, call, register # noqa: E402 + + +def compose(*args): + return subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, check=True, capture_output=True, text=True) + + +def wait(predicate, seconds=60): + deadline = time.time() + seconds + while time.time() < deadline: + try: + if predicate(): + return + except Exception: + pass + time.sleep(0.5) + raise RuntimeError("условие не выполнено за %d с" % seconds) + + +class DependencyTest(unittest.TestCase): + def tearDown(self): + compose("start", "postgres", "valkey") + wait(lambda: call("GET", "/health/ready")[0] == 200) + + def test_valkey_down_is_bypassed(self): + actor = register("dep") + self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200) + compose("stop", "valkey") + started = time.time() + for _ in range(5): + self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200) + self.assertLess(time.time() - started, 3, "circuit breaker не должен давать каждому запросу ждать таймаут") + # вход и выход работают без кэша + s = call("POST", "/api/v1/auth/sessions", headers=basic(actor["user"]["username"], actor["password"])) + self.assertEqual(s[0], 201) + self.assertEqual(call("DELETE", "/api/v1/auth/sessions/current", token=s[2]["token"])[0], 204) + self.assertEqual(call("GET", "/api/v1/users/me", token=s[2]["token"])[0], 401) # отзыв виден без кэша + self.assertEqual(call("GET", "/health/ready")[0], 200) # кэш не критичен для readiness + + def test_postgres_down_gives_bounded_503_never_anonymous_access(self): + actor = register("dep") + self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200) # токен в кэше + compose("stop", "postgres") + # Wait past HAProxy's fall=2 health checks: cover the all-unready state, + # not just the short interval while the load balancer still routes normally. + time.sleep(5) + for method, path, body, token in (("GET", "/api/v1/users/me", None, actor["token"]), + ("GET", "/api/v1/chats", None, actor["token"]), + ("POST", "/api/v1/chats", {"title": "x", "member_ids": [actor["user"]["id"]]}, actor["token"]), + ("GET", "/api/v1/users/me", None, "Z" * 43)): # неизвестный токен: кэш пуст -> БД -> 503 + r = call(method, path, body, token) + self.assertEqual(r[0], 503, (path, r)) + self.assertEqual(r[2]["error"]["code"], "unavailable") + self.assertGreaterEqual(int(r[1]["Retry-After"]), 1) + self.assertEqual(r[2]["error"]["request_id"], r[1]["X-Request-Id"]) + r = call("POST", "/api/v1/auth/sessions", headers=basic(actor["user"]["username"], actor["password"])) + self.assertEqual(r[0], 503) + self.assertEqual(call("GET", "/health/ready")[0], 503) + self.assertEqual(call("GET", "/health/live")[0], 200) + self.assertEqual(call("GET", "/api/v1/users/me")[0], 401) # без токена — по-прежнему 401 + compose("start", "postgres") + wait(lambda: call("GET", "/health/ready")[0] == 200) + self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200) # сессия пережила простой БД + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_expiry.py b/student-tests/test_expiry.py new file mode 100644 index 0000000..293fd05 --- /dev/null +++ b/student-tests/test_expiry.py @@ -0,0 +1,69 @@ +"""Проверка срока жизни сессии при коротком SESSION_TTL_SECONDS (по умолчанию 2). + + python student-tests/test_expiry.py +Скрипт пересоздаёт только контейнер api с SESSION_TTL_SECONDS=2, проверяет expiry и возвращает настройки +по умолчанию. Данные и volumes не затрагиваются. +""" +import os +import subprocess +import sys +import time +import unittest + +sys.path.insert(0, os.path.dirname(__file__)) +from test_api import ROOT, call, register # noqa: E402 + +TTL = 2 + + +def compose(*args, env=None): + return subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, check=True, + capture_output=True, text=True, env=dict(os.environ, **(env or {}))) + + +def wait_ready(seconds=60): + deadline = time.time() + seconds + while time.time() < deadline: + try: + if call("GET", "/health/ready")[0] == 200: + return + except Exception: + pass + time.sleep(0.5) + raise RuntimeError("api не стал ready") + + +class ExpiryTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + compose("up", "-d", "--no-deps", "--force-recreate", "api", env={"SESSION_TTL_SECONDS": str(TTL)}) + wait_ready() + + @classmethod + def tearDownClass(cls): + compose("up", "-d", "--no-deps", "--force-recreate", "api") + wait_ready() + + def test_session_expires_after_ttl(self): + actor = register("exp") + session = actor["session"] + self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200) # валидна, кэш прогрет + time.sleep(TTL + 1.5) + r = call("GET", "/api/v1/users/me", token=actor["token"]) + self.assertEqual(r[0], 401, "после TTL токен должен быть отвергнут (и кэшем, и БД)") + self.assertEqual(r[1]["WWW-Authenticate"], 'Bearer realm="messenger"') + self.assertEqual(r[2]["error"]["code"], "unauthorized") + self.assertEqual(call("DELETE", "/api/v1/auth/sessions/current", token=actor["token"])[0], 401) + self.assertIn("expires_at", session) + + def test_expiry_is_not_extended_by_use(self): + actor = register("exp") + for _ in range(3): + call("GET", "/api/v1/users/me", token=actor["token"]) + time.sleep(0.5) + time.sleep(1.5) + self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 401) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_reconcile5.py b/student-tests/test_reconcile5.py new file mode 100644 index 0000000..d2abceb --- /dev/null +++ b/student-tests/test_reconcile5.py @@ -0,0 +1,89 @@ +"""Reconciliation must preserve variants, in-flight effects and durable staging.""" +import hashlib +import json +import os +import subprocess +import sys +import unittest +import uuid + +sys.path.insert(0, os.path.dirname(__file__)) +from test_api import ROOT, call, register +from test_attachments import PNG, download, upload +from test_pipeline import wait_status + + +def compose(*args): + return subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT, + check=True, capture_output=True, text=True, encoding='utf-8').stdout + + +def reconcile(prefix, grace=0): + return json.loads(compose('run', '--rm', '-T', 'api', 'python', '-m', 'app.reconcile', + '--apply', '--grace', str(grace), '--prefix', prefix)) + + +def s3_action(action, key): + code = ('import asyncio\nfrom app import config\nfrom app.s3 import S3\nasync def m():\n' + ' s=S3(); await s.start()\n' + + {'put': ' await s.put_bytes(%r, b"temporary", "application/octet-stream")\n', + 'head': ' print((await s.head(%r)) is not None)\n', + 'delete': ' await s.delete(%r)\n'}[action] % key + + ' await s.stop()\nasyncio.run(m())') + return compose('exec', '-T', 'api', 'python', '-c', code).strip() + + +class ReconcilePipelineTest(unittest.TestCase): + def setUp(self): + self.actor = register('reconcile') + self.token = self.actor['token'] + self.chat = call('POST', '/api/v1/chats', {'title': 'reconcile', 'member_ids': [self.actor['user']['id']]}, self.token)[2] + + def accept(self): + status, _, att = upload(self.token, self.chat['id'], PNG) + self.assertEqual(status, 202) + return att + + def test_ready_variants_and_uncommitted_side_effect_are_preserved(self): + att = wait_status(self.token, self.accept()['id']) + key = 'att/%s/v1/uncommitted' % att['id'] + s3_action('put', key) + try: + report = reconcile('att/%s/' % att['id']) + self.assertEqual(report['fixed'], {'deleted_orphans': 0, 'marked_failed': 0}) + self.assertEqual(report['orphan_objects_count'], 0) + self.assertEqual(report['owned_objects_skipped'], 1) + self.assertEqual(s3_action('head', key), 'True') + self.assertEqual(download(self.token, att['id'])[2], PNG) + for variant in att['variants']: + code, _, data = download(self.token, att['id'], '/content?variant=' + variant['name']) + self.assertEqual(code, 200) + self.assertEqual(hashlib.sha256(data).hexdigest(), variant['sha256']) + finally: + s3_action('delete', key) + + def test_staging_is_protected_and_unowned_orphan_obeys_grace(self): + compose('stop', 'worker') + att = None + orphan = 'staging/' + str(uuid.uuid4()) + try: + att = self.accept() + report = reconcile('staging/' + att['id']) + self.assertEqual(report['fixed'], {'deleted_orphans': 0, 'marked_failed': 0}) + self.assertEqual(s3_action('head', 'staging/' + att['id']), 'True') + s3_action('put', orphan) + fresh = reconcile(orphan, grace=900) + self.assertEqual(fresh['orphans_skipped_fresh'], 1) + self.assertEqual(fresh['fixed']['deleted_orphans'], 0) + self.assertEqual(reconcile(orphan)['fixed']['deleted_orphans'], 1) + self.assertEqual(s3_action('head', orphan), 'False') + self.assertEqual(reconcile(orphan)['fixed']['deleted_orphans'], 0) + finally: + s3_action('delete', orphan) + compose('start', 'worker') + self.assertEqual(wait_status(self.token, att['id'])['status'], 'ready') + self.assertEqual(download(self.token, att['id'])[2], PNG) + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/test_resilience5.py b/student-tests/test_resilience5.py index 95ce7e2..1e25e5b 100644 --- a/student-tests/test_resilience5.py +++ b/student-tests/test_resilience5.py @@ -56,7 +56,7 @@ class ResilienceTest(unittest.TestCase): compose("up", "-d", "--no-deps", "rabbitmq", "api") def setUp(self): - compose("start", "rabbitmq", "postgres", "minio", "valkey", check=False) + compose("start", "rabbitmq", "postgres", "seaweedfs", "valkey") wait(lambda: healthy("rabbitmq") and healthy("postgres"), 90, "rabbitmq/postgres healthy") wait(api_ready, 60, "api ready") self.a = register("res")