forked from mtusi2026/mtusi2026lab6
fix(lab6): preserve lab5 resilience and reproducible recovery
This commit is contained in:
@@ -0,0 +1,2 @@
|
||||
# Keep shell hooks runnable after a Windows clone with core.autocrlf=true.
|
||||
*.sh text eol=lf
|
||||
@@ -0,0 +1,25 @@
|
||||
name: Application smoke
|
||||
on: [push, pull_request]
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
application:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- name: Prepare isolated lab credentials
|
||||
run: |
|
||||
sh scripts/init-secrets.sh
|
||||
cp .env.example .env
|
||||
docker compose --env-file .env config --quiet
|
||||
- name: Build and wait for application
|
||||
run: docker compose --env-file .env up -d --build --wait --wait-timeout 120
|
||||
- name: Check contract and public scenario
|
||||
run: make check test PYTHON=python BASE_URL=http://127.0.0.1:8080
|
||||
- name: Stop disposable CI stack
|
||||
if: always()
|
||||
run: docker compose --env-file .env down
|
||||
@@ -17,7 +17,8 @@ defaults
|
||||
timeout queue 5s
|
||||
retries 3
|
||||
option redispatch 1
|
||||
retry-on conn-failure empty-response response-timeout 0rtt-rejected
|
||||
# Only retry before a connection is established: upload POST has no idempotency key.
|
||||
retry-on conn-failure
|
||||
|
||||
# Docker DNS (127.0.0.11): адреса реплик API обновляются без перезапуска LB (scale up/down, пересоздание контейнеров).
|
||||
resolvers docker
|
||||
@@ -33,6 +34,9 @@ resolvers docker
|
||||
|
||||
frontend public
|
||||
bind :8080
|
||||
# If the shared DB is down, live API processes still produce contract JSON,
|
||||
# validate missing credentials, and serve /health/live independently of DB.
|
||||
use_backend api_live if { nbsrv(api) eq 0 }
|
||||
default_backend api
|
||||
|
||||
# Внутренний healthcheck самого LB (контейнерный HEALTHCHECK); наружу не публикуется.
|
||||
@@ -48,3 +52,11 @@ backend api
|
||||
http-check expect status 200
|
||||
default-server check inter 1s fall 2 rise 2 resolvers docker init-addr none
|
||||
server-template api 1-8 api:8080
|
||||
|
||||
backend api_live
|
||||
balance roundrobin
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/live
|
||||
http-check expect status 200
|
||||
default-server check inter 1s fall 2 rise 2 resolvers docker init-addr none
|
||||
server-template api 1-8 api:8080
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
aiohttp==3.14.3
|
||||
Pillow==12.3.0
|
||||
matplotlib==3.11.2
|
||||
+2
-2
@@ -5,5 +5,5 @@ docker compose --env-file .env restart postgres valkey seaweedfs rabbitmq worker
|
||||
if errorlevel 1 exit /b 1
|
||||
docker compose --env-file .env up -d --wait --no-recreate --no-deps postgres valkey seaweedfs rabbitmq worker api lb otel-collector prometheus loki tempo grafana >NUL 2>&1
|
||||
if errorlevel 1 exit /b 1
|
||||
ping -n 12 127.0.0.1 >NUL
|
||||
exit /b 0
|
||||
docker compose --env-file .env exec -T api python -m app.s3_recovery
|
||||
exit /b %errorlevel%
|
||||
|
||||
+1
-3
@@ -5,6 +5,4 @@ set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose --env-file .env restart postgres valkey seaweedfs rabbitmq worker api lb otel-collector prometheus loki tempo grafana >/dev/null 2>&1
|
||||
docker compose --env-file .env up -d --wait --no-recreate --no-deps postgres valkey seaweedfs rabbitmq worker api lb otel-collector prometheus loki tempo grafana >/dev/null 2>&1
|
||||
# стабилизация: API готов и отвечает через LB (повторные проверки до 60 с), затем короткая пауза для S3/брокера
|
||||
i=0; until curl -fsS -o /dev/null http://127.0.0.1:8080/health/ready 2>/dev/null; do i=$((i+1)); [ "$i" -gt 60 ] && break; sleep 1; done
|
||||
sleep 6
|
||||
docker compose --env-file .env exec -T api python -m app.s3_recovery
|
||||
|
||||
+34
-7
@@ -15,6 +15,7 @@ import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import asyncpg
|
||||
@@ -23,6 +24,17 @@ from . import config
|
||||
from .s3 import S3
|
||||
|
||||
|
||||
def object_owner(key):
|
||||
"""Recognize server-generated prefixes, including effects not committed yet."""
|
||||
parts = key.split("/")
|
||||
if (parts[0] == "att" and len(parts) >= 3) or (parts[0] == "staging" and len(parts) == 2):
|
||||
try:
|
||||
return uuid.UUID(parts[1])
|
||||
except ValueError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
async def reconcile(apply, grace, prefix):
|
||||
started = time.perf_counter()
|
||||
s3 = S3()
|
||||
@@ -30,9 +42,15 @@ async def reconcile(apply, grace, prefix):
|
||||
conn = await asyncpg.connect(config.DATABASE_URL, timeout=5)
|
||||
try:
|
||||
t0 = time.perf_counter()
|
||||
rows = await conn.fetch("SELECT id, s3_key, size_bytes, status, created_at FROM attachments")
|
||||
rows = await conn.fetch("SELECT id, s3_key, staging_key, size_bytes, status, created_at FROM attachments")
|
||||
variants = await conn.fetch(
|
||||
"SELECT a.id, v.s3_key, v.size_bytes, a.status, a.created_at "
|
||||
"FROM attachment_variants v JOIN attachments a ON a.id = v.attachment_id")
|
||||
db_ms = (time.perf_counter() - t0) * 1000
|
||||
by_key = {r["s3_key"]: r for r in rows}
|
||||
by_key = {r["s3_key"]: dict(r, check=True) for r in [*rows, *variants]}
|
||||
# Staging may already be removed after ready, or retained for failed/replay.
|
||||
by_key.update({r["staging_key"]: dict(r, check=False) for r in rows if r["staging_key"]})
|
||||
owners = {r["id"] for r in rows}
|
||||
t0 = time.perf_counter()
|
||||
objects = {}
|
||||
async for key, size, modified in s3.list_keys(prefix):
|
||||
@@ -41,23 +59,32 @@ async def reconcile(apply, grace, prefix):
|
||||
now = datetime.now(timezone.utc)
|
||||
cutoff = now - timedelta(seconds=grace)
|
||||
report = {"mode": "apply" if apply else "dry-run", "grace_seconds": grace, "db_rows": len(rows), "s3_objects": len(objects),
|
||||
"orphan_objects": [], "orphans_skipped_fresh": 0, "missing_objects": [], "size_mismatch": [], "fixed": {"deleted_orphans": 0, "marked_failed": 0}}
|
||||
"orphan_objects": [], "orphans_skipped_fresh": 0, "owned_objects_skipped": 0,
|
||||
"missing_objects": [], "size_mismatch": [], "fixed": {"deleted_orphans": 0, "marked_failed": 0}}
|
||||
for key, (size, modified) in objects.items():
|
||||
row = by_key.get(key)
|
||||
if row is None:
|
||||
if modified < cutoff:
|
||||
if object_owner(key) in owners:
|
||||
# Worker can write S3 variants before committing their metadata.
|
||||
report["owned_objects_skipped"] += 1
|
||||
elif modified < cutoff:
|
||||
report["orphan_objects"].append(key)
|
||||
else:
|
||||
report["orphans_skipped_fresh"] += 1
|
||||
elif row["status"] == "ready" and row["size_bytes"] != size:
|
||||
elif row["check"] and row["status"] == "ready" and row["size_bytes"] != size:
|
||||
report["size_mismatch"].append(str(row["id"]))
|
||||
for key, row in by_key.items():
|
||||
if key.startswith(prefix) and key not in objects and row["status"] == "ready" and row["created_at"] < cutoff:
|
||||
if row["check"] and key.startswith(prefix) and key not in objects and row["status"] == "ready" and row["created_at"] < cutoff:
|
||||
report["missing_objects"].append(str(row["id"]))
|
||||
for field in ("missing_objects", "size_mismatch"):
|
||||
report[field] = sorted(set(report[field]))
|
||||
if apply:
|
||||
for key in report["orphan_objects"]:
|
||||
# повторная проверка перед удалением: за время обхода строка могла появиться
|
||||
if await conn.fetchval("SELECT 1 FROM attachments WHERE s3_key = $1", key) is None:
|
||||
referenced = await conn.fetchval(
|
||||
"SELECT EXISTS (SELECT 1 FROM attachments WHERE s3_key = $1 OR staging_key = $1 OR id = $2) "
|
||||
"OR EXISTS (SELECT 1 FROM attachment_variants WHERE s3_key = $1)", key, object_owner(key))
|
||||
if not referenced:
|
||||
await s3.delete(key)
|
||||
report["fixed"]["deleted_orphans"] += 1
|
||||
for ids, code in ((report["missing_objects"], "object_missing"), (report["size_mismatch"], "size_mismatch")):
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Read-only readiness probe after restart: ranged GET of recent ready originals.
|
||||
|
||||
An open S3 socket / HEAD bucket does not prove that SeaweedFS volume locations
|
||||
have been registered. Read one byte from up to 16 recent ready originals.
|
||||
This is a bounded recovery hook, not an audit of every historical object.
|
||||
"""
|
||||
import asyncio
|
||||
import sys
|
||||
|
||||
import asyncpg
|
||||
|
||||
from . import config
|
||||
from .s3 import S3
|
||||
|
||||
|
||||
async def probe(s3):
|
||||
count = 0
|
||||
batch = []
|
||||
|
||||
async def read(key, size):
|
||||
params = {"Bucket": config.S3_BUCKET, "Key": key}
|
||||
if size:
|
||||
params["Range"] = "bytes=0-0"
|
||||
response = await s3.client.get_object(**params)
|
||||
async with response["Body"] as stream:
|
||||
data = await stream.read()
|
||||
if len(data) != min(size, 1):
|
||||
raise ValueError("unexpected ranged read size")
|
||||
|
||||
async def flush():
|
||||
results = await asyncio.gather(*(read(key, size) for key, size in batch), return_exceptions=True)
|
||||
for result in results:
|
||||
if isinstance(result, BaseException):
|
||||
raise result
|
||||
|
||||
await s3.ping()
|
||||
conn = await asyncpg.connect(config.DATABASE_URL, timeout=5)
|
||||
try:
|
||||
rows = await conn.fetch("SELECT s3_key, size_bytes FROM attachments "
|
||||
"WHERE status = 'ready' ORDER BY accepted_at DESC LIMIT 16")
|
||||
finally:
|
||||
await conn.close()
|
||||
for row in rows:
|
||||
key, size = row["s3_key"], row["size_bytes"]
|
||||
batch.append((key, size))
|
||||
count += 1
|
||||
if len(batch) == 8:
|
||||
await flush()
|
||||
batch.clear()
|
||||
if batch:
|
||||
await flush()
|
||||
return count
|
||||
|
||||
|
||||
async def main():
|
||||
s3 = S3()
|
||||
await s3.start()
|
||||
try:
|
||||
async with asyncio.timeout(45):
|
||||
while True:
|
||||
try:
|
||||
count = await probe(s3)
|
||||
print(f"PASS: S3 recovery, {count} recent ready originals readable")
|
||||
return 0
|
||||
except Exception as exc:
|
||||
# Do not print SDK requests/credentials or object contents.
|
||||
print(f"Waiting for S3 data volumes: {type(exc).__name__}", flush=True)
|
||||
await asyncio.sleep(1)
|
||||
except TimeoutError:
|
||||
print("FAIL: S3 data recovery exceeded 45 seconds")
|
||||
return 1
|
||||
finally:
|
||||
await s3.stop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(asyncio.run(main()))
|
||||
@@ -0,0 +1,32 @@
|
||||
# Собственные проверки ЛР6
|
||||
|
||||
Запустите стенд по REPORT.md. Python 3.11+ с `python -m pip install -r research/requirements.txt` включает Pillow для пиксельных проверок. Публичным tests/ достаточно Python 3.9+. В Windows используйте `PYTHONUTF8=1`.
|
||||
|
||||
Все сценарии выполняются **последовательно** на учебном стенде: часть тестов явно останавливает сервисы, внедряет отказы и восстанавливает настройки. Volumes не удаляются. Профиль для проверок наблюдаемости — OTEL_ENABLED=1, head sampling 1.0, обычный collector.yaml.
|
||||
|
||||
```sh
|
||||
python scripts/check_contract.py
|
||||
python scripts/validate_openapi.py # предварительно установить requirements-dev.txt
|
||||
python tests/smoke.py --base-url http://127.0.0.1:8080
|
||||
for suite in test_api test_idempotency test_attachments test_topology test_pipeline test_observability test_observability_retry test_reconcile5 test_resilience5 test_expiry test_dependencies; do
|
||||
python -m unittest discover -s student-tests -p "$suite.py" -v || exit 1
|
||||
done
|
||||
python tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url http://127.0.0.1:8080
|
||||
python tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url http://127.0.0.1:8080
|
||||
sh scripts/restore-replicas.sh
|
||||
```
|
||||
|
||||
В Windows публичным resilience передаются restart.cmd / stop-one.cmd; восстановление API — `docker compose up -d --wait --no-recreate --no-deps api`. Failover повторяется с восстановлением, чтобы проверить обе реплики. Фактические остановленные контейнеры и перезапуск всех сервисов сохранены в evidence/recheck_2026-11-12/.
|
||||
|
||||
Набор содержит 66 тестов: API 12, идемпотентность 9, вложения 11, топология 6, pipeline 9, наблюдаемость 5, retry trace 1, reconciler 2, отказы pipeline 7, TTL 2, зависимости 2. Проверки наблюдаемости ищут реальные metrics/logs/traces, связь API → producer → consumer → S3/DB, retry в той же трассе, низкую кардинальность и отсутствие синтетических секретов в сигналах.
|
||||
|
||||
Исследования для выбранного трека (запускать после тестов, не одновременно):
|
||||
|
||||
```sh
|
||||
mkdir -p evidence/my-run
|
||||
python research/incidents.py --out evidence/my-run
|
||||
python research/obs_cost.py --out evidence/my-run --profiles off on100 head10 tail --repeat 2
|
||||
python research/plot_obs_cost.py --out evidence/my-run
|
||||
```
|
||||
|
||||
Четыре контролируемых инцидента проверяют firing/resolved; off/on100/head10/tail сравниваются при одинаковых ресурсах, 64 клиентах, 8 процессах, 5 с прогрева и 30 с измерения (2 повтора). Эти прогоны занимают около 20 минут. Секреты создаются scripts/init-secrets.sh и в Git не входят. Все команды и реальные результаты текущей проверки — evidence/checks_2026-11-12.txt и evidence/recheck_2026-11-12/.
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Поведение при недоступности зависимостей: Valkey (кэш) и PostgreSQL (источник истины).
|
||||
|
||||
python student-tests/test_dependencies.py
|
||||
Контейнеры останавливаются и запускаются обратно (volumes сохраняются). Ожидания:
|
||||
* Valkey остановлен -> кэш обходится, API работает (чтение сессии из PostgreSQL), logout работает;
|
||||
* PostgreSQL остановлен -> 503 в контрактном формате с Retry-After (не 500, не «анонимный» доступ),
|
||||
/health/ready = 503, запрос без токена по-прежнему 401; после запуска БД API восстанавливается сам.
|
||||
"""
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from test_api import ROOT, basic, call, register # noqa: E402
|
||||
|
||||
|
||||
def compose(*args):
|
||||
return subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, check=True, capture_output=True, text=True)
|
||||
|
||||
|
||||
def wait(predicate, seconds=60):
|
||||
deadline = time.time() + seconds
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
if predicate():
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
raise RuntimeError("условие не выполнено за %d с" % seconds)
|
||||
|
||||
|
||||
class DependencyTest(unittest.TestCase):
|
||||
def tearDown(self):
|
||||
compose("start", "postgres", "valkey")
|
||||
wait(lambda: call("GET", "/health/ready")[0] == 200)
|
||||
|
||||
def test_valkey_down_is_bypassed(self):
|
||||
actor = register("dep")
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200)
|
||||
compose("stop", "valkey")
|
||||
started = time.time()
|
||||
for _ in range(5):
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200)
|
||||
self.assertLess(time.time() - started, 3, "circuit breaker не должен давать каждому запросу ждать таймаут")
|
||||
# вход и выход работают без кэша
|
||||
s = call("POST", "/api/v1/auth/sessions", headers=basic(actor["user"]["username"], actor["password"]))
|
||||
self.assertEqual(s[0], 201)
|
||||
self.assertEqual(call("DELETE", "/api/v1/auth/sessions/current", token=s[2]["token"])[0], 204)
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", token=s[2]["token"])[0], 401) # отзыв виден без кэша
|
||||
self.assertEqual(call("GET", "/health/ready")[0], 200) # кэш не критичен для readiness
|
||||
|
||||
def test_postgres_down_gives_bounded_503_never_anonymous_access(self):
|
||||
actor = register("dep")
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200) # токен в кэше
|
||||
compose("stop", "postgres")
|
||||
# Wait past HAProxy's fall=2 health checks: cover the all-unready state,
|
||||
# not just the short interval while the load balancer still routes normally.
|
||||
time.sleep(5)
|
||||
for method, path, body, token in (("GET", "/api/v1/users/me", None, actor["token"]),
|
||||
("GET", "/api/v1/chats", None, actor["token"]),
|
||||
("POST", "/api/v1/chats", {"title": "x", "member_ids": [actor["user"]["id"]]}, actor["token"]),
|
||||
("GET", "/api/v1/users/me", None, "Z" * 43)): # неизвестный токен: кэш пуст -> БД -> 503
|
||||
r = call(method, path, body, token)
|
||||
self.assertEqual(r[0], 503, (path, r))
|
||||
self.assertEqual(r[2]["error"]["code"], "unavailable")
|
||||
self.assertGreaterEqual(int(r[1]["Retry-After"]), 1)
|
||||
self.assertEqual(r[2]["error"]["request_id"], r[1]["X-Request-Id"])
|
||||
r = call("POST", "/api/v1/auth/sessions", headers=basic(actor["user"]["username"], actor["password"]))
|
||||
self.assertEqual(r[0], 503)
|
||||
self.assertEqual(call("GET", "/health/ready")[0], 503)
|
||||
self.assertEqual(call("GET", "/health/live")[0], 200)
|
||||
self.assertEqual(call("GET", "/api/v1/users/me")[0], 401) # без токена — по-прежнему 401
|
||||
compose("start", "postgres")
|
||||
wait(lambda: call("GET", "/health/ready")[0] == 200)
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200) # сессия пережила простой БД
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Проверка срока жизни сессии при коротком SESSION_TTL_SECONDS (по умолчанию 2).
|
||||
|
||||
python student-tests/test_expiry.py
|
||||
Скрипт пересоздаёт только контейнер api с SESSION_TTL_SECONDS=2, проверяет expiry и возвращает настройки
|
||||
по умолчанию. Данные и volumes не затрагиваются.
|
||||
"""
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from test_api import ROOT, call, register # noqa: E402
|
||||
|
||||
TTL = 2
|
||||
|
||||
|
||||
def compose(*args, env=None):
|
||||
return subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, check=True,
|
||||
capture_output=True, text=True, env=dict(os.environ, **(env or {})))
|
||||
|
||||
|
||||
def wait_ready(seconds=60):
|
||||
deadline = time.time() + seconds
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
if call("GET", "/health/ready")[0] == 200:
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
raise RuntimeError("api не стал ready")
|
||||
|
||||
|
||||
class ExpiryTest(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
compose("up", "-d", "--no-deps", "--force-recreate", "api", env={"SESSION_TTL_SECONDS": str(TTL)})
|
||||
wait_ready()
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
compose("up", "-d", "--no-deps", "--force-recreate", "api")
|
||||
wait_ready()
|
||||
|
||||
def test_session_expires_after_ttl(self):
|
||||
actor = register("exp")
|
||||
session = actor["session"]
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200) # валидна, кэш прогрет
|
||||
time.sleep(TTL + 1.5)
|
||||
r = call("GET", "/api/v1/users/me", token=actor["token"])
|
||||
self.assertEqual(r[0], 401, "после TTL токен должен быть отвергнут (и кэшем, и БД)")
|
||||
self.assertEqual(r[1]["WWW-Authenticate"], 'Bearer realm="messenger"')
|
||||
self.assertEqual(r[2]["error"]["code"], "unauthorized")
|
||||
self.assertEqual(call("DELETE", "/api/v1/auth/sessions/current", token=actor["token"])[0], 401)
|
||||
self.assertIn("expires_at", session)
|
||||
|
||||
def test_expiry_is_not_extended_by_use(self):
|
||||
actor = register("exp")
|
||||
for _ in range(3):
|
||||
call("GET", "/api/v1/users/me", token=actor["token"])
|
||||
time.sleep(0.5)
|
||||
time.sleep(1.5)
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 401)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,89 @@
|
||||
"""Reconciliation must preserve variants, in-flight effects and durable staging."""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import unittest
|
||||
import uuid
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from test_api import ROOT, call, register
|
||||
from test_attachments import PNG, download, upload
|
||||
from test_pipeline import wait_status
|
||||
|
||||
|
||||
def compose(*args):
|
||||
return subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT,
|
||||
check=True, capture_output=True, text=True, encoding='utf-8').stdout
|
||||
|
||||
|
||||
def reconcile(prefix, grace=0):
|
||||
return json.loads(compose('run', '--rm', '-T', 'api', 'python', '-m', 'app.reconcile',
|
||||
'--apply', '--grace', str(grace), '--prefix', prefix))
|
||||
|
||||
|
||||
def s3_action(action, key):
|
||||
code = ('import asyncio\nfrom app import config\nfrom app.s3 import S3\nasync def m():\n'
|
||||
' s=S3(); await s.start()\n'
|
||||
+ {'put': ' await s.put_bytes(%r, b"temporary", "application/octet-stream")\n',
|
||||
'head': ' print((await s.head(%r)) is not None)\n',
|
||||
'delete': ' await s.delete(%r)\n'}[action] % key
|
||||
+ ' await s.stop()\nasyncio.run(m())')
|
||||
return compose('exec', '-T', 'api', 'python', '-c', code).strip()
|
||||
|
||||
|
||||
class ReconcilePipelineTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.actor = register('reconcile')
|
||||
self.token = self.actor['token']
|
||||
self.chat = call('POST', '/api/v1/chats', {'title': 'reconcile', 'member_ids': [self.actor['user']['id']]}, self.token)[2]
|
||||
|
||||
def accept(self):
|
||||
status, _, att = upload(self.token, self.chat['id'], PNG)
|
||||
self.assertEqual(status, 202)
|
||||
return att
|
||||
|
||||
def test_ready_variants_and_uncommitted_side_effect_are_preserved(self):
|
||||
att = wait_status(self.token, self.accept()['id'])
|
||||
key = 'att/%s/v1/uncommitted' % att['id']
|
||||
s3_action('put', key)
|
||||
try:
|
||||
report = reconcile('att/%s/' % att['id'])
|
||||
self.assertEqual(report['fixed'], {'deleted_orphans': 0, 'marked_failed': 0})
|
||||
self.assertEqual(report['orphan_objects_count'], 0)
|
||||
self.assertEqual(report['owned_objects_skipped'], 1)
|
||||
self.assertEqual(s3_action('head', key), 'True')
|
||||
self.assertEqual(download(self.token, att['id'])[2], PNG)
|
||||
for variant in att['variants']:
|
||||
code, _, data = download(self.token, att['id'], '/content?variant=' + variant['name'])
|
||||
self.assertEqual(code, 200)
|
||||
self.assertEqual(hashlib.sha256(data).hexdigest(), variant['sha256'])
|
||||
finally:
|
||||
s3_action('delete', key)
|
||||
|
||||
def test_staging_is_protected_and_unowned_orphan_obeys_grace(self):
|
||||
compose('stop', 'worker')
|
||||
att = None
|
||||
orphan = 'staging/' + str(uuid.uuid4())
|
||||
try:
|
||||
att = self.accept()
|
||||
report = reconcile('staging/' + att['id'])
|
||||
self.assertEqual(report['fixed'], {'deleted_orphans': 0, 'marked_failed': 0})
|
||||
self.assertEqual(s3_action('head', 'staging/' + att['id']), 'True')
|
||||
s3_action('put', orphan)
|
||||
fresh = reconcile(orphan, grace=900)
|
||||
self.assertEqual(fresh['orphans_skipped_fresh'], 1)
|
||||
self.assertEqual(fresh['fixed']['deleted_orphans'], 0)
|
||||
self.assertEqual(reconcile(orphan)['fixed']['deleted_orphans'], 1)
|
||||
self.assertEqual(s3_action('head', orphan), 'False')
|
||||
self.assertEqual(reconcile(orphan)['fixed']['deleted_orphans'], 0)
|
||||
finally:
|
||||
s3_action('delete', orphan)
|
||||
compose('start', 'worker')
|
||||
self.assertEqual(wait_status(self.token, att['id'])['status'], 'ready')
|
||||
self.assertEqual(download(self.token, att['id'])[2], PNG)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -56,7 +56,7 @@ class ResilienceTest(unittest.TestCase):
|
||||
compose("up", "-d", "--no-deps", "rabbitmq", "api")
|
||||
|
||||
def setUp(self):
|
||||
compose("start", "rabbitmq", "postgres", "minio", "valkey", check=False)
|
||||
compose("start", "rabbitmq", "postgres", "seaweedfs", "valkey")
|
||||
wait(lambda: healthy("rabbitmq") and healthy("postgres"), 90, "rabbitmq/postgres healthy")
|
||||
wait(api_ready, 60, "api ready")
|
||||
self.a = register("res")
|
||||
|
||||
Reference in New Issue
Block a user