prepared skeleton for lab5

This commit is contained in:
2026-11-07 16:10:32 +03:00
parent 276dbdad8a
commit 0a70358829
135 changed files with 9647 additions and 34 deletions
+1
View File
@@ -0,0 +1 @@
"""Команды управления учебным стендом."""
+355
View File
@@ -0,0 +1,355 @@
"""Проверяет инфраструктуру, SIGTERM и обновление реального стенда."""
import asyncio
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
import hashlib
import json
from pathlib import Path
import subprocess
from threading import Event
import time
from uuid import uuid4
import asyncpg
from scripts.deployment import ComposeProject, ROOT, deployment_lock
from scripts.failover_all import exercise, failure_kind, history
from scripts.rolling_update import update
from tests.client import Client, Failure, arguments, check, run
PROBE = """import json, sys, urllib.request, urllib.error
payload = json.load(sys.stdin)
results = []
for item in payload:
data = json.dumps(item['body']).encode() if 'body' in item else None
request = urllib.request.Request(item['url'], data=data, headers=item.get('headers', {}), method=item.get('method', 'GET'))
try:
response = urllib.request.urlopen(request, timeout=7)
except urllib.error.HTTPError as error:
response = error
with response:
results.append({'status': response.status, 'body': json.loads(response.read())})
print(json.dumps(results))
"""
def probe(project, runner, requests):
"""Обращается к внутреннему API, передавая токен только через stdin."""
result = subprocess.run(
["docker", "exec", "-i", runner, "python", "-c", PROBE],
input=json.dumps(requests),
text=True,
capture_output=True,
timeout=10,
)
check(result.returncode == 0, "internal HTTP probe failed")
return json.loads(result.stdout)
def audit(project):
"""Проверяет фактические лимиты, порты и область доступа к секретам."""
allowed = {
"postgres": {"db_password"},
"redis": {"redis_password"},
"migrate": {"db_password"},
"api": {"db_password", "redis_password", "s3_access_key_id", "s3_secret_access_key"},
"minio": {"minio_root_user", "minio_root_password"},
"minio-bootstrap": {"minio_root_user", "minio_root_password", "s3_access_key_id", "s3_secret_access_key"},
"cleanup": {"db_password", "s3_access_key_id", "s3_secret_access_key"},
"nginx": set(),
}
ids = project.run("ps", "--all", "--quiet").splitlines()
containers = json.loads(project.docker("inspect", *ids))
for item in containers:
service = item["Config"]["Labels"]["com.docker.compose.service"]
check(service in allowed, "unexpected service")
host = item["HostConfig"]
check(
host["Memory"] > 0 and host["NanoCpus"] > 0,
service + ": missing CPU/RAM limit",
)
env_names = {entry.split("=", 1)[0] for entry in item["Config"]["Env"]}
check(
not env_names
& {
"DB_PASSWORD",
"REDIS_PASSWORD",
"POSTGRES_PASSWORD",
"REDISCLI_AUTH",
"DATABASE_URL",
"MINIO_ROOT_USER", "MINIO_ROOT_PASSWORD", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY",
},
service + ": password in environment",
)
secrets = {
Path(mount["Destination"]).name
for mount in item["Mounts"]
if mount["Destination"].startswith("/run/secrets/")
}
check(secrets == allowed[service], service + ": incorrect secret scope")
if service in {"api", "minio"}:
check(not host["PortBindings"], service + " publishes ports")
for mount in item["Mounts"]:
if mount["Destination"].startswith("/run/secrets/"):
check(not mount["RW"], "writable secret mount")
check(
"container_name" not in project.config["services"][service],
"fixed container name",
)
check(
len(containers) == len(allowed) - 1 + len(project.replicas()),
"unexpected container count",
)
print(
"PASS infrastructure: CPU/RAM, private API ports, scoped read-only file secrets, no environment passwords",
flush=True,
)
async def graceful(project, base_url):
"""Удерживает реальный POST в БД и завершает его после SIGTERM."""
replicas = project.replicas()
target, runner = replicas[:2]
client = Client(base_url)
actor, _, _, chat = await asyncio.to_thread(client.bootstrap)
path = f"/api/v1/chats/{chat['id']}/messages"
key = uuid4().hex
raw = json.dumps([actor["id"], "POST", path, key]).encode()
lock_id = int.from_bytes(hashlib.sha256(raw).digest()[:8], signed=True)
database = project.config["services"]["postgres"]
env = database["environment"]
conn = await asyncpg.connect(
host="127.0.0.1",
port=int(database["ports"][0]["published"]),
database=env["POSTGRES_DB"],
user=env["POSTGRES_USER"],
password=(ROOT / "secrets/db_password").read_text().strip(),
timeout=3,
)
network = project.config['networks']['default']['name']
address_template = '{{with index .NetworkSettings.Networks ' + json.dumps(network) + '}}{{.IPAddress}}{{end}}'
address = project.docker(
"inspect",
"--format",
address_template,
target.id,
)
check(bool(address), 'API has no address in the application network')
port = project.config["services"]["api"]["environment"]["APP_PORT"]
direct = f"http://{address}:{port}"
started = time.monotonic()
task = None
try:
async with conn.transaction():
await conn.execute("SELECT pg_advisory_xact_lock($1)", lock_id)
task = asyncio.create_task(
asyncio.to_thread(
probe,
project,
runner.id,
[
{
"url": direct + path,
"method": "POST",
"body": {"text": "graceful-" + key},
"headers": {
"Authorization": "Bearer " + actor["token"],
"Idempotency-Key": key,
"Content-Type": "application/json",
},
}
],
)
)
deadline = time.monotonic() + 3
while not await conn.fetchval(
"SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE wait_event = 'advisory' AND datname = current_database())"
):
check(time.monotonic() < deadline, "POST did not reach database lock")
await asyncio.sleep(0.02)
started = time.monotonic()
await asyncio.to_thread(
project.docker, "kill", "--signal", "SIGTERM", target.id
)
results = await asyncio.to_thread(
probe,
project,
runner.id,
[
{"url": direct + "/health/ready"},
{"url": direct + "/health/live"},
{"url": direct + "/api/v1/users/me"},
],
)
check(
[item["status"] for item in results] == [503, 200, 503],
"incorrect drain readiness/liveness/new work",
)
await asyncio.sleep(max(0, 1.2 - (time.monotonic() - started)))
result = (await task)[0]
check(result["status"] == 201, "in-flight POST failed during SIGTERM")
await asyncio.to_thread(project.docker, "wait", target.id, timeout=10)
state = json.loads(
project.docker("inspect", "--format", "{{json .State}}", target.id)
)
check(
not state["OOMKilled"] and state["ExitCode"] != 137,
"forced shutdown instead of graceful completion",
)
check(time.monotonic() - started < 8, "shutdown exceeded Docker grace budget")
items = await asyncio.to_thread(history, client, actor, chat)
check(
result["body"]["id"] in {item["id"] for item in items},
"acknowledged in-flight message lost",
)
print(
f"PASS SIGTERM: readiness 503, live 200, new work 503, in-flight POST 201; exit in {time.monotonic() - started:.3f}s",
flush=True,
)
finally:
await conn.close()
if task:
await asyncio.gather(task, return_exceptions=True)
if any(item.id == target.id and item.running for item in project.replicas()):
await asyncio.to_thread(project.docker, "stop", target.id)
await asyncio.to_thread(project.docker, "start", target.id)
await asyncio.to_thread(project.wait_healthy, {item.id for item in replicas})
await asyncio.to_thread(project.wait_routed, target.instance_id, base_url)
class ObservedProject(ComposeProject):
"""Подсчитывает миграции и проверяет готовность перед заменой."""
def __init__(self):
self.migrations = 0
self.replacements = 0
self.replacement_started = []
super().__init__()
def run(self, *args, **kwargs):
if args[:6] == ("run", "--rm", "--no-deps", "--pull", "never", "migrate"):
self.migrations += 1
return super().run(*args, **kwargs)
def docker(self, *args, **kwargs):
if args[0] == "stop":
healthy = [
item
for item in self.replicas()
if item.id != args[1] and item.running and item.health == "healthy"
]
check(len(healthy) >= 2, "rolling update reduces healthy capacity")
self.replacements += 1
self.replacement_started.append(time.monotonic())
return super().docker(*args, **kwargs)
def rolling(base_url):
"""Заменяет образ под постоянной записью с проверкой сохранности."""
project = ObservedProject()
before = project.replicas()
client = Client(base_url)
actor, _, _, chat = client.bootstrap()
stop = Event()
results, errors = [], []
def worker():
local = Client(base_url, timeout=2)
key = uuid4().hex
while not stop.is_set():
started = time.monotonic()
try:
result = local.message(actor, chat, "rolling-" + key, key=key)
results.append(
(started, time.monotonic(), result["json"]["id"], result["elapsed"])
)
key = uuid4().hex
except Failure as error:
errors.append(failure_kind(error))
stop.wait(0.08)
with ThreadPoolExecutor(max_workers=2) as pool:
jobs = [pool.submit(worker) for _ in range(2)]
try:
time.sleep(1)
started = time.monotonic()
update(project, base_url, revision="verification-" + uuid4().hex[:12])
finished = time.monotonic()
time.sleep(1)
finally:
stop.set()
for job in jobs:
job.result(timeout=5)
after = project.replicas()
check(
project.migrations == 1 and project.replacements == len(before),
"migration/replacement count incorrect",
)
check(
{item.image for item in before}.isdisjoint({item.image for item in after}),
"new image was not deployed",
)
check(
{item.id for item in before}.isdisjoint({item.id for item in after}),
"old containers remain",
)
writes = [row for row in results if started <= row[0] <= finished]
check(bool(writes), "no writes during update")
check(
any(project.replacement_started[0] <= row[0] <= finished for row in writes),
"no writes during container replacement",
)
items = history(client, actor, chat)
check(
{row[2] for row in results} <= {item["id"] for item in items},
"rolling update lost confirmed messages",
)
check(
len(items) == len({item["text"] for item in items}),
"rolling update created duplicate messages",
)
latencies = sorted(row[3] for row in writes)
print(
json.dumps(
{
"scenario": "rolling update",
"replacements": project.replacements,
"migration_runs": project.migrations,
"successful_writes": len(writes),
"failed_attempts": len(errors),
"errors_by_kind": dict(Counter(errors)),
"elapsed_seconds": round(finished - started, 3),
"successful_rps": round(len(writes) / (finished - started), 3),
"p95_seconds": round(
latencies[min(len(latencies) - 1, int(len(latencies) * 0.95))], 3
),
}
),
flush=True,
)
def main():
"""Выполняет проверки только на отдельном тестовом проекте."""
def configure(parser):
parser.add_argument(
"--phase", choices=("all", "update-and-failover"), default="all"
)
args = arguments("Grade 4 integration checks", configure)
project = ComposeProject()
check(project.name.endswith("-test"), "use isolated test Compose project")
with deployment_lock(project):
audit(project)
if args.phase == "all":
asyncio.run(graceful(project, args.base_url))
rolling(args.base_url)
args.duration, args.concurrency = 10, 2
exercise(project, args)
audit(project)
if __name__ == "__main__":
run(main)
+39
View File
@@ -0,0 +1,39 @@
"""Проверяет конкурентные повторы через балансировщик."""
from concurrent.futures import ThreadPoolExecutor
from uuid import uuid4
from tests.client import Client, arguments, check, run
def main() -> None:
"""Отправляет один ключ параллельно через разные реплики."""
args = arguments("Concurrent idempotency through load balancer")
client = Client(args.base_url, args.ca_file)
alice, _, _, chat = client.bootstrap()
key = uuid4().hex
body = "parallel-" + key
def send(_):
return Client(args.base_url, args.ca_file).message(alice, chat, body, key=key)
with ThreadPoolExecutor(max_workers=8) as pool:
results = list(pool.map(send, range(24)))
ids = {result["json"]["id"] for result in results}
replicas = {result["headers"]["X-Instance-Id"] for result in results}
check(len(ids) == 1, "parallel retries created duplicate messages")
check(len(replicas) >= 2, "parallel retries did not reach two API replicas")
history = client.call(
"GET",
f"/api/v1/chats/{chat['id']}/messages?limit=100",
actor=alice,
schema="MessagePage",
)["json"]["items"]
check(
len(history) == 1 and history[0] == results[0]["json"], "history has duplicates"
)
print(f"PASS: 24 concurrent replays, {len(replicas)} replicas, one message")
if __name__ == "__main__":
run(main)
+261
View File
@@ -0,0 +1,261 @@
#!/usr/bin/env python3
"""Проверки реального nginx в отдельном Compose-проекте без БД и приложения."""
from concurrent.futures import ThreadPoolExecutor
import gzip
import json
from pathlib import Path
import subprocess
import tempfile
import time
from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen
from uuid import uuid4
ROOT = Path(__file__).resolve().parents[1]
IMAGE = "nginx:1.28.0-alpine"
PAYLOAD = json.dumps({"text": "message " * 256}).encode()
def check(condition: bool, message: str) -> None:
if not condition:
raise AssertionError(message)
class Stand:
def __init__(self, directory: str) -> None:
self.directory = Path(directory)
self.project = "mtusi-lab3-nginx-check-" + uuid4().hex[:12]
self.base_url = ""
backend = self.directory / "backend.conf"
backend.write_text(
"worker_processes 1;\n"
"events { worker_connections 1024; }\n"
"http {\n"
" log_format backend 'BACKEND $request_method $uri';\n"
" access_log /dev/stdout backend;\n"
" server {\n"
" listen 8080;\n"
" default_type application/json;\n"
" add_header X-Instance-Id $hostname always;\n"
" add_header X-Request-Id $request_id always;\n"
" add_header Retry-After 7 always;\n"
' location /health/ { return 200 \'{"status":"ok"}\'; }\n'
" location = /api/limited {\n"
' return 429 \'{"error":{"code":"rate_limited",'
'"message":"upstream limit","request_id":"$request_id"}}\';\n'
" }\n"
" location = /api/post-failure {\n"
' return 503 \'{"error":{"code":"unavailable",'
'"message":"upstream failure","request_id":"$request_id"}}\';\n'
" }\n"
" location / { return 200 '" + PAYLOAD.decode() + "'; }\n"
" }\n"
"}\n"
)
self.compose_file = self.directory / "compose.json"
self.compose_file.write_text(
json.dumps(
{
"services": {
"api": {
"image": IMAGE,
"volumes": [f"{backend}:/etc/nginx/nginx.conf:ro"],
},
"nginx": {
"image": IMAGE,
"volumes": [
f"{ROOT / 'infra/nginx/nginx.conf'}:/etc/nginx/nginx.conf:ro"
],
"ports": ["127.0.0.1::8080"],
},
}
}
)
)
def compose(self, *args: str) -> str:
# Явный файл, уникальный проект и пустой env-file исключают основной стенд.
result = subprocess.run(
[
"docker",
"compose",
"--env-file",
"/dev/null",
"-f",
str(self.compose_file),
"-p",
self.project,
*args,
],
cwd=self.directory,
capture_output=True,
text=True,
timeout=45,
)
if result.returncode:
raise RuntimeError(result.stdout + result.stderr)
return result.stdout + result.stderr
def call(self, path: str, *, method: str = "GET", encoding: str = "identity"):
request = Request(
self.base_url + path,
method=method,
headers={"Accept-Encoding": encoding},
data=b"{}" if method == "POST" else None,
)
try:
response = urlopen(request, timeout=3)
except HTTPError as error:
response = error
except (URLError, TimeoutError):
return 0, {}, b""
with response:
return (
response.status,
{key.lower(): value for key, value in response.headers.items()},
response.read(),
)
def wait_instances(self, count: int) -> set[str]:
seen: set[str] = set()
deadline = time.monotonic() + 15
while time.monotonic() < deadline:
status, headers, _ = self.call("/health/live")
if status == 200:
seen.add(headers["x-instance-id"])
if len(seen) == count:
return seen
time.sleep(0.05)
raise AssertionError(f"Ожидалось {count} реплик, наблюдалось {len(seen)}")
def check_error(response, status: int, code: str) -> None:
actual_status, headers, body = response
check(actual_status == status, f"Ожидался HTTP {status}, получен {actual_status}")
check(
headers.get("content-type", "").startswith("application/json"), "Ошибка не JSON"
)
error = json.loads(body)["error"]
check(error["code"] == code, "Неверный код ошибки")
check(error["request_id"] == headers.get("x-request-id"), "request_id не совпадает")
def run_checks(stand: Stand) -> None:
stand.compose("up", "-d", "--pull", "never", "--scale", "api=2", "api", "nginx")
address = stand.compose("port", "nginx", "8080").strip()
stand.base_url = "http://" + address
stand.wait_instances(2)
print(stand.compose("exec", "-T", "nginx", "nginx", "-v").strip())
stand.compose("exec", "-T", "nginx", "nginx", "-t")
print("PASS: nginx -t и балансировка двух реплик")
status, headers, body = stand.call("/api/large", encoding="gzip")
check(
status == 200 and headers.get("content-encoding") == "gzip", "Нет gzip для JSON"
)
check(gzip.decompress(body) == PAYLOAD, "gzip изменил содержимое")
check(len(body) < len(PAYLOAD), "gzip не уменьшил тестовый ответ")
check(
"accept-encoding" in headers.get("vary", "").lower(),
"Нет Vary: Accept-Encoding",
)
check(stand.call("/api/large")[2] == PAYLOAD, "Ответ без gzip изменён")
small = stand.call("/health/live", encoding="gzip")
check(
small[0] == 200 and "content-encoding" not in small[1], "Маленький ответ сжат"
)
print(f"PASS: gzip {len(PAYLOAD)} → {len(body)} байт, содержимое сохранено")
limited = stand.call("/api/limited")
check_error(limited, 429, "rate_limited")
check(limited[1].get("retry-after") == "7", "Подменён Retry-After приложения")
check("x-instance-id" in limited[1], "Потерян id backend на ошибке API")
print("PASS: JSON и заголовки ошибки API передаются без подмены")
failure = stand.call("/api/post-failure", method="POST")
check_error(failure, 503, "unavailable")
backend_logs = stand.compose("logs", "--no-color", "--no-log-prefix", "api")
check(backend_logs.count("BACKEND POST /api/post-failure") == 1, "POST повторился")
print("PASS: отправленный POST не повторяется на другой реплике")
with ThreadPoolExecutor(max_workers=32) as pool:
responses = list(pool.map(lambda _: stand.call("/api/large"), range(500)))
rejected = [response for response in responses if response[0] == 429]
check(bool(rejected), "Не удалось вызвать ограничение частоты")
check(
all(response[0] in (200, 429) for response in responses), "Ошибка при нагрузке"
)
for response in rejected:
check_error(response, 429, "rate_limited")
check(response[1].get("retry-after") == "1", "Нет Retry-After балансировщика")
check("x-instance-id" not in response[1], "LB выдал себя за API-реплику")
with ThreadPoolExecutor(max_workers=16) as pool:
health = list(pool.map(lambda _: stand.call("/health/ready"), range(100)))
check(all(response[0] == 200 for response in health), "Лимит затронул health")
check(stand.call("/lb-health")[0] == 200, "Проверка LB недоступна")
print(f"PASS: лимит вернул {len(rejected)} ответов 429; health доступен")
stand.compose(
"up", "-d", "--pull", "never", "--scale", "api=3", "--no-recreate", "api"
)
instances = stand.wait_instances(3)
print("PASS: третья реплика обнаружена через DNS без reload nginx")
target = sorted(instances)[0]
containers = stand.compose("ps", "-q", "api").splitlines()
matching = [container for container in containers if container.startswith(target)]
check(len(matching) == 1, "Не найдена тестовая реплика для остановки")
# Все id взяты только из уникального временного Compose-проекта.
started = time.monotonic()
subprocess.run(
["docker", "stop", "--time", "1", matching[0]], check=True, capture_output=True
)
recovery = None
for _ in range(20):
status, headers, _ = stand.call("/health/live")
if status == 200:
check(headers["x-instance-id"] != target, "Остановленная реплика отвечает")
if recovery is None:
recovery = time.monotonic() - started
check(recovery is not None and recovery < 10, "Чтение не восстановилось за 10 с")
print(f"PASS: отказ реплики, первое успешное чтение через {recovery:.3f} с")
stand.compose("stop", "--timeout", "1", "api")
unavailable = stand.call("/health/ready")
check_error(unavailable, 503, "unavailable")
check(
unavailable[1].get("retry-after") == "1", "Нет Retry-After при отказе всех API"
)
check("x-instance-id" not in unavailable[1], "У ошибки LB есть id реплики")
check(stand.call("/lb-health")[0] == 200, "Отказ API сломал проверку самого LB")
print("PASS: недоступность всех backend возвращает JSON 503")
logs = stand.compose("logs", "--no-color", "--no-log-prefix", "nginx")
entries = [json.loads(line) for line in logs.splitlines() if line.startswith("{")]
check(bool(entries), "Нет JSON access log")
check(
all("request_id" in entry and "upstream_time" in entry for entry in entries),
"Неполный лог",
)
check(
any(entry["limit_status"] == "REJECTED" for entry in entries),
"Лимит не отражён в логе",
)
print("PASS: JSON access log содержит время backend и результат лимита")
def main() -> None:
with tempfile.TemporaryDirectory(prefix="mtusi-lab3-nginx-check-") as directory:
stand = Stand(directory)
try:
run_checks(stand)
except Exception:
print(stand.compose("logs", "--no-color", "--tail", "20"), flush=True)
raise
finally:
stand.compose("down", "--remove-orphans", "--timeout", "1")
if __name__ == "__main__":
main()
+137
View File
@@ -0,0 +1,137 @@
"""Реальный MinIO: повтор bootstrap, private bucket, ограниченные права и отказ."""
import json
import base64
from pathlib import Path
import subprocess
import time
from scripts.deployment import ComposeProject
from tests.client import Client, arguments, check, run
PROBE = r'''
import asyncio, json, sys
from pathlib import Path
from uuid import UUID
from urllib.request import Request, urlopen
from urllib.error import HTTPError, URLError
from app.core.config import Settings
from app.core.database import create_db_engine, create_session_factory
from app.storage.s3 import create_s3_storage
from app.models.attachment import Attachment
from minio.error import S3Error
async def main():
settings = Settings()
engine = create_db_engine(settings)
storage = create_s3_storage(settings.s3)
try:
async with create_session_factory(engine)() as session:
record = await session.get(Attachment, UUID(json.load(sys.stdin)['attachment_id']))
key = record.object_key
results = {}
for method in ('GET', 'PUT'):
request = Request(settings.s3.endpoint + '/' + settings.s3.bucket + '/' + key,
method=method, data=b'x' if method == 'PUT' else None)
try:
with urlopen(request, timeout=3) as response:
results[method] = response.status
except HTTPError as error:
results[method] = error.code
error.close()
try:
storage._client.get_bucket_policy(settings.s3.bucket)
results['admin_denied'] = False
except S3Error as error:
results['admin_denied'] = error.code == 'AccessDenied'
try:
buckets = storage._client.list_buckets()
results['other_buckets_denied'] = all(bucket.name == settings.s3.bucket for bucket in buckets)
except S3Error as error:
results['other_buckets_denied'] = error.code == 'AccessDenied'
try:
with urlopen(settings.s3.endpoint.replace(':9000', ':9001'), timeout=1):
results['console_unreachable'] = False
except HTTPError as error:
results['console_unreachable'] = False
error.close()
except URLError:
results['console_unreachable'] = True
results['temp_originals_absent'] = not any(Path('/tmp').glob('messenger-upload-*'))
results['root_files_absent'] = not any(Path('/run/secrets', name).exists()
for name in ('minio_root_user', 'minio_root_password'))
print(json.dumps(results))
finally:
await storage.aclose()
await engine.dispose()
asyncio.run(main())
'''
def main():
def configure(parser):
parser.add_argument('--with-outage', action='store_true', help='временно остановить MinIO выбранного проекта')
parser.add_argument('--with-faults', action='store_true', help='проверить компенсацию и очистку на реальных S3/БД')
args = arguments('MinIO private bucket and failure checks', configure)
project = ComposeProject()
client = Client(args.base_url, args.ca_file, timeout=40)
alice, _, _, chat = client.bootstrap()
attachment, original = client.attachment(alice, chat)
# Повторный bootstrap не должен очистить bucket или сменить используемые credentials.
for _ in range(2):
project.run('run', '--rm', '--no-deps', 'minio-bootstrap', timeout=100)
replicas = project.replicas()
check(bool(replicas), 'API not running')
result = subprocess.run(['docker', 'exec', '-i', replicas[0].id, 'python', '-c', PROBE],
input=json.dumps({'attachment_id': attachment['id']}),
text=True, capture_output=True, timeout=20)
check(result.returncode == 0, 'private bucket probe failed (details hidden)')
statuses = json.loads(result.stdout)
check(statuses['GET'] == statuses['PUT'] == 403, 'anonymous S3 access allowed')
check(statuses['admin_denied'] and statuses['other_buckets_denied'] and statuses['root_files_absent'], 'API has excessive access')
check(statuses['console_unreachable'], 'console reachable from API network')
check(statuses['temp_originals_absent'], 'API retained temporary originals')
check(not project.config['services']['minio'].get('ports'), 'MinIO publishes host ports')
env = project.config['services']['minio']['environment']
check(env['MINIO_BROWSER'] == 'off', 'browser enabled')
check('127.0.0.1:9001' in project.config['services']['minio']['command'], 'console is not bound to loopback')
path = '/api/v1/attachments/' + attachment['id'] + '/content'
check(client.call('GET', path, actor=alice)['bytes'] == original, 'bootstrap changed original')
if args.with_faults:
project.run('stop', 'cleanup')
try:
fault_script = Path(__file__).with_name('s3_faults.py').read_text()
result = subprocess.run(
['docker', 'exec', '-i', replicas[0].id, 'python', '-c', fault_script],
input=json.dumps({'user': alice['user'], 'chat_id': chat['id'],
'image': base64.b64encode(original).decode()}),
text=True, capture_output=True, timeout=60,
)
check(result.returncode == 0, 'real S3/database compensation probe failed (details hidden)')
check(all(json.loads(result.stdout).values()), 'real S3/database invariants failed')
finally:
project.run('start', 'cleanup')
check(client.call('GET', path, actor=alice)['bytes'] == original, 'cleanup damaged confirmed original')
if args.with_outage:
try:
project.run('stop', 'minio')
client.call('GET', path, expected=503, actor=alice)
finally:
project.run('start', 'minio')
deadline = time.monotonic() + 60
while True:
try:
client.call('GET', '/health/ready', schema='Health')
break
except Exception:
if time.monotonic() >= deadline:
raise
time.sleep(0.5)
check(client.call('GET', path, actor=alice)['bytes'] == original, 'original lost after MinIO restart')
print('PASS: idempotent bootstrap, anonymous GET/PUT denied, restricted API credentials, private console'
+ (', MinIO outage/recovery' if args.with_outage else ''))
if args.with_faults:
print('PASS: real S3 PUT/database failure, compensation, durable journal, grace period, dry-run, repeated cleanup')
if __name__ == '__main__':
run(main)
+204
View File
@@ -0,0 +1,204 @@
"""Повторяемый прогон lab4; публикует только успешные обезличенные evidence."""
from datetime import datetime, timezone
import hashlib
from importlib.metadata import version
import json
import os
from pathlib import Path
import platform
import re
import shlex
import subprocess
from tempfile import TemporaryDirectory
from zoneinfo import ZoneInfo
ROOT = Path(__file__).resolve().parents[1]
PROJECT = 'mtusi-messenger-lab5-test'
EVIDENCE = ROOT / 'evidence'
ENVIRONMENT = {
**os.environ,
'COMPOSE_PROJECT_NAME': PROJECT,
'COMPOSE_FILE': 'compose.yaml:compose.test.yaml',
'LB_HOST_PORT': '28080',
}
COMPOSE = ['docker', 'compose', '--env-file', '.env']
NONSECRET_SETTINGS = {
'APP_PORT', 'API_REPLICAS', 'MAX_MESSAGES', 'MAX_IN_FLIGHT',
'DB_POOL_SIZE', 'DB_MAX_OVERFLOW', 'SESSION_TTL_SECONDS',
'IDEMPOTENCY_TTL_SECONDS', 'SHUTDOWN_DRAIN_SECONDS',
'GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS', 'READINESS_TIMEOUT_SECONDS',
'S3_ENDPOINT', 'S3_BUCKET', 'S3_REGION', 'S3_CONNECT_TIMEOUT_SECONDS',
'S3_READ_TIMEOUT_SECONDS', 'S3_UPLOAD_TIMEOUT_SECONDS',
'S3_DOWNLOAD_TIMEOUT_SECONDS', 'S3_IMAGE_TIMEOUT_SECONDS',
'S3_MAX_UPLOAD_BYTES', 'S3_MAX_IMAGE_PIXELS', 'S3_CHUNK_BYTES',
'S3_MAX_CONCURRENT_UPLOADS', 'S3_CLEANUP_GRACE_SECONDS',
'S3_CLEANUP_INTERVAL_SECONDS',
}
SNAPSHOT_FORMAT = '{"id":{{json .Id}},"name":{{json .Name}},"service":{{json (index .Config.Labels "com.docker.compose.service")}},"image_id":{{json .Image}},"status":{{json .State.Status}},"started_at":{{json .State.StartedAt}},"pid":{{json .State.Pid}},"exit_code":{{json .State.ExitCode}},"oom_killed":{{json .State.OOMKilled}}}'
IMAGE_FORMAT = '{"id":{{json .Id}},"repo_digests":{{json .RepoDigests}},"architecture":{{json .Architecture}},"os":{{json .Os}}}'
def timestamp():
return datetime.now(timezone.utc).isoformat()
def invoke(command):
return subprocess.run(command, cwd=ROOT, env=ENVIRONMENT, text=True,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
def read_command(command):
result = invoke(command)
if result.returncode:
raise RuntimeError('Не выполнена команда: ' + shlex.join(command))
return result.stdout.strip()
def source_manifest():
names = {
'.dockerignore', '.env.example',
'Dockerfile', 'Makefile', 'alembic.ini', 'compose.yaml',
'compose.test.yaml', 'pytest.ini', 'requirements.txt', 'requirements-dev.txt', 'lab.json',
}
for directory in ('app', 'migrations', 'student-tests', 'scripts', 'infra', 'contracts', 'tests'):
names.update(str(path.relative_to(ROOT)) for path in (ROOT / directory).rglob('*')
if path.is_file() and '__pycache__' not in path.parts
and path.suffix != '.md' and not path.name.startswith('.'))
return ''.join(f'{hashlib.sha256((ROOT / name).read_bytes()).hexdigest()} {name}\n' for name in sorted(names))
def sanitize(text, secrets, staging):
clean = text.replace(str(ROOT), '.').replace(str(staging), '<temporary>')
for secret in secrets:
clean = clean.replace(secret, '<redacted>')
clean = re.sub(r'(?i)\b(Bearer|Basic)\s+[A-Za-z0-9._~+/=-]+', r'\1 <redacted>', clean)
clean = re.sub(r"(?i)([\"'](?:token|password|secret_access_key)[\"']\s*:\s*[\"'])[^\"']*", r'\1<redacted>', clean)
return clean
def container_snapshot(images):
ids = read_command(['docker', 'ps', '-aq', '--filter', f'label=com.docker.compose.project={PROJECT}']).splitlines()
rows = []
if ids:
output = read_command(['docker', 'inspect', '--format', SNAPSHOT_FORMAT, *ids])
rows = [json.loads(line) for line in output.splitlines()]
for row in rows:
image = row['image_id']
if image not in images:
images[image] = json.loads(read_command(['docker', 'image', 'inspect', '--format', IMAGE_FORMAT, image]))
return rows
def remaining_resources():
return {resource: read_command(['docker', resource, 'ls', '-aq' if resource == 'container' else '-q', '--filter',
f'label=com.docker.compose.project={PROJECT}']).splitlines()
for resource in ('container', 'network', 'volume')}
def extract_metrics(output, upload_file):
rows = [json.loads(line) for line in output.splitlines() if line.startswith('{') and line.endswith('}')]
pytest = re.search(r'(\d+) passed(?:, [^\n]*?)? in ([\d.]+)s', output)
if pytest is None:
raise RuntimeError('В выводе нет успешного итогового результата pytest')
return {
'pytest': {'passed': int(pytest[1]), 'elapsed_seconds': float(pytest[2])},
'uploads': json.loads(upload_file.read_text()),
'public_failover': [row for row in rows if 'observed_instances_before' in row],
'rolling_update': [row for row in rows if row.get('scenario') == 'rolling update'],
'replica_failures': [row for row in rows if 'first_recovery_seconds' in row and 'instance' in row],
}
def main():
if any(remaining_resources().values()):
raise RuntimeError('Тестовый проект уже существует; сначала завершите его отдельно')
started = timestamp()
manifest = source_manifest()
secrets = [path.read_text().strip() for path in (ROOT / 'secrets').iterdir() if path.is_file()]
secrets = [value for value in secrets if value]
snapshots, images, commands, combined, nginx_output = {}, {}, [], [], ''
info = json.loads(read_command(['docker', 'info', '--format', '{{json .}}']))
config = json.loads(read_command([*COMPOSE, 'config', '--format', 'json']))
services = {name: {
'image': service.get('image'),
'limits': service.get('deploy', {}).get('resources', {}).get('limits', {}),
'settings': {key: value for key, value in service.get('environment', {}).items() if key in NONSECRET_SETTINGS},
'ports': service.get('ports', []), 'networks': list(service.get('networks', {})),
'secret_names': [entry['source'] for entry in service.get('secrets', [])],
'tmpfs': service.get('tmpfs', []),
} for name, service in config['services'].items()}
with TemporaryDirectory(prefix='lab4-evidence-') as folder:
staging = Path(folder)
upload_file = staging / 'uploads.json'
def execute(command):
begin = timestamp()
result = invoke(command)
output = sanitize(result.stdout, secrets, staging)
display = sanitize(shlex.join(command), secrets, staging)
commands.append({'command': display, 'started_at_utc': begin,
'finished_at_utc': timestamp(), 'exit_code': result.returncode})
print(f'{"PASS" if result.returncode == 0 else "FAIL"}: {display}', flush=True)
if result.returncode:
print(output[-5000:], flush=True)
raise RuntimeError('Прогон остановлен; неуспешные evidence не опубликованы')
return f'$ {display}\n{output}\n'
cleanup_output = ''
try:
combined.append(execute(['make', 'check', 'validate', 'test-stand-up', f'TEST_PROJECT={PROJECT}']))
snapshots['started'] = container_snapshot(images)
combined.append(execute(['make', 'test-docker', f'TEST_PROJECT={PROJECT}', 'PYTEST_ARGS=--tb=short']))
combined.append(execute(['make', 'test-stand-smoke', 'test-stand-idempotency', f'TEST_PROJECT={PROJECT}']))
snapshots['before_restart'] = container_snapshot(images)
combined.append(execute(['make', 'test-stand-persistence', f'TEST_PROJECT={PROJECT}']))
snapshots['after_restart'] = container_snapshot(images)
combined.append(execute(['make', 'test-stand-failover', f'TEST_PROJECT={PROJECT}']))
snapshots['after_failover'] = container_snapshot(images)
combined.append(execute(['.venv/bin/python', '-m', 'scripts.check_s3', '--base-url', 'http://localhost:28080', '--with-outage', '--with-faults']))
combined.append(execute(['.venv/bin/python', '-m', 'scripts.upload_load', '--base-url', 'http://localhost:28080', '--concurrency', '10', '--output', str(upload_file)]))
combined.append(execute(['.venv/bin/python', '-m', 'scripts.check_grade4', '--base-url', 'http://localhost:28080']))
snapshots['after_update_and_failures'] = container_snapshot(images)
nginx_output = execute(['make', 'test-nginx'])
finally:
cleanup_output = execute([*COMPOSE, 'down', '--volumes', '--remove-orphans'])
remaining = remaining_resources()
if any(remaining.values()):
raise RuntimeError('После очистки остались ресурсы тестового проекта')
if source_manifest() != manifest:
raise RuntimeError('Во время прогона изменились проверяемые исходники')
output = ''.join(combined)
metrics = extract_metrics(output, upload_file)
run = {
'started_at_utc': started, 'finished_at_utc': timestamp(),
'started_at_moscow': datetime.fromisoformat(started).astimezone(ZoneInfo('Europe/Moscow')).isoformat(),
'base_commit': read_command(['git', 'rev-parse', 'HEAD']),
'worktree_dirty': bool(read_command(['git', 'status', '--porcelain'])),
'source_manifest_sha256': hashlib.sha256(manifest.encode()).hexdigest(),
'project': PROJECT, 'runtime': {'python': platform.python_version(),
'packages': {name: version(name) for name in ('fastapi', 'SQLAlchemy', 'pydantic', 'minio', 'Pillow', 'pytest')},
'compose': read_command(['docker', 'compose', 'version'])},
'docker_host': {key: info[key] for key in ('ServerVersion', 'OperatingSystem', 'Architecture', 'NCPU', 'MemTotal', 'CgroupVersion')},
'services': services, 'container_snapshots': snapshots, 'images': images,
'commands': commands, 'remaining_test_resources': remaining,
'minio_source_release': 'RELEASE.2025-10-15T17-29-55Z',
'mc_source_release': 'RELEASE.2025-08-13T08-35-41Z',
}
artifacts = {
'test-output.txt': output, 'nginx-output.txt': nginx_output,
'cleanup.txt': cleanup_output, 'source-sha256.txt': manifest,
'metrics.json': json.dumps(metrics, ensure_ascii=False, indent=2) + '\n',
'run.json': json.dumps(run, ensure_ascii=False, indent=2) + '\n',
}
for name, content in artifacts.items():
if any(secret in content for secret in secrets):
raise RuntimeError('Обнаружено значение секрета; evidence не опубликованы')
EVIDENCE.mkdir(exist_ok=True)
for name, content in artifacts.items():
(EVIDENCE / name).write_text(content)
print('Evidence опубликованы; тестовые ресурсы удалены.', flush=True)
if __name__ == '__main__':
main()
+107
View File
@@ -0,0 +1,107 @@
"""Общие операции с репликами текущего Compose-проекта."""
from contextlib import contextmanager
from dataclasses import dataclass
import fcntl
import json
import os
from pathlib import Path
import ssl
import subprocess
import tempfile
import time
from urllib.error import HTTPError, URLError
from urllib.request import urlopen
ROOT = Path(__file__).resolve().parents[1]
@dataclass(frozen=True)
class Replica:
"""Состояние контейнера API."""
id: str
image: str
instance_id: str
running: bool
health: str | None
class ComposeProject:
"""Работает только с выбранным Compose-проектом."""
def __init__(self) -> None:
self.command = ["docker", "compose", "--env-file", ".env"]
self.config = json.loads(self.run("config", "--format", "json"))
self.name = self.config["name"]
def docker(self, *args: str, timeout: int = 90, env=None) -> str:
"""Выполняет Docker-команду с ограничением времени."""
result = subprocess.run(
["docker", *args], cwd=ROOT, text=True, capture_output=True,
timeout=timeout, env={**os.environ, **(env or {})},
)
if result.returncode:
raise RuntimeError(result.stderr.strip() or result.stdout.strip())
return result.stdout.strip()
def run(self, *args: str, timeout: int = 90, env=None) -> str:
"""Выполняет команду Compose с текущими overrides."""
return self.docker(*self.command[1:], *args, timeout=timeout, env=env)
def replicas(self) -> list[Replica]:
"""Читает состояние API без окружения контейнеров."""
ids = self.run("ps", "--all", "--quiet", "api").splitlines()
if not ids:
return []
template = (
'{"id":{{json .Id}},"image":{{json .Image}},'
'"hostname":{{json .Config.Hostname}},"running":{{json .State.Running}},'
'"health":{{if .State.Health}}{{json .State.Health.Status}}{{else}}null{{end}}}'
)
rows = self.docker("inspect", "--format", template, *ids).splitlines()
configured = str(self.config["services"]["api"].get("environment", {}).get("INSTANCE_ID") or "").strip()
return [
Replica(row["id"], row["image"], configured or row["hostname"], row["running"], row["health"])
for row in map(json.loads, rows)
]
def wait_healthy(self, ids: set[str], timeout: float = 40) -> None:
"""Ожидает готовность конкретных реплик."""
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
current = {replica.id: replica for replica in self.replicas()}
if not ids <= current.keys():
raise RuntimeError("Контейнер API исчез во время развёртывания")
if all(current[id].running and current[id].health == "healthy" for id in ids):
return
if any(not current[id].running for id in ids):
raise RuntimeError("Контейнер API завершился до готовности")
time.sleep(0.25)
raise TimeoutError("API не стал healthy за отведённое время")
def wait_routed(self, instance_id: str, base_url: str, ca_file=None) -> None:
"""Ожидает ответа реплики через балансировщик."""
context = ssl.create_default_context(cafile=ca_file)
deadline = time.monotonic() + 40
while time.monotonic() < deadline:
try:
with urlopen(base_url.rstrip("/") + "/health/live", timeout=3, context=context) as response:
if response.status == 200 and response.headers.get("X-Instance-Id") == instance_id:
return
except (HTTPError, URLError, TimeoutError, OSError):
pass
time.sleep(0.1)
raise TimeoutError("Реплика не появилась в ответах балансировщика")
@contextmanager
def deployment_lock(project: ComposeProject):
"""Исключает параллельные изменения одного стенда."""
path = Path(tempfile.gettempdir()) / f"{project.name}-deployment.lock"
with path.open("a") as lock:
try:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
raise RuntimeError("Другой процесс уже изменяет этот Compose-проект") from None
yield
+28
View File
@@ -0,0 +1,28 @@
#!/bin/sh
set -eu
BASE_URL="${BASE_URL:-http://localhost:8080}"
TTL="${TTL:?укажите TTL, с которым запущен API (SESSION_TTL_SECONDS)}"
API="$BASE_URL/api/v1"
USERNAME="expiry_$(date +%s)"
PASSWORD="expiry-password-123"
code() {
curl -s -o /dev/null -w '%{http_code}' "$@"
}
curl -sf -o /dev/null -X POST "$API/users" -H 'Content-Type: application/json' \
-d "{\"username\":\"$USERNAME\",\"display_name\":\"Expiry\",\"password\":\"$PASSWORD\"}"
SESSION=$(curl -sf -X POST "$API/auth/sessions" -u "$USERNAME:$PASSWORD")
TOKEN=$(printf '%s' "$SESSION" | python3 -c 'import json, sys; print(json.load(sys.stdin)["token"])')
EXPIRES_AT=$(printf '%s' "$SESSION" | python3 -c 'import json, sys; print(json.load(sys.stdin)["expires_at"])')
AUTH="Authorization: Bearer $TOKEN"
echo "SESSION_TTL_SECONDS=$TTL, login at $(date -u +%H:%M:%S)Z, expires_at=$EXPIRES_AT"
echo "GET /users/me right after login -> $(code "$API/users/me" -H "$AUTH")"
sleep $((TTL + 1))
printf "%-35s -> %s\n" "GET /users/me after $((TTL + 1)) s" "$(code "$API/users/me" -H "$AUTH")"
echo "DELETE /auth/sessions/current after -> $(code -X DELETE "$API/auth/sessions/current" -H "$AUTH")"
NEW_TOKEN=$(curl -sf -X POST "$API/auth/sessions" -u "$USERNAME:$PASSWORD" \
| python3 -c 'import json, sys; print(json.load(sys.stdin)["token"])')
echo "GET /users/me with new login token -> $(code "$API/users/me" -H "Authorization: Bearer $NEW_TOKEN")"
+132
View File
@@ -0,0 +1,132 @@
"""EXPLAIN (ANALYZE, BUFFERS) запроса истории на синтетических данных.
Данные создаются в одной транзакции и откатываются в конце: база не меняется.
"""
import asyncio
from uuid import uuid4
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncConnection
from app.core.config import Settings
from app.core.database import create_db_engine
TARGET_MESSAGES = 10_000
NOISE_MESSAGES = 40_000
PAGE = 51
HISTORY_FIRST_PAGE = """
SELECT * FROM messages
WHERE chat_id = :chat_id
ORDER BY created_at, id
LIMIT :page
"""
HISTORY_AFTER_CURSOR = """
SELECT * FROM messages
WHERE chat_id = :chat_id AND (created_at, id) > (:after_created_at, :after_id)
ORDER BY created_at, id
LIMIT :page
"""
async def seed(connection: AsyncConnection) -> dict[str, object]:
user_id, target_id, noise_id = uuid4(), uuid4(), uuid4()
await connection.execute(
text(
"INSERT INTO users (id, username, display_name, password_hash) "
"VALUES (:id, :username, 'Explain', 'x')"
),
{"id": user_id, "username": "explain_" + uuid4().hex[:8]},
)
await connection.execute(
text("INSERT INTO chats (id, title) VALUES (:a, 'target'), (:b, 'noise')"),
{"a": target_id, "b": noise_id},
)
await connection.execute(
text("INSERT INTO chat_members (chat_id, user_id) VALUES (:a, :u), (:b, :u)"),
{"a": target_id, "b": noise_id, "u": user_id},
)
for chat_id, count in ((target_id, TARGET_MESSAGES), (noise_id, NOISE_MESSAGES)):
await connection.execute(
text(
"INSERT INTO messages (id, chat_id, sender_id, text, created_at) "
"SELECT gen_random_uuid(), :chat_id, :user_id, 'message ' || g, "
"now() - make_interval(secs => :count - g) "
"FROM generate_series(1, :count) AS g"
),
{"chat_id": chat_id, "user_id": user_id, "count": count},
)
await connection.execute(text("ANALYZE messages"))
middle = (
await connection.execute(
text(
"SELECT created_at, id FROM messages WHERE chat_id = :c "
"ORDER BY created_at, id OFFSET :o LIMIT 1"
),
{"c": target_id, "o": TARGET_MESSAGES // 2},
)
).one()
return {
"chat_id": target_id,
"after_created_at": middle.created_at,
"after_id": middle.id,
"page": PAGE,
}
async def explain(
connection: AsyncConnection, title: str, query: str, params: dict[str, object]
) -> None:
rows = await connection.execute(
text(f"EXPLAIN (ANALYZE, BUFFERS, COSTS OFF, TIMING ON) {query}"), params
)
print(f"== {title}")
print(query.strip())
print()
for row in rows:
print(row[0])
print()
async def main() -> None:
engine = create_db_engine(Settings()) # type: ignore[call-arg]
async with engine.connect() as connection:
transaction = await connection.begin()
params = await seed(connection)
total = (
await connection.execute(text("SELECT count(*) FROM messages"))
).scalar_one()
print(
f"messages in table: {total}; target chat: {TARGET_MESSAGES}; noise chat: {NOISE_MESSAGES}"
)
print()
first_page = {"chat_id": params["chat_id"], "page": PAGE}
await explain(
connection, "first page (limit+1 = 51)", HISTORY_FIRST_PAGE, first_page
)
await explain(
connection,
"page after cursor in the middle of history",
HISTORY_AFTER_CURSOR,
params,
)
await connection.execute(text("SET LOCAL enable_indexscan = off"))
await connection.execute(text("SET LOCAL enable_bitmapscan = off"))
await explain(
connection,
"same page with index scans disabled (for comparison)",
HISTORY_AFTER_CURSOR,
params,
)
await transaction.rollback()
await engine.dispose()
if __name__ == "__main__":
asyncio.run(main())
+148
View File
@@ -0,0 +1,148 @@
"""Сценарий поочерёдного отказа всех API-реплик под записью."""
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
import json
import os
import re
import subprocess
from threading import Event, Lock
import time
from urllib.parse import quote
from uuid import uuid4
from scripts.deployment import ComposeProject, ROOT, deployment_lock
from tests.client import Client, Failure, arguments, check, run
def failure_kind(error: Failure) -> str:
"""Различает HTTP-отказы, ошибки соединения и проверки контракта."""
match = re.search(r"received (\d{3})$", str(error))
if match:
return "http_" + match.group(1)
return "connection" if "connection/TLS failure" in str(error) else "contract"
def snapshot(project: ComposeProject, phase: str, target: str) -> list:
"""Печатает состояние API без окружения и секретов."""
replicas = project.replicas()
print(json.dumps({
"scenario": "replica_failure", "phase": phase, "target": target,
"replicas": [{"instance": item.instance_id, "running": item.running, "health": item.health} for item in replicas],
}), flush=True)
return replicas
def history(client: Client, actor: dict, chat: dict) -> list[dict]:
"""Читает всю историю без повторов курсора."""
path = f"/api/v1/chats/{chat['id']}/messages?limit=100"
items = []
cursors = set()
for _ in range(1000):
page = client.call("GET", path, actor=actor, schema="MessagePage")["json"]
items.extend(page["items"])
cursor = page["next_cursor"]
if cursor is None:
return items
check(cursor not in cursors, "cursor cycle")
cursors.add(cursor)
path = f"/api/v1/chats/{chat['id']}/messages?limit=100&cursor={quote(cursor, safe='')}"
raise Failure("history exceeded safety bound")
def exercise(project: ComposeProject, args) -> None:
"""Останавливает каждую реплику и восстанавливает её перед следующей."""
replicas = project.replicas()
check(len(replicas) >= 2, "at least two API replicas are required")
project.wait_healthy({item.id for item in replicas})
client = Client(args.base_url, args.ca_file, timeout=2)
for item in replicas:
project.wait_routed(item.instance_id, client.base, args.ca_file)
actor, _, _, chat = client.bootstrap()
confirmed = set()
for target in replicas:
snapshot(project, "before", target.instance_id)
stop = Event()
lock = Lock()
successes = []
errors = []
def worker():
local = Client(client.base, args.ca_file, timeout=2)
key = uuid4().hex
while not stop.is_set():
started = time.monotonic()
try:
result = local.message(actor, chat, "failover-" + key, key=key)
with lock:
successes.append((started, time.monotonic(), result["json"]["id"]))
key = uuid4().hex
except Failure as error:
with lock:
errors.append(failure_kind(error))
time.sleep(0.1)
started = time.monotonic()
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
jobs = [pool.submit(worker) for _ in range(args.concurrency)]
try:
time.sleep(2)
stop_started = time.monotonic()
subprocess.run(
[str(ROOT / "scripts/stop-one.sh")], cwd=ROOT,
env={**os.environ, "TARGET_INSTANCE_ID": target.instance_id},
check=True, timeout=10, capture_output=True,
)
stop_finished = time.monotonic()
stopped = snapshot(project, "stopped", target.instance_id)
check(any(item.id == target.id and not item.running for item in stopped), "target API was not stopped")
check(any(item.id != target.id and item.running and item.health == "healthy" for item in stopped), "no healthy surviving API")
time.sleep(args.duration)
finally:
stop.set()
try:
for job in jobs:
job.result(timeout=5)
finally:
project.docker("start", target.id)
project.wait_healthy({item.id for item in replicas})
project.wait_routed(target.instance_id, client.base, args.ca_file)
snapshot(project, "restored", target.instance_id)
elapsed = max(end for _, end, _ in successes) - started if successes else 0
recovered = [end for begin, end, _ in successes if begin >= stop_finished]
check(bool(recovered), "no confirmed writes after failure")
recovery = min(recovered) - stop_started
check(recovery <= 10, "writes did not recover within 10 seconds")
confirmed.update(id for _, _, id in successes)
items = history(client, actor, chat)
check(confirmed <= {item["id"] for item in items}, "confirmed messages were lost")
texts = [item["text"] for item in items]
check(len(texts) == len(set(texts)), "uncertain retry created duplicates")
print(json.dumps({
"instance": target.instance_id,
"successful_writes": len(successes),
"failed_attempts": len(errors),
"errors_by_kind": dict(Counter(errors)),
"elapsed_seconds": round(elapsed, 3),
"successful_rps": round(len(successes) / elapsed, 3) if elapsed else 0,
"first_recovery_seconds": round(recovery, 3),
}), flush=True)
def main() -> None:
"""Готовит параметры эксперимента отказа реплик."""
def configure(parser):
parser.add_argument("--duration", type=float, default=15)
parser.add_argument("--concurrency", type=int, default=2)
args = arguments("Fail each API replica under write load", configure)
check(1 <= args.duration <= 120 and 1 <= args.concurrency <= 8, "invalid workload")
project = ComposeProject()
with deployment_lock(project):
exercise(project, args)
if __name__ == "__main__":
run(main)
+64
View File
@@ -0,0 +1,64 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
BASE_URL="${BASE_URL:-http://localhost:8080}"
API="$BASE_URL/api/v1"
USERNAME="demo_$(date +%s)"
PASSWORD="demo-password-123"
compose() {
docker compose --env-file .env "$@"
}
code() {
curl -s -o /dev/null -w '%{http_code} (%{time_total}s)' "$@"
}
wait_healthy() {
attempts=0
until [ "$(compose ps --format '{{.Health}}' "$1")" = "healthy" ]; do
attempts=$((attempts + 1))
[ "$attempts" -lt 60 ] || { echo "$1 не стал healthy" >&2; exit 1; }
sleep 1
done
}
curl -sf -o /dev/null -X POST "$API/users" -H 'Content-Type: application/json' \
-d "{\"username\":\"$USERNAME\",\"display_name\":\"Demo\",\"password\":\"$PASSWORD\"}"
TOKEN=$(curl -sf -X POST "$API/auth/sessions" -u "$USERNAME:$PASSWORD" \
| python3 -c 'import json, sys; print(json.load(sys.stdin)["token"])')
AUTH="Authorization: Bearer $TOKEN"
echo "== baseline"
echo "GET /users/me (Bearer) -> $(code "$API/users/me" -H "$AUTH")"
echo
echo "== redis stopped"
compose stop redis >/dev/null 2>&1
echo "GET /users/me (Bearer) -> $(code "$API/users/me" -H "$AUTH")"
echo "POST /auth/sessions (Basic) -> $(code -X POST "$API/auth/sessions" -u "$USERNAME:$PASSWORD")"
echo "GET /users/me (no token) -> $(code "$API/users/me")"
echo "GET /health/ready -> $(code "$BASE_URL/health/ready")"
echo "GET /health/live -> $(code "$BASE_URL/health/live")"
compose start redis >/dev/null 2>&1
wait_healthy redis
echo "-- redis started again (AOF restored)"
echo "GET /users/me (same token) -> $(code "$API/users/me" -H "$AUTH")"
echo
echo "== postgres stopped"
compose stop postgres >/dev/null 2>&1
echo "GET /users/me (Bearer) -> $(code "$API/users/me" -H "$AUTH")"
echo "GET /api/v1/chats (Bearer) -> $(code "$API/chats" -H "$AUTH")"
echo "GET /health/ready -> $(code "$BASE_URL/health/ready")"
echo "GET /health/live -> $(code "$BASE_URL/health/live")"
compose start postgres >/dev/null 2>&1
wait_healthy postgres
echo "-- postgres started again"
echo "GET /users/me (same token) -> $(code "$API/users/me" -H "$AUTH")"
echo
echo "== logout"
echo "DELETE /auth/sessions/current -> $(code -X DELETE "$API/auth/sessions/current" -H "$AUTH")"
echo "GET /users/me (same token) -> $(code "$API/users/me" -H "$AUTH")"
+30
View File
@@ -0,0 +1,30 @@
"""Создаёт только отсутствующие локальные настройки и файловые секреты."""
import os
from pathlib import Path
import secrets
def main() -> None:
root = Path(__file__).resolve().parents[1]
destination = root / ".env"
if not destination.exists():
with destination.open("x") as target:
target.write((root / ".env.example").read_text())
destination.chmod(0o600)
directory = root / "secrets"
directory.mkdir(mode=0o700, exist_ok=True)
for name in ("db_password", "redis_password", "minio_root_user", "minio_root_password", "s3_access_key_id", "s3_secret_access_key"):
path = directory / name
try:
descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o444)
except FileExistsError:
continue
with os.fdopen(descriptor, "w") as target:
target.write(secrets.token_urlsafe(24))
print("Создан файл " + str(path.relative_to(root)))
print("Настройки готовы; существующие значения сохранены")
if __name__ == "__main__":
main()
+32
View File
@@ -0,0 +1,32 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
compose() {
docker compose --env-file .env "$@"
}
wait_healthy() {
for service in "$@"; do
attempts=0
until states=$(compose ps --format '{{.Health}}' "$service") \
&& [ -n "$states" ] \
&& [ -z "$(printf '%s\n' "$states" | grep -v '^healthy$')" ]; do
attempts=$((attempts + 1))
if [ "$attempts" -ge 60 ]; then
echo "$service не стал healthy за 60 секунд" >&2
exit 1
fi
sleep 1
done
done
}
compose stop cleanup
compose restart postgres redis minio
wait_healthy postgres redis minio
compose restart api
wait_healthy api
compose restart nginx
wait_healthy nginx
compose start cleanup
+92
View File
@@ -0,0 +1,92 @@
"""Поочерёдно заменяет API с одной дополнительной репликой."""
import argparse
import sys
from scripts.deployment import ComposeProject, deployment_lock
def update(project: ComposeProject, base_url: str, ca_file=None, revision: str | None = None) -> None:
"""Собирает образ, применяет миграции и заменяет реплики."""
original = project.replicas()
if len(original) < 2 or any(not item.running or item.health != "healthy" for item in original):
raise RuntimeError("До обновления нужны минимум две здоровые API-реплики")
old_images = {item.image for item in original}
if len(old_images) != 1:
raise RuntimeError("Сначала восстановите единый образ работающих реплик")
old_image = old_images.pop()
profile = project.docker(
"image", "inspect", old_image,
"--format", '{{index .Config.Labels "io.mtusi.messenger.secrets"}}',
)
if profile != "file-v1":
raise RuntimeError("Сначала выполните первоначальный переход на файловые секреты через make up")
count = len(original)
print("Сборка нового образа API", flush=True)
build_args = ("--build-arg", f"API_BUILD_REVISION={revision}") if revision else ()
project.run("build", *build_args, "api", timeout=300)
tag = project.config["services"]["api"]["image"]
new_image = project.docker("image", "inspect", tag, "--format", "{{.Id}}")
print("Однократное применение совместимых миграций", flush=True)
project.run("run", "--rm", "--no-deps", "--pull", "never", "migrate")
if old_image == new_image:
print("Образ не изменился; замена контейнеров не требуется")
return
try:
for old in original:
project.run(
"up", "-d", "--no-deps", "--no-build", "--pull", "never",
"--no-recreate", "--scale", f"api={count + 1}", "api",
)
current = project.replicas()
additions = [item for item in current if item.id not in {replica.id for replica in original} and item.image == new_image]
if not additions:
raise RuntimeError("Новая реплика не создана")
project.wait_healthy({item.id for item in additions})
for added in additions:
project.wait_routed(added.instance_id, base_url, ca_file)
survivors = {item.id for item in project.replicas() if item.id != old.id}
project.wait_healthy(survivors)
project.docker("stop", old.id)
project.docker("rm", old.id)
print(f"Заменена реплика {old.instance_id}", flush=True)
final = project.replicas()
if len(final) != count or any(item.image != new_image for item in final):
raise RuntimeError("Состав реплик после обновления не соответствует плану")
project.wait_healthy({item.id for item in final})
except Exception:
print("Обновление прервано; восстанавливается прежний образ без отката БД", file=sys.stderr)
project.run(
"up", "-d", "--no-deps", "--no-build", "--pull", "never",
"--force-recreate", "--scale", f"api={count}", "api",
env={"API_IMAGE": old_image},
)
project.wait_healthy({item.id for item in project.replicas()})
raise
dangling = project.docker("image", "ls", "--quiet", "--no-trunc", "--filter", "dangling=true").splitlines()
if old_image in dangling:
references = project.docker("ps", "--all", "--quiet", "--filter", f"ancestor={old_image}")
if not references:
try:
project.docker("image", "rm", old_image)
except RuntimeError:
print("Прежний образ не удалён: состояние Docker изменилось", file=sys.stderr)
print(f"Готово: {count} здоровых реплик нового образа")
def main() -> None:
"""Запускает обновление с блокировкой проекта."""
parser = argparse.ArgumentParser(description="Rolling update of API replicas")
parser.add_argument("--base-url", default="http://localhost:8080")
parser.add_argument("--ca-file", default=None)
parser.add_argument("--revision", default=None)
args = parser.parse_args()
project = ComposeProject()
with deployment_lock(project):
update(project, args.base_url or "http://localhost:8080", args.ca_file, args.revision)
if __name__ == "__main__":
main()
+108
View File
@@ -0,0 +1,108 @@
"""Исполняется внутри API-контейнера: реальные S3/БД и управляемые отказы."""
import asyncio
import base64
from datetime import datetime, timedelta, timezone
import json
import sys
from uuid import UUID
from minio.error import S3Error
from sqlalchemy import select
from sqlalchemy.exc import OperationalError
from app.core.config import Settings
from app.core.database import create_db_engine, create_session_factory
from app.domain import attachments as domain
from app.errors.exceptions import unavailable
from app.models.attachment import UploadIntent
from app.schemas.user import User
from app.storage import attachments as repository
from app.storage.s3 import create_s3_storage
async def object_exists(storage, key):
exists = True
try:
await storage.stat_object(key)
except S3Error as error:
if error.code not in {'NoSuchKey', 'NoSuchObject'}:
raise
exists = False
return exists
async def main():
payload = json.load(sys.stdin)
settings = Settings()
engine = create_db_engine(settings)
factory = create_session_factory(engine)
storage = create_s3_storage(settings.s3)
user = User.model_validate(payload['user'])
chat_id = UUID(payload['chat_id'])
original = base64.b64decode(payload['image'])
keys = []
create_record = repository.create_attachment
put_file, delete_object = storage.put_file, storage.delete_object
async def capture_put(key, path, **metadata):
keys.append(key)
await put_file(key, path, **metadata)
async def fail_metadata(*args, **kwargs):
raise OperationalError('injected metadata failure after real S3 PUT', {}, None)
async def fail_delete(*args):
raise unavailable('injected compensation failure')
async def chunks():
yield original
async def attempt():
raised = False
async with factory() as session:
try:
await domain.upload_attachment(session, storage, user, chat_id,
filename='fault.png', content_type='image/png', chunks=chunks())
except OperationalError:
raised = True
assert raised, 'injected database failure was not raised'
storage.put_file = capture_put
repository.create_attachment = fail_metadata
try:
await attempt()
assert not await object_exists(storage, keys[-1]), 'compensation left original'
async with factory() as session:
intent = await session.scalar(select(UploadIntent).where(UploadIntent.object_key == keys[-1]))
assert intent is None, 'compensated upload left intent'
storage.delete_object = fail_delete
await attempt()
key = keys[-1]
assert await object_exists(storage, key), 'fault did not leave real S3 object'
storage.delete_object = delete_object
async with factory() as session:
result = await domain.reconcile_attachments(session, storage, dry_run=False)
assert result.deleted == 0, 'fresh object deleted before grace period'
intent = await session.scalar(select(UploadIntent).where(UploadIntent.object_key == key))
assert intent is not None, 'durable intent missing'
intent.created_at = datetime.now(timezone.utc) - timedelta(days=2)
await session.commit()
dry_run = await domain.reconcile_attachments(session, storage)
assert dry_run.candidates == 1 and dry_run.deleted == 0
assert await object_exists(storage, key), 'dry-run removed original'
applied = await domain.reconcile_attachments(session, storage, dry_run=False)
assert applied.deleted == 1 and not await object_exists(storage, key)
repeated = await domain.reconcile_attachments(session, storage, dry_run=False)
assert repeated.examined == repeated.deleted == 0
print(json.dumps({'compensation': True, 'durable_intent': True, 'grace_period': True,
'dry_run': True, 'cleanup': True, 'idempotent_cleanup': True}))
finally:
repository.create_attachment = create_record
storage.put_file, storage.delete_object = put_file, delete_object
await storage.aclose()
await engine.dispose()
if __name__ == '__main__':
asyncio.run(main())
+30
View File
@@ -0,0 +1,30 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
case "$TARGET_INSTANCE_ID" in
*[!a-zA-Z0-9_.:-]*|'') exit 1 ;;
esac
target=""
for container in $(docker compose --env-file .env ps -q api); do
hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container")
configured=$(docker inspect --format '{{range .Config.Env}}{{if eq (index (split . "=") 0) "INSTANCE_ID"}}{{println .}}{{end}}{{end}}' "$container" \
| sed -n 's/^INSTANCE_ID=//p' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
if [ "$TARGET_INSTANCE_ID" = "$hostname" ] || [ "$TARGET_INSTANCE_ID" = "$configured" ]; then
if [ -n "$target" ]; then
echo 'Instance id matches multiple API containers' >&2
exit 1
fi
target="$container"
fi
done
if [ -z "$target" ]; then
echo 'Target API instance not found in this Compose project' >&2
exit 1
fi
echo "Stopping API container $target" >&2
docker stop --timeout 8 "$target" >/dev/null
+113
View File
@@ -0,0 +1,113 @@
"""Повторяемые параллельные upload и память cgroup API, без записи токенов."""
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
import hashlib
from io import BytesIO
import json
from pathlib import Path
import random
import subprocess
from threading import Barrier
import time
from PIL import Image
from scripts.deployment import ComposeProject
from tests.client import Client, Failure, arguments, check, run
MONITOR = r'''
import json, pathlib, sys, threading
root = pathlib.Path('/sys/fs/cgroup')
def current():
return int((root / 'memory.current').read_text())
def peak():
path = root / 'memory.peak'
return int(path.read_text()) if path.exists() else None
stop = threading.Event()
threading.Thread(target=lambda: (sys.stdin.read(), stop.set()), daemon=True).start()
initial, before = current(), peak()
sampled = initial
print('ready', flush=True)
while not stop.wait(0.02):
sampled = max(sampled, current())
print(json.dumps({'initial_bytes': initial, 'sampled_peak_bytes': sampled,
'cgroup_peak_before_bytes': before, 'cgroup_peak_after_bytes': peak(), 'interval_ms': 20}), flush=True)
'''
def main():
def configure(parser):
parser.add_argument('--concurrency', type=int, default=10)
parser.add_argument('--rounds', type=int, default=3)
parser.add_argument('--output', type=Path)
args = arguments('Concurrent attachment uploads and API memory', configure)
check(10 <= args.concurrency <= 32 and 1 <= args.rounds <= 10, 'use concurrency 10..32 and rounds 1..10')
project = ComposeProject()
client = Client(args.base_url, args.ca_file, timeout=65)
alice, _, _, chat = client.bootstrap()
random_bytes = random.Random(42).randbytes(1500 * 1500 * 3)
image = Image.frombytes('RGB', (1500, 1500), random_bytes)
buffer = BytesIO()
image.save(buffer, format='PNG')
data = buffer.getvalue()
check(len(data) <= 10485760, 'fixture exceeds upload limit')
monitors = []
measurements = {}
rows = []
started = time.monotonic()
try:
for replica in project.replicas():
process = subprocess.Popen(['docker', 'exec', '-i', replica.id, 'python', '-c', MONITOR],
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
monitors.append((replica.instance_id, process))
check(process.stdout.readline().strip() == 'ready', 'memory monitor failed')
check(bool(monitors), 'API not running')
for _ in range(args.rounds):
barrier = Barrier(args.concurrency)
def upload(index):
local = Client(args.base_url, args.ca_file, timeout=65)
barrier.wait(timeout=15)
request_started = time.monotonic()
try:
return local.call('POST', f"/api/v1/chats/{chat['id']}/attachments?filename=load.png",
expected=[201, 400, 401, 404, 409, 413, 415, 429, 500, 502, 503, 504], actor=alice, raw=data,
headers={'Content-Type': 'image/png'}, schema='Attachment')
except Failure:
# При раннем отказе сервер может закрыть ещё отправляемое тело.
return {'status': 'failed_request', 'elapsed': time.monotonic() - request_started}
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
rows.extend(pool.map(upload, range(args.concurrency)))
finally:
elapsed = time.monotonic() - started
for alias, process in monitors:
output, _ = process.communicate(input='', timeout=10)
check(process.returncode == 0, 'memory monitor failed')
measurements[alias] = json.loads(output)
succeeded = [row for row in rows if row['status'] == 201]
check(bool(succeeded), 'all uploads were rejected')
digest = hashlib.sha256(data).hexdigest()
for row in succeeded:
check(row['json']['sha256'] == digest, 'upload checksum differs')
original = client.call('GET', '/api/v1/attachments/' + row['json']['id'] + '/content', actor=alice)['bytes']
check(hashlib.sha256(original).hexdigest() == digest, 'download checksum differs')
latencies = sorted(row['elapsed'] for row in rows)
result = {
'concurrency': args.concurrency, 'rounds': args.rounds, 'attempts': len(rows),
'fixture': {'format': 'PNG', 'width': 1500, 'height': 1500, 'size_bytes': len(data), 'sha256': digest},
'statuses': dict(Counter(str(row['status']) for row in rows)),
'elapsed_seconds': round(elapsed, 3), 'successful_uploads': len(succeeded),
'p95_seconds': round(latencies[min(len(latencies) - 1, int(len(latencies) * .95))], 3),
'memory': measurements, 'api_memory_limit': project.config['services']['api']['deploy']['resources']['limits']['memory'],
'notes': '20 ms samples include monitor and decoder processes; cgroup peak is cumulative since container creation; 429 and failed requests counted separately; no upload retries',
}
encoded = json.dumps(result, ensure_ascii=False, indent=2)
if args.output:
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(encoded + '\n')
print(encoded)
check(all(row['status'] in {201, 429} for row in rows), 'unexpected HTTP/transport errors during upload experiment; inspect status counts')
if __name__ == '__main__':
run(main)