diff --git a/.github/workflows/app.yml b/.github/workflows/app.yml new file mode 100644 index 0000000..039bcd6 --- /dev/null +++ b/.github/workflows/app.yml @@ -0,0 +1,42 @@ +# Проверка РЕАЛИЗАЦИИ (аудит F15); contracts.yml проверяет только skeleton. +# Изолированный стенд Compose с одноразовыми секретами: сборка, готовность, публичный smoke, +# регрессии аудита, не требующие ручного вмешательства. Логи при ошибке, очистка всегда. +# Секреты CI не нужны: init-secrets.sh генерирует их на раннере. pull_request_target не используется. +# Ограничение: локально workflow не исполнялся (нет раннера); проверен синтаксис YAML. +name: Application +on: [push, pull_request] +permissions: + contents: read +jobs: + app: + runs-on: ubuntu-latest + timeout-minutes: 40 + env: + COMPOSE_PROJECT_NAME: mtusi-lab5-ci + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Secrets and config + run: ./scripts/init-secrets.sh && docker compose --env-file .env config --quiet + - name: Build and start + run: docker compose --env-file .env up -d --build --wait --wait-timeout 300 + - name: Public smoke + run: make check test + - name: Audit regressions (HTTP) + run: python3 student-tests/api_negative_lab5.py + - name: Variants and transform limits + run: | + docker compose --profile tools run --rm -T checks /app/student-tests/variants_check.py --base-url http://lb:8080 + docker compose --profile tools run --rm -T checks /app/student-tests/transform_limits.py + - name: Worker outage and redelivery + run: python3 student-tests/worker_outage.py + - name: Outbox crash windows + run: python3 student-tests/outbox_gap.py + - name: Logs on failure + if: failure() + run: docker compose logs --no-color --tail 300 + - name: Cleanup + if: always() + run: docker compose --profile tools down -v --remove-orphans diff --git a/evidence/failover-bench.json b/evidence/failover-bench.json new file mode 100644 index 0000000..1de9f63 --- /dev/null +++ b/evidence/failover-bench.json @@ -0,0 +1,292 @@ +{ + "label": "", + "concurrency": 4, + "duration_s": 20.0, + "request_timeout_s": 2.0, + "action": { + "command": "env TARGET_INSTANCE_ID=78ea3303cfbc scripts/stop-one.sh", + "started_s": 5.01, + "finished_s": 8.521, + "exit_code": 0 + }, + "baseline_ok_rps": 486.0, + "overall_ok_rps": 262.1, + "requests_ok": 5242, + "errors_total": 0, + "error_rate": 0.0, + "errors_by_kind": {}, + "error_window_s": null, + "recovery_s": 0.0, + "post_action_ok_rps": 137.8, + "capacity_ratio": 0.28, + "latency_ok_ms": { + "p50": 5.1, + "p95": 76.6, + "p99": 94.2 + }, + "target_instance": "78ea3303cfbc", + "target_last_seen_s": 1.629, + "lb_events": [ + { + "t": 1.7, + "server": "api1", + "addr": "172.23.0.9:8080", + "from": "UP", + "to": "DOWN" + }, + { + "t": 6.02, + "server": "api1", + "addr": "", + "from": "DOWN", + "to": "MAINT" + } + ], + "posts_retried_same_key": 0, + "confirmed_messages": 2622, + "history_messages": 2622, + "lost_confirmed": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "timeline": [ + { + "t": 0, + "ok": 448, + "err": 0, + "instances": { + "78ea3303cfbc": 224, + "ad0d1b2c4289": 224 + }, + "versions": { + "2": 448 + } + }, + { + "t": 1, + "ok": 545, + "err": 0, + "instances": { + "78ea3303cfbc": 273, + "ad0d1b2c4289": 272 + }, + "versions": { + "2": 545 + } + }, + { + "t": 2, + "ok": 404, + "err": 0, + "instances": { + "ad0d1b2c4289": 204, + "78ea3303cfbc": 200 + }, + "versions": { + "2": 404 + } + }, + { + "t": 3, + "ok": 601, + "err": 0, + "instances": { + "78ea3303cfbc": 302, + "ad0d1b2c4289": 299 + }, + "versions": { + "2": 601 + } + }, + { + "t": 4, + "ok": 394, + "err": 0, + "instances": { + "78ea3303cfbc": 198, + "ad0d1b2c4289": 196 + }, + "versions": { + "2": 394 + } + }, + { + "t": 5, + "ok": 476, + "err": 0, + "instances": { + "ad0d1b2c4289": 239, + "78ea3303cfbc": 237 + }, + "versions": { + "2": 476 + } + }, + { + "t": 6, + "ok": 468, + "err": 0, + "instances": { + "78ea3303cfbc": 206, + "ad0d1b2c4289": 262 + }, + "versions": { + "2": 468 + } + }, + { + "t": 7, + "ok": 166, + "err": 0, + "instances": { + "ad0d1b2c4289": 166 + }, + "versions": { + "2": 166 + } + }, + { + "t": 8, + "ok": 184, + "err": 0, + "instances": { + "ad0d1b2c4289": 184 + }, + "versions": { + "2": 184 + } + }, + { + "t": 9, + "ok": 151, + "err": 0, + "instances": { + "ad0d1b2c4289": 151 + }, + "versions": { + "2": 151 + } + }, + { + "t": 10, + "ok": 79, + "err": 0, + "instances": { + "ad0d1b2c4289": 79 + }, + "versions": { + "2": 79 + } + }, + { + "t": 11, + "ok": 93, + "err": 0, + "instances": { + "ad0d1b2c4289": 93 + }, + "versions": { + "2": 93 + } + }, + { + "t": 12, + "ok": 172, + "err": 0, + "instances": { + "ad0d1b2c4289": 172 + }, + "versions": { + "2": 172 + } + }, + { + "t": 13, + "ok": 201, + "err": 0, + "instances": { + "ad0d1b2c4289": 201 + }, + "versions": { + "2": 201 + } + }, + { + "t": 14, + "ok": 149, + "err": 0, + "instances": { + "ad0d1b2c4289": 149 + }, + "versions": { + "2": 149 + } + }, + { + "t": 15, + "ok": 171, + "err": 0, + "instances": { + "ad0d1b2c4289": 171 + }, + "versions": { + "2": 171 + } + }, + { + "t": 16, + "ok": 115, + "err": 0, + "instances": { + "ad0d1b2c4289": 115 + }, + "versions": { + "2": 115 + } + }, + { + "t": 17, + "ok": 108, + "err": 0, + "instances": { + "ad0d1b2c4289": 108 + }, + "versions": { + "2": 108 + } + }, + { + "t": 18, + "ok": 152, + "err": 0, + "instances": { + "ad0d1b2c4289": 152 + }, + "versions": { + "2": 152 + } + }, + { + "t": 19, + "ok": 157, + "err": 0, + "instances": { + "ad0d1b2c4289": 157 + }, + "versions": { + "2": 157 + } + }, + { + "t": 20, + "ok": 8, + "err": 0, + "instances": { + "ad0d1b2c4289": 8 + }, + "versions": { + "2": 8 + } + } + ], + "pass": true +} diff --git a/evidence/rolling-update.txt b/evidence/rolling-update.txt new file mode 100644 index 0000000..6f6c978 --- /dev/null +++ b/evidence/rolling-update.txt @@ -0,0 +1,15 @@ +# 2026-11-13T17:56:27Z ./scripts/rolling-update.sh 2 +2026-11-13T17:56:28Z build version 2 + migrate Built +2026-11-13T17:56:29Z migrate (once) +2026-11-13 17:56:30,183 INFO migrations done; applied now: nothing +2026-11-13T17:56:30Z replacing 2 replicas +2026-11-13T17:56:36Z up mtusi-lab5-api-3 version=2 ip=172.23.0.12 UP in LB (UP slots: 4) +2026-11-13T17:56:39Z gone mtusi-lab5-api-1 (graceful SIGTERM) +2026-11-13T17:56:45Z up mtusi-lab5-api-4 version=2 ip=172.23.0.7 UP in LB (UP slots: 4) +2026-11-13T17:56:49Z gone mtusi-lab5-api-2 (graceful SIGTERM) +2026-11-13T17:56:49Z done: mtusi-lab5-api-3 mtusi-lab5-api-4 +2026-11-13T17:56:49Z verified: 10/10 responses via LB have X-App-Version=2 +exit=0 +# make test после обновления +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/s3-outage.json b/evidence/s3-outage.json new file mode 100644 index 0000000..aa275cc --- /dev/null +++ b/evidence/s3-outage.json @@ -0,0 +1,235 @@ +{ + "scenario": "S3 outage: API answers 503 fast, accepted jobs retry and complete", + "date_utc": "2026-11-13T17:58:27+00:00", + "steps": [ + { + "t": 2.04, + "utc": "17:57:29.727", + "step": "worker stopped, job accepted before outage", + "attachment_id": "19dac41c-4059-4326-8c97-93a0566b8979" + }, + { + "t": 24.72, + "utc": "17:57:52.405", + "step": "S3 `stop` for 5 s", + "upload": { + "status": 503, + "ms": 14, + "code": "unavailable", + "retry_after": "1" + }, + "download_ready": { + "status": 503, + "ms": 10, + "code": "unavailable", + "retry_after": "1" + }, + "metadata_queued": { + "status": 200, + "ms": 6, + "code": null, + "retry_after": null + }, + "text_message": { + "status": 201, + "ms": 6, + "code": null, + "retry_after": null + }, + "ready": { + "status": 200, + "ms": 2, + "code": null, + "retry_after": null + } + }, + { + "t": 25.83, + "utc": "17:57:53.513", + "step": "worker meets S3 outage -> retry with backoff", + "events": [ + "enqueued", + "claimed", + "retry_scheduled" + ], + "retries": [ + { + "error_code": "storage_unavailable", + "next_attempt": 2, + "delay_seconds": 1.499 + } + ] + }, + { + "t": 46.46, + "utc": "17:58:14.145", + "step": "S3 `start` -> healthy", + "seconds": 20.4 + }, + { + "t": 51.61, + "utc": "17:58:19.296", + "step": "S3 `pause` for 5 s", + "upload": { + "status": 503, + "ms": 14, + "code": "unavailable", + "retry_after": "1" + }, + "download_ready": { + "status": 503, + "ms": 11, + "code": "unavailable", + "retry_after": "1" + }, + "metadata_queued": { + "status": 200, + "ms": 7, + "code": null, + "retry_after": null + }, + "text_message": { + "status": 201, + "ms": 9, + "code": null, + "retry_after": null + }, + "ready": { + "status": 200, + "ms": 2, + "code": null, + "retry_after": null + } + }, + { + "t": 57.47, + "utc": "17:58:25.154", + "step": "S3 `unpause` -> healthy", + "seconds": 5.7 + }, + { + "t": 58.24, + "utc": "17:58:25.925", + "step": "job accepted before outage completed", + "final": "ready", + "replay": null, + "attempts": 4, + "events": [ + "enqueued", + "claimed", + "retry_scheduled", + "claimed", + "retry_scheduled", + "claimed", + "retry_scheduled", + "claimed", + "ready", + "staging_deleted" + ] + }, + { + "t": 58.58, + "utc": "17:58:26.262", + "step": "upload after recovery", + "status": "ready" + } + ], + "results": { + "s3_stop": { + "upload": { + "status": 503, + "ms": 14, + "code": "unavailable", + "retry_after": "1" + }, + "download_ready": { + "status": 503, + "ms": 10, + "code": "unavailable", + "retry_after": "1" + }, + "metadata_queued": { + "status": 200, + "ms": 6, + "code": null, + "retry_after": null + }, + "text_message": { + "status": 201, + "ms": 6, + "code": null, + "retry_after": null + }, + "ready": { + "status": 200, + "ms": 2, + "code": null, + "retry_after": null + } + }, + "worker_during_outage": { + "events": [ + "enqueued", + "claimed", + "retry_scheduled" + ], + "retries": [ + { + "error_code": "storage_unavailable", + "next_attempt": 2, + "delay_seconds": 1.499 + } + ] + }, + "s3_pause": { + "upload": { + "status": 503, + "ms": 14, + "code": "unavailable", + "retry_after": "1" + }, + "download_ready": { + "status": 503, + "ms": 11, + "code": "unavailable", + "retry_after": "1" + }, + "metadata_queued": { + "status": 200, + "ms": 7, + "code": null, + "retry_after": null + }, + "text_message": { + "status": 201, + "ms": 9, + "code": null, + "retry_after": null + }, + "ready": { + "status": 200, + "ms": 2, + "code": null, + "retry_after": null + } + }, + "accepted_job": { + "final": "ready", + "replay": null, + "attempts": 4, + "events": [ + "enqueued", + "claimed", + "retry_scheduled", + "claimed", + "retry_scheduled", + "claimed", + "retry_scheduled", + "claimed", + "ready", + "staging_deleted" + ] + }, + "upload_after_recovery": "ready" + } +} diff --git a/evidence/upload-memory-c12.json b/evidence/upload-memory-c12.json new file mode 100644 index 0000000..b1beaf3 --- /dev/null +++ b/evidence/upload-memory-c12.json @@ -0,0 +1,77 @@ +{ + "concurrency": 12, + "rounds": 2, + "files": [ + { + "type": "image/png", + "bytes": 10482871, + "sha256": "184263681fcc647c" + }, + { + "type": "image/jpeg", + "bytes": 8051317, + "sha256": "65d91ac28a9d4d68" + }, + { + "type": "image/jpeg", + "bytes": 890394, + "sha256": "e7f29bc3786e6d91" + } + ], + "uploads": 24, + "status_counts": { + "202": 24 + }, + "megabytes_uploaded_ok": 148.2, + "wall_s": 5.2, + "latency_s_p50_max": [ + 2.07, + 2.7 + ], + "replicas": { + "78ea3303cfbc": { + "rss_mib": 51, + "hwm_mib_after_warmup": 82, + "samples": 100, + "peak_rss_mib_sampled": 81.1, + "peak_anon_mib": 290.0, + "peak_file_cache_mib": 46.6, + "vmhwm_mib": 82, + "cgroup_memory_peak_mib": 331.9, + "memory_limit_mib": 512, + "uploads_served": 12 + }, + "ad0d1b2c4289": { + "rss_mib": 51, + "hwm_mib_after_warmup": 83, + "samples": 95, + "peak_rss_mib_sampled": 88.5, + "peak_anon_mib": 297.5, + "peak_file_cache_mib": 37.1, + "vmhwm_mib": 88, + "cgroup_memory_peak_mib": 339.7, + "memory_limit_mib": 512, + "uploads_served": 12 + } + }, + "s3": { + "rss_mib": 413, + "oom_killed": "false", + "peak_rss_mib_sampled": 477.2, + "peak_anon_mib": 396.8, + "restarts": 0, + "oom_killed_after": "false", + "memory_limit_mib": 768 + }, + "oom_kills": [ + "false", + "false" + ], + "criteria": { + "all_responses_allowed": true, + "no_oom_kills": true, + "no_replica_restarts": true, + "cgroup_peak_below_90pct": true + }, + "pass": true +} \ No newline at end of file diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh index e3e130a..0fa7700 100755 --- a/scripts/rolling-update.sh +++ b/scripts/rolling-update.sh @@ -2,7 +2,9 @@ # Поочерёдная замена API-реплик на новую версию образа без остановки сервиса. # scripts/rolling-update.sh # Порядок (expand -> replace): -# 1. собрать новый образ (тот же тег mtusi-messenger:lab5, build-arg APP_VERSION); +# 1. собрать новый образ (тот же тег mtusi-messenger:lab5, build-arg APP_VERSION). Секция build +# есть только у сервиса migrate (аудит F10: `build api` молча ничего не собирал), поэтому +# собираем migrate и проверяем, что в образе действительно новая версия; # 2. ОДИН раз применить миграции (job migrate). Допускаются только обратно совместимые # (expand-only) миграции: старые реплики продолжают работать на новой схеме; # 3. для каждой старой реплики: поднять +1 новую -> дождаться healthy и UP в HAProxy -> @@ -25,7 +27,9 @@ lb_has_up() { } echo "$(ts) build version $VERSION" -docker compose --env-file .env build api >/dev/null +docker compose --env-file .env build migrate >/dev/null +built=$(docker run --rm --entrypoint printenv mtusi-messenger:lab5 APP_VERSION) +[ "$built" = "$VERSION" ] || { echo "image has APP_VERSION=$built, expected $VERSION" >&2; exit 1; } echo "$(ts) migrate (once)" docker compose --env-file .env run --rm --no-deps migrate @@ -44,7 +48,9 @@ for old in $OLD; do ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$new") for _ in $(seq 1 30); do lb_has_up "$ip" && break; sleep 0.5; done lb_has_up "$ip" || { echo "HAProxy did not mark new replica $ip UP" >&2; exit 1; } - echo "$(ts) up $(docker inspect -f '{{.Name}}' "$new" | tr -d /) version=$(docker exec "$new" printenv APP_VERSION) ip=$ip UP in LB (UP slots: $(lb_up_count))" + got=$(docker exec "$new" printenv APP_VERSION) + [ "$got" = "$VERSION" ] || { echo "new replica runs APP_VERSION=$got, expected $VERSION" >&2; exit 1; } + echo "$(ts) up $(docker inspect -f '{{.Name}}' "$new" | tr -d /) version=$got ip=$ip UP in LB (UP slots: $(lb_up_count))" name=$(docker inspect -f '{{.Name}}' "$old" | tr -d /) docker stop -t 20 "$old" >/dev/null && docker rm "$old" >/dev/null echo "$(ts) gone $name (graceful SIGTERM)" @@ -55,3 +61,9 @@ else echo "APP_VERSION=$VERSION" >> .env fi echo "$(ts) done: $(docker compose ps --format '{{.Name}}' api | tr '\n' ' ')" +# Проверка через LB: все ответы — от новой версии. +for _ in $(seq 1 10); do + v=$(curl -s -D - -o /dev/null http://127.0.0.1:${LB_PORT:-8080}/health/ready | tr -d '\r' | awk -F': ' 'tolower($1)=="x-app-version"{print $2}') + [ "$v" = "$VERSION" ] || { echo "LB served X-App-Version=$v, expected $VERSION" >&2; exit 1; } +done +echo "$(ts) verified: 10/10 responses via LB have X-App-Version=$VERSION" diff --git a/student-tests/consistency_faults.py b/student-tests/consistency_faults.py deleted file mode 100644 index 9c35702..0000000 --- a/student-tests/consistency_faults.py +++ /dev/null @@ -1,179 +0,0 @@ -#!/usr/bin/env python3 -"""Отказы между записью в S3 и PostgreSQL и очистка janitor (уровень 4). - -Требует API с FAULT_INJECTION=1 (учебный хук, по умолчанию выключен): - FAULT_INJECTION=1 docker compose --env-file .env up -d --no-deps --wait api - python3 student-tests/consistency_faults.py --output evidence/consistency-faults.json - docker compose --env-file .env up -d --no-deps --wait api # вернуть FAULT_INJECTION=0 - -Сценарии (каждый — реальный HTTP upload через LB): - db-fail-after-s3-put PUT прошёл, «COMMIT» ready упал -> 503; компенсация удаляет объект - и pending-строку -> ничего не остаётся; - crash-after-s3-put процесс API умирает (os._exit) после PUT -> pending + объект; - crash-after-pending процесс умирает до PUT -> pending без объекта; - обрыв клиента Content-Length 5 MiB, отправлен 1 MiB, сокет закрыт -> ничего; - orphan pending-строку удаляем вручную -> объект без строки; - missing объект ready-вложения удаляем в S3 -> только отчёт, download = 503. -Затем janitor: dry-run и боевой проход с grace 3600 (свежее не трогает), ожидание -grace-seconds, dry-run -> боевой -> повторный (идемпотентность). Контрольное ready-вложение -должно остаться доступным побайтно. -""" -import argparse -import hashlib -import json -import socket -import subprocess -import sys -import time -import uuid -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT / "tests")) -from client import Client, check # noqa: E402 - -PNG = (ROOT / "tests/fixtures/sample.png").read_bytes() -log = [] - - -def sh(*cmd, timeout=120): - r = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, timeout=timeout) - if r.returncode != 0: - raise SystemExit("command failed: %s\n%s" % (" ".join(cmd[:4]), r.stderr[-500:])) - return r.stdout - - -def psql(sql): - return sh("docker", "compose", "exec", "-T", "db", "sh", "-c", - 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -tAc "$0"', sql).strip() - - -def janitor(grace, dry_run): - cmd = ["docker", "compose", "run", "--rm", "--no-deps", "janitor", "python", "/app/src/janitor.py", - "--once", "--grace-seconds", str(grace)] + (["--dry-run"] if dry_run else []) - out = sh(*cmd) - report = json.loads([line for line in out.splitlines() if line.startswith("{")][-1]) - keep = ("dry_run", "grace_seconds", "rows_by_status", "objects_total", "abandoned_pending", - "orphan_objects", "missing_objects", "deleted_objects", "deleted_rows", "errors", "duration_ms") - return {k: report.get(k) for k in keep} - - -def step(name, **data): - entry = dict(step=name, t=round(time.time(), 1), **data) - log.append(entry) - print(json.dumps(entry, ensure_ascii=False), flush=True) - return entry - - -def wait_api_healthy(): - deadline = time.time() + 90 - while time.time() < deadline: - states = sh("docker", "compose", "ps", "-a", "--format", "{{.Health}}", "api").split() - if states and all(s == "healthy" for s in states): - time.sleep(3) # HAProxy rise - return - time.sleep(1) - raise SystemExit("api did not become healthy") - - -def upload(c, actor, chat, fault=None, expected=201): - h = {"Content-Type": "image/png"} - if fault: - h["X-Fault-Inject"] = fault - try: - r = c.call("POST", "/api/v1/chats/%s/attachments?filename=f.png" % chat["id"], expected, - actor=actor, raw=PNG, headers=h, schema="Attachment" if expected == 201 else None) - return r["status"], r["json"] - except Exception as exc: # обрыв соединения при crash-* тоже допустимый исход - return type(exc).__name__, str(exc)[:120] - - -def interrupted_upload(base, token, chat_id): - host, port = base.split("//")[1].split(":") - s = socket.create_connection((host, int(port)), timeout=10) - s.sendall(("POST /api/v1/chats/%s/attachments?filename=cut.png HTTP/1.1\r\nHost: %s\r\n" - "Authorization: Bearer %s\r\nContent-Type: image/png\r\nContent-Length: %d\r\n\r\n" - % (chat_id, host, token, 5 * 1024 * 1024)).encode()) - s.sendall(PNG[:8] + b"\0" * (1024 * 1024 - 8)) - s.close() # клиент «пропал» на середине - - -def main(): - ap = argparse.ArgumentParser() - ap.add_argument("--base-url", default="http://localhost:8080") - ap.add_argument("--grace-seconds", type=int, default=20) - ap.add_argument("--output") - args = ap.parse_args() - env = sh("docker", "compose", "exec", "-T", "api", "printenv", "FAULT_INJECTION").strip() - check(env == "1", "start api with FAULT_INJECTION=1 first (see docstring)") - c = Client(args.base_url, timeout=30) - alice, bob, _eve, chat = c.bootstrap() - started = psql("SELECT now()") - base = janitor(3600, True) - step("baseline janitor dry-run", report=base) - - _, control = upload(c, alice, chat) - _, victim = upload(c, alice, chat) - step("control + victim uploaded (ready)", control=control["id"], victim=victim["id"]) - - st, body = upload(c, alice, chat, "db-fail-after-s3-put", expected=503) - after_comp = janitor(3600, True) - step("db-fail-after-s3-put -> compensation", status=st, report=after_comp, - leftover_pending=after_comp["rows_by_status"].get("pending", 0) - base["rows_by_status"].get("pending", 0), - leftover_objects=after_comp["objects_total"] - base["objects_total"] - 2) - - for fault in ("crash-after-s3-put", "crash-after-s3-put", "crash-after-pending"): - st, body = upload(c, alice, chat, fault, expected=[502, 503]) - step(fault, client_saw=st) - wait_api_healthy() - - interrupted_upload(args.base_url, alice["token"], chat["id"]) - time.sleep(2) - step("client aborted upload at 1 of 5 MiB", report=janitor(3600, True)) - - pending = psql("SELECT id || ' ' || (object_key IS NOT NULL) FROM attachments WHERE status='pending' " - "AND created_at >= '%s' ORDER BY created_at" % started).splitlines() - step("pending rows left by crashes", pending=len(pending)) - check(len(pending) == 3, "expected 3 pending rows from crashes") - orphan_id = pending[0].split()[0] - psql("DELETE FROM attachments WHERE id = '%s'" % orphan_id) - step("orphan injected: pending row deleted, object kept", row=orphan_id) - victim_key = psql("SELECT object_key FROM attachments WHERE id = '%s'" % victim["id"]) - sh("docker", "compose", "--profile", "tools", "run", "--rm", "--entrypoint", "python", "s3-check", "-c", - "import sys; sys.argv=['x']; exec(open('/app/student-tests/s3_private_check.py').read().split('results = []')[0]);" - "client('api').remove_object(BUCKET, %r)" % victim_key) - dl = c.call("GET", "/api/v1/attachments/%s/content" % victim["id"], 503, actor=bob) - step("missing injected: victim object deleted in S3", download_status=dl["status"]) - - fresh_dry = janitor(3600, True) - fresh_run = janitor(3600, False) - step("janitor grace=3600 (all fresh): dry-run", report=fresh_dry) - step("janitor grace=3600 (all fresh): real run deletes nothing", report=fresh_run) - check(fresh_run["deleted_objects"] == 0 and fresh_run["deleted_rows"] == 0, "fresh data deleted") - - time.sleep(args.grace_seconds + 2) - dry = janitor(args.grace_seconds, True) - step("after grace: dry-run", report=dry) - real = janitor(args.grace_seconds, False) - step("after grace: real run", report=real) - again = janitor(args.grace_seconds, False) - step("after grace: repeated run (idempotent)", report=again) - d = c.call("GET", "/api/v1/attachments/%s/content" % control["id"], actor=bob) - control_ok = hashlib.sha256(d["bytes"]).hexdigest() == control["sha256"] - still = psql("SELECT count(*) FROM attachments WHERE status='pending' AND created_at >= '%s'" % started) - step("control attachment still intact", sha256_ok=control_ok, pending_left=int(still)) - - ok = (log[2]["leftover_pending"] == 0 and log[2]["leftover_objects"] == 0 - and dry["abandoned_pending"] >= 2 and dry["orphan_objects"] >= 1 and dry["missing_objects"] >= 1 - and dry["deleted_objects"] == 0 - and real["deleted_rows"] >= 2 and again["abandoned_pending"] == 0 and again["orphan_objects"] == 0 - and again["deleted_objects"] == 0 and again["missing_objects"] >= 1 - and control_ok and int(still) == 0) - print(json.dumps({"pass": ok})) - if args.output: - Path(args.output).write_text(json.dumps({"pass": ok, "steps": log}, ensure_ascii=False, indent=1)) - return 0 if ok else 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/student-tests/run_all.sh b/student-tests/run_all.sh index b8d3bb2..2653e94 100755 --- a/student-tests/run_all.sh +++ b/student-tests/run_all.sh @@ -2,6 +2,7 @@ # Полный прогон проверок lab5 на запущенном стенде с записью результатов в evidence/. # Каждый сценарий сам включает и выключает свои учебные инъекции отказов. # ./student-tests/run_all.sh +# Для make validate и сверки со схемой нужен venv (создаётся автоматически в .venv/). set -u cd "$(dirname "$0")/.." E=evidence @@ -17,7 +18,13 @@ run() { # run states() { docker compose ps --format '{{.Name}} {{.State}}' | sort; \ for c in $(docker compose ps -q); do docker inspect -f '{{.Name}} StartedAt={{.State.StartedAt}}' "$c"; done | sort; } +if [ ! -x .venv/bin/python ]; then + python3 -m venv .venv && .venv/bin/python -m pip install -q -r requirements-dev.txt rfc3339-validator +fi +VPY=.venv/bin/python + run check make check +run validate make validate PYTHON=$VPY run smoke make test { echo "# containers before persistence hook"; states; } > "$E/containers-before-restart.txt" run persistence make test-persistence ACTION_SCRIPT=scripts/restart.sh @@ -38,8 +45,25 @@ run s3-private-check docker compose --profile tools run --rm -T checks /app/stud run attachments-negative python3 student-tests/attachments_negative.py --output "$E/attachments-negative.json" run attachment-failover python3 student-tests/attachment_failover.py --output "$E/attachment-failover.json" ./scripts/start-all.sh >/dev/null 2>&1 +# --- регрессии внешнего аудита (F01–F17) --- +run api-negative-lab5 python3 student-tests/api_negative_lab5.py --output "$E/api-negative-lab5.json" +run slow-client python3 student-tests/slow_client.py --output "$E/slow-client.json" +run janitor-replay-race python3 student-tests/janitor_replay_race.py --output "$E/janitor-replay-race.json" +run queue-contract-check $VPY student-tests/queue_contract_check.py --output "$E/queue-contract-check.json" +run s3-outage python3 student-tests/s3_outage.py --output "$E/s3-outage.json" +run upload-memory python3 student-tests/upload_memory.py --concurrency 12 --rounds 2 --output "$E/upload-memory-c12.json" +# --- регрессия lab3 --- +run cursor-validation python3 student-tests/cursor_validation.py +run idempotency-concurrency python3 student-tests/idempotency_concurrency.py --output "$E/idempotency-concurrency.json" +T=$(docker compose ps -q api | head -1 | cut -c1-12) +run failover-bench python3 student-tests/failover_bench.py --action "env TARGET_INSTANCE_ID=$T scripts/stop-one.sh" \ + --target-instance "$T" --duration 20 --action-at 5 --concurrency 4 --output "$E/failover-bench.json" +./scripts/start-all.sh >/dev/null 2>&1 +NEXT=$(( $(grep -E '^APP_VERSION=' .env | cut -d= -f2) + 1 )) +run rolling-update sh -c "./scripts/rolling-update.sh $NEXT && make test" run janitor-dry-run docker compose exec -T janitor python /app/src/janitor.py --once --dry-run run queues docker compose exec -T broker rabbitmqctl -q -p messenger list_queues name type durable messages consumers run failed-registry docker compose --profile tools run --rm -T ops failed --limit 10 +run login-flood python3 student-tests/login_flood.py --output "$E/login-flood.json" # последним: исчерпывает лимит входов echo "overall=$status" exit $status diff --git a/student-tests/s3_outage.py b/student-tests/s3_outage.py index 0802626..367fd57 100644 --- a/student-tests/s3_outage.py +++ b/student-tests/s3_outage.py @@ -1,63 +1,48 @@ #!/usr/bin/env python3 -"""Недоступность S3: stop (порт закрыт) и pause (S3 «завис», TCP не отвечает). +"""Недоступность S3 в lab5 (переписано из lab4 по аудиту F14: там ожидался синхронный 201). - python3 student-tests/s3_outage.py [--output evidence/s3-outage.json] + python3 student-tests/s3_outage.py --output evidence/s3-outage.json -Ожидания: upload и download — быстрый 503 unavailable c Retry-After (не 500 и не -таймаут LB), текстовые сообщения и /health/ready продолжают работать (S3 не входит в -readiness), после восстановления S3 загрузка снова 201 без перезапуска API. +API (как в lab4): при остановленном (`stop`) и зависшем (`pause`) S3 upload и download — быстрый +503 unavailable с Retry-After, текстовые сообщения 201, /health/ready 200, GET метаданных 200. +Worker (новое): задание, принятое ДО отказа S3, при обработке получает storage_unavailable и +уходит в retry с backoff; после восстановления S3 оно доходит до ready. Если отказ длился +дольше всех попыток — failed(retry_exhausted), и оператор делает replay (это тоже проверяется). """ import argparse import json -import subprocess -import sys import time import uuid -from pathlib import Path -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT / "tests")) -from client import Client # noqa: E402 - -PNG = (ROOT / "tests/fixtures/sample.png").read_bytes() -log = [] +from lab5lib import (Client, Log, SAMPLE_PNG, check, compose, events, ops, upload, wait_status, wait_until) -def probe(c, alice, chat, att): +def probe(c, alice, chat, att_ready, att_queued): out = {} - for name, args in ( - ("upload", ("POST", "/api/v1/chats/%s/attachments?filename=o.png" % chat["id"])), - ("download", ("GET", "/api/v1/attachments/%s/content" % att["id"])), - ("text_message", ("POST", "/api/v1/chats/%s/messages" % chat["id"])), - ("ready", ("GET", "/health/ready"))): - kw = {} - if name == "upload": - kw = {"raw": PNG, "headers": {"Content-Type": "image/png"}} - if name == "text_message": - kw = {"body": {"text": "без S3"}, "headers": {"Idempotency-Key": uuid.uuid4().hex}} + calls = { + "upload": ("POST", "/api/v1/chats/%s/attachments?filename=o.png" % chat["id"], + {"raw": SAMPLE_PNG, "headers": {"Content-Type": "image/png"}}), + "download_ready": ("GET", "/api/v1/attachments/%s/content" % att_ready["id"], {}), + "metadata_queued": ("GET", "/api/v1/attachments/%s" % att_queued["id"], {}), + "text_message": ("POST", "/api/v1/chats/%s/messages" % chat["id"], + {"body": {"text": "без S3"}, "headers": {"Idempotency-Key": uuid.uuid4().hex}}), + "ready": ("GET", "/health/ready", {}), + } + for name, (method, path, kw) in calls.items(): t = time.monotonic() - r = c.call(*args, [200, 201, 503, 502, 504], actor=alice if name != "ready" else None, **kw) + r = c.call(method, path, [200, 201, 202, 503], actor=None if name == "ready" else alice, **kw) out[name] = {"status": r["status"], "ms": round((time.monotonic() - t) * 1000), "code": (r["json"] or {}).get("error", {}).get("code"), "retry_after": r["headers"].get("Retry-After")} return out -def phase(name, c, alice, chat, att): - entry = {"phase": name, "probe": probe(c, alice, chat, att)} - log.append(entry) - print(json.dumps(entry, ensure_ascii=False), flush=True) - return entry - - -def recover(c, alice, chat, started): - while time.monotonic() - started < 120: - r = c.call("POST", "/api/v1/chats/%s/attachments?filename=r.png" % chat["id"], [201, 503], actor=alice, - raw=PNG, headers={"Content-Type": "image/png"}) - if r["status"] == 201: - return round(time.monotonic() - started, 1) - time.sleep(0.5) - return None +def s3_healthy(): + cid = compose("ps", "-q", "s3").stdout.strip() + import subprocess + st = subprocess.run(["docker", "inspect", "-f", "{{.State.Health.Status}}", cid], capture_output=True, + text=True).stdout.strip() + return st == "healthy" def main(): @@ -65,45 +50,57 @@ def main(): ap.add_argument("--base-url", default="http://localhost:8080") ap.add_argument("--output") args = ap.parse_args() + log = Log("S3 outage: API answers 503 fast, accepted jobs retry and complete") c = Client(args.base_url, timeout=30) - alice, _bob, _eve, chat = c.bootstrap() - att = c.call("POST", "/api/v1/chats/%s/attachments?filename=a.png" % chat["id"], 201, actor=alice, raw=PNG, - headers={"Content-Type": "image/png"}, schema="Attachment")["json"] - phase("baseline", c, alice, chat, att) - for action, undo in ((["stop", "s3"], ["start", "s3"]), (["pause", "s3"], ["unpause", "s3"])): - subprocess.run(["docker", "compose"] + action, cwd=ROOT, check=True, capture_output=True) - time.sleep(1) - phase("immediately after `%s` (S3Health not yet stale)" % " ".join(action), c, alice, chat, att) - time.sleep(4) - phase("S3 down for 5 s (`%s`)" % " ".join(action), c, alice, chat, att) - subprocess.run(["docker", "compose"] + undo, cwd=ROOT, check=True, capture_output=True) - t0 = time.monotonic() - rec = recover(c, alice, chat, t0) - log.append({"phase": "recovery after `%s`" % " ".join(undo), "upload_201_after_s": rec}) - print(json.dumps(log[-1]), flush=True) - # Оставляем стенд рабочим: ждём, пока healthcheck S3 снова станет healthy (после pause - # он несколько проверок подряд был unhealthy, а `docker compose up api` при нездоровой - # зависимости не запускает реплики). - cid = subprocess.run(["docker", "compose", "ps", "-q", "s3"], cwd=ROOT, capture_output=True, text=True).stdout.strip() - t0 = time.monotonic() - while time.monotonic() - t0 < 120: - st = subprocess.run(["docker", "inspect", "-f", "{{.State.Health.Status}}", cid], capture_output=True, text=True).stdout.strip() - if st == "healthy": - break - time.sleep(1) - log.append({"phase": "s3 healthcheck healthy again", "after_s": round(time.monotonic() - t0, 1), "status": st}) - print(json.dumps(log[-1]), flush=True) - down = [e for e in log if e.get("phase", "").startswith("S3 down")] - ok = all(e["probe"]["upload"]["status"] == 503 and e["probe"]["download"]["status"] == 503 - and e["probe"]["text_message"]["status"] == 201 and e["probe"]["ready"]["status"] == 200 - for e in down) and all(e.get("upload_201_after_s") is not None for e in log if "upload_201_after_s" in e) - no5xx = all(p["status"] in (200, 201, 503) for e in log if "probe" in e for p in e["probe"].values()) - ok = ok and st == "healthy" - print(json.dumps({"pass": ok and no5xx, "only_200_201_503": no5xx})) - if args.output: - Path(args.output).write_text(json.dumps({"pass": ok and no5xx, "phases": log}, ensure_ascii=False, indent=1)) - return 0 if ok and no5xx else 1 + alice, bob, _eve, chat = c.bootstrap() + att_ready = c.wait_ready(alice, upload(c, alice, chat, SAMPLE_PNG, "before.png")) + res = {} + try: + compose("stop", "worker") + queued = upload(c, alice, chat, SAMPLE_PNG, "accepted-before-outage.png") + log.step("worker stopped, job accepted before outage", attachment_id=queued["id"]) + for action, undo in (("stop", "start"), ("pause", "unpause")): + compose(action, "s3") + time.sleep(5) # S3Health (1 с) заметил отказ, данные устарели (3 с) + pr = probe(c, alice, chat, att_ready, queued) + res["s3_" + action] = pr + log.step("S3 `%s` for 5 s" % action, **pr) + check(pr["upload"]["status"] == 503 and pr["download_ready"]["status"] == 503 + and pr["text_message"]["status"] == 201 and pr["ready"]["status"] == 200 + and pr["metadata_queued"]["status"] == 200, "API during S3 %s: %s" % (action, pr)) + if action == "stop": + compose("up", "-d", "--no-deps", "--no-recreate", "worker") + info = wait_until(lambda: (lambda i: i if "retry_scheduled" in events(i) else None)( + ops("inspect", queued["id"])), 60, 1, "worker retry on storage error") + retries = [e["detail"] for e in info["events"] if e["event"] == "retry_scheduled"] + res["worker_during_outage"] = {"events": events(info), "retries": retries} + log.step("worker meets S3 outage -> retry with backoff", **res["worker_during_outage"]) + check(all(r["error_code"] == "storage_unavailable" for r in retries), "retry reason") + compose(undo, "s3") + t0 = time.monotonic() + wait_until(s3_healthy, 120, 1, "s3 healthy") + log.step("S3 `%s` -> healthy" % undo, seconds=round(time.monotonic() - t0, 1)) + final = wait_status(c, bob, queued["id"], {"ready", "failed"}, 120) + replayed = None + if final["status"] == "failed": + check(final["error_code"] == "retry_exhausted", "unexpected failure " + str(final["error_code"])) + replayed = ops("replay", queued["id"], "--reason", "S3 restored")["replay"][0]["result"] + final = wait_status(c, bob, queued["id"], {"ready", "failed"}, 60) + info = ops("inspect", queued["id"]) + res["accepted_job"] = {"final": final["status"], "replay": replayed, "attempts": info["jobs"][0]["attempt"], + "events": events(info)} + check(final["status"] == "ready", "accepted job not completed") + log.step("job accepted before outage completed", **res["accepted_job"]) + after = c.wait_ready(alice, upload(c, alice, chat, SAMPLE_PNG, "after.png")) + res["upload_after_recovery"] = after["status"] + log.step("upload after recovery", status=after["status"]) + finally: + compose("unpause", "s3", ok=False) + compose("start", "s3", ok=False) + compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "worker", ok=False) + log.save(args.output, results=res) + print("PASS: S3 outage handled by API (503) and worker (retry), accepted work completed") if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/student-tests/upload_memory.py b/student-tests/upload_memory.py index 915be3d..29845f5 100644 --- a/student-tests/upload_memory.py +++ b/student-tests/upload_memory.py @@ -4,9 +4,14 @@ python3 student-tests/upload_memory.py --concurrency 12 --rounds 3 --output evidence/upload-memory-c12.json Гипотеза: потоковый приём (буфер 64 KiB -> безымянный временный файл), multipart PUT -частями по 5 MiB и draft-декодирование JPEG держат RAM реплики ограниченной и не -зависящей линейно от числа одновременных загрузок; 10+ загрузок по ~10 MiB не -приближают процесс к лимиту контейнера 256 MiB. +частями по 5 MiB и проверка JPEG в подпроцессе с лимитом (src/jpeg_probe.py) держат RAM +реплики ограниченной и не зависящей линейно от числа одновременных загрузок. + +Lab5 (аудит F16): явные критерии успеха и ненулевой код выхода при нарушении — все ответы 202 +(503 только с --allow-503), ни одного OOM-kill и перезапуска реплик, пик памяти cgroup каждой +реплики < 90% лимита. Объём считается по размеру файла в каждой записи. В смесь входят +несжимаемый PNG ~10 MiB, baseline JPEG и progressive JPEG 4:4:4 ~40 Мпикс (самый тяжёлый +по памяти вход, аудит F02) — JPEG генерируются в контейнере checks (Pillow). Методика: перед прогоном API-реплики перезапускаются (VmHWM считается с нуля). В каждой реплике фоново (docker exec ... while sleep 0.05) читаются VmRSS процесса (PID 1) и @@ -67,18 +72,28 @@ class Sampler: self.p.terminate() -def make_jpeg(): - try: - from PIL import Image - except ImportError: - return None +GEN = r""" +import io, sys +from PIL import Image +kind = sys.argv[1] +if kind == "baseline": im = Image.effect_noise((6000, 6000), 40).convert("RGB") for q in (60, 45, 30, 20): - b = io.BytesIO() - im.save(b, "JPEG", quality=q) + b = io.BytesIO(); im.save(b, "JPEG", quality=q) if b.tell() <= 10 * 1024 * 1024: - return b.getvalue() - return None + break +else: + g = Image.radial_gradient("L").resize((6324, 6324)) + b = io.BytesIO(); Image.merge("RGB", (g, g.rotate(90), g)).save(b, "JPEG", quality=85, progressive=True, subsampling=0) +sys.stdout.buffer.write(b.getvalue()) +""" + + +def make_jpeg(kind): + """JPEG генерируется в контейнере checks (на хосте Pillow может не быть).""" + proc = subprocess.run(["docker", "compose", "--profile", "tools", "run", "--rm", "-T", "checks", "-c", GEN, kind], + cwd=ROOT, capture_output=True, check=True) + return proc.stdout def main(): @@ -87,12 +102,12 @@ def main(): ap.add_argument("--concurrency", type=int, default=12) ap.add_argument("--rounds", type=int, default=3) ap.add_argument("--no-restart", action="store_true") + ap.add_argument("--allow-503", action="store_true", help="считать 503 (перегрузка) допустимым ответом") ap.add_argument("--output") args = ap.parse_args() files = [("image/png", noise_png(10 * 1024 * 1024))] - jpeg = make_jpeg() - if jpeg: - files.append(("image/jpeg", jpeg)) + files.append(("image/jpeg", make_jpeg("baseline"))) + files.append(("image/jpeg", make_jpeg("progressive444"))) if not args.no_restart: sh("docker", "compose", "restart", "api") sh("docker", "compose", "up", "-d", "--no-deps", "--no-recreate", "--wait", "api") @@ -104,6 +119,7 @@ def main(): c.call("POST", "/api/v1/chats/%s/attachments?filename=w" % chat["id"], 202, actor=alice, raw=data, headers={"Content-Type": ctype}) cids = containers() + restarts_before = sh("docker", "inspect", "-f", "{{.RestartCount}}", *cids).split() idle = {cid[:12]: {"rss_mib": proc_status(cid, "VmRSS"), "hwm_mib_after_warmup": proc_status(cid, "VmHWM")} for cid in cids} s3_cid = sh("docker", "compose", "ps", "-q", "s3").split()[0] @@ -123,7 +139,9 @@ def main(): except Exception as exc: # noqa: BLE001 st, inst = type(exc).__name__, None with lock: - results.append({"type": ctype, "status": st, "instance": inst, "s": round(time.monotonic() - t, 2)}) + # Размер — в самой записи: потоки завершаются не по порядку (аудит F16). + results.append({"type": ctype, "bytes": len(data), "status": st, "instance": inst, + "s": round(time.monotonic() - t, 2)}) started = time.monotonic() for _ in range(args.rounds): @@ -154,8 +172,7 @@ def main(): "files": [{"type": t, "bytes": len(d), "sha256": hashlib.sha256(d).hexdigest()[:16]} for t, d in files], "uploads": len(results), "status_counts": {str(k): sum(1 for r in results if r["status"] == k) for k in {r["status"] for r in results}}, - "megabytes_uploaded_ok": round(sum(len(files[i % len(files)][1]) for i, r in enumerate(results) - if r["status"] == 202) / 2**20, 1), + "megabytes_uploaded_ok": round(sum(r["bytes"] for r in results if r["status"] == 202) / 2**20, 1), "wall_s": round(wall, 1), "latency_s_p50_max": [durations[len(durations) // 2], durations[-1]] if durations else None, "replicas": per, @@ -167,10 +184,20 @@ def main(): memory_limit_mib=round(int(sh("docker", "exec", s3_cid, "cat", "/sys/fs/cgroup/memory.max")) / 2**20)), "oom_kills": sh("docker", "inspect", "-f", "{{.State.OOMKilled}}", *cids).split(), } + allowed = {202, 503} if args.allow_503 else {202} + restarts_after = sh("docker", "inspect", "-f", "{{.RestartCount}}", *cids).split() + criteria = { + "all_responses_allowed": all(r["status"] in allowed for r in results), + "no_oom_kills": all(x == "false" for x in summary["oom_kills"]), + "no_replica_restarts": restarts_before == restarts_after, + "cgroup_peak_below_90pct": all(v["cgroup_memory_peak_mib"] < 0.9 * v["memory_limit_mib"] for v in per.values()), + } + summary["criteria"] = criteria + summary["pass"] = all(criteria.values()) print(json.dumps(summary, ensure_ascii=False, indent=1)) if args.output: Path(args.output).write_text(json.dumps(summary, ensure_ascii=False, indent=1)) - return 0 + return 0 if summary["pass"] else 1 if __name__ == "__main__":