diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml index bae9be48..101463de 100644 --- a/.github/workflows/application.yml +++ b/.github/workflows/application.yml @@ -19,7 +19,7 @@ jobs: - run: make student-test VENV=.venv acceptance: - name: Стенд Compose за HAProxy, smoke, отказ реплики, рестарт и отказы зависимостей + name: Стенд Compose с RustFS за HAProxy, smoke, вложения, отказ реплики, рестарт и отказы зависимостей runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -34,6 +34,14 @@ jobs: - name: Публичный smoke через балансировщик run: make test + - name: Вложение после остановки принявшей его реплики и при недоступном S3 + run: | + make scenario SCENARIO=failover-upload + make scenario SCENARIO=s3-outage + + - name: Приватный bucket, ограниченный ключ API и повторный bootstrap + run: make s3-check + - name: Несколько реплик на одном токене и гонка одного ключа run: make lb-check diff --git a/Makefile b/Makefile index c3b05ed5..59c8a4f2 100644 --- a/Makefile +++ b/Makefile @@ -10,14 +10,16 @@ COMPOSE ?= docker compose RATE ?= 100 API_IMAGE ?= mtusi-messenger:lab4 FROM_REV ?= -SCRIPTS_PY = scripts/probe.py scripts/limits.py +SCRIPTS_PY = scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py +SCENARIO ?= failover-upload PSQL = $(COMPOSE) exec -T postgres psql -U messenger -d messenger -X -q .PHONY: help check validate test load test-persistence test-failover \ install lint format typecheck student-test \ secrets up down reset-data migrate ps logs restore lb-check failover-evidence stack-evidence \ persistence-evidence storage-evidence cache-evidence test-faults seed explain \ - limits detection failover-each rolling-update level4 + limits detection failover-each rolling-update level4 \ + scenario s3-check attachments-evidence help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -25,6 +27,7 @@ help: @echo "secrets/up/down/migrate/ps/logs: стенд Docker Compose; reset-data: намеренное удаление volumes" @echo "restore: вернуть остановленные реплики; lb-check: реплики на одном токене и 20 гонок одного ключа" @echo "level4 = limits, lb-check, detection, failover-each, rolling-update [FROM_REV=<коммит>]; RATE=$(RATE) rps" + @echo "scenario SCENARIO=failover-upload|s3-outage → evidence/<сценарий>.json; s3-check; attachments-evidence" check: $(PYTHON) scripts/check_contract.py @@ -114,6 +117,16 @@ endif level4: limits lb-check detection failover-each rolling-update +scenario: + $(PYTHON) scripts/lab4_scenarios.py $(SCENARIO) --base-url "$(BASE_URL)" --restore --output evidence/$(SCENARIO).json + scripts/lb-wait.sh + +s3-check: + sh scripts/s3-check.sh > evidence/s3-security.txt; status=$$?; cat evidence/s3-security.txt; exit $$status + +attachments-evidence: + sh scripts/evidence-attachments.sh + failover-evidence: sh scripts/failover-evidence.sh "$(BASE_URL)" diff --git a/openspec/changes/add-lab4-image-attachments/tasks.md b/openspec/changes/add-lab4-image-attachments/tasks.md index 42b21066..a518473f 100644 --- a/openspec/changes/add-lab4-image-attachments/tasks.md +++ b/openspec/changes/add-lab4-image-attachments/tasks.md @@ -45,8 +45,8 @@ ## 8. Сценарии отказов и обвязка -- [ ] 8.1 `scripts/lab4_scenarios.py` (`failover-upload`, `s3-outage`), `scripts/s3-check.sh`, `scripts/evidence-attachments.sh`; цели Makefile `scenario`, `s3-check`, `attachments-evidence`; ruff для новых `scripts/*.py` -- [ ] 8.2 CI-job под лабу 4: сценарии вложений после smoke +- [x] 8.1 `scripts/lab4_scenarios.py` (`failover-upload`, `s3-outage`), `scripts/s3-check.sh`, `scripts/evidence-attachments.sh`; цели Makefile `scenario`, `s3-check`, `attachments-evidence`; ruff для новых `scripts/*.py` +- [x] 8.2 CI-job под лабу 4: сценарии вложений после smoke - [ ] 8.3 Проверка группы: `make test`, `make test-persistence ACTION_SCRIPT=scripts/restart.sh`, `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` по обеим репликам с `make restore`, `make scenario SCENARIO=failover-upload`, `make scenario SCENARIO=s3-outage`, `make s3-check` ## 9. Отчёт diff --git a/scripts/evidence-attachments.sh b/scripts/evidence-attachments.sh new file mode 100755 index 00000000..87c241a2 --- /dev/null +++ b/scripts/evidence-attachments.sh @@ -0,0 +1,40 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +psql() { docker compose exec -T postgres psql -U messenger -d messenger -X -c "$1"; } +value() { docker compose exec -T postgres psql -U messenger -d messenger -X -tA -c "$1"; } +cli() { docker compose run --rm --no-deps -T s3-cli "$@"; } +bucket=${S3_BUCKET:-messenger-attachments} +{ + echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)" + echo "\$ docker compose ps --format '{{.Service}} {{.Status}} {{.Ports}}'" + docker compose ps -a --format '{{.Service}} {{.Status}} {{.Ports}}' + for table in attachments message_attachments; do + echo "\$ psql -c '\\d $table'" + psql "\\d $table" + done + echo "\$ psql: столбцы bytea во всей схеме (хеши токенов и отпечатки, не файлы)" + psql "SELECT table_name, column_name FROM information_schema.columns WHERE table_schema = 'public' AND data_type = 'bytea' ORDER BY 1, 2" + echo "\$ psql: вложения по статусу и максимальный размер строки attachments" + psql "SELECT status, count(*), max(pg_column_size(a.*)) AS max_row_bytes FROM attachments a GROUP BY status" + echo "\$ rc object list --recursive root/$bucket/attachments/ | count" + cli object list --recursive "root/$bucket/attachments/" --json | grep -c '"key"' || true + echo "\$ rc object list root/$bucket/ (первые 3 объекта, ключи назначены сервером)" + cli object list --recursive "root/$bucket/attachments/" | head -3 + row=$(value "SELECT object_key || ' ' || sha256 || ' ' || size_bytes FROM attachments WHERE status = 'ready' ORDER BY created_at DESC LIMIT 1") + key=${row%% *} + echo "\$ сверка вне API: sha256 и размер последнего ready-вложения в PostgreSQL против байтов объекта в bucket" + echo "postgres: ${row#* }" + echo "s3: $(cli sh -c "rc object show root/$bucket/$key | sha256sum | cut -d' ' -f1") $(cli sh -c "rc object show root/$bucket/$key | wc -c" | tr -d ' ')" + echo "\$ docker inspect api: ReadonlyRootfs, tmpfs, volumes/bind mounts" + for id in $(docker compose ps -q api); do + docker inspect -f '{{.Config.Hostname}} ro={{.HostConfig.ReadonlyRootfs}} tmpfs={{json .HostConfig.Tmpfs}} mounts={{range .Mounts}}{{.Type}}:{{.Destination}} {{end}}' "$id" + done + echo "\$ docker inspect s3: volume данных" + docker inspect -f '{{range .Mounts}}{{.Type}}:{{.Name}}:{{.Destination}} {{end}}' "$(docker compose ps -q s3)" + echo "\$ секреты сервисов: root-ключи S3 только у s3/s3-init/s3-cli" + docker compose config --format json | python3 -c 'import json,sys +for name, service in sorted(json.load(sys.stdin)["services"].items()): + print(name, sorted(s["source"] for s in service.get("secrets", [])))' +} > evidence/storage-layout.txt +cat evidence/storage-layout.txt diff --git a/scripts/lab4_scenarios.py b/scripts/lab4_scenarios.py new file mode 100755 index 00000000..32c35d6f --- /dev/null +++ b/scripts/lab4_scenarios.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +import argparse +import hashlib +import http.client +import json +import os +import subprocess +import sys +import time +import uuid +from pathlib import Path +from urllib.parse import urlsplit + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +client = __import__("client") +Client, Failure, check = client.Client, client.Failure, client.check +FIXTURE = (ROOT / "tests/fixtures/sample.png").read_bytes() + + +def compose(*args, stdin=None): + cmd = ["docker", "compose", *args] + return subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, text=True, check=False) + + +def upload(c, actor, chat, expected=201): + path = f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png" + schema = "Attachment" if expected == 201 else None + return c.call( + "POST", + path, + expected, + actor=actor, + raw=FIXTURE, + headers={"Content-Type": "image/png"}, + schema=schema, + ) + + +def content(c, actor, attachment, expected=200): + return c.call("GET", f"/api/v1/attachments/{attachment['id']}/content", expected, actor=actor) + + +def timed(fn): + started = time.monotonic() + try: + response = fn() + except Failure as exc: + return { + "status": "failure", + "seconds": round(time.monotonic() - started, 3), + "error": str(exc), + } + seconds = round(time.monotonic() - started, 3) + return { + "status": response["status"], + "seconds": seconds, + "instance": response["headers"].get("X-Instance-Id"), + } + + +def wait(fn, seconds): + deadline = time.monotonic() + seconds + while True: + try: + return fn() + except Failure: + if time.monotonic() > deadline: + raise + time.sleep(0.2) + + +def failover_upload(c, args): + alice, bob, _, chat = c.bootstrap() + sent = upload(c, alice, chat) + att, target = sent["json"], sent["headers"]["X-Instance-Id"] + started = time.monotonic() + hook = subprocess.run( + [str(ROOT / "scripts/stop-one.sh")], + cwd=ROOT, + env=dict(os.environ, TARGET_INSTANCE_ID=target), + capture_output=True, + text=True, + check=False, + ) + check(hook.returncode == 0, f"stop-one.sh failed: {hook.stderr.strip()}") + hook_seconds = time.monotonic() - started + got = wait(lambda: content(c, bob, att), 15) + recovered = time.monotonic() - started + served = got["headers"].get("X-Instance-Id") + check(served != target, "download served by the stopped replica") + digest = hashlib.sha256(got["bytes"]).hexdigest() + check( + digest == att["sha256"] == hashlib.sha256(FIXTURE).hexdigest(), + "sha256 mismatch after failover", + ) + if args.restore: + compose("up", "-d", "--wait", "--no-recreate", "--no-deps", args.service) + return { + "scenario": "failover-upload", + "upload_instance": target, + "download_instance": served, + "sha256_equal": True, + "hook_seconds": round(hook_seconds, 3), + "download_ok_after_seconds": round(recovered, 3), + "restored": args.restore, + } + + +def s3_outage(c, args): + alice, bob, _, chat = c.bootstrap() + att = upload(c, alice, chat)["json"] + check( + compose("stop", "-t", "10", args.s3_service).returncode == 0, "docker compose stop failed" + ) + try: + during = { + "upload": timed(lambda: upload(c, alice, chat, [503])), + "metadata": timed( + lambda: c.call("GET", f"/api/v1/attachments/{att['id']}", [200, 503], actor=bob) + ), + "download": timed(lambda: content(c, bob, att, 503)), + "text_message": timed(lambda: c.message(alice, chat, "S3 недоступен, текст работает")), + "health_ready": timed(lambda: c.call("GET", "/health/ready", [200, 503])), + } + finally: + started = time.monotonic() + compose("up", "-d", "--wait", "--no-deps", "--no-recreate", args.s3_service) + recovered = round(time.monotonic() - started, 3) + got = wait(lambda: content(c, bob, att), 30) + check(got["bytes"] == FIXTURE, "attachment changed across S3 outage") + expected = {"upload": 503, "download": 503, "text_message": 201} + return { + "scenario": "s3-outage", + "during_outage": during, + "expected": expected, + "as_expected": all(during[k]["status"] == v for k, v in expected.items()), + "s3_back_healthy_seconds": recovered, + "download_after_recovery": "sha256 equal", + } + + +def abort_upload(c, args): + alice, _, _, chat = c.bootstrap() + name = f"abort-{uuid.uuid4().hex}.png" + parts = urlsplit(c.base) + factory = http.client.HTTPSConnection if parts.scheme == "https" else http.client.HTTPConnection + conn = factory(parts.netloc, timeout=10) + conn.putrequest("POST", f"/api/v1/chats/{chat['id']}/attachments?filename={name}") + headers = { + "Authorization": f"Bearer {alice['token']}", + "Content-Type": "image/png", + "Content-Length": str(args.declared), + "Accept": "application/json", + } + for key, value in headers.items(): + conn.putheader(key, value) + conn.endheaders() + conn.send(FIXTURE[:8] + os.urandom(args.sent - 8)) + conn.close() + time.sleep(args.settle) + user, db = os.environ.get("PGUSER", "messenger"), os.environ.get("PGDATABASE", "messenger") + sql = "SELECT status, count(*) FROM attachments WHERE filename = :'name' GROUP BY status;\n" + out = compose( + "exec", + "-T", + "postgres", + "psql", + "-qAt", + "-F", + " ", + "-U", + user, + "-d", + db, + "-v", + f"name={name}", + stdin=sql, + ) + rows = out.stdout.split() + counts = dict(zip(rows[::2], (int(x) for x in rows[1::2]))) + alive = c.call("GET", "/api/v1/users/me", actor=alice, schema="User")["status"] + return { + "scenario": "abort-upload", + "declared_bytes": args.declared, + "sent_bytes": args.sent, + "settle_seconds": args.settle, + "rows_by_status": counts, + "ready_rows": counts.get("ready", 0), + "api_still_serves": alive == 200, + } + + +def main(): + p = argparse.ArgumentParser( + description="Lab4 failure scenarios over the public HTTP contract plus docker compose" + ) + p.add_argument("scenario", choices=["failover-upload", "s3-outage", "abort-upload"]) + p.add_argument("--base-url", default=None) + p.add_argument("--ca-file", default=None) + p.add_argument("--timeout", type=float, default=15) + p.add_argument("--service", default="api") + p.add_argument("--s3-service", default="s3") + p.add_argument( + "--restore", action="store_true", help="failover-upload: start the stopped replica again" + ) + p.add_argument("--declared", type=int, default=5 * 1024 * 1024) + p.add_argument("--sent", type=int, default=256 * 1024) + p.add_argument("--settle", type=float, default=3) + p.add_argument("--output", metavar="FILE") + args = p.parse_args() + c = Client(args.base_url, args.ca_file, timeout=args.timeout) + run = { + "failover-upload": failover_upload, + "s3-outage": s3_outage, + "abort-upload": abort_upload, + }[args.scenario] + try: + report = {"ok": True, **run(c, args)} + except Failure as exc: + report = {"scenario": args.scenario, "ok": False, "error": str(exc)} + text = json.dumps(report, ensure_ascii=False, indent=2) + if args.output: + Path(args.output).write_text(text + "\n") + print(text) + sys.exit(0 if report["ok"] else 1) + + +if __name__ == "__main__": + main() diff --git a/scripts/s3-check.sh b/scripts/s3-check.sh new file mode 100755 index 00000000..fb6c6db8 --- /dev/null +++ b/scripts/s3-check.sh @@ -0,0 +1,62 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +cli() { docker compose run --rm --no-deps -T s3-cli "$@"; } +snapshot() { + cli sh -s <<'EOF' | grep -v '_date' +ak=$(cat /run/secrets/s3_access_key_id) +for c in "bucket list root" "bucket anonymous get root/$S3_BUCKET" "admin policy info root messenger-api" "admin user info root $ak" "bucket lifecycle rule export root/$S3_BUCKET"; do + rc $c --json +done | sed "s/$ak//g" +EOF +} +failed=0 +verdict() { if [ "$2" = "$3" ]; then echo "PASS $1: $3"; else echo "FAIL $1: ожидалось $2, получено $3"; failed=1; fi; } +echo "== $(date -u +%FT%TZ) S3 privacy and bootstrap checks" +verdict "s3: опубликованные host-порты" "нет" "$(docker inspect -f '{{json .NetworkSettings.Ports}}' "$(docker compose ps -q s3)" | grep -q HostPort && echo есть || echo нет)" +verdict "s3: консоль :9001 внутри контейнера" "000" "$(docker compose exec -T s3 curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:9001/ || true)" +before=$(snapshot) +first=$(docker compose run --rm --no-deps -T s3-init >/dev/null 2>&1 && echo 0 || echo 1) +second=$(docker compose run --rm --no-deps -T s3-init 2>/dev/null || echo FAILED) +after=$(snapshot) +verdict "s3-init: повторный запуск 1" "0" "$first" +verdict "s3-init: повторный запуск 2" "0" "$(echo "$second" | grep -q '^s3-init:' && echo 0 || echo 1)" +verdict "s3-init: состояние IAM/bucket не изменилось" "same" "$([ "$before" = "$after" ] && echo same || echo changed)" +cli sh -s <<'EOF' || failed=1 +b=$S3_BUCKET +key="attachments/s3-check-$(date +%s)-$$" +other="s3-check-$(date +%s)-$$" +head -c 4096 /dev/urandom > /tmp/probe +result=0 +expect() { + want=$1; name=$2; shift 2 + if "$@" >/tmp/out 2>&1; then got=allow; elif grep -qE '403|AccessDenied|set_bucket_policy: service error' /tmp/out; then got=deny; else got=error; fi + if [ "$got" = "$want" ]; then echo "PASS $name: $got"; else echo "FAIL $name: ожидалось $want, получено $got"; result=1; fi +} +expect allow "api: PutObject" rc put /tmp/probe "api/$b/$key" +expect allow "api: HeadObject" rc object stat "api/$b/$key" +expect allow "api: GetObject" rc object show "api/$b/$key" +expect allow "api: ListBucket" rc object list "api/$b/attachments/" +expect deny "api: CreateBucket" rc bucket create "api/$other" +expect deny "api: DeleteBucket" rc bucket remove "api/$b" +expect deny "api: PutBucketPolicy (public)" rc bucket anonymous set public "api/$b" +expect deny "api: PutBucketLifecycle" rc bucket lifecycle rule import "api/$b" /s3/lifecycle.json +expect deny "api: admin ListUsers" rc admin user list api +rc bucket create --ignore-existing "root/$other" >/dev/null +expect deny "api: PutObject в чужой bucket" rc put /tmp/probe "api/$other/x" +expect deny "api: ListBucket чужого bucket" rc object list "api/$other" +rc bucket remove "root/$other" >/dev/null +expect deny "anonymous: HeadObject" rc object stat "anon/$b/$key" +expect deny "anonymous: ListBucket" rc object list "anon/$b" +expect deny "anonymous: PutObject" rc put /tmp/probe "anon/$b/attachments/anonymous" +status=$(wget -S -O /dev/null "$S3_ENDPOINT/$b/$key" 2>&1 | sed -n 's/^ *HTTP\/[0-9.]* \([0-9]*\).*/\1/p' | head -n 1) +if [ "$status" = 403 ]; then echo "PASS anonymous: прямой HTTP GET объекта из отдельного контейнера: $status"; else echo "FAIL anonymous: прямой HTTP GET: $status"; result=1; fi +permission=$(rc bucket anonymous get "root/$b" --json | sed -n 's/.*"permission": *"\([a-z]*\)".*/\1/p') +if [ "$permission" = private ]; then echo "PASS bucket: anonymous policy: $permission"; else echo "FAIL bucket: anonymous policy: $permission"; result=1; fi +expect allow "api: DeleteObject" rc object remove --force "api/$b/$key" +exit $result +EOF +echo "== IAM/bucket snapshot (без секретов)" +echo "$after" +if [ "$failed" != 0 ]; then echo "RESULT: FAIL"; exit 1; fi +echo "RESULT: PASS"