Add the attachment failure scenarios and S3 checks

lab4_scenarios.py drives the public contract through tests/client.py:
failover-upload stops the replica that answered the upload with the
stop-one hook and downloads through another one comparing SHA-256, and
s3-outage stops S3 and records status and time of upload, download,
metadata, a text message and readiness before bringing it back. Both
print JSON with an ok field. s3-check.sh checks published ports, the
console, two more bootstrap runs without state changes, the allowed
and denied operations of the API key and anonymous access including a
plain HTTP GET from a separate container. evidence-attachments.sh
shows where the bytes live and compares a stored object with its
metadata outside the API.

Makefile gains scenario, s3-check and attachments-evidence, ruff covers
the new script, and the CI acceptance job runs both scenarios and the
S3 check after the smoke test.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Artyom
2026-10-28 20:50:03 +03:00
co-authored by Claude Opus 5.5
parent 5aa766acff
commit 24f9d6195b
6 changed files with 358 additions and 5 deletions
+9 -1
View File
@@ -19,7 +19,7 @@ jobs:
- run: make student-test VENV=.venv
acceptance:
name: Стенд Compose за HAProxy, smoke, отказ реплики, рестарт и отказы зависимостей
name: Стенд Compose с RustFS за HAProxy, smoke, вложения, отказ реплики, рестарт и отказы зависимостей
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
@@ -34,6 +34,14 @@ jobs:
- name: Публичный smoke через балансировщик
run: make test
- name: Вложение после остановки принявшей его реплики и при недоступном S3
run: |
make scenario SCENARIO=failover-upload
make scenario SCENARIO=s3-outage
- name: Приватный bucket, ограниченный ключ API и повторный bootstrap
run: make s3-check
- name: Несколько реплик на одном токене и гонка одного ключа
run: make lb-check
+15 -2
View File
@@ -10,14 +10,16 @@ COMPOSE ?= docker compose
RATE ?= 100
API_IMAGE ?= mtusi-messenger:lab4
FROM_REV ?=
SCRIPTS_PY = scripts/probe.py scripts/limits.py
SCRIPTS_PY = scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py
SCENARIO ?= failover-upload
PSQL = $(COMPOSE) exec -T postgres psql -U messenger -d messenger -X -q
.PHONY: help check validate test load test-persistence test-failover \
install lint format typecheck student-test \
secrets up down reset-data migrate ps logs restore lb-check failover-evidence stack-evidence \
persistence-evidence storage-evidence cache-evidence test-faults seed explain \
limits detection failover-each rolling-update level4
limits detection failover-each rolling-update level4 \
scenario s3-check attachments-evidence
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@@ -25,6 +27,7 @@ help:
@echo "secrets/up/down/migrate/ps/logs: стенд Docker Compose; reset-data: намеренное удаление volumes"
@echo "restore: вернуть остановленные реплики; lb-check: реплики на одном токене и 20 гонок одного ключа"
@echo "level4 = limits, lb-check, detection, failover-each, rolling-update [FROM_REV=<коммит>]; RATE=$(RATE) rps"
@echo "scenario SCENARIO=failover-upload|s3-outage → evidence/<сценарий>.json; s3-check; attachments-evidence"
check:
$(PYTHON) scripts/check_contract.py
@@ -114,6 +117,16 @@ endif
level4: limits lb-check detection failover-each rolling-update
scenario:
$(PYTHON) scripts/lab4_scenarios.py $(SCENARIO) --base-url "$(BASE_URL)" --restore --output evidence/$(SCENARIO).json
scripts/lb-wait.sh
s3-check:
sh scripts/s3-check.sh > evidence/s3-security.txt; status=$$?; cat evidence/s3-security.txt; exit $$status
attachments-evidence:
sh scripts/evidence-attachments.sh
failover-evidence:
sh scripts/failover-evidence.sh "$(BASE_URL)"
@@ -45,8 +45,8 @@
## 8. Сценарии отказов и обвязка
- [ ] 8.1 `scripts/lab4_scenarios.py` (`failover-upload`, `s3-outage`), `scripts/s3-check.sh`, `scripts/evidence-attachments.sh`; цели Makefile `scenario`, `s3-check`, `attachments-evidence`; ruff для новых `scripts/*.py`
- [ ] 8.2 CI-job под лабу 4: сценарии вложений после smoke
- [x] 8.1 `scripts/lab4_scenarios.py` (`failover-upload`, `s3-outage`), `scripts/s3-check.sh`, `scripts/evidence-attachments.sh`; цели Makefile `scenario`, `s3-check`, `attachments-evidence`; ruff для новых `scripts/*.py`
- [x] 8.2 CI-job под лабу 4: сценарии вложений после smoke
- [ ] 8.3 Проверка группы: `make test`, `make test-persistence ACTION_SCRIPT=scripts/restart.sh`, `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` по обеим репликам с `make restore`, `make scenario SCENARIO=failover-upload`, `make scenario SCENARIO=s3-outage`, `make s3-check`
## 9. Отчёт
+40
View File
@@ -0,0 +1,40 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
psql() { docker compose exec -T postgres psql -U messenger -d messenger -X -c "$1"; }
value() { docker compose exec -T postgres psql -U messenger -d messenger -X -tA -c "$1"; }
cli() { docker compose run --rm --no-deps -T s3-cli "$@"; }
bucket=${S3_BUCKET:-messenger-attachments}
{
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"
echo "\$ docker compose ps --format '{{.Service}} {{.Status}} {{.Ports}}'"
docker compose ps -a --format '{{.Service}} {{.Status}} {{.Ports}}'
for table in attachments message_attachments; do
echo "\$ psql -c '\\d $table'"
psql "\\d $table"
done
echo "\$ psql: столбцы bytea во всей схеме (хеши токенов и отпечатки, не файлы)"
psql "SELECT table_name, column_name FROM information_schema.columns WHERE table_schema = 'public' AND data_type = 'bytea' ORDER BY 1, 2"
echo "\$ psql: вложения по статусу и максимальный размер строки attachments"
psql "SELECT status, count(*), max(pg_column_size(a.*)) AS max_row_bytes FROM attachments a GROUP BY status"
echo "\$ rc object list --recursive root/$bucket/attachments/ | count"
cli object list --recursive "root/$bucket/attachments/" --json | grep -c '"key"' || true
echo "\$ rc object list root/$bucket/ (первые 3 объекта, ключи назначены сервером)"
cli object list --recursive "root/$bucket/attachments/" | head -3
row=$(value "SELECT object_key || ' ' || sha256 || ' ' || size_bytes FROM attachments WHERE status = 'ready' ORDER BY created_at DESC LIMIT 1")
key=${row%% *}
echo "\$ сверка вне API: sha256 и размер последнего ready-вложения в PostgreSQL против байтов объекта в bucket"
echo "postgres: ${row#* }"
echo "s3: $(cli sh -c "rc object show root/$bucket/$key | sha256sum | cut -d' ' -f1") $(cli sh -c "rc object show root/$bucket/$key | wc -c" | tr -d ' ')"
echo "\$ docker inspect api: ReadonlyRootfs, tmpfs, volumes/bind mounts"
for id in $(docker compose ps -q api); do
docker inspect -f '{{.Config.Hostname}} ro={{.HostConfig.ReadonlyRootfs}} tmpfs={{json .HostConfig.Tmpfs}} mounts={{range .Mounts}}{{.Type}}:{{.Destination}} {{end}}' "$id"
done
echo "\$ docker inspect s3: volume данных"
docker inspect -f '{{range .Mounts}}{{.Type}}:{{.Name}}:{{.Destination}} {{end}}' "$(docker compose ps -q s3)"
echo "\$ секреты сервисов: root-ключи S3 только у s3/s3-init/s3-cli"
docker compose config --format json | python3 -c 'import json,sys
for name, service in sorted(json.load(sys.stdin)["services"].items()):
print(name, sorted(s["source"] for s in service.get("secrets", [])))'
} > evidence/storage-layout.txt
cat evidence/storage-layout.txt
+230
View File
@@ -0,0 +1,230 @@
#!/usr/bin/env python3
import argparse
import hashlib
import http.client
import json
import os
import subprocess
import sys
import time
import uuid
from pathlib import Path
from urllib.parse import urlsplit
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
client = __import__("client")
Client, Failure, check = client.Client, client.Failure, client.check
FIXTURE = (ROOT / "tests/fixtures/sample.png").read_bytes()
def compose(*args, stdin=None):
cmd = ["docker", "compose", *args]
return subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, text=True, check=False)
def upload(c, actor, chat, expected=201):
path = f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png"
schema = "Attachment" if expected == 201 else None
return c.call(
"POST",
path,
expected,
actor=actor,
raw=FIXTURE,
headers={"Content-Type": "image/png"},
schema=schema,
)
def content(c, actor, attachment, expected=200):
return c.call("GET", f"/api/v1/attachments/{attachment['id']}/content", expected, actor=actor)
def timed(fn):
started = time.monotonic()
try:
response = fn()
except Failure as exc:
return {
"status": "failure",
"seconds": round(time.monotonic() - started, 3),
"error": str(exc),
}
seconds = round(time.monotonic() - started, 3)
return {
"status": response["status"],
"seconds": seconds,
"instance": response["headers"].get("X-Instance-Id"),
}
def wait(fn, seconds):
deadline = time.monotonic() + seconds
while True:
try:
return fn()
except Failure:
if time.monotonic() > deadline:
raise
time.sleep(0.2)
def failover_upload(c, args):
alice, bob, _, chat = c.bootstrap()
sent = upload(c, alice, chat)
att, target = sent["json"], sent["headers"]["X-Instance-Id"]
started = time.monotonic()
hook = subprocess.run(
[str(ROOT / "scripts/stop-one.sh")],
cwd=ROOT,
env=dict(os.environ, TARGET_INSTANCE_ID=target),
capture_output=True,
text=True,
check=False,
)
check(hook.returncode == 0, f"stop-one.sh failed: {hook.stderr.strip()}")
hook_seconds = time.monotonic() - started
got = wait(lambda: content(c, bob, att), 15)
recovered = time.monotonic() - started
served = got["headers"].get("X-Instance-Id")
check(served != target, "download served by the stopped replica")
digest = hashlib.sha256(got["bytes"]).hexdigest()
check(
digest == att["sha256"] == hashlib.sha256(FIXTURE).hexdigest(),
"sha256 mismatch after failover",
)
if args.restore:
compose("up", "-d", "--wait", "--no-recreate", "--no-deps", args.service)
return {
"scenario": "failover-upload",
"upload_instance": target,
"download_instance": served,
"sha256_equal": True,
"hook_seconds": round(hook_seconds, 3),
"download_ok_after_seconds": round(recovered, 3),
"restored": args.restore,
}
def s3_outage(c, args):
alice, bob, _, chat = c.bootstrap()
att = upload(c, alice, chat)["json"]
check(
compose("stop", "-t", "10", args.s3_service).returncode == 0, "docker compose stop failed"
)
try:
during = {
"upload": timed(lambda: upload(c, alice, chat, [503])),
"metadata": timed(
lambda: c.call("GET", f"/api/v1/attachments/{att['id']}", [200, 503], actor=bob)
),
"download": timed(lambda: content(c, bob, att, 503)),
"text_message": timed(lambda: c.message(alice, chat, "S3 недоступен, текст работает")),
"health_ready": timed(lambda: c.call("GET", "/health/ready", [200, 503])),
}
finally:
started = time.monotonic()
compose("up", "-d", "--wait", "--no-deps", "--no-recreate", args.s3_service)
recovered = round(time.monotonic() - started, 3)
got = wait(lambda: content(c, bob, att), 30)
check(got["bytes"] == FIXTURE, "attachment changed across S3 outage")
expected = {"upload": 503, "download": 503, "text_message": 201}
return {
"scenario": "s3-outage",
"during_outage": during,
"expected": expected,
"as_expected": all(during[k]["status"] == v for k, v in expected.items()),
"s3_back_healthy_seconds": recovered,
"download_after_recovery": "sha256 equal",
}
def abort_upload(c, args):
alice, _, _, chat = c.bootstrap()
name = f"abort-{uuid.uuid4().hex}.png"
parts = urlsplit(c.base)
factory = http.client.HTTPSConnection if parts.scheme == "https" else http.client.HTTPConnection
conn = factory(parts.netloc, timeout=10)
conn.putrequest("POST", f"/api/v1/chats/{chat['id']}/attachments?filename={name}")
headers = {
"Authorization": f"Bearer {alice['token']}",
"Content-Type": "image/png",
"Content-Length": str(args.declared),
"Accept": "application/json",
}
for key, value in headers.items():
conn.putheader(key, value)
conn.endheaders()
conn.send(FIXTURE[:8] + os.urandom(args.sent - 8))
conn.close()
time.sleep(args.settle)
user, db = os.environ.get("PGUSER", "messenger"), os.environ.get("PGDATABASE", "messenger")
sql = "SELECT status, count(*) FROM attachments WHERE filename = :'name' GROUP BY status;\n"
out = compose(
"exec",
"-T",
"postgres",
"psql",
"-qAt",
"-F",
" ",
"-U",
user,
"-d",
db,
"-v",
f"name={name}",
stdin=sql,
)
rows = out.stdout.split()
counts = dict(zip(rows[::2], (int(x) for x in rows[1::2])))
alive = c.call("GET", "/api/v1/users/me", actor=alice, schema="User")["status"]
return {
"scenario": "abort-upload",
"declared_bytes": args.declared,
"sent_bytes": args.sent,
"settle_seconds": args.settle,
"rows_by_status": counts,
"ready_rows": counts.get("ready", 0),
"api_still_serves": alive == 200,
}
def main():
p = argparse.ArgumentParser(
description="Lab4 failure scenarios over the public HTTP contract plus docker compose"
)
p.add_argument("scenario", choices=["failover-upload", "s3-outage", "abort-upload"])
p.add_argument("--base-url", default=None)
p.add_argument("--ca-file", default=None)
p.add_argument("--timeout", type=float, default=15)
p.add_argument("--service", default="api")
p.add_argument("--s3-service", default="s3")
p.add_argument(
"--restore", action="store_true", help="failover-upload: start the stopped replica again"
)
p.add_argument("--declared", type=int, default=5 * 1024 * 1024)
p.add_argument("--sent", type=int, default=256 * 1024)
p.add_argument("--settle", type=float, default=3)
p.add_argument("--output", metavar="FILE")
args = p.parse_args()
c = Client(args.base_url, args.ca_file, timeout=args.timeout)
run = {
"failover-upload": failover_upload,
"s3-outage": s3_outage,
"abort-upload": abort_upload,
}[args.scenario]
try:
report = {"ok": True, **run(c, args)}
except Failure as exc:
report = {"scenario": args.scenario, "ok": False, "error": str(exc)}
text = json.dumps(report, ensure_ascii=False, indent=2)
if args.output:
Path(args.output).write_text(text + "\n")
print(text)
sys.exit(0 if report["ok"] else 1)
if __name__ == "__main__":
main()
+62
View File
@@ -0,0 +1,62 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
cli() { docker compose run --rm --no-deps -T s3-cli "$@"; }
snapshot() {
cli sh -s <<'EOF' | grep -v '_date'
ak=$(cat /run/secrets/s3_access_key_id)
for c in "bucket list root" "bucket anonymous get root/$S3_BUCKET" "admin policy info root messenger-api" "admin user info root $ak" "bucket lifecycle rule export root/$S3_BUCKET"; do
rc $c --json
done | sed "s/$ak/<api-key>/g"
EOF
}
failed=0
verdict() { if [ "$2" = "$3" ]; then echo "PASS $1: $3"; else echo "FAIL $1: ожидалось $2, получено $3"; failed=1; fi; }
echo "== $(date -u +%FT%TZ) S3 privacy and bootstrap checks"
verdict "s3: опубликованные host-порты" "нет" "$(docker inspect -f '{{json .NetworkSettings.Ports}}' "$(docker compose ps -q s3)" | grep -q HostPort && echo есть || echo нет)"
verdict "s3: консоль :9001 внутри контейнера" "000" "$(docker compose exec -T s3 curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:9001/ || true)"
before=$(snapshot)
first=$(docker compose run --rm --no-deps -T s3-init >/dev/null 2>&1 && echo 0 || echo 1)
second=$(docker compose run --rm --no-deps -T s3-init 2>/dev/null || echo FAILED)
after=$(snapshot)
verdict "s3-init: повторный запуск 1" "0" "$first"
verdict "s3-init: повторный запуск 2" "0" "$(echo "$second" | grep -q '^s3-init:' && echo 0 || echo 1)"
verdict "s3-init: состояние IAM/bucket не изменилось" "same" "$([ "$before" = "$after" ] && echo same || echo changed)"
cli sh -s <<'EOF' || failed=1
b=$S3_BUCKET
key="attachments/s3-check-$(date +%s)-$$"
other="s3-check-$(date +%s)-$$"
head -c 4096 /dev/urandom > /tmp/probe
result=0
expect() {
want=$1; name=$2; shift 2
if "$@" >/tmp/out 2>&1; then got=allow; elif grep -qE '403|AccessDenied|set_bucket_policy: service error' /tmp/out; then got=deny; else got=error; fi
if [ "$got" = "$want" ]; then echo "PASS $name: $got"; else echo "FAIL $name: ожидалось $want, получено $got"; result=1; fi
}
expect allow "api: PutObject" rc put /tmp/probe "api/$b/$key"
expect allow "api: HeadObject" rc object stat "api/$b/$key"
expect allow "api: GetObject" rc object show "api/$b/$key"
expect allow "api: ListBucket" rc object list "api/$b/attachments/"
expect deny "api: CreateBucket" rc bucket create "api/$other"
expect deny "api: DeleteBucket" rc bucket remove "api/$b"
expect deny "api: PutBucketPolicy (public)" rc bucket anonymous set public "api/$b"
expect deny "api: PutBucketLifecycle" rc bucket lifecycle rule import "api/$b" /s3/lifecycle.json
expect deny "api: admin ListUsers" rc admin user list api
rc bucket create --ignore-existing "root/$other" >/dev/null
expect deny "api: PutObject в чужой bucket" rc put /tmp/probe "api/$other/x"
expect deny "api: ListBucket чужого bucket" rc object list "api/$other"
rc bucket remove "root/$other" >/dev/null
expect deny "anonymous: HeadObject" rc object stat "anon/$b/$key"
expect deny "anonymous: ListBucket" rc object list "anon/$b"
expect deny "anonymous: PutObject" rc put /tmp/probe "anon/$b/attachments/anonymous"
status=$(wget -S -O /dev/null "$S3_ENDPOINT/$b/$key" 2>&1 | sed -n 's/^ *HTTP\/[0-9.]* \([0-9]*\).*/\1/p' | head -n 1)
if [ "$status" = 403 ]; then echo "PASS anonymous: прямой HTTP GET объекта из отдельного контейнера: $status"; else echo "FAIL anonymous: прямой HTTP GET: $status"; result=1; fi
permission=$(rc bucket anonymous get "root/$b" --json | sed -n 's/.*"permission": *"\([a-z]*\)".*/\1/p')
if [ "$permission" = private ]; then echo "PASS bucket: anonymous policy: $permission"; else echo "FAIL bucket: anonymous policy: $permission"; result=1; fi
expect allow "api: DeleteObject" rc object remove --force "api/$b/$key"
exit $result
EOF
echo "== IAM/bucket snapshot (без секретов)"
echo "$after"
if [ "$failed" != 0 ]; then echo "RESULT: FAIL"; exit 1; fi
echo "RESULT: PASS"