Аудит F10, F14, F15, F16: rolling update, тесты lab4, CI приложения, тест памяти

- F10: rolling-update.sh собирает migrate (единственный сервис с build),
  проверяет APP_VERSION в образе, в новой реплике и в ответах через LB;
  реально выполнено обновление 1 -> 2 (evidence/rolling-update.txt)
- F14: s3_outage.py переписан под lab5 (503 в API, retry в worker, принятое
  задание доходит до ready); устаревший consistency_faults.py удалён
- F15: .github/workflows/app.yml — сборка, готовность, smoke и регрессии на
  изолированном стенде, логи при ошибке, очистка
- F16: upload_memory.py с явными критериями и ненулевым кодом выхода, размер
  в каждой записи, progressive JPEG в смеси
- run_all.sh: все регрессии аудита, make validate, регрессия lab3, rolling update
This commit is contained in:
2026-11-13 21:02:54 +03:00
parent a71b445019
commit 5d60a85cc1
10 changed files with 822 additions and 280 deletions
+42
View File
@@ -0,0 +1,42 @@
# Проверка РЕАЛИЗАЦИИ (аудит F15); contracts.yml проверяет только skeleton.
# Изолированный стенд Compose с одноразовыми секретами: сборка, готовность, публичный smoke,
# регрессии аудита, не требующие ручного вмешательства. Логи при ошибке, очистка всегда.
# Секреты CI не нужны: init-secrets.sh генерирует их на раннере. pull_request_target не используется.
# Ограничение: локально workflow не исполнялся (нет раннера); проверен синтаксис YAML.
name: Application
on: [push, pull_request]
permissions:
contents: read
jobs:
app:
runs-on: ubuntu-latest
timeout-minutes: 40
env:
COMPOSE_PROJECT_NAME: mtusi-lab5-ci
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Secrets and config
run: ./scripts/init-secrets.sh && docker compose --env-file .env config --quiet
- name: Build and start
run: docker compose --env-file .env up -d --build --wait --wait-timeout 300
- name: Public smoke
run: make check test
- name: Audit regressions (HTTP)
run: python3 student-tests/api_negative_lab5.py
- name: Variants and transform limits
run: |
docker compose --profile tools run --rm -T checks /app/student-tests/variants_check.py --base-url http://lb:8080
docker compose --profile tools run --rm -T checks /app/student-tests/transform_limits.py
- name: Worker outage and redelivery
run: python3 student-tests/worker_outage.py
- name: Outbox crash windows
run: python3 student-tests/outbox_gap.py
- name: Logs on failure
if: failure()
run: docker compose logs --no-color --tail 300
- name: Cleanup
if: always()
run: docker compose --profile tools down -v --remove-orphans
+292
View File
@@ -0,0 +1,292 @@
{
"label": "",
"concurrency": 4,
"duration_s": 20.0,
"request_timeout_s": 2.0,
"action": {
"command": "env TARGET_INSTANCE_ID=78ea3303cfbc scripts/stop-one.sh",
"started_s": 5.01,
"finished_s": 8.521,
"exit_code": 0
},
"baseline_ok_rps": 486.0,
"overall_ok_rps": 262.1,
"requests_ok": 5242,
"errors_total": 0,
"error_rate": 0.0,
"errors_by_kind": {},
"error_window_s": null,
"recovery_s": 0.0,
"post_action_ok_rps": 137.8,
"capacity_ratio": 0.28,
"latency_ok_ms": {
"p50": 5.1,
"p95": 76.6,
"p99": 94.2
},
"target_instance": "78ea3303cfbc",
"target_last_seen_s": 1.629,
"lb_events": [
{
"t": 1.7,
"server": "api1",
"addr": "172.23.0.9:8080",
"from": "UP",
"to": "DOWN"
},
{
"t": 6.02,
"server": "api1",
"addr": "",
"from": "DOWN",
"to": "MAINT"
}
],
"posts_retried_same_key": 0,
"confirmed_messages": 2622,
"history_messages": 2622,
"lost_confirmed": 0,
"duplicate_ids": 0,
"duplicate_texts": 0,
"timeline": [
{
"t": 0,
"ok": 448,
"err": 0,
"instances": {
"78ea3303cfbc": 224,
"ad0d1b2c4289": 224
},
"versions": {
"2": 448
}
},
{
"t": 1,
"ok": 545,
"err": 0,
"instances": {
"78ea3303cfbc": 273,
"ad0d1b2c4289": 272
},
"versions": {
"2": 545
}
},
{
"t": 2,
"ok": 404,
"err": 0,
"instances": {
"ad0d1b2c4289": 204,
"78ea3303cfbc": 200
},
"versions": {
"2": 404
}
},
{
"t": 3,
"ok": 601,
"err": 0,
"instances": {
"78ea3303cfbc": 302,
"ad0d1b2c4289": 299
},
"versions": {
"2": 601
}
},
{
"t": 4,
"ok": 394,
"err": 0,
"instances": {
"78ea3303cfbc": 198,
"ad0d1b2c4289": 196
},
"versions": {
"2": 394
}
},
{
"t": 5,
"ok": 476,
"err": 0,
"instances": {
"ad0d1b2c4289": 239,
"78ea3303cfbc": 237
},
"versions": {
"2": 476
}
},
{
"t": 6,
"ok": 468,
"err": 0,
"instances": {
"78ea3303cfbc": 206,
"ad0d1b2c4289": 262
},
"versions": {
"2": 468
}
},
{
"t": 7,
"ok": 166,
"err": 0,
"instances": {
"ad0d1b2c4289": 166
},
"versions": {
"2": 166
}
},
{
"t": 8,
"ok": 184,
"err": 0,
"instances": {
"ad0d1b2c4289": 184
},
"versions": {
"2": 184
}
},
{
"t": 9,
"ok": 151,
"err": 0,
"instances": {
"ad0d1b2c4289": 151
},
"versions": {
"2": 151
}
},
{
"t": 10,
"ok": 79,
"err": 0,
"instances": {
"ad0d1b2c4289": 79
},
"versions": {
"2": 79
}
},
{
"t": 11,
"ok": 93,
"err": 0,
"instances": {
"ad0d1b2c4289": 93
},
"versions": {
"2": 93
}
},
{
"t": 12,
"ok": 172,
"err": 0,
"instances": {
"ad0d1b2c4289": 172
},
"versions": {
"2": 172
}
},
{
"t": 13,
"ok": 201,
"err": 0,
"instances": {
"ad0d1b2c4289": 201
},
"versions": {
"2": 201
}
},
{
"t": 14,
"ok": 149,
"err": 0,
"instances": {
"ad0d1b2c4289": 149
},
"versions": {
"2": 149
}
},
{
"t": 15,
"ok": 171,
"err": 0,
"instances": {
"ad0d1b2c4289": 171
},
"versions": {
"2": 171
}
},
{
"t": 16,
"ok": 115,
"err": 0,
"instances": {
"ad0d1b2c4289": 115
},
"versions": {
"2": 115
}
},
{
"t": 17,
"ok": 108,
"err": 0,
"instances": {
"ad0d1b2c4289": 108
},
"versions": {
"2": 108
}
},
{
"t": 18,
"ok": 152,
"err": 0,
"instances": {
"ad0d1b2c4289": 152
},
"versions": {
"2": 152
}
},
{
"t": 19,
"ok": 157,
"err": 0,
"instances": {
"ad0d1b2c4289": 157
},
"versions": {
"2": 157
}
},
{
"t": 20,
"ok": 8,
"err": 0,
"instances": {
"ad0d1b2c4289": 8
},
"versions": {
"2": 8
}
}
],
"pass": true
}
+15
View File
@@ -0,0 +1,15 @@
# 2026-11-13T17:56:27Z ./scripts/rolling-update.sh 2
2026-11-13T17:56:28Z build version 2
migrate Built
2026-11-13T17:56:29Z migrate (once)
2026-11-13 17:56:30,183 INFO migrations done; applied now: nothing
2026-11-13T17:56:30Z replacing 2 replicas
2026-11-13T17:56:36Z up mtusi-lab5-api-3 version=2 ip=172.23.0.12 UP in LB (UP slots: 4)
2026-11-13T17:56:39Z gone mtusi-lab5-api-1 (graceful SIGTERM)
2026-11-13T17:56:45Z up mtusi-lab5-api-4 version=2 ip=172.23.0.7 UP in LB (UP slots: 4)
2026-11-13T17:56:49Z gone mtusi-lab5-api-2 (graceful SIGTERM)
2026-11-13T17:56:49Z done: mtusi-lab5-api-3 mtusi-lab5-api-4
2026-11-13T17:56:49Z verified: 10/10 responses via LB have X-App-Version=2
exit=0
# make test после обновления
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
+235
View File
@@ -0,0 +1,235 @@
{
"scenario": "S3 outage: API answers 503 fast, accepted jobs retry and complete",
"date_utc": "2026-11-13T17:58:27+00:00",
"steps": [
{
"t": 2.04,
"utc": "17:57:29.727",
"step": "worker stopped, job accepted before outage",
"attachment_id": "19dac41c-4059-4326-8c97-93a0566b8979"
},
{
"t": 24.72,
"utc": "17:57:52.405",
"step": "S3 `stop` for 5 s",
"upload": {
"status": 503,
"ms": 14,
"code": "unavailable",
"retry_after": "1"
},
"download_ready": {
"status": 503,
"ms": 10,
"code": "unavailable",
"retry_after": "1"
},
"metadata_queued": {
"status": 200,
"ms": 6,
"code": null,
"retry_after": null
},
"text_message": {
"status": 201,
"ms": 6,
"code": null,
"retry_after": null
},
"ready": {
"status": 200,
"ms": 2,
"code": null,
"retry_after": null
}
},
{
"t": 25.83,
"utc": "17:57:53.513",
"step": "worker meets S3 outage -> retry with backoff",
"events": [
"enqueued",
"claimed",
"retry_scheduled"
],
"retries": [
{
"error_code": "storage_unavailable",
"next_attempt": 2,
"delay_seconds": 1.499
}
]
},
{
"t": 46.46,
"utc": "17:58:14.145",
"step": "S3 `start` -> healthy",
"seconds": 20.4
},
{
"t": 51.61,
"utc": "17:58:19.296",
"step": "S3 `pause` for 5 s",
"upload": {
"status": 503,
"ms": 14,
"code": "unavailable",
"retry_after": "1"
},
"download_ready": {
"status": 503,
"ms": 11,
"code": "unavailable",
"retry_after": "1"
},
"metadata_queued": {
"status": 200,
"ms": 7,
"code": null,
"retry_after": null
},
"text_message": {
"status": 201,
"ms": 9,
"code": null,
"retry_after": null
},
"ready": {
"status": 200,
"ms": 2,
"code": null,
"retry_after": null
}
},
{
"t": 57.47,
"utc": "17:58:25.154",
"step": "S3 `unpause` -> healthy",
"seconds": 5.7
},
{
"t": 58.24,
"utc": "17:58:25.925",
"step": "job accepted before outage completed",
"final": "ready",
"replay": null,
"attempts": 4,
"events": [
"enqueued",
"claimed",
"retry_scheduled",
"claimed",
"retry_scheduled",
"claimed",
"retry_scheduled",
"claimed",
"ready",
"staging_deleted"
]
},
{
"t": 58.58,
"utc": "17:58:26.262",
"step": "upload after recovery",
"status": "ready"
}
],
"results": {
"s3_stop": {
"upload": {
"status": 503,
"ms": 14,
"code": "unavailable",
"retry_after": "1"
},
"download_ready": {
"status": 503,
"ms": 10,
"code": "unavailable",
"retry_after": "1"
},
"metadata_queued": {
"status": 200,
"ms": 6,
"code": null,
"retry_after": null
},
"text_message": {
"status": 201,
"ms": 6,
"code": null,
"retry_after": null
},
"ready": {
"status": 200,
"ms": 2,
"code": null,
"retry_after": null
}
},
"worker_during_outage": {
"events": [
"enqueued",
"claimed",
"retry_scheduled"
],
"retries": [
{
"error_code": "storage_unavailable",
"next_attempt": 2,
"delay_seconds": 1.499
}
]
},
"s3_pause": {
"upload": {
"status": 503,
"ms": 14,
"code": "unavailable",
"retry_after": "1"
},
"download_ready": {
"status": 503,
"ms": 11,
"code": "unavailable",
"retry_after": "1"
},
"metadata_queued": {
"status": 200,
"ms": 7,
"code": null,
"retry_after": null
},
"text_message": {
"status": 201,
"ms": 9,
"code": null,
"retry_after": null
},
"ready": {
"status": 200,
"ms": 2,
"code": null,
"retry_after": null
}
},
"accepted_job": {
"final": "ready",
"replay": null,
"attempts": 4,
"events": [
"enqueued",
"claimed",
"retry_scheduled",
"claimed",
"retry_scheduled",
"claimed",
"retry_scheduled",
"claimed",
"ready",
"staging_deleted"
]
},
"upload_after_recovery": "ready"
}
}
+77
View File
@@ -0,0 +1,77 @@
{
"concurrency": 12,
"rounds": 2,
"files": [
{
"type": "image/png",
"bytes": 10482871,
"sha256": "184263681fcc647c"
},
{
"type": "image/jpeg",
"bytes": 8051317,
"sha256": "65d91ac28a9d4d68"
},
{
"type": "image/jpeg",
"bytes": 890394,
"sha256": "e7f29bc3786e6d91"
}
],
"uploads": 24,
"status_counts": {
"202": 24
},
"megabytes_uploaded_ok": 148.2,
"wall_s": 5.2,
"latency_s_p50_max": [
2.07,
2.7
],
"replicas": {
"78ea3303cfbc": {
"rss_mib": 51,
"hwm_mib_after_warmup": 82,
"samples": 100,
"peak_rss_mib_sampled": 81.1,
"peak_anon_mib": 290.0,
"peak_file_cache_mib": 46.6,
"vmhwm_mib": 82,
"cgroup_memory_peak_mib": 331.9,
"memory_limit_mib": 512,
"uploads_served": 12
},
"ad0d1b2c4289": {
"rss_mib": 51,
"hwm_mib_after_warmup": 83,
"samples": 95,
"peak_rss_mib_sampled": 88.5,
"peak_anon_mib": 297.5,
"peak_file_cache_mib": 37.1,
"vmhwm_mib": 88,
"cgroup_memory_peak_mib": 339.7,
"memory_limit_mib": 512,
"uploads_served": 12
}
},
"s3": {
"rss_mib": 413,
"oom_killed": "false",
"peak_rss_mib_sampled": 477.2,
"peak_anon_mib": 396.8,
"restarts": 0,
"oom_killed_after": "false",
"memory_limit_mib": 768
},
"oom_kills": [
"false",
"false"
],
"criteria": {
"all_responses_allowed": true,
"no_oom_kills": true,
"no_replica_restarts": true,
"cgroup_peak_below_90pct": true
},
"pass": true
}
+15 -3
View File
@@ -2,7 +2,9 @@
# Поочерёдная замена API-реплик на новую версию образа без остановки сервиса.
# scripts/rolling-update.sh <APP_VERSION>
# Порядок (expand -> replace):
# 1. собрать новый образ (тот же тег mtusi-messenger:lab5, build-arg APP_VERSION);
# 1. собрать новый образ (тот же тег mtusi-messenger:lab5, build-arg APP_VERSION). Секция build
# есть только у сервиса migrate (аудит F10: `build api` молча ничего не собирал), поэтому
# собираем migrate и проверяем, что в образе действительно новая версия;
# 2. ОДИН раз применить миграции (job migrate). Допускаются только обратно совместимые
# (expand-only) миграции: старые реплики продолжают работать на новой схеме;
# 3. для каждой старой реплики: поднять +1 новую -> дождаться healthy и UP в HAProxy ->
@@ -25,7 +27,9 @@ lb_has_up() {
}
echo "$(ts) build version $VERSION"
docker compose --env-file .env build api >/dev/null
docker compose --env-file .env build migrate >/dev/null
built=$(docker run --rm --entrypoint printenv mtusi-messenger:lab5 APP_VERSION)
[ "$built" = "$VERSION" ] || { echo "image has APP_VERSION=$built, expected $VERSION" >&2; exit 1; }
echo "$(ts) migrate (once)"
docker compose --env-file .env run --rm --no-deps migrate
@@ -44,7 +48,9 @@ for old in $OLD; do
ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$new")
for _ in $(seq 1 30); do lb_has_up "$ip" && break; sleep 0.5; done
lb_has_up "$ip" || { echo "HAProxy did not mark new replica $ip UP" >&2; exit 1; }
echo "$(ts) up $(docker inspect -f '{{.Name}}' "$new" | tr -d /) version=$(docker exec "$new" printenv APP_VERSION) ip=$ip UP in LB (UP slots: $(lb_up_count))"
got=$(docker exec "$new" printenv APP_VERSION)
[ "$got" = "$VERSION" ] || { echo "new replica runs APP_VERSION=$got, expected $VERSION" >&2; exit 1; }
echo "$(ts) up $(docker inspect -f '{{.Name}}' "$new" | tr -d /) version=$got ip=$ip UP in LB (UP slots: $(lb_up_count))"
name=$(docker inspect -f '{{.Name}}' "$old" | tr -d /)
docker stop -t 20 "$old" >/dev/null && docker rm "$old" >/dev/null
echo "$(ts) gone $name (graceful SIGTERM)"
@@ -55,3 +61,9 @@ else
echo "APP_VERSION=$VERSION" >> .env
fi
echo "$(ts) done: $(docker compose ps --format '{{.Name}}' api | tr '\n' ' ')"
# Проверка через LB: все ответы — от новой версии.
for _ in $(seq 1 10); do
v=$(curl -s -D - -o /dev/null http://127.0.0.1:${LB_PORT:-8080}/health/ready | tr -d '\r' | awk -F': ' 'tolower($1)=="x-app-version"{print $2}')
[ "$v" = "$VERSION" ] || { echo "LB served X-App-Version=$v, expected $VERSION" >&2; exit 1; }
done
echo "$(ts) verified: 10/10 responses via LB have X-App-Version=$VERSION"
-179
View File
@@ -1,179 +0,0 @@
#!/usr/bin/env python3
"""Отказы между записью в S3 и PostgreSQL и очистка janitor (уровень 4).
Требует API с FAULT_INJECTION=1 (учебный хук, по умолчанию выключен):
FAULT_INJECTION=1 docker compose --env-file .env up -d --no-deps --wait api
python3 student-tests/consistency_faults.py --output evidence/consistency-faults.json
docker compose --env-file .env up -d --no-deps --wait api # вернуть FAULT_INJECTION=0
Сценарии (каждый — реальный HTTP upload через LB):
db-fail-after-s3-put PUT прошёл, «COMMIT» ready упал -> 503; компенсация удаляет объект
и pending-строку -> ничего не остаётся;
crash-after-s3-put процесс API умирает (os._exit) после PUT -> pending + объект;
crash-after-pending процесс умирает до PUT -> pending без объекта;
обрыв клиента Content-Length 5 MiB, отправлен 1 MiB, сокет закрыт -> ничего;
orphan pending-строку удаляем вручную -> объект без строки;
missing объект ready-вложения удаляем в S3 -> только отчёт, download = 503.
Затем janitor: dry-run и боевой проход с grace 3600 (свежее не трогает), ожидание
grace-seconds, dry-run -> боевой -> повторный (идемпотентность). Контрольное ready-вложение
должно остаться доступным побайтно.
"""
import argparse
import hashlib
import json
import socket
import subprocess
import sys
import time
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, check # noqa: E402
PNG = (ROOT / "tests/fixtures/sample.png").read_bytes()
log = []
def sh(*cmd, timeout=120):
r = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, timeout=timeout)
if r.returncode != 0:
raise SystemExit("command failed: %s\n%s" % (" ".join(cmd[:4]), r.stderr[-500:]))
return r.stdout
def psql(sql):
return sh("docker", "compose", "exec", "-T", "db", "sh", "-c",
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -tAc "$0"', sql).strip()
def janitor(grace, dry_run):
cmd = ["docker", "compose", "run", "--rm", "--no-deps", "janitor", "python", "/app/src/janitor.py",
"--once", "--grace-seconds", str(grace)] + (["--dry-run"] if dry_run else [])
out = sh(*cmd)
report = json.loads([line for line in out.splitlines() if line.startswith("{")][-1])
keep = ("dry_run", "grace_seconds", "rows_by_status", "objects_total", "abandoned_pending",
"orphan_objects", "missing_objects", "deleted_objects", "deleted_rows", "errors", "duration_ms")
return {k: report.get(k) for k in keep}
def step(name, **data):
entry = dict(step=name, t=round(time.time(), 1), **data)
log.append(entry)
print(json.dumps(entry, ensure_ascii=False), flush=True)
return entry
def wait_api_healthy():
deadline = time.time() + 90
while time.time() < deadline:
states = sh("docker", "compose", "ps", "-a", "--format", "{{.Health}}", "api").split()
if states and all(s == "healthy" for s in states):
time.sleep(3) # HAProxy rise
return
time.sleep(1)
raise SystemExit("api did not become healthy")
def upload(c, actor, chat, fault=None, expected=201):
h = {"Content-Type": "image/png"}
if fault:
h["X-Fault-Inject"] = fault
try:
r = c.call("POST", "/api/v1/chats/%s/attachments?filename=f.png" % chat["id"], expected,
actor=actor, raw=PNG, headers=h, schema="Attachment" if expected == 201 else None)
return r["status"], r["json"]
except Exception as exc: # обрыв соединения при crash-* тоже допустимый исход
return type(exc).__name__, str(exc)[:120]
def interrupted_upload(base, token, chat_id):
host, port = base.split("//")[1].split(":")
s = socket.create_connection((host, int(port)), timeout=10)
s.sendall(("POST /api/v1/chats/%s/attachments?filename=cut.png HTTP/1.1\r\nHost: %s\r\n"
"Authorization: Bearer %s\r\nContent-Type: image/png\r\nContent-Length: %d\r\n\r\n"
% (chat_id, host, token, 5 * 1024 * 1024)).encode())
s.sendall(PNG[:8] + b"\0" * (1024 * 1024 - 8))
s.close() # клиент «пропал» на середине
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--base-url", default="http://localhost:8080")
ap.add_argument("--grace-seconds", type=int, default=20)
ap.add_argument("--output")
args = ap.parse_args()
env = sh("docker", "compose", "exec", "-T", "api", "printenv", "FAULT_INJECTION").strip()
check(env == "1", "start api with FAULT_INJECTION=1 first (see docstring)")
c = Client(args.base_url, timeout=30)
alice, bob, _eve, chat = c.bootstrap()
started = psql("SELECT now()")
base = janitor(3600, True)
step("baseline janitor dry-run", report=base)
_, control = upload(c, alice, chat)
_, victim = upload(c, alice, chat)
step("control + victim uploaded (ready)", control=control["id"], victim=victim["id"])
st, body = upload(c, alice, chat, "db-fail-after-s3-put", expected=503)
after_comp = janitor(3600, True)
step("db-fail-after-s3-put -> compensation", status=st, report=after_comp,
leftover_pending=after_comp["rows_by_status"].get("pending", 0) - base["rows_by_status"].get("pending", 0),
leftover_objects=after_comp["objects_total"] - base["objects_total"] - 2)
for fault in ("crash-after-s3-put", "crash-after-s3-put", "crash-after-pending"):
st, body = upload(c, alice, chat, fault, expected=[502, 503])
step(fault, client_saw=st)
wait_api_healthy()
interrupted_upload(args.base_url, alice["token"], chat["id"])
time.sleep(2)
step("client aborted upload at 1 of 5 MiB", report=janitor(3600, True))
pending = psql("SELECT id || ' ' || (object_key IS NOT NULL) FROM attachments WHERE status='pending' "
"AND created_at >= '%s' ORDER BY created_at" % started).splitlines()
step("pending rows left by crashes", pending=len(pending))
check(len(pending) == 3, "expected 3 pending rows from crashes")
orphan_id = pending[0].split()[0]
psql("DELETE FROM attachments WHERE id = '%s'" % orphan_id)
step("orphan injected: pending row deleted, object kept", row=orphan_id)
victim_key = psql("SELECT object_key FROM attachments WHERE id = '%s'" % victim["id"])
sh("docker", "compose", "--profile", "tools", "run", "--rm", "--entrypoint", "python", "s3-check", "-c",
"import sys; sys.argv=['x']; exec(open('/app/student-tests/s3_private_check.py').read().split('results = []')[0]);"
"client('api').remove_object(BUCKET, %r)" % victim_key)
dl = c.call("GET", "/api/v1/attachments/%s/content" % victim["id"], 503, actor=bob)
step("missing injected: victim object deleted in S3", download_status=dl["status"])
fresh_dry = janitor(3600, True)
fresh_run = janitor(3600, False)
step("janitor grace=3600 (all fresh): dry-run", report=fresh_dry)
step("janitor grace=3600 (all fresh): real run deletes nothing", report=fresh_run)
check(fresh_run["deleted_objects"] == 0 and fresh_run["deleted_rows"] == 0, "fresh data deleted")
time.sleep(args.grace_seconds + 2)
dry = janitor(args.grace_seconds, True)
step("after grace: dry-run", report=dry)
real = janitor(args.grace_seconds, False)
step("after grace: real run", report=real)
again = janitor(args.grace_seconds, False)
step("after grace: repeated run (idempotent)", report=again)
d = c.call("GET", "/api/v1/attachments/%s/content" % control["id"], actor=bob)
control_ok = hashlib.sha256(d["bytes"]).hexdigest() == control["sha256"]
still = psql("SELECT count(*) FROM attachments WHERE status='pending' AND created_at >= '%s'" % started)
step("control attachment still intact", sha256_ok=control_ok, pending_left=int(still))
ok = (log[2]["leftover_pending"] == 0 and log[2]["leftover_objects"] == 0
and dry["abandoned_pending"] >= 2 and dry["orphan_objects"] >= 1 and dry["missing_objects"] >= 1
and dry["deleted_objects"] == 0
and real["deleted_rows"] >= 2 and again["abandoned_pending"] == 0 and again["orphan_objects"] == 0
and again["deleted_objects"] == 0 and again["missing_objects"] >= 1
and control_ok and int(still) == 0)
print(json.dumps({"pass": ok}))
if args.output:
Path(args.output).write_text(json.dumps({"pass": ok, "steps": log}, ensure_ascii=False, indent=1))
return 0 if ok else 1
if __name__ == "__main__":
sys.exit(main())
+24
View File
@@ -2,6 +2,7 @@
# Полный прогон проверок lab5 на запущенном стенде с записью результатов в evidence/.
# Каждый сценарий сам включает и выключает свои учебные инъекции отказов.
# ./student-tests/run_all.sh
# Для make validate и сверки со схемой нужен venv (создаётся автоматически в .venv/).
set -u
cd "$(dirname "$0")/.."
E=evidence
@@ -17,7 +18,13 @@ run() { # run <name> <command...>
states() { docker compose ps --format '{{.Name}} {{.State}}' | sort; \
for c in $(docker compose ps -q); do docker inspect -f '{{.Name}} StartedAt={{.State.StartedAt}}' "$c"; done | sort; }
if [ ! -x .venv/bin/python ]; then
python3 -m venv .venv && .venv/bin/python -m pip install -q -r requirements-dev.txt rfc3339-validator
fi
VPY=.venv/bin/python
run check make check
run validate make validate PYTHON=$VPY
run smoke make test
{ echo "# containers before persistence hook"; states; } > "$E/containers-before-restart.txt"
run persistence make test-persistence ACTION_SCRIPT=scripts/restart.sh
@@ -38,8 +45,25 @@ run s3-private-check docker compose --profile tools run --rm -T checks /app/stud
run attachments-negative python3 student-tests/attachments_negative.py --output "$E/attachments-negative.json"
run attachment-failover python3 student-tests/attachment_failover.py --output "$E/attachment-failover.json"
./scripts/start-all.sh >/dev/null 2>&1
# --- регрессии внешнего аудита (F01–F17) ---
run api-negative-lab5 python3 student-tests/api_negative_lab5.py --output "$E/api-negative-lab5.json"
run slow-client python3 student-tests/slow_client.py --output "$E/slow-client.json"
run janitor-replay-race python3 student-tests/janitor_replay_race.py --output "$E/janitor-replay-race.json"
run queue-contract-check $VPY student-tests/queue_contract_check.py --output "$E/queue-contract-check.json"
run s3-outage python3 student-tests/s3_outage.py --output "$E/s3-outage.json"
run upload-memory python3 student-tests/upload_memory.py --concurrency 12 --rounds 2 --output "$E/upload-memory-c12.json"
# --- регрессия lab3 ---
run cursor-validation python3 student-tests/cursor_validation.py
run idempotency-concurrency python3 student-tests/idempotency_concurrency.py --output "$E/idempotency-concurrency.json"
T=$(docker compose ps -q api | head -1 | cut -c1-12)
run failover-bench python3 student-tests/failover_bench.py --action "env TARGET_INSTANCE_ID=$T scripts/stop-one.sh" \
--target-instance "$T" --duration 20 --action-at 5 --concurrency 4 --output "$E/failover-bench.json"
./scripts/start-all.sh >/dev/null 2>&1
NEXT=$(( $(grep -E '^APP_VERSION=' .env | cut -d= -f2) + 1 ))
run rolling-update sh -c "./scripts/rolling-update.sh $NEXT && make test"
run janitor-dry-run docker compose exec -T janitor python /app/src/janitor.py --once --dry-run
run queues docker compose exec -T broker rabbitmqctl -q -p messenger list_queues name type durable messages consumers
run failed-registry docker compose --profile tools run --rm -T ops failed --limit 10
run login-flood python3 student-tests/login_flood.py --output "$E/login-flood.json" # последним: исчерпывает лимит входов
echo "overall=$status"
exit $status
+76 -79
View File
@@ -1,63 +1,48 @@
#!/usr/bin/env python3
"""Недоступность S3: stop (порт закрыт) и pause (S3 «завис», TCP не отвечает).
"""Недоступность S3 в lab5 (переписано из lab4 по аудиту F14: там ожидался синхронный 201).
python3 student-tests/s3_outage.py [--output evidence/s3-outage.json]
python3 student-tests/s3_outage.py --output evidence/s3-outage.json
Ожидания: upload и download — быстрый 503 unavailable c Retry-After (не 500 и не
таймаут LB), текстовые сообщения и /health/ready продолжают работать (S3 не входит в
readiness), после восстановления S3 загрузка снова 201 без перезапуска API.
API (как в lab4): при остановленном (`stop`) и зависшем (`pause`) S3 upload и download — быстрый
503 unavailable с Retry-After, текстовые сообщения 201, /health/ready 200, GET метаданных 200.
Worker (новое): задание, принятое ДО отказа S3, при обработке получает storage_unavailable и
уходит в retry с backoff; после восстановления S3 оно доходит до ready. Если отказ длился
дольше всех попыток — failed(retry_exhausted), и оператор делает replay (это тоже проверяется).
"""
import argparse
import json
import subprocess
import sys
import time
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client # noqa: E402
PNG = (ROOT / "tests/fixtures/sample.png").read_bytes()
log = []
from lab5lib import (Client, Log, SAMPLE_PNG, check, compose, events, ops, upload, wait_status, wait_until)
def probe(c, alice, chat, att):
def probe(c, alice, chat, att_ready, att_queued):
out = {}
for name, args in (
("upload", ("POST", "/api/v1/chats/%s/attachments?filename=o.png" % chat["id"])),
("download", ("GET", "/api/v1/attachments/%s/content" % att["id"])),
("text_message", ("POST", "/api/v1/chats/%s/messages" % chat["id"])),
("ready", ("GET", "/health/ready"))):
kw = {}
if name == "upload":
kw = {"raw": PNG, "headers": {"Content-Type": "image/png"}}
if name == "text_message":
kw = {"body": {"text": "без S3"}, "headers": {"Idempotency-Key": uuid.uuid4().hex}}
calls = {
"upload": ("POST", "/api/v1/chats/%s/attachments?filename=o.png" % chat["id"],
{"raw": SAMPLE_PNG, "headers": {"Content-Type": "image/png"}}),
"download_ready": ("GET", "/api/v1/attachments/%s/content" % att_ready["id"], {}),
"metadata_queued": ("GET", "/api/v1/attachments/%s" % att_queued["id"], {}),
"text_message": ("POST", "/api/v1/chats/%s/messages" % chat["id"],
{"body": {"text": "без S3"}, "headers": {"Idempotency-Key": uuid.uuid4().hex}}),
"ready": ("GET", "/health/ready", {}),
}
for name, (method, path, kw) in calls.items():
t = time.monotonic()
r = c.call(*args, [200, 201, 503, 502, 504], actor=alice if name != "ready" else None, **kw)
r = c.call(method, path, [200, 201, 202, 503], actor=None if name == "ready" else alice, **kw)
out[name] = {"status": r["status"], "ms": round((time.monotonic() - t) * 1000),
"code": (r["json"] or {}).get("error", {}).get("code"),
"retry_after": r["headers"].get("Retry-After")}
return out
def phase(name, c, alice, chat, att):
entry = {"phase": name, "probe": probe(c, alice, chat, att)}
log.append(entry)
print(json.dumps(entry, ensure_ascii=False), flush=True)
return entry
def recover(c, alice, chat, started):
while time.monotonic() - started < 120:
r = c.call("POST", "/api/v1/chats/%s/attachments?filename=r.png" % chat["id"], [201, 503], actor=alice,
raw=PNG, headers={"Content-Type": "image/png"})
if r["status"] == 201:
return round(time.monotonic() - started, 1)
time.sleep(0.5)
return None
def s3_healthy():
cid = compose("ps", "-q", "s3").stdout.strip()
import subprocess
st = subprocess.run(["docker", "inspect", "-f", "{{.State.Health.Status}}", cid], capture_output=True,
text=True).stdout.strip()
return st == "healthy"
def main():
@@ -65,45 +50,57 @@ def main():
ap.add_argument("--base-url", default="http://localhost:8080")
ap.add_argument("--output")
args = ap.parse_args()
log = Log("S3 outage: API answers 503 fast, accepted jobs retry and complete")
c = Client(args.base_url, timeout=30)
alice, _bob, _eve, chat = c.bootstrap()
att = c.call("POST", "/api/v1/chats/%s/attachments?filename=a.png" % chat["id"], 201, actor=alice, raw=PNG,
headers={"Content-Type": "image/png"}, schema="Attachment")["json"]
phase("baseline", c, alice, chat, att)
for action, undo in ((["stop", "s3"], ["start", "s3"]), (["pause", "s3"], ["unpause", "s3"])):
subprocess.run(["docker", "compose"] + action, cwd=ROOT, check=True, capture_output=True)
time.sleep(1)
phase("immediately after `%s` (S3Health not yet stale)" % " ".join(action), c, alice, chat, att)
time.sleep(4)
phase("S3 down for 5 s (`%s`)" % " ".join(action), c, alice, chat, att)
subprocess.run(["docker", "compose"] + undo, cwd=ROOT, check=True, capture_output=True)
t0 = time.monotonic()
rec = recover(c, alice, chat, t0)
log.append({"phase": "recovery after `%s`" % " ".join(undo), "upload_201_after_s": rec})
print(json.dumps(log[-1]), flush=True)
# Оставляем стенд рабочим: ждём, пока healthcheck S3 снова станет healthy (после pause
# он несколько проверок подряд был unhealthy, а `docker compose up api` при нездоровой
# зависимости не запускает реплики).
cid = subprocess.run(["docker", "compose", "ps", "-q", "s3"], cwd=ROOT, capture_output=True, text=True).stdout.strip()
t0 = time.monotonic()
while time.monotonic() - t0 < 120:
st = subprocess.run(["docker", "inspect", "-f", "{{.State.Health.Status}}", cid], capture_output=True, text=True).stdout.strip()
if st == "healthy":
break
time.sleep(1)
log.append({"phase": "s3 healthcheck healthy again", "after_s": round(time.monotonic() - t0, 1), "status": st})
print(json.dumps(log[-1]), flush=True)
down = [e for e in log if e.get("phase", "").startswith("S3 down")]
ok = all(e["probe"]["upload"]["status"] == 503 and e["probe"]["download"]["status"] == 503
and e["probe"]["text_message"]["status"] == 201 and e["probe"]["ready"]["status"] == 200
for e in down) and all(e.get("upload_201_after_s") is not None for e in log if "upload_201_after_s" in e)
no5xx = all(p["status"] in (200, 201, 503) for e in log if "probe" in e for p in e["probe"].values())
ok = ok and st == "healthy"
print(json.dumps({"pass": ok and no5xx, "only_200_201_503": no5xx}))
if args.output:
Path(args.output).write_text(json.dumps({"pass": ok and no5xx, "phases": log}, ensure_ascii=False, indent=1))
return 0 if ok and no5xx else 1
alice, bob, _eve, chat = c.bootstrap()
att_ready = c.wait_ready(alice, upload(c, alice, chat, SAMPLE_PNG, "before.png"))
res = {}
try:
compose("stop", "worker")
queued = upload(c, alice, chat, SAMPLE_PNG, "accepted-before-outage.png")
log.step("worker stopped, job accepted before outage", attachment_id=queued["id"])
for action, undo in (("stop", "start"), ("pause", "unpause")):
compose(action, "s3")
time.sleep(5) # S3Health (1 с) заметил отказ, данные устарели (3 с)
pr = probe(c, alice, chat, att_ready, queued)
res["s3_" + action] = pr
log.step("S3 `%s` for 5 s" % action, **pr)
check(pr["upload"]["status"] == 503 and pr["download_ready"]["status"] == 503
and pr["text_message"]["status"] == 201 and pr["ready"]["status"] == 200
and pr["metadata_queued"]["status"] == 200, "API during S3 %s: %s" % (action, pr))
if action == "stop":
compose("up", "-d", "--no-deps", "--no-recreate", "worker")
info = wait_until(lambda: (lambda i: i if "retry_scheduled" in events(i) else None)(
ops("inspect", queued["id"])), 60, 1, "worker retry on storage error")
retries = [e["detail"] for e in info["events"] if e["event"] == "retry_scheduled"]
res["worker_during_outage"] = {"events": events(info), "retries": retries}
log.step("worker meets S3 outage -> retry with backoff", **res["worker_during_outage"])
check(all(r["error_code"] == "storage_unavailable" for r in retries), "retry reason")
compose(undo, "s3")
t0 = time.monotonic()
wait_until(s3_healthy, 120, 1, "s3 healthy")
log.step("S3 `%s` -> healthy" % undo, seconds=round(time.monotonic() - t0, 1))
final = wait_status(c, bob, queued["id"], {"ready", "failed"}, 120)
replayed = None
if final["status"] == "failed":
check(final["error_code"] == "retry_exhausted", "unexpected failure " + str(final["error_code"]))
replayed = ops("replay", queued["id"], "--reason", "S3 restored")["replay"][0]["result"]
final = wait_status(c, bob, queued["id"], {"ready", "failed"}, 60)
info = ops("inspect", queued["id"])
res["accepted_job"] = {"final": final["status"], "replay": replayed, "attempts": info["jobs"][0]["attempt"],
"events": events(info)}
check(final["status"] == "ready", "accepted job not completed")
log.step("job accepted before outage completed", **res["accepted_job"])
after = c.wait_ready(alice, upload(c, alice, chat, SAMPLE_PNG, "after.png"))
res["upload_after_recovery"] = after["status"]
log.step("upload after recovery", status=after["status"])
finally:
compose("unpause", "s3", ok=False)
compose("start", "s3", ok=False)
compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "worker", ok=False)
log.save(args.output, results=res)
print("PASS: S3 outage handled by API (503) and worker (retry), accepted work completed")
if __name__ == "__main__":
sys.exit(main())
main()
+46 -19
View File
@@ -4,9 +4,14 @@
python3 student-tests/upload_memory.py --concurrency 12 --rounds 3 --output evidence/upload-memory-c12.json
Гипотеза: потоковый приём (буфер 64 KiB -> безымянный временный файл), multipart PUT
частями по 5 MiB и draft-декодирование JPEG держат RAM реплики ограниченной и не
зависящей линейно от числа одновременных загрузок; 10+ загрузок по ~10 MiB не
приближают процесс к лимиту контейнера 256 MiB.
частями по 5 MiB и проверка JPEG в подпроцессе с лимитом (src/jpeg_probe.py) держат RAM
реплики ограниченной и не зависящей линейно от числа одновременных загрузок.
Lab5 (аудит F16): явные критерии успеха и ненулевой код выхода при нарушении — все ответы 202
(503 только с --allow-503), ни одного OOM-kill и перезапуска реплик, пик памяти cgroup каждой
реплики < 90% лимита. Объём считается по размеру файла в каждой записи. В смесь входят
несжимаемый PNG ~10 MiB, baseline JPEG и progressive JPEG 4:4:4 ~40 Мпикс (самый тяжёлый
по памяти вход, аудит F02) — JPEG генерируются в контейнере checks (Pillow).
Методика: перед прогоном API-реплики перезапускаются (VmHWM считается с нуля). В каждой
реплике фоново (docker exec ... while sleep 0.05) читаются VmRSS процесса (PID 1) и
@@ -67,18 +72,28 @@ class Sampler:
self.p.terminate()
def make_jpeg():
try:
from PIL import Image
except ImportError:
return None
GEN = r"""
import io, sys
from PIL import Image
kind = sys.argv[1]
if kind == "baseline":
im = Image.effect_noise((6000, 6000), 40).convert("RGB")
for q in (60, 45, 30, 20):
b = io.BytesIO()
im.save(b, "JPEG", quality=q)
b = io.BytesIO(); im.save(b, "JPEG", quality=q)
if b.tell() <= 10 * 1024 * 1024:
return b.getvalue()
return None
break
else:
g = Image.radial_gradient("L").resize((6324, 6324))
b = io.BytesIO(); Image.merge("RGB", (g, g.rotate(90), g)).save(b, "JPEG", quality=85, progressive=True, subsampling=0)
sys.stdout.buffer.write(b.getvalue())
"""
def make_jpeg(kind):
"""JPEG генерируется в контейнере checks (на хосте Pillow может не быть)."""
proc = subprocess.run(["docker", "compose", "--profile", "tools", "run", "--rm", "-T", "checks", "-c", GEN, kind],
cwd=ROOT, capture_output=True, check=True)
return proc.stdout
def main():
@@ -87,12 +102,12 @@ def main():
ap.add_argument("--concurrency", type=int, default=12)
ap.add_argument("--rounds", type=int, default=3)
ap.add_argument("--no-restart", action="store_true")
ap.add_argument("--allow-503", action="store_true", help="считать 503 (перегрузка) допустимым ответом")
ap.add_argument("--output")
args = ap.parse_args()
files = [("image/png", noise_png(10 * 1024 * 1024))]
jpeg = make_jpeg()
if jpeg:
files.append(("image/jpeg", jpeg))
files.append(("image/jpeg", make_jpeg("baseline")))
files.append(("image/jpeg", make_jpeg("progressive444")))
if not args.no_restart:
sh("docker", "compose", "restart", "api")
sh("docker", "compose", "up", "-d", "--no-deps", "--no-recreate", "--wait", "api")
@@ -104,6 +119,7 @@ def main():
c.call("POST", "/api/v1/chats/%s/attachments?filename=w" % chat["id"], 202, actor=alice, raw=data,
headers={"Content-Type": ctype})
cids = containers()
restarts_before = sh("docker", "inspect", "-f", "{{.RestartCount}}", *cids).split()
idle = {cid[:12]: {"rss_mib": proc_status(cid, "VmRSS"), "hwm_mib_after_warmup": proc_status(cid, "VmHWM")}
for cid in cids}
s3_cid = sh("docker", "compose", "ps", "-q", "s3").split()[0]
@@ -123,7 +139,9 @@ def main():
except Exception as exc: # noqa: BLE001
st, inst = type(exc).__name__, None
with lock:
results.append({"type": ctype, "status": st, "instance": inst, "s": round(time.monotonic() - t, 2)})
# Размер — в самой записи: потоки завершаются не по порядку (аудит F16).
results.append({"type": ctype, "bytes": len(data), "status": st, "instance": inst,
"s": round(time.monotonic() - t, 2)})
started = time.monotonic()
for _ in range(args.rounds):
@@ -154,8 +172,7 @@ def main():
"files": [{"type": t, "bytes": len(d), "sha256": hashlib.sha256(d).hexdigest()[:16]} for t, d in files],
"uploads": len(results), "status_counts": {str(k): sum(1 for r in results if r["status"] == k)
for k in {r["status"] for r in results}},
"megabytes_uploaded_ok": round(sum(len(files[i % len(files)][1]) for i, r in enumerate(results)
if r["status"] == 202) / 2**20, 1),
"megabytes_uploaded_ok": round(sum(r["bytes"] for r in results if r["status"] == 202) / 2**20, 1),
"wall_s": round(wall, 1),
"latency_s_p50_max": [durations[len(durations) // 2], durations[-1]] if durations else None,
"replicas": per,
@@ -167,10 +184,20 @@ def main():
memory_limit_mib=round(int(sh("docker", "exec", s3_cid, "cat", "/sys/fs/cgroup/memory.max")) / 2**20)),
"oom_kills": sh("docker", "inspect", "-f", "{{.State.OOMKilled}}", *cids).split(),
}
allowed = {202, 503} if args.allow_503 else {202}
restarts_after = sh("docker", "inspect", "-f", "{{.RestartCount}}", *cids).split()
criteria = {
"all_responses_allowed": all(r["status"] in allowed for r in results),
"no_oom_kills": all(x == "false" for x in summary["oom_kills"]),
"no_replica_restarts": restarts_before == restarts_after,
"cgroup_peak_below_90pct": all(v["cgroup_memory_peak_mib"] < 0.9 * v["memory_limit_mib"] for v in per.values()),
}
summary["criteria"] = criteria
summary["pass"] = all(criteria.values())
print(json.dumps(summary, ensure_ascii=False, indent=1))
if args.output:
Path(args.output).write_text(json.dumps(summary, ensure_ascii=False, indent=1))
return 0
return 0 if summary["pass"] else 1
if __name__ == "__main__":