forked from mtusi2026/mtusi2026lab5
Аудит F10, F14, F15, F16: rolling update, тесты lab4, CI приложения, тест памяти
- F10: rolling-update.sh собирает migrate (единственный сервис с build), проверяет APP_VERSION в образе, в новой реплике и в ответах через LB; реально выполнено обновление 1 -> 2 (evidence/rolling-update.txt) - F14: s3_outage.py переписан под lab5 (503 в API, retry в worker, принятое задание доходит до ready); устаревший consistency_faults.py удалён - F15: .github/workflows/app.yml — сборка, готовность, smoke и регрессии на изолированном стенде, логи при ошибке, очистка - F16: upload_memory.py с явными критериями и ненулевым кодом выхода, размер в каждой записи, progressive JPEG в смеси - run_all.sh: все регрессии аудита, make validate, регрессия lab3, rolling update
This commit is contained in:
@@ -0,0 +1,42 @@
|
||||
# Проверка РЕАЛИЗАЦИИ (аудит F15); contracts.yml проверяет только skeleton.
|
||||
# Изолированный стенд Compose с одноразовыми секретами: сборка, готовность, публичный smoke,
|
||||
# регрессии аудита, не требующие ручного вмешательства. Логи при ошибке, очистка всегда.
|
||||
# Секреты CI не нужны: init-secrets.sh генерирует их на раннере. pull_request_target не используется.
|
||||
# Ограничение: локально workflow не исполнялся (нет раннера); проверен синтаксис YAML.
|
||||
name: Application
|
||||
on: [push, pull_request]
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
app:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 40
|
||||
env:
|
||||
COMPOSE_PROJECT_NAME: mtusi-lab5-ci
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- name: Secrets and config
|
||||
run: ./scripts/init-secrets.sh && docker compose --env-file .env config --quiet
|
||||
- name: Build and start
|
||||
run: docker compose --env-file .env up -d --build --wait --wait-timeout 300
|
||||
- name: Public smoke
|
||||
run: make check test
|
||||
- name: Audit regressions (HTTP)
|
||||
run: python3 student-tests/api_negative_lab5.py
|
||||
- name: Variants and transform limits
|
||||
run: |
|
||||
docker compose --profile tools run --rm -T checks /app/student-tests/variants_check.py --base-url http://lb:8080
|
||||
docker compose --profile tools run --rm -T checks /app/student-tests/transform_limits.py
|
||||
- name: Worker outage and redelivery
|
||||
run: python3 student-tests/worker_outage.py
|
||||
- name: Outbox crash windows
|
||||
run: python3 student-tests/outbox_gap.py
|
||||
- name: Logs on failure
|
||||
if: failure()
|
||||
run: docker compose logs --no-color --tail 300
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: docker compose --profile tools down -v --remove-orphans
|
||||
@@ -0,0 +1,292 @@
|
||||
{
|
||||
"label": "",
|
||||
"concurrency": 4,
|
||||
"duration_s": 20.0,
|
||||
"request_timeout_s": 2.0,
|
||||
"action": {
|
||||
"command": "env TARGET_INSTANCE_ID=78ea3303cfbc scripts/stop-one.sh",
|
||||
"started_s": 5.01,
|
||||
"finished_s": 8.521,
|
||||
"exit_code": 0
|
||||
},
|
||||
"baseline_ok_rps": 486.0,
|
||||
"overall_ok_rps": 262.1,
|
||||
"requests_ok": 5242,
|
||||
"errors_total": 0,
|
||||
"error_rate": 0.0,
|
||||
"errors_by_kind": {},
|
||||
"error_window_s": null,
|
||||
"recovery_s": 0.0,
|
||||
"post_action_ok_rps": 137.8,
|
||||
"capacity_ratio": 0.28,
|
||||
"latency_ok_ms": {
|
||||
"p50": 5.1,
|
||||
"p95": 76.6,
|
||||
"p99": 94.2
|
||||
},
|
||||
"target_instance": "78ea3303cfbc",
|
||||
"target_last_seen_s": 1.629,
|
||||
"lb_events": [
|
||||
{
|
||||
"t": 1.7,
|
||||
"server": "api1",
|
||||
"addr": "172.23.0.9:8080",
|
||||
"from": "UP",
|
||||
"to": "DOWN"
|
||||
},
|
||||
{
|
||||
"t": 6.02,
|
||||
"server": "api1",
|
||||
"addr": "",
|
||||
"from": "DOWN",
|
||||
"to": "MAINT"
|
||||
}
|
||||
],
|
||||
"posts_retried_same_key": 0,
|
||||
"confirmed_messages": 2622,
|
||||
"history_messages": 2622,
|
||||
"lost_confirmed": 0,
|
||||
"duplicate_ids": 0,
|
||||
"duplicate_texts": 0,
|
||||
"timeline": [
|
||||
{
|
||||
"t": 0,
|
||||
"ok": 448,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"78ea3303cfbc": 224,
|
||||
"ad0d1b2c4289": 224
|
||||
},
|
||||
"versions": {
|
||||
"2": 448
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 1,
|
||||
"ok": 545,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"78ea3303cfbc": 273,
|
||||
"ad0d1b2c4289": 272
|
||||
},
|
||||
"versions": {
|
||||
"2": 545
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 2,
|
||||
"ok": 404,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"ad0d1b2c4289": 204,
|
||||
"78ea3303cfbc": 200
|
||||
},
|
||||
"versions": {
|
||||
"2": 404
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 3,
|
||||
"ok": 601,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"78ea3303cfbc": 302,
|
||||
"ad0d1b2c4289": 299
|
||||
},
|
||||
"versions": {
|
||||
"2": 601
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 4,
|
||||
"ok": 394,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"78ea3303cfbc": 198,
|
||||
"ad0d1b2c4289": 196
|
||||
},
|
||||
"versions": {
|
||||
"2": 394
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 5,
|
||||
"ok": 476,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"ad0d1b2c4289": 239,
|
||||
"78ea3303cfbc": 237
|
||||
},
|
||||
"versions": {
|
||||
"2": 476
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 6,
|
||||
"ok": 468,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"78ea3303cfbc": 206,
|
||||
"ad0d1b2c4289": 262
|
||||
},
|
||||
"versions": {
|
||||
"2": 468
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 7,
|
||||
"ok": 166,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"ad0d1b2c4289": 166
|
||||
},
|
||||
"versions": {
|
||||
"2": 166
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 8,
|
||||
"ok": 184,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"ad0d1b2c4289": 184
|
||||
},
|
||||
"versions": {
|
||||
"2": 184
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 9,
|
||||
"ok": 151,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"ad0d1b2c4289": 151
|
||||
},
|
||||
"versions": {
|
||||
"2": 151
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 10,
|
||||
"ok": 79,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"ad0d1b2c4289": 79
|
||||
},
|
||||
"versions": {
|
||||
"2": 79
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 11,
|
||||
"ok": 93,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"ad0d1b2c4289": 93
|
||||
},
|
||||
"versions": {
|
||||
"2": 93
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 12,
|
||||
"ok": 172,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"ad0d1b2c4289": 172
|
||||
},
|
||||
"versions": {
|
||||
"2": 172
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 13,
|
||||
"ok": 201,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"ad0d1b2c4289": 201
|
||||
},
|
||||
"versions": {
|
||||
"2": 201
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 14,
|
||||
"ok": 149,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"ad0d1b2c4289": 149
|
||||
},
|
||||
"versions": {
|
||||
"2": 149
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 15,
|
||||
"ok": 171,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"ad0d1b2c4289": 171
|
||||
},
|
||||
"versions": {
|
||||
"2": 171
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 16,
|
||||
"ok": 115,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"ad0d1b2c4289": 115
|
||||
},
|
||||
"versions": {
|
||||
"2": 115
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 17,
|
||||
"ok": 108,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"ad0d1b2c4289": 108
|
||||
},
|
||||
"versions": {
|
||||
"2": 108
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 18,
|
||||
"ok": 152,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"ad0d1b2c4289": 152
|
||||
},
|
||||
"versions": {
|
||||
"2": 152
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 19,
|
||||
"ok": 157,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"ad0d1b2c4289": 157
|
||||
},
|
||||
"versions": {
|
||||
"2": 157
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 20,
|
||||
"ok": 8,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"ad0d1b2c4289": 8
|
||||
},
|
||||
"versions": {
|
||||
"2": 8
|
||||
}
|
||||
}
|
||||
],
|
||||
"pass": true
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
# 2026-11-13T17:56:27Z ./scripts/rolling-update.sh 2
|
||||
2026-11-13T17:56:28Z build version 2
|
||||
migrate Built
|
||||
2026-11-13T17:56:29Z migrate (once)
|
||||
2026-11-13 17:56:30,183 INFO migrations done; applied now: nothing
|
||||
2026-11-13T17:56:30Z replacing 2 replicas
|
||||
2026-11-13T17:56:36Z up mtusi-lab5-api-3 version=2 ip=172.23.0.12 UP in LB (UP slots: 4)
|
||||
2026-11-13T17:56:39Z gone mtusi-lab5-api-1 (graceful SIGTERM)
|
||||
2026-11-13T17:56:45Z up mtusi-lab5-api-4 version=2 ip=172.23.0.7 UP in LB (UP slots: 4)
|
||||
2026-11-13T17:56:49Z gone mtusi-lab5-api-2 (graceful SIGTERM)
|
||||
2026-11-13T17:56:49Z done: mtusi-lab5-api-3 mtusi-lab5-api-4
|
||||
2026-11-13T17:56:49Z verified: 10/10 responses via LB have X-App-Version=2
|
||||
exit=0
|
||||
# make test после обновления
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,235 @@
|
||||
{
|
||||
"scenario": "S3 outage: API answers 503 fast, accepted jobs retry and complete",
|
||||
"date_utc": "2026-11-13T17:58:27+00:00",
|
||||
"steps": [
|
||||
{
|
||||
"t": 2.04,
|
||||
"utc": "17:57:29.727",
|
||||
"step": "worker stopped, job accepted before outage",
|
||||
"attachment_id": "19dac41c-4059-4326-8c97-93a0566b8979"
|
||||
},
|
||||
{
|
||||
"t": 24.72,
|
||||
"utc": "17:57:52.405",
|
||||
"step": "S3 `stop` for 5 s",
|
||||
"upload": {
|
||||
"status": 503,
|
||||
"ms": 14,
|
||||
"code": "unavailable",
|
||||
"retry_after": "1"
|
||||
},
|
||||
"download_ready": {
|
||||
"status": 503,
|
||||
"ms": 10,
|
||||
"code": "unavailable",
|
||||
"retry_after": "1"
|
||||
},
|
||||
"metadata_queued": {
|
||||
"status": 200,
|
||||
"ms": 6,
|
||||
"code": null,
|
||||
"retry_after": null
|
||||
},
|
||||
"text_message": {
|
||||
"status": 201,
|
||||
"ms": 6,
|
||||
"code": null,
|
||||
"retry_after": null
|
||||
},
|
||||
"ready": {
|
||||
"status": 200,
|
||||
"ms": 2,
|
||||
"code": null,
|
||||
"retry_after": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 25.83,
|
||||
"utc": "17:57:53.513",
|
||||
"step": "worker meets S3 outage -> retry with backoff",
|
||||
"events": [
|
||||
"enqueued",
|
||||
"claimed",
|
||||
"retry_scheduled"
|
||||
],
|
||||
"retries": [
|
||||
{
|
||||
"error_code": "storage_unavailable",
|
||||
"next_attempt": 2,
|
||||
"delay_seconds": 1.499
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 46.46,
|
||||
"utc": "17:58:14.145",
|
||||
"step": "S3 `start` -> healthy",
|
||||
"seconds": 20.4
|
||||
},
|
||||
{
|
||||
"t": 51.61,
|
||||
"utc": "17:58:19.296",
|
||||
"step": "S3 `pause` for 5 s",
|
||||
"upload": {
|
||||
"status": 503,
|
||||
"ms": 14,
|
||||
"code": "unavailable",
|
||||
"retry_after": "1"
|
||||
},
|
||||
"download_ready": {
|
||||
"status": 503,
|
||||
"ms": 11,
|
||||
"code": "unavailable",
|
||||
"retry_after": "1"
|
||||
},
|
||||
"metadata_queued": {
|
||||
"status": 200,
|
||||
"ms": 7,
|
||||
"code": null,
|
||||
"retry_after": null
|
||||
},
|
||||
"text_message": {
|
||||
"status": 201,
|
||||
"ms": 9,
|
||||
"code": null,
|
||||
"retry_after": null
|
||||
},
|
||||
"ready": {
|
||||
"status": 200,
|
||||
"ms": 2,
|
||||
"code": null,
|
||||
"retry_after": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 57.47,
|
||||
"utc": "17:58:25.154",
|
||||
"step": "S3 `unpause` -> healthy",
|
||||
"seconds": 5.7
|
||||
},
|
||||
{
|
||||
"t": 58.24,
|
||||
"utc": "17:58:25.925",
|
||||
"step": "job accepted before outage completed",
|
||||
"final": "ready",
|
||||
"replay": null,
|
||||
"attempts": 4,
|
||||
"events": [
|
||||
"enqueued",
|
||||
"claimed",
|
||||
"retry_scheduled",
|
||||
"claimed",
|
||||
"retry_scheduled",
|
||||
"claimed",
|
||||
"retry_scheduled",
|
||||
"claimed",
|
||||
"ready",
|
||||
"staging_deleted"
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 58.58,
|
||||
"utc": "17:58:26.262",
|
||||
"step": "upload after recovery",
|
||||
"status": "ready"
|
||||
}
|
||||
],
|
||||
"results": {
|
||||
"s3_stop": {
|
||||
"upload": {
|
||||
"status": 503,
|
||||
"ms": 14,
|
||||
"code": "unavailable",
|
||||
"retry_after": "1"
|
||||
},
|
||||
"download_ready": {
|
||||
"status": 503,
|
||||
"ms": 10,
|
||||
"code": "unavailable",
|
||||
"retry_after": "1"
|
||||
},
|
||||
"metadata_queued": {
|
||||
"status": 200,
|
||||
"ms": 6,
|
||||
"code": null,
|
||||
"retry_after": null
|
||||
},
|
||||
"text_message": {
|
||||
"status": 201,
|
||||
"ms": 6,
|
||||
"code": null,
|
||||
"retry_after": null
|
||||
},
|
||||
"ready": {
|
||||
"status": 200,
|
||||
"ms": 2,
|
||||
"code": null,
|
||||
"retry_after": null
|
||||
}
|
||||
},
|
||||
"worker_during_outage": {
|
||||
"events": [
|
||||
"enqueued",
|
||||
"claimed",
|
||||
"retry_scheduled"
|
||||
],
|
||||
"retries": [
|
||||
{
|
||||
"error_code": "storage_unavailable",
|
||||
"next_attempt": 2,
|
||||
"delay_seconds": 1.499
|
||||
}
|
||||
]
|
||||
},
|
||||
"s3_pause": {
|
||||
"upload": {
|
||||
"status": 503,
|
||||
"ms": 14,
|
||||
"code": "unavailable",
|
||||
"retry_after": "1"
|
||||
},
|
||||
"download_ready": {
|
||||
"status": 503,
|
||||
"ms": 11,
|
||||
"code": "unavailable",
|
||||
"retry_after": "1"
|
||||
},
|
||||
"metadata_queued": {
|
||||
"status": 200,
|
||||
"ms": 7,
|
||||
"code": null,
|
||||
"retry_after": null
|
||||
},
|
||||
"text_message": {
|
||||
"status": 201,
|
||||
"ms": 9,
|
||||
"code": null,
|
||||
"retry_after": null
|
||||
},
|
||||
"ready": {
|
||||
"status": 200,
|
||||
"ms": 2,
|
||||
"code": null,
|
||||
"retry_after": null
|
||||
}
|
||||
},
|
||||
"accepted_job": {
|
||||
"final": "ready",
|
||||
"replay": null,
|
||||
"attempts": 4,
|
||||
"events": [
|
||||
"enqueued",
|
||||
"claimed",
|
||||
"retry_scheduled",
|
||||
"claimed",
|
||||
"retry_scheduled",
|
||||
"claimed",
|
||||
"retry_scheduled",
|
||||
"claimed",
|
||||
"ready",
|
||||
"staging_deleted"
|
||||
]
|
||||
},
|
||||
"upload_after_recovery": "ready"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
{
|
||||
"concurrency": 12,
|
||||
"rounds": 2,
|
||||
"files": [
|
||||
{
|
||||
"type": "image/png",
|
||||
"bytes": 10482871,
|
||||
"sha256": "184263681fcc647c"
|
||||
},
|
||||
{
|
||||
"type": "image/jpeg",
|
||||
"bytes": 8051317,
|
||||
"sha256": "65d91ac28a9d4d68"
|
||||
},
|
||||
{
|
||||
"type": "image/jpeg",
|
||||
"bytes": 890394,
|
||||
"sha256": "e7f29bc3786e6d91"
|
||||
}
|
||||
],
|
||||
"uploads": 24,
|
||||
"status_counts": {
|
||||
"202": 24
|
||||
},
|
||||
"megabytes_uploaded_ok": 148.2,
|
||||
"wall_s": 5.2,
|
||||
"latency_s_p50_max": [
|
||||
2.07,
|
||||
2.7
|
||||
],
|
||||
"replicas": {
|
||||
"78ea3303cfbc": {
|
||||
"rss_mib": 51,
|
||||
"hwm_mib_after_warmup": 82,
|
||||
"samples": 100,
|
||||
"peak_rss_mib_sampled": 81.1,
|
||||
"peak_anon_mib": 290.0,
|
||||
"peak_file_cache_mib": 46.6,
|
||||
"vmhwm_mib": 82,
|
||||
"cgroup_memory_peak_mib": 331.9,
|
||||
"memory_limit_mib": 512,
|
||||
"uploads_served": 12
|
||||
},
|
||||
"ad0d1b2c4289": {
|
||||
"rss_mib": 51,
|
||||
"hwm_mib_after_warmup": 83,
|
||||
"samples": 95,
|
||||
"peak_rss_mib_sampled": 88.5,
|
||||
"peak_anon_mib": 297.5,
|
||||
"peak_file_cache_mib": 37.1,
|
||||
"vmhwm_mib": 88,
|
||||
"cgroup_memory_peak_mib": 339.7,
|
||||
"memory_limit_mib": 512,
|
||||
"uploads_served": 12
|
||||
}
|
||||
},
|
||||
"s3": {
|
||||
"rss_mib": 413,
|
||||
"oom_killed": "false",
|
||||
"peak_rss_mib_sampled": 477.2,
|
||||
"peak_anon_mib": 396.8,
|
||||
"restarts": 0,
|
||||
"oom_killed_after": "false",
|
||||
"memory_limit_mib": 768
|
||||
},
|
||||
"oom_kills": [
|
||||
"false",
|
||||
"false"
|
||||
],
|
||||
"criteria": {
|
||||
"all_responses_allowed": true,
|
||||
"no_oom_kills": true,
|
||||
"no_replica_restarts": true,
|
||||
"cgroup_peak_below_90pct": true
|
||||
},
|
||||
"pass": true
|
||||
}
|
||||
@@ -2,7 +2,9 @@
|
||||
# Поочерёдная замена API-реплик на новую версию образа без остановки сервиса.
|
||||
# scripts/rolling-update.sh <APP_VERSION>
|
||||
# Порядок (expand -> replace):
|
||||
# 1. собрать новый образ (тот же тег mtusi-messenger:lab5, build-arg APP_VERSION);
|
||||
# 1. собрать новый образ (тот же тег mtusi-messenger:lab5, build-arg APP_VERSION). Секция build
|
||||
# есть только у сервиса migrate (аудит F10: `build api` молча ничего не собирал), поэтому
|
||||
# собираем migrate и проверяем, что в образе действительно новая версия;
|
||||
# 2. ОДИН раз применить миграции (job migrate). Допускаются только обратно совместимые
|
||||
# (expand-only) миграции: старые реплики продолжают работать на новой схеме;
|
||||
# 3. для каждой старой реплики: поднять +1 новую -> дождаться healthy и UP в HAProxy ->
|
||||
@@ -25,7 +27,9 @@ lb_has_up() {
|
||||
}
|
||||
|
||||
echo "$(ts) build version $VERSION"
|
||||
docker compose --env-file .env build api >/dev/null
|
||||
docker compose --env-file .env build migrate >/dev/null
|
||||
built=$(docker run --rm --entrypoint printenv mtusi-messenger:lab5 APP_VERSION)
|
||||
[ "$built" = "$VERSION" ] || { echo "image has APP_VERSION=$built, expected $VERSION" >&2; exit 1; }
|
||||
echo "$(ts) migrate (once)"
|
||||
docker compose --env-file .env run --rm --no-deps migrate
|
||||
|
||||
@@ -44,7 +48,9 @@ for old in $OLD; do
|
||||
ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$new")
|
||||
for _ in $(seq 1 30); do lb_has_up "$ip" && break; sleep 0.5; done
|
||||
lb_has_up "$ip" || { echo "HAProxy did not mark new replica $ip UP" >&2; exit 1; }
|
||||
echo "$(ts) up $(docker inspect -f '{{.Name}}' "$new" | tr -d /) version=$(docker exec "$new" printenv APP_VERSION) ip=$ip UP in LB (UP slots: $(lb_up_count))"
|
||||
got=$(docker exec "$new" printenv APP_VERSION)
|
||||
[ "$got" = "$VERSION" ] || { echo "new replica runs APP_VERSION=$got, expected $VERSION" >&2; exit 1; }
|
||||
echo "$(ts) up $(docker inspect -f '{{.Name}}' "$new" | tr -d /) version=$got ip=$ip UP in LB (UP slots: $(lb_up_count))"
|
||||
name=$(docker inspect -f '{{.Name}}' "$old" | tr -d /)
|
||||
docker stop -t 20 "$old" >/dev/null && docker rm "$old" >/dev/null
|
||||
echo "$(ts) gone $name (graceful SIGTERM)"
|
||||
@@ -55,3 +61,9 @@ else
|
||||
echo "APP_VERSION=$VERSION" >> .env
|
||||
fi
|
||||
echo "$(ts) done: $(docker compose ps --format '{{.Name}}' api | tr '\n' ' ')"
|
||||
# Проверка через LB: все ответы — от новой версии.
|
||||
for _ in $(seq 1 10); do
|
||||
v=$(curl -s -D - -o /dev/null http://127.0.0.1:${LB_PORT:-8080}/health/ready | tr -d '\r' | awk -F': ' 'tolower($1)=="x-app-version"{print $2}')
|
||||
[ "$v" = "$VERSION" ] || { echo "LB served X-App-Version=$v, expected $VERSION" >&2; exit 1; }
|
||||
done
|
||||
echo "$(ts) verified: 10/10 responses via LB have X-App-Version=$VERSION"
|
||||
|
||||
@@ -1,179 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Отказы между записью в S3 и PostgreSQL и очистка janitor (уровень 4).
|
||||
|
||||
Требует API с FAULT_INJECTION=1 (учебный хук, по умолчанию выключен):
|
||||
FAULT_INJECTION=1 docker compose --env-file .env up -d --no-deps --wait api
|
||||
python3 student-tests/consistency_faults.py --output evidence/consistency-faults.json
|
||||
docker compose --env-file .env up -d --no-deps --wait api # вернуть FAULT_INJECTION=0
|
||||
|
||||
Сценарии (каждый — реальный HTTP upload через LB):
|
||||
db-fail-after-s3-put PUT прошёл, «COMMIT» ready упал -> 503; компенсация удаляет объект
|
||||
и pending-строку -> ничего не остаётся;
|
||||
crash-after-s3-put процесс API умирает (os._exit) после PUT -> pending + объект;
|
||||
crash-after-pending процесс умирает до PUT -> pending без объекта;
|
||||
обрыв клиента Content-Length 5 MiB, отправлен 1 MiB, сокет закрыт -> ничего;
|
||||
orphan pending-строку удаляем вручную -> объект без строки;
|
||||
missing объект ready-вложения удаляем в S3 -> только отчёт, download = 503.
|
||||
Затем janitor: dry-run и боевой проход с grace 3600 (свежее не трогает), ожидание
|
||||
grace-seconds, dry-run -> боевой -> повторный (идемпотентность). Контрольное ready-вложение
|
||||
должно остаться доступным побайтно.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, check # noqa: E402
|
||||
|
||||
PNG = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
log = []
|
||||
|
||||
|
||||
def sh(*cmd, timeout=120):
|
||||
r = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, timeout=timeout)
|
||||
if r.returncode != 0:
|
||||
raise SystemExit("command failed: %s\n%s" % (" ".join(cmd[:4]), r.stderr[-500:]))
|
||||
return r.stdout
|
||||
|
||||
|
||||
def psql(sql):
|
||||
return sh("docker", "compose", "exec", "-T", "db", "sh", "-c",
|
||||
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -tAc "$0"', sql).strip()
|
||||
|
||||
|
||||
def janitor(grace, dry_run):
|
||||
cmd = ["docker", "compose", "run", "--rm", "--no-deps", "janitor", "python", "/app/src/janitor.py",
|
||||
"--once", "--grace-seconds", str(grace)] + (["--dry-run"] if dry_run else [])
|
||||
out = sh(*cmd)
|
||||
report = json.loads([line for line in out.splitlines() if line.startswith("{")][-1])
|
||||
keep = ("dry_run", "grace_seconds", "rows_by_status", "objects_total", "abandoned_pending",
|
||||
"orphan_objects", "missing_objects", "deleted_objects", "deleted_rows", "errors", "duration_ms")
|
||||
return {k: report.get(k) for k in keep}
|
||||
|
||||
|
||||
def step(name, **data):
|
||||
entry = dict(step=name, t=round(time.time(), 1), **data)
|
||||
log.append(entry)
|
||||
print(json.dumps(entry, ensure_ascii=False), flush=True)
|
||||
return entry
|
||||
|
||||
|
||||
def wait_api_healthy():
|
||||
deadline = time.time() + 90
|
||||
while time.time() < deadline:
|
||||
states = sh("docker", "compose", "ps", "-a", "--format", "{{.Health}}", "api").split()
|
||||
if states and all(s == "healthy" for s in states):
|
||||
time.sleep(3) # HAProxy rise
|
||||
return
|
||||
time.sleep(1)
|
||||
raise SystemExit("api did not become healthy")
|
||||
|
||||
|
||||
def upload(c, actor, chat, fault=None, expected=201):
|
||||
h = {"Content-Type": "image/png"}
|
||||
if fault:
|
||||
h["X-Fault-Inject"] = fault
|
||||
try:
|
||||
r = c.call("POST", "/api/v1/chats/%s/attachments?filename=f.png" % chat["id"], expected,
|
||||
actor=actor, raw=PNG, headers=h, schema="Attachment" if expected == 201 else None)
|
||||
return r["status"], r["json"]
|
||||
except Exception as exc: # обрыв соединения при crash-* тоже допустимый исход
|
||||
return type(exc).__name__, str(exc)[:120]
|
||||
|
||||
|
||||
def interrupted_upload(base, token, chat_id):
|
||||
host, port = base.split("//")[1].split(":")
|
||||
s = socket.create_connection((host, int(port)), timeout=10)
|
||||
s.sendall(("POST /api/v1/chats/%s/attachments?filename=cut.png HTTP/1.1\r\nHost: %s\r\n"
|
||||
"Authorization: Bearer %s\r\nContent-Type: image/png\r\nContent-Length: %d\r\n\r\n"
|
||||
% (chat_id, host, token, 5 * 1024 * 1024)).encode())
|
||||
s.sendall(PNG[:8] + b"\0" * (1024 * 1024 - 8))
|
||||
s.close() # клиент «пропал» на середине
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--base-url", default="http://localhost:8080")
|
||||
ap.add_argument("--grace-seconds", type=int, default=20)
|
||||
ap.add_argument("--output")
|
||||
args = ap.parse_args()
|
||||
env = sh("docker", "compose", "exec", "-T", "api", "printenv", "FAULT_INJECTION").strip()
|
||||
check(env == "1", "start api with FAULT_INJECTION=1 first (see docstring)")
|
||||
c = Client(args.base_url, timeout=30)
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
started = psql("SELECT now()")
|
||||
base = janitor(3600, True)
|
||||
step("baseline janitor dry-run", report=base)
|
||||
|
||||
_, control = upload(c, alice, chat)
|
||||
_, victim = upload(c, alice, chat)
|
||||
step("control + victim uploaded (ready)", control=control["id"], victim=victim["id"])
|
||||
|
||||
st, body = upload(c, alice, chat, "db-fail-after-s3-put", expected=503)
|
||||
after_comp = janitor(3600, True)
|
||||
step("db-fail-after-s3-put -> compensation", status=st, report=after_comp,
|
||||
leftover_pending=after_comp["rows_by_status"].get("pending", 0) - base["rows_by_status"].get("pending", 0),
|
||||
leftover_objects=after_comp["objects_total"] - base["objects_total"] - 2)
|
||||
|
||||
for fault in ("crash-after-s3-put", "crash-after-s3-put", "crash-after-pending"):
|
||||
st, body = upload(c, alice, chat, fault, expected=[502, 503])
|
||||
step(fault, client_saw=st)
|
||||
wait_api_healthy()
|
||||
|
||||
interrupted_upload(args.base_url, alice["token"], chat["id"])
|
||||
time.sleep(2)
|
||||
step("client aborted upload at 1 of 5 MiB", report=janitor(3600, True))
|
||||
|
||||
pending = psql("SELECT id || ' ' || (object_key IS NOT NULL) FROM attachments WHERE status='pending' "
|
||||
"AND created_at >= '%s' ORDER BY created_at" % started).splitlines()
|
||||
step("pending rows left by crashes", pending=len(pending))
|
||||
check(len(pending) == 3, "expected 3 pending rows from crashes")
|
||||
orphan_id = pending[0].split()[0]
|
||||
psql("DELETE FROM attachments WHERE id = '%s'" % orphan_id)
|
||||
step("orphan injected: pending row deleted, object kept", row=orphan_id)
|
||||
victim_key = psql("SELECT object_key FROM attachments WHERE id = '%s'" % victim["id"])
|
||||
sh("docker", "compose", "--profile", "tools", "run", "--rm", "--entrypoint", "python", "s3-check", "-c",
|
||||
"import sys; sys.argv=['x']; exec(open('/app/student-tests/s3_private_check.py').read().split('results = []')[0]);"
|
||||
"client('api').remove_object(BUCKET, %r)" % victim_key)
|
||||
dl = c.call("GET", "/api/v1/attachments/%s/content" % victim["id"], 503, actor=bob)
|
||||
step("missing injected: victim object deleted in S3", download_status=dl["status"])
|
||||
|
||||
fresh_dry = janitor(3600, True)
|
||||
fresh_run = janitor(3600, False)
|
||||
step("janitor grace=3600 (all fresh): dry-run", report=fresh_dry)
|
||||
step("janitor grace=3600 (all fresh): real run deletes nothing", report=fresh_run)
|
||||
check(fresh_run["deleted_objects"] == 0 and fresh_run["deleted_rows"] == 0, "fresh data deleted")
|
||||
|
||||
time.sleep(args.grace_seconds + 2)
|
||||
dry = janitor(args.grace_seconds, True)
|
||||
step("after grace: dry-run", report=dry)
|
||||
real = janitor(args.grace_seconds, False)
|
||||
step("after grace: real run", report=real)
|
||||
again = janitor(args.grace_seconds, False)
|
||||
step("after grace: repeated run (idempotent)", report=again)
|
||||
d = c.call("GET", "/api/v1/attachments/%s/content" % control["id"], actor=bob)
|
||||
control_ok = hashlib.sha256(d["bytes"]).hexdigest() == control["sha256"]
|
||||
still = psql("SELECT count(*) FROM attachments WHERE status='pending' AND created_at >= '%s'" % started)
|
||||
step("control attachment still intact", sha256_ok=control_ok, pending_left=int(still))
|
||||
|
||||
ok = (log[2]["leftover_pending"] == 0 and log[2]["leftover_objects"] == 0
|
||||
and dry["abandoned_pending"] >= 2 and dry["orphan_objects"] >= 1 and dry["missing_objects"] >= 1
|
||||
and dry["deleted_objects"] == 0
|
||||
and real["deleted_rows"] >= 2 and again["abandoned_pending"] == 0 and again["orphan_objects"] == 0
|
||||
and again["deleted_objects"] == 0 and again["missing_objects"] >= 1
|
||||
and control_ok and int(still) == 0)
|
||||
print(json.dumps({"pass": ok}))
|
||||
if args.output:
|
||||
Path(args.output).write_text(json.dumps({"pass": ok, "steps": log}, ensure_ascii=False, indent=1))
|
||||
return 0 if ok else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -2,6 +2,7 @@
|
||||
# Полный прогон проверок lab5 на запущенном стенде с записью результатов в evidence/.
|
||||
# Каждый сценарий сам включает и выключает свои учебные инъекции отказов.
|
||||
# ./student-tests/run_all.sh
|
||||
# Для make validate и сверки со схемой нужен venv (создаётся автоматически в .venv/).
|
||||
set -u
|
||||
cd "$(dirname "$0")/.."
|
||||
E=evidence
|
||||
@@ -17,7 +18,13 @@ run() { # run <name> <command...>
|
||||
states() { docker compose ps --format '{{.Name}} {{.State}}' | sort; \
|
||||
for c in $(docker compose ps -q); do docker inspect -f '{{.Name}} StartedAt={{.State.StartedAt}}' "$c"; done | sort; }
|
||||
|
||||
if [ ! -x .venv/bin/python ]; then
|
||||
python3 -m venv .venv && .venv/bin/python -m pip install -q -r requirements-dev.txt rfc3339-validator
|
||||
fi
|
||||
VPY=.venv/bin/python
|
||||
|
||||
run check make check
|
||||
run validate make validate PYTHON=$VPY
|
||||
run smoke make test
|
||||
{ echo "# containers before persistence hook"; states; } > "$E/containers-before-restart.txt"
|
||||
run persistence make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
@@ -38,8 +45,25 @@ run s3-private-check docker compose --profile tools run --rm -T checks /app/stud
|
||||
run attachments-negative python3 student-tests/attachments_negative.py --output "$E/attachments-negative.json"
|
||||
run attachment-failover python3 student-tests/attachment_failover.py --output "$E/attachment-failover.json"
|
||||
./scripts/start-all.sh >/dev/null 2>&1
|
||||
# --- регрессии внешнего аудита (F01–F17) ---
|
||||
run api-negative-lab5 python3 student-tests/api_negative_lab5.py --output "$E/api-negative-lab5.json"
|
||||
run slow-client python3 student-tests/slow_client.py --output "$E/slow-client.json"
|
||||
run janitor-replay-race python3 student-tests/janitor_replay_race.py --output "$E/janitor-replay-race.json"
|
||||
run queue-contract-check $VPY student-tests/queue_contract_check.py --output "$E/queue-contract-check.json"
|
||||
run s3-outage python3 student-tests/s3_outage.py --output "$E/s3-outage.json"
|
||||
run upload-memory python3 student-tests/upload_memory.py --concurrency 12 --rounds 2 --output "$E/upload-memory-c12.json"
|
||||
# --- регрессия lab3 ---
|
||||
run cursor-validation python3 student-tests/cursor_validation.py
|
||||
run idempotency-concurrency python3 student-tests/idempotency_concurrency.py --output "$E/idempotency-concurrency.json"
|
||||
T=$(docker compose ps -q api | head -1 | cut -c1-12)
|
||||
run failover-bench python3 student-tests/failover_bench.py --action "env TARGET_INSTANCE_ID=$T scripts/stop-one.sh" \
|
||||
--target-instance "$T" --duration 20 --action-at 5 --concurrency 4 --output "$E/failover-bench.json"
|
||||
./scripts/start-all.sh >/dev/null 2>&1
|
||||
NEXT=$(( $(grep -E '^APP_VERSION=' .env | cut -d= -f2) + 1 ))
|
||||
run rolling-update sh -c "./scripts/rolling-update.sh $NEXT && make test"
|
||||
run janitor-dry-run docker compose exec -T janitor python /app/src/janitor.py --once --dry-run
|
||||
run queues docker compose exec -T broker rabbitmqctl -q -p messenger list_queues name type durable messages consumers
|
||||
run failed-registry docker compose --profile tools run --rm -T ops failed --limit 10
|
||||
run login-flood python3 student-tests/login_flood.py --output "$E/login-flood.json" # последним: исчерпывает лимит входов
|
||||
echo "overall=$status"
|
||||
exit $status
|
||||
|
||||
+76
-79
@@ -1,63 +1,48 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Недоступность S3: stop (порт закрыт) и pause (S3 «завис», TCP не отвечает).
|
||||
"""Недоступность S3 в lab5 (переписано из lab4 по аудиту F14: там ожидался синхронный 201).
|
||||
|
||||
python3 student-tests/s3_outage.py [--output evidence/s3-outage.json]
|
||||
python3 student-tests/s3_outage.py --output evidence/s3-outage.json
|
||||
|
||||
Ожидания: upload и download — быстрый 503 unavailable c Retry-After (не 500 и не
|
||||
таймаут LB), текстовые сообщения и /health/ready продолжают работать (S3 не входит в
|
||||
readiness), после восстановления S3 загрузка снова 201 без перезапуска API.
|
||||
API (как в lab4): при остановленном (`stop`) и зависшем (`pause`) S3 upload и download — быстрый
|
||||
503 unavailable с Retry-After, текстовые сообщения 201, /health/ready 200, GET метаданных 200.
|
||||
Worker (новое): задание, принятое ДО отказа S3, при обработке получает storage_unavailable и
|
||||
уходит в retry с backoff; после восстановления S3 оно доходит до ready. Если отказ длился
|
||||
дольше всех попыток — failed(retry_exhausted), и оператор делает replay (это тоже проверяется).
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client # noqa: E402
|
||||
|
||||
PNG = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
log = []
|
||||
from lab5lib import (Client, Log, SAMPLE_PNG, check, compose, events, ops, upload, wait_status, wait_until)
|
||||
|
||||
|
||||
def probe(c, alice, chat, att):
|
||||
def probe(c, alice, chat, att_ready, att_queued):
|
||||
out = {}
|
||||
for name, args in (
|
||||
("upload", ("POST", "/api/v1/chats/%s/attachments?filename=o.png" % chat["id"])),
|
||||
("download", ("GET", "/api/v1/attachments/%s/content" % att["id"])),
|
||||
("text_message", ("POST", "/api/v1/chats/%s/messages" % chat["id"])),
|
||||
("ready", ("GET", "/health/ready"))):
|
||||
kw = {}
|
||||
if name == "upload":
|
||||
kw = {"raw": PNG, "headers": {"Content-Type": "image/png"}}
|
||||
if name == "text_message":
|
||||
kw = {"body": {"text": "без S3"}, "headers": {"Idempotency-Key": uuid.uuid4().hex}}
|
||||
calls = {
|
||||
"upload": ("POST", "/api/v1/chats/%s/attachments?filename=o.png" % chat["id"],
|
||||
{"raw": SAMPLE_PNG, "headers": {"Content-Type": "image/png"}}),
|
||||
"download_ready": ("GET", "/api/v1/attachments/%s/content" % att_ready["id"], {}),
|
||||
"metadata_queued": ("GET", "/api/v1/attachments/%s" % att_queued["id"], {}),
|
||||
"text_message": ("POST", "/api/v1/chats/%s/messages" % chat["id"],
|
||||
{"body": {"text": "без S3"}, "headers": {"Idempotency-Key": uuid.uuid4().hex}}),
|
||||
"ready": ("GET", "/health/ready", {}),
|
||||
}
|
||||
for name, (method, path, kw) in calls.items():
|
||||
t = time.monotonic()
|
||||
r = c.call(*args, [200, 201, 503, 502, 504], actor=alice if name != "ready" else None, **kw)
|
||||
r = c.call(method, path, [200, 201, 202, 503], actor=None if name == "ready" else alice, **kw)
|
||||
out[name] = {"status": r["status"], "ms": round((time.monotonic() - t) * 1000),
|
||||
"code": (r["json"] or {}).get("error", {}).get("code"),
|
||||
"retry_after": r["headers"].get("Retry-After")}
|
||||
return out
|
||||
|
||||
|
||||
def phase(name, c, alice, chat, att):
|
||||
entry = {"phase": name, "probe": probe(c, alice, chat, att)}
|
||||
log.append(entry)
|
||||
print(json.dumps(entry, ensure_ascii=False), flush=True)
|
||||
return entry
|
||||
|
||||
|
||||
def recover(c, alice, chat, started):
|
||||
while time.monotonic() - started < 120:
|
||||
r = c.call("POST", "/api/v1/chats/%s/attachments?filename=r.png" % chat["id"], [201, 503], actor=alice,
|
||||
raw=PNG, headers={"Content-Type": "image/png"})
|
||||
if r["status"] == 201:
|
||||
return round(time.monotonic() - started, 1)
|
||||
time.sleep(0.5)
|
||||
return None
|
||||
def s3_healthy():
|
||||
cid = compose("ps", "-q", "s3").stdout.strip()
|
||||
import subprocess
|
||||
st = subprocess.run(["docker", "inspect", "-f", "{{.State.Health.Status}}", cid], capture_output=True,
|
||||
text=True).stdout.strip()
|
||||
return st == "healthy"
|
||||
|
||||
|
||||
def main():
|
||||
@@ -65,45 +50,57 @@ def main():
|
||||
ap.add_argument("--base-url", default="http://localhost:8080")
|
||||
ap.add_argument("--output")
|
||||
args = ap.parse_args()
|
||||
log = Log("S3 outage: API answers 503 fast, accepted jobs retry and complete")
|
||||
c = Client(args.base_url, timeout=30)
|
||||
alice, _bob, _eve, chat = c.bootstrap()
|
||||
att = c.call("POST", "/api/v1/chats/%s/attachments?filename=a.png" % chat["id"], 201, actor=alice, raw=PNG,
|
||||
headers={"Content-Type": "image/png"}, schema="Attachment")["json"]
|
||||
phase("baseline", c, alice, chat, att)
|
||||
for action, undo in ((["stop", "s3"], ["start", "s3"]), (["pause", "s3"], ["unpause", "s3"])):
|
||||
subprocess.run(["docker", "compose"] + action, cwd=ROOT, check=True, capture_output=True)
|
||||
time.sleep(1)
|
||||
phase("immediately after `%s` (S3Health not yet stale)" % " ".join(action), c, alice, chat, att)
|
||||
time.sleep(4)
|
||||
phase("S3 down for 5 s (`%s`)" % " ".join(action), c, alice, chat, att)
|
||||
subprocess.run(["docker", "compose"] + undo, cwd=ROOT, check=True, capture_output=True)
|
||||
t0 = time.monotonic()
|
||||
rec = recover(c, alice, chat, t0)
|
||||
log.append({"phase": "recovery after `%s`" % " ".join(undo), "upload_201_after_s": rec})
|
||||
print(json.dumps(log[-1]), flush=True)
|
||||
# Оставляем стенд рабочим: ждём, пока healthcheck S3 снова станет healthy (после pause
|
||||
# он несколько проверок подряд был unhealthy, а `docker compose up api` при нездоровой
|
||||
# зависимости не запускает реплики).
|
||||
cid = subprocess.run(["docker", "compose", "ps", "-q", "s3"], cwd=ROOT, capture_output=True, text=True).stdout.strip()
|
||||
t0 = time.monotonic()
|
||||
while time.monotonic() - t0 < 120:
|
||||
st = subprocess.run(["docker", "inspect", "-f", "{{.State.Health.Status}}", cid], capture_output=True, text=True).stdout.strip()
|
||||
if st == "healthy":
|
||||
break
|
||||
time.sleep(1)
|
||||
log.append({"phase": "s3 healthcheck healthy again", "after_s": round(time.monotonic() - t0, 1), "status": st})
|
||||
print(json.dumps(log[-1]), flush=True)
|
||||
down = [e for e in log if e.get("phase", "").startswith("S3 down")]
|
||||
ok = all(e["probe"]["upload"]["status"] == 503 and e["probe"]["download"]["status"] == 503
|
||||
and e["probe"]["text_message"]["status"] == 201 and e["probe"]["ready"]["status"] == 200
|
||||
for e in down) and all(e.get("upload_201_after_s") is not None for e in log if "upload_201_after_s" in e)
|
||||
no5xx = all(p["status"] in (200, 201, 503) for e in log if "probe" in e for p in e["probe"].values())
|
||||
ok = ok and st == "healthy"
|
||||
print(json.dumps({"pass": ok and no5xx, "only_200_201_503": no5xx}))
|
||||
if args.output:
|
||||
Path(args.output).write_text(json.dumps({"pass": ok and no5xx, "phases": log}, ensure_ascii=False, indent=1))
|
||||
return 0 if ok and no5xx else 1
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
att_ready = c.wait_ready(alice, upload(c, alice, chat, SAMPLE_PNG, "before.png"))
|
||||
res = {}
|
||||
try:
|
||||
compose("stop", "worker")
|
||||
queued = upload(c, alice, chat, SAMPLE_PNG, "accepted-before-outage.png")
|
||||
log.step("worker stopped, job accepted before outage", attachment_id=queued["id"])
|
||||
for action, undo in (("stop", "start"), ("pause", "unpause")):
|
||||
compose(action, "s3")
|
||||
time.sleep(5) # S3Health (1 с) заметил отказ, данные устарели (3 с)
|
||||
pr = probe(c, alice, chat, att_ready, queued)
|
||||
res["s3_" + action] = pr
|
||||
log.step("S3 `%s` for 5 s" % action, **pr)
|
||||
check(pr["upload"]["status"] == 503 and pr["download_ready"]["status"] == 503
|
||||
and pr["text_message"]["status"] == 201 and pr["ready"]["status"] == 200
|
||||
and pr["metadata_queued"]["status"] == 200, "API during S3 %s: %s" % (action, pr))
|
||||
if action == "stop":
|
||||
compose("up", "-d", "--no-deps", "--no-recreate", "worker")
|
||||
info = wait_until(lambda: (lambda i: i if "retry_scheduled" in events(i) else None)(
|
||||
ops("inspect", queued["id"])), 60, 1, "worker retry on storage error")
|
||||
retries = [e["detail"] for e in info["events"] if e["event"] == "retry_scheduled"]
|
||||
res["worker_during_outage"] = {"events": events(info), "retries": retries}
|
||||
log.step("worker meets S3 outage -> retry with backoff", **res["worker_during_outage"])
|
||||
check(all(r["error_code"] == "storage_unavailable" for r in retries), "retry reason")
|
||||
compose(undo, "s3")
|
||||
t0 = time.monotonic()
|
||||
wait_until(s3_healthy, 120, 1, "s3 healthy")
|
||||
log.step("S3 `%s` -> healthy" % undo, seconds=round(time.monotonic() - t0, 1))
|
||||
final = wait_status(c, bob, queued["id"], {"ready", "failed"}, 120)
|
||||
replayed = None
|
||||
if final["status"] == "failed":
|
||||
check(final["error_code"] == "retry_exhausted", "unexpected failure " + str(final["error_code"]))
|
||||
replayed = ops("replay", queued["id"], "--reason", "S3 restored")["replay"][0]["result"]
|
||||
final = wait_status(c, bob, queued["id"], {"ready", "failed"}, 60)
|
||||
info = ops("inspect", queued["id"])
|
||||
res["accepted_job"] = {"final": final["status"], "replay": replayed, "attempts": info["jobs"][0]["attempt"],
|
||||
"events": events(info)}
|
||||
check(final["status"] == "ready", "accepted job not completed")
|
||||
log.step("job accepted before outage completed", **res["accepted_job"])
|
||||
after = c.wait_ready(alice, upload(c, alice, chat, SAMPLE_PNG, "after.png"))
|
||||
res["upload_after_recovery"] = after["status"]
|
||||
log.step("upload after recovery", status=after["status"])
|
||||
finally:
|
||||
compose("unpause", "s3", ok=False)
|
||||
compose("start", "s3", ok=False)
|
||||
compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "worker", ok=False)
|
||||
log.save(args.output, results=res)
|
||||
print("PASS: S3 outage handled by API (503) and worker (retry), accepted work completed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
main()
|
||||
|
||||
@@ -4,9 +4,14 @@
|
||||
python3 student-tests/upload_memory.py --concurrency 12 --rounds 3 --output evidence/upload-memory-c12.json
|
||||
|
||||
Гипотеза: потоковый приём (буфер 64 KiB -> безымянный временный файл), multipart PUT
|
||||
частями по 5 MiB и draft-декодирование JPEG держат RAM реплики ограниченной и не
|
||||
зависящей линейно от числа одновременных загрузок; 10+ загрузок по ~10 MiB не
|
||||
приближают процесс к лимиту контейнера 256 MiB.
|
||||
частями по 5 MiB и проверка JPEG в подпроцессе с лимитом (src/jpeg_probe.py) держат RAM
|
||||
реплики ограниченной и не зависящей линейно от числа одновременных загрузок.
|
||||
|
||||
Lab5 (аудит F16): явные критерии успеха и ненулевой код выхода при нарушении — все ответы 202
|
||||
(503 только с --allow-503), ни одного OOM-kill и перезапуска реплик, пик памяти cgroup каждой
|
||||
реплики < 90% лимита. Объём считается по размеру файла в каждой записи. В смесь входят
|
||||
несжимаемый PNG ~10 MiB, baseline JPEG и progressive JPEG 4:4:4 ~40 Мпикс (самый тяжёлый
|
||||
по памяти вход, аудит F02) — JPEG генерируются в контейнере checks (Pillow).
|
||||
|
||||
Методика: перед прогоном API-реплики перезапускаются (VmHWM считается с нуля). В каждой
|
||||
реплике фоново (docker exec ... while sleep 0.05) читаются VmRSS процесса (PID 1) и
|
||||
@@ -67,18 +72,28 @@ class Sampler:
|
||||
self.p.terminate()
|
||||
|
||||
|
||||
def make_jpeg():
|
||||
try:
|
||||
from PIL import Image
|
||||
except ImportError:
|
||||
return None
|
||||
GEN = r"""
|
||||
import io, sys
|
||||
from PIL import Image
|
||||
kind = sys.argv[1]
|
||||
if kind == "baseline":
|
||||
im = Image.effect_noise((6000, 6000), 40).convert("RGB")
|
||||
for q in (60, 45, 30, 20):
|
||||
b = io.BytesIO()
|
||||
im.save(b, "JPEG", quality=q)
|
||||
b = io.BytesIO(); im.save(b, "JPEG", quality=q)
|
||||
if b.tell() <= 10 * 1024 * 1024:
|
||||
return b.getvalue()
|
||||
return None
|
||||
break
|
||||
else:
|
||||
g = Image.radial_gradient("L").resize((6324, 6324))
|
||||
b = io.BytesIO(); Image.merge("RGB", (g, g.rotate(90), g)).save(b, "JPEG", quality=85, progressive=True, subsampling=0)
|
||||
sys.stdout.buffer.write(b.getvalue())
|
||||
"""
|
||||
|
||||
|
||||
def make_jpeg(kind):
|
||||
"""JPEG генерируется в контейнере checks (на хосте Pillow может не быть)."""
|
||||
proc = subprocess.run(["docker", "compose", "--profile", "tools", "run", "--rm", "-T", "checks", "-c", GEN, kind],
|
||||
cwd=ROOT, capture_output=True, check=True)
|
||||
return proc.stdout
|
||||
|
||||
|
||||
def main():
|
||||
@@ -87,12 +102,12 @@ def main():
|
||||
ap.add_argument("--concurrency", type=int, default=12)
|
||||
ap.add_argument("--rounds", type=int, default=3)
|
||||
ap.add_argument("--no-restart", action="store_true")
|
||||
ap.add_argument("--allow-503", action="store_true", help="считать 503 (перегрузка) допустимым ответом")
|
||||
ap.add_argument("--output")
|
||||
args = ap.parse_args()
|
||||
files = [("image/png", noise_png(10 * 1024 * 1024))]
|
||||
jpeg = make_jpeg()
|
||||
if jpeg:
|
||||
files.append(("image/jpeg", jpeg))
|
||||
files.append(("image/jpeg", make_jpeg("baseline")))
|
||||
files.append(("image/jpeg", make_jpeg("progressive444")))
|
||||
if not args.no_restart:
|
||||
sh("docker", "compose", "restart", "api")
|
||||
sh("docker", "compose", "up", "-d", "--no-deps", "--no-recreate", "--wait", "api")
|
||||
@@ -104,6 +119,7 @@ def main():
|
||||
c.call("POST", "/api/v1/chats/%s/attachments?filename=w" % chat["id"], 202, actor=alice, raw=data,
|
||||
headers={"Content-Type": ctype})
|
||||
cids = containers()
|
||||
restarts_before = sh("docker", "inspect", "-f", "{{.RestartCount}}", *cids).split()
|
||||
idle = {cid[:12]: {"rss_mib": proc_status(cid, "VmRSS"), "hwm_mib_after_warmup": proc_status(cid, "VmHWM")}
|
||||
for cid in cids}
|
||||
s3_cid = sh("docker", "compose", "ps", "-q", "s3").split()[0]
|
||||
@@ -123,7 +139,9 @@ def main():
|
||||
except Exception as exc: # noqa: BLE001
|
||||
st, inst = type(exc).__name__, None
|
||||
with lock:
|
||||
results.append({"type": ctype, "status": st, "instance": inst, "s": round(time.monotonic() - t, 2)})
|
||||
# Размер — в самой записи: потоки завершаются не по порядку (аудит F16).
|
||||
results.append({"type": ctype, "bytes": len(data), "status": st, "instance": inst,
|
||||
"s": round(time.monotonic() - t, 2)})
|
||||
|
||||
started = time.monotonic()
|
||||
for _ in range(args.rounds):
|
||||
@@ -154,8 +172,7 @@ def main():
|
||||
"files": [{"type": t, "bytes": len(d), "sha256": hashlib.sha256(d).hexdigest()[:16]} for t, d in files],
|
||||
"uploads": len(results), "status_counts": {str(k): sum(1 for r in results if r["status"] == k)
|
||||
for k in {r["status"] for r in results}},
|
||||
"megabytes_uploaded_ok": round(sum(len(files[i % len(files)][1]) for i, r in enumerate(results)
|
||||
if r["status"] == 202) / 2**20, 1),
|
||||
"megabytes_uploaded_ok": round(sum(r["bytes"] for r in results if r["status"] == 202) / 2**20, 1),
|
||||
"wall_s": round(wall, 1),
|
||||
"latency_s_p50_max": [durations[len(durations) // 2], durations[-1]] if durations else None,
|
||||
"replicas": per,
|
||||
@@ -167,10 +184,20 @@ def main():
|
||||
memory_limit_mib=round(int(sh("docker", "exec", s3_cid, "cat", "/sys/fs/cgroup/memory.max")) / 2**20)),
|
||||
"oom_kills": sh("docker", "inspect", "-f", "{{.State.OOMKilled}}", *cids).split(),
|
||||
}
|
||||
allowed = {202, 503} if args.allow_503 else {202}
|
||||
restarts_after = sh("docker", "inspect", "-f", "{{.RestartCount}}", *cids).split()
|
||||
criteria = {
|
||||
"all_responses_allowed": all(r["status"] in allowed for r in results),
|
||||
"no_oom_kills": all(x == "false" for x in summary["oom_kills"]),
|
||||
"no_replica_restarts": restarts_before == restarts_after,
|
||||
"cgroup_peak_below_90pct": all(v["cgroup_memory_peak_mib"] < 0.9 * v["memory_limit_mib"] for v in per.values()),
|
||||
}
|
||||
summary["criteria"] = criteria
|
||||
summary["pass"] = all(criteria.values())
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=1))
|
||||
if args.output:
|
||||
Path(args.output).write_text(json.dumps(summary, ensure_ascii=False, indent=1))
|
||||
return 0
|
||||
return 0 if summary["pass"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user