тесты: ждать обновления счётчиков rabbitmqctl; collect_evidence и CI для lab5

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Alexey Bodanyuk
2026-10-31 17:02:24 +03:00
co-authored by Claude Opus 5.5
parent 962adbe03b
commit a3fa7713fa
4 changed files with 46 additions and 22 deletions
+8 -6
View File
@@ -1,4 +1,4 @@
name: Application (lab4)
name: Application (lab5)
on: [push, pull_request]
permissions:
contents: read
@@ -13,26 +13,28 @@ jobs:
python-version: '3.13'
- name: Synthetic secrets and env (CI only, never committed)
run: python scripts/init_secrets.py && cp .env.example .env
- name: Build, migrate once, bootstrap the S3 bucket, start 2 API replicas behind HAProxy
- name: Build, migrate once, bootstrap both S3 buckets, start broker, 2 API replicas, LB, relay, worker
run: docker compose --env-file .env up -d --build --wait
- name: Contract and public smoke
run: make check test PYTHON=python
- name: Persistence across restart of LB, API replicas, PostgreSQL, Valkey and S3
- name: Persistence across restart of LB, API replicas, relay, worker, PostgreSQL, Valkey, S3, RabbitMQ
run: make test-persistence ACTION_SCRIPT=scripts/restart.sh PYTHON=python
- name: Failover of the replica that served the test, then restore it
run: |
make test-failover ACTION_SCRIPT=scripts/stop-one.sh PYTHON=python
sh scripts/restore-api.sh && python scripts/lbctl.py wait-up 2
- name: Reconciler (track 2) dry-run on the stand - read-only report
run: docker compose run --rm reconciler
- name: Defense scenario - worker stopped, 202, 409, restart API and broker, ready
run: python -m pip install -q pillow==12.3.0 && python scripts/demo_pipeline.py
- name: Own tests (unit + stand; every failure is restored in finally)
run: |
python -m pip install --require-hashes --no-deps -r requirements.txt
python -m pip install -r requirements-test.txt
make student-test PYTHON=python
- name: Track 2 invariants (read-only audit of PostgreSQL and both buckets)
run: docker compose run --rm --no-deps auditor --verify-checksums
- name: Logs
if: always()
run: docker compose logs --tail 100 api lb s3 s3-init
run: docker compose logs --tail 100 api lb s3 s3-init broker relay worker
- name: Remove CI stand and its synthetic data
if: always()
run: docker compose down -v
+32 -13
View File
@@ -1,12 +1,13 @@
#!/usr/bin/env python3
"""Run every check of REPORT.md on the current checkout and save the real output in evidence/.
python scripts/collect_evidence.py # ~60 min; the stand must NOT be running yet
python scripts/collect_evidence.py # ~90 min; the stand must NOT be running yet
Intended for a CLEAN CLONE (fresh volumes): it creates secrets and .env, builds and starts the
stand, then runs make check/validate/test/test-persistence/test-failover, the own tests, the
lab 4 experiments (memory under concurrent uploads, reconciler faults and cost), and records
container states before/after every hook. Prints no secrets
stand, then runs make check/validate/test/test-persistence/test-failover, the lab 5 defense
scenario (scripts/demo_pipeline.py), the own tests, the track 2 fault-injection matrix
(bench/pipeline_faults.py) and the invariant audit, and records container states before/after
every hook and the real resource limits (docker inspect). Prints no secrets
(hook output is shown by the public tests as suppressed; our scripts print names and states only).
On Windows the hooks are the scripts/*.cmd wrappers (Git Bash) and make needs Git's usr/bin.
"""
@@ -98,6 +99,13 @@ def main() -> int:
log.run(["docker", "compose", "logs", "--no-log-prefix", "migrate"])
log.run(["docker", "compose", "logs", "--no-log-prefix", "s3-init"]) # bucket + access-model verdicts
log.run([PYTHON, "scripts/lbctl.py", "replicas"]); log.run([PYTHON, "scripts/lbctl.py", "status"])
log.run(["docker", "compose", "exec", "-T", "broker", "rabbitmqctl", "-q", "list_users"])
log.run(["docker", "compose", "exec", "-T", "broker", "rabbitmqctl", "-q", "list_queues", "name", "type",
"durable", "messages", "consumers", "arguments"])
log.run(["docker", "compose", "exec", "-T", "broker", "rabbitmqctl", "-q", "list_permissions"])
ids = subprocess.run(["docker", "compose", "ps", "-q"], cwd=ROOT, capture_output=True, text=True).stdout.split()
log.run(["docker", "inspect", "-f", "{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}} "
"memswap={{.HostConfig.MemorySwap}} image={{.Config.Image}}", *ids], shown="docker inspect (limits)")
# venv for validate / own tests
if not Path(VENV_PY).exists():
log.run([PYTHON, "-m", "venv", ".venv"])
@@ -111,11 +119,13 @@ def main() -> int:
log.run([make, "test", f"PYTHON={MAKE_PY}"], shown="make test")
# 3. persistence with states before/after
log = Log("test-persistence.txt", "make test-persistence: restart of LB, all API replicas, PostgreSQL, Valkey, S3")
log = Log("test-persistence.txt", "make test-persistence: restart of LB, API replicas, relay, worker, PostgreSQL, "
"Valkey, S3, RabbitMQ")
log.note("Containers BEFORE:\n" + "\n".join(states()))
log.run([make, "test-persistence", f"ACTION_SCRIPT=scripts/restart{HOOK}", f"PYTHON={MAKE_PY}"],
shown=f"make test-persistence ACTION_SCRIPT=scripts/restart{HOOK}")
log.note("Containers AFTER (new pid/started for lb, api-*, postgres, valkey, s3; volumes kept):\n" + "\n".join(states()))
log.note("Containers AFTER (new pid/started for lb, api-*, relay, worker, postgres, valkey, s3, broker; volumes "
"kept):\n" + "\n".join(states()))
log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"])
# 4. failover: until BOTH replicas have been the stopped one (each run is a real public test)
@@ -144,16 +154,25 @@ def main() -> int:
break
log.note(f"Replicas stopped by the public test across runs: {sorted(stopped)}")
# 5. own tests
# 5. lab 5 defense scenario: worker stopped -> 202 -> 409 -> restart api+broker -> worker -> ready
log = Log("demo-pipeline.txt", "Defense steps 2-3: worker stopped, upload 202, content 409, restart API and broker "
"with volumes, worker started, ready + thumbnail")
log.note("Containers BEFORE:\n" + "\n".join(states()))
log.run([VENV_PY, "scripts/demo_pipeline.py"], shown="python scripts/demo_pipeline.py", timeout=900)
log.note("Containers AFTER (api-*, broker restarted; worker stopped and started):\n" + "\n".join(states()))
# 6. own tests
log = Log("student-tests.txt", "Own tests: unit + stand (pytest -v); failures restored in finally")
log.run([VENV_PY, "-m", "pytest", "student-tests", "-v", "-p", "no:cacheprovider"], shown="python -m pytest student-tests -v",
timeout=3600)
timeout=5400)
# 6. lab 4 experiments (level 4 memory, track 2 reconciler) and the public load script
log = Log("lab4-runs.txt", "Lab 4: memory under 16 simultaneous uploads, reconciler faults + cost, load.py")
log.run([VENV_PY, "bench/upload_memory.py", "--output", "evidence/memory"], timeout=3600)
log.run([VENV_PY, "bench/reconcile_faults.py", "--output", "evidence/reconcile"], timeout=1800)
log.run([VENV_PY, "bench/reconcile_cost.py", "--output", "evidence/reconcile"], timeout=3600)
# 7. track 2: fault-injection matrix + audit; lab 3 load script
log = Log("track2-runs.txt", "Track 2: bench/pipeline_faults.py (duplicate, reorder, crashes, stale versions, hung "
"worker) + read-only audit of the invariants")
log.run([VENV_PY, "bench/pipeline_faults.py", "--output", "evidence/track2", "--uploads", "5"], timeout=5400)
log.run(["docker", "compose", "run", "--rm", "--no-deps", "-T", "auditor", "--verify-checksums"],
shown="docker compose run --rm auditor --verify-checksums", timeout=900)
log.run(["docker", "compose", "run", "--rm", "--no-deps", "-T", "jobctl", "stats"], shown="docker compose run --rm jobctl stats")
log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"])
log.run([PYTHON, "tests/load.py", "--duration", "30", "--concurrency", "4", "--output", "evidence/load.json"])
log.run(["docker", "compose", "ps", "-a", "--format", "table {{.Service}}\t{{.Status}}"])
+3 -1
View File
@@ -34,6 +34,8 @@ def psql(sql: str) -> str:
def queue_depth() -> str:
"""messages/consumers of the jobs queue; rabbitmqctl refreshes these counters every ~5 s."""
time.sleep(6)
out = compose("exec", "-T", "broker", "rabbitmqctl", "-q", "list_queues", "name", "messages", "consumers")
line = next(l for l in out.splitlines() if l.startswith("messenger.attachment-jobs\t"))
_, messages, consumers = line.split("\t")
@@ -69,7 +71,7 @@ def main() -> int:
try:
step("1. docker compose stop worker")
compose("stop", "worker")
print(" " + compose("ps", "--format", "{{.Service}} {{.State}}", "worker").strip())
print(" " + compose("ps", "-a", "--format", "{{.Service}} {{.State}}", "worker").strip())
step("2. upload while the worker is stopped")
st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png", token=alice["token"],
+3 -2
View File
@@ -144,11 +144,12 @@ def test_worker_stopped_upload_202_content_409_restart_api_and_broker_then_ready
api.send(bob, chat, "текст работает без worker")
msg = api.post_message(alice, chat, {"text": "фото", "attachment_ids": [att["id"]]}).json()
wait_for(lambda: all(o["published"] for o in outbox_of(att["id"])), 15, "relay did not publish the job")
assert queue_depth() >= 1 # durable message waits in the queue
# rabbitmqctl counters are refreshed every ~5 s (statistics interval): wait, do not sample once.
wait_for(lambda: queue_depth() >= 1, 20, "no message in the queue") # durable message waits in the queue
compose("restart", "api", "broker") # volumes preserved
wait_healthy("api", "broker", "relay")
wait_stand()
assert queue_depth() >= 1, "the accepted job was lost by the broker restart"
wait_for(lambda: queue_depth() >= 1, 20, "the accepted job was lost by the broker restart")
assert api.attachment(bob, att["id"])["status"] == "queued"
finally:
compose("start", "worker")