тесты: ждать обновления счётчиков rabbitmqctl; collect_evidence и CI для lab5

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Alexey Bodanyuk
2026-10-31 17:02:24 +03:00
co-authored by Claude Opus 5.5
parent 962adbe03b
commit a3fa7713fa
4 changed files with 46 additions and 22 deletions
+32 -13
View File
@@ -1,12 +1,13 @@
#!/usr/bin/env python3
"""Run every check of REPORT.md on the current checkout and save the real output in evidence/.
python scripts/collect_evidence.py # ~60 min; the stand must NOT be running yet
python scripts/collect_evidence.py # ~90 min; the stand must NOT be running yet
Intended for a CLEAN CLONE (fresh volumes): it creates secrets and .env, builds and starts the
stand, then runs make check/validate/test/test-persistence/test-failover, the own tests, the
lab 4 experiments (memory under concurrent uploads, reconciler faults and cost), and records
container states before/after every hook. Prints no secrets
stand, then runs make check/validate/test/test-persistence/test-failover, the lab 5 defense
scenario (scripts/demo_pipeline.py), the own tests, the track 2 fault-injection matrix
(bench/pipeline_faults.py) and the invariant audit, and records container states before/after
every hook and the real resource limits (docker inspect). Prints no secrets
(hook output is shown by the public tests as suppressed; our scripts print names and states only).
On Windows the hooks are the scripts/*.cmd wrappers (Git Bash) and make needs Git's usr/bin.
"""
@@ -98,6 +99,13 @@ def main() -> int:
log.run(["docker", "compose", "logs", "--no-log-prefix", "migrate"])
log.run(["docker", "compose", "logs", "--no-log-prefix", "s3-init"]) # bucket + access-model verdicts
log.run([PYTHON, "scripts/lbctl.py", "replicas"]); log.run([PYTHON, "scripts/lbctl.py", "status"])
log.run(["docker", "compose", "exec", "-T", "broker", "rabbitmqctl", "-q", "list_users"])
log.run(["docker", "compose", "exec", "-T", "broker", "rabbitmqctl", "-q", "list_queues", "name", "type",
"durable", "messages", "consumers", "arguments"])
log.run(["docker", "compose", "exec", "-T", "broker", "rabbitmqctl", "-q", "list_permissions"])
ids = subprocess.run(["docker", "compose", "ps", "-q"], cwd=ROOT, capture_output=True, text=True).stdout.split()
log.run(["docker", "inspect", "-f", "{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}} "
"memswap={{.HostConfig.MemorySwap}} image={{.Config.Image}}", *ids], shown="docker inspect (limits)")
# venv for validate / own tests
if not Path(VENV_PY).exists():
log.run([PYTHON, "-m", "venv", ".venv"])
@@ -111,11 +119,13 @@ def main() -> int:
log.run([make, "test", f"PYTHON={MAKE_PY}"], shown="make test")
# 3. persistence with states before/after
log = Log("test-persistence.txt", "make test-persistence: restart of LB, all API replicas, PostgreSQL, Valkey, S3")
log = Log("test-persistence.txt", "make test-persistence: restart of LB, API replicas, relay, worker, PostgreSQL, "
"Valkey, S3, RabbitMQ")
log.note("Containers BEFORE:\n" + "\n".join(states()))
log.run([make, "test-persistence", f"ACTION_SCRIPT=scripts/restart{HOOK}", f"PYTHON={MAKE_PY}"],
shown=f"make test-persistence ACTION_SCRIPT=scripts/restart{HOOK}")
log.note("Containers AFTER (new pid/started for lb, api-*, postgres, valkey, s3; volumes kept):\n" + "\n".join(states()))
log.note("Containers AFTER (new pid/started for lb, api-*, relay, worker, postgres, valkey, s3, broker; volumes "
"kept):\n" + "\n".join(states()))
log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"])
# 4. failover: until BOTH replicas have been the stopped one (each run is a real public test)
@@ -144,16 +154,25 @@ def main() -> int:
break
log.note(f"Replicas stopped by the public test across runs: {sorted(stopped)}")
# 5. own tests
# 5. lab 5 defense scenario: worker stopped -> 202 -> 409 -> restart api+broker -> worker -> ready
log = Log("demo-pipeline.txt", "Defense steps 2-3: worker stopped, upload 202, content 409, restart API and broker "
"with volumes, worker started, ready + thumbnail")
log.note("Containers BEFORE:\n" + "\n".join(states()))
log.run([VENV_PY, "scripts/demo_pipeline.py"], shown="python scripts/demo_pipeline.py", timeout=900)
log.note("Containers AFTER (api-*, broker restarted; worker stopped and started):\n" + "\n".join(states()))
# 6. own tests
log = Log("student-tests.txt", "Own tests: unit + stand (pytest -v); failures restored in finally")
log.run([VENV_PY, "-m", "pytest", "student-tests", "-v", "-p", "no:cacheprovider"], shown="python -m pytest student-tests -v",
timeout=3600)
timeout=5400)
# 6. lab 4 experiments (level 4 memory, track 2 reconciler) and the public load script
log = Log("lab4-runs.txt", "Lab 4: memory under 16 simultaneous uploads, reconciler faults + cost, load.py")
log.run([VENV_PY, "bench/upload_memory.py", "--output", "evidence/memory"], timeout=3600)
log.run([VENV_PY, "bench/reconcile_faults.py", "--output", "evidence/reconcile"], timeout=1800)
log.run([VENV_PY, "bench/reconcile_cost.py", "--output", "evidence/reconcile"], timeout=3600)
# 7. track 2: fault-injection matrix + audit; lab 3 load script
log = Log("track2-runs.txt", "Track 2: bench/pipeline_faults.py (duplicate, reorder, crashes, stale versions, hung "
"worker) + read-only audit of the invariants")
log.run([VENV_PY, "bench/pipeline_faults.py", "--output", "evidence/track2", "--uploads", "5"], timeout=5400)
log.run(["docker", "compose", "run", "--rm", "--no-deps", "-T", "auditor", "--verify-checksums"],
shown="docker compose run --rm auditor --verify-checksums", timeout=900)
log.run(["docker", "compose", "run", "--rm", "--no-deps", "-T", "jobctl", "stats"], shown="docker compose run --rm jobctl stats")
log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"])
log.run([PYTHON, "tests/load.py", "--duration", "30", "--concurrency", "4", "--output", "evidence/load.json"])
log.run(["docker", "compose", "ps", "-a", "--format", "table {{.Service}}\t{{.Status}}"])
+3 -1
View File
@@ -34,6 +34,8 @@ def psql(sql: str) -> str:
def queue_depth() -> str:
"""messages/consumers of the jobs queue; rabbitmqctl refreshes these counters every ~5 s."""
time.sleep(6)
out = compose("exec", "-T", "broker", "rabbitmqctl", "-q", "list_queues", "name", "messages", "consumers")
line = next(l for l in out.splitlines() if l.startswith("messenger.attachment-jobs\t"))
_, messages, consumers = line.split("\t")
@@ -69,7 +71,7 @@ def main() -> int:
try:
step("1. docker compose stop worker")
compose("stop", "worker")
print(" " + compose("ps", "--format", "{{.Service}} {{.State}}", "worker").strip())
print(" " + compose("ps", "-a", "--format", "{{.Service}} {{.State}}", "worker").strip())
step("2. upload while the worker is stopped")
st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png", token=alice["token"],