forked from mtusi2026/mtusi2026lab5
тесты: ждать обновления счётчиков rabbitmqctl; collect_evidence и CI для lab5
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
962adbe03b
commit
a3fa7713fa
@@ -1,4 +1,4 @@
|
||||
name: Application (lab4)
|
||||
name: Application (lab5)
|
||||
on: [push, pull_request]
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -13,26 +13,28 @@ jobs:
|
||||
python-version: '3.13'
|
||||
- name: Synthetic secrets and env (CI only, never committed)
|
||||
run: python scripts/init_secrets.py && cp .env.example .env
|
||||
- name: Build, migrate once, bootstrap the S3 bucket, start 2 API replicas behind HAProxy
|
||||
- name: Build, migrate once, bootstrap both S3 buckets, start broker, 2 API replicas, LB, relay, worker
|
||||
run: docker compose --env-file .env up -d --build --wait
|
||||
- name: Contract and public smoke
|
||||
run: make check test PYTHON=python
|
||||
- name: Persistence across restart of LB, API replicas, PostgreSQL, Valkey and S3
|
||||
- name: Persistence across restart of LB, API replicas, relay, worker, PostgreSQL, Valkey, S3, RabbitMQ
|
||||
run: make test-persistence ACTION_SCRIPT=scripts/restart.sh PYTHON=python
|
||||
- name: Failover of the replica that served the test, then restore it
|
||||
run: |
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh PYTHON=python
|
||||
sh scripts/restore-api.sh && python scripts/lbctl.py wait-up 2
|
||||
- name: Reconciler (track 2) dry-run on the stand - read-only report
|
||||
run: docker compose run --rm reconciler
|
||||
- name: Defense scenario - worker stopped, 202, 409, restart API and broker, ready
|
||||
run: python -m pip install -q pillow==12.3.0 && python scripts/demo_pipeline.py
|
||||
- name: Own tests (unit + stand; every failure is restored in finally)
|
||||
run: |
|
||||
python -m pip install --require-hashes --no-deps -r requirements.txt
|
||||
python -m pip install -r requirements-test.txt
|
||||
make student-test PYTHON=python
|
||||
- name: Track 2 invariants (read-only audit of PostgreSQL and both buckets)
|
||||
run: docker compose run --rm --no-deps auditor --verify-checksums
|
||||
- name: Logs
|
||||
if: always()
|
||||
run: docker compose logs --tail 100 api lb s3 s3-init
|
||||
run: docker compose logs --tail 100 api lb s3 s3-init broker relay worker
|
||||
- name: Remove CI stand and its synthetic data
|
||||
if: always()
|
||||
run: docker compose down -v
|
||||
|
||||
+32
-13
@@ -1,12 +1,13 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run every check of REPORT.md on the current checkout and save the real output in evidence/.
|
||||
|
||||
python scripts/collect_evidence.py # ~60 min; the stand must NOT be running yet
|
||||
python scripts/collect_evidence.py # ~90 min; the stand must NOT be running yet
|
||||
|
||||
Intended for a CLEAN CLONE (fresh volumes): it creates secrets and .env, builds and starts the
|
||||
stand, then runs make check/validate/test/test-persistence/test-failover, the own tests, the
|
||||
lab 4 experiments (memory under concurrent uploads, reconciler faults and cost), and records
|
||||
container states before/after every hook. Prints no secrets
|
||||
stand, then runs make check/validate/test/test-persistence/test-failover, the lab 5 defense
|
||||
scenario (scripts/demo_pipeline.py), the own tests, the track 2 fault-injection matrix
|
||||
(bench/pipeline_faults.py) and the invariant audit, and records container states before/after
|
||||
every hook and the real resource limits (docker inspect). Prints no secrets
|
||||
(hook output is shown by the public tests as suppressed; our scripts print names and states only).
|
||||
On Windows the hooks are the scripts/*.cmd wrappers (Git Bash) and make needs Git's usr/bin.
|
||||
"""
|
||||
@@ -98,6 +99,13 @@ def main() -> int:
|
||||
log.run(["docker", "compose", "logs", "--no-log-prefix", "migrate"])
|
||||
log.run(["docker", "compose", "logs", "--no-log-prefix", "s3-init"]) # bucket + access-model verdicts
|
||||
log.run([PYTHON, "scripts/lbctl.py", "replicas"]); log.run([PYTHON, "scripts/lbctl.py", "status"])
|
||||
log.run(["docker", "compose", "exec", "-T", "broker", "rabbitmqctl", "-q", "list_users"])
|
||||
log.run(["docker", "compose", "exec", "-T", "broker", "rabbitmqctl", "-q", "list_queues", "name", "type",
|
||||
"durable", "messages", "consumers", "arguments"])
|
||||
log.run(["docker", "compose", "exec", "-T", "broker", "rabbitmqctl", "-q", "list_permissions"])
|
||||
ids = subprocess.run(["docker", "compose", "ps", "-q"], cwd=ROOT, capture_output=True, text=True).stdout.split()
|
||||
log.run(["docker", "inspect", "-f", "{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}} "
|
||||
"memswap={{.HostConfig.MemorySwap}} image={{.Config.Image}}", *ids], shown="docker inspect (limits)")
|
||||
# venv for validate / own tests
|
||||
if not Path(VENV_PY).exists():
|
||||
log.run([PYTHON, "-m", "venv", ".venv"])
|
||||
@@ -111,11 +119,13 @@ def main() -> int:
|
||||
log.run([make, "test", f"PYTHON={MAKE_PY}"], shown="make test")
|
||||
|
||||
# 3. persistence with states before/after
|
||||
log = Log("test-persistence.txt", "make test-persistence: restart of LB, all API replicas, PostgreSQL, Valkey, S3")
|
||||
log = Log("test-persistence.txt", "make test-persistence: restart of LB, API replicas, relay, worker, PostgreSQL, "
|
||||
"Valkey, S3, RabbitMQ")
|
||||
log.note("Containers BEFORE:\n" + "\n".join(states()))
|
||||
log.run([make, "test-persistence", f"ACTION_SCRIPT=scripts/restart{HOOK}", f"PYTHON={MAKE_PY}"],
|
||||
shown=f"make test-persistence ACTION_SCRIPT=scripts/restart{HOOK}")
|
||||
log.note("Containers AFTER (new pid/started for lb, api-*, postgres, valkey, s3; volumes kept):\n" + "\n".join(states()))
|
||||
log.note("Containers AFTER (new pid/started for lb, api-*, relay, worker, postgres, valkey, s3, broker; volumes "
|
||||
"kept):\n" + "\n".join(states()))
|
||||
log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"])
|
||||
|
||||
# 4. failover: until BOTH replicas have been the stopped one (each run is a real public test)
|
||||
@@ -144,16 +154,25 @@ def main() -> int:
|
||||
break
|
||||
log.note(f"Replicas stopped by the public test across runs: {sorted(stopped)}")
|
||||
|
||||
# 5. own tests
|
||||
# 5. lab 5 defense scenario: worker stopped -> 202 -> 409 -> restart api+broker -> worker -> ready
|
||||
log = Log("demo-pipeline.txt", "Defense steps 2-3: worker stopped, upload 202, content 409, restart API and broker "
|
||||
"with volumes, worker started, ready + thumbnail")
|
||||
log.note("Containers BEFORE:\n" + "\n".join(states()))
|
||||
log.run([VENV_PY, "scripts/demo_pipeline.py"], shown="python scripts/demo_pipeline.py", timeout=900)
|
||||
log.note("Containers AFTER (api-*, broker restarted; worker stopped and started):\n" + "\n".join(states()))
|
||||
|
||||
# 6. own tests
|
||||
log = Log("student-tests.txt", "Own tests: unit + stand (pytest -v); failures restored in finally")
|
||||
log.run([VENV_PY, "-m", "pytest", "student-tests", "-v", "-p", "no:cacheprovider"], shown="python -m pytest student-tests -v",
|
||||
timeout=3600)
|
||||
timeout=5400)
|
||||
|
||||
# 6. lab 4 experiments (level 4 memory, track 2 reconciler) and the public load script
|
||||
log = Log("lab4-runs.txt", "Lab 4: memory under 16 simultaneous uploads, reconciler faults + cost, load.py")
|
||||
log.run([VENV_PY, "bench/upload_memory.py", "--output", "evidence/memory"], timeout=3600)
|
||||
log.run([VENV_PY, "bench/reconcile_faults.py", "--output", "evidence/reconcile"], timeout=1800)
|
||||
log.run([VENV_PY, "bench/reconcile_cost.py", "--output", "evidence/reconcile"], timeout=3600)
|
||||
# 7. track 2: fault-injection matrix + audit; lab 3 load script
|
||||
log = Log("track2-runs.txt", "Track 2: bench/pipeline_faults.py (duplicate, reorder, crashes, stale versions, hung "
|
||||
"worker) + read-only audit of the invariants")
|
||||
log.run([VENV_PY, "bench/pipeline_faults.py", "--output", "evidence/track2", "--uploads", "5"], timeout=5400)
|
||||
log.run(["docker", "compose", "run", "--rm", "--no-deps", "-T", "auditor", "--verify-checksums"],
|
||||
shown="docker compose run --rm auditor --verify-checksums", timeout=900)
|
||||
log.run(["docker", "compose", "run", "--rm", "--no-deps", "-T", "jobctl", "stats"], shown="docker compose run --rm jobctl stats")
|
||||
log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"])
|
||||
log.run([PYTHON, "tests/load.py", "--duration", "30", "--concurrency", "4", "--output", "evidence/load.json"])
|
||||
log.run(["docker", "compose", "ps", "-a", "--format", "table {{.Service}}\t{{.Status}}"])
|
||||
|
||||
@@ -34,6 +34,8 @@ def psql(sql: str) -> str:
|
||||
|
||||
|
||||
def queue_depth() -> str:
|
||||
"""messages/consumers of the jobs queue; rabbitmqctl refreshes these counters every ~5 s."""
|
||||
time.sleep(6)
|
||||
out = compose("exec", "-T", "broker", "rabbitmqctl", "-q", "list_queues", "name", "messages", "consumers")
|
||||
line = next(l for l in out.splitlines() if l.startswith("messenger.attachment-jobs\t"))
|
||||
_, messages, consumers = line.split("\t")
|
||||
@@ -69,7 +71,7 @@ def main() -> int:
|
||||
try:
|
||||
step("1. docker compose stop worker")
|
||||
compose("stop", "worker")
|
||||
print(" " + compose("ps", "--format", "{{.Service}} {{.State}}", "worker").strip())
|
||||
print(" " + compose("ps", "-a", "--format", "{{.Service}} {{.State}}", "worker").strip())
|
||||
|
||||
step("2. upload while the worker is stopped")
|
||||
st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png", token=alice["token"],
|
||||
|
||||
@@ -144,11 +144,12 @@ def test_worker_stopped_upload_202_content_409_restart_api_and_broker_then_ready
|
||||
api.send(bob, chat, "текст работает без worker")
|
||||
msg = api.post_message(alice, chat, {"text": "фото", "attachment_ids": [att["id"]]}).json()
|
||||
wait_for(lambda: all(o["published"] for o in outbox_of(att["id"])), 15, "relay did not publish the job")
|
||||
assert queue_depth() >= 1 # durable message waits in the queue
|
||||
# rabbitmqctl counters are refreshed every ~5 s (statistics interval): wait, do not sample once.
|
||||
wait_for(lambda: queue_depth() >= 1, 20, "no message in the queue") # durable message waits in the queue
|
||||
compose("restart", "api", "broker") # volumes preserved
|
||||
wait_healthy("api", "broker", "relay")
|
||||
wait_stand()
|
||||
assert queue_depth() >= 1, "the accepted job was lost by the broker restart"
|
||||
wait_for(lambda: queue_depth() >= 1, 20, "the accepted job was lost by the broker restart")
|
||||
assert api.attachment(bob, att["id"])["status"] == "queued"
|
||||
finally:
|
||||
compose("start", "worker")
|
||||
|
||||
Reference in New Issue
Block a user