diff --git a/.github/workflows/app.yml b/.github/workflows/app.yml index c838522..8e83f1b 100644 --- a/.github/workflows/app.yml +++ b/.github/workflows/app.yml @@ -1,4 +1,4 @@ -name: Application (lab4) +name: Application (lab5) on: [push, pull_request] permissions: contents: read @@ -13,26 +13,28 @@ jobs: python-version: '3.13' - name: Synthetic secrets and env (CI only, never committed) run: python scripts/init_secrets.py && cp .env.example .env - - name: Build, migrate once, bootstrap the S3 bucket, start 2 API replicas behind HAProxy + - name: Build, migrate once, bootstrap both S3 buckets, start broker, 2 API replicas, LB, relay, worker run: docker compose --env-file .env up -d --build --wait - name: Contract and public smoke run: make check test PYTHON=python - - name: Persistence across restart of LB, API replicas, PostgreSQL, Valkey and S3 + - name: Persistence across restart of LB, API replicas, relay, worker, PostgreSQL, Valkey, S3, RabbitMQ run: make test-persistence ACTION_SCRIPT=scripts/restart.sh PYTHON=python - name: Failover of the replica that served the test, then restore it run: | make test-failover ACTION_SCRIPT=scripts/stop-one.sh PYTHON=python sh scripts/restore-api.sh && python scripts/lbctl.py wait-up 2 - - name: Reconciler (track 2) dry-run on the stand - read-only report - run: docker compose run --rm reconciler + - name: Defense scenario - worker stopped, 202, 409, restart API and broker, ready + run: python -m pip install -q pillow==12.3.0 && python scripts/demo_pipeline.py - name: Own tests (unit + stand; every failure is restored in finally) run: | python -m pip install --require-hashes --no-deps -r requirements.txt python -m pip install -r requirements-test.txt make student-test PYTHON=python + - name: Track 2 invariants (read-only audit of PostgreSQL and both buckets) + run: docker compose run --rm --no-deps auditor --verify-checksums - name: Logs if: always() - run: docker compose logs --tail 100 api lb s3 s3-init + run: docker compose logs --tail 100 api lb s3 s3-init broker relay worker - name: Remove CI stand and its synthetic data if: always() run: docker compose down -v diff --git a/scripts/collect_evidence.py b/scripts/collect_evidence.py index 8a68c54..ae5ef3b 100644 --- a/scripts/collect_evidence.py +++ b/scripts/collect_evidence.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 """Run every check of REPORT.md on the current checkout and save the real output in evidence/. - python scripts/collect_evidence.py # ~60 min; the stand must NOT be running yet + python scripts/collect_evidence.py # ~90 min; the stand must NOT be running yet Intended for a CLEAN CLONE (fresh volumes): it creates secrets and .env, builds and starts the -stand, then runs make check/validate/test/test-persistence/test-failover, the own tests, the -lab 4 experiments (memory under concurrent uploads, reconciler faults and cost), and records -container states before/after every hook. Prints no secrets +stand, then runs make check/validate/test/test-persistence/test-failover, the lab 5 defense +scenario (scripts/demo_pipeline.py), the own tests, the track 2 fault-injection matrix +(bench/pipeline_faults.py) and the invariant audit, and records container states before/after +every hook and the real resource limits (docker inspect). Prints no secrets (hook output is shown by the public tests as suppressed; our scripts print names and states only). On Windows the hooks are the scripts/*.cmd wrappers (Git Bash) and make needs Git's usr/bin. """ @@ -98,6 +99,13 @@ def main() -> int: log.run(["docker", "compose", "logs", "--no-log-prefix", "migrate"]) log.run(["docker", "compose", "logs", "--no-log-prefix", "s3-init"]) # bucket + access-model verdicts log.run([PYTHON, "scripts/lbctl.py", "replicas"]); log.run([PYTHON, "scripts/lbctl.py", "status"]) + log.run(["docker", "compose", "exec", "-T", "broker", "rabbitmqctl", "-q", "list_users"]) + log.run(["docker", "compose", "exec", "-T", "broker", "rabbitmqctl", "-q", "list_queues", "name", "type", + "durable", "messages", "consumers", "arguments"]) + log.run(["docker", "compose", "exec", "-T", "broker", "rabbitmqctl", "-q", "list_permissions"]) + ids = subprocess.run(["docker", "compose", "ps", "-q"], cwd=ROOT, capture_output=True, text=True).stdout.split() + log.run(["docker", "inspect", "-f", "{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}} " + "memswap={{.HostConfig.MemorySwap}} image={{.Config.Image}}", *ids], shown="docker inspect (limits)") # venv for validate / own tests if not Path(VENV_PY).exists(): log.run([PYTHON, "-m", "venv", ".venv"]) @@ -111,11 +119,13 @@ def main() -> int: log.run([make, "test", f"PYTHON={MAKE_PY}"], shown="make test") # 3. persistence with states before/after - log = Log("test-persistence.txt", "make test-persistence: restart of LB, all API replicas, PostgreSQL, Valkey, S3") + log = Log("test-persistence.txt", "make test-persistence: restart of LB, API replicas, relay, worker, PostgreSQL, " + "Valkey, S3, RabbitMQ") log.note("Containers BEFORE:\n" + "\n".join(states())) log.run([make, "test-persistence", f"ACTION_SCRIPT=scripts/restart{HOOK}", f"PYTHON={MAKE_PY}"], shown=f"make test-persistence ACTION_SCRIPT=scripts/restart{HOOK}") - log.note("Containers AFTER (new pid/started for lb, api-*, postgres, valkey, s3; volumes kept):\n" + "\n".join(states())) + log.note("Containers AFTER (new pid/started for lb, api-*, relay, worker, postgres, valkey, s3, broker; volumes " + "kept):\n" + "\n".join(states())) log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"]) # 4. failover: until BOTH replicas have been the stopped one (each run is a real public test) @@ -144,16 +154,25 @@ def main() -> int: break log.note(f"Replicas stopped by the public test across runs: {sorted(stopped)}") - # 5. own tests + # 5. lab 5 defense scenario: worker stopped -> 202 -> 409 -> restart api+broker -> worker -> ready + log = Log("demo-pipeline.txt", "Defense steps 2-3: worker stopped, upload 202, content 409, restart API and broker " + "with volumes, worker started, ready + thumbnail") + log.note("Containers BEFORE:\n" + "\n".join(states())) + log.run([VENV_PY, "scripts/demo_pipeline.py"], shown="python scripts/demo_pipeline.py", timeout=900) + log.note("Containers AFTER (api-*, broker restarted; worker stopped and started):\n" + "\n".join(states())) + + # 6. own tests log = Log("student-tests.txt", "Own tests: unit + stand (pytest -v); failures restored in finally") log.run([VENV_PY, "-m", "pytest", "student-tests", "-v", "-p", "no:cacheprovider"], shown="python -m pytest student-tests -v", - timeout=3600) + timeout=5400) - # 6. lab 4 experiments (level 4 memory, track 2 reconciler) and the public load script - log = Log("lab4-runs.txt", "Lab 4: memory under 16 simultaneous uploads, reconciler faults + cost, load.py") - log.run([VENV_PY, "bench/upload_memory.py", "--output", "evidence/memory"], timeout=3600) - log.run([VENV_PY, "bench/reconcile_faults.py", "--output", "evidence/reconcile"], timeout=1800) - log.run([VENV_PY, "bench/reconcile_cost.py", "--output", "evidence/reconcile"], timeout=3600) + # 7. track 2: fault-injection matrix + audit; lab 3 load script + log = Log("track2-runs.txt", "Track 2: bench/pipeline_faults.py (duplicate, reorder, crashes, stale versions, hung " + "worker) + read-only audit of the invariants") + log.run([VENV_PY, "bench/pipeline_faults.py", "--output", "evidence/track2", "--uploads", "5"], timeout=5400) + log.run(["docker", "compose", "run", "--rm", "--no-deps", "-T", "auditor", "--verify-checksums"], + shown="docker compose run --rm auditor --verify-checksums", timeout=900) + log.run(["docker", "compose", "run", "--rm", "--no-deps", "-T", "jobctl", "stats"], shown="docker compose run --rm jobctl stats") log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"]) log.run([PYTHON, "tests/load.py", "--duration", "30", "--concurrency", "4", "--output", "evidence/load.json"]) log.run(["docker", "compose", "ps", "-a", "--format", "table {{.Service}}\t{{.Status}}"]) diff --git a/scripts/demo_pipeline.py b/scripts/demo_pipeline.py index d2f848c..79bb19e 100644 --- a/scripts/demo_pipeline.py +++ b/scripts/demo_pipeline.py @@ -34,6 +34,8 @@ def psql(sql: str) -> str: def queue_depth() -> str: + """messages/consumers of the jobs queue; rabbitmqctl refreshes these counters every ~5 s.""" + time.sleep(6) out = compose("exec", "-T", "broker", "rabbitmqctl", "-q", "list_queues", "name", "messages", "consumers") line = next(l for l in out.splitlines() if l.startswith("messenger.attachment-jobs\t")) _, messages, consumers = line.split("\t") @@ -69,7 +71,7 @@ def main() -> int: try: step("1. docker compose stop worker") compose("stop", "worker") - print(" " + compose("ps", "--format", "{{.Service}} {{.State}}", "worker").strip()) + print(" " + compose("ps", "-a", "--format", "{{.Service}} {{.State}}", "worker").strip()) step("2. upload while the worker is stopped") st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png", token=alice["token"], diff --git a/student-tests/test_pipeline_flow.py b/student-tests/test_pipeline_flow.py index 681fe0b..3e12911 100644 --- a/student-tests/test_pipeline_flow.py +++ b/student-tests/test_pipeline_flow.py @@ -144,11 +144,12 @@ def test_worker_stopped_upload_202_content_409_restart_api_and_broker_then_ready api.send(bob, chat, "текст работает без worker") msg = api.post_message(alice, chat, {"text": "фото", "attachment_ids": [att["id"]]}).json() wait_for(lambda: all(o["published"] for o in outbox_of(att["id"])), 15, "relay did not publish the job") - assert queue_depth() >= 1 # durable message waits in the queue + # rabbitmqctl counters are refreshed every ~5 s (statistics interval): wait, do not sample once. + wait_for(lambda: queue_depth() >= 1, 20, "no message in the queue") # durable message waits in the queue compose("restart", "api", "broker") # volumes preserved wait_healthy("api", "broker", "relay") wait_stand() - assert queue_depth() >= 1, "the accepted job was lost by the broker restart" + wait_for(lambda: queue_depth() >= 1, 20, "the accepted job was lost by the broker restart") assert api.attachment(bob, att["id"])["status"] == "queued" finally: compose("start", "worker")