diff --git a/Makefile b/Makefile index 5d86d171..af3f116c 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load test-persistence test-failover up down test-files test-pipeline test-faults failed replay evidence help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -28,3 +28,33 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +# --- Студенческие цели (lab5) --- +up: + sh scripts/init-secrets.sh + docker compose --env-file .env up -d --build --wait + +down: + docker compose --env-file .env stop + +test-files: + $(PYTHON) student-tests/attachments_negative.py --base-url "$(BASE_URL)" + +# Сценарии pipeline на стенде: make test-pipeline SCENARIO=demo (список — student-tests/pipeline_ops.py) +SCENARIO ?= demo +test-pipeline: + $(PYTHON) student-tests/pipeline_ops.py $(SCENARIO) --base-url "$(BASE_URL)" + +# Интеграционный fault-injection pipeline (трек 5); нужен PostgreSQL: TEST_DATABASE_URL=postgres://... +test-faults: + go test ./internal/worker/ -run Pipeline -count=1 -v + +failed: + docker compose run --rm --no-deps -T --entrypoint /app/pipeline relay failed + +# make replay ID= +replay: + docker compose run --rm --no-deps -T --entrypoint /app/pipeline relay replay $(ID) --actor operator + +evidence: + sh scripts/collect-evidence.sh all diff --git a/student-tests/pipeline_ops.py b/student-tests/pipeline_ops.py index 84800dc1..32499078 100755 --- a/student-tests/pipeline_ops.py +++ b/student-tests/pipeline_ops.py @@ -260,21 +260,38 @@ def scenario_broker_restart(c): time.sleep(0.5) before = queues() compose('restart', 'rabbitmq') - after = queues() - compose('start', 'worker') + # Quorum-очередь после рестарта выбирает лидера и восстанавливает журнал: ждём. + after = {} + for _ in range(30): + after = queues() + if after.get('attachment.process', 0) >= 1: + break + time.sleep(1) + sh('docker', 'start', compose('ps', '-aq', 'worker').stdout.split()[0]) # без запуска зависимостей wait_status(c, bob, att['id'], ('ready',)) + published = psql("SELECT publish_attempts FROM outbox WHERE job_id='%s'" % jid) check(after.get('attachment.process', 0) >= 1, 'message lost on broker restart: %s' % after) + check(published == '1', 'message was republished from outbox (%s): not a broker durability proof' % published) return {'attachment_id': att['id'], 'queue_before_restart': before, 'queue_after_restart': after, + 'outbox_publish_attempts': int(published), 'invariants': invariants(att['id'], chat['id'])} def scenario_transient(c): alice, bob, _, chat = c.bootstrap() + worker_cid = compose('ps', '-q', 'worker').stdout.split()[0] + s3_cid = compose('ps', '-q', 's3').stdout.split()[0] compose('stop', 'worker') att = upload(c, alice, chat, sample(), name='retry.png') jid, _, _ = job_of(att['id']) - compose('stop', 's3') - compose('start', 'worker') + for _ in range(30): + if psql("SELECT count(*) FROM outbox WHERE job_id='%s' AND published_at IS NOT NULL" % jid) == '1': + break + time.sleep(0.5) + # Прямые docker stop/start: `docker compose start worker` поднял бы и S3 + # (worker → s3-init → s3), и временного отказа бы не было. + sh('docker', 'stop', s3_cid) + sh('docker', 'start', worker_cid) seen = [] t0 = time.time() while time.time() - t0 < 25: @@ -283,7 +300,7 @@ def scenario_transient(c): if seen.count('retry_scheduled') >= 2: break time.sleep(1) - compose('start', 's3') + sh('docker', 'start', s3_cid) meta = wait_status(c, bob, att['id'], ('ready',), timeout=150) _, state, attempt = job_of(att['id']) details = psql("SELECT string_agg(detail, ' | ' ORDER BY id) FROM job_events WHERE job_id='%s' AND event='retry_scheduled'" % jid)