tests(lab5): прямой docker stop/start для S3 и worker, ожидание лидера quorum-очереди; цели Makefile

This commit is contained in:
2026-11-09 17:53:38 +03:00
parent 16442809f1
commit 6a797e468c
2 changed files with 53 additions and 6 deletions
+31 -1
View File
@@ -4,7 +4,7 @@ CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover
.PHONY: help check validate test load test-persistence test-failover up down test-files test-pipeline test-faults failed replay evidence
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@@ -28,3 +28,33 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
# --- Студенческие цели (lab5) ---
up:
sh scripts/init-secrets.sh
docker compose --env-file .env up -d --build --wait
down:
docker compose --env-file .env stop
test-files:
$(PYTHON) student-tests/attachments_negative.py --base-url "$(BASE_URL)"
# Сценарии pipeline на стенде: make test-pipeline SCENARIO=demo (список — student-tests/pipeline_ops.py)
SCENARIO ?= demo
test-pipeline:
$(PYTHON) student-tests/pipeline_ops.py $(SCENARIO) --base-url "$(BASE_URL)"
# Интеграционный fault-injection pipeline (трек 5); нужен PostgreSQL: TEST_DATABASE_URL=postgres://...
test-faults:
go test ./internal/worker/ -run Pipeline -count=1 -v
failed:
docker compose run --rm --no-deps -T --entrypoint /app/pipeline relay failed
# make replay ID=<attachment_id>
replay:
docker compose run --rm --no-deps -T --entrypoint /app/pipeline relay replay $(ID) --actor operator
evidence:
sh scripts/collect-evidence.sh all
+22 -5
View File
@@ -260,21 +260,38 @@ def scenario_broker_restart(c):
time.sleep(0.5)
before = queues()
compose('restart', 'rabbitmq')
after = queues()
compose('start', 'worker')
# Quorum-очередь после рестарта выбирает лидера и восстанавливает журнал: ждём.
after = {}
for _ in range(30):
after = queues()
if after.get('attachment.process', 0) >= 1:
break
time.sleep(1)
sh('docker', 'start', compose('ps', '-aq', 'worker').stdout.split()[0]) # без запуска зависимостей
wait_status(c, bob, att['id'], ('ready',))
published = psql("SELECT publish_attempts FROM outbox WHERE job_id='%s'" % jid)
check(after.get('attachment.process', 0) >= 1, 'message lost on broker restart: %s' % after)
check(published == '1', 'message was republished from outbox (%s): not a broker durability proof' % published)
return {'attachment_id': att['id'], 'queue_before_restart': before, 'queue_after_restart': after,
'outbox_publish_attempts': int(published),
'invariants': invariants(att['id'], chat['id'])}
def scenario_transient(c):
alice, bob, _, chat = c.bootstrap()
worker_cid = compose('ps', '-q', 'worker').stdout.split()[0]
s3_cid = compose('ps', '-q', 's3').stdout.split()[0]
compose('stop', 'worker')
att = upload(c, alice, chat, sample(), name='retry.png')
jid, _, _ = job_of(att['id'])
compose('stop', 's3')
compose('start', 'worker')
for _ in range(30):
if psql("SELECT count(*) FROM outbox WHERE job_id='%s' AND published_at IS NOT NULL" % jid) == '1':
break
time.sleep(0.5)
# Прямые docker stop/start: `docker compose start worker` поднял бы и S3
# (worker → s3-init → s3), и временного отказа бы не было.
sh('docker', 'stop', s3_cid)
sh('docker', 'start', worker_cid)
seen = []
t0 = time.time()
while time.time() - t0 < 25:
@@ -283,7 +300,7 @@ def scenario_transient(c):
if seen.count('retry_scheduled') >= 2:
break
time.sleep(1)
compose('start', 's3')
sh('docker', 'start', s3_cid)
meta = wait_status(c, bob, att['id'], ('ready',), timeout=150)
_, state, attempt = job_of(att['id'])
details = psql("SELECT string_agg(detail, ' | ' ORDER BY id) FROM job_events WHERE job_id='%s' AND event='retry_scheduled'" % jid)