tests(lab5): прямой docker stop/start для S3 и worker, ожидание лидера quorum-очереди; цели Makefile
This commit is contained in:
@@ -4,7 +4,7 @@ CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover
|
||||
.PHONY: help check validate test load test-persistence test-failover up down test-files test-pipeline test-faults failed replay evidence
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@@ -28,3 +28,33 @@ test-persistence:
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
# --- Студенческие цели (lab5) ---
|
||||
up:
|
||||
sh scripts/init-secrets.sh
|
||||
docker compose --env-file .env up -d --build --wait
|
||||
|
||||
down:
|
||||
docker compose --env-file .env stop
|
||||
|
||||
test-files:
|
||||
$(PYTHON) student-tests/attachments_negative.py --base-url "$(BASE_URL)"
|
||||
|
||||
# Сценарии pipeline на стенде: make test-pipeline SCENARIO=demo (список — student-tests/pipeline_ops.py)
|
||||
SCENARIO ?= demo
|
||||
test-pipeline:
|
||||
$(PYTHON) student-tests/pipeline_ops.py $(SCENARIO) --base-url "$(BASE_URL)"
|
||||
|
||||
# Интеграционный fault-injection pipeline (трек 5); нужен PostgreSQL: TEST_DATABASE_URL=postgres://...
|
||||
test-faults:
|
||||
go test ./internal/worker/ -run Pipeline -count=1 -v
|
||||
|
||||
failed:
|
||||
docker compose run --rm --no-deps -T --entrypoint /app/pipeline relay failed
|
||||
|
||||
# make replay ID=<attachment_id>
|
||||
replay:
|
||||
docker compose run --rm --no-deps -T --entrypoint /app/pipeline relay replay $(ID) --actor operator
|
||||
|
||||
evidence:
|
||||
sh scripts/collect-evidence.sh all
|
||||
|
||||
@@ -260,21 +260,38 @@ def scenario_broker_restart(c):
|
||||
time.sleep(0.5)
|
||||
before = queues()
|
||||
compose('restart', 'rabbitmq')
|
||||
after = queues()
|
||||
compose('start', 'worker')
|
||||
# Quorum-очередь после рестарта выбирает лидера и восстанавливает журнал: ждём.
|
||||
after = {}
|
||||
for _ in range(30):
|
||||
after = queues()
|
||||
if after.get('attachment.process', 0) >= 1:
|
||||
break
|
||||
time.sleep(1)
|
||||
sh('docker', 'start', compose('ps', '-aq', 'worker').stdout.split()[0]) # без запуска зависимостей
|
||||
wait_status(c, bob, att['id'], ('ready',))
|
||||
published = psql("SELECT publish_attempts FROM outbox WHERE job_id='%s'" % jid)
|
||||
check(after.get('attachment.process', 0) >= 1, 'message lost on broker restart: %s' % after)
|
||||
check(published == '1', 'message was republished from outbox (%s): not a broker durability proof' % published)
|
||||
return {'attachment_id': att['id'], 'queue_before_restart': before, 'queue_after_restart': after,
|
||||
'outbox_publish_attempts': int(published),
|
||||
'invariants': invariants(att['id'], chat['id'])}
|
||||
|
||||
|
||||
def scenario_transient(c):
|
||||
alice, bob, _, chat = c.bootstrap()
|
||||
worker_cid = compose('ps', '-q', 'worker').stdout.split()[0]
|
||||
s3_cid = compose('ps', '-q', 's3').stdout.split()[0]
|
||||
compose('stop', 'worker')
|
||||
att = upload(c, alice, chat, sample(), name='retry.png')
|
||||
jid, _, _ = job_of(att['id'])
|
||||
compose('stop', 's3')
|
||||
compose('start', 'worker')
|
||||
for _ in range(30):
|
||||
if psql("SELECT count(*) FROM outbox WHERE job_id='%s' AND published_at IS NOT NULL" % jid) == '1':
|
||||
break
|
||||
time.sleep(0.5)
|
||||
# Прямые docker stop/start: `docker compose start worker` поднял бы и S3
|
||||
# (worker → s3-init → s3), и временного отказа бы не было.
|
||||
sh('docker', 'stop', s3_cid)
|
||||
sh('docker', 'start', worker_cid)
|
||||
seen = []
|
||||
t0 = time.time()
|
||||
while time.time() - t0 < 25:
|
||||
@@ -283,7 +300,7 @@ def scenario_transient(c):
|
||||
if seen.count('retry_scheduled') >= 2:
|
||||
break
|
||||
time.sleep(1)
|
||||
compose('start', 's3')
|
||||
sh('docker', 'start', s3_cid)
|
||||
meta = wait_status(c, bob, att['id'], ('ready',), timeout=150)
|
||||
_, state, attempt = job_of(att['id'])
|
||||
details = psql("SELECT string_agg(detail, ' | ' ORDER BY id) FROM job_events WHERE job_id='%s' AND event='retry_scheduled'" % jid)
|
||||
|
||||
Reference in New Issue
Block a user