forked from mtusi2026/mtusi2026lab6
chore: port lab5 solution as lab6 base
This commit is contained in:
Executable
+21
@@ -0,0 +1,21 @@
|
||||
#!/usr/bin/env python3
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
bucket, key = sys.argv[1:3]
|
||||
base = 'http://s3:9000/' + bucket
|
||||
attempts = (
|
||||
('GET', '/<object>', base + '/' + key, None),
|
||||
('GET', '/', base + '/', None),
|
||||
('PUT', '/anon.txt', base + '/anon.txt', b'x'),
|
||||
)
|
||||
failed = False
|
||||
for method, label, url, data in attempts:
|
||||
try:
|
||||
code = urllib.request.urlopen(urllib.request.Request(url, data=data, method=method), timeout=5).status
|
||||
except urllib.error.HTTPError as error:
|
||||
code = error.code
|
||||
print(method, bucket + label, '->', code)
|
||||
failed = failed or code != 403
|
||||
sys.exit(1 if failed else 0)
|
||||
Executable
+46
@@ -0,0 +1,46 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
bucket="${S3_BUCKET:-messenger-attachments}"
|
||||
object="check/$(date +%s)-$$.txt"
|
||||
|
||||
as_api() {
|
||||
docker compose run --rm --no-deps -T -e BUCKET="$bucket" -e OBJECT="$object" --entrypoint sh s3-init -c "$1"
|
||||
}
|
||||
|
||||
echo "## S3 не опубликован наружу, консоль выключена"
|
||||
docker compose ps s3 --format '{{.Service}} ports={{.Ports}}'
|
||||
echo "MINIO_BROWSER=$(docker compose exec -T s3 printenv MINIO_BROWSER)"
|
||||
|
||||
echo
|
||||
echo "## Права пользователя API: только свой bucket"
|
||||
as_api '
|
||||
set -e
|
||||
cd /run/secrets
|
||||
export MC_HOST_api="http://$(cat s3_access_key_id):$(cat s3_secret_access_key)@s3:9000"
|
||||
export MC_HOST_root="http://$(cat s3_root_user):$(cat s3_root_password)@s3:9000"
|
||||
expect() { want=$1; label=$2; shift 2; if "$@" >/dev/null 2>&1; then got=allowed; else got=denied; fi; echo "$label -> $got"; [ "$got" = "$want" ]; }
|
||||
echo probe > /tmp/probe.txt
|
||||
mc mb --ignore-existing root/check-other-bucket >/dev/null
|
||||
mc cp /tmp/probe.txt root/check-other-bucket/probe.txt >/dev/null
|
||||
trap "mc rb --force root/check-other-bucket >/dev/null" EXIT
|
||||
expect allowed "PUT в свой bucket" mc cp /tmp/probe.txt "api/$BUCKET/$OBJECT"
|
||||
expect allowed "GET из своего bucket" mc cat "api/$BUCKET/$OBJECT"
|
||||
expect denied "создание другого bucket" mc mb api/forbidden-bucket
|
||||
expect denied "GET из другого bucket" mc cat api/check-other-bucket/probe.txt
|
||||
expect denied "PUT в другой bucket" mc cp /tmp/probe.txt api/check-other-bucket/new.txt
|
||||
expect denied "изменение политики bucket" mc anonymous set public "api/$BUCKET"
|
||||
expect denied "администрирование пользователей" mc admin user list api
|
||||
'
|
||||
|
||||
echo
|
||||
echo "## Анонимный доступ напрямую к S3, мимо API"
|
||||
python3 scripts/check-s3-anonymous.py "$bucket" "$object"
|
||||
|
||||
as_api '
|
||||
cd /run/secrets
|
||||
export MC_HOST_api="http://$(cat s3_access_key_id):$(cat s3_secret_access_key)@s3:9000"
|
||||
mc rm "api/$BUCKET/$OBJECT" >/dev/null
|
||||
'
|
||||
echo
|
||||
echo "PASS: bucket приватный, у API нет прав вне своего bucket"
|
||||
Executable
+123
@@ -0,0 +1,123 @@
|
||||
#!/bin/sh
|
||||
set -u
|
||||
cd "$(dirname "$0")/.."
|
||||
mkdir -p evidence
|
||||
|
||||
COMMIT="$(git rev-parse --short HEAD)$(git diff --quiet HEAD -- src prisma test scripts infra compose.yaml Dockerfile || echo ' (dirty)')"
|
||||
|
||||
header() {
|
||||
printf '# %s\n# date: %s\n# commit: %s\n\n' "$1" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$COMMIT"
|
||||
}
|
||||
|
||||
run() {
|
||||
echo "\$ $*"
|
||||
"$@" 2>&1 || echo "EXIT CODE $?"
|
||||
echo
|
||||
}
|
||||
|
||||
started() {
|
||||
docker inspect -f '{{.Name}} {{.State.Status}} started={{.State.StartedAt}}' $(docker compose ps -aq postgres redis s3 rabbitmq api lb)
|
||||
}
|
||||
|
||||
api_states() {
|
||||
docker inspect -f '{{.Name}} id={{slice .Id 0 12}} {{.State.Status}} exit={{.State.ExitCode}} finished={{.State.FinishedAt}}' $(docker compose ps -aq api)
|
||||
}
|
||||
|
||||
restore_api() {
|
||||
for id in $(docker compose ps -aq --status exited api); do
|
||||
docker start "$id" >/dev/null
|
||||
done
|
||||
python3 scripts/probe.py ready >/dev/null
|
||||
sleep 3
|
||||
}
|
||||
|
||||
echo 'container.txt'
|
||||
{
|
||||
header 'Стенд: сервисы, порты, лимиты, секреты, миграции, топология брокера'
|
||||
run docker compose ps -a --format 'table {{.Service}}\t{{.Name}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}'
|
||||
echo '## CPU/RAM-лимиты (NanoCpus: 1e9 = 1 CPU; Memory в байтах)'
|
||||
run sh -c "docker inspect -f '{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}}' \$(docker compose ps -aq)"
|
||||
echo '## Секреты: каждый сервис видит только свои файлы (содержимое не выводится)'
|
||||
for service in postgres redis s3 api reconciler relay worker; do
|
||||
run docker compose exec -T "$service" ls /run/secrets
|
||||
done
|
||||
run docker compose exec -T rabbitmq ls /etc/rabbitmq/conf.d
|
||||
run docker compose exec -T lb sh -c 'ls /run/secrets 2>/dev/null || echo "no /run/secrets"'
|
||||
echo '## Версии'
|
||||
run docker compose exec -T lb haproxy -v
|
||||
run docker compose exec -T postgres postgres --version
|
||||
run docker compose exec -T redis redis-server --version
|
||||
run docker compose exec -T s3 minio --version
|
||||
run docker compose exec -T rabbitmq rabbitmqctl -q version
|
||||
echo '## Топология брокера: durable quorum-очереди, delivery-limit и DLQ'
|
||||
run docker compose exec -T rabbitmq rabbitmqctl -q list_queues name type durable arguments messages
|
||||
run docker compose exec -T rabbitmq rabbitmqctl -q list_consumers queue_name prefetch_count ack_required
|
||||
echo '## Таблицы'
|
||||
run docker compose exec -T postgres psql -U messenger -d messenger -c '\d attachments' -c '\d attachment_variants' -c '\d attachment_jobs' -c '\d outbox' -c '\d job_events'
|
||||
echo '## Миграции: отдельный one-shot сервис, повторный запуск ничего не применяет, схема совпадает с БД'
|
||||
run docker compose run --rm migrate
|
||||
run docker compose run --rm --no-deps migrate sh -c 'export DATABASE_URL="$(cat "$DATABASE_URL_FILE")"; npx prisma migrate diff --from-url "$DATABASE_URL" --to-schema-datamodel prisma/schema.prisma --exit-code'
|
||||
run docker compose exec -T postgres psql -U messenger -d messenger -c 'SELECT migration_name, finished_at FROM _prisma_migrations ORDER BY started_at'
|
||||
} > evidence/container.txt
|
||||
|
||||
echo 'tests.txt'
|
||||
{
|
||||
header 'Собственные e2e-тесты (PostgreSQL + Redis + MinIO из compose.test.yaml)'
|
||||
docker compose -f compose.test.yaml build -q tests
|
||||
run docker compose -f compose.test.yaml run --rm tests npx jest --config ./test/jest-e2e.json --verbose
|
||||
docker compose -f compose.test.yaml down -v >/dev/null 2>&1
|
||||
} > evidence/tests.txt
|
||||
|
||||
echo 'smoke.txt'
|
||||
{
|
||||
header 'Публичные проверки через LB; разные реплики обслуживают одну сессию'
|
||||
run make check
|
||||
run make test
|
||||
run python3 scripts/probe.py instances 6
|
||||
} > evidence/smoke.txt
|
||||
|
||||
echo 'persistence.txt'
|
||||
{
|
||||
header 'make test-persistence: restart.sh перезапускает postgres, redis, s3, rabbitmq, api, lb с сохранением volumes'
|
||||
echo '## до'
|
||||
started
|
||||
echo
|
||||
run make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
echo '## после'
|
||||
started
|
||||
} > evidence/persistence.txt
|
||||
|
||||
echo 'failover.txt'
|
||||
{
|
||||
header 'make test-failover: stop-one.sh останавливает реплику, обслужившую тест'
|
||||
echo '## до'
|
||||
api_states
|
||||
echo
|
||||
run make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
echo '## после: остановленная реплика Exited (0) — штатное завершение по SIGTERM'
|
||||
api_states
|
||||
restore_api
|
||||
} > evidence/failover.txt
|
||||
|
||||
echo 's3.txt'
|
||||
{
|
||||
header 'S3 (минимум lab4): идемпотентный bootstrap, приватность мимо API, права пользователя API'
|
||||
run docker compose run --rm --no-deps -T s3-init
|
||||
run docker compose run --rm --no-deps -T s3-init
|
||||
run sh scripts/check-s3.sh
|
||||
} > evidence/s3.txt
|
||||
|
||||
echo 'jobs.txt'
|
||||
{
|
||||
header 'Фоновая обработка: сценарии защиты уровней 3 и 4 (scripts/job-scenario.py)'
|
||||
for scenario in restart crash-after-store crash-after-commit outbox-gap relay-crash retry-replay broker; do
|
||||
echo "## $scenario"
|
||||
run python3 scripts/job-scenario.py "$scenario"
|
||||
done
|
||||
echo '## Итог по всем заданиям стенда'
|
||||
run docker compose exec -T postgres psql -U messenger -d messenger -c 'SELECT status, count(*) FROM attachment_jobs GROUP BY status' -c 'SELECT count(*) AS unpublished_outbox FROM outbox WHERE published_at IS NULL' -c 'SELECT count(*) AS staging_left_on_ready FROM attachments WHERE status = '"'ready'"' AND staging_key IS NOT NULL'
|
||||
echo '## Последний проход reconciler'
|
||||
docker compose logs --no-log-prefix --tail 1 reconciler 2>&1
|
||||
} > evidence/jobs.txt
|
||||
|
||||
echo 'done'
|
||||
Executable
+394
@@ -0,0 +1,394 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Defense scenarios for background attachment processing; run inside the tools container."""
|
||||
import atexit
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
BASE = os.environ.get('BASE_URL', 'http://localhost:8080')
|
||||
SAMPLE = open(os.path.join(os.path.dirname(__file__), '..', 'tests', 'fixtures', 'sample.png'), 'rb').read()
|
||||
STARTED = time.monotonic()
|
||||
ANSI = re.compile(chr(27) + re.escape('[') + '[0-9;]*m')
|
||||
|
||||
|
||||
def log(*parts):
|
||||
print('[%6.1fs]' % (time.monotonic() - STARTED), *parts, flush=True)
|
||||
|
||||
|
||||
def call(method, path, body=None, headers=None, raw=None):
|
||||
data = raw if raw is not None else (json.dumps(body).encode() if body is not None else None)
|
||||
all_headers = dict(headers or {})
|
||||
if body is not None:
|
||||
all_headers['Content-Type'] = 'application/json'
|
||||
request = urllib.request.Request(BASE + path, data=data, method=method, headers=all_headers)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=15) as response:
|
||||
return response.status, response.read(), response.headers
|
||||
except urllib.error.HTTPError as error:
|
||||
return error.code, error.read(), error.headers
|
||||
except OSError as error:
|
||||
return 0, str(error).encode(), {}
|
||||
|
||||
|
||||
def json_of(raw):
|
||||
try:
|
||||
return json.loads(raw or b'null')
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def compose(*args):
|
||||
log('$ docker compose', ' '.join(args))
|
||||
subprocess.run(['docker', 'compose', *args], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
|
||||
|
||||
def start_only(service):
|
||||
"""docker compose start also starts dependencies (worker -> s3-init -> s3); start just this container."""
|
||||
log('$ docker start', service, '(without dependencies)')
|
||||
ids = subprocess.run(['docker', 'compose', 'ps', '-aq', service], check=True, capture_output=True, text=True).stdout.split()
|
||||
subprocess.run(['docker', 'start', *ids], check=True, stdout=subprocess.DEVNULL)
|
||||
|
||||
|
||||
def psql(sql):
|
||||
out = subprocess.run(
|
||||
['docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-tA', '-F', ' | ', '-c', sql],
|
||||
check=True, capture_output=True, text=True,
|
||||
)
|
||||
return out.stdout.strip()
|
||||
|
||||
|
||||
def queue_depth():
|
||||
time.sleep(6)
|
||||
out = subprocess.run(
|
||||
['docker', 'compose', 'exec', '-T', 'rabbitmq', 'rabbitmqctl', '-q', 'list_queues', 'name', 'type', 'durable', 'messages_ready', 'messages_unacknowledged'],
|
||||
capture_output=True, text=True,
|
||||
)
|
||||
return out.stdout.strip() or out.stderr.strip()
|
||||
|
||||
|
||||
def wait_ready(timeout=90):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if call('GET', '/health/ready')[0] == 200:
|
||||
return
|
||||
time.sleep(1)
|
||||
sys.exit('API did not become ready')
|
||||
|
||||
|
||||
def session():
|
||||
username = 'job_' + uuid.uuid4().hex[:8]
|
||||
password = 'Job-scenario-' + uuid.uuid4().hex
|
||||
code, raw, _ = call('POST', '/api/v1/users', {'username': username, 'display_name': 'Job scenario', 'password': password})
|
||||
if code != 201:
|
||||
sys.exit('register failed: %s' % code)
|
||||
user = json_of(raw)
|
||||
credential = base64.b64encode((username + ':' + password).encode()).decode()
|
||||
code, raw, _ = call('POST', '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credential})
|
||||
if code != 201:
|
||||
sys.exit('login failed: %s' % code)
|
||||
auth = {'Authorization': 'Bearer ' + json_of(raw)['token']}
|
||||
code, raw, _ = call('POST', '/api/v1/chats', {'title': 'Job scenario', 'member_ids': [user['id']]}, auth)
|
||||
if code != 201:
|
||||
sys.exit('chat failed: %s' % code)
|
||||
chat = json_of(raw)['id']
|
||||
log('user', user['id'], 'chat', chat)
|
||||
return auth, chat
|
||||
|
||||
|
||||
def upload(auth, chat, body=SAMPLE):
|
||||
code, raw, headers = call(
|
||||
'POST', '/api/v1/chats/%s/attachments?filename=sample.png' % chat,
|
||||
headers={**auth, 'Content-Type': 'image/png'}, raw=body,
|
||||
)
|
||||
attachment = json_of(raw)
|
||||
log('POST attachment ->', code, 'status=%s' % attachment.get('status'), 'id=%s' % attachment.get('id'),
|
||||
'instance=%s' % headers.get('X-Instance-Id'))
|
||||
if code != 202:
|
||||
sys.exit('upload failed')
|
||||
return attachment['id']
|
||||
|
||||
|
||||
def message(auth, chat, text, attachments=()):
|
||||
code, raw, _ = call(
|
||||
'POST', '/api/v1/chats/%s/messages' % chat, {'text': text, 'attachment_ids': list(attachments)},
|
||||
{**auth, 'Idempotency-Key': uuid.uuid4().hex},
|
||||
)
|
||||
log('POST message', 'with %d attachment(s)' % len(attachments), '->', code, 'id=%s' % (json_of(raw) or {}).get('id'))
|
||||
return code
|
||||
|
||||
|
||||
def metadata(auth, attachment):
|
||||
code, raw, _ = call('GET', '/api/v1/attachments/' + attachment, headers=auth)
|
||||
return code, json_of(raw)
|
||||
|
||||
|
||||
def content_status(auth, attachment, variant='original'):
|
||||
return call('GET', '/api/v1/attachments/%s/content?variant=%s' % (attachment, variant), headers=auth)[0]
|
||||
|
||||
|
||||
def wait_status(auth, attachment, wanted, timeout=150):
|
||||
deadline = time.monotonic() + timeout
|
||||
last = None
|
||||
while time.monotonic() < deadline:
|
||||
code, meta = metadata(auth, attachment)
|
||||
if code == 200 and meta['status'] != last:
|
||||
last = meta['status']
|
||||
log('GET attachment -> status=%s error_code=%s' % (last, meta['error_code']))
|
||||
if code == 200 and meta['status'] in wanted:
|
||||
return meta
|
||||
time.sleep(1)
|
||||
sys.exit('attachment did not reach %s' % '/'.join(wanted))
|
||||
|
||||
|
||||
def check_ready(auth, attachment, meta):
|
||||
code, raw, _ = call('GET', '/api/v1/attachments/%s/content' % attachment, headers=auth)
|
||||
log('original ->', code, 'sha256 matches upload:', code == 200 and hashlib.sha256(raw).hexdigest() == hashlib.sha256(SAMPLE).hexdigest())
|
||||
for variant in meta['variants']:
|
||||
code, raw, headers = call('GET', '/api/v1/attachments/%s/content?variant=%s' % (attachment, variant['name']), headers=auth)
|
||||
log('%-9s -> %s %s %dx%d sha256 matches metadata: %s' % (
|
||||
variant['name'], code, headers.get('Content-Type'), variant['width'], variant['height'],
|
||||
hashlib.sha256(raw).hexdigest() == variant['sha256']))
|
||||
|
||||
|
||||
def job_state(attachment):
|
||||
log('job:', psql("SELECT id, status, attempt, max_attempts, error_code FROM attachment_jobs WHERE attachment_id = '%s'" % attachment))
|
||||
log('variants rows:', psql("SELECT count(*) FROM attachment_variants WHERE attachment_id = '%s'" % attachment))
|
||||
log('events:', psql("SELECT string_agg(e.event, ' -> ' ORDER BY e.id) FROM job_events e JOIN attachment_jobs j ON j.id = e.job_id WHERE j.attachment_id = '%s'" % attachment))
|
||||
|
||||
|
||||
def scenario_restart():
|
||||
"""Level 3: worker down -> upload 202 -> restart API and broker -> worker up -> ready."""
|
||||
wait_ready()
|
||||
auth, chat = session()
|
||||
compose('stop', 'worker')
|
||||
attachment = upload(auth, chat)
|
||||
log('content while queued ->', content_status(auth, attachment))
|
||||
message(auth, chat, '', [attachment])
|
||||
message(auth, chat, 'other messages keep working')
|
||||
log('queues:', queue_depth())
|
||||
compose('restart', 'api', 'rabbitmq')
|
||||
wait_ready()
|
||||
log('queues after broker restart:', queue_depth())
|
||||
log('status after restarts:', metadata(auth, attachment)[1]['status'])
|
||||
compose('start', 'worker')
|
||||
meta = wait_status(auth, attachment, ['ready', 'failed'])
|
||||
check_ready(auth, attachment, meta)
|
||||
job_state(attachment)
|
||||
|
||||
|
||||
def remove_container(name):
|
||||
subprocess.run(['docker', 'rm', '-f', name], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
|
||||
|
||||
def plain(line):
|
||||
return ANSI.sub('', line).split('] ')[-1].strip()
|
||||
|
||||
|
||||
def faulty(service, env):
|
||||
"""Run a one-off copy of the service with fault injection; no restart policy, so it dies once."""
|
||||
name = 'lab6-fault-%s-%s' % (service, uuid.uuid4().hex[:8])
|
||||
atexit.register(remove_container, name)
|
||||
args = ['docker', 'compose', 'run', '-d', '--no-deps', '--name', name]
|
||||
for key, value in env.items():
|
||||
args += ['-e', '%s=%s' % (key, value)]
|
||||
log('$ docker compose run --no-deps', ' '.join('-e %s=%s' % item for item in env.items()), service)
|
||||
subprocess.run(args + [service], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
deadline = time.monotonic() + 30
|
||||
while time.monotonic() < deadline:
|
||||
if 'connected to broker' in subprocess.run(['docker', 'logs', name], capture_output=True, text=True).stdout:
|
||||
return name
|
||||
time.sleep(0.5)
|
||||
sys.exit('%s with fault did not start' % service)
|
||||
|
||||
|
||||
def wait_exit(name, timeout=90):
|
||||
try:
|
||||
code = subprocess.run(['docker', 'wait', name], capture_output=True, text=True, timeout=timeout).stdout.strip()
|
||||
except subprocess.TimeoutExpired:
|
||||
code = 'timeout'
|
||||
for line in subprocess.run(['docker', 'logs', name], capture_output=True, text=True).stdout.splitlines():
|
||||
if 'fault injection' in line or 'published' in line or 'job ' in line:
|
||||
log(' one-off:', plain(line))
|
||||
remove_container(name)
|
||||
log('one-off process exit code', code, '(137 = killed without ack)')
|
||||
|
||||
|
||||
def service_logs(service, since, needle):
|
||||
out = subprocess.run(['docker', 'compose', 'logs', '--no-log-prefix', '--no-color', '--since', since, service],
|
||||
capture_output=True, text=True).stdout
|
||||
for line in out.splitlines():
|
||||
if needle in line:
|
||||
log(' %s:' % service, plain(line))
|
||||
|
||||
|
||||
def utc_now():
|
||||
return time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
|
||||
|
||||
|
||||
def staging_key(attachment):
|
||||
return psql("SELECT coalesce(staging_key, 'NULL') FROM attachments WHERE id = '%s'" % attachment)
|
||||
|
||||
|
||||
def job_id(attachment):
|
||||
return psql("SELECT id FROM attachment_jobs WHERE attachment_id = '%s'" % attachment)
|
||||
|
||||
|
||||
def scenario_crash_after_store():
|
||||
"""Level 4: worker killed after writing S3 objects, before COMMIT and ack; lease expires; same result."""
|
||||
wait_ready()
|
||||
since = utc_now()
|
||||
auth, chat = session()
|
||||
compose('stop', 'worker')
|
||||
proc = faulty('worker', {'WORKER_FAULT': 'exit-after-store', 'JOB_LEASE_SECONDS': '15'})
|
||||
attachment = upload(auth, chat)
|
||||
wait_exit(proc)
|
||||
log('status after crash:', metadata(auth, attachment)[1]['status'])
|
||||
log('lease:', psql("SELECT status, attempt, lease_until > now() AS lease_active FROM attachment_jobs WHERE attachment_id = '%s'" % attachment))
|
||||
log('queues (unacked message returned):', queue_depth())
|
||||
compose('start', 'worker')
|
||||
meta = wait_status(auth, attachment, ['ready', 'failed'])
|
||||
check_ready(auth, attachment, meta)
|
||||
job_state(attachment)
|
||||
service_logs('worker', since, job_id(attachment))
|
||||
service_logs('relay', since, 'lease expired')
|
||||
|
||||
|
||||
def scenario_crash_after_commit():
|
||||
"""Level 4: worker killed after COMMIT ready, before ack; redelivery is a no-op and cleans staging."""
|
||||
wait_ready()
|
||||
since = utc_now()
|
||||
auth, chat = session()
|
||||
compose('stop', 'worker')
|
||||
proc = faulty('worker', {'WORKER_FAULT': 'exit-after-commit'})
|
||||
attachment = upload(auth, chat)
|
||||
wait_exit(proc)
|
||||
log('status after crash:', metadata(auth, attachment)[1]['status'], 'staging_key:', staging_key(attachment))
|
||||
log('queues (unacked message returned):', queue_depth())
|
||||
compose('start', 'worker')
|
||||
deadline = time.monotonic() + 30
|
||||
while staging_key(attachment) != 'NULL' and time.monotonic() < deadline:
|
||||
time.sleep(1)
|
||||
log('staging_key after redelivery:', staging_key(attachment))
|
||||
check_ready(auth, attachment, metadata(auth, attachment)[1])
|
||||
job_state(attachment)
|
||||
service_logs('worker', since, job_id(attachment))
|
||||
|
||||
|
||||
def scenario_outbox_gap():
|
||||
"""Level 4: crash between COMMIT and publish; the outbox row is published after recovery."""
|
||||
wait_ready()
|
||||
auth, chat = session()
|
||||
compose('stop', 'relay')
|
||||
attachment = upload(auth, chat)
|
||||
outbox = "SELECT o.id, coalesce(o.published_at::text, 'NOT PUBLISHED'), o.publish_attempts FROM outbox o JOIN attachment_jobs j ON j.id = o.job_id WHERE j.attachment_id = '%s'" % attachment
|
||||
log('outbox after COMMIT:', psql(outbox))
|
||||
log('queues:', queue_depth())
|
||||
compose('kill', 'api')
|
||||
log('API killed (SIGKILL) after COMMIT, nothing was published')
|
||||
compose('start', 'api')
|
||||
wait_ready()
|
||||
log('status after API restart:', metadata(auth, attachment)[1]['status'])
|
||||
compose('start', 'relay')
|
||||
meta = wait_status(auth, attachment, ['ready', 'failed'])
|
||||
log('outbox after relay recovery:', psql(outbox))
|
||||
check_ready(auth, attachment, meta)
|
||||
job_state(attachment)
|
||||
|
||||
|
||||
def scenario_relay_crash():
|
||||
"""Level 4: relay killed after broker confirm, before marking outbox; duplicate delivery is a no-op."""
|
||||
wait_ready()
|
||||
since = utc_now()
|
||||
auth, chat = session()
|
||||
compose('stop', 'relay')
|
||||
proc = faulty('relay', {'RELAY_FAULT': 'exit-after-confirm'})
|
||||
attachment = upload(auth, chat)
|
||||
wait_exit(proc)
|
||||
outbox = "SELECT o.id, coalesce(o.published_at::text, 'NOT MARKED'), o.publish_attempts FROM outbox o JOIN attachment_jobs j ON j.id = o.job_id WHERE j.attachment_id = '%s'" % attachment
|
||||
log('outbox after relay crash:', psql(outbox))
|
||||
compose('start', 'relay')
|
||||
meta = wait_status(auth, attachment, ['ready', 'failed'])
|
||||
time.sleep(3)
|
||||
log('outbox after relay restart:', psql(outbox))
|
||||
check_ready(auth, attachment, meta)
|
||||
job_state(attachment)
|
||||
service_logs('worker', since, job_id(attachment))
|
||||
|
||||
|
||||
def replay(*args):
|
||||
out = subprocess.run(['docker', 'compose', 'exec', '-T', '-e', 'OPERATOR=defense-demo', 'worker', 'node', 'dist/replay.js', *args],
|
||||
capture_output=True, text=True)
|
||||
for line in (out.stdout.strip() or out.stderr.strip()).splitlines():
|
||||
log(' replay:', line)
|
||||
|
||||
|
||||
def scenario_retry_replay():
|
||||
"""Level 4: S3 down -> retries with backoff and jitter -> failed retry_exhausted -> explicit replay -> ready."""
|
||||
wait_ready()
|
||||
auth, chat = session()
|
||||
compose('stop', 'worker')
|
||||
attachment = upload(auth, chat)
|
||||
compose('stop', 's3')
|
||||
start_only('worker')
|
||||
meta = wait_status(auth, attachment, ['failed'], timeout=120)
|
||||
log('failed attachment in a new message ->', message(auth, chat, '', [attachment]))
|
||||
log('retries:')
|
||||
for line in psql("SELECT e.attempt, e.event, coalesce(e.detail, ''), to_char(e.created_at, 'HH24:MI:SS.MS') FROM job_events e JOIN attachment_jobs j ON j.id = e.job_id WHERE j.attachment_id = '%s' ORDER BY e.id" % attachment).splitlines():
|
||||
log(' ', line)
|
||||
log('$ replay.js --list')
|
||||
replay('--list')
|
||||
compose('start', 's3')
|
||||
wait_ready()
|
||||
log('$ replay.js', attachment)
|
||||
replay(attachment)
|
||||
meta = wait_status(auth, attachment, ['ready', 'failed'])
|
||||
check_ready(auth, attachment, meta)
|
||||
log('$ replay.js', attachment, '(already ready)')
|
||||
replay(attachment)
|
||||
job_state(attachment)
|
||||
|
||||
|
||||
def scenario_broker():
|
||||
"""Level 4: prefetch/concurrency, limits and dead letter queue for a malformed message."""
|
||||
wait_ready()
|
||||
since = utc_now()
|
||||
log('consumers:', subprocess.run(
|
||||
['docker', 'compose', 'exec', '-T', 'rabbitmq', 'rabbitmqctl', '-q', 'list_consumers', 'queue_name', 'prefetch_count', 'ack_required'],
|
||||
capture_output=True, text=True).stdout.strip())
|
||||
log('worker limits:', subprocess.run(
|
||||
['sh', '-c', "docker inspect -f 'cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}}' $(docker compose ps -q worker)"],
|
||||
capture_output=True, text=True).stdout.strip())
|
||||
publish = (
|
||||
"const amqp=require('amqplib');const fs=require('fs');(async()=>{"
|
||||
"const c=await amqp.connect(fs.readFileSync(process.env.BROKER_URL_FILE,'utf8').trim());"
|
||||
"const ch=await c.createConfirmChannel();"
|
||||
"ch.sendToQueue('attachment.jobs',Buffer.from(JSON.stringify({job_id:'not-a-uuid',url:'http://example.invalid/x'})),{persistent:true});"
|
||||
"await ch.waitForConfirms();await c.close();console.log('malformed message published');})()"
|
||||
)
|
||||
log(subprocess.run(['docker', 'compose', 'exec', '-T', 'worker', 'node', '-e', publish], capture_output=True, text=True).stdout.strip())
|
||||
log('queues:', queue_depth())
|
||||
service_logs('worker', since, 'malformed')
|
||||
|
||||
|
||||
SCENARIOS = {
|
||||
'restart': scenario_restart,
|
||||
'crash-after-store': scenario_crash_after_store,
|
||||
'crash-after-commit': scenario_crash_after_commit,
|
||||
'outbox-gap': scenario_outbox_gap,
|
||||
'relay-crash': scenario_relay_crash,
|
||||
'retry-replay': scenario_retry_replay,
|
||||
'broker': scenario_broker,
|
||||
}
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) != 2 or sys.argv[1] not in SCENARIOS:
|
||||
sys.exit('usage: job-scenario.py %s' % '|'.join(SCENARIOS))
|
||||
SCENARIOS[sys.argv[1]]()
|
||||
Executable
+88
@@ -0,0 +1,88 @@
|
||||
#!/usr/bin/env python3
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
BASE = os.environ.get('BASE_URL', 'http://localhost:8080')
|
||||
|
||||
|
||||
def call(method, path, body=None, headers=None, with_headers=False):
|
||||
data = json.dumps(body).encode() if body is not None else None
|
||||
content_type = {'Content-Type': 'application/json'} if data else {}
|
||||
request = urllib.request.Request(BASE + path, data=data, method=method, headers={**content_type, **(headers or {})})
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=15) as response:
|
||||
result = response.status, json.loads(response.read() or b'null'), response.headers
|
||||
except urllib.error.HTTPError as error:
|
||||
result = error.code, json.loads(error.read() or b'null'), error.headers
|
||||
except OSError as error:
|
||||
result = 0, str(error), {}
|
||||
return result if with_headers else result[:2]
|
||||
|
||||
|
||||
def login():
|
||||
username = 'probe_' + uuid.uuid4().hex[:8]
|
||||
password = 'Probe-check-' + uuid.uuid4().hex
|
||||
print('register ->', call('POST', '/api/v1/users', {'username': username, 'display_name': 'Probe', 'password': password})[0])
|
||||
credential = base64.b64encode((username + ':' + password).encode()).decode()
|
||||
code, session = call('POST', '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credential})
|
||||
if code != 201:
|
||||
sys.exit('login failed: %s %s' % (code, session))
|
||||
print('login ->', code, 'expires_at:', session['expires_at'])
|
||||
return {'Authorization': 'Bearer ' + session['token']}
|
||||
|
||||
|
||||
def instances(count):
|
||||
auth = login()
|
||||
for _ in range(count):
|
||||
code, _, headers = call('GET', '/api/v1/users/me', headers=auth, with_headers=True)
|
||||
print('GET /api/v1/users/me ->', code, 'X-Instance-Id:', headers.get('X-Instance-Id'))
|
||||
|
||||
|
||||
def logout():
|
||||
auth = login()
|
||||
print('GET /api/v1/users/me ->', call('GET', '/api/v1/users/me', headers=auth)[0])
|
||||
code, body = call('DELETE', '/api/v1/auth/sessions/current', headers=auth)
|
||||
print('DELETE /api/v1/auth/sessions/current ->', code, json.dumps(body, ensure_ascii=False) if body else '')
|
||||
print('GET /api/v1/users/me ->', call('GET', '/api/v1/users/me', headers=auth)[0])
|
||||
|
||||
|
||||
def ready(timeout=90):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if call('GET', '/health/ready')[0] == 200:
|
||||
print('ready')
|
||||
return
|
||||
time.sleep(0.5)
|
||||
sys.exit('not ready within %ds' % timeout)
|
||||
|
||||
|
||||
def status(method, path, authorization=None):
|
||||
code, body = call(method, path, headers={'Authorization': authorization} if authorization else None)
|
||||
print(method, path, '(%s)' % (authorization or 'anonymous'), '->', code, json.dumps(body, ensure_ascii=False))
|
||||
|
||||
|
||||
def expiry(ttl):
|
||||
auth = login()
|
||||
print('GET /api/v1/users/me (fresh) ->', call('GET', '/api/v1/users/me', headers=auth)[0])
|
||||
time.sleep(ttl + 1)
|
||||
print('GET /api/v1/users/me (after %ds) ->' % (ttl + 1), call('GET', '/api/v1/users/me', headers=auth)[0])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
command = sys.argv[1]
|
||||
if command == 'ready':
|
||||
ready()
|
||||
elif command == 'status':
|
||||
status(*sys.argv[2:5])
|
||||
elif command == 'expiry':
|
||||
expiry(int(sys.argv[2]))
|
||||
elif command == 'instances':
|
||||
instances(int(sys.argv[2]))
|
||||
elif command == 'logout':
|
||||
logout()
|
||||
Executable
+9
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
started() {
|
||||
docker inspect -f '{{.Name}} {{.State.Status}} started={{.State.StartedAt}}' $(docker compose ps -aq postgres redis s3 rabbitmq api lb)
|
||||
}
|
||||
started
|
||||
docker compose restart postgres redis s3 rabbitmq api lb
|
||||
started
|
||||
Executable
+11
@@ -0,0 +1,11 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
project="${COMPOSE_PROJECT_NAME:-messenger-lab6}"
|
||||
labels=$(docker inspect -f '{{index .Config.Labels "com.docker.compose.project"}}/{{index .Config.Labels "com.docker.compose.service"}}' "$TARGET_INSTANCE_ID")
|
||||
if [ "$labels" != "$project/api" ]; then
|
||||
echo "refusing to stop $TARGET_INSTANCE_ID: not an api replica of $project ($labels)" >&2
|
||||
exit 1
|
||||
fi
|
||||
docker stop -t 8 "$TARGET_INSTANCE_ID" >/dev/null
|
||||
docker inspect -f '{{.Name}} {{.State.Status}} finished={{.State.FinishedAt}}' "$TARGET_INSTANCE_ID"
|
||||
Reference in New Issue
Block a user