feat: add upload experiment, attachment scenarios and health extension

This commit is contained in:
2026-11-05 16:26:15 +03:00
parent a7980a4cb8
commit 5c3cba4951
8 changed files with 552 additions and 8 deletions
+2
View File
@@ -18,3 +18,5 @@ S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
RECONCILE_GRACE_SECONDS=3600
RECONCILE_INTERVAL_SECONDS=300
+36
View File
@@ -0,0 +1,36 @@
{
"openapi": "3.1.0",
"info": {
"title": "MTUSI Messenger — расширения лабораторной 4",
"version": "1.4.0",
"description": "Обратно совместимое дополнение к contracts/openapi.json (см. COURSE.md, раздел «Контракт и совместимость»). Обязательный контракт не меняется. Обоснование — REPORT.md, раздел «Критичные зависимости и проверка готовности»."
},
"paths": {
"/health/core": {
"get": {
"operationId": "getCoreReadiness",
"summary": "Служебная проверка для балансировщика и Docker healthcheck: может ли реплика обслуживать API без вложений",
"description": "В отличие от /health/ready не учитывает S3. Проверяет PostgreSQL и то, что процесс не останавливается. Падение общего S3 не выводит реплики из балансировки: сообщения продолжают работать, вложения отвечают 503.",
"responses": {
"200": {
"description": "Реплика готова",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": { "status": { "const": "ok" } },
"required": ["status"],
"additionalProperties": false
}
}
}
},
"503": {
"description": "PostgreSQL недоступен или процесс останавливается; тело в общем формате Error",
"content": { "application/json": { "schema": { "type": "object" } } }
}
}
}
}
}
}
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "mtusi-messenger-lab4",
"version": "1.0.0",
"description": "Lab 1: in-memory messenger API (NestJS + Fastify)",
"description": "Lab 4: messenger API with S3 attachments (NestJS + Fastify)",
"private": true,
"license": "UNLICENSED",
"scripts": {
+190
View File
@@ -0,0 +1,190 @@
#!/usr/bin/env python3
import hashlib
import http.client
import json
import os
from pathlib import Path
import socket
import subprocess
import sys
import time
import urllib.error
import urllib.request
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client
HOST = os.environ.get('LB_HOST', 'lb')
PORT = int(os.environ.get('LB_PORT', '8080'))
SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes()
SAMPLE_SHA = hashlib.sha256(SAMPLE).hexdigest()
BUCKET = os.environ.get('S3_BUCKET', 'messenger-attachments')
FAULT_NAME = 'lab4_fault'
def sh(*args, check=True):
return subprocess.run(args, check=check, capture_output=True, text=True).stdout.strip()
def sql(statement):
return sh('docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-tAc', statement)
def request(method, path, token=None, body=None, content_type=None):
headers = {}
if token:
headers['Authorization'] = 'Bearer ' + token
if content_type:
headers['Content-Type'] = content_type
conn = http.client.HTTPConnection(HOST, PORT, timeout=60)
conn.request(method, path, body=body, headers=headers)
response = conn.getresponse()
data = response.read()
conn.close()
return response, data
def upload(token, chat):
return request('POST', '/api/v1/chats/' + chat + '/attachments?filename=sample.png', token, SAMPLE, 'image/png')
def download(token, attachment):
return request('GET', '/api/v1/attachments/' + attachment + '/content', token)
def reconcile(*flags):
output = sh('docker', 'compose', 'run', '--rm', '--no-deps', '-T', 'reconciler', 'node', 'dist/reconcile.js', *flags)
line = output.splitlines()[-1]
print('$ reconcile', ' '.join(flags))
print(line)
return json.loads(line)
def replicas():
return [cid[:12] for cid in sh('docker', 'compose', 'ps', '-q', 'api').split()]
def wait_ready():
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
try:
response, _ = request('GET', '/health/ready')
if response.status == 200:
return
except OSError:
pass
time.sleep(0.3)
raise SystemExit('not ready')
def header(text):
print('\n## ' + text, flush=True)
def defence(client):
header('1. Сценарий защиты: загрузка, сообщение, второй участник, посторонний, S3 мимо API')
alice, bob, eve, chat = client.bootstrap()
attachment, _ = client.attachment(alice, chat)
print('POST /attachments -> 201 status=%s variants=%s error_code=%s sha256=%s' % (attachment['status'], attachment['variants'], attachment['error_code'], attachment['sha256']))
message = client.message(alice, chat, '', attachments=[attachment['id']])['json']
print('POST /messages text="" -> 201 attachment_ids совпадают:', message['attachment_ids'] == [attachment['id']])
response, data = download(bob['token'], attachment['id'])
print('Bob GET /content -> %d %s sha256 совпадает: %s' % (response.status, response.getheader('Content-Type'), hashlib.sha256(data).hexdigest() == SAMPLE_SHA))
for suffix in ('', '/content'):
response, _ = request('GET', '/api/v1/attachments/' + attachment['id'] + suffix, eve['token'])
print('Eve GET /attachments/{id}%s -> %d' % (suffix, response.status))
key = sql("SELECT object_key FROM attachments WHERE id = '%s'" % attachment['id'])
print('object_key в БД: attachments/<uuid>, filename в ключ не входит:', 'sample' not in key)
try:
code = urllib.request.urlopen('http://s3:9000/%s/%s' % (BUCKET, key), timeout=5).status
except urllib.error.HTTPError as error:
code = error.code
print('Анонимный GET объекта напрямую в S3 ->', code)
return alice, chat['id']
def failover(alice, chat):
header('2. Остановка реплики, обслужившей загрузку; скачивание через другую')
token = alice['token']
tested = set()
while len(tested) < 2:
response, data = upload(token, chat)
target = response.getheader('X-Instance-Id')
if target in tested:
continue
tested.add(target)
attachment = json.loads(data)
print('upload -> %d, обслужила реплика %s' % (response.status, target))
result = subprocess.run(['sh', 'scripts/stop-one.sh'], env={**os.environ, 'TARGET_INSTANCE_ID': target}, capture_output=True, text=True)
print('stop-one.sh:', result.stdout.strip())
others = [cid for cid in replicas() if cid != target]
started = time.monotonic()
served = []
for _ in range(6):
response, data = download(token, attachment['id'])
served.append(response.getheader('X-Instance-Id'))
ok = response.status == 200 and hashlib.sha256(data).hexdigest() == SAMPLE_SHA
print('download -> %d реплика=%s sha256 совпадает: %s' % (response.status, served[-1], ok))
if not ok:
raise SystemExit('download failed')
print('остановленная реплика среди обслуживших:', target in served, '; %.2fs' % (time.monotonic() - started))
print('Файл оригинала на диске обслужившей реплики (поиск по точному размеру %d байт):' % len(SAMPLE))
for other in others:
found = sh('docker', 'exec', other, 'sh', '-c', 'find /tmp /app /home -xdev -type f -size %dc 2>/dev/null | grep -v node_modules || true' % len(SAMPLE))
print(' %s -> %s' % (other, found or 'нет'))
sh('docker', 'start', target)
wait_ready()
time.sleep(3)
print('реплика %s возвращена' % target)
def interrupted(alice, chat):
header('3. Прерванная загрузка через LB: ни строки, ни объекта, ни временного файла')
before_rows = sql('SELECT count(*) FROM attachments')
before = reconcile('--grace-seconds=3600')['scanned']
connection = socket.create_connection((HOST, PORT))
connection.sendall(('POST /api/v1/chats/%s/attachments?filename=cut.png HTTP/1.1\r\nHost: lb\r\nAuthorization: Bearer %s\r\nContent-Type: image/png\r\nContent-Length: %d\r\n\r\n' % (chat, alice['token'], len(SAMPLE))).encode())
connection.sendall(SAMPLE[:20000])
time.sleep(0.5)
connection.close()
time.sleep(1)
after = reconcile('--grace-seconds=3600')['scanned']
print('строк attachments: %s -> %s; объектов: %d -> %d' % (before_rows, sql('SELECT count(*) FROM attachments'), before, after))
for cid in replicas():
print('временных каталогов в %s: %s' % (cid, sh('docker', 'exec', cid, 'sh', '-c', 'ls /tmp | grep -c messenger-upload || true') or '0'))
def inject(alice, chat, title, body):
header(title)
token = alice['token']
before = reconcile('--grace-seconds=3600')['scanned']
rows = sql('SELECT count(*) FROM attachments')
sql("CREATE FUNCTION %s() RETURNS trigger AS $$ BEGIN %s RETURN NEW; END $$ LANGUAGE plpgsql; CREATE TRIGGER %s BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION %s();" % (FAULT_NAME, body, FAULT_NAME, FAULT_NAME))
try:
response, data = upload(token, chat)
print('upload при отказе INSERT -> %d %s' % (response.status, json.loads(data)['error']['code']))
finally:
sql('DROP TRIGGER %s ON attachments; DROP FUNCTION %s();' % (FAULT_NAME, FAULT_NAME))
print('строк attachments: %s -> %s' % (rows, sql('SELECT count(*) FROM attachments')))
fresh = reconcile('--apply', '--grace-seconds=3600')
print('объектов было %d, стало %d; свежий объект защищён grace: удалено %d' % (before, fresh['scanned'], fresh['deleted']))
time.sleep(2)
cleaned = reconcile('--apply', '--grace-seconds=1')
print('после grace: сирот %d, удалено %d, привязанные объекты не тронуты: %d' % (cleaned['orphans'], cleaned['deleted'], cleaned['referenced']))
response, data = upload(token, chat)
print('следующая загрузка без отказа ->', response.status)
wait_ready()
def main():
client = Client('http://%s:%d' % (HOST, PORT))
alice, chat = defence(client)
failover(alice, chat)
interrupted(alice, chat)
inject(alice, chat, '4. Реальный отказ PostgreSQL между PUT и INSERT: триггер отклоняет вставку', "RAISE EXCEPTION 'injected fault';")
inject(alice, chat, '5. Потеря соединения с PostgreSQL между PUT и INSERT: исход неясен, объект остаётся до сверки', 'PERFORM pg_terminate_backend(pg_backend_pid());')
if __name__ == '__main__':
main()
+37 -7
View File
@@ -16,7 +16,7 @@ run() {
}
started() {
docker inspect -f '{{.Name}} {{.State.Status}} started={{.State.StartedAt}}' $(docker compose ps -aq postgres redis api lb)
docker inspect -f '{{.Name}} {{.State.Status}} started={{.State.StartedAt}}' $(docker compose ps -aq postgres redis s3 api lb)
}
api_states() {
@@ -36,17 +36,21 @@ echo 'container.txt'
header 'Стенд: сервисы, порты, лимиты, секреты, миграции'
run docker compose ps -a --format 'table {{.Service}}\t{{.Name}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}'
echo '## CPU/RAM-лимиты (NanoCpus: 1e9 = 1 CPU; Memory в байтах)'
run sh -c "docker inspect -f '{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}}' \$(docker compose ps -q)"
run sh -c "docker inspect -f '{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}}' \$(docker compose ps -aq)"
echo '## container_name не задан (одно имя сервиса масштабируется)'
run sh -c "docker compose config | grep -c container_name || true"
echo '## Секреты: каждый сервис видит только свои файлы'
for service in postgres redis api; do
for service in postgres redis s3 api reconciler; do
run docker compose exec -T "$service" ls /run/secrets
done
run docker compose run --rm --no-deps -T --entrypoint ls s3-init /run/secrets
run docker compose exec -T lb sh -c 'ls /run/secrets 2>/dev/null || echo "no /run/secrets"'
run docker compose exec -T lb haproxy -v
run docker compose exec -T postgres postgres --version
run docker compose exec -T redis redis-server --version
run docker compose exec -T s3 minio --version
echo '## Таблицы вложений'
run docker compose exec -T postgres psql -U messenger -d messenger -c '\d attachments' -c '\d message_attachments'
echo '## Миграции: отдельный one-shot сервис, повторный запуск ничего не применяет'
run docker compose run --rm migrate
run docker compose run --rm --no-deps migrate sh -c 'export DATABASE_URL="$(cat "$DATABASE_URL_FILE")"; npx prisma migrate diff --from-url "$DATABASE_URL" --to-schema-datamodel prisma/schema.prisma --exit-code'
@@ -55,10 +59,10 @@ echo 'container.txt'
echo 'tests.txt'
{
header 'Собственные e2e-тесты (PostgreSQL + Redis из compose.test.yaml)'
header 'Собственные e2e-тесты (PostgreSQL + Redis + MinIO из compose.test.yaml)'
docker compose -f compose.test.yaml build -q tests
run docker compose -f compose.test.yaml run --rm tests npx jest --config ./test/jest-e2e.json --verbose
docker compose -f compose.test.yaml down >/dev/null 2>&1
docker compose -f compose.test.yaml down -v >/dev/null 2>&1
} > evidence/tests.txt
echo 'smoke.txt'
@@ -72,7 +76,7 @@ echo 'smoke.txt'
echo 'persistence.txt'
{
header 'make test-persistence: restart.sh перезапускает postgres, redis, api, lb с сохранением volumes'
header 'make test-persistence: restart.sh перезапускает postgres, redis, s3, api, lb с сохранением volumes'
echo '## до'
started
echo
@@ -104,6 +108,32 @@ echo 'failover.txt'
run env TARGET_INSTANCE_ID="$(docker compose ps -q postgres | cut -c1-12)" sh scripts/stop-one.sh
} > evidence/failover.txt
echo 'attachments.txt'
{
header 'Вложения: сценарий защиты, остановка загрузившей реплики, прерванная загрузка, отказ между S3 и БД'
run python3 scripts/attachment-scenario.py
} > evidence/attachments.txt
echo 's3.txt'
{
header 'S3: идемпотентный bootstrap, приватность мимо API, отказ S3, сверка'
echo '## Повторный bootstrap ничего не ломает'
run docker compose run --rm --no-deps -T s3-init
run docker compose run --rm --no-deps -T s3-init
run sh scripts/check-s3.sh
echo '## Отказ: контейнер S3 остановлен'
run sh scripts/s3-outage.sh stop
echo '## Отказ: контейнер S3 завис (docker pause)'
run sh scripts/s3-outage.sh pause
echo '## Сверка: объект без метаданных'
run sh scripts/reconcile-demo.sh
echo '## Журнал постоянного сервиса reconciler'
docker compose logs --no-log-prefix --tail 3 reconciler 2>&1
} > evidence/s3.txt
echo 'upload-memory.json'
python3 scripts/upload-experiment.py --output evidence/upload-memory.json >/dev/null 2>&1
echo 'experiments: idempotency, failover, failover --kill, rolling'
python3 scripts/experiment.py idempotency --output evidence/idempotency.json >/dev/null 2>&1
python3 scripts/experiment.py failover --output evidence/failover-stop.json >/dev/null 2>&1
@@ -142,7 +172,7 @@ echo 'failures.txt'
docker compose exec -T postgres sh -c "cp $hba /tmp/pg_hba.bak && { echo 'host all all $ip/32 reject'; cat /tmp/pg_hba.bak; } > $hba && chown postgres:postgres $hba && chmod 600 $hba"
docker compose exec -T postgres psql -U messenger -d messenger -tAc "SELECT pg_reload_conf(); SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE client_addr = '$ip'" >/dev/null
sleep 4
run docker exec "$target" node -e "let n=0;const g=p=>require('http').get('http://127.0.0.1:8080'+p,r=>{console.log(p,r.statusCode);r.resume();if(++n===2)process.exit(0)});g('/health/live');g('/health/ready')"
run docker exec "$target" node -e "let n=0;const g=p=>require('http').get('http://127.0.0.1:8080'+p,r=>{console.log(p,r.statusCode);r.resume();if(++n===2)process.exit(0)});g('/health/live');g('/health/core')"
run python3 scripts/probe.py instances 10
docker compose logs --no-log-prefix --since "$since" lb 2>&1 | grep -E 'is DOWN|is UP' | grep -v '^\[WARNING\]'
echo
+264
View File
@@ -0,0 +1,264 @@
#!/usr/bin/env python3
import argparse
import datetime
import hashlib
import http.client
import json
import math
import os
from pathlib import Path
import re
import struct
import subprocess
import sys
import threading
import time
import zlib
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client
HOST = os.environ.get('LB_HOST', 'lb')
PORT = int(os.environ.get('LB_PORT', '8080'))
SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes()
MiB = 1024 * 1024
def docker(*args, text=True):
return subprocess.run(['docker', *args], check=True, capture_output=True, text=text).stdout
def replicas():
return [cid[:12] for cid in docker('compose', 'ps', '-q', 'api').split()]
def chunk(kind, data):
return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data) & 0xffffffff)
def padded_png(size):
return SAMPLE[:33] + chunk(b'abCd', b'\x01' * (size - len(SAMPLE) - 12)) + SAMPLE[33:]
def big_png(width, height):
rows = (b'\0' + b'\x55' * (width * 3)) * height
header = struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)
return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', header) + chunk(b'IDAT', zlib.compress(rows, 9)) + chunk(b'IEND', b'')
def big_jpeg(width, height):
script = (
"require('sharp')({create:{width:%d,height:%d,channels:3,background:'#336699'}})"
".jpeg({quality:80}).toBuffer().then(b=>process.stdout.write(b))" % (width, height)
)
return docker('exec', replicas()[0], 'node', '-e', script, text=False)
def cgroup_peak(cid):
raw = docker('exec', cid, 'sh', '-c', 'cat /sys/fs/cgroup/memory/memory.max_usage_in_bytes 2>/dev/null || cat /sys/fs/cgroup/memory.peak 2>/dev/null || echo 0')
return int(raw.strip()) // MiB
def percentile(values, q):
ordered = sorted(values)
return round(ordered[max(0, math.ceil(q * len(ordered)) - 1)], 3)
class Sampler:
def __init__(self, cids):
self.rss = {cid: 0 for cid in cids}
self.cpu = {}
self.stop_flag = threading.Event()
self.threads = [threading.Thread(target=self.watch, args=(cid,)) for cid in cids]
self.threads.append(threading.Thread(target=self.watch_cpu))
def watch(self, cid):
script = 'while :; do grep VmRSS /proc/1/status; sleep 0.2; done'
process = subprocess.Popen(['docker', 'exec', cid, 'sh', '-c', script], stdout=subprocess.PIPE, text=True)
for line in process.stdout:
if self.stop_flag.is_set():
break
match = re.match(r'VmRSS:\s+(\d+)', line)
if match:
self.rss[cid] = max(self.rss[cid], int(match.group(1)) // 1024)
process.kill()
process.wait()
def watch_cpu(self):
while not self.stop_flag.is_set():
lines = docker('stats', '--no-stream', '--format', '{{.Name}} {{.CPUPerc}} {{.MemUsage}}')
for line in lines.splitlines():
name, cpu, usage = line.split(' ', 2)
value = float(cpu.rstrip('%'))
entry = self.cpu.setdefault(name, {'max_cpu_percent': 0, 'last_mem': ''})
entry['max_cpu_percent'] = max(entry['max_cpu_percent'], value)
entry['last_mem'] = usage.split(' / ')[0]
def __enter__(self):
[thread.start() for thread in self.threads]
return self
def __exit__(self, *_):
self.stop_flag.set()
for thread in self.threads:
thread.join(timeout=5)
def restart_replicas():
docker('compose', 'restart', 'api')
deadline = time.monotonic() + 90
expected = set(replicas())
while time.monotonic() < deadline:
seen = set()
for _ in range(8):
try:
conn = http.client.HTTPConnection(HOST, PORT, timeout=3)
conn.request('GET', '/health/ready')
response = conn.getresponse()
response.read()
if response.status == 200:
seen.add(response.getheader('X-Instance-Id'))
conn.close()
except OSError:
pass
if seen >= expected:
return
time.sleep(0.5)
raise SystemExit('replicas did not return')
def upload(path, token, body, content_type):
conn = http.client.HTTPConnection(HOST, PORT, timeout=120)
started = time.monotonic()
conn.request('POST', path, body=body, headers={'Authorization': 'Bearer ' + token, 'Content-Type': content_type})
response = conn.getresponse()
data = response.read()
elapsed = time.monotonic() - started
conn.close()
return response.status, elapsed, json.loads(data), response.getheader('X-Instance-Id')
def download_sha(attachment_id, token):
conn = http.client.HTTPConnection(HOST, PORT, timeout=60)
conn.request('GET', '/api/v1/attachments/' + attachment_id + '/content', headers={'Authorization': 'Bearer ' + token})
response = conn.getresponse()
data = response.read()
conn.close()
return response.status, hashlib.sha256(data).hexdigest()
def run_profile(name, body, content_type, concurrency, rounds, token, chat):
restart_replicas()
cids = replicas()
idle = {cid: int(re.search(r'VmRSS:\s+(\d+)', docker('exec', cid, 'cat', '/proc/1/status')).group(1)) // 1024 for cid in cids}
path = '/api/v1/chats/' + chat + '/attachments?filename=load.bin'
expected_sha = hashlib.sha256(body).hexdigest()
results = []
lock = threading.Lock()
def worker():
try:
outcome = upload(path, token, body, content_type)
except OSError as error:
outcome = (0, 0.0, {'error': {'code': type(error).__name__}}, None)
with lock:
results.append(outcome)
with Sampler(cids) as sampler:
started = time.monotonic()
for _ in range(rounds):
threads = [threading.Thread(target=worker) for _ in range(concurrency)]
[thread.start() for thread in threads]
[thread.join() for thread in threads]
wall = time.monotonic() - started
time.sleep(0.5)
created = [item for item in results if item[0] == 201]
latencies = [item[1] for item in created]
statuses = {}
for item in results:
statuses[str(item[0])] = statuses.get(str(item[0]), 0) + 1
sha_ok = all(item[2]['sha256'] == expected_sha for item in created)
stored_ok = None
if created:
status, sha = download_sha(created[0][2]['id'], token)
stored_ok = status == 200 and sha == expected_sha
per_replica = {}
for cid in cids:
info = docker('exec', cid, 'cat', '/proc/1/status')
hwm = int(re.search(r'VmHWM:\s+(\d+)', info).group(1)) // 1024
oom = docker('inspect', '-f', '{{.State.OOMKilled}}', cid).strip()
temp = docker('exec', cid, 'sh', '-c', 'ls /tmp | grep -c messenger-upload || true').strip()
per_replica[cid] = {
'idle_rss_mib': idle[cid],
'sampled_peak_rss_mib': sampler.rss[cid],
'vm_hwm_mib': hwm,
'cgroup_max_usage_with_page_cache_mib': cgroup_peak(cid),
'oom_killed': oom,
'temp_dirs_left': int(temp or 0),
'served': sum(1 for item in results if item[3] == cid),
}
return {
'profile': name,
'body_bytes': len(body),
'content_type': content_type,
'concurrency': concurrency,
'rounds': rounds,
'requests': len(results),
'statuses': statuses,
'wall_seconds': round(wall, 2),
'successful_rps': round(len(created) / wall, 2),
'error_rate': round(1 - len(created) / len(results), 4),
'latency_seconds': None if not latencies else {
'p50': percentile(latencies, 0.5),
'p95': percentile(latencies, 0.95),
'p99': percentile(latencies, 0.99),
},
'sha256_in_responses_ok': sha_ok,
'download_sha256_ok': stored_ok,
'replicas': per_replica,
'docker_stats': sampler.cpu,
}
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--output')
parser.add_argument('--rounds', type=int, default=3)
args = parser.parse_args()
client = Client('http://%s:%d' % (HOST, PORT))
alice = client.user('upload_load')
chat = client.call('POST', '/api/v1/chats', 201, actor=alice, body={'title': 'upload experiment', 'member_ids': [alice['id']]}, schema='Chat')['json']
token, chat_id = alice['token'], chat['id']
jpeg_40mp = big_jpeg(8000, 5000)
png_40mp = big_png(8000, 5000)
profiles = [
('sample 112 KiB x10', SAMPLE, 'image/png', 10),
('png 10 MiB x1', padded_png(10 * MiB), 'image/png', 1),
('png 10 MiB x10', padded_png(10 * MiB), 'image/png', 10),
('png 40 MP x10', png_40mp, 'image/png', 10),
('jpeg 40 MP x10', jpeg_40mp, 'image/jpeg', 10),
('png 10 MiB x20', padded_png(10 * MiB), 'image/png', 20),
('png 10 MiB x40 (выше лимита 16 на реплику)', padded_png(10 * MiB), 'image/png', 40),
]
results = []
for name, body, content_type, concurrency in profiles:
print('profile:', name, flush=True)
results.append(run_profile(name, body, content_type, concurrency, args.rounds, token, chat_id))
count = docker('compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-tAc', 'SELECT (SELECT count(*) FROM attachments), (SELECT count(*) FROM messages)').strip()
limits = docker('inspect', '-f', '{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}', replicas()[0]).split()
report = {
'date': datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'),
'api_replica_limits': {'cpus': int(limits[0]) / 1e9, 'memory_mib': int(limits[1]) // MiB},
'rows_after_attachments_messages': count,
'results': results,
}
text = json.dumps(report, ensure_ascii=False, indent=2)
if args.output:
Path(args.output).write_text(text + '\n', encoding='utf-8')
print(text)
if __name__ == '__main__':
main()
+4
View File
@@ -137,6 +137,10 @@ export class AttachmentsService implements OnModuleInit {
this.logger.warn(`compensation failed for ${objectKey}`);
});
}
if (error instanceof Prisma.PrismaClientUnknownRequestError) {
this.logger.error(`metadata insert failed for ${objectKey}`);
throw new ApiError('unavailable', 'database unavailable');
}
throw error;
}
}
+18
View File
@@ -325,6 +325,24 @@ describe('Загрузка вложений (e2e)', () => {
}
});
it('необработанная ошибка PostgreSQL при INSERT — 503, объект не удаляется', async () => {
const prisma = app.get(PrismaService);
const s3 = app.get(S3Service);
const remove = jest.spyOn(s3, 'delete');
const create = jest.spyOn(prisma.attachment, 'create').mockRejectedValueOnce(
new Prisma.PrismaClientUnknownRequestError('server closed the connection', {
clientVersion: 'test',
}),
);
try {
expectError(await upload(SAMPLE_PNG), 503, 'unavailable');
expect(remove).not.toHaveBeenCalled();
} finally {
create.mockRestore();
remove.mockRestore();
}
});
it('при неясном исходе COMMIT объект не удаляется', async () => {
const prisma = app.get(PrismaService);
const s3 = app.get(S3Service);