forked from mtusi2026/mtusi2026lab4
feat: add upload experiment, attachment scenarios and health extension
This commit is contained in:
@@ -18,3 +18,5 @@ S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
RECONCILE_GRACE_SECONDS=3600
|
||||
RECONCILE_INTERVAL_SECONDS=300
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
{
|
||||
"openapi": "3.1.0",
|
||||
"info": {
|
||||
"title": "MTUSI Messenger — расширения лабораторной 4",
|
||||
"version": "1.4.0",
|
||||
"description": "Обратно совместимое дополнение к contracts/openapi.json (см. COURSE.md, раздел «Контракт и совместимость»). Обязательный контракт не меняется. Обоснование — REPORT.md, раздел «Критичные зависимости и проверка готовности»."
|
||||
},
|
||||
"paths": {
|
||||
"/health/core": {
|
||||
"get": {
|
||||
"operationId": "getCoreReadiness",
|
||||
"summary": "Служебная проверка для балансировщика и Docker healthcheck: может ли реплика обслуживать API без вложений",
|
||||
"description": "В отличие от /health/ready не учитывает S3. Проверяет PostgreSQL и то, что процесс не останавливается. Падение общего S3 не выводит реплики из балансировки: сообщения продолжают работать, вложения отвечают 503.",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Реплика готова",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": { "status": { "const": "ok" } },
|
||||
"required": ["status"],
|
||||
"additionalProperties": false
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"503": {
|
||||
"description": "PostgreSQL недоступен или процесс останавливается; тело в общем формате Error",
|
||||
"content": { "application/json": { "schema": { "type": "object" } } }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "mtusi-messenger-lab4",
|
||||
"version": "1.0.0",
|
||||
"description": "Lab 1: in-memory messenger API (NestJS + Fastify)",
|
||||
"description": "Lab 4: messenger API with S3 attachments (NestJS + Fastify)",
|
||||
"private": true,
|
||||
"license": "UNLICENSED",
|
||||
"scripts": {
|
||||
|
||||
Executable
+190
@@ -0,0 +1,190 @@
|
||||
#!/usr/bin/env python3
|
||||
import hashlib
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client
|
||||
|
||||
HOST = os.environ.get('LB_HOST', 'lb')
|
||||
PORT = int(os.environ.get('LB_PORT', '8080'))
|
||||
SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
SAMPLE_SHA = hashlib.sha256(SAMPLE).hexdigest()
|
||||
BUCKET = os.environ.get('S3_BUCKET', 'messenger-attachments')
|
||||
FAULT_NAME = 'lab4_fault'
|
||||
|
||||
|
||||
def sh(*args, check=True):
|
||||
return subprocess.run(args, check=check, capture_output=True, text=True).stdout.strip()
|
||||
|
||||
|
||||
def sql(statement):
|
||||
return sh('docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-tAc', statement)
|
||||
|
||||
|
||||
def request(method, path, token=None, body=None, content_type=None):
|
||||
headers = {}
|
||||
if token:
|
||||
headers['Authorization'] = 'Bearer ' + token
|
||||
if content_type:
|
||||
headers['Content-Type'] = content_type
|
||||
conn = http.client.HTTPConnection(HOST, PORT, timeout=60)
|
||||
conn.request(method, path, body=body, headers=headers)
|
||||
response = conn.getresponse()
|
||||
data = response.read()
|
||||
conn.close()
|
||||
return response, data
|
||||
|
||||
|
||||
def upload(token, chat):
|
||||
return request('POST', '/api/v1/chats/' + chat + '/attachments?filename=sample.png', token, SAMPLE, 'image/png')
|
||||
|
||||
|
||||
def download(token, attachment):
|
||||
return request('GET', '/api/v1/attachments/' + attachment + '/content', token)
|
||||
|
||||
|
||||
def reconcile(*flags):
|
||||
output = sh('docker', 'compose', 'run', '--rm', '--no-deps', '-T', 'reconciler', 'node', 'dist/reconcile.js', *flags)
|
||||
line = output.splitlines()[-1]
|
||||
print('$ reconcile', ' '.join(flags))
|
||||
print(line)
|
||||
return json.loads(line)
|
||||
|
||||
|
||||
def replicas():
|
||||
return [cid[:12] for cid in sh('docker', 'compose', 'ps', '-q', 'api').split()]
|
||||
|
||||
|
||||
def wait_ready():
|
||||
deadline = time.monotonic() + 90
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
response, _ = request('GET', '/health/ready')
|
||||
if response.status == 200:
|
||||
return
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(0.3)
|
||||
raise SystemExit('not ready')
|
||||
|
||||
|
||||
def header(text):
|
||||
print('\n## ' + text, flush=True)
|
||||
|
||||
|
||||
def defence(client):
|
||||
header('1. Сценарий защиты: загрузка, сообщение, второй участник, посторонний, S3 мимо API')
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
attachment, _ = client.attachment(alice, chat)
|
||||
print('POST /attachments -> 201 status=%s variants=%s error_code=%s sha256=%s' % (attachment['status'], attachment['variants'], attachment['error_code'], attachment['sha256']))
|
||||
message = client.message(alice, chat, '', attachments=[attachment['id']])['json']
|
||||
print('POST /messages text="" -> 201 attachment_ids совпадают:', message['attachment_ids'] == [attachment['id']])
|
||||
response, data = download(bob['token'], attachment['id'])
|
||||
print('Bob GET /content -> %d %s sha256 совпадает: %s' % (response.status, response.getheader('Content-Type'), hashlib.sha256(data).hexdigest() == SAMPLE_SHA))
|
||||
for suffix in ('', '/content'):
|
||||
response, _ = request('GET', '/api/v1/attachments/' + attachment['id'] + suffix, eve['token'])
|
||||
print('Eve GET /attachments/{id}%s -> %d' % (suffix, response.status))
|
||||
key = sql("SELECT object_key FROM attachments WHERE id = '%s'" % attachment['id'])
|
||||
print('object_key в БД: attachments/<uuid>, filename в ключ не входит:', 'sample' not in key)
|
||||
try:
|
||||
code = urllib.request.urlopen('http://s3:9000/%s/%s' % (BUCKET, key), timeout=5).status
|
||||
except urllib.error.HTTPError as error:
|
||||
code = error.code
|
||||
print('Анонимный GET объекта напрямую в S3 ->', code)
|
||||
return alice, chat['id']
|
||||
|
||||
|
||||
def failover(alice, chat):
|
||||
header('2. Остановка реплики, обслужившей загрузку; скачивание через другую')
|
||||
token = alice['token']
|
||||
tested = set()
|
||||
while len(tested) < 2:
|
||||
response, data = upload(token, chat)
|
||||
target = response.getheader('X-Instance-Id')
|
||||
if target in tested:
|
||||
continue
|
||||
tested.add(target)
|
||||
attachment = json.loads(data)
|
||||
print('upload -> %d, обслужила реплика %s' % (response.status, target))
|
||||
result = subprocess.run(['sh', 'scripts/stop-one.sh'], env={**os.environ, 'TARGET_INSTANCE_ID': target}, capture_output=True, text=True)
|
||||
print('stop-one.sh:', result.stdout.strip())
|
||||
others = [cid for cid in replicas() if cid != target]
|
||||
started = time.monotonic()
|
||||
served = []
|
||||
for _ in range(6):
|
||||
response, data = download(token, attachment['id'])
|
||||
served.append(response.getheader('X-Instance-Id'))
|
||||
ok = response.status == 200 and hashlib.sha256(data).hexdigest() == SAMPLE_SHA
|
||||
print('download -> %d реплика=%s sha256 совпадает: %s' % (response.status, served[-1], ok))
|
||||
if not ok:
|
||||
raise SystemExit('download failed')
|
||||
print('остановленная реплика среди обслуживших:', target in served, '; %.2fs' % (time.monotonic() - started))
|
||||
print('Файл оригинала на диске обслужившей реплики (поиск по точному размеру %d байт):' % len(SAMPLE))
|
||||
for other in others:
|
||||
found = sh('docker', 'exec', other, 'sh', '-c', 'find /tmp /app /home -xdev -type f -size %dc 2>/dev/null | grep -v node_modules || true' % len(SAMPLE))
|
||||
print(' %s -> %s' % (other, found or 'нет'))
|
||||
sh('docker', 'start', target)
|
||||
wait_ready()
|
||||
time.sleep(3)
|
||||
print('реплика %s возвращена' % target)
|
||||
|
||||
|
||||
def interrupted(alice, chat):
|
||||
header('3. Прерванная загрузка через LB: ни строки, ни объекта, ни временного файла')
|
||||
before_rows = sql('SELECT count(*) FROM attachments')
|
||||
before = reconcile('--grace-seconds=3600')['scanned']
|
||||
connection = socket.create_connection((HOST, PORT))
|
||||
connection.sendall(('POST /api/v1/chats/%s/attachments?filename=cut.png HTTP/1.1\r\nHost: lb\r\nAuthorization: Bearer %s\r\nContent-Type: image/png\r\nContent-Length: %d\r\n\r\n' % (chat, alice['token'], len(SAMPLE))).encode())
|
||||
connection.sendall(SAMPLE[:20000])
|
||||
time.sleep(0.5)
|
||||
connection.close()
|
||||
time.sleep(1)
|
||||
after = reconcile('--grace-seconds=3600')['scanned']
|
||||
print('строк attachments: %s -> %s; объектов: %d -> %d' % (before_rows, sql('SELECT count(*) FROM attachments'), before, after))
|
||||
for cid in replicas():
|
||||
print('временных каталогов в %s: %s' % (cid, sh('docker', 'exec', cid, 'sh', '-c', 'ls /tmp | grep -c messenger-upload || true') or '0'))
|
||||
|
||||
|
||||
def inject(alice, chat, title, body):
|
||||
header(title)
|
||||
token = alice['token']
|
||||
before = reconcile('--grace-seconds=3600')['scanned']
|
||||
rows = sql('SELECT count(*) FROM attachments')
|
||||
sql("CREATE FUNCTION %s() RETURNS trigger AS $$ BEGIN %s RETURN NEW; END $$ LANGUAGE plpgsql; CREATE TRIGGER %s BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION %s();" % (FAULT_NAME, body, FAULT_NAME, FAULT_NAME))
|
||||
try:
|
||||
response, data = upload(token, chat)
|
||||
print('upload при отказе INSERT -> %d %s' % (response.status, json.loads(data)['error']['code']))
|
||||
finally:
|
||||
sql('DROP TRIGGER %s ON attachments; DROP FUNCTION %s();' % (FAULT_NAME, FAULT_NAME))
|
||||
print('строк attachments: %s -> %s' % (rows, sql('SELECT count(*) FROM attachments')))
|
||||
fresh = reconcile('--apply', '--grace-seconds=3600')
|
||||
print('объектов было %d, стало %d; свежий объект защищён grace: удалено %d' % (before, fresh['scanned'], fresh['deleted']))
|
||||
time.sleep(2)
|
||||
cleaned = reconcile('--apply', '--grace-seconds=1')
|
||||
print('после grace: сирот %d, удалено %d, привязанные объекты не тронуты: %d' % (cleaned['orphans'], cleaned['deleted'], cleaned['referenced']))
|
||||
response, data = upload(token, chat)
|
||||
print('следующая загрузка без отказа ->', response.status)
|
||||
wait_ready()
|
||||
|
||||
|
||||
def main():
|
||||
client = Client('http://%s:%d' % (HOST, PORT))
|
||||
alice, chat = defence(client)
|
||||
failover(alice, chat)
|
||||
interrupted(alice, chat)
|
||||
inject(alice, chat, '4. Реальный отказ PostgreSQL между PUT и INSERT: триггер отклоняет вставку', "RAISE EXCEPTION 'injected fault';")
|
||||
inject(alice, chat, '5. Потеря соединения с PostgreSQL между PUT и INSERT: исход неясен, объект остаётся до сверки', 'PERFORM pg_terminate_backend(pg_backend_pid());')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -16,7 +16,7 @@ run() {
|
||||
}
|
||||
|
||||
started() {
|
||||
docker inspect -f '{{.Name}} {{.State.Status}} started={{.State.StartedAt}}' $(docker compose ps -aq postgres redis api lb)
|
||||
docker inspect -f '{{.Name}} {{.State.Status}} started={{.State.StartedAt}}' $(docker compose ps -aq postgres redis s3 api lb)
|
||||
}
|
||||
|
||||
api_states() {
|
||||
@@ -36,17 +36,21 @@ echo 'container.txt'
|
||||
header 'Стенд: сервисы, порты, лимиты, секреты, миграции'
|
||||
run docker compose ps -a --format 'table {{.Service}}\t{{.Name}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}'
|
||||
echo '## CPU/RAM-лимиты (NanoCpus: 1e9 = 1 CPU; Memory в байтах)'
|
||||
run sh -c "docker inspect -f '{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}}' \$(docker compose ps -q)"
|
||||
run sh -c "docker inspect -f '{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}}' \$(docker compose ps -aq)"
|
||||
echo '## container_name не задан (одно имя сервиса масштабируется)'
|
||||
run sh -c "docker compose config | grep -c container_name || true"
|
||||
echo '## Секреты: каждый сервис видит только свои файлы'
|
||||
for service in postgres redis api; do
|
||||
for service in postgres redis s3 api reconciler; do
|
||||
run docker compose exec -T "$service" ls /run/secrets
|
||||
done
|
||||
run docker compose run --rm --no-deps -T --entrypoint ls s3-init /run/secrets
|
||||
run docker compose exec -T lb sh -c 'ls /run/secrets 2>/dev/null || echo "no /run/secrets"'
|
||||
run docker compose exec -T lb haproxy -v
|
||||
run docker compose exec -T postgres postgres --version
|
||||
run docker compose exec -T redis redis-server --version
|
||||
run docker compose exec -T s3 minio --version
|
||||
echo '## Таблицы вложений'
|
||||
run docker compose exec -T postgres psql -U messenger -d messenger -c '\d attachments' -c '\d message_attachments'
|
||||
echo '## Миграции: отдельный one-shot сервис, повторный запуск ничего не применяет'
|
||||
run docker compose run --rm migrate
|
||||
run docker compose run --rm --no-deps migrate sh -c 'export DATABASE_URL="$(cat "$DATABASE_URL_FILE")"; npx prisma migrate diff --from-url "$DATABASE_URL" --to-schema-datamodel prisma/schema.prisma --exit-code'
|
||||
@@ -55,10 +59,10 @@ echo 'container.txt'
|
||||
|
||||
echo 'tests.txt'
|
||||
{
|
||||
header 'Собственные e2e-тесты (PostgreSQL + Redis из compose.test.yaml)'
|
||||
header 'Собственные e2e-тесты (PostgreSQL + Redis + MinIO из compose.test.yaml)'
|
||||
docker compose -f compose.test.yaml build -q tests
|
||||
run docker compose -f compose.test.yaml run --rm tests npx jest --config ./test/jest-e2e.json --verbose
|
||||
docker compose -f compose.test.yaml down >/dev/null 2>&1
|
||||
docker compose -f compose.test.yaml down -v >/dev/null 2>&1
|
||||
} > evidence/tests.txt
|
||||
|
||||
echo 'smoke.txt'
|
||||
@@ -72,7 +76,7 @@ echo 'smoke.txt'
|
||||
|
||||
echo 'persistence.txt'
|
||||
{
|
||||
header 'make test-persistence: restart.sh перезапускает postgres, redis, api, lb с сохранением volumes'
|
||||
header 'make test-persistence: restart.sh перезапускает postgres, redis, s3, api, lb с сохранением volumes'
|
||||
echo '## до'
|
||||
started
|
||||
echo
|
||||
@@ -104,6 +108,32 @@ echo 'failover.txt'
|
||||
run env TARGET_INSTANCE_ID="$(docker compose ps -q postgres | cut -c1-12)" sh scripts/stop-one.sh
|
||||
} > evidence/failover.txt
|
||||
|
||||
echo 'attachments.txt'
|
||||
{
|
||||
header 'Вложения: сценарий защиты, остановка загрузившей реплики, прерванная загрузка, отказ между S3 и БД'
|
||||
run python3 scripts/attachment-scenario.py
|
||||
} > evidence/attachments.txt
|
||||
|
||||
echo 's3.txt'
|
||||
{
|
||||
header 'S3: идемпотентный bootstrap, приватность мимо API, отказ S3, сверка'
|
||||
echo '## Повторный bootstrap ничего не ломает'
|
||||
run docker compose run --rm --no-deps -T s3-init
|
||||
run docker compose run --rm --no-deps -T s3-init
|
||||
run sh scripts/check-s3.sh
|
||||
echo '## Отказ: контейнер S3 остановлен'
|
||||
run sh scripts/s3-outage.sh stop
|
||||
echo '## Отказ: контейнер S3 завис (docker pause)'
|
||||
run sh scripts/s3-outage.sh pause
|
||||
echo '## Сверка: объект без метаданных'
|
||||
run sh scripts/reconcile-demo.sh
|
||||
echo '## Журнал постоянного сервиса reconciler'
|
||||
docker compose logs --no-log-prefix --tail 3 reconciler 2>&1
|
||||
} > evidence/s3.txt
|
||||
|
||||
echo 'upload-memory.json'
|
||||
python3 scripts/upload-experiment.py --output evidence/upload-memory.json >/dev/null 2>&1
|
||||
|
||||
echo 'experiments: idempotency, failover, failover --kill, rolling'
|
||||
python3 scripts/experiment.py idempotency --output evidence/idempotency.json >/dev/null 2>&1
|
||||
python3 scripts/experiment.py failover --output evidence/failover-stop.json >/dev/null 2>&1
|
||||
@@ -142,7 +172,7 @@ echo 'failures.txt'
|
||||
docker compose exec -T postgres sh -c "cp $hba /tmp/pg_hba.bak && { echo 'host all all $ip/32 reject'; cat /tmp/pg_hba.bak; } > $hba && chown postgres:postgres $hba && chmod 600 $hba"
|
||||
docker compose exec -T postgres psql -U messenger -d messenger -tAc "SELECT pg_reload_conf(); SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE client_addr = '$ip'" >/dev/null
|
||||
sleep 4
|
||||
run docker exec "$target" node -e "let n=0;const g=p=>require('http').get('http://127.0.0.1:8080'+p,r=>{console.log(p,r.statusCode);r.resume();if(++n===2)process.exit(0)});g('/health/live');g('/health/ready')"
|
||||
run docker exec "$target" node -e "let n=0;const g=p=>require('http').get('http://127.0.0.1:8080'+p,r=>{console.log(p,r.statusCode);r.resume();if(++n===2)process.exit(0)});g('/health/live');g('/health/core')"
|
||||
run python3 scripts/probe.py instances 10
|
||||
docker compose logs --no-log-prefix --since "$since" lb 2>&1 | grep -E 'is DOWN|is UP' | grep -v '^\[WARNING\]'
|
||||
echo
|
||||
|
||||
Executable
+264
@@ -0,0 +1,264 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import datetime
|
||||
import hashlib
|
||||
import http.client
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import zlib
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client
|
||||
|
||||
HOST = os.environ.get('LB_HOST', 'lb')
|
||||
PORT = int(os.environ.get('LB_PORT', '8080'))
|
||||
SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
MiB = 1024 * 1024
|
||||
|
||||
|
||||
def docker(*args, text=True):
|
||||
return subprocess.run(['docker', *args], check=True, capture_output=True, text=text).stdout
|
||||
|
||||
|
||||
def replicas():
|
||||
return [cid[:12] for cid in docker('compose', 'ps', '-q', 'api').split()]
|
||||
|
||||
|
||||
def chunk(kind, data):
|
||||
return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data) & 0xffffffff)
|
||||
|
||||
|
||||
def padded_png(size):
|
||||
return SAMPLE[:33] + chunk(b'abCd', b'\x01' * (size - len(SAMPLE) - 12)) + SAMPLE[33:]
|
||||
|
||||
|
||||
def big_png(width, height):
|
||||
rows = (b'\0' + b'\x55' * (width * 3)) * height
|
||||
header = struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)
|
||||
return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', header) + chunk(b'IDAT', zlib.compress(rows, 9)) + chunk(b'IEND', b'')
|
||||
|
||||
|
||||
def big_jpeg(width, height):
|
||||
script = (
|
||||
"require('sharp')({create:{width:%d,height:%d,channels:3,background:'#336699'}})"
|
||||
".jpeg({quality:80}).toBuffer().then(b=>process.stdout.write(b))" % (width, height)
|
||||
)
|
||||
return docker('exec', replicas()[0], 'node', '-e', script, text=False)
|
||||
|
||||
|
||||
def cgroup_peak(cid):
|
||||
raw = docker('exec', cid, 'sh', '-c', 'cat /sys/fs/cgroup/memory/memory.max_usage_in_bytes 2>/dev/null || cat /sys/fs/cgroup/memory.peak 2>/dev/null || echo 0')
|
||||
return int(raw.strip()) // MiB
|
||||
|
||||
|
||||
def percentile(values, q):
|
||||
ordered = sorted(values)
|
||||
return round(ordered[max(0, math.ceil(q * len(ordered)) - 1)], 3)
|
||||
|
||||
|
||||
class Sampler:
|
||||
def __init__(self, cids):
|
||||
self.rss = {cid: 0 for cid in cids}
|
||||
self.cpu = {}
|
||||
self.stop_flag = threading.Event()
|
||||
self.threads = [threading.Thread(target=self.watch, args=(cid,)) for cid in cids]
|
||||
self.threads.append(threading.Thread(target=self.watch_cpu))
|
||||
|
||||
def watch(self, cid):
|
||||
script = 'while :; do grep VmRSS /proc/1/status; sleep 0.2; done'
|
||||
process = subprocess.Popen(['docker', 'exec', cid, 'sh', '-c', script], stdout=subprocess.PIPE, text=True)
|
||||
for line in process.stdout:
|
||||
if self.stop_flag.is_set():
|
||||
break
|
||||
match = re.match(r'VmRSS:\s+(\d+)', line)
|
||||
if match:
|
||||
self.rss[cid] = max(self.rss[cid], int(match.group(1)) // 1024)
|
||||
process.kill()
|
||||
process.wait()
|
||||
|
||||
def watch_cpu(self):
|
||||
while not self.stop_flag.is_set():
|
||||
lines = docker('stats', '--no-stream', '--format', '{{.Name}} {{.CPUPerc}} {{.MemUsage}}')
|
||||
for line in lines.splitlines():
|
||||
name, cpu, usage = line.split(' ', 2)
|
||||
value = float(cpu.rstrip('%'))
|
||||
entry = self.cpu.setdefault(name, {'max_cpu_percent': 0, 'last_mem': ''})
|
||||
entry['max_cpu_percent'] = max(entry['max_cpu_percent'], value)
|
||||
entry['last_mem'] = usage.split(' / ')[0]
|
||||
|
||||
def __enter__(self):
|
||||
[thread.start() for thread in self.threads]
|
||||
return self
|
||||
|
||||
def __exit__(self, *_):
|
||||
self.stop_flag.set()
|
||||
for thread in self.threads:
|
||||
thread.join(timeout=5)
|
||||
|
||||
|
||||
def restart_replicas():
|
||||
docker('compose', 'restart', 'api')
|
||||
deadline = time.monotonic() + 90
|
||||
expected = set(replicas())
|
||||
while time.monotonic() < deadline:
|
||||
seen = set()
|
||||
for _ in range(8):
|
||||
try:
|
||||
conn = http.client.HTTPConnection(HOST, PORT, timeout=3)
|
||||
conn.request('GET', '/health/ready')
|
||||
response = conn.getresponse()
|
||||
response.read()
|
||||
if response.status == 200:
|
||||
seen.add(response.getheader('X-Instance-Id'))
|
||||
conn.close()
|
||||
except OSError:
|
||||
pass
|
||||
if seen >= expected:
|
||||
return
|
||||
time.sleep(0.5)
|
||||
raise SystemExit('replicas did not return')
|
||||
|
||||
|
||||
def upload(path, token, body, content_type):
|
||||
conn = http.client.HTTPConnection(HOST, PORT, timeout=120)
|
||||
started = time.monotonic()
|
||||
conn.request('POST', path, body=body, headers={'Authorization': 'Bearer ' + token, 'Content-Type': content_type})
|
||||
response = conn.getresponse()
|
||||
data = response.read()
|
||||
elapsed = time.monotonic() - started
|
||||
conn.close()
|
||||
return response.status, elapsed, json.loads(data), response.getheader('X-Instance-Id')
|
||||
|
||||
|
||||
def download_sha(attachment_id, token):
|
||||
conn = http.client.HTTPConnection(HOST, PORT, timeout=60)
|
||||
conn.request('GET', '/api/v1/attachments/' + attachment_id + '/content', headers={'Authorization': 'Bearer ' + token})
|
||||
response = conn.getresponse()
|
||||
data = response.read()
|
||||
conn.close()
|
||||
return response.status, hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def run_profile(name, body, content_type, concurrency, rounds, token, chat):
|
||||
restart_replicas()
|
||||
cids = replicas()
|
||||
idle = {cid: int(re.search(r'VmRSS:\s+(\d+)', docker('exec', cid, 'cat', '/proc/1/status')).group(1)) // 1024 for cid in cids}
|
||||
path = '/api/v1/chats/' + chat + '/attachments?filename=load.bin'
|
||||
expected_sha = hashlib.sha256(body).hexdigest()
|
||||
results = []
|
||||
lock = threading.Lock()
|
||||
|
||||
def worker():
|
||||
try:
|
||||
outcome = upload(path, token, body, content_type)
|
||||
except OSError as error:
|
||||
outcome = (0, 0.0, {'error': {'code': type(error).__name__}}, None)
|
||||
with lock:
|
||||
results.append(outcome)
|
||||
|
||||
with Sampler(cids) as sampler:
|
||||
started = time.monotonic()
|
||||
for _ in range(rounds):
|
||||
threads = [threading.Thread(target=worker) for _ in range(concurrency)]
|
||||
[thread.start() for thread in threads]
|
||||
[thread.join() for thread in threads]
|
||||
wall = time.monotonic() - started
|
||||
time.sleep(0.5)
|
||||
|
||||
created = [item for item in results if item[0] == 201]
|
||||
latencies = [item[1] for item in created]
|
||||
statuses = {}
|
||||
for item in results:
|
||||
statuses[str(item[0])] = statuses.get(str(item[0]), 0) + 1
|
||||
sha_ok = all(item[2]['sha256'] == expected_sha for item in created)
|
||||
stored_ok = None
|
||||
if created:
|
||||
status, sha = download_sha(created[0][2]['id'], token)
|
||||
stored_ok = status == 200 and sha == expected_sha
|
||||
per_replica = {}
|
||||
for cid in cids:
|
||||
info = docker('exec', cid, 'cat', '/proc/1/status')
|
||||
hwm = int(re.search(r'VmHWM:\s+(\d+)', info).group(1)) // 1024
|
||||
oom = docker('inspect', '-f', '{{.State.OOMKilled}}', cid).strip()
|
||||
temp = docker('exec', cid, 'sh', '-c', 'ls /tmp | grep -c messenger-upload || true').strip()
|
||||
per_replica[cid] = {
|
||||
'idle_rss_mib': idle[cid],
|
||||
'sampled_peak_rss_mib': sampler.rss[cid],
|
||||
'vm_hwm_mib': hwm,
|
||||
'cgroup_max_usage_with_page_cache_mib': cgroup_peak(cid),
|
||||
'oom_killed': oom,
|
||||
'temp_dirs_left': int(temp or 0),
|
||||
'served': sum(1 for item in results if item[3] == cid),
|
||||
}
|
||||
return {
|
||||
'profile': name,
|
||||
'body_bytes': len(body),
|
||||
'content_type': content_type,
|
||||
'concurrency': concurrency,
|
||||
'rounds': rounds,
|
||||
'requests': len(results),
|
||||
'statuses': statuses,
|
||||
'wall_seconds': round(wall, 2),
|
||||
'successful_rps': round(len(created) / wall, 2),
|
||||
'error_rate': round(1 - len(created) / len(results), 4),
|
||||
'latency_seconds': None if not latencies else {
|
||||
'p50': percentile(latencies, 0.5),
|
||||
'p95': percentile(latencies, 0.95),
|
||||
'p99': percentile(latencies, 0.99),
|
||||
},
|
||||
'sha256_in_responses_ok': sha_ok,
|
||||
'download_sha256_ok': stored_ok,
|
||||
'replicas': per_replica,
|
||||
'docker_stats': sampler.cpu,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--output')
|
||||
parser.add_argument('--rounds', type=int, default=3)
|
||||
args = parser.parse_args()
|
||||
client = Client('http://%s:%d' % (HOST, PORT))
|
||||
alice = client.user('upload_load')
|
||||
chat = client.call('POST', '/api/v1/chats', 201, actor=alice, body={'title': 'upload experiment', 'member_ids': [alice['id']]}, schema='Chat')['json']
|
||||
token, chat_id = alice['token'], chat['id']
|
||||
jpeg_40mp = big_jpeg(8000, 5000)
|
||||
png_40mp = big_png(8000, 5000)
|
||||
profiles = [
|
||||
('sample 112 KiB x10', SAMPLE, 'image/png', 10),
|
||||
('png 10 MiB x1', padded_png(10 * MiB), 'image/png', 1),
|
||||
('png 10 MiB x10', padded_png(10 * MiB), 'image/png', 10),
|
||||
('png 40 MP x10', png_40mp, 'image/png', 10),
|
||||
('jpeg 40 MP x10', jpeg_40mp, 'image/jpeg', 10),
|
||||
('png 10 MiB x20', padded_png(10 * MiB), 'image/png', 20),
|
||||
('png 10 MiB x40 (выше лимита 16 на реплику)', padded_png(10 * MiB), 'image/png', 40),
|
||||
]
|
||||
results = []
|
||||
for name, body, content_type, concurrency in profiles:
|
||||
print('profile:', name, flush=True)
|
||||
results.append(run_profile(name, body, content_type, concurrency, args.rounds, token, chat_id))
|
||||
count = docker('compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-tAc', 'SELECT (SELECT count(*) FROM attachments), (SELECT count(*) FROM messages)').strip()
|
||||
limits = docker('inspect', '-f', '{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}', replicas()[0]).split()
|
||||
report = {
|
||||
'date': datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'),
|
||||
'api_replica_limits': {'cpus': int(limits[0]) / 1e9, 'memory_mib': int(limits[1]) // MiB},
|
||||
'rows_after_attachments_messages': count,
|
||||
'results': results,
|
||||
}
|
||||
text = json.dumps(report, ensure_ascii=False, indent=2)
|
||||
if args.output:
|
||||
Path(args.output).write_text(text + '\n', encoding='utf-8')
|
||||
print(text)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -137,6 +137,10 @@ export class AttachmentsService implements OnModuleInit {
|
||||
this.logger.warn(`compensation failed for ${objectKey}`);
|
||||
});
|
||||
}
|
||||
if (error instanceof Prisma.PrismaClientUnknownRequestError) {
|
||||
this.logger.error(`metadata insert failed for ${objectKey}`);
|
||||
throw new ApiError('unavailable', 'database unavailable');
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -325,6 +325,24 @@ describe('Загрузка вложений (e2e)', () => {
|
||||
}
|
||||
});
|
||||
|
||||
it('необработанная ошибка PostgreSQL при INSERT — 503, объект не удаляется', async () => {
|
||||
const prisma = app.get(PrismaService);
|
||||
const s3 = app.get(S3Service);
|
||||
const remove = jest.spyOn(s3, 'delete');
|
||||
const create = jest.spyOn(prisma.attachment, 'create').mockRejectedValueOnce(
|
||||
new Prisma.PrismaClientUnknownRequestError('server closed the connection', {
|
||||
clientVersion: 'test',
|
||||
}),
|
||||
);
|
||||
try {
|
||||
expectError(await upload(SAMPLE_PNG), 503, 'unavailable');
|
||||
expect(remove).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
create.mockRestore();
|
||||
remove.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('при неясном исходе COMMIT объект не удаляется', async () => {
|
||||
const prisma = app.get(PrismaService);
|
||||
const s3 = app.get(S3Service);
|
||||
|
||||
Reference in New Issue
Block a user