From 5c3cba4951cee63af641bd4bb58e3540f10ece7a Mon Sep 17 00:00:00 2001 From: aegor Date: Thu, 5 Nov 2026 16:26:15 +0300 Subject: [PATCH] feat: add upload experiment, attachment scenarios and health extension --- .env.example | 2 + contracts/extensions.openapi.json | 36 ++++ package.json | 2 +- scripts/attachment-scenario.py | 190 ++++++++++++++++++ scripts/collect-evidence.sh | 44 ++++- scripts/upload-experiment.py | 264 +++++++++++++++++++++++++ src/attachments/attachments.service.ts | 4 + test/attachments.e2e-spec.ts | 18 ++ 8 files changed, 552 insertions(+), 8 deletions(-) create mode 100644 contracts/extensions.openapi.json create mode 100755 scripts/attachment-scenario.py create mode 100755 scripts/upload-experiment.py diff --git a/.env.example b/.env.example index d99d26a..6860c71 100644 --- a/.env.example +++ b/.env.example @@ -18,3 +18,5 @@ S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 +RECONCILE_GRACE_SECONDS=3600 +RECONCILE_INTERVAL_SECONDS=300 diff --git a/contracts/extensions.openapi.json b/contracts/extensions.openapi.json new file mode 100644 index 0000000..f79db1a --- /dev/null +++ b/contracts/extensions.openapi.json @@ -0,0 +1,36 @@ +{ + "openapi": "3.1.0", + "info": { + "title": "MTUSI Messenger — расширения лабораторной 4", + "version": "1.4.0", + "description": "Обратно совместимое дополнение к contracts/openapi.json (см. COURSE.md, раздел «Контракт и совместимость»). Обязательный контракт не меняется. Обоснование — REPORT.md, раздел «Критичные зависимости и проверка готовности»." + }, + "paths": { + "/health/core": { + "get": { + "operationId": "getCoreReadiness", + "summary": "Служебная проверка для балансировщика и Docker healthcheck: может ли реплика обслуживать API без вложений", + "description": "В отличие от /health/ready не учитывает S3. Проверяет PostgreSQL и то, что процесс не останавливается. Падение общего S3 не выводит реплики из балансировки: сообщения продолжают работать, вложения отвечают 503.", + "responses": { + "200": { + "description": "Реплика готова", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { "status": { "const": "ok" } }, + "required": ["status"], + "additionalProperties": false + } + } + } + }, + "503": { + "description": "PostgreSQL недоступен или процесс останавливается; тело в общем формате Error", + "content": { "application/json": { "schema": { "type": "object" } } } + } + } + } + } + } +} diff --git a/package.json b/package.json index b8ee565..efa1196 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "mtusi-messenger-lab4", "version": "1.0.0", - "description": "Lab 1: in-memory messenger API (NestJS + Fastify)", + "description": "Lab 4: messenger API with S3 attachments (NestJS + Fastify)", "private": true, "license": "UNLICENSED", "scripts": { diff --git a/scripts/attachment-scenario.py b/scripts/attachment-scenario.py new file mode 100755 index 0000000..fad53f2 --- /dev/null +++ b/scripts/attachment-scenario.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +import hashlib +import http.client +import json +import os +from pathlib import Path +import socket +import subprocess +import sys +import time +import urllib.error +import urllib.request + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client + +HOST = os.environ.get('LB_HOST', 'lb') +PORT = int(os.environ.get('LB_PORT', '8080')) +SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes() +SAMPLE_SHA = hashlib.sha256(SAMPLE).hexdigest() +BUCKET = os.environ.get('S3_BUCKET', 'messenger-attachments') +FAULT_NAME = 'lab4_fault' + + +def sh(*args, check=True): + return subprocess.run(args, check=check, capture_output=True, text=True).stdout.strip() + + +def sql(statement): + return sh('docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-tAc', statement) + + +def request(method, path, token=None, body=None, content_type=None): + headers = {} + if token: + headers['Authorization'] = 'Bearer ' + token + if content_type: + headers['Content-Type'] = content_type + conn = http.client.HTTPConnection(HOST, PORT, timeout=60) + conn.request(method, path, body=body, headers=headers) + response = conn.getresponse() + data = response.read() + conn.close() + return response, data + + +def upload(token, chat): + return request('POST', '/api/v1/chats/' + chat + '/attachments?filename=sample.png', token, SAMPLE, 'image/png') + + +def download(token, attachment): + return request('GET', '/api/v1/attachments/' + attachment + '/content', token) + + +def reconcile(*flags): + output = sh('docker', 'compose', 'run', '--rm', '--no-deps', '-T', 'reconciler', 'node', 'dist/reconcile.js', *flags) + line = output.splitlines()[-1] + print('$ reconcile', ' '.join(flags)) + print(line) + return json.loads(line) + + +def replicas(): + return [cid[:12] for cid in sh('docker', 'compose', 'ps', '-q', 'api').split()] + + +def wait_ready(): + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + try: + response, _ = request('GET', '/health/ready') + if response.status == 200: + return + except OSError: + pass + time.sleep(0.3) + raise SystemExit('not ready') + + +def header(text): + print('\n## ' + text, flush=True) + + +def defence(client): + header('1. Сценарий защиты: загрузка, сообщение, второй участник, посторонний, S3 мимо API') + alice, bob, eve, chat = client.bootstrap() + attachment, _ = client.attachment(alice, chat) + print('POST /attachments -> 201 status=%s variants=%s error_code=%s sha256=%s' % (attachment['status'], attachment['variants'], attachment['error_code'], attachment['sha256'])) + message = client.message(alice, chat, '', attachments=[attachment['id']])['json'] + print('POST /messages text="" -> 201 attachment_ids совпадают:', message['attachment_ids'] == [attachment['id']]) + response, data = download(bob['token'], attachment['id']) + print('Bob GET /content -> %d %s sha256 совпадает: %s' % (response.status, response.getheader('Content-Type'), hashlib.sha256(data).hexdigest() == SAMPLE_SHA)) + for suffix in ('', '/content'): + response, _ = request('GET', '/api/v1/attachments/' + attachment['id'] + suffix, eve['token']) + print('Eve GET /attachments/{id}%s -> %d' % (suffix, response.status)) + key = sql("SELECT object_key FROM attachments WHERE id = '%s'" % attachment['id']) + print('object_key в БД: attachments/, filename в ключ не входит:', 'sample' not in key) + try: + code = urllib.request.urlopen('http://s3:9000/%s/%s' % (BUCKET, key), timeout=5).status + except urllib.error.HTTPError as error: + code = error.code + print('Анонимный GET объекта напрямую в S3 ->', code) + return alice, chat['id'] + + +def failover(alice, chat): + header('2. Остановка реплики, обслужившей загрузку; скачивание через другую') + token = alice['token'] + tested = set() + while len(tested) < 2: + response, data = upload(token, chat) + target = response.getheader('X-Instance-Id') + if target in tested: + continue + tested.add(target) + attachment = json.loads(data) + print('upload -> %d, обслужила реплика %s' % (response.status, target)) + result = subprocess.run(['sh', 'scripts/stop-one.sh'], env={**os.environ, 'TARGET_INSTANCE_ID': target}, capture_output=True, text=True) + print('stop-one.sh:', result.stdout.strip()) + others = [cid for cid in replicas() if cid != target] + started = time.monotonic() + served = [] + for _ in range(6): + response, data = download(token, attachment['id']) + served.append(response.getheader('X-Instance-Id')) + ok = response.status == 200 and hashlib.sha256(data).hexdigest() == SAMPLE_SHA + print('download -> %d реплика=%s sha256 совпадает: %s' % (response.status, served[-1], ok)) + if not ok: + raise SystemExit('download failed') + print('остановленная реплика среди обслуживших:', target in served, '; %.2fs' % (time.monotonic() - started)) + print('Файл оригинала на диске обслужившей реплики (поиск по точному размеру %d байт):' % len(SAMPLE)) + for other in others: + found = sh('docker', 'exec', other, 'sh', '-c', 'find /tmp /app /home -xdev -type f -size %dc 2>/dev/null | grep -v node_modules || true' % len(SAMPLE)) + print(' %s -> %s' % (other, found or 'нет')) + sh('docker', 'start', target) + wait_ready() + time.sleep(3) + print('реплика %s возвращена' % target) + + +def interrupted(alice, chat): + header('3. Прерванная загрузка через LB: ни строки, ни объекта, ни временного файла') + before_rows = sql('SELECT count(*) FROM attachments') + before = reconcile('--grace-seconds=3600')['scanned'] + connection = socket.create_connection((HOST, PORT)) + connection.sendall(('POST /api/v1/chats/%s/attachments?filename=cut.png HTTP/1.1\r\nHost: lb\r\nAuthorization: Bearer %s\r\nContent-Type: image/png\r\nContent-Length: %d\r\n\r\n' % (chat, alice['token'], len(SAMPLE))).encode()) + connection.sendall(SAMPLE[:20000]) + time.sleep(0.5) + connection.close() + time.sleep(1) + after = reconcile('--grace-seconds=3600')['scanned'] + print('строк attachments: %s -> %s; объектов: %d -> %d' % (before_rows, sql('SELECT count(*) FROM attachments'), before, after)) + for cid in replicas(): + print('временных каталогов в %s: %s' % (cid, sh('docker', 'exec', cid, 'sh', '-c', 'ls /tmp | grep -c messenger-upload || true') or '0')) + + +def inject(alice, chat, title, body): + header(title) + token = alice['token'] + before = reconcile('--grace-seconds=3600')['scanned'] + rows = sql('SELECT count(*) FROM attachments') + sql("CREATE FUNCTION %s() RETURNS trigger AS $$ BEGIN %s RETURN NEW; END $$ LANGUAGE plpgsql; CREATE TRIGGER %s BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION %s();" % (FAULT_NAME, body, FAULT_NAME, FAULT_NAME)) + try: + response, data = upload(token, chat) + print('upload при отказе INSERT -> %d %s' % (response.status, json.loads(data)['error']['code'])) + finally: + sql('DROP TRIGGER %s ON attachments; DROP FUNCTION %s();' % (FAULT_NAME, FAULT_NAME)) + print('строк attachments: %s -> %s' % (rows, sql('SELECT count(*) FROM attachments'))) + fresh = reconcile('--apply', '--grace-seconds=3600') + print('объектов было %d, стало %d; свежий объект защищён grace: удалено %d' % (before, fresh['scanned'], fresh['deleted'])) + time.sleep(2) + cleaned = reconcile('--apply', '--grace-seconds=1') + print('после grace: сирот %d, удалено %d, привязанные объекты не тронуты: %d' % (cleaned['orphans'], cleaned['deleted'], cleaned['referenced'])) + response, data = upload(token, chat) + print('следующая загрузка без отказа ->', response.status) + wait_ready() + + +def main(): + client = Client('http://%s:%d' % (HOST, PORT)) + alice, chat = defence(client) + failover(alice, chat) + interrupted(alice, chat) + inject(alice, chat, '4. Реальный отказ PostgreSQL между PUT и INSERT: триггер отклоняет вставку', "RAISE EXCEPTION 'injected fault';") + inject(alice, chat, '5. Потеря соединения с PostgreSQL между PUT и INSERT: исход неясен, объект остаётся до сверки', 'PERFORM pg_terminate_backend(pg_backend_pid());') + + +if __name__ == '__main__': + main() diff --git a/scripts/collect-evidence.sh b/scripts/collect-evidence.sh index 4271e5f..0347f21 100755 --- a/scripts/collect-evidence.sh +++ b/scripts/collect-evidence.sh @@ -16,7 +16,7 @@ run() { } started() { - docker inspect -f '{{.Name}} {{.State.Status}} started={{.State.StartedAt}}' $(docker compose ps -aq postgres redis api lb) + docker inspect -f '{{.Name}} {{.State.Status}} started={{.State.StartedAt}}' $(docker compose ps -aq postgres redis s3 api lb) } api_states() { @@ -36,17 +36,21 @@ echo 'container.txt' header 'Стенд: сервисы, порты, лимиты, секреты, миграции' run docker compose ps -a --format 'table {{.Service}}\t{{.Name}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}' echo '## CPU/RAM-лимиты (NanoCpus: 1e9 = 1 CPU; Memory в байтах)' - run sh -c "docker inspect -f '{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}}' \$(docker compose ps -q)" + run sh -c "docker inspect -f '{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}}' \$(docker compose ps -aq)" echo '## container_name не задан (одно имя сервиса масштабируется)' run sh -c "docker compose config | grep -c container_name || true" echo '## Секреты: каждый сервис видит только свои файлы' - for service in postgres redis api; do + for service in postgres redis s3 api reconciler; do run docker compose exec -T "$service" ls /run/secrets done + run docker compose run --rm --no-deps -T --entrypoint ls s3-init /run/secrets run docker compose exec -T lb sh -c 'ls /run/secrets 2>/dev/null || echo "no /run/secrets"' run docker compose exec -T lb haproxy -v run docker compose exec -T postgres postgres --version run docker compose exec -T redis redis-server --version + run docker compose exec -T s3 minio --version + echo '## Таблицы вложений' + run docker compose exec -T postgres psql -U messenger -d messenger -c '\d attachments' -c '\d message_attachments' echo '## Миграции: отдельный one-shot сервис, повторный запуск ничего не применяет' run docker compose run --rm migrate run docker compose run --rm --no-deps migrate sh -c 'export DATABASE_URL="$(cat "$DATABASE_URL_FILE")"; npx prisma migrate diff --from-url "$DATABASE_URL" --to-schema-datamodel prisma/schema.prisma --exit-code' @@ -55,10 +59,10 @@ echo 'container.txt' echo 'tests.txt' { - header 'Собственные e2e-тесты (PostgreSQL + Redis из compose.test.yaml)' + header 'Собственные e2e-тесты (PostgreSQL + Redis + MinIO из compose.test.yaml)' docker compose -f compose.test.yaml build -q tests run docker compose -f compose.test.yaml run --rm tests npx jest --config ./test/jest-e2e.json --verbose - docker compose -f compose.test.yaml down >/dev/null 2>&1 + docker compose -f compose.test.yaml down -v >/dev/null 2>&1 } > evidence/tests.txt echo 'smoke.txt' @@ -72,7 +76,7 @@ echo 'smoke.txt' echo 'persistence.txt' { - header 'make test-persistence: restart.sh перезапускает postgres, redis, api, lb с сохранением volumes' + header 'make test-persistence: restart.sh перезапускает postgres, redis, s3, api, lb с сохранением volumes' echo '## до' started echo @@ -104,6 +108,32 @@ echo 'failover.txt' run env TARGET_INSTANCE_ID="$(docker compose ps -q postgres | cut -c1-12)" sh scripts/stop-one.sh } > evidence/failover.txt +echo 'attachments.txt' +{ + header 'Вложения: сценарий защиты, остановка загрузившей реплики, прерванная загрузка, отказ между S3 и БД' + run python3 scripts/attachment-scenario.py +} > evidence/attachments.txt + +echo 's3.txt' +{ + header 'S3: идемпотентный bootstrap, приватность мимо API, отказ S3, сверка' + echo '## Повторный bootstrap ничего не ломает' + run docker compose run --rm --no-deps -T s3-init + run docker compose run --rm --no-deps -T s3-init + run sh scripts/check-s3.sh + echo '## Отказ: контейнер S3 остановлен' + run sh scripts/s3-outage.sh stop + echo '## Отказ: контейнер S3 завис (docker pause)' + run sh scripts/s3-outage.sh pause + echo '## Сверка: объект без метаданных' + run sh scripts/reconcile-demo.sh + echo '## Журнал постоянного сервиса reconciler' + docker compose logs --no-log-prefix --tail 3 reconciler 2>&1 +} > evidence/s3.txt + +echo 'upload-memory.json' +python3 scripts/upload-experiment.py --output evidence/upload-memory.json >/dev/null 2>&1 + echo 'experiments: idempotency, failover, failover --kill, rolling' python3 scripts/experiment.py idempotency --output evidence/idempotency.json >/dev/null 2>&1 python3 scripts/experiment.py failover --output evidence/failover-stop.json >/dev/null 2>&1 @@ -142,7 +172,7 @@ echo 'failures.txt' docker compose exec -T postgres sh -c "cp $hba /tmp/pg_hba.bak && { echo 'host all all $ip/32 reject'; cat /tmp/pg_hba.bak; } > $hba && chown postgres:postgres $hba && chmod 600 $hba" docker compose exec -T postgres psql -U messenger -d messenger -tAc "SELECT pg_reload_conf(); SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE client_addr = '$ip'" >/dev/null sleep 4 - run docker exec "$target" node -e "let n=0;const g=p=>require('http').get('http://127.0.0.1:8080'+p,r=>{console.log(p,r.statusCode);r.resume();if(++n===2)process.exit(0)});g('/health/live');g('/health/ready')" + run docker exec "$target" node -e "let n=0;const g=p=>require('http').get('http://127.0.0.1:8080'+p,r=>{console.log(p,r.statusCode);r.resume();if(++n===2)process.exit(0)});g('/health/live');g('/health/core')" run python3 scripts/probe.py instances 10 docker compose logs --no-log-prefix --since "$since" lb 2>&1 | grep -E 'is DOWN|is UP' | grep -v '^\[WARNING\]' echo diff --git a/scripts/upload-experiment.py b/scripts/upload-experiment.py new file mode 100755 index 0000000..e91fcf3 --- /dev/null +++ b/scripts/upload-experiment.py @@ -0,0 +1,264 @@ +#!/usr/bin/env python3 +import argparse +import datetime +import hashlib +import http.client +import json +import math +import os +from pathlib import Path +import re +import struct +import subprocess +import sys +import threading +import time +import zlib + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client + +HOST = os.environ.get('LB_HOST', 'lb') +PORT = int(os.environ.get('LB_PORT', '8080')) +SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes() +MiB = 1024 * 1024 + + +def docker(*args, text=True): + return subprocess.run(['docker', *args], check=True, capture_output=True, text=text).stdout + + +def replicas(): + return [cid[:12] for cid in docker('compose', 'ps', '-q', 'api').split()] + + +def chunk(kind, data): + return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data) & 0xffffffff) + + +def padded_png(size): + return SAMPLE[:33] + chunk(b'abCd', b'\x01' * (size - len(SAMPLE) - 12)) + SAMPLE[33:] + + +def big_png(width, height): + rows = (b'\0' + b'\x55' * (width * 3)) * height + header = struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0) + return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', header) + chunk(b'IDAT', zlib.compress(rows, 9)) + chunk(b'IEND', b'') + + +def big_jpeg(width, height): + script = ( + "require('sharp')({create:{width:%d,height:%d,channels:3,background:'#336699'}})" + ".jpeg({quality:80}).toBuffer().then(b=>process.stdout.write(b))" % (width, height) + ) + return docker('exec', replicas()[0], 'node', '-e', script, text=False) + + +def cgroup_peak(cid): + raw = docker('exec', cid, 'sh', '-c', 'cat /sys/fs/cgroup/memory/memory.max_usage_in_bytes 2>/dev/null || cat /sys/fs/cgroup/memory.peak 2>/dev/null || echo 0') + return int(raw.strip()) // MiB + + +def percentile(values, q): + ordered = sorted(values) + return round(ordered[max(0, math.ceil(q * len(ordered)) - 1)], 3) + + +class Sampler: + def __init__(self, cids): + self.rss = {cid: 0 for cid in cids} + self.cpu = {} + self.stop_flag = threading.Event() + self.threads = [threading.Thread(target=self.watch, args=(cid,)) for cid in cids] + self.threads.append(threading.Thread(target=self.watch_cpu)) + + def watch(self, cid): + script = 'while :; do grep VmRSS /proc/1/status; sleep 0.2; done' + process = subprocess.Popen(['docker', 'exec', cid, 'sh', '-c', script], stdout=subprocess.PIPE, text=True) + for line in process.stdout: + if self.stop_flag.is_set(): + break + match = re.match(r'VmRSS:\s+(\d+)', line) + if match: + self.rss[cid] = max(self.rss[cid], int(match.group(1)) // 1024) + process.kill() + process.wait() + + def watch_cpu(self): + while not self.stop_flag.is_set(): + lines = docker('stats', '--no-stream', '--format', '{{.Name}} {{.CPUPerc}} {{.MemUsage}}') + for line in lines.splitlines(): + name, cpu, usage = line.split(' ', 2) + value = float(cpu.rstrip('%')) + entry = self.cpu.setdefault(name, {'max_cpu_percent': 0, 'last_mem': ''}) + entry['max_cpu_percent'] = max(entry['max_cpu_percent'], value) + entry['last_mem'] = usage.split(' / ')[0] + + def __enter__(self): + [thread.start() for thread in self.threads] + return self + + def __exit__(self, *_): + self.stop_flag.set() + for thread in self.threads: + thread.join(timeout=5) + + +def restart_replicas(): + docker('compose', 'restart', 'api') + deadline = time.monotonic() + 90 + expected = set(replicas()) + while time.monotonic() < deadline: + seen = set() + for _ in range(8): + try: + conn = http.client.HTTPConnection(HOST, PORT, timeout=3) + conn.request('GET', '/health/ready') + response = conn.getresponse() + response.read() + if response.status == 200: + seen.add(response.getheader('X-Instance-Id')) + conn.close() + except OSError: + pass + if seen >= expected: + return + time.sleep(0.5) + raise SystemExit('replicas did not return') + + +def upload(path, token, body, content_type): + conn = http.client.HTTPConnection(HOST, PORT, timeout=120) + started = time.monotonic() + conn.request('POST', path, body=body, headers={'Authorization': 'Bearer ' + token, 'Content-Type': content_type}) + response = conn.getresponse() + data = response.read() + elapsed = time.monotonic() - started + conn.close() + return response.status, elapsed, json.loads(data), response.getheader('X-Instance-Id') + + +def download_sha(attachment_id, token): + conn = http.client.HTTPConnection(HOST, PORT, timeout=60) + conn.request('GET', '/api/v1/attachments/' + attachment_id + '/content', headers={'Authorization': 'Bearer ' + token}) + response = conn.getresponse() + data = response.read() + conn.close() + return response.status, hashlib.sha256(data).hexdigest() + + +def run_profile(name, body, content_type, concurrency, rounds, token, chat): + restart_replicas() + cids = replicas() + idle = {cid: int(re.search(r'VmRSS:\s+(\d+)', docker('exec', cid, 'cat', '/proc/1/status')).group(1)) // 1024 for cid in cids} + path = '/api/v1/chats/' + chat + '/attachments?filename=load.bin' + expected_sha = hashlib.sha256(body).hexdigest() + results = [] + lock = threading.Lock() + + def worker(): + try: + outcome = upload(path, token, body, content_type) + except OSError as error: + outcome = (0, 0.0, {'error': {'code': type(error).__name__}}, None) + with lock: + results.append(outcome) + + with Sampler(cids) as sampler: + started = time.monotonic() + for _ in range(rounds): + threads = [threading.Thread(target=worker) for _ in range(concurrency)] + [thread.start() for thread in threads] + [thread.join() for thread in threads] + wall = time.monotonic() - started + time.sleep(0.5) + + created = [item for item in results if item[0] == 201] + latencies = [item[1] for item in created] + statuses = {} + for item in results: + statuses[str(item[0])] = statuses.get(str(item[0]), 0) + 1 + sha_ok = all(item[2]['sha256'] == expected_sha for item in created) + stored_ok = None + if created: + status, sha = download_sha(created[0][2]['id'], token) + stored_ok = status == 200 and sha == expected_sha + per_replica = {} + for cid in cids: + info = docker('exec', cid, 'cat', '/proc/1/status') + hwm = int(re.search(r'VmHWM:\s+(\d+)', info).group(1)) // 1024 + oom = docker('inspect', '-f', '{{.State.OOMKilled}}', cid).strip() + temp = docker('exec', cid, 'sh', '-c', 'ls /tmp | grep -c messenger-upload || true').strip() + per_replica[cid] = { + 'idle_rss_mib': idle[cid], + 'sampled_peak_rss_mib': sampler.rss[cid], + 'vm_hwm_mib': hwm, + 'cgroup_max_usage_with_page_cache_mib': cgroup_peak(cid), + 'oom_killed': oom, + 'temp_dirs_left': int(temp or 0), + 'served': sum(1 for item in results if item[3] == cid), + } + return { + 'profile': name, + 'body_bytes': len(body), + 'content_type': content_type, + 'concurrency': concurrency, + 'rounds': rounds, + 'requests': len(results), + 'statuses': statuses, + 'wall_seconds': round(wall, 2), + 'successful_rps': round(len(created) / wall, 2), + 'error_rate': round(1 - len(created) / len(results), 4), + 'latency_seconds': None if not latencies else { + 'p50': percentile(latencies, 0.5), + 'p95': percentile(latencies, 0.95), + 'p99': percentile(latencies, 0.99), + }, + 'sha256_in_responses_ok': sha_ok, + 'download_sha256_ok': stored_ok, + 'replicas': per_replica, + 'docker_stats': sampler.cpu, + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--output') + parser.add_argument('--rounds', type=int, default=3) + args = parser.parse_args() + client = Client('http://%s:%d' % (HOST, PORT)) + alice = client.user('upload_load') + chat = client.call('POST', '/api/v1/chats', 201, actor=alice, body={'title': 'upload experiment', 'member_ids': [alice['id']]}, schema='Chat')['json'] + token, chat_id = alice['token'], chat['id'] + jpeg_40mp = big_jpeg(8000, 5000) + png_40mp = big_png(8000, 5000) + profiles = [ + ('sample 112 KiB x10', SAMPLE, 'image/png', 10), + ('png 10 MiB x1', padded_png(10 * MiB), 'image/png', 1), + ('png 10 MiB x10', padded_png(10 * MiB), 'image/png', 10), + ('png 40 MP x10', png_40mp, 'image/png', 10), + ('jpeg 40 MP x10', jpeg_40mp, 'image/jpeg', 10), + ('png 10 MiB x20', padded_png(10 * MiB), 'image/png', 20), + ('png 10 MiB x40 (выше лимита 16 на реплику)', padded_png(10 * MiB), 'image/png', 40), + ] + results = [] + for name, body, content_type, concurrency in profiles: + print('profile:', name, flush=True) + results.append(run_profile(name, body, content_type, concurrency, args.rounds, token, chat_id)) + count = docker('compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-tAc', 'SELECT (SELECT count(*) FROM attachments), (SELECT count(*) FROM messages)').strip() + limits = docker('inspect', '-f', '{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}', replicas()[0]).split() + report = { + 'date': datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'), + 'api_replica_limits': {'cpus': int(limits[0]) / 1e9, 'memory_mib': int(limits[1]) // MiB}, + 'rows_after_attachments_messages': count, + 'results': results, + } + text = json.dumps(report, ensure_ascii=False, indent=2) + if args.output: + Path(args.output).write_text(text + '\n', encoding='utf-8') + print(text) + + +if __name__ == '__main__': + main() diff --git a/src/attachments/attachments.service.ts b/src/attachments/attachments.service.ts index 035d171..27f5dd1 100644 --- a/src/attachments/attachments.service.ts +++ b/src/attachments/attachments.service.ts @@ -137,6 +137,10 @@ export class AttachmentsService implements OnModuleInit { this.logger.warn(`compensation failed for ${objectKey}`); }); } + if (error instanceof Prisma.PrismaClientUnknownRequestError) { + this.logger.error(`metadata insert failed for ${objectKey}`); + throw new ApiError('unavailable', 'database unavailable'); + } throw error; } } diff --git a/test/attachments.e2e-spec.ts b/test/attachments.e2e-spec.ts index ec0cec8..7bdfb20 100644 --- a/test/attachments.e2e-spec.ts +++ b/test/attachments.e2e-spec.ts @@ -325,6 +325,24 @@ describe('Загрузка вложений (e2e)', () => { } }); + it('необработанная ошибка PostgreSQL при INSERT — 503, объект не удаляется', async () => { + const prisma = app.get(PrismaService); + const s3 = app.get(S3Service); + const remove = jest.spyOn(s3, 'delete'); + const create = jest.spyOn(prisma.attachment, 'create').mockRejectedValueOnce( + new Prisma.PrismaClientUnknownRequestError('server closed the connection', { + clientVersion: 'test', + }), + ); + try { + expectError(await upload(SAMPLE_PNG), 503, 'unavailable'); + expect(remove).not.toHaveBeenCalled(); + } finally { + create.mockRestore(); + remove.mockRestore(); + } + }); + it('при неясном исходе COMMIT объект не удаляется', async () => { const prisma = app.get(PrismaService); const s3 = app.get(S3Service);