From 3e0033fc79fda6b9d32dd0468a4147ef6d5b658c Mon Sep 17 00:00:00 2001 From: aegor Date: Mon, 16 Nov 2026 14:02:27 +0300 Subject: [PATCH] feat: add operator replay, fault injection scenarios and staging cleanup --- compose.yaml | 1 + scripts/job-scenario.py | 214 +++++++++++++++++++++++++++++++++- src/attachments/reconciler.ts | 3 +- src/jobs/processor.ts | 3 + src/relay.ts | 3 +- src/replay.ts | 54 +++++++++ src/worker.ts | 3 +- test/jobs.e2e-spec.ts | 33 +++++- test/reconciler.e2e-spec.ts | 29 +++++ 9 files changed, 335 insertions(+), 8 deletions(-) create mode 100644 src/replay.ts diff --git a/compose.yaml b/compose.yaml index e2aa7224..3702fff6 100644 --- a/compose.yaml +++ b/compose.yaml @@ -227,6 +227,7 @@ services: timeout: 10s retries: 12 start_period: 20s + stop_grace_period: 30s depends_on: secrets-init: condition: service_completed_successfully diff --git a/scripts/job-scenario.py b/scripts/job-scenario.py index ea3abd7a..9fb6c360 100644 --- a/scripts/job-scenario.py +++ b/scripts/job-scenario.py @@ -1,9 +1,11 @@ #!/usr/bin/env python3 """Defense scenarios for background attachment processing; run inside the tools container.""" +import atexit import base64 import hashlib import json import os +import re import subprocess import sys import time @@ -14,6 +16,7 @@ import uuid BASE = os.environ.get('BASE_URL', 'http://localhost:8080') SAMPLE = open(os.path.join(os.path.dirname(__file__), '..', 'tests', 'fixtures', 'sample.png'), 'rb').read() STARTED = time.monotonic() +ANSI = re.compile(chr(27) + re.escape('[') + '[0-9;]*m') def log(*parts): @@ -47,6 +50,13 @@ def compose(*args): subprocess.run(['docker', 'compose', *args], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) +def start_only(service): + """docker compose start also starts dependencies (worker -> s3-init -> s3); start just this container.""" + log('$ docker start', service, '(without dependencies)') + ids = subprocess.run(['docker', 'compose', 'ps', '-aq', service], check=True, capture_output=True, text=True).stdout.split() + subprocess.run(['docker', 'start', *ids], check=True, stdout=subprocess.DEVNULL) + + def psql(sql): out = subprocess.run( ['docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-tA', '-F', ' | ', '-c', sql], @@ -174,7 +184,209 @@ def scenario_restart(): job_state(attachment) -SCENARIOS = {'restart': scenario_restart} +def remove_container(name): + subprocess.run(['docker', 'rm', '-f', name], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +def plain(line): + return ANSI.sub('', line).split('] ')[-1].strip() + + +def faulty(service, env): + """Run a one-off copy of the service with fault injection; no restart policy, so it dies once.""" + name = 'lab5-fault-%s-%s' % (service, uuid.uuid4().hex[:8]) + atexit.register(remove_container, name) + args = ['docker', 'compose', 'run', '-d', '--no-deps', '--name', name] + for key, value in env.items(): + args += ['-e', '%s=%s' % (key, value)] + log('$ docker compose run --no-deps', ' '.join('-e %s=%s' % item for item in env.items()), service) + subprocess.run(args + [service], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + if 'connected to broker' in subprocess.run(['docker', 'logs', name], capture_output=True, text=True).stdout: + return name + time.sleep(0.5) + sys.exit('%s with fault did not start' % service) + + +def wait_exit(name, timeout=90): + try: + code = subprocess.run(['docker', 'wait', name], capture_output=True, text=True, timeout=timeout).stdout.strip() + except subprocess.TimeoutExpired: + code = 'timeout' + for line in subprocess.run(['docker', 'logs', name], capture_output=True, text=True).stdout.splitlines(): + if 'fault injection' in line or 'published' in line or 'job ' in line: + log(' one-off:', plain(line)) + remove_container(name) + log('one-off process exit code', code, '(137 = killed without ack)') + + +def service_logs(service, since, needle): + out = subprocess.run(['docker', 'compose', 'logs', '--no-log-prefix', '--no-color', '--since', since, service], + capture_output=True, text=True).stdout + for line in out.splitlines(): + if needle in line: + log(' %s:' % service, plain(line)) + + +def utc_now(): + return time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) + + +def staging_key(attachment): + return psql("SELECT coalesce(staging_key, 'NULL') FROM attachments WHERE id = '%s'" % attachment) + + +def job_id(attachment): + return psql("SELECT id FROM attachment_jobs WHERE attachment_id = '%s'" % attachment) + + +def scenario_crash_after_store(): + """Level 4: worker killed after writing S3 objects, before COMMIT and ack; lease expires; same result.""" + wait_ready() + since = utc_now() + auth, chat = session() + compose('stop', 'worker') + proc = faulty('worker', {'WORKER_FAULT': 'exit-after-store', 'JOB_LEASE_SECONDS': '15'}) + attachment = upload(auth, chat) + wait_exit(proc) + log('status after crash:', metadata(auth, attachment)[1]['status']) + log('lease:', psql("SELECT status, attempt, lease_until > now() AS lease_active FROM attachment_jobs WHERE attachment_id = '%s'" % attachment)) + log('queues (unacked message returned):', queue_depth()) + compose('start', 'worker') + meta = wait_status(auth, attachment, ['ready', 'failed']) + check_ready(auth, attachment, meta) + job_state(attachment) + service_logs('worker', since, job_id(attachment)) + service_logs('relay', since, 'lease expired') + + +def scenario_crash_after_commit(): + """Level 4: worker killed after COMMIT ready, before ack; redelivery is a no-op and cleans staging.""" + wait_ready() + since = utc_now() + auth, chat = session() + compose('stop', 'worker') + proc = faulty('worker', {'WORKER_FAULT': 'exit-after-commit'}) + attachment = upload(auth, chat) + wait_exit(proc) + log('status after crash:', metadata(auth, attachment)[1]['status'], 'staging_key:', staging_key(attachment)) + log('queues (unacked message returned):', queue_depth()) + compose('start', 'worker') + deadline = time.monotonic() + 30 + while staging_key(attachment) != 'NULL' and time.monotonic() < deadline: + time.sleep(1) + log('staging_key after redelivery:', staging_key(attachment)) + check_ready(auth, attachment, metadata(auth, attachment)[1]) + job_state(attachment) + service_logs('worker', since, job_id(attachment)) + + +def scenario_outbox_gap(): + """Level 4: crash between COMMIT and publish; the outbox row is published after recovery.""" + wait_ready() + auth, chat = session() + compose('stop', 'relay') + attachment = upload(auth, chat) + outbox = "SELECT o.id, coalesce(o.published_at::text, 'NOT PUBLISHED'), o.publish_attempts FROM outbox o JOIN attachment_jobs j ON j.id = o.job_id WHERE j.attachment_id = '%s'" % attachment + log('outbox after COMMIT:', psql(outbox)) + log('queues:', queue_depth()) + compose('kill', 'api') + log('API killed (SIGKILL) after COMMIT, nothing was published') + compose('start', 'api') + wait_ready() + log('status after API restart:', metadata(auth, attachment)[1]['status']) + compose('start', 'relay') + meta = wait_status(auth, attachment, ['ready', 'failed']) + log('outbox after relay recovery:', psql(outbox)) + check_ready(auth, attachment, meta) + job_state(attachment) + + +def scenario_relay_crash(): + """Level 4: relay killed after broker confirm, before marking outbox; duplicate delivery is a no-op.""" + wait_ready() + since = utc_now() + auth, chat = session() + compose('stop', 'relay') + proc = faulty('relay', {'RELAY_FAULT': 'exit-after-confirm'}) + attachment = upload(auth, chat) + wait_exit(proc) + outbox = "SELECT o.id, coalesce(o.published_at::text, 'NOT MARKED'), o.publish_attempts FROM outbox o JOIN attachment_jobs j ON j.id = o.job_id WHERE j.attachment_id = '%s'" % attachment + log('outbox after relay crash:', psql(outbox)) + compose('start', 'relay') + meta = wait_status(auth, attachment, ['ready', 'failed']) + time.sleep(3) + log('outbox after relay restart:', psql(outbox)) + check_ready(auth, attachment, meta) + job_state(attachment) + service_logs('worker', since, job_id(attachment)) + + +def replay(*args): + out = subprocess.run(['docker', 'compose', 'exec', '-T', '-e', 'OPERATOR=defense-demo', 'worker', 'node', 'dist/replay.js', *args], + capture_output=True, text=True) + for line in (out.stdout.strip() or out.stderr.strip()).splitlines(): + log(' replay:', line) + + +def scenario_retry_replay(): + """Level 4: S3 down -> retries with backoff and jitter -> failed retry_exhausted -> explicit replay -> ready.""" + wait_ready() + auth, chat = session() + compose('stop', 'worker') + attachment = upload(auth, chat) + compose('stop', 's3') + start_only('worker') + meta = wait_status(auth, attachment, ['failed'], timeout=120) + log('failed attachment in a new message ->', message(auth, chat, '', [attachment])) + log('retries:') + for line in psql("SELECT e.attempt, e.event, coalesce(e.detail, ''), to_char(e.created_at, 'HH24:MI:SS.MS') FROM job_events e JOIN attachment_jobs j ON j.id = e.job_id WHERE j.attachment_id = '%s' ORDER BY e.id" % attachment).splitlines(): + log(' ', line) + log('$ replay.js --list') + replay('--list') + compose('start', 's3') + wait_ready() + log('$ replay.js', attachment) + replay(attachment) + meta = wait_status(auth, attachment, ['ready', 'failed']) + check_ready(auth, attachment, meta) + log('$ replay.js', attachment, '(already ready)') + replay(attachment) + job_state(attachment) + + +def scenario_broker(): + """Level 4: prefetch/concurrency, limits and dead letter queue for a malformed message.""" + wait_ready() + since = utc_now() + log('consumers:', subprocess.run( + ['docker', 'compose', 'exec', '-T', 'rabbitmq', 'rabbitmqctl', '-q', 'list_consumers', 'queue_name', 'prefetch_count', 'ack_required'], + capture_output=True, text=True).stdout.strip()) + log('worker limits:', subprocess.run( + ['sh', '-c', "docker inspect -f 'cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}}' $(docker compose ps -q worker)"], + capture_output=True, text=True).stdout.strip()) + publish = ( + "const amqp=require('amqplib');const fs=require('fs');(async()=>{" + "const c=await amqp.connect(fs.readFileSync(process.env.BROKER_URL_FILE,'utf8').trim());" + "const ch=await c.createConfirmChannel();" + "ch.sendToQueue('attachment.jobs',Buffer.from(JSON.stringify({job_id:'not-a-uuid',url:'http://example.invalid/x'})),{persistent:true});" + "await ch.waitForConfirms();await c.close();console.log('malformed message published');})()" + ) + log(subprocess.run(['docker', 'compose', 'exec', '-T', 'worker', 'node', '-e', publish], capture_output=True, text=True).stdout.strip()) + log('queues:', queue_depth()) + service_logs('worker', since, 'malformed') + + +SCENARIOS = { + 'restart': scenario_restart, + 'crash-after-store': scenario_crash_after_store, + 'crash-after-commit': scenario_crash_after_commit, + 'outbox-gap': scenario_outbox_gap, + 'relay-crash': scenario_relay_crash, + 'retry-replay': scenario_retry_replay, + 'broker': scenario_broker, +} if __name__ == '__main__': if len(sys.argv) != 2 or sys.argv[1] not in SCENARIOS: diff --git a/src/attachments/reconciler.ts b/src/attachments/reconciler.ts index f6116d2e..e164e5e0 100644 --- a/src/attachments/reconciler.ts +++ b/src/attachments/reconciler.ts @@ -72,9 +72,8 @@ async function expireFailedStaging( ): Promise { const rows = await prisma.attachment.findMany({ where: { - status: 'failed', stagingKey: { not: null }, - updatedAt: { lt: new Date(before) }, + OR: [{ status: 'ready' }, { status: 'failed', updatedAt: { lt: new Date(before) } }], }, select: { id: true, stagingKey: true }, take: 500, diff --git a/src/jobs/processor.ts b/src/jobs/processor.ts index 13ceea2d..8d9603a5 100644 --- a/src/jobs/processor.ts +++ b/src/jobs/processor.ts @@ -71,6 +71,9 @@ export class JobProcessor { } const lease = await this.claim(job); if (!lease) { + if (job.status === 'ready') { + await this.removeStaging(job.attachment); + } this.logger.log(`job ${job.id}: ${job.status}, not due or leased elsewhere, skipped`); return 'skipped'; } diff --git a/src/relay.ts b/src/relay.ts index 082d6b98..5f549879 100644 --- a/src/relay.ts +++ b/src/relay.ts @@ -39,8 +39,7 @@ async function publishBatch(channel: ConfirmChannel): Promise { logger.log(`published ${rows.length} job message(s), broker confirmed`); if (FAULT === 'exit-after-confirm') { logger.warn('fault injection: exit after broker confirm, before outbox is marked'); - process.kill(process.pid, 'SIGKILL'); - await sleep(60000); + process.exit(137); } await tx.outboxEntry.updateMany({ where: { id: { in: rows.map((row) => row.id) } }, diff --git a/src/replay.ts b/src/replay.ts new file mode 100644 index 00000000..30148a8b --- /dev/null +++ b/src/replay.ts @@ -0,0 +1,54 @@ +import { Logger } from '@nestjs/common'; +import { validate as isUuid } from './common/uuid'; +import { PrismaService } from './db/prisma.service'; +import { PIPELINE_VERSION } from './jobs/job-message'; +import { replayJob } from './jobs/job-store'; + +const USAGE = 'usage: replay.js --list | ...'; + +async function main(): Promise { + const args = process.argv.slice(2); + if (args.length === 0 || (args[0] !== '--list' && args.some((arg) => !isUuid(arg)))) { + throw new Error(USAGE); + } + const prisma = new PrismaService(); + try { + if (args[0] === '--list') { + const failed = await prisma.attachmentJob.findMany({ + where: { status: 'failed' }, + orderBy: { finishedAt: 'asc' }, + take: 100, + }); + for (const job of failed) { + process.stdout.write( + `${JSON.stringify({ + job_id: job.id, + attachment_id: job.attachmentId, + error_code: job.errorCode, + attempt: job.attempt, + last_error: job.lastError, + finished_at: job.finishedAt?.toISOString(), + })}\n`, + ); + } + return; + } + const actor = process.env.OPERATOR ?? 'operator'; + for (const id of args) { + const job = await prisma.attachmentJob.findFirst({ + where: { + OR: [{ id }, { attachmentId: id, pipelineVersion: PIPELINE_VERSION }], + }, + }); + const result = job ? await replayJob(prisma, job.id, actor) : 'not_found'; + process.stdout.write(`${JSON.stringify({ id, job_id: job?.id ?? null, result })}\n`); + } + } finally { + await prisma.$disconnect(); + } +} + +main().catch((error) => { + Logger.error(error instanceof Error ? error.message : 'fatal error', 'Replay'); + process.exit(1); +}); diff --git a/src/worker.ts b/src/worker.ts index 079a1a25..cec9146f 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -21,8 +21,7 @@ let inFlight = 0; function faultHooks(): ProcessorHooks { const crash = (point: string) => async () => { logger.warn(`fault injection: worker killed ${point}`); - process.kill(process.pid, 'SIGKILL'); - await sleep(60000); + process.exit(137); }; if (FAULT === 'exit-after-store') { return { afterStore: crash('after storing objects, before commit and ack') }; diff --git a/test/jobs.e2e-spec.ts b/test/jobs.e2e-spec.ts index 85573e0d..fd8af118 100644 --- a/test/jobs.e2e-spec.ts +++ b/test/jobs.e2e-spec.ts @@ -7,7 +7,7 @@ import { PrismaService } from '../src/db/prisma.service'; import { jobMessage, parseJobMessage } from '../src/jobs/job-message'; import { backoffMs, recoverExpiredLeases, replayJob, republishStale } from '../src/jobs/job-store'; import { JobProcessor, processorOptions } from '../src/jobs/processor'; -import { fitInside } from '../src/jobs/variants'; +import { fitInside, InvalidImageError, renderVariants } from '../src/jobs/variants'; import { S3Service } from '../src/storage/s3.service'; import { bearer, @@ -172,6 +172,14 @@ describe('Фоновая обработка вложений (e2e)', () => { expect([thumbnail.width, thumbnail.height]).toEqual([160, 256]); }); + it('worker ограничивает пиксели до распаковки: превышение — InvalidImageError', async () => { + const image = await sharp({ create: { width: 300, height: 200, channels: 3, background: '#000' } }) + .png() + .toBuffer(); + await expect(renderVariants(image, 300 * 200 - 1)).rejects.toBeInstanceOf(InvalidImageError); + await expect(renderVariants(image, 300 * 200)).resolves.toHaveLength(3); + }); + it('размеры округляются вниз, минимум 1 px', () => { expect(fitInside(320, 200, 256)).toEqual({ width: 256, height: 160 }); expect(fitInside(1000, 3, 256)).toEqual({ width: 256, height: 1 }); @@ -235,6 +243,29 @@ describe('Фоновая обработка вложений (e2e)', () => { expect(await events(id)).toEqual(['enqueued', 'processing', 'lease_expired', 'processing', 'ready']); }); + it('worker упал после COMMIT, до ack: повтор — no-op, staging убирается при повторной доставке', async () => { + const id = await upload(); + await expect( + processAttachment(app, id, { + afterCommit: async () => { + throw new Error('worker crashed'); + }, + }), + ).rejects.toThrow('worker crashed'); + expect((await metadata(id)).status).toBe('ready'); + expect((await prisma.attachment.findUniqueOrThrow({ where: { id } })).stagingKey).toBe(`staging/${id}`); + const put = jest.spyOn(s3, 'putBuffer'); + try { + expect(await processAttachment(app, id)).toBe('skipped'); + expect(put).not.toHaveBeenCalled(); + } finally { + put.mockRestore(); + } + expect((await prisma.attachment.findUniqueOrThrow({ where: { id } })).stagingKey).toBeNull(); + expect(await prisma.attachmentVariant.count({ where: { attachmentId: id } })).toBe(3); + expect(await events(id)).toEqual(['enqueued', 'processing', 'ready']); + }); + it('сообщение потеряно брокером: queued без свежей публикации публикуется повторно один раз', async () => { const id = await upload(); const row = await job(id); diff --git a/test/reconciler.e2e-spec.ts b/test/reconciler.e2e-spec.ts index abb41153..0c12f53a 100644 --- a/test/reconciler.e2e-spec.ts +++ b/test/reconciler.e2e-spec.ts @@ -241,6 +241,35 @@ describe('Сверка объектов S3 с метаданными (e2e)', () const row = await prisma.attachment.findUniqueOrThrow({ where: { id: res.body.id } }); expect(row.stagingKey).toBeNull(); }); + + it('staging ready-вложения, оставшийся после сбоя worker, удаляется без ожидания срока хранения', async () => { + const res = await request(server) + .post(`/api/v1/chats/${chatId}/attachments?filename=a.png`) + .set('Authorization', bearer(alice)) + .set('Content-Type', 'image/png') + .send(SAMPLE_PNG) + .expect(202); + await expect( + processAttachment(app, res.body.id, { + afterCommit: async () => { + throw new Error('worker crashed'); + }, + }), + ).rejects.toThrow('worker crashed'); + expect(await exists(`staging/${res.body.id}`)).toBe(true); + const report = await reconcile(prisma, s3, { + apply: true, + graceMs: HOUR_MS, + prefix: `staging/${res.body.id}`, + stagingRetentionMs: 24 * HOUR_MS, + }); + expect(report.staging_expired).toBeGreaterThanOrEqual(1); + expect(await exists(`staging/${res.body.id}`)).toBe(false); + await request(server) + .get(`/api/v1/attachments/${res.body.id}/content`) + .set('Authorization', bearer(alice)) + .expect(200); + }); }); describe('Готовность при отказе S3 (e2e)', () => {