forked from mtusi2026/mtusi2026lab5
feat: add operator replay, fault injection scenarios and staging cleanup
This commit is contained in:
@@ -227,6 +227,7 @@ services:
|
||||
timeout: 10s
|
||||
retries: 12
|
||||
start_period: 20s
|
||||
stop_grace_period: 30s
|
||||
depends_on:
|
||||
secrets-init:
|
||||
condition: service_completed_successfully
|
||||
|
||||
+213
-1
@@ -1,9 +1,11 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Defense scenarios for background attachment processing; run inside the tools container."""
|
||||
import atexit
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
@@ -14,6 +16,7 @@ import uuid
|
||||
BASE = os.environ.get('BASE_URL', 'http://localhost:8080')
|
||||
SAMPLE = open(os.path.join(os.path.dirname(__file__), '..', 'tests', 'fixtures', 'sample.png'), 'rb').read()
|
||||
STARTED = time.monotonic()
|
||||
ANSI = re.compile(chr(27) + re.escape('[') + '[0-9;]*m')
|
||||
|
||||
|
||||
def log(*parts):
|
||||
@@ -47,6 +50,13 @@ def compose(*args):
|
||||
subprocess.run(['docker', 'compose', *args], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
|
||||
|
||||
def start_only(service):
|
||||
"""docker compose start also starts dependencies (worker -> s3-init -> s3); start just this container."""
|
||||
log('$ docker start', service, '(without dependencies)')
|
||||
ids = subprocess.run(['docker', 'compose', 'ps', '-aq', service], check=True, capture_output=True, text=True).stdout.split()
|
||||
subprocess.run(['docker', 'start', *ids], check=True, stdout=subprocess.DEVNULL)
|
||||
|
||||
|
||||
def psql(sql):
|
||||
out = subprocess.run(
|
||||
['docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-tA', '-F', ' | ', '-c', sql],
|
||||
@@ -174,7 +184,209 @@ def scenario_restart():
|
||||
job_state(attachment)
|
||||
|
||||
|
||||
SCENARIOS = {'restart': scenario_restart}
|
||||
def remove_container(name):
|
||||
subprocess.run(['docker', 'rm', '-f', name], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
|
||||
|
||||
def plain(line):
|
||||
return ANSI.sub('', line).split('] ')[-1].strip()
|
||||
|
||||
|
||||
def faulty(service, env):
|
||||
"""Run a one-off copy of the service with fault injection; no restart policy, so it dies once."""
|
||||
name = 'lab5-fault-%s-%s' % (service, uuid.uuid4().hex[:8])
|
||||
atexit.register(remove_container, name)
|
||||
args = ['docker', 'compose', 'run', '-d', '--no-deps', '--name', name]
|
||||
for key, value in env.items():
|
||||
args += ['-e', '%s=%s' % (key, value)]
|
||||
log('$ docker compose run --no-deps', ' '.join('-e %s=%s' % item for item in env.items()), service)
|
||||
subprocess.run(args + [service], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
deadline = time.monotonic() + 30
|
||||
while time.monotonic() < deadline:
|
||||
if 'connected to broker' in subprocess.run(['docker', 'logs', name], capture_output=True, text=True).stdout:
|
||||
return name
|
||||
time.sleep(0.5)
|
||||
sys.exit('%s with fault did not start' % service)
|
||||
|
||||
|
||||
def wait_exit(name, timeout=90):
|
||||
try:
|
||||
code = subprocess.run(['docker', 'wait', name], capture_output=True, text=True, timeout=timeout).stdout.strip()
|
||||
except subprocess.TimeoutExpired:
|
||||
code = 'timeout'
|
||||
for line in subprocess.run(['docker', 'logs', name], capture_output=True, text=True).stdout.splitlines():
|
||||
if 'fault injection' in line or 'published' in line or 'job ' in line:
|
||||
log(' one-off:', plain(line))
|
||||
remove_container(name)
|
||||
log('one-off process exit code', code, '(137 = killed without ack)')
|
||||
|
||||
|
||||
def service_logs(service, since, needle):
|
||||
out = subprocess.run(['docker', 'compose', 'logs', '--no-log-prefix', '--no-color', '--since', since, service],
|
||||
capture_output=True, text=True).stdout
|
||||
for line in out.splitlines():
|
||||
if needle in line:
|
||||
log(' %s:' % service, plain(line))
|
||||
|
||||
|
||||
def utc_now():
|
||||
return time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
|
||||
|
||||
|
||||
def staging_key(attachment):
|
||||
return psql("SELECT coalesce(staging_key, 'NULL') FROM attachments WHERE id = '%s'" % attachment)
|
||||
|
||||
|
||||
def job_id(attachment):
|
||||
return psql("SELECT id FROM attachment_jobs WHERE attachment_id = '%s'" % attachment)
|
||||
|
||||
|
||||
def scenario_crash_after_store():
|
||||
"""Level 4: worker killed after writing S3 objects, before COMMIT and ack; lease expires; same result."""
|
||||
wait_ready()
|
||||
since = utc_now()
|
||||
auth, chat = session()
|
||||
compose('stop', 'worker')
|
||||
proc = faulty('worker', {'WORKER_FAULT': 'exit-after-store', 'JOB_LEASE_SECONDS': '15'})
|
||||
attachment = upload(auth, chat)
|
||||
wait_exit(proc)
|
||||
log('status after crash:', metadata(auth, attachment)[1]['status'])
|
||||
log('lease:', psql("SELECT status, attempt, lease_until > now() AS lease_active FROM attachment_jobs WHERE attachment_id = '%s'" % attachment))
|
||||
log('queues (unacked message returned):', queue_depth())
|
||||
compose('start', 'worker')
|
||||
meta = wait_status(auth, attachment, ['ready', 'failed'])
|
||||
check_ready(auth, attachment, meta)
|
||||
job_state(attachment)
|
||||
service_logs('worker', since, job_id(attachment))
|
||||
service_logs('relay', since, 'lease expired')
|
||||
|
||||
|
||||
def scenario_crash_after_commit():
|
||||
"""Level 4: worker killed after COMMIT ready, before ack; redelivery is a no-op and cleans staging."""
|
||||
wait_ready()
|
||||
since = utc_now()
|
||||
auth, chat = session()
|
||||
compose('stop', 'worker')
|
||||
proc = faulty('worker', {'WORKER_FAULT': 'exit-after-commit'})
|
||||
attachment = upload(auth, chat)
|
||||
wait_exit(proc)
|
||||
log('status after crash:', metadata(auth, attachment)[1]['status'], 'staging_key:', staging_key(attachment))
|
||||
log('queues (unacked message returned):', queue_depth())
|
||||
compose('start', 'worker')
|
||||
deadline = time.monotonic() + 30
|
||||
while staging_key(attachment) != 'NULL' and time.monotonic() < deadline:
|
||||
time.sleep(1)
|
||||
log('staging_key after redelivery:', staging_key(attachment))
|
||||
check_ready(auth, attachment, metadata(auth, attachment)[1])
|
||||
job_state(attachment)
|
||||
service_logs('worker', since, job_id(attachment))
|
||||
|
||||
|
||||
def scenario_outbox_gap():
|
||||
"""Level 4: crash between COMMIT and publish; the outbox row is published after recovery."""
|
||||
wait_ready()
|
||||
auth, chat = session()
|
||||
compose('stop', 'relay')
|
||||
attachment = upload(auth, chat)
|
||||
outbox = "SELECT o.id, coalesce(o.published_at::text, 'NOT PUBLISHED'), o.publish_attempts FROM outbox o JOIN attachment_jobs j ON j.id = o.job_id WHERE j.attachment_id = '%s'" % attachment
|
||||
log('outbox after COMMIT:', psql(outbox))
|
||||
log('queues:', queue_depth())
|
||||
compose('kill', 'api')
|
||||
log('API killed (SIGKILL) after COMMIT, nothing was published')
|
||||
compose('start', 'api')
|
||||
wait_ready()
|
||||
log('status after API restart:', metadata(auth, attachment)[1]['status'])
|
||||
compose('start', 'relay')
|
||||
meta = wait_status(auth, attachment, ['ready', 'failed'])
|
||||
log('outbox after relay recovery:', psql(outbox))
|
||||
check_ready(auth, attachment, meta)
|
||||
job_state(attachment)
|
||||
|
||||
|
||||
def scenario_relay_crash():
|
||||
"""Level 4: relay killed after broker confirm, before marking outbox; duplicate delivery is a no-op."""
|
||||
wait_ready()
|
||||
since = utc_now()
|
||||
auth, chat = session()
|
||||
compose('stop', 'relay')
|
||||
proc = faulty('relay', {'RELAY_FAULT': 'exit-after-confirm'})
|
||||
attachment = upload(auth, chat)
|
||||
wait_exit(proc)
|
||||
outbox = "SELECT o.id, coalesce(o.published_at::text, 'NOT MARKED'), o.publish_attempts FROM outbox o JOIN attachment_jobs j ON j.id = o.job_id WHERE j.attachment_id = '%s'" % attachment
|
||||
log('outbox after relay crash:', psql(outbox))
|
||||
compose('start', 'relay')
|
||||
meta = wait_status(auth, attachment, ['ready', 'failed'])
|
||||
time.sleep(3)
|
||||
log('outbox after relay restart:', psql(outbox))
|
||||
check_ready(auth, attachment, meta)
|
||||
job_state(attachment)
|
||||
service_logs('worker', since, job_id(attachment))
|
||||
|
||||
|
||||
def replay(*args):
|
||||
out = subprocess.run(['docker', 'compose', 'exec', '-T', '-e', 'OPERATOR=defense-demo', 'worker', 'node', 'dist/replay.js', *args],
|
||||
capture_output=True, text=True)
|
||||
for line in (out.stdout.strip() or out.stderr.strip()).splitlines():
|
||||
log(' replay:', line)
|
||||
|
||||
|
||||
def scenario_retry_replay():
|
||||
"""Level 4: S3 down -> retries with backoff and jitter -> failed retry_exhausted -> explicit replay -> ready."""
|
||||
wait_ready()
|
||||
auth, chat = session()
|
||||
compose('stop', 'worker')
|
||||
attachment = upload(auth, chat)
|
||||
compose('stop', 's3')
|
||||
start_only('worker')
|
||||
meta = wait_status(auth, attachment, ['failed'], timeout=120)
|
||||
log('failed attachment in a new message ->', message(auth, chat, '', [attachment]))
|
||||
log('retries:')
|
||||
for line in psql("SELECT e.attempt, e.event, coalesce(e.detail, ''), to_char(e.created_at, 'HH24:MI:SS.MS') FROM job_events e JOIN attachment_jobs j ON j.id = e.job_id WHERE j.attachment_id = '%s' ORDER BY e.id" % attachment).splitlines():
|
||||
log(' ', line)
|
||||
log('$ replay.js --list')
|
||||
replay('--list')
|
||||
compose('start', 's3')
|
||||
wait_ready()
|
||||
log('$ replay.js', attachment)
|
||||
replay(attachment)
|
||||
meta = wait_status(auth, attachment, ['ready', 'failed'])
|
||||
check_ready(auth, attachment, meta)
|
||||
log('$ replay.js', attachment, '(already ready)')
|
||||
replay(attachment)
|
||||
job_state(attachment)
|
||||
|
||||
|
||||
def scenario_broker():
|
||||
"""Level 4: prefetch/concurrency, limits and dead letter queue for a malformed message."""
|
||||
wait_ready()
|
||||
since = utc_now()
|
||||
log('consumers:', subprocess.run(
|
||||
['docker', 'compose', 'exec', '-T', 'rabbitmq', 'rabbitmqctl', '-q', 'list_consumers', 'queue_name', 'prefetch_count', 'ack_required'],
|
||||
capture_output=True, text=True).stdout.strip())
|
||||
log('worker limits:', subprocess.run(
|
||||
['sh', '-c', "docker inspect -f 'cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}}' $(docker compose ps -q worker)"],
|
||||
capture_output=True, text=True).stdout.strip())
|
||||
publish = (
|
||||
"const amqp=require('amqplib');const fs=require('fs');(async()=>{"
|
||||
"const c=await amqp.connect(fs.readFileSync(process.env.BROKER_URL_FILE,'utf8').trim());"
|
||||
"const ch=await c.createConfirmChannel();"
|
||||
"ch.sendToQueue('attachment.jobs',Buffer.from(JSON.stringify({job_id:'not-a-uuid',url:'http://example.invalid/x'})),{persistent:true});"
|
||||
"await ch.waitForConfirms();await c.close();console.log('malformed message published');})()"
|
||||
)
|
||||
log(subprocess.run(['docker', 'compose', 'exec', '-T', 'worker', 'node', '-e', publish], capture_output=True, text=True).stdout.strip())
|
||||
log('queues:', queue_depth())
|
||||
service_logs('worker', since, 'malformed')
|
||||
|
||||
|
||||
SCENARIOS = {
|
||||
'restart': scenario_restart,
|
||||
'crash-after-store': scenario_crash_after_store,
|
||||
'crash-after-commit': scenario_crash_after_commit,
|
||||
'outbox-gap': scenario_outbox_gap,
|
||||
'relay-crash': scenario_relay_crash,
|
||||
'retry-replay': scenario_retry_replay,
|
||||
'broker': scenario_broker,
|
||||
}
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) != 2 or sys.argv[1] not in SCENARIOS:
|
||||
|
||||
@@ -72,9 +72,8 @@ async function expireFailedStaging(
|
||||
): Promise<number> {
|
||||
const rows = await prisma.attachment.findMany({
|
||||
where: {
|
||||
status: 'failed',
|
||||
stagingKey: { not: null },
|
||||
updatedAt: { lt: new Date(before) },
|
||||
OR: [{ status: 'ready' }, { status: 'failed', updatedAt: { lt: new Date(before) } }],
|
||||
},
|
||||
select: { id: true, stagingKey: true },
|
||||
take: 500,
|
||||
|
||||
@@ -71,6 +71,9 @@ export class JobProcessor {
|
||||
}
|
||||
const lease = await this.claim(job);
|
||||
if (!lease) {
|
||||
if (job.status === 'ready') {
|
||||
await this.removeStaging(job.attachment);
|
||||
}
|
||||
this.logger.log(`job ${job.id}: ${job.status}, not due or leased elsewhere, skipped`);
|
||||
return 'skipped';
|
||||
}
|
||||
|
||||
+1
-2
@@ -39,8 +39,7 @@ async function publishBatch(channel: ConfirmChannel): Promise<number> {
|
||||
logger.log(`published ${rows.length} job message(s), broker confirmed`);
|
||||
if (FAULT === 'exit-after-confirm') {
|
||||
logger.warn('fault injection: exit after broker confirm, before outbox is marked');
|
||||
process.kill(process.pid, 'SIGKILL');
|
||||
await sleep(60000);
|
||||
process.exit(137);
|
||||
}
|
||||
await tx.outboxEntry.updateMany({
|
||||
where: { id: { in: rows.map((row) => row.id) } },
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { validate as isUuid } from './common/uuid';
|
||||
import { PrismaService } from './db/prisma.service';
|
||||
import { PIPELINE_VERSION } from './jobs/job-message';
|
||||
import { replayJob } from './jobs/job-store';
|
||||
|
||||
const USAGE = 'usage: replay.js --list | <job_id|attachment_id>...';
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const args = process.argv.slice(2);
|
||||
if (args.length === 0 || (args[0] !== '--list' && args.some((arg) => !isUuid(arg)))) {
|
||||
throw new Error(USAGE);
|
||||
}
|
||||
const prisma = new PrismaService();
|
||||
try {
|
||||
if (args[0] === '--list') {
|
||||
const failed = await prisma.attachmentJob.findMany({
|
||||
where: { status: 'failed' },
|
||||
orderBy: { finishedAt: 'asc' },
|
||||
take: 100,
|
||||
});
|
||||
for (const job of failed) {
|
||||
process.stdout.write(
|
||||
`${JSON.stringify({
|
||||
job_id: job.id,
|
||||
attachment_id: job.attachmentId,
|
||||
error_code: job.errorCode,
|
||||
attempt: job.attempt,
|
||||
last_error: job.lastError,
|
||||
finished_at: job.finishedAt?.toISOString(),
|
||||
})}\n`,
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
const actor = process.env.OPERATOR ?? 'operator';
|
||||
for (const id of args) {
|
||||
const job = await prisma.attachmentJob.findFirst({
|
||||
where: {
|
||||
OR: [{ id }, { attachmentId: id, pipelineVersion: PIPELINE_VERSION }],
|
||||
},
|
||||
});
|
||||
const result = job ? await replayJob(prisma, job.id, actor) : 'not_found';
|
||||
process.stdout.write(`${JSON.stringify({ id, job_id: job?.id ?? null, result })}\n`);
|
||||
}
|
||||
} finally {
|
||||
await prisma.$disconnect();
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
Logger.error(error instanceof Error ? error.message : 'fatal error', 'Replay');
|
||||
process.exit(1);
|
||||
});
|
||||
+1
-2
@@ -21,8 +21,7 @@ let inFlight = 0;
|
||||
function faultHooks(): ProcessorHooks {
|
||||
const crash = (point: string) => async () => {
|
||||
logger.warn(`fault injection: worker killed ${point}`);
|
||||
process.kill(process.pid, 'SIGKILL');
|
||||
await sleep(60000);
|
||||
process.exit(137);
|
||||
};
|
||||
if (FAULT === 'exit-after-store') {
|
||||
return { afterStore: crash('after storing objects, before commit and ack') };
|
||||
|
||||
+32
-1
@@ -7,7 +7,7 @@ import { PrismaService } from '../src/db/prisma.service';
|
||||
import { jobMessage, parseJobMessage } from '../src/jobs/job-message';
|
||||
import { backoffMs, recoverExpiredLeases, replayJob, republishStale } from '../src/jobs/job-store';
|
||||
import { JobProcessor, processorOptions } from '../src/jobs/processor';
|
||||
import { fitInside } from '../src/jobs/variants';
|
||||
import { fitInside, InvalidImageError, renderVariants } from '../src/jobs/variants';
|
||||
import { S3Service } from '../src/storage/s3.service';
|
||||
import {
|
||||
bearer,
|
||||
@@ -172,6 +172,14 @@ describe('Фоновая обработка вложений (e2e)', () => {
|
||||
expect([thumbnail.width, thumbnail.height]).toEqual([160, 256]);
|
||||
});
|
||||
|
||||
it('worker ограничивает пиксели до распаковки: превышение — InvalidImageError', async () => {
|
||||
const image = await sharp({ create: { width: 300, height: 200, channels: 3, background: '#000' } })
|
||||
.png()
|
||||
.toBuffer();
|
||||
await expect(renderVariants(image, 300 * 200 - 1)).rejects.toBeInstanceOf(InvalidImageError);
|
||||
await expect(renderVariants(image, 300 * 200)).resolves.toHaveLength(3);
|
||||
});
|
||||
|
||||
it('размеры округляются вниз, минимум 1 px', () => {
|
||||
expect(fitInside(320, 200, 256)).toEqual({ width: 256, height: 160 });
|
||||
expect(fitInside(1000, 3, 256)).toEqual({ width: 256, height: 1 });
|
||||
@@ -235,6 +243,29 @@ describe('Фоновая обработка вложений (e2e)', () => {
|
||||
expect(await events(id)).toEqual(['enqueued', 'processing', 'lease_expired', 'processing', 'ready']);
|
||||
});
|
||||
|
||||
it('worker упал после COMMIT, до ack: повтор — no-op, staging убирается при повторной доставке', async () => {
|
||||
const id = await upload();
|
||||
await expect(
|
||||
processAttachment(app, id, {
|
||||
afterCommit: async () => {
|
||||
throw new Error('worker crashed');
|
||||
},
|
||||
}),
|
||||
).rejects.toThrow('worker crashed');
|
||||
expect((await metadata(id)).status).toBe('ready');
|
||||
expect((await prisma.attachment.findUniqueOrThrow({ where: { id } })).stagingKey).toBe(`staging/${id}`);
|
||||
const put = jest.spyOn(s3, 'putBuffer');
|
||||
try {
|
||||
expect(await processAttachment(app, id)).toBe('skipped');
|
||||
expect(put).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
put.mockRestore();
|
||||
}
|
||||
expect((await prisma.attachment.findUniqueOrThrow({ where: { id } })).stagingKey).toBeNull();
|
||||
expect(await prisma.attachmentVariant.count({ where: { attachmentId: id } })).toBe(3);
|
||||
expect(await events(id)).toEqual(['enqueued', 'processing', 'ready']);
|
||||
});
|
||||
|
||||
it('сообщение потеряно брокером: queued без свежей публикации публикуется повторно один раз', async () => {
|
||||
const id = await upload();
|
||||
const row = await job(id);
|
||||
|
||||
@@ -241,6 +241,35 @@ describe('Сверка объектов S3 с метаданными (e2e)', ()
|
||||
const row = await prisma.attachment.findUniqueOrThrow({ where: { id: res.body.id } });
|
||||
expect(row.stagingKey).toBeNull();
|
||||
});
|
||||
|
||||
it('staging ready-вложения, оставшийся после сбоя worker, удаляется без ожидания срока хранения', async () => {
|
||||
const res = await request(server)
|
||||
.post(`/api/v1/chats/${chatId}/attachments?filename=a.png`)
|
||||
.set('Authorization', bearer(alice))
|
||||
.set('Content-Type', 'image/png')
|
||||
.send(SAMPLE_PNG)
|
||||
.expect(202);
|
||||
await expect(
|
||||
processAttachment(app, res.body.id, {
|
||||
afterCommit: async () => {
|
||||
throw new Error('worker crashed');
|
||||
},
|
||||
}),
|
||||
).rejects.toThrow('worker crashed');
|
||||
expect(await exists(`staging/${res.body.id}`)).toBe(true);
|
||||
const report = await reconcile(prisma, s3, {
|
||||
apply: true,
|
||||
graceMs: HOUR_MS,
|
||||
prefix: `staging/${res.body.id}`,
|
||||
stagingRetentionMs: 24 * HOUR_MS,
|
||||
});
|
||||
expect(report.staging_expired).toBeGreaterThanOrEqual(1);
|
||||
expect(await exists(`staging/${res.body.id}`)).toBe(false);
|
||||
await request(server)
|
||||
.get(`/api/v1/attachments/${res.body.id}/content`)
|
||||
.set('Authorization', bearer(alice))
|
||||
.expect(200);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Готовность при отказе S3 (e2e)', () => {
|
||||
|
||||
Reference in New Issue
Block a user