From d19824f5467971e8e0531b411879271fec34d936 Mon Sep 17 00:00:00 2001 From: Ilya Date: Thu, 5 Nov 2026 01:17:58 +0300 Subject: [PATCH] =?UTF-8?q?lab5:=20=D0=B1=D1=80=D0=BE=D0=BA=D0=B5=D1=80=20?= =?UTF-8?q?RabbitMQ,=20worker=20=D0=B8=20=D0=B0=D1=81=D0=B8=D0=BD=D1=85?= =?UTF-8?q?=D1=80=D0=BE=D0=BD=D0=BD=D0=B0=D1=8F=20=D0=BE=D0=B1=D1=80=D0=B0?= =?UTF-8?q?=D0=B1=D0=BE=D1=82=D0=BA=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 27 +- app/package-lock.json | 10 + app/package.json | 6 +- .../attachments/attachment-sweeper.service.ts | 4 +- app/src/attachments/attachment.ts | 76 +- app/src/attachments/attachments.controller.ts | 13 +- app/src/attachments/attachments.module.ts | 6 + app/src/attachments/attachments.repository.ts | 66 +- .../attachments/attachments.service.spec.ts | 62 +- app/src/attachments/attachments.service.ts | 102 +- app/src/attachments/direct-uploads.service.ts | 79 +- .../attachments/image/image-decoder.spec.ts | 2 +- app/src/attachments/image/image-decoder.ts | 2 +- app/src/config/app-config.schema.ts | 38 + .../migrations/007_attachment_pipeline.sql | 81 ++ app/src/jobs/backoff.spec.ts | 19 + app/src/jobs/backoff.ts | 14 + app/src/jobs/broker.client.ts | 259 +++++ app/src/jobs/broker.topology.ts | 37 + app/src/jobs/errors.spec.ts | 33 + app/src/jobs/errors.ts | 65 ++ app/src/jobs/fault-injector.ts | 131 +++ app/src/jobs/fit-inside.spec.ts | 14 + app/src/jobs/fit-inside.ts | 15 + app/src/jobs/job-message.spec.ts | 47 + app/src/jobs/job-message.ts | 53 + app/src/jobs/jobs.cli.ts | 234 +++++ app/src/jobs/jobs.module.ts | 14 + app/src/jobs/jobs.repository.ts | 910 ++++++++++++++++++ app/src/jobs/outbox-relay.service.ts | 92 ++ app/src/messages/messages.repository.ts | 3 +- app/src/messages/messages.service.ts | 15 +- app/src/storage/object-storage.ts | 48 +- app/src/worker.main.ts | 67 ++ app/src/worker/job-processor.service.ts | 405 ++++++++ app/src/worker/lease-reaper.service.ts | 64 ++ app/src/worker/pipelines.spec.ts | 90 ++ app/src/worker/pipelines.ts | 142 +++ app/src/worker/staging-cleaner.service.ts | 84 ++ app/src/worker/worker.module.ts | 23 + compose.yaml | 107 +- infra/minio/bootstrap.sh | 48 +- scripts/init-env.sh | 25 + scripts/restart.sh | 4 +- 44 files changed, 3486 insertions(+), 150 deletions(-) create mode 100644 app/src/database/migrations/007_attachment_pipeline.sql create mode 100644 app/src/jobs/backoff.spec.ts create mode 100644 app/src/jobs/backoff.ts create mode 100644 app/src/jobs/broker.client.ts create mode 100644 app/src/jobs/broker.topology.ts create mode 100644 app/src/jobs/errors.spec.ts create mode 100644 app/src/jobs/errors.ts create mode 100644 app/src/jobs/fault-injector.ts create mode 100644 app/src/jobs/fit-inside.spec.ts create mode 100644 app/src/jobs/fit-inside.ts create mode 100644 app/src/jobs/job-message.spec.ts create mode 100644 app/src/jobs/job-message.ts create mode 100644 app/src/jobs/jobs.cli.ts create mode 100644 app/src/jobs/jobs.module.ts create mode 100644 app/src/jobs/jobs.repository.ts create mode 100644 app/src/jobs/outbox-relay.service.ts create mode 100644 app/src/worker.main.ts create mode 100644 app/src/worker/job-processor.service.ts create mode 100644 app/src/worker/lease-reaper.service.ts create mode 100644 app/src/worker/pipelines.spec.ts create mode 100644 app/src/worker/pipelines.ts create mode 100644 app/src/worker/staging-cleaner.service.ts create mode 100644 app/src/worker/worker.module.ts diff --git a/.env.example b/.env.example index 2bae9d1..dc7db80 100644 --- a/.env.example +++ b/.env.example @@ -81,11 +81,32 @@ DIRECT_UPLOAD_MAX_PENDING_PER_USER=10 # ATTACHMENT_SWEEP_INTERVAL_MS задаются в compose.yaml (environment) с умолчаниями # и переопределяются из shell только для экспериментов. -# lab5: очередь и асинхронная обработка изображений. Пока не реализовано — -# приложение эти переменные не читает. URL брокера берётся из файла секрета, -# байты и credentials в сообщения не попадают. +# lab5: очередь и асинхронная обработка изображений. URL брокера берётся из +# файла секрета (scripts/init-env.sh), байты и credentials в сообщения не +# попадают: только job_id/attachment_id/pipeline_version/attempt. BROKER_URL_FILE=/run/secrets/broker_url +BROKER_DELIVERY_LIMIT=10 +# Сколько заданий worker обрабатывает параллельно; prefetch равен этому числу. WORKER_CONCURRENCY=2 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 +JOB_LEASE_SECONDS=30 +JOB_RETRY_BASE_MS=1000 +JOB_RETRY_MAX_MS=30000 +PIPELINE_VERSION=1 +OUTBOX_POLL_INTERVAL_MS=500 +LEASE_REAPER_INTERVAL_MS=2000 +# 0 выключает фоновую уборку staging (ручной запуск остаётся). +STAGING_CLEANUP_INTERVAL_MS=60000 +# Сколько staging-объект неуспешной загрузки живёт до удаления. STAGING_RETENTION_SECONDS=86400 +WORKER_HEALTH_PORT=8081 +# 1 включает чтение fault: из Redis; в обычном стенде — 0. +FAULT_INJECTION_ENABLED=0 +WORKER_REPLICAS=1 +WORKER_CPUS=1.5 +# Бюджет worker: база ≈250 MiB + WORKER_CONCURRENCY × (10 MiB исходник + +# ≈340 MiB худший декод 40 Мп) ≈ 950 MiB при concurrency 2; лимит 1280m. +WORKER_MEMORY=1280m +BROKER_CPUS=0.5 +BROKER_MEMORY=512m diff --git a/app/package-lock.json b/app/package-lock.json index 32fbef7..34fa690 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -17,6 +17,7 @@ "@nestjs/platform-express": "^12.0.1", "@nestjs/platform-fastify": "^12.0.3", "@smithy/node-http-handler": "^4.12.1", + "amqplib": "^2.2.1", "argon2": "^0.45.1", "class-transformer": "^0.5.1", "class-validator": "^0.15.1", @@ -3806,6 +3807,15 @@ } } }, + "node_modules/amqplib": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/amqplib/-/amqplib-2.2.1.tgz", + "integrity": "sha512-J3JjAydC2vJ/AgkgVAE0kVH5M4RNfm51zUaspHpwSai3W35lD3hKryvYY/xqMnYcNfR66NYL06Eik+Adn293dQ==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/ansi-escapes": { "version": "4.3.2", "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", diff --git a/app/package.json b/app/package.json index 30a2c2b..e9277cd 100644 --- a/app/package.json +++ b/app/package.json @@ -16,7 +16,7 @@ "start:prod": "node dist/main", "lint": "oxlint src/ test/", "test": "vitest run", - "test:unit": "vitest run --include 'src/**/*.spec.ts'", + "test:unit": "vitest run src/", "test:watch": "vitest", "test:cov": "vitest run --coverage", "test:debug": "vitest --inspect-brk --no-file-parallelism", @@ -24,7 +24,8 @@ "test:e2e": "vitest run --config ./vitest.config.e2e.ts", "test:dependencies": "E2E_DEPENDENCY_FAULTS=1 vitest run --config ./vitest.config.e2e.ts dependency-failures", "attachments:sweep": "tsx src/attachments/attachments-sweep.cli.ts", - "idempotency:gc": "tsx src/messages/idempotency/idempotency-gc.cli.ts" + "idempotency:gc": "tsx src/messages/idempotency/idempotency-gc.cli.ts", + "jobs": "node dist/jobs/jobs.cli.js" }, "dependencies": { "@aws-sdk/client-s3": "^3.1156.0", @@ -35,6 +36,7 @@ "@nestjs/platform-express": "^12.0.1", "@nestjs/platform-fastify": "^12.0.3", "@smithy/node-http-handler": "^4.12.1", + "amqplib": "^2.2.1", "argon2": "^0.45.1", "class-transformer": "^0.5.1", "class-validator": "^0.15.1", diff --git a/app/src/attachments/attachment-sweeper.service.ts b/app/src/attachments/attachment-sweeper.service.ts index 83b429c..ba17ae3 100644 --- a/app/src/attachments/attachment-sweeper.service.ts +++ b/app/src/attachments/attachment-sweeper.service.ts @@ -62,7 +62,9 @@ export class AttachmentSweeper implements OnModuleInit, OnModuleDestroy { let deleted = 0; for (const item of claimed) { try { - await this.storage.delete(item.object_key); + // New uploads live in staging; lab4 rows have no staging key and + // their final object is the only copy. + await this.storage.delete(item.staging_key ?? item.object_key); await this.repository.deleteClaimed(item.id); deleted += 1; this.metrics.increment('attachments.sweep.deleted'); diff --git a/app/src/attachments/attachment.ts b/app/src/attachments/attachment.ts index c3df9ec..999c97d 100644 --- a/app/src/attachments/attachment.ts +++ b/app/src/attachments/attachment.ts @@ -1,3 +1,17 @@ +export type AttachmentStatus = 'queued' | 'processing' | 'ready' | 'failed'; +export type VariantName = 'thumbnail' | 'crop' | 'optimized'; + +export const VARIANT_ORDER: VariantName[] = ['thumbnail', 'crop', 'optimized']; + +export type Variant = { + name: VariantName; + content_type: 'image/jpeg' | 'image/png' | 'image/webp'; + size_bytes: number; + width: number; + height: number; + sha256: string; +}; + export type Attachment = { id: string; chat_id: string; @@ -6,9 +20,9 @@ export type Attachment = { content_type: 'image/png' | 'image/jpeg'; size_bytes: number; sha256: string; - status: 'ready'; - variants: never[]; - error_code: null; + status: AttachmentStatus; + variants: Variant[]; + error_code: string | null; created_at: string; updated_at: string; }; @@ -20,18 +34,64 @@ export type AttachmentRow = { filename: string; content_type: 'image/png' | 'image/jpeg'; upload_mode: 'proxy' | 'direct'; - status: 'pending' | 'ready' | 'deleting' | 'rejected'; + status: + | 'pending' + | 'queued' + | 'processing' + | 'ready' + | 'failed' + | 'deleting' + | 'rejected'; object_key: string; + staging_key: string | null; size_bytes: string | number | null; sha256: string | null; width: number | null; height: number | null; + error_code: string | null; + ready_pipeline_version: number | null; expires_at: Date | null; created_at: Date; updated_at: Date; }; -export function toAttachment(row: AttachmentRow): Attachment { +export type VariantRow = { + attachment_id: string; + pipeline_version: number; + variant: VariantName; + object_key: string; + content_type: 'image/jpeg' | 'image/png' | 'image/webp'; + size_bytes: string | number; + width: number; + height: number; + sha256: string; + job_id: string; + created_at: Date; +}; + +function toVariant(row: VariantRow): Variant { + return { + name: row.variant, + content_type: row.content_type, + size_bytes: Number(row.size_bytes), + width: row.width, + height: row.height, + sha256: row.sha256, + }; +} + +export function toAttachment( + row: AttachmentRow, + variants: VariantRow[] = [], +): Attachment { + const ordered = + row.status === 'ready' + ? [...variants].sort( + (left, right) => + VARIANT_ORDER.indexOf(left.variant) - + VARIANT_ORDER.indexOf(right.variant), + ) + : []; return { id: row.id, chat_id: row.chat_id, @@ -40,9 +100,9 @@ export function toAttachment(row: AttachmentRow): Attachment { content_type: row.content_type, size_bytes: Number(row.size_bytes), sha256: row.sha256 as string, - status: 'ready', - variants: [], - error_code: null, + status: row.status as AttachmentStatus, + variants: ordered.map(toVariant), + error_code: row.status === 'failed' ? row.error_code : null, created_at: row.created_at.toISOString(), updated_at: row.updated_at.toISOString(), }; diff --git a/app/src/attachments/attachments.controller.ts b/app/src/attachments/attachments.controller.ts index 6435849..6335d6e 100644 --- a/app/src/attachments/attachments.controller.ts +++ b/app/src/attachments/attachments.controller.ts @@ -40,7 +40,7 @@ export class AttachmentsController { } @Post('chats/:chatId/attachments') - @HttpCode(201) + @HttpCode(202) upload( @Param('chatId', new ParseUUIDPipe()) chatId: string, @CurrentUser() user: User, @@ -96,16 +96,13 @@ export class AttachmentsController { @Res({ passthrough: true }) reply: FastifyReply, @Query('variant') variant?: string, ) { - const { row, stream } = await this.attachments.openContent( - attachmentId, - user, - variant, - ); + const { row, contentType, sizeBytes, stream } = + await this.attachments.openContent(attachmentId, user, variant); reply.header('X-Content-Type-Options', 'nosniff'); reply.header('Cache-Control', 'private, no-store'); return new StreamableFile(stream, { - type: row.content_type, - length: Number(row.size_bytes), + type: contentType, + length: sizeBytes, disposition: contentDisposition(row.filename), }); } diff --git a/app/src/attachments/attachments.module.ts b/app/src/attachments/attachments.module.ts index 951c8d6..7a8fbdc 100644 --- a/app/src/attachments/attachments.module.ts +++ b/app/src/attachments/attachments.module.ts @@ -5,6 +5,8 @@ import { UsersModule } from '../users/users.module.js'; import { RedisModule } from '../redis/redis.module.js'; import { MetricsModule } from '../metrics/metrics.module.js'; import { StorageModule } from '../storage/storage.module.js'; +import { JobsModule } from '../jobs/jobs.module.js'; +import { OutboxRelay } from '../jobs/outbox-relay.service.js'; import { AttachmentSweeper } from './attachment-sweeper.service.js'; import { AttachmentsController } from './attachments.controller.js'; import { AttachmentsRepository } from './attachments.repository.js'; @@ -21,6 +23,7 @@ import { AttachmentsService } from './attachments.service.js'; RedisModule, MetricsModule, StorageModule, + JobsModule, ], controllers: [AttachmentsController], providers: [ @@ -30,6 +33,9 @@ import { AttachmentsService } from './attachments.service.js'; ImageDecoder, UploadSlots, AttachmentSweeper, + // The relay runs only in the API process; the worker never imports + // AttachmentsModule. + OutboxRelay, ], exports: [AttachmentsRepository], }) diff --git a/app/src/attachments/attachments.repository.ts b/app/src/attachments/attachments.repository.ts index d5f36c0..721eb01 100644 --- a/app/src/attachments/attachments.repository.ts +++ b/app/src/attachments/attachments.repository.ts @@ -10,18 +10,12 @@ export type InsertPending = { contentType: string; uploadMode: 'proxy' | 'direct'; objectKey: string; + stagingKey: string | null; sizeBytes: number | null; sha256: string | null; ttlMs: number; }; -export type ReadyFields = { - sizeBytes: number; - sha256: string; - width: number; - height: number; -}; - const VISIBLE_MEMBER = `JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = $2`; @Injectable() @@ -33,9 +27,9 @@ export class AttachmentsRepository { 'attachments.insert_pending', `INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, upload_mode, status, - object_key, size_bytes, sha256, expires_at) - VALUES ($1, $2, $3, $4, $5, $6, 'pending', $7, $8, $9, - now() + make_interval(secs => $10::double precision / 1000.0))`, + object_key, staging_key, size_bytes, sha256, expires_at) + VALUES ($1, $2, $3, $4, $5, $6, 'pending', $7, $8, $9, $10, + now() + make_interval(secs => $11::double precision / 1000.0))`, [ input.id, input.chatId, @@ -44,6 +38,7 @@ export class AttachmentsRepository { input.contentType, input.uploadMode, input.objectKey, + input.stagingKey, input.sizeBytes, input.sha256, input.ttlMs, @@ -51,22 +46,6 @@ export class AttachmentsRepository { ); } - async markReady( - id: string, - fields: ReadyFields, - ): Promise { - const result = await this.database.query( - 'attachments.mark_ready', - `UPDATE attachments - SET status = 'ready', size_bytes = $2, sha256 = $3, width = $4, - height = $5, expires_at = NULL, updated_at = now() - WHERE id = $1 AND status = 'pending' - RETURNING *`, - [id, fields.sizeBytes, fields.sha256, fields.width, fields.height], - ); - return result.rows[0]; - } - async deletePending(id: string): Promise { await this.database.query( 'attachments.delete_pending', @@ -116,7 +95,7 @@ export class AttachmentsRepository { const result = await this.database.query( 'attachments.find_visible', `SELECT a.* FROM attachments a ${VISIBLE_MEMBER} - WHERE a.id = $1 AND a.status = 'ready'`, + WHERE a.id = $1 AND a.status IN ('queued','processing','ready','failed')`, [id, userId], ); return result.rows[0]; @@ -130,7 +109,7 @@ export class AttachmentsRepository { 'attachments.find_for_complete', `SELECT a.* FROM attachments a ${VISIBLE_MEMBER} WHERE a.id = $1 AND a.upload_mode = 'direct' AND a.owner_id = $2 - AND a.status IN ('pending', 'ready')`, + AND a.status IN ('pending','queued','processing','ready','failed')`, [id, userId], ); return result.rows[0]; @@ -147,23 +126,38 @@ export class AttachmentsRepository { return Number(result.rows[0].count); } - async allLinkable(chatId: string, ids: string[]): Promise { - const result = await this.database.query<{ count: string }>( - 'attachments.all_linkable', - `SELECT count(*) AS count FROM attachments - WHERE chat_id = $1 AND status = 'ready' AND id = ANY($2::uuid[])`, + async linkState( + chatId: string, + ids: string[], + ): Promise<'ok' | 'failed' | 'missing'> { + const result = await this.database.query<{ + linkable: string; + failed: string; + }>( + 'attachments.link_state', + `SELECT count(*) FILTER (WHERE status IN ('queued','processing','ready')) AS linkable, + count(*) FILTER (WHERE status = 'failed') AS failed + FROM attachments + WHERE chat_id = $1 AND id = ANY($2::uuid[])`, [chatId, ids], ); - return Number(result.rows[0].count) === ids.length; + const linkable = Number(result.rows[0]?.linkable ?? 0); + const failed = Number(result.rows[0]?.failed ?? 0); + if (linkable === ids.length) return 'ok'; + if (linkable + failed === ids.length) return 'failed'; + return 'missing'; } async claimExpired( graceSeconds: number, limit: number, - ): Promise<{ id: string; object_key: string }[]> { + ): Promise< + { id: string; object_key: string; staging_key: string | null }[] + > { const result = await this.database.query<{ id: string; object_key: string; + staging_key: string | null; }>( 'attachments.claim_expired', `UPDATE attachments SET status = 'deleting', updated_at = now() @@ -175,7 +169,7 @@ export class AttachmentsRepository { ORDER BY expires_at NULLS FIRST LIMIT $2 FOR UPDATE SKIP LOCKED) - RETURNING id, object_key`, + RETURNING id, object_key, staging_key`, [graceSeconds, limit], ); return result.rows; diff --git a/app/src/attachments/attachments.service.spec.ts b/app/src/attachments/attachments.service.spec.ts index b79287a..2aac0ae 100644 --- a/app/src/attachments/attachments.service.spec.ts +++ b/app/src/attachments/attachments.service.spec.ts @@ -15,11 +15,17 @@ const settings: Record = { MAX_UPLOAD_BYTES: 10_485_760, MAX_IMAGE_PIXELS: 40_000_000, UPLOAD_TIMEOUT_MS: 10_000, + PIPELINE_VERSION: 1, + JOB_MAX_ATTEMPTS: 5, }; -function setup(repository: Record) { +function setup( + repository: Record, + jobs: Record = {}, +) { const storage = { objectKey: (id: string) => `attachments/${id}/original`, + stagingKey: (id: string) => `staging/${id}/original`, putStream: vi.fn(async (_key: string, body: Readable) => { for await (const _chunk of body) { // drain like S3 does @@ -34,6 +40,9 @@ function setup(repository: Record) { deleteClaimed: vi.fn(), ...repository, } as never, + { acceptUpload: vi.fn(), ...jobs } as never, + { kick: vi.fn() } as never, + { point: vi.fn() } as never, storage as never, { getForUser: vi.fn() } as never, { increment: vi.fn(), add: vi.fn() } as never, @@ -63,49 +72,58 @@ const readyRow = (id: string): AttachmentRow => ({ upload_mode: 'proxy', status: 'ready', object_key: `attachments/${id}/original`, + staging_key: `staging/${id}/original`, size_bytes: png.length, sha256: 'a'.repeat(64), width: 320, height: 200, + error_code: null, + ready_pipeline_version: 1, expires_at: null, created_at: new Date(), updated_at: new Date(), }); describe('AttachmentsService compensation', () => { - it('keeps the object and answers 201 when ready was committed but its reply was lost', async () => { - const { storage, upload } = setup({ - markReady: vi.fn().mockRejectedValue(lostReply()), - claimForCompensation: vi.fn().mockResolvedValue(false), - findById: vi.fn((id: string) => Promise.resolve(readyRow(id))), - }); + it('keeps the object and answers 202 when the job commit reply was lost', async () => { + const { storage, upload } = setup( + { + claimForCompensation: vi.fn().mockResolvedValue(false), + findById: vi.fn((id: string) => Promise.resolve(readyRow(id))), + }, + { acceptUpload: vi.fn().mockRejectedValue(lostReply()) }, + ); const attachment = await upload(); expect(attachment.status).toBe('ready'); expect(storage.delete).not.toHaveBeenCalled(); }); it('deletes nothing while PostgreSQL is unreachable', async () => { - const { storage, upload } = setup({ - markReady: vi.fn().mockRejectedValue(lostReply()), - claimForCompensation: vi.fn().mockRejectedValue(lostReply()), - findById: vi.fn().mockRejectedValue(lostReply()), - }); + const { storage, upload } = setup( + { + claimForCompensation: vi.fn().mockRejectedValue(lostReply()), + findById: vi.fn().mockRejectedValue(lostReply()), + }, + { acceptUpload: vi.fn().mockRejectedValue(lostReply()) }, + ); await expect(upload()).rejects.toThrow(); expect(storage.delete).not.toHaveBeenCalled(); }); it('deletes the object only after claiming the pending row', async () => { const order: string[] = []; - const { storage, upload } = setup({ - markReady: vi.fn().mockRejectedValue(lostReply()), - claimForCompensation: vi.fn(async () => { - order.push('claim'); - return true; - }), - deleteClaimed: vi.fn(async () => { - order.push('delete-row'); - }), - }); + const { storage, upload } = setup( + { + claimForCompensation: vi.fn(async () => { + order.push('claim'); + return true; + }), + deleteClaimed: vi.fn(async () => { + order.push('delete-row'); + }), + }, + { acceptUpload: vi.fn().mockRejectedValue(lostReply()) }, + ); storage.delete.mockImplementation(async () => { order.push('delete-object'); }); diff --git a/app/src/attachments/attachments.service.ts b/app/src/attachments/attachments.service.ts index b346b6e..72a9b56 100644 --- a/app/src/attachments/attachments.service.ts +++ b/app/src/attachments/attachments.service.ts @@ -1,5 +1,6 @@ import { BadRequestException, + ConflictException, HttpException, Injectable, InternalServerErrorException, @@ -14,6 +15,9 @@ import { createHash, randomUUID } from 'node:crypto'; import { pipeline, Readable, Transform } from 'node:stream'; import type { AppConfig } from '../config/app-config.schema.js'; import { ChatsService } from '../chats/chats.service.js'; +import { FaultInjector } from '../jobs/fault-injector.js'; +import { JobsRepository } from '../jobs/jobs.repository.js'; +import { OutboxRelay } from '../jobs/outbox-relay.service.js'; import { MetricsService } from '../metrics/metrics.service.js'; import { ObjectStorage } from '../storage/object-storage.js'; import type { User } from '../users/user.js'; @@ -53,10 +57,15 @@ export class AttachmentsService { private readonly maxUploadBytes: number; private readonly maxPixels: number; private readonly uploadTimeoutMs: number; + private readonly pipelineVersion: number; + private readonly maxAttempts: number; constructor( config: ConfigService, private readonly repository: AttachmentsRepository, + private readonly jobs: JobsRepository, + private readonly relay: OutboxRelay, + private readonly faults: FaultInjector, private readonly storage: ObjectStorage, private readonly chats: ChatsService, private readonly metrics: MetricsService, @@ -66,6 +75,8 @@ export class AttachmentsService { this.maxUploadBytes = config.getOrThrow('MAX_UPLOAD_BYTES'); this.maxPixels = config.getOrThrow('MAX_IMAGE_PIXELS'); this.uploadTimeoutMs = config.getOrThrow('UPLOAD_TIMEOUT_MS'); + this.pipelineVersion = config.getOrThrow('PIPELINE_VERSION'); + this.maxAttempts = config.getOrThrow('JOB_MAX_ATTEMPTS'); } async uploadProxy( @@ -162,7 +173,7 @@ export class AttachmentsService { ); id = randomUUID(); - key = this.storage.objectKey(id); + key = this.storage.stagingKey(id); await this.repository.insertPending({ id, chatId, @@ -170,7 +181,8 @@ export class AttachmentsService { filename, contentType: declared, uploadMode: 'proxy', - objectKey: key, + objectKey: this.storage.objectKey(id), + stagingKey: key, sizeBytes: null, sha256: null, ttlMs: this.uploadTimeoutMs, @@ -251,17 +263,27 @@ export class AttachmentsService { contentLength, ); stored = true; - const row = await this.repository.markReady(id, { - sizeBytes: total, - sha256: sha256.digest('hex'), - width: header.width, - height: header.height, - }); + const row = await this.jobs.acceptUpload( + id, + { + sizeBytes: total, + sha256: sha256.digest('hex'), + width: header.width, + height: header.height, + }, + this.pipelineVersion, + this.maxAttempts, + ); if (row === undefined) { claimed = true; throw new ServiceUnavailableException(); } - this.metrics.increment('attachments.upload.created'); + // Durable: staging object + attachment + job + outbox are committed. + // The fault point deliberately sits between COMMIT and publish; the + // relay recovers the outbox row after a crash. + await this.faults.point('api.after_commit_before_publish'); + this.relay.kick(); + this.metrics.increment('attachments.upload.queued'); return toAttachment(row); } catch (error) { readable.destroy(); @@ -272,14 +294,22 @@ export class AttachmentsService { const outcome = await this.compensate(id, key, { keepRow: allSent && !stored, }); - // markReady failed on our side, but the row is no longer + // acceptUpload failed on our side, but the row is no longer // pending: the commit may have succeeded with its reply lost. - // If so, the upload is complete and 201 is the truth. + // If so, the upload is accepted and 202 is the truth. if (outcome === 'not_pending' && stored) { const row = await this.repository .findById(id) .catch(() => undefined); - if (row?.status === 'ready') { + if ( + row && + [ + 'queued', + 'processing', + 'ready', + 'failed', + ].includes(row.status) + ) { this.metrics.increment( 'attachments.upload.recovered', ); @@ -337,22 +367,54 @@ export class AttachmentsService { async getMetadata(id: string, user: User): Promise { const row = await this.repository.findVisible(id, user.id); if (!row) throw new NotFoundException(); - return toAttachment(row); + const variants = + row.status === 'ready' && row.ready_pipeline_version !== null + ? await this.jobs.variantsFor( + row.id, + row.ready_pipeline_version, + ) + : []; + return toAttachment(row, variants); } async openContent( id: string, user: User, variant: string | undefined, - ): Promise<{ row: AttachmentRow; stream: Readable }> { - if (variant !== undefined && variant !== 'original') { - if (['thumbnail', 'crop', 'optimized'].includes(variant)) - throw new NotFoundException(); + ): Promise<{ + row: AttachmentRow; + contentType: string; + sizeBytes: number; + stream: Readable; + }> { + if ( + variant !== undefined && + !['original', 'thumbnail', 'crop', 'optimized'].includes(variant) + ) throw new BadRequestException('variant is invalid'); - } const row = await this.repository.findVisible(id, user.id); if (!row) throw new NotFoundException(); - const source = await this.storage.getStream(row.object_key); + if (row.status !== 'ready') + throw new ConflictException('Attachment is not ready'); + + let key = row.object_key; + let contentType: string = row.content_type; + let sizeBytes = Number(row.size_bytes); + if (variant !== undefined && variant !== 'original') { + if (row.ready_pipeline_version === null) + throw new NotFoundException(); + const variants = await this.jobs.variantsFor( + row.id, + row.ready_pipeline_version, + ); + const match = variants.find((item) => item.variant === variant); + if (!match) throw new NotFoundException(); + key = match.object_key; + contentType = match.content_type; + sizeBytes = Number(match.size_bytes); + } + + const source = await this.storage.getStream(key); if (source === null) { this.metrics.increment('attachments.object_missing'); this.logger.error( @@ -368,6 +430,6 @@ export class AttachmentsService { }, }); pipeline(source, counter, () => undefined); - return { row, stream: counter }; + return { row, contentType, sizeBytes, stream: counter }; } } diff --git a/app/src/attachments/direct-uploads.service.ts b/app/src/attachments/direct-uploads.service.ts index 1b738dd..b0751ad 100644 --- a/app/src/attachments/direct-uploads.service.ts +++ b/app/src/attachments/direct-uploads.service.ts @@ -10,6 +10,8 @@ import { ConfigService } from '@nestjs/config'; import { randomUUID } from 'node:crypto'; import type { AppConfig } from '../config/app-config.schema.js'; import { ChatsService } from '../chats/chats.service.js'; +import { JobsRepository } from '../jobs/jobs.repository.js'; +import { OutboxRelay } from '../jobs/outbox-relay.service.js'; import { MetricsService } from '../metrics/metrics.service.js'; import { ObjectStorage, @@ -50,10 +52,14 @@ export class DirectUploadsService { private readonly uploadTimeoutMs: number; private readonly ttlSeconds: number; private readonly maxPending: number; + private readonly pipelineVersion: number; + private readonly maxAttempts: number; constructor( config: ConfigService, private readonly repository: AttachmentsRepository, + private readonly jobs: JobsRepository, + private readonly relay: OutboxRelay, private readonly storage: ObjectStorage, private readonly chats: ChatsService, private readonly metrics: MetricsService, @@ -67,6 +73,8 @@ export class DirectUploadsService { this.maxPending = config.getOrThrow( 'DIRECT_UPLOAD_MAX_PENDING_PER_USER', ); + this.pipelineVersion = config.getOrThrow('PIPELINE_VERSION'); + this.maxAttempts = config.getOrThrow('JOB_MAX_ATTEMPTS'); } async init( @@ -95,7 +103,7 @@ export class DirectUploadsService { ); const id = randomUUID(); - const key = this.storage.objectKey(id); + const stagingKey = this.storage.stagingKey(id); await this.repository.insertPending({ id, chatId, @@ -103,7 +111,8 @@ export class DirectUploadsService { filename, contentType: dto.content_type, uploadMode: 'direct', - objectKey: key, + objectKey: this.storage.objectKey(id), + stagingKey, sizeBytes: dto.size_bytes, sha256: dto.sha256, ttlMs: this.ttlSeconds * 1000, @@ -111,7 +120,7 @@ export class DirectUploadsService { const checksum = Buffer.from(dto.sha256, 'hex').toString('base64'); let url: string; try { - url = await this.storage.presignPut(key, { + url = await this.storage.presignPut(stagingKey, { contentType: dto.content_type, contentLength: dto.size_bytes, checksumSha256Base64: checksum, @@ -142,21 +151,35 @@ export class DirectUploadsService { user.id, ); if (!row) throw new NotFoundException(); - if (row.status === 'ready') return toAttachment(row); - return this.slots.run(user.id, () => this.verifyAndMarkReady(row)); + if (row.status !== 'pending') return this.describe(row); + return this.slots.run(user.id, () => this.verifyAndAccept(row)); } - private async verifyAndMarkReady(row: AttachmentRow): Promise { + private async describe(row: AttachmentRow): Promise { + const variants = + row.status === 'ready' && row.ready_pipeline_version !== null + ? await this.jobs.variantsFor( + row.id, + row.ready_pipeline_version, + ) + : []; + return toAttachment(row, variants); + } + + private async verifyAndAccept(row: AttachmentRow): Promise { const size = Number(row.size_bytes); + const stagingKey = row.staging_key; + if (stagingKey === null) + throw new ConflictException('Upload has not been received'); const expectedChecksum = Buffer.from( row.sha256 as string, 'hex', ).toString('base64'); - const head = await this.storage.head(row.object_key); + const head = await this.storage.head(stagingKey); if (head === null) throw new ConflictException('Upload has not been received'); if (head.size !== size || head.checksumSha256 !== expectedChecksum) { - await this.reject(row); + await this.reject(row, stagingKey); throw new ConflictException( 'Uploaded object does not match the declaration', ); @@ -164,37 +187,44 @@ export class DirectUploadsService { let header: ImageHeader; try { - header = await this.verifyObject(row.object_key, size); + header = await this.verifyObject(stagingKey, size); } catch (error) { if (error instanceof ImageRejectedError) { - await this.reject(row); + await this.reject(row, stagingKey); throw imageRejection(error); } throw error; } if (header.contentType !== row.content_type) { - await this.reject(row); + await this.reject(row, stagingKey); throw new UnsupportedMediaTypeException( 'Image content does not match content_type', ); } - const ready = await this.repository.markReady(row.id, { - sizeBytes: size, - sha256: row.sha256 as string, - width: header.width, - height: header.height, - }); - if (!ready) { + const accepted = await this.jobs.acceptUpload( + row.id, + { + sizeBytes: size, + sha256: row.sha256 as string, + width: header.width, + height: header.height, + }, + this.pipelineVersion, + this.maxAttempts, + ); + if (!accepted) { // A parallel complete of the same upload won: answer like a repeat. const current = await this.repository.findForComplete( row.id, row.owner_id, ); - if (current?.status === 'ready') return toAttachment(current); + if (current && current.status !== 'pending') + return this.describe(current); throw new ConflictException('Upload is no longer pending'); } + this.relay.kick(); this.metrics.increment('attachments.direct.completed'); - return toAttachment(ready); + return toAttachment(accepted); } /** @@ -247,12 +277,15 @@ export class DirectUploadsService { * the signed URL may still be used: a later re-PUT would otherwise leave an * object that no row refers to. The sweeper removes both after expiry + grace. */ - private async reject(row: AttachmentRow): Promise { - // Row first: a row that is not pending any more (ready after a parallel + private async reject( + row: AttachmentRow, + stagingKey: string, + ): Promise { + // Row first: a row that is not pending any more (accepted after a parallel // complete, or claimed by the sweeper) keeps its object. if (!(await this.repository.rejectPending(row.id))) return; try { - await this.storage.delete(row.object_key); + await this.storage.delete(stagingKey); } catch { // The sweeper deletes the object again after expiry. } diff --git a/app/src/attachments/image/image-decoder.spec.ts b/app/src/attachments/image/image-decoder.spec.ts index e5d82cf..cc98444 100644 --- a/app/src/attachments/image/image-decoder.spec.ts +++ b/app/src/attachments/image/image-decoder.spec.ts @@ -55,7 +55,7 @@ describe('ImageDecoder', () => { it('accepts the heaviest valid input: 40 MP RGBA 16-bit', async () => { await decoder.verify(await maxPixelsRgba16Png()); - }); + }, 30_000); it('drops an aborted waiter from the queue without running its decode', async () => { const busy = decoder.verify(noisePng(4_000_000)); diff --git a/app/src/attachments/image/image-decoder.ts b/app/src/attachments/image/image-decoder.ts index fc1ee06..c383101 100644 --- a/app/src/attachments/image/image-decoder.ts +++ b/app/src/attachments/image/image-decoder.ts @@ -19,7 +19,7 @@ sharp.concurrency(1); // client 415. libvips spills large images to a temp file unless // VIPS_DISC_THRESHOLD (compose.yaml) keeps them in memory; a full /tmp or an // allocation failure ends up here. -const SERVER_FAULT = +export const SERVER_FAULT = /no space left|out of memory|unable to allocate|cannot allocate|write failed|unable to open|tmp/iu; /** diff --git a/app/src/config/app-config.schema.ts b/app/src/config/app-config.schema.ts index 86fd5a2..604d088 100644 --- a/app/src/config/app-config.schema.ts +++ b/app/src/config/app-config.schema.ts @@ -35,6 +35,7 @@ export const appConfigSchema = z SESSION_TTL_SECONDS: numeric('SESSION_TTL_SECONDS', 2, undefined, 3600), DATABASE_URL: requiredString('DATABASE_URL'), REDIS_URL: requiredString('REDIS_URL'), + BROKER_URL: requiredString('BROKER_URL'), DATABASE_POOL_MAX: numeric('DATABASE_POOL_MAX', 1, 100, 10), DATABASE_CONNECTION_TIMEOUT_MS: numeric( 'DATABASE_CONNECTION_TIMEOUT_MS', @@ -123,6 +124,41 @@ export const appConfigSchema = z 1000, 10, ), + WORKER_CONCURRENCY: numeric('WORKER_CONCURRENCY', 1, 16, 2), + JOB_MAX_ATTEMPTS: numeric('JOB_MAX_ATTEMPTS', 1, 20, 5), + JOB_TIMEOUT_SECONDS: numeric('JOB_TIMEOUT_SECONDS', 1, 600, 60), + JOB_LEASE_SECONDS: numeric('JOB_LEASE_SECONDS', 3, 600, 30), + JOB_RETRY_BASE_MS: numeric('JOB_RETRY_BASE_MS', 10, undefined, 1000), + JOB_RETRY_MAX_MS: numeric('JOB_RETRY_MAX_MS', 10, undefined, 30000), + PIPELINE_VERSION: numeric('PIPELINE_VERSION', 1, 2, 1), + BROKER_DELIVERY_LIMIT: numeric('BROKER_DELIVERY_LIMIT', 1, 100, 10), + OUTBOX_POLL_INTERVAL_MS: numeric( + 'OUTBOX_POLL_INTERVAL_MS', + 50, + undefined, + 500, + ), + LEASE_REAPER_INTERVAL_MS: numeric( + 'LEASE_REAPER_INTERVAL_MS', + 100, + undefined, + 2000, + ), + // 0 отключает фоновую уборку staging (ручной запуск остаётся). + STAGING_CLEANUP_INTERVAL_MS: numeric( + 'STAGING_CLEANUP_INTERVAL_MS', + 0, + undefined, + 60000, + ), + STAGING_RETENTION_SECONDS: numeric( + 'STAGING_RETENTION_SECONDS', + 60, + undefined, + 86400, + ), + WORKER_HEALTH_PORT: numeric('WORKER_HEALTH_PORT', 1, 65535, 8081), + FAULT_INJECTION_ENABLED: numeric('FAULT_INJECTION_ENABLED', 0, 1, 0), }) .strip(); @@ -144,6 +180,7 @@ function resolveSecret( key: | 'DATABASE_URL' | 'REDIS_URL' + | 'BROKER_URL' | 'S3_ACCESS_KEY_ID' | 'S3_SECRET_ACCESS_KEY', ): unknown { @@ -170,6 +207,7 @@ export function parseAppConfig(raw: Record): AppConfig { ...raw, DATABASE_URL: resolveSecret(raw, 'DATABASE_URL'), REDIS_URL: resolveSecret(raw, 'REDIS_URL'), + BROKER_URL: resolveSecret(raw, 'BROKER_URL'), S3_ACCESS_KEY_ID: resolveSecret(raw, 'S3_ACCESS_KEY_ID'), S3_SECRET_ACCESS_KEY: resolveSecret(raw, 'S3_SECRET_ACCESS_KEY'), }); diff --git a/app/src/database/migrations/007_attachment_pipeline.sql b/app/src/database/migrations/007_attachment_pipeline.sql new file mode 100644 index 0000000..c2244aa --- /dev/null +++ b/app/src/database/migrations/007_attachment_pipeline.sql @@ -0,0 +1,81 @@ +ALTER TABLE attachments DROP CONSTRAINT attachments_status_check; +ALTER TABLE attachments ADD CONSTRAINT attachments_status_check CHECK (status IN + ('pending','queued','processing','ready','failed','deleting','rejected')); +ALTER TABLE attachments DROP CONSTRAINT attachments_ready_complete; +ALTER TABLE attachments ADD CONSTRAINT attachments_accepted_complete CHECK ( + status NOT IN ('queued','processing','ready','failed') OR + (size_bytes IS NOT NULL AND sha256 IS NOT NULL AND width IS NOT NULL AND height IS NOT NULL)); +ALTER TABLE attachments DROP CONSTRAINT attachments_unfinished_expiry; +ALTER TABLE attachments ADD CONSTRAINT attachments_unfinished_expiry CHECK ( + status NOT IN ('pending','rejected','deleting') OR expires_at IS NOT NULL); +ALTER TABLE attachments + ADD COLUMN staging_key text UNIQUE, + ADD COLUMN error_code text CHECK (error_code ~ '^[a-z_]{1,64}$'), + ADD COLUMN ready_pipeline_version integer CHECK (ready_pipeline_version >= 1); +ALTER TABLE attachments ADD CONSTRAINT attachments_failed_code + CHECK (status <> 'failed' OR error_code IS NOT NULL); +DROP INDEX attachments_unfinished_idx; +CREATE INDEX attachments_unfinished_idx ON attachments (expires_at) + WHERE status IN ('pending','rejected','deleting'); +CREATE INDEX attachments_staging_idx ON attachments (updated_at) + WHERE staging_key IS NOT NULL AND status IN ('ready','failed'); + +-- Rows accepted by lab4 already have their final original and complete metadata. +UPDATE attachments SET ready_pipeline_version = 1 + WHERE status = 'ready' AND ready_pipeline_version IS NULL; + +CREATE TABLE attachment_jobs ( + job_id uuid PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + pipeline_version integer NOT NULL CHECK (pipeline_version >= 1), + status text NOT NULL CHECK (status IN ('queued','processing','succeeded','failed','superseded')), + attempt integer NOT NULL DEFAULT 1 CHECK (attempt >= 1), + max_attempts integer NOT NULL CHECK (max_attempts >= 1), + next_attempt_at timestamptz(3) NOT NULL DEFAULT now(), + lease_owner text, lease_token uuid, lease_until timestamptz(3), + last_error_code text, + created_at timestamptz(3) NOT NULL DEFAULT now(), + updated_at timestamptz(3) NOT NULL DEFAULT now(), + finished_at timestamptz(3), + CONSTRAINT attachment_jobs_effect_key UNIQUE (attachment_id, pipeline_version), + CONSTRAINT attachment_jobs_lease CHECK ((status = 'processing') = + (lease_owner IS NOT NULL AND lease_token IS NOT NULL AND lease_until IS NOT NULL)) +); +CREATE INDEX attachment_jobs_lease_idx ON attachment_jobs (lease_until) WHERE status = 'processing'; +CREATE INDEX attachment_jobs_failed_idx ON attachment_jobs (updated_at) WHERE status = 'failed'; + +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + pipeline_version integer NOT NULL CHECK (pipeline_version >= 1), + variant text NOT NULL CHECK (variant IN ('thumbnail','crop','optimized')), + object_key text NOT NULL UNIQUE, + content_type text NOT NULL CHECK (content_type IN ('image/jpeg','image/png','image/webp')), + size_bytes bigint NOT NULL CHECK (size_bytes >= 1), + width integer NOT NULL CHECK (width >= 1), + height integer NOT NULL CHECK (height >= 1), + sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + job_id uuid NOT NULL REFERENCES attachment_jobs(job_id) ON DELETE CASCADE, + created_at timestamptz(3) NOT NULL DEFAULT now(), + PRIMARY KEY (attachment_id, pipeline_version, variant) +); + +CREATE TABLE job_outbox ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs(job_id) ON DELETE CASCADE, + payload jsonb NOT NULL, + available_at timestamptz(3) NOT NULL DEFAULT now(), + published_at timestamptz(3), + created_at timestamptz(3) NOT NULL DEFAULT now() +); +CREATE INDEX job_outbox_pending_idx ON job_outbox (available_at, id) WHERE published_at IS NULL; + +CREATE TABLE job_events ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs(job_id) ON DELETE CASCADE, + event text NOT NULL CHECK (event IN ('created','claimed','retry_scheduled','lease_expired', + 'succeeded','failed','superseded','replayed','duplicate_skipped','staging_deleted')), + attempt integer NOT NULL, + detail jsonb NOT NULL DEFAULT '{}'::jsonb, + created_at timestamptz(3) NOT NULL DEFAULT now() +); +CREATE INDEX job_events_job_idx ON job_events (job_id, id); diff --git a/app/src/jobs/backoff.spec.ts b/app/src/jobs/backoff.spec.ts new file mode 100644 index 0000000..1255ae3 --- /dev/null +++ b/app/src/jobs/backoff.spec.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from 'vitest'; +import { retryDelayMs } from './backoff.js'; + +describe('retryDelayMs', () => { + it('uses equal jitter', () => { + expect(retryDelayMs(1, 1000, 30000, () => 0)).toBe(500); + expect(retryDelayMs(1, 1000, 30000, () => 1)).toBe(1000); + expect(retryDelayMs(10, 1000, 30000, () => 1)).toBe(30000); + }); + + it('never exceeds the cap and grows monotonically at random 0.5', () => { + const values = Array.from({ length: 6 }, (_, index) => + retryDelayMs(index + 1, 1000, 30000, () => 0.5), + ); + for (const value of values) expect(value).toBeLessThanOrEqual(30000); + for (let index = 1; index < values.length; index += 1) + expect(values[index]).toBeGreaterThanOrEqual(values[index - 1]); + }); +}); diff --git a/app/src/jobs/backoff.ts b/app/src/jobs/backoff.ts new file mode 100644 index 0000000..33645ff --- /dev/null +++ b/app/src/jobs/backoff.ts @@ -0,0 +1,14 @@ +/** + * Equal jitter: half of the exponential delay is fixed, half is random. The + * result never exceeds `capMs`, so `max_attempts` bounds both attempts and + * total wait time. + */ +export function retryDelayMs( + attempt: number, + baseMs: number, + capMs: number, + random: () => number = Math.random, +): number { + const exponential = Math.min(capMs, baseMs * 2 ** (attempt - 1)); + return Math.floor(exponential * (0.5 + 0.5 * random())); +} diff --git a/app/src/jobs/broker.client.ts b/app/src/jobs/broker.client.ts new file mode 100644 index 0000000..e0b5599 --- /dev/null +++ b/app/src/jobs/broker.client.ts @@ -0,0 +1,259 @@ +import { + connect as amqpConnect, + type Channel, + type ChannelModel, + type ConfirmChannel, + type ConsumeMessage, +} from 'amqplib'; +import { + Injectable, + Logger, + OnModuleDestroy, + ServiceUnavailableException, +} from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import type { AppConfig } from '../config/app-config.schema.js'; +import { MetricsService } from '../metrics/metrics.service.js'; +import { + assertTopology, + JOB_EXCHANGE, + JOB_QUEUE, + JOB_ROUTING_KEY, +} from './broker.topology.js'; +import type { JobMessage } from './job-message.js'; + +export class BrokerUnavailableError extends ServiceUnavailableException { + constructor() { + super('Broker unavailable'); + } +} + +export type ConsumeHandler = ( + message: ConsumeMessage, + channel: Channel, +) => Promise; + +const MAX_RECONNECT_DELAY_MS = 10_000; + +@Injectable() +export class BrokerClient implements OnModuleDestroy { + private readonly logger = new Logger(BrokerClient.name); + private readonly url: string; + private readonly deliveryLimit: number; + private connection: ChannelModel | undefined; + private confirmChannel: ConfirmChannel | undefined; + private consumeChannel: Channel | undefined; + private consumerTag: string | undefined; + private prefetch = 0; + private handler: ConsumeHandler | undefined; + private connecting: Promise | undefined; + private reconnectTimer: NodeJS.Timeout | undefined; + private reconnectDelayMs = 1000; + private closing = false; + + constructor( + config: ConfigService, + private readonly metrics: MetricsService, + ) { + this.url = config.getOrThrow('BROKER_URL'); + this.deliveryLimit = config.getOrThrow('BROKER_DELIVERY_LIMIT'); + } + + async publish(message: JobMessage | Buffer): Promise { + const channel = await this.ensureConnected(); + const content = Buffer.isBuffer(message) + ? message + : Buffer.from(JSON.stringify(message)); + const options = Buffer.isBuffer(message) + ? { persistent: true, contentType: 'application/json' } + : { + persistent: true, + contentType: 'application/json', + messageId: message.job_id, + type: message.type, + headers: { 'x-attempt': message.attempt }, + }; + const { promise, resolve, reject } = Promise.withResolvers(); + try { + channel.publish( + JOB_EXCHANGE, + JOB_ROUTING_KEY, + content, + options, + (error) => { + if (error) { + this.metrics.increment('broker.publish_failed'); + reject(new BrokerUnavailableError()); + return; + } + this.metrics.increment('broker.published'); + resolve(); + }, + ); + } catch { + this.metrics.increment('broker.publish_failed'); + reject(new BrokerUnavailableError()); + } + return promise; + } + + consume(prefetch: number, handler: ConsumeHandler): void { + this.prefetch = prefetch; + this.handler = handler; + this.ensureConnected() + .then(() => this.subscribe()) + .catch(() => this.scheduleReconnect()); + } + + isConsuming(): boolean { + return this.consumerTag !== undefined; + } + + async queueDepth( + queue: string, + ): Promise<{ messages: number; consumers: number }> { + const channel = await this.ensureConnected(); + const result = await channel.checkQueue(queue); + return { + messages: result.messageCount, + consumers: result.consumerCount, + }; + } + + async stopConsuming(): Promise { + const channel = this.consumeChannel; + const tag = this.consumerTag; + this.consumerTag = undefined; + this.consumeChannel = undefined; + this.handler = undefined; + if (channel && tag) { + try { + await channel.cancel(tag); + } catch { + // The channel is already closing; there is nothing to cancel. + } + } + if (channel) { + try { + await channel.close(); + } catch { + // Closing an already closed channel is harmless. + } + } + } + + async onModuleDestroy(): Promise { + this.closing = true; + clearTimeout(this.reconnectTimer); + this.reconnectTimer = undefined; + await this.stopConsuming(); + const connection = this.connection; + this.connection = undefined; + this.confirmChannel = undefined; + if (connection) { + try { + await connection.close(); + } catch { + // Shutdown is best effort; channels are already unusable. + } + } + } + + private async ensureConnected(): Promise { + if (this.closing) throw new BrokerUnavailableError(); + if (this.confirmChannel) return this.confirmChannel; + if (!this.connecting) { + this.connecting = this.connect() + .then(() => undefined) + .finally(() => { + this.connecting = undefined; + }); + } + await this.connecting; + if (!this.confirmChannel) throw new BrokerUnavailableError(); + return this.confirmChannel; + } + + private async connect(): Promise { + const connection = await amqpConnect(this.url); + connection.on('error', (error: Error) => { + this.metrics.increment('broker.error'); + this.logger.warn(`Broker connection error: ${error.message}`); + }); + connection.on('close', () => { + if (this.connection !== connection) return; + this.handleClose(); + }); + const confirm = await connection.createConfirmChannel(); + await assertTopology(confirm, this.deliveryLimit); + this.connection = connection; + this.confirmChannel = confirm; + this.reconnectDelayMs = 1000; + clearTimeout(this.reconnectTimer); + this.reconnectTimer = undefined; + this.metrics.increment('broker.connected'); + if (this.handler) await this.subscribe(); + } + + private async subscribe(): Promise { + const connection = this.connection; + const handler = this.handler; + if (!connection || !handler || this.consumerTag) return; + const channel = await connection.createChannel(); + channel.on('error', (error: Error) => { + this.metrics.increment('broker.consumer_error'); + this.logger.warn(`Broker consumer channel error: ${error.message}`); + }); + channel.on('close', () => { + if (this.consumeChannel !== channel) return; + this.consumeChannel = undefined; + this.consumerTag = undefined; + }); + await channel.prefetch(this.prefetch); + await assertTopology(channel, this.deliveryLimit); + const consumer = await channel.consume( + JOB_QUEUE, + (message) => { + if (!message) return; + handler(message, channel).catch((error: unknown) => { + this.metrics.increment('broker.handler_error'); + this.logger.error( + `Job handler crashed: ${error instanceof Error ? error.message : 'unknown error'}`, + ); + try { + channel.nack(message, false, true); + } catch { + // The channel is gone; the broker redelivers after the lease. + } + }); + }, + { noAck: false }, + ); + this.consumeChannel = channel; + this.consumerTag = consumer.consumerTag; + } + + private handleClose(): void { + this.metrics.increment('broker.closed'); + this.connection = undefined; + this.confirmChannel = undefined; + this.consumeChannel = undefined; + this.consumerTag = undefined; + if (this.closing) return; + this.scheduleReconnect(); + } + + private scheduleReconnect(): void { + if (this.closing || this.reconnectTimer) return; + const delay = this.reconnectDelayMs; + this.reconnectDelayMs = Math.min( + this.reconnectDelayMs * 2, + MAX_RECONNECT_DELAY_MS, + ); + this.metrics.increment('broker.reconnect_scheduled'); + this.reconnectTimer = setTimeout(() => { + this.reconnectTimer = undefined; + this.ensureConnected().catch(() => this.scheduleReconnect()); + }, delay); + } +} diff --git a/app/src/jobs/broker.topology.ts b/app/src/jobs/broker.topology.ts new file mode 100644 index 0000000..6921d74 --- /dev/null +++ b/app/src/jobs/broker.topology.ts @@ -0,0 +1,37 @@ +import type { Channel } from 'amqplib'; + +export const JOB_EXCHANGE = 'attachments.jobs'; +export const JOB_ROUTING_KEY = 'attachment.process.v1'; +export const JOB_QUEUE = 'attachments.process'; +export const DEAD_LETTER_EXCHANGE = 'attachments.dlx'; +export const DEAD_LETTER_QUEUE = 'attachments.process.dlq'; + +/** + * Idempotent declaration used by both the producer and the consumer on every + * (re)connect: a stand started from an empty volume and a reconnect after a + * broker restart converge to the same durable topology. + */ +export async function assertTopology( + channel: Channel, + deliveryLimit: number, +): Promise { + await channel.assertExchange(DEAD_LETTER_EXCHANGE, 'fanout', { + durable: true, + }); + await channel.assertQueue(DEAD_LETTER_QUEUE, { + durable: true, + arguments: { 'x-queue-type': 'quorum' }, + }); + await channel.bindQueue(DEAD_LETTER_QUEUE, DEAD_LETTER_EXCHANGE, ''); + + await channel.assertExchange(JOB_EXCHANGE, 'direct', { durable: true }); + await channel.assertQueue(JOB_QUEUE, { + durable: true, + arguments: { + 'x-queue-type': 'quorum', + 'x-dead-letter-exchange': DEAD_LETTER_EXCHANGE, + 'x-delivery-limit': deliveryLimit, + }, + }); + await channel.bindQueue(JOB_QUEUE, JOB_EXCHANGE, JOB_ROUTING_KEY); +} diff --git a/app/src/jobs/errors.spec.ts b/app/src/jobs/errors.spec.ts new file mode 100644 index 0000000..6c37c3f --- /dev/null +++ b/app/src/jobs/errors.spec.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from 'vitest'; +import { ObjectStorageUnavailableError } from '../storage/object-storage.js'; +import { classifyError, JobTimeoutError } from './errors.js'; + +describe('classifyError', () => { + it('classifies unsupported sharp input as permanent invalid_image', () => { + expect( + classifyError( + new Error('Input buffer contains unsupported image format'), + ), + ).toEqual({ kind: 'permanent', code: 'invalid_image' }); + }); + + it('classifies the pixel limit as permanent image_too_large', () => { + expect( + classifyError(new Error('Input image exceeds pixel limit')), + ).toEqual({ kind: 'permanent', code: 'image_too_large' }); + }); + + it('classifies object storage outages as transient', () => { + expect(classifyError(new ObjectStorageUnavailableError())).toEqual({ + kind: 'transient', + code: 'processing_failed', + }); + }); + + it('classifies a job timeout as transient processing_timeout', () => { + expect(classifyError(new JobTimeoutError(10))).toEqual({ + kind: 'transient', + code: 'processing_timeout', + }); + }); +}); diff --git a/app/src/jobs/errors.ts b/app/src/jobs/errors.ts new file mode 100644 index 0000000..9c121e1 --- /dev/null +++ b/app/src/jobs/errors.ts @@ -0,0 +1,65 @@ +import { SERVER_FAULT } from '../attachments/image/image-decoder.js'; +import { InjectedFaultError } from './fault-injector.js'; + +export class SourceMissingError extends Error { + constructor(message = 'Source object is missing') { + super(message); + this.name = 'SourceMissingError'; + } +} + +export class SourceMismatchError extends Error { + constructor(message = 'Source object does not match its metadata') { + super(message); + this.name = 'SourceMismatchError'; + } +} + +export class UnsupportedPipelineError extends Error { + constructor(version: number) { + super(`Unsupported pipeline version: ${version}`); + this.name = 'UnsupportedPipelineError'; + } +} + +export class JobTimeoutError extends Error { + constructor(seconds: number) { + super(`Job exceeded ${seconds}s deadline`); + this.name = 'JobTimeoutError'; + } +} + +export type ErrorClass = { + kind: 'permanent' | 'transient'; + code: string; +}; + +const SHARP_INPUT = + /unsupported image format|input buffer|input file|bad seek|premature end|vipsjpeg|vippng|corrupt|truncated|invalid (?:image|pixel)|not a valid image|exceeded.*limit/iu; + +/** + * Maps a processing failure to the contract's safe error class. Permanent + * errors are never retried; transient ones are retried until max_attempts, + * after which the job is failed as `retry_exhausted`. + */ +export function classifyError(error: unknown): ErrorClass { + if (error instanceof InjectedFaultError) + return { kind: error.kind, code: 'injected_fault' }; + if (error instanceof SourceMissingError) + return { kind: 'permanent', code: 'source_missing' }; + if (error instanceof SourceMismatchError) + return { kind: 'permanent', code: 'source_mismatch' }; + if (error instanceof UnsupportedPipelineError) + return { kind: 'permanent', code: 'unsupported_pipeline' }; + if (error instanceof JobTimeoutError) + return { kind: 'transient', code: 'processing_timeout' }; + + const message = error instanceof Error ? error.message : String(error); + if (/pixel limit/iu.test(message)) + return { kind: 'permanent', code: 'image_too_large' }; + if (SERVER_FAULT.test(message)) + return { kind: 'transient', code: 'processing_failed' }; + if (SHARP_INPUT.test(message)) + return { kind: 'permanent', code: 'invalid_image' }; + return { kind: 'transient', code: 'processing_failed' }; +} diff --git a/app/src/jobs/fault-injector.ts b/app/src/jobs/fault-injector.ts new file mode 100644 index 0000000..c60b3d9 --- /dev/null +++ b/app/src/jobs/fault-injector.ts @@ -0,0 +1,131 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import type { AppConfig } from '../config/app-config.schema.js'; +import { MetricsService } from '../metrics/metrics.service.js'; +import { RedisClient } from '../redis/redis.client.js'; + +export type FaultPoint = + | 'api.after_commit_before_publish' + | 'relay.after_publish_before_mark' + | 'worker.before_side_effect' + | 'worker.after_side_effect' + | 'worker.after_commit_before_ack'; + +export type FaultTarget = { + attachmentId: string; + pipelineVersion: number; +}; + +export type FaultKind = 'permanent' | 'transient'; + +export class InjectedFaultError extends Error { + constructor(readonly kind: FaultKind) { + super(`Injected ${kind} fault`); + this.name = 'InjectedFaultError'; + } +} + +/** + * Active only with FAULT_INJECTION_ENABLED=1. The switch is deliberately + * checked before every Redis call so a normal stand never depends on Redis for + * job correctness. + * + * `transient` stays until the key is deleted; every other value is consumed by + * the first point that observes it. Worker points are addressed to one + * attachment and pipeline version; the untargeted key is a fallback. + */ +@Injectable() +export class FaultInjector { + private readonly logger = new Logger(FaultInjector.name); + private readonly enabled: boolean; + + constructor( + config: ConfigService, + private readonly redis: RedisClient, + private readonly metrics: MetricsService, + ) { + this.enabled = config.getOrThrow('FAULT_INJECTION_ENABLED') === 1; + } + + async point( + name: FaultPoint, + target?: FaultTarget, + signal?: AbortSignal, + ): Promise { + if (!this.enabled) return; + const keys = target + ? [ + `fault:${name}:${target.attachmentId}:${target.pipelineVersion}`, + `fault:${name}`, + ] + : [`fault:${name}`]; + for (const key of keys) { + let value: string | null; + let deleted: number; + try { + value = await this.redis.get(key); + if (value === null) continue; + if (value === 'transient') { + this.metrics.increment('faults.triggered'); + throw new InjectedFaultError('transient'); + } + deleted = await this.redis.del(key); + } catch (error) { + if (error instanceof InjectedFaultError) throw error; + this.logger.warn( + `Fault point ${name} could not read Redis: ${error instanceof Error ? error.message : 'unknown error'}`, + ); + return; + } + if (deleted !== 1) continue; + // The action is outside the Redis try/catch: an abort or an injected + // fault must reach the job processor, not be logged as an outage. + await this.act(value, signal); + return; + } + } + + private async act(value: string, signal?: AbortSignal): Promise { + if (value === 'crash') { + this.metrics.increment('faults.triggered'); + // PID 1 in a container namespace may not be killable by SIGKILL; + // exit 137 gives the same observable crash (no graceful shutdown). + process.kill(process.pid, 'SIGKILL'); + process.exit(137); + } + if (value === 'permanent') { + this.metrics.increment('faults.triggered'); + throw new InjectedFaultError('permanent'); + } + if (value.startsWith('delay:')) { + const milliseconds = Number(value.slice('delay:'.length)); + if (!Number.isFinite(milliseconds) || milliseconds < 0) return; + this.metrics.increment('faults.triggered'); + await this.sleep(milliseconds, signal); + } + } + + private sleep(milliseconds: number, signal?: AbortSignal): Promise { + if (signal?.aborted) + return Promise.reject( + signal.reason instanceof Error + ? signal.reason + : new Error('Aborted'), + ); + const { promise, resolve, reject } = Promise.withResolvers(); + const timer = setTimeout(() => { + signal?.removeEventListener('abort', cancel); + resolve(); + }, milliseconds); + const cancel = (): void => { + clearTimeout(timer); + reject( + signal?.reason instanceof Error + ? signal.reason + : new Error('Aborted'), + ); + }; + signal?.addEventListener('abort', cancel, { once: true }); + return promise; + } +} diff --git a/app/src/jobs/fit-inside.spec.ts b/app/src/jobs/fit-inside.spec.ts new file mode 100644 index 0000000..a2cbb07 --- /dev/null +++ b/app/src/jobs/fit-inside.spec.ts @@ -0,0 +1,14 @@ +import { describe, expect, it } from 'vitest'; +import { fitInside } from './fit-inside.js'; + +describe('fitInside', () => { + it.each([ + [320, 200, 256, [256, 160]], + [1000, 335, 256, [256, 85]], + [100, 50, 256, [100, 50]], + [10000, 1, 256, [256, 1]], + [1, 10000, 256, [1, 256]], + ])('fits %ix%i into %i as %j', (width, height, box, expected) => { + expect(fitInside(width, height, box)).toEqual(expected); + }); +}); diff --git a/app/src/jobs/fit-inside.ts b/app/src/jobs/fit-inside.ts new file mode 100644 index 0000000..cff80c6 --- /dev/null +++ b/app/src/jobs/fit-inside.ts @@ -0,0 +1,15 @@ +/** + * Fits `width`x`height` into a `box`x`box` square without upscaling. + * Integer arithmetic keeps the result exact for the contract's fixtures: + * 320x200 -> 256x160, 1000x335 -> 256x85, 10000x1 -> 256x1. + */ +export function fitInside( + width: number, + height: number, + box: number, +): [number, number] { + if (width <= box && height <= box) return [width, height]; + if (width >= height) + return [box, Math.max(1, Math.floor((height * box) / width))]; + return [Math.max(1, Math.floor((width * box) / height)), box]; +} diff --git a/app/src/jobs/job-message.spec.ts b/app/src/jobs/job-message.spec.ts new file mode 100644 index 0000000..4016bd0 --- /dev/null +++ b/app/src/jobs/job-message.spec.ts @@ -0,0 +1,47 @@ +import { readFileSync } from 'node:fs'; +import { describe, expect, it } from 'vitest'; +import { parseJobMessage } from './job-message.js'; + +const contract = JSON.parse( + readFileSync( + new URL('../../../contracts/jobs.schema.json', import.meta.url), + 'utf8', + ), +) as { examples: Record[] }; +const example = contract.examples[0]; + +describe('parseJobMessage', () => { + it('accepts the contract example', () => { + expect(parseJobMessage(Buffer.from(JSON.stringify(example)))).toEqual( + example, + ); + }); + + it('rejects an extra property', () => { + expect( + parseJobMessage( + Buffer.from(JSON.stringify({ ...example, extra: true })), + ), + ).toBeUndefined(); + }); + + it('rejects attempt 0', () => { + expect( + parseJobMessage( + Buffer.from(JSON.stringify({ ...example, attempt: 0 })), + ), + ).toBeUndefined(); + }); + + it('rejects a wrong type', () => { + expect( + parseJobMessage( + Buffer.from(JSON.stringify({ ...example, type: 'other' })), + ), + ).toBeUndefined(); + }); + + it('rejects non-JSON', () => { + expect(parseJobMessage(Buffer.from('not-json'))).toBeUndefined(); + }); +}); diff --git a/app/src/jobs/job-message.ts b/app/src/jobs/job-message.ts new file mode 100644 index 0000000..2e9c724 --- /dev/null +++ b/app/src/jobs/job-message.ts @@ -0,0 +1,53 @@ +import { z } from 'zod'; + +export const JOB_TYPE = 'attachment.process.v1'; + +/** + * Mirrors contracts/jobs.schema.json. The broker message carries identifiers + * and scheduling data only: no bytes, URLs, file names or credentials. + */ +export const jobMessageSchema = z.strictObject({ + schema_version: z.literal(1), + job_id: z.uuid(), + type: z.literal(JOB_TYPE), + attachment_id: z.uuid(), + pipeline_version: z.number().int().min(1), + attempt: z.number().int().min(1), + created_at: z.iso.datetime(), + traceparent: z + .string() + .regex(/^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$/u) + .optional(), + tracestate: z.string().max(512).optional(), +}); + +export type JobMessage = z.infer; + +export function buildJobMessage(job: { + job_id: string; + attachment_id: string; + pipeline_version: number; + attempt: number; + created_at: Date; +}): JobMessage { + return { + schema_version: 1, + job_id: job.job_id, + type: JOB_TYPE, + attachment_id: job.attachment_id, + pipeline_version: job.pipeline_version, + attempt: job.attempt, + created_at: job.created_at.toISOString(), + }; +} + +export function parseJobMessage(content: Buffer): JobMessage | undefined { + let value: unknown; + try { + value = JSON.parse(content.toString('utf8')); + } catch { + return undefined; + } + const parsed = jobMessageSchema.safeParse(value); + return parsed.success ? parsed.data : undefined; +} diff --git a/app/src/jobs/jobs.cli.ts b/app/src/jobs/jobs.cli.ts new file mode 100644 index 0000000..447eb61 --- /dev/null +++ b/app/src/jobs/jobs.cli.ts @@ -0,0 +1,234 @@ +import { Module } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { NestFactory } from '@nestjs/core'; +import type { AppConfig } from '../config/app-config.schema.js'; +import { AppConfigModule } from '../config/app-config.module.js'; +import { DatabaseModule } from '../database/database.module.js'; +import { MetricsModule } from '../metrics/metrics.module.js'; +import { RedisModule } from '../redis/redis.module.js'; +import { ObjectStorage } from '../storage/object-storage.js'; +import { StorageModule } from '../storage/storage.module.js'; +import { DEAD_LETTER_QUEUE, JOB_QUEUE } from './broker.topology.js'; +import { BrokerClient } from './broker.client.js'; +import { JobsModule } from './jobs.module.js'; +import { JobsRepository } from './jobs.repository.js'; + +@Module({ + imports: [ + AppConfigModule, + DatabaseModule, + RedisModule, + MetricsModule, + StorageModule, + JobsModule, + ], +}) +class JobsCliModule {} + +const SUPPORTED_PIPELINES = [1, 2]; + +function option(args: string[], name: string): string | undefined { + const index = args.indexOf(name); + if (index === -1) return undefined; + return args[index + 1]; +} + +async function readStdin(): Promise { + const chunks: Buffer[] = []; + for await (const chunk of process.stdin) + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + return Buffer.concat(chunks).toString('utf8'); +} + +async function s3State( + storage: ObjectStorage, + key: string, +): Promise<{ + exists: boolean; + etag: string | null; + last_modified: string | null; +}> { + const head = await storage.head(key); + return head + ? { + exists: true, + etag: head.etag ?? null, + last_modified: head.lastModified?.toISOString() ?? null, + } + : { exists: false, etag: null, last_modified: null }; +} + +async function main(): Promise { + const args = process.argv.slice(2); + const command = args[0]; + const app = await NestFactory.createApplicationContext(JobsCliModule, { + logger: false, + }); + try { + const repository = app.get(JobsRepository); + const storage = app.get(ObjectStorage); + const broker = app.get(BrokerClient); + const config = app.get(ConfigService); + + if (command === 'status') { + const attachmentId = args[1]; + if (!attachmentId) return usage(); + const status = await repository.statusOf(attachmentId); + if (!status) { + console.log(JSON.stringify({ error: 'attachment_not_found' })); + return 1; + } + const variants = []; + for (const variant of status.variants) { + variants.push({ + pipeline_version: variant.pipeline_version, + variant: variant.variant, + sha256: variant.sha256, + size_bytes: Number(variant.size_bytes), + width: variant.width, + height: variant.height, + object_key: variant.object_key, + s3: await s3State(storage, variant.object_key), + }); + } + console.log( + JSON.stringify({ + attachment: { + id: status.attachment.id, + status: status.attachment.status, + error_code: status.attachment.error_code, + ready_pipeline_version: + status.attachment.ready_pipeline_version, + staging_present: status.attachment.staging_key !== null, + }, + jobs: status.jobs.map((job) => ({ + job_id: job.job_id, + pipeline_version: job.pipeline_version, + status: job.status, + attempt: job.attempt, + max_attempts: job.max_attempts, + last_error_code: job.last_error_code, + })), + variants, + original: await s3State( + storage, + status.attachment.object_key, + ), + events: status.events.map((event) => ({ + job_id: event.job_id, + event: event.event, + attempt: event.attempt, + detail: event.detail, + created_at: event.created_at.toISOString(), + })), + outbox: status.outbox, + }), + ); + return 0; + } + + if (command === 'find-by-chat') { + const chatId = args[1]; + if (!chatId) return usage(); + console.log(JSON.stringify(await repository.findByChat(chatId))); + return 0; + } + + if (command === 'failed') { + console.log(JSON.stringify(await repository.failedJobs())); + return 0; + } + + if (command === 'replay') { + const attachmentId = args[1]; + const reason = option(args, '--reason') ?? 'operator replay'; + const operator = + option(args, '--operator') ?? process.env.USER ?? 'operator'; + if (!attachmentId) return usage(); + const candidate = await repository.replayCandidate(attachmentId); + let sourceAvailable = true; + if (candidate && candidate.staging_key === null) { + const original = await storage.head( + storage.objectKey(attachmentId), + ); + sourceAvailable = original !== null; + } + const result = await repository.replay(attachmentId, { + reason, + operator, + maxAttempts: config.getOrThrow('JOB_MAX_ATTEMPTS'), + sourceAvailable, + }); + console.log(JSON.stringify(result)); + return result.kind === 'replayed' ? 0 : 3; + } + + if (command === 'reprocess') { + const attachmentId = args[1]; + const rawVersion = option(args, '--pipeline-version'); + const version = Number(rawVersion); + if ( + !attachmentId || + rawVersion === undefined || + !SUPPORTED_PIPELINES.includes(version) + ) + return usage(); + const result = await repository.reprocess(attachmentId, version, { + reason: option(args, '--reason') ?? 'reprocess', + operator: + option(args, '--operator') ?? + process.env.USER ?? + 'operator', + maxAttempts: config.getOrThrow('JOB_MAX_ATTEMPTS'), + }); + console.log(JSON.stringify(result)); + return result.kind === 'created' ? 0 : 3; + } + + if (command === 'publish-raw') { + if (config.getOrThrow('FAULT_INJECTION_ENABLED') !== 1) { + console.log( + JSON.stringify({ refused: 'fault_injection_disabled' }), + ); + return 3; + } + const text = await readStdin(); + const lines = text.split(/\r?\n/u).filter((line) => line !== ''); + for (const line of lines) await broker.publish(Buffer.from(line)); + console.log(JSON.stringify({ published: lines.length })); + return 0; + } + + if (command === 'queues') { + console.log( + JSON.stringify({ + process: await broker.queueDepth(JOB_QUEUE), + dlq: await broker.queueDepth(DEAD_LETTER_QUEUE), + }), + ); + return 0; + } + + return usage(); + } finally { + await app.close(); + } +} + +function usage(): number { + console.error( + 'usage: jobs.cli.js status | find-by-chat | failed | replay --reason [--operator ] | reprocess --pipeline-version [--reason ] | publish-raw | queues', + ); + return 2; +} + +main() + .then((code) => process.exit(code)) + .catch((error: unknown) => { + console.log( + JSON.stringify({ + error: error instanceof Error ? error.message : 'unknown error', + }), + ); + process.exit(1); + }); diff --git a/app/src/jobs/jobs.module.ts b/app/src/jobs/jobs.module.ts new file mode 100644 index 0000000..26e251d --- /dev/null +++ b/app/src/jobs/jobs.module.ts @@ -0,0 +1,14 @@ +import { Module } from '@nestjs/common'; +import { DatabaseModule } from '../database/database.module.js'; +import { MetricsModule } from '../metrics/metrics.module.js'; +import { RedisModule } from '../redis/redis.module.js'; +import { BrokerClient } from './broker.client.js'; +import { FaultInjector } from './fault-injector.js'; +import { JobsRepository } from './jobs.repository.js'; + +@Module({ + imports: [DatabaseModule, RedisModule, MetricsModule], + providers: [BrokerClient, JobsRepository, FaultInjector], + exports: [BrokerClient, JobsRepository, FaultInjector], +}) +export class JobsModule {} diff --git a/app/src/jobs/jobs.repository.ts b/app/src/jobs/jobs.repository.ts new file mode 100644 index 0000000..f066935 --- /dev/null +++ b/app/src/jobs/jobs.repository.ts @@ -0,0 +1,910 @@ +import { Injectable } from '@nestjs/common'; +import { randomUUID } from 'node:crypto'; +import type { PoolClient } from 'pg'; +import type { AttachmentRow, VariantRow } from '../attachments/attachment.js'; +import { DatabaseClient } from '../database/database.client.js'; +import { buildJobMessage, type JobMessage } from './job-message.js'; + +export type JobStatus = + 'queued' | 'processing' | 'succeeded' | 'failed' | 'superseded'; + +export type AttachmentJobRow = { + job_id: string; + attachment_id: string; + pipeline_version: number; + status: JobStatus; + attempt: number; + max_attempts: number; + next_attempt_at: Date; + lease_owner: string | null; + lease_token: string | null; + lease_until: Date | null; + last_error_code: string | null; + created_at: Date; + updated_at: Date; + finished_at: Date | null; +}; + +export type VariantWrite = { + name: 'thumbnail' | 'crop' | 'optimized'; + objectKey: string; + contentType: 'image/jpeg' | 'image/png' | 'image/webp'; + sizeBytes: number; + width: number; + height: number; + sha256: string; +}; + +export type ClaimResult = + | { kind: 'skip'; reason: string } + | { + kind: 'claimed'; + job: AttachmentJobRow; + attachment: AttachmentRow; + leaseToken: string; + }; + +export type CommitFailureResult = 'retry' | 'failed' | 'lease_lost'; +export type CommitSuccessResult = 'succeeded' | 'superseded' | 'lease_lost'; + +export type ReplayResult = + | { kind: 'replayed'; job_id: string; attempt: number } + | { kind: 'refused'; reason: string; job_id?: string }; + +export type ReprocessResult = + | { kind: 'created'; job_id: string } + | { kind: 'refused'; reason: string; job_id?: string }; + +type AttachmentJobExtra = AttachmentJobRow & { + attachment_status: AttachmentRow['status']; + ready_pipeline_version: number | null; +}; + +@Injectable() +export class JobsRepository { + constructor(private readonly database: DatabaseClient) {} + + async acceptUpload( + attachmentId: string, + fields: { + sizeBytes: number; + sha256: string; + width: number; + height: number; + }, + pipelineVersion: number, + maxAttempts: number, + ): Promise { + return this.database.withTransaction(async (client) => { + const accepted = await client.query( + `UPDATE attachments + SET status = 'queued', size_bytes = $2, sha256 = $3, + width = $4, height = $5, expires_at = NULL, updated_at = now() + WHERE id = $1 AND status = 'pending' + RETURNING *`, + [ + attachmentId, + fields.sizeBytes, + fields.sha256, + fields.width, + fields.height, + ], + ); + const attachment = accepted.rows[0]; + if (!attachment) return undefined; + + const jobId = randomUUID(); + const created = await client.query( + `INSERT INTO attachment_jobs + (job_id, attachment_id, pipeline_version, status, attempt, + max_attempts, next_attempt_at) + VALUES ($1, $2, $3, 'queued', 1, $4, now()) + RETURNING *`, + [jobId, attachmentId, pipelineVersion, maxAttempts], + ); + const job = created.rows[0]; + await client.query( + `INSERT INTO job_outbox (job_id, payload, available_at) + VALUES ($1, $2::jsonb, now())`, + [ + job.job_id, + JSON.stringify( + buildJobMessage({ + ...job, + created_at: job.created_at, + }), + ), + ], + ); + await this.event(client, job.job_id, 'created', 1, {}); + return attachment; + }); + } + + async claim( + message: JobMessage, + workerId: string, + leaseSeconds: number, + ): Promise { + return this.database.withTransaction(async (client) => { + const found = await client.query( + `SELECT j.*, a.status AS attachment_status, + a.ready_pipeline_version + FROM attachment_jobs j + JOIN attachments a ON a.id = j.attachment_id + WHERE j.job_id = $1 + FOR UPDATE OF j, a`, + [message.job_id], + ); + const job = found.rows[0]; + if (!job) return { kind: 'skip', reason: 'unknown_job' }; + + if ( + job.attachment_id !== message.attachment_id || + job.pipeline_version !== message.pipeline_version + ) + return this.skip(client, message, 'mismatch'); + + if (job.status === 'succeeded') + return this.skip(client, message, 'already_succeeded'); + if (job.status === 'failed') + return this.skip(client, message, 'already_failed'); + if (job.status === 'superseded') + return this.skip(client, message, 'superseded'); + if (job.status === 'processing') + return this.skip(client, message, 'in_progress'); + if (job.attempt !== message.attempt) + return this.skip(client, message, 'stale_attempt'); + if (new Date(job.next_attempt_at).getTime() > Date.now()) + return this.skip(client, message, 'not_due'); + + if ( + job.ready_pipeline_version !== null && + job.ready_pipeline_version > job.pipeline_version + ) { + await client.query( + `UPDATE attachment_jobs + SET status = 'superseded', updated_at = now(), finished_at = now() + WHERE job_id = $1`, + [job.job_id], + ); + await this.event( + client, + job.job_id, + 'superseded', + job.attempt, + { + ready_pipeline_version: job.ready_pipeline_version, + }, + ); + return this.skip(client, message, 'superseded'); + } + + const leaseToken = randomUUID(); + const claimed = await client.query( + `UPDATE attachment_jobs + SET status = 'processing', lease_owner = $2, lease_token = $3, + lease_until = now() + make_interval(secs => $4::double precision), + updated_at = now() + WHERE job_id = $1 + RETURNING *`, + [job.job_id, workerId, leaseToken, leaseSeconds], + ); + const attachment = await client.query( + `SELECT * FROM attachments WHERE id = $1`, + [job.attachment_id], + ); + await client.query( + `UPDATE attachments SET status = 'processing', updated_at = now() + WHERE id = $1 AND status = 'queued'`, + [job.attachment_id], + ); + await this.event( + client, + job.job_id, + 'claimed', + claimed.rows[0].attempt, + { lease_owner: workerId }, + ); + return { + kind: 'claimed', + job: claimed.rows[0], + attachment: attachment.rows[0], + leaseToken, + }; + }); + } + + async extendLease( + jobId: string, + token: string, + leaseSeconds: number, + ): Promise { + const result = await this.database.query( + 'jobs.extend_lease', + `UPDATE attachment_jobs + SET lease_until = now() + make_interval(secs => $3::double precision), + updated_at = now() + WHERE job_id = $1 AND lease_token = $2 AND status = 'processing'`, + [jobId, token, leaseSeconds], + ); + return result.rowCount === 1; + } + + async commitSuccess( + jobId: string, + token: string, + variants: VariantWrite[], + ): Promise { + return this.database.withTransaction(async (client) => { + const succeeded = await client.query( + `UPDATE attachment_jobs + SET status = 'succeeded', lease_owner = NULL, lease_token = NULL, + lease_until = NULL, updated_at = now(), finished_at = now() + WHERE job_id = $1 AND lease_token = $2 AND status = 'processing' + RETURNING *`, + [jobId, token], + ); + const job = succeeded.rows[0]; + if (!job) return 'lease_lost'; + + const attachment = await client.query( + `SELECT * FROM attachments WHERE id = $1 FOR UPDATE`, + [job.attachment_id], + ); + const row = attachment.rows[0]; + if ( + row.ready_pipeline_version !== null && + row.ready_pipeline_version > job.pipeline_version + ) { + await client.query( + `UPDATE attachment_jobs + SET status = 'superseded', updated_at = now() + WHERE job_id = $1`, + [job.job_id], + ); + await this.event( + client, + job.job_id, + 'superseded', + job.attempt, + { + ready_pipeline_version: row.ready_pipeline_version, + }, + ); + return 'superseded'; + } + + for (const variant of variants) { + await client.query( + `INSERT INTO attachment_variants + (attachment_id, pipeline_version, variant, object_key, + content_type, size_bytes, width, height, sha256, job_id) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + ON CONFLICT (attachment_id, pipeline_version, variant) + DO UPDATE SET object_key = EXCLUDED.object_key, + content_type = EXCLUDED.content_type, + size_bytes = EXCLUDED.size_bytes, + width = EXCLUDED.width, + height = EXCLUDED.height, + sha256 = EXCLUDED.sha256, + job_id = EXCLUDED.job_id`, + [ + job.attachment_id, + job.pipeline_version, + variant.name, + variant.objectKey, + variant.contentType, + variant.sizeBytes, + variant.width, + variant.height, + variant.sha256, + job.job_id, + ], + ); + } + await client.query( + `UPDATE attachments + SET status = 'ready', ready_pipeline_version = $2, + error_code = NULL, updated_at = now() + WHERE id = $1`, + [job.attachment_id, job.pipeline_version], + ); + await this.event(client, job.job_id, 'succeeded', job.attempt, {}); + return 'succeeded'; + }); + } + + async commitFailure( + jobId: string, + token: string, + failure: { kind: 'permanent' | 'transient'; code: string }, + delayMs: number, + ): Promise { + return this.database.withTransaction(async (client) => { + const found = await client.query( + `SELECT * FROM attachment_jobs + WHERE job_id = $1 AND lease_token = $2 AND status = 'processing' + FOR UPDATE`, + [jobId, token], + ); + const job = found.rows[0]; + if (!job) return 'lease_lost'; + const finalCode = + failure.kind === 'permanent' || job.attempt >= job.max_attempts + ? failure.kind === 'transient' + ? 'retry_exhausted' + : failure.code + : failure.code; + + if ( + failure.kind === 'permanent' || + job.attempt >= job.max_attempts + ) { + await client.query( + `UPDATE attachment_jobs + SET status = 'failed', last_error_code = $2, + lease_owner = NULL, lease_token = NULL, lease_until = NULL, + updated_at = now(), finished_at = now() + WHERE job_id = $1`, + [job.job_id, finalCode], + ); + await client.query( + `UPDATE attachments + SET status = 'failed', error_code = $2, updated_at = now() + WHERE id = $1 AND status <> 'ready'`, + [job.attachment_id, finalCode], + ); + await this.event(client, job.job_id, 'failed', job.attempt, { + code: finalCode, + cause: failure.code, + }); + return 'failed'; + } + + const nextAttempt = job.attempt + 1; + await client.query( + `UPDATE attachment_jobs + SET status = 'queued', attempt = $2, + next_attempt_at = now() + make_interval(secs => $3::double precision), + lease_owner = NULL, lease_token = NULL, lease_until = NULL, + last_error_code = $4, updated_at = now() + WHERE job_id = $1`, + [job.job_id, nextAttempt, delayMs / 1000, failure.code], + ); + await client.query( + `UPDATE attachments SET status = 'queued', updated_at = now() + WHERE id = $1 AND status = 'processing'`, + [job.attachment_id], + ); + await client.query( + `INSERT INTO job_outbox (job_id, payload, available_at) + VALUES ($1, $2::jsonb, + now() + make_interval(secs => $3::double precision))`, + [ + job.job_id, + JSON.stringify( + buildJobMessage({ ...job, attempt: nextAttempt }), + ), + delayMs / 1000, + ], + ); + await this.event( + client, + job.job_id, + 'retry_scheduled', + nextAttempt, + { + code: failure.code, + delay_ms: delayMs, + }, + ); + return 'retry'; + }); + } + + async reapExpiredLeases( + limit: number, + delayFor: (attempt: number) => number, + ): Promise { + return this.database.withTransaction(async (client) => { + const expired = await client.query( + `SELECT j.*, a.status AS attachment_status, + a.ready_pipeline_version + FROM attachment_jobs j + JOIN attachments a ON a.id = j.attachment_id + WHERE j.status = 'processing' AND j.lease_until < now() + ORDER BY j.lease_until + LIMIT $1 + FOR UPDATE OF j, a`, + [limit], + ); + for (const job of expired.rows) { + if (job.attempt >= job.max_attempts) { + await client.query( + `UPDATE attachment_jobs + SET status = 'failed', last_error_code = 'retry_exhausted', + lease_owner = NULL, lease_token = NULL, + lease_until = NULL, updated_at = now(), + finished_at = now() + WHERE job_id = $1`, + [job.job_id], + ); + await client.query( + `UPDATE attachments + SET status = 'failed', error_code = 'retry_exhausted', + updated_at = now() + WHERE id = $1 AND status <> 'ready'`, + [job.attachment_id], + ); + await this.event( + client, + job.job_id, + 'lease_expired', + job.attempt, + { + lease_owner: job.lease_owner, + code: 'retry_exhausted', + }, + ); + continue; + } + + const nextAttempt = job.attempt + 1; + const delayMs = delayFor(job.attempt); + await client.query( + `UPDATE attachment_jobs + SET status = 'queued', attempt = $2, + next_attempt_at = now() + make_interval(secs => $3::double precision), + lease_owner = NULL, lease_token = NULL, + lease_until = NULL, last_error_code = 'lease_expired', + updated_at = now() + WHERE job_id = $1`, + [job.job_id, nextAttempt, delayMs / 1000], + ); + await client.query( + `UPDATE attachments SET status = 'queued', updated_at = now() + WHERE id = $1 AND status = 'processing'`, + [job.attachment_id], + ); + await client.query( + `INSERT INTO job_outbox (job_id, payload, available_at) + VALUES ($1, $2::jsonb, + now() + make_interval(secs => $3::double precision))`, + [ + job.job_id, + JSON.stringify( + buildJobMessage({ ...job, attempt: nextAttempt }), + ), + delayMs / 1000, + ], + ); + await this.event( + client, + job.job_id, + 'lease_expired', + job.attempt, + { lease_owner: job.lease_owner }, + ); + } + return expired.rows.length; + }); + } + + async publishPending( + limit: number, + publish: (message: JobMessage) => Promise, + afterPublish: () => Promise, + ): Promise { + return this.database.withTransaction(async (client) => { + const pending = await client.query<{ + id: string; + payload: JobMessage; + }>( + `SELECT id, payload FROM job_outbox + WHERE published_at IS NULL AND available_at <= now() + ORDER BY available_at, id + LIMIT $1 + FOR UPDATE SKIP LOCKED`, + [limit], + ); + const published: string[] = []; + let failure: unknown; + for (const row of pending.rows) { + try { + await publish(row.payload); + published.push(row.id); + } catch (error) { + failure = error; + break; + } + } + if (published.length === 0) { + if (failure) throw failure; + return 0; + } + await afterPublish(); + await client.query( + `UPDATE job_outbox SET published_at = now() WHERE id = ANY($1::bigint[])`, + [published], + ); + if (failure) throw failure; + return published.length; + }); + } + + async variantsFor( + attachmentId: string, + pipelineVersion: number, + ): Promise { + const result = await this.database.query( + 'jobs.variants', + `SELECT * FROM attachment_variants + WHERE attachment_id = $1 AND pipeline_version = $2 + ORDER BY array_position(ARRAY['thumbnail','crop','optimized']::text[], variant)`, + [attachmentId, pipelineVersion], + ); + return result.rows; + } + + async stagingCleanupCandidates( + retentionSeconds: number, + limit: number, + ): Promise<{ id: string; staging_key: string }[]> { + const result = await this.database.query<{ + id: string; + staging_key: string; + }>( + 'jobs.staging_candidates', + `SELECT id, staging_key FROM attachments + WHERE staging_key IS NOT NULL + AND ((status = 'ready' AND updated_at < now() - interval '1 minute') + OR (status = 'failed' AND updated_at < now() - make_interval(secs => $1::double precision))) + ORDER BY updated_at + LIMIT $2`, + [retentionSeconds, limit], + ); + return result.rows; + } + + async clearStaging(id: string, key: string): Promise { + return this.database.withTransaction(async (client) => { + const cleared = await client.query( + `UPDATE attachments SET staging_key = NULL, updated_at = now() + WHERE id = $1 AND staging_key = $2 AND status IN ('ready','failed')`, + [id, key], + ); + if (cleared.rowCount !== 1) return false; + await client.query( + `INSERT INTO job_events (job_id, event, attempt, detail) + SELECT job_id, 'staging_deleted', attempt, $2::jsonb + FROM attachment_jobs + WHERE attachment_id = $1 + ORDER BY created_at DESC + LIMIT 1`, + [id, JSON.stringify({ staging_key: key })], + ); + return true; + }); + } + + async statusOf(attachmentId: string): Promise< + | { + attachment: AttachmentRow; + jobs: AttachmentJobRow[]; + variants: VariantRow[]; + events: { + job_id: string; + event: string; + attempt: number; + detail: unknown; + created_at: Date; + }[]; + outbox: { pending: number; published: number }; + } + | undefined + > { + const attachment = await this.database.query( + 'jobs.status_attachment', + `SELECT * FROM attachments WHERE id = $1`, + [attachmentId], + ); + if (!attachment.rows[0]) return undefined; + const jobs = await this.database.query( + 'jobs.status_jobs', + `SELECT * FROM attachment_jobs WHERE attachment_id = $1 ORDER BY created_at`, + [attachmentId], + ); + const variants = await this.database.query( + 'jobs.status_variants', + `SELECT * FROM attachment_variants WHERE attachment_id = $1 + ORDER BY pipeline_version, + array_position(ARRAY['thumbnail','crop','optimized']::text[], variant)`, + [attachmentId], + ); + const events = await this.database.query<{ + job_id: string; + event: string; + attempt: number; + detail: unknown; + created_at: Date; + }>( + 'jobs.status_events', + `SELECT e.job_id, e.event, e.attempt, e.detail, e.created_at + FROM job_events e + JOIN attachment_jobs j ON j.job_id = e.job_id + WHERE j.attachment_id = $1 + ORDER BY e.id`, + [attachmentId], + ); + const outbox = await this.database.query<{ + pending: string; + published: string; + }>( + 'jobs.status_outbox', + `SELECT count(*) FILTER (WHERE o.published_at IS NULL) AS pending, + count(*) FILTER (WHERE o.published_at IS NOT NULL) AS published + FROM job_outbox o + JOIN attachment_jobs j ON j.job_id = o.job_id + WHERE j.attachment_id = $1`, + [attachmentId], + ); + return { + attachment: attachment.rows[0], + jobs: jobs.rows, + variants: variants.rows, + events: events.rows, + outbox: { + pending: Number(outbox.rows[0]?.pending ?? 0), + published: Number(outbox.rows[0]?.published ?? 0), + }, + }; + } + + async findByChat( + chatId: string, + ): Promise<{ id: string; status: string; created_at: Date }[]> { + const result = await this.database.query<{ + id: string; + status: string; + created_at: Date; + }>( + 'jobs.find_by_chat', + `SELECT id, status, created_at FROM attachments + WHERE chat_id = $1 ORDER BY created_at`, + [chatId], + ); + return result.rows; + } + + async failedJobs(): Promise< + { + attachment_id: string; + job_id: string; + pipeline_version: number; + attempt: number; + last_error_code: string | null; + updated_at: Date; + }[] + > { + const result = await this.database.query<{ + attachment_id: string; + job_id: string; + pipeline_version: number; + attempt: number; + last_error_code: string | null; + updated_at: Date; + }>( + 'jobs.failed', + `SELECT attachment_id, job_id, pipeline_version, attempt, + last_error_code, updated_at + FROM attachment_jobs WHERE status = 'failed' + ORDER BY updated_at`, + ); + return result.rows; + } + + async replayCandidate(attachmentId: string): Promise< + | { + job_id: string; + pipeline_version: number; + staging_key: string | null; + } + | undefined + > { + const result = await this.database.query<{ + job_id: string; + pipeline_version: number; + staging_key: string | null; + }>( + 'jobs.replay_candidate', + `SELECT j.job_id, j.pipeline_version, a.staging_key + FROM attachment_jobs j + JOIN attachments a ON a.id = j.attachment_id + WHERE j.attachment_id = $1 AND j.status = 'failed' + ORDER BY j.updated_at DESC + LIMIT 1`, + [attachmentId], + ); + return result.rows[0]; + } + + async replay( + attachmentId: string, + options: { + reason: string; + operator: string; + maxAttempts: number; + sourceAvailable: boolean; + }, + ): Promise { + return this.database.withTransaction(async (client) => { + const found = await client.query< + AttachmentJobRow & { staging_key: string | null } + >( + `SELECT j.*, a.staging_key + FROM attachment_jobs j + JOIN attachments a ON a.id = j.attachment_id + WHERE j.attachment_id = $1 AND j.status = 'failed' + ORDER BY j.updated_at DESC + LIMIT 1 + FOR UPDATE OF j, a`, + [attachmentId], + ); + const job = found.rows[0]; + if (!job) return { kind: 'refused', reason: 'no_failed_job' }; + + const attachment = await client.query( + `SELECT * FROM attachments WHERE id = $1`, + [attachmentId], + ); + const row = attachment.rows[0]; + if ( + row.status === 'ready' && + row.ready_pipeline_version !== null && + row.ready_pipeline_version >= job.pipeline_version + ) + return { + kind: 'refused', + reason: 'already_ready', + job_id: job.job_id, + }; + + const active = await client.query( + `SELECT 1 FROM attachment_jobs + WHERE attachment_id = $1 AND status IN ('queued','processing') + LIMIT 1`, + [attachmentId], + ); + if (active.rowCount !== 0) + return { + kind: 'refused', + reason: 'job_active', + job_id: job.job_id, + }; + + if (row.staging_key === null && !options.sourceAvailable) + return { + kind: 'refused', + reason: 'source_expired', + job_id: job.job_id, + }; + + const attempt = job.attempt + 1; + await client.query( + `UPDATE attachment_jobs + SET status = 'queued', attempt = $2, + max_attempts = $2::integer + $3::integer - 1, + next_attempt_at = now(), lease_owner = NULL, + lease_token = NULL, lease_until = NULL, + updated_at = now(), finished_at = NULL + WHERE job_id = $1`, + [job.job_id, attempt, options.maxAttempts], + ); + await client.query( + `UPDATE attachments SET status = 'queued', error_code = NULL, + updated_at = now() + WHERE id = $1 AND status = 'failed'`, + [attachmentId], + ); + await client.query( + `INSERT INTO job_outbox (job_id, payload, available_at) + VALUES ($1, $2::jsonb, now())`, + [ + job.job_id, + JSON.stringify(buildJobMessage({ ...job, attempt })), + ], + ); + await this.event(client, job.job_id, 'replayed', attempt, { + operator: options.operator, + reason: options.reason, + previous_error: job.last_error_code, + }); + return { kind: 'replayed', job_id: job.job_id, attempt }; + }); + } + + async reprocess( + attachmentId: string, + pipelineVersion: number, + options: { reason: string; operator: string; maxAttempts: number }, + ): Promise { + return this.database.withTransaction(async (client) => { + const attachment = await client.query( + `SELECT * FROM attachments WHERE id = $1 FOR UPDATE`, + [attachmentId], + ); + const row = attachment.rows[0]; + if ( + !row || + !['queued', 'processing', 'ready', 'failed'].includes( + row.status, + ) + ) + return { kind: 'refused', reason: 'not_accepted' }; + + const jobId = randomUUID(); + const created = await client.query( + `INSERT INTO attachment_jobs + (job_id, attachment_id, pipeline_version, status, attempt, + max_attempts, next_attempt_at) + VALUES ($1, $2, $3, 'queued', 1, $4, now()) + ON CONFLICT (attachment_id, pipeline_version) DO NOTHING + RETURNING *`, + [jobId, attachmentId, pipelineVersion, options.maxAttempts], + ); + const job = created.rows[0]; + if (!job) { + const existing = await client.query<{ job_id: string }>( + `SELECT job_id FROM attachment_jobs + WHERE attachment_id = $1 AND pipeline_version = $2`, + [attachmentId, pipelineVersion], + ); + return { + kind: 'refused', + reason: 'job_exists', + job_id: existing.rows[0]?.job_id, + }; + } + await client.query( + `INSERT INTO job_outbox (job_id, payload, available_at) + VALUES ($1, $2::jsonb, now())`, + [job.job_id, JSON.stringify(buildJobMessage(job))], + ); + await this.event(client, job.job_id, 'created', 1, { + reason: options.reason, + operator: options.operator, + }); + return { kind: 'created', job_id: job.job_id }; + }); + } + + private async skip( + client: PoolClient, + message: JobMessage, + reason: string, + ): Promise { + await this.event( + client, + message.job_id, + 'duplicate_skipped', + message.attempt, + { reason, message_attempt: message.attempt }, + ); + return { kind: 'skip', reason }; + } + + private async event( + client: PoolClient, + jobId: string, + event: string, + attempt: number, + detail: Record, + ): Promise { + await client.query( + `INSERT INTO job_events (job_id, event, attempt, detail) + VALUES ($1, $2, $3, $4::jsonb)`, + [jobId, event, attempt, JSON.stringify(detail)], + ); + } +} diff --git a/app/src/jobs/outbox-relay.service.ts b/app/src/jobs/outbox-relay.service.ts new file mode 100644 index 0000000..b734f59 --- /dev/null +++ b/app/src/jobs/outbox-relay.service.ts @@ -0,0 +1,92 @@ +import { + Injectable, + Logger, + OnApplicationBootstrap, + OnModuleDestroy, +} from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import type { AppConfig } from '../config/app-config.schema.js'; +import { MetricsService } from '../metrics/metrics.service.js'; +import { BrokerClient } from './broker.client.js'; +import { FaultInjector } from './fault-injector.js'; +import { JobsRepository } from './jobs.repository.js'; + +const BATCH_SIZE = 100; + +/** + * Runs only in the API process. `publishPending` holds one transaction around + * the broker confirms and the `published_at` update; a crash before the mark + * rolls the transaction back and republishes the same message after restart. + * That duplicate is expected: the worker fences it on job status. + */ +@Injectable() +export class OutboxRelay implements OnApplicationBootstrap, OnModuleDestroy { + private readonly logger = new Logger(OutboxRelay.name); + private timer: NodeJS.Timeout | undefined; + private running = false; + private again = false; + private current: Promise | undefined; + private failing = false; + + constructor( + private readonly config: ConfigService, + private readonly repository: JobsRepository, + private readonly broker: BrokerClient, + private readonly faults: FaultInjector, + private readonly metrics: MetricsService, + ) {} + + onApplicationBootstrap(): void { + const interval = this.config.getOrThrow('OUTBOX_POLL_INTERVAL_MS'); + this.timer = setInterval(() => this.kick(), interval); + this.timer.unref(); + this.kick(); + } + + async onModuleDestroy(): Promise { + clearInterval(this.timer); + this.timer = undefined; + await this.current; + } + + kick(): void { + if (this.running) { + this.again = true; + return; + } + this.running = true; + this.current = this.drain() + .catch((error: unknown) => { + this.metrics.increment('outbox.publish_failed'); + if (!this.failing) { + this.failing = true; + this.logger.warn( + `Outbox publish failed: ${error instanceof Error ? error.message : 'unknown error'}`, + ); + } + }) + .finally(() => { + this.running = false; + this.current = undefined; + if (this.again) { + this.again = false; + this.kick(); + } + }); + } + + private async drain(): Promise { + const published = await this.repository.publishPending( + BATCH_SIZE, + (message) => this.broker.publish(message), + () => this.faults.point('relay.after_publish_before_mark'), + ); + if (published > 0) { + this.metrics.add('outbox.published', published); + if (this.failing) { + this.failing = false; + this.logger.log('Outbox publishing recovered'); + } + } + } +} diff --git a/app/src/messages/messages.repository.ts b/app/src/messages/messages.repository.ts index 5b49919..a3f17ce 100644 --- a/app/src/messages/messages.repository.ts +++ b/app/src/messages/messages.repository.ts @@ -60,7 +60,8 @@ WITH claim AS ( SELECT 1 FROM chat_members WHERE chat_id = $7::uuid AND user_id = $1 ) AND (SELECT count(*) FROM attachments a - WHERE a.chat_id = $7::uuid AND a.status = 'ready' + WHERE a.chat_id = $7::uuid + AND a.status IN ('queued','processing','ready') AND a.id = ANY($8::uuid[])) = cardinality($8::uuid[]) ON CONFLICT DO NOTHING RETURNING response_body diff --git a/app/src/messages/messages.service.ts b/app/src/messages/messages.service.ts index 34bee78..1ab2f96 100644 --- a/app/src/messages/messages.service.ts +++ b/app/src/messages/messages.service.ts @@ -97,12 +97,15 @@ export class MessagesService { // чужую заявку сняли между нашими двумя запросами (503). if (!(await this.idempotency.isMember(chatId, currentUser.id))) throw new NotFoundException(); - // Unknown, foreign-chat or unfinished attachment: not found. - if ( - ids.length > 0 && - !(await this.attachments.allLinkable(chatId, ids)) - ) - throw new NotFoundException(); + // Unknown or foreign-chat ids are 404; an attachment whose + // processing failed is a conflict, not a missing object. + const linkState = + ids.length > 0 + ? await this.attachments.linkState(chatId, ids) + : 'ok'; + if (linkState === 'failed') + throw new ConflictException('Attachment processing failed'); + if (linkState === 'missing') throw new NotFoundException(); this.metrics.increment('idempotency.claim_lost'); throw new ServiceUnavailableException( 'Idempotency key could not be claimed', diff --git a/app/src/storage/object-storage.ts b/app/src/storage/object-storage.ts index cddbb5f..81f8ea3 100644 --- a/app/src/storage/object-storage.ts +++ b/app/src/storage/object-storage.ts @@ -48,9 +48,11 @@ export class ObjectStorage { private readonly bucket: string; private readonly internal: S3Client; private readonly presign: S3Client; + private readonly requestTimeoutMs: number; constructor(config: ConfigService) { this.bucket = config.getOrThrow('S3_BUCKET'); + this.requestTimeoutMs = config.getOrThrow('S3_REQUEST_TIMEOUT_MS'); const common = { region: config.getOrThrow('S3_REGION'), forcePathStyle: true, @@ -89,12 +91,25 @@ export class ObjectStorage { return `attachments/${attachmentId}/original`; } + stagingKey(attachmentId: string): string { + return `staging/${attachmentId}/original`; + } + + variantKey( + attachmentId: string, + pipelineVersion: number, + variant: 'thumbnail' | 'crop' | 'optimized', + extension: 'jpg' | 'webp', + ): string { + return `attachments/${attachmentId}/v${pipelineVersion}/${variant}.${extension}`; + } + async putStream( key: string, body: Readable, contentLength: number, contentType: string, - signal: AbortSignal, + signal?: AbortSignal, ): Promise { try { await this.internal.send( @@ -107,13 +122,31 @@ export class ObjectStorage { }), // The upload deadline (UPLOAD_TIMEOUT_MS, via `signal`) bounds the // PUT: it lasts as long as the client sends, not S3_REQUEST_TIMEOUT_MS. - { abortSignal: signal, requestTimeout: 0 }, + // A worker buffer has no client deadline, so it uses the SDK timeout. + signal + ? { abortSignal: signal, requestTimeout: 0 } + : { requestTimeout: this.requestTimeoutMs }, ); } catch (error) { throw this.fail('put', error); } } + async putBuffer( + key: string, + data: Buffer, + contentType: string, + signal?: AbortSignal, + ): Promise { + await this.putStream( + key, + Readable.from([data]), + data.length, + contentType, + signal, + ); + } + async getStream( key: string, signal?: AbortSignal, @@ -130,9 +163,12 @@ export class ObjectStorage { } } - async head( - key: string, - ): Promise<{ size: number; checksumSha256?: string } | null> { + async head(key: string): Promise<{ + size: number; + checksumSha256?: string; + etag?: string; + lastModified?: Date; + } | null> { try { const result = await this.internal.send( new HeadObjectCommand({ @@ -144,6 +180,8 @@ export class ObjectStorage { return { size: Number(result.ContentLength ?? 0), checksumSha256: result.ChecksumSHA256, + etag: result.ETag, + lastModified: result.LastModified, }; } catch (error) { if (isMissingKey(error)) return null; diff --git a/app/src/worker.main.ts b/app/src/worker.main.ts new file mode 100644 index 0000000..77dfea6 --- /dev/null +++ b/app/src/worker.main.ts @@ -0,0 +1,67 @@ +import { Logger } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { NestFactory } from '@nestjs/core'; +import http from 'node:http'; +import type { AppConfig } from './config/app-config.schema.js'; +import { BrokerClient } from './jobs/broker.client.js'; +import { JobProcessor } from './worker/job-processor.service.js'; +import { WorkerModule } from './worker/worker.module.js'; + +const DRAIN_MS = 30_000; + +async function bootstrap(): Promise { + const app = await NestFactory.createApplicationContext(WorkerModule, { + logger: ['error', 'warn', 'log'], + }); + const config = app.get(ConfigService); + const broker = app.get(BrokerClient); + const processor = app.get(JobProcessor); + const logger = new Logger('worker'); + + const server = http.createServer((request, response) => { + if (request.method === 'GET' && request.url === '/health') { + const consuming = broker.isConsuming(); + response.writeHead(consuming ? 200 : 503, { + 'content-type': 'application/json', + }); + response.end( + JSON.stringify({ + status: consuming ? 'ok' : 'not_consuming', + }), + ); + return; + } + response.writeHead(404, { 'content-type': 'application/json' }); + response.end(JSON.stringify({ status: 'not_found' })); + }); + await new Promise((resolve) => + server.listen( + config.getOrThrow('WORKER_HEALTH_PORT'), + '127.0.0.1', + resolve, + ), + ); + logger.log( + `worker ready: id=${process.env.INSTANCE_ID ?? process.env.HOSTNAME ?? 'worker'}`, + ); + + let shuttingDown = false; + const shutdown = async (signal: string): Promise => { + if (shuttingDown) return; + shuttingDown = true; + logger.log( + `${signal}: stopping consumption and draining in-flight jobs`, + ); + await broker.stopConsuming(); + await processor.drain(DRAIN_MS); + await new Promise((resolve) => server.close(() => resolve())); + await app.close(); + process.exit(0); + }; + for (const signal of ['SIGTERM', 'SIGINT'] as const) + process.once(signal, () => { + void shutdown(signal); + }); +} + +await bootstrap(); diff --git a/app/src/worker/job-processor.service.ts b/app/src/worker/job-processor.service.ts new file mode 100644 index 0000000..db69131 --- /dev/null +++ b/app/src/worker/job-processor.service.ts @@ -0,0 +1,405 @@ +import { Injectable, Logger, OnApplicationBootstrap } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import type { Channel, ConsumeMessage } from 'amqplib'; +import { createHash } from 'node:crypto'; +import os from 'node:os'; +import type { AttachmentRow } from '../attachments/attachment.js'; +import type { AppConfig } from '../config/app-config.schema.js'; +import { ObjectStorage } from '../storage/object-storage.js'; +import { retryDelayMs } from '../jobs/backoff.js'; +import { BrokerClient } from '../jobs/broker.client.js'; +import { + classifyError, + JobTimeoutError, + SourceMismatchError, + SourceMissingError, + type ErrorClass, +} from '../jobs/errors.js'; +import { FaultInjector } from '../jobs/fault-injector.js'; +import { parseJobMessage } from '../jobs/job-message.js'; +import { + JobsRepository, + type ClaimResult, + type VariantWrite, +} from '../jobs/jobs.repository.js'; +import { MetricsService } from '../metrics/metrics.service.js'; +import { renderVariants, type RenderedVariant } from './pipelines.js'; + +const SHUTDOWN_DRAIN_MS = 30_000; + +function throwIfAborted(signal: AbortSignal): void { + if (!signal.aborted) return; + throw signal.reason instanceof Error ? signal.reason : new Error('Aborted'); +} + +function sha256(data: Buffer): string { + return createHash('sha256').update(data).digest('hex'); +} + +@Injectable() +export class JobProcessor implements OnApplicationBootstrap { + private readonly logger = new Logger(JobProcessor.name); + private readonly workerId = process.env.INSTANCE_ID ?? os.hostname(); + private readonly maxPixels: number; + private readonly timeoutSeconds: number; + private readonly leaseSeconds: number; + private readonly retryBaseMs: number; + private readonly retryMaxMs: number; + private readonly maxUploadBytes: number; + private active = 0; + private readonly idleWaiters: (() => void)[] = []; + + constructor( + private readonly config: ConfigService, + private readonly broker: BrokerClient, + private readonly jobs: JobsRepository, + private readonly faults: FaultInjector, + private readonly storage: ObjectStorage, + private readonly metrics: MetricsService, + ) { + this.maxPixels = config.getOrThrow('MAX_IMAGE_PIXELS'); + this.timeoutSeconds = config.getOrThrow('JOB_TIMEOUT_SECONDS'); + this.leaseSeconds = config.getOrThrow('JOB_LEASE_SECONDS'); + this.retryBaseMs = config.getOrThrow('JOB_RETRY_BASE_MS'); + this.retryMaxMs = config.getOrThrow('JOB_RETRY_MAX_MS'); + this.maxUploadBytes = config.getOrThrow('MAX_UPLOAD_BYTES'); + } + + onApplicationBootstrap(): void { + this.broker.consume( + this.config.getOrThrow('WORKER_CONCURRENCY'), + (message, channel) => this.handle(message, channel), + ); + } + + async drain(timeoutMs = SHUTDOWN_DRAIN_MS): Promise { + if (this.active === 0) return; + const { promise, resolve } = Promise.withResolvers(); + this.idleWaiters.push(resolve); + const timer = setTimeout(resolve, timeoutMs); + await promise; + clearTimeout(timer); + } + + private async handle( + message: ConsumeMessage, + channel: Channel, + ): Promise { + const parsed = parseJobMessage(message.content); + if (!parsed) { + this.metrics.increment('worker.messages.poison'); + this.safeNack(channel, message, false); + return; + } + + let claim: ClaimResult; + try { + claim = await this.jobs.claim( + parsed, + this.workerId, + this.leaseSeconds, + ); + } catch (error) { + this.logger.warn( + `Claim failed: ${error instanceof Error ? error.message : 'unknown error'}`, + ); + this.safeNack(channel, message, true); + return; + } + if (claim.kind === 'skip') { + if (claim.reason === 'mismatch') { + this.safeNack(channel, message, false); + return; + } + this.metrics.increment(`worker.jobs.skipped.${claim.reason}`); + this.safeAck(channel, message); + return; + } + + this.begin(); + try { + await this.process( + claim.job, + claim.attachment, + claim.leaseToken, + message, + channel, + ); + } finally { + this.end(); + } + } + + private async process( + job: { + job_id: string; + attachment_id: string; + pipeline_version: number; + attempt: number; + }, + attachment: AttachmentRow, + leaseToken: string, + message: ConsumeMessage, + channel: Channel, + ): Promise { + this.metrics.increment('worker.jobs.claimed'); + const controller = new AbortController(); + const target = { + attachmentId: job.attachment_id, + pipelineVersion: job.pipeline_version, + }; + const timeout = setTimeout( + () => controller.abort(new JobTimeoutError(this.timeoutSeconds)), + this.timeoutSeconds * 1000, + ); + const heartbeat = setInterval( + () => { + this.jobs + .extendLease(job.job_id, leaseToken, this.leaseSeconds) + .then((extended) => { + if (!extended && !controller.signal.aborted) + controller.abort(new Error('Lease lost')); + }) + .catch((error: unknown) => { + this.logger.warn( + `Lease heartbeat failed: ${error instanceof Error ? error.message : 'unknown error'}`, + ); + }); + }, + Math.max(1000, (this.leaseSeconds * 1000) / 3), + ); + let committed = false; + + try { + await this.faults.point( + 'worker.before_side_effect', + target, + controller.signal, + ); + throwIfAborted(controller.signal); + const source = await this.ensureFinalOriginal( + attachment, + controller.signal, + ); + throwIfAborted(controller.signal); + const variants = await renderVariants( + source, + job.pipeline_version, + { + maxPixels: this.maxPixels, + timeoutSeconds: this.timeoutSeconds, + }, + ); + throwIfAborted(controller.signal); + for (const variant of variants) { + await this.storage.putBuffer( + this.variantKey(job, variant), + variant.data, + variant.contentType, + controller.signal, + ); + } + await this.faults.point( + 'worker.after_side_effect', + target, + controller.signal, + ); + throwIfAborted(controller.signal); + + const result = await this.jobs.commitSuccess( + job.job_id, + leaseToken, + variants.map((variant) => this.variantWrite(job, variant)), + ); + committed = true; + this.metrics.increment(`worker.jobs.${result}`); + await this.faults.point( + 'worker.after_commit_before_ack', + target, + controller.signal, + ); + this.safeAck(channel, message); + if (result === 'succeeded') await this.cleanupStaging(attachment); + } catch (error) { + if (committed) { + // The durable effect is already committed; a failure at the ack + // fault point or a lost reply must not turn into a retry. + this.safeAck(channel, message); + return; + } + const failure = this.classify(error, controller.signal); + const delay = retryDelayMs( + job.attempt, + this.retryBaseMs, + this.retryMaxMs, + ); + try { + const outcome = await this.jobs.commitFailure( + job.job_id, + leaseToken, + failure, + delay, + ); + this.metrics.increment(`worker.jobs.${outcome}`); + this.safeAck(channel, message); + } catch (commitError) { + this.metrics.increment('worker.jobs.commit_failed'); + this.logger.warn( + `Failure commit failed: ${commitError instanceof Error ? commitError.message : 'unknown error'}`, + ); + setTimeout(() => this.safeNack(channel, message, true), 1000); + } + } finally { + clearTimeout(timeout); + clearInterval(heartbeat); + } + } + + private classify(error: unknown, signal: AbortSignal): ErrorClass { + if (signal.aborted && signal.reason instanceof JobTimeoutError) + return classifyError(signal.reason); + return classifyError(error); + } + + private async ensureFinalOriginal( + attachment: AttachmentRow, + signal: AbortSignal, + ): Promise { + const expectedSize = Number(attachment.size_bytes); + const head = await this.storage.head(attachment.object_key); + if (head !== null && head.size === expectedSize) { + const data = await this.readObject(attachment.object_key, signal); + this.verifySource(attachment, data); + return data; + } + + const stagingKey = attachment.staging_key; + if (stagingKey === null) throw new SourceMissingError(); + const data = await this.readObject(stagingKey, signal); + this.verifySource(attachment, data); + await this.storage.putBuffer( + attachment.object_key, + data, + attachment.content_type, + signal, + ); + this.metrics.increment('worker.original.published'); + return data; + } + + private async readObject( + key: string, + signal: AbortSignal, + ): Promise { + const stream = await this.storage.getStream(key, signal); + if (stream === null) throw new SourceMissingError(); + const chunks: Buffer[] = []; + let total = 0; + const destroy = (): void => { + stream.destroy( + signal.reason instanceof Error + ? signal.reason + : new Error('Aborted'), + ); + }; + signal.addEventListener('abort', destroy, { once: true }); + try { + for await (const chunk of stream as AsyncIterable) { + total += chunk.length; + if (total > this.maxUploadBytes) + throw new SourceMismatchError( + 'Source exceeds MAX_UPLOAD_BYTES', + ); + chunks.push(chunk); + } + } catch (error) { + if (signal.aborted) + throw signal.reason instanceof Error ? signal.reason : error; + throw error; + } finally { + signal.removeEventListener('abort', destroy); + stream.destroy(); + } + return Buffer.concat(chunks, total); + } + + private verifySource(attachment: AttachmentRow, data: Buffer): void { + if ( + data.length !== Number(attachment.size_bytes) || + sha256(data) !== attachment.sha256 + ) + throw new SourceMismatchError(); + } + + private variantKey( + job: { attachment_id: string; pipeline_version: number }, + variant: RenderedVariant, + ): string { + return this.storage.variantKey( + job.attachment_id, + job.pipeline_version, + variant.name, + variant.ext, + ); + } + + private variantWrite( + job: { attachment_id: string; pipeline_version: number }, + variant: RenderedVariant, + ): VariantWrite { + return { + name: variant.name, + objectKey: this.variantKey(job, variant), + contentType: variant.contentType, + sizeBytes: variant.data.length, + width: variant.width, + height: variant.height, + sha256: variant.sha256, + }; + } + + private async cleanupStaging(attachment: AttachmentRow): Promise { + const key = attachment.staging_key; + if (key === null) return; + try { + await this.storage.delete(key); + await this.jobs.clearStaging(attachment.id, key); + this.metrics.increment('worker.staging.deleted'); + } catch (error) { + this.metrics.increment('worker.staging.error'); + this.logger.warn( + `Staging cleanup for ${attachment.id} deferred: ${error instanceof Error ? error.message : 'unknown error'}`, + ); + } + } + + private safeAck(channel: Channel, message: ConsumeMessage): void { + try { + channel.ack(message); + } catch { + // A closed channel redelivers; the DB fence makes the duplicate a no-op. + } + } + + private safeNack( + channel: Channel, + message: ConsumeMessage, + requeue: boolean, + ): void { + try { + channel.nack(message, false, requeue); + } catch { + // The broker redelivers after the delivery limit or lease recovery. + } + } + + private begin(): void { + this.active += 1; + } + + private end(): void { + this.active -= 1; + if (this.active !== 0) return; + for (const resolve of this.idleWaiters.splice(0)) resolve(); + } +} diff --git a/app/src/worker/lease-reaper.service.ts b/app/src/worker/lease-reaper.service.ts new file mode 100644 index 0000000..1da1367 --- /dev/null +++ b/app/src/worker/lease-reaper.service.ts @@ -0,0 +1,64 @@ +import { + Injectable, + Logger, + OnModuleDestroy, + OnModuleInit, +} from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import type { AppConfig } from '../config/app-config.schema.js'; +import { retryDelayMs } from '../jobs/backoff.js'; +import { JobsRepository } from '../jobs/jobs.repository.js'; +import { MetricsService } from '../metrics/metrics.service.js'; + +const REAP_BATCH = 20; + +/** + * The reaper is the backstop for a worker that was killed mid-job: the lease + * expires and the job returns to `queued` without waiting for the broker's + * delivery limit. + */ +@Injectable() +export class LeaseReaper implements OnModuleInit, OnModuleDestroy { + private readonly logger = new Logger(LeaseReaper.name); + private timer: NodeJS.Timeout | undefined; + private running = false; + + constructor( + private readonly config: ConfigService, + private readonly jobs: JobsRepository, + private readonly metrics: MetricsService, + ) {} + + onModuleInit(): void { + const interval = this.config.getOrThrow('LEASE_REAPER_INTERVAL_MS'); + const base = this.config.getOrThrow('JOB_RETRY_BASE_MS'); + const cap = this.config.getOrThrow('JOB_RETRY_MAX_MS'); + this.timer = setInterval(() => { + if (this.running) return; + this.running = true; + this.jobs + .reapExpiredLeases(REAP_BATCH, (attempt) => + retryDelayMs(attempt, base, cap), + ) + .then((count) => { + if (count > 0) + this.metrics.add('worker.leases.reaped', count); + }) + .catch((error: unknown) => { + this.metrics.increment('worker.leases.reap_error'); + this.logger.warn( + `Lease reaper failed: ${error instanceof Error ? error.message : 'unknown error'}`, + ); + }) + .finally(() => { + this.running = false; + }); + }, interval); + this.timer.unref(); + } + + onModuleDestroy(): void { + clearInterval(this.timer); + this.timer = undefined; + } +} diff --git a/app/src/worker/pipelines.spec.ts b/app/src/worker/pipelines.spec.ts new file mode 100644 index 0000000..43f0858 --- /dev/null +++ b/app/src/worker/pipelines.spec.ts @@ -0,0 +1,90 @@ +import { readFileSync } from 'node:fs'; +import { describe, expect, it } from 'vitest'; +import sharp from 'sharp'; +import { renderVariants, type RenderedVariant } from './pipelines.js'; + +const sample = readFileSync( + new URL('../../../tests/fixtures/sample.png', import.meta.url), +); +const limits = { maxPixels: 40_000_000, timeoutSeconds: 30 }; + +function byName( + variants: RenderedVariant[], + name: 'thumbnail' | 'crop' | 'optimized', +): RenderedVariant { + const variant = variants.find((item) => item.name === name); + if (!variant) throw new Error(`missing ${name}`); + return variant; +} + +describe('renderVariants', () => { + it('renders the v1 contract geometry and formats', async () => { + const variants = await renderVariants(sample, 1, limits); + const thumbnail = byName(variants, 'thumbnail'); + const crop = byName(variants, 'crop'); + const optimized = byName(variants, 'optimized'); + + expect([thumbnail.width, thumbnail.height]).toEqual([256, 160]); + expect(thumbnail.contentType).toBe('image/jpeg'); + expect((await sharp(thumbnail.data).metadata()).format).toBe('jpeg'); + + expect([crop.width, crop.height]).toEqual([128, 128]); + expect((await sharp(crop.data).metadata()).format).toBe('jpeg'); + + expect([optimized.width, optimized.height]).toEqual([320, 200]); + expect(optimized.contentType).toBe('image/webp'); + expect((await sharp(optimized.data).metadata()).format).toBe('webp'); + }); + + it('applies EXIF orientation before fitting', async () => { + const oriented = await sharp({ + create: { + width: 300, + height: 200, + channels: 3, + background: { r: 20, g: 80, b: 140 }, + }, + }) + .jpeg() + .withMetadata({ orientation: 6 }) + .toBuffer(); + const variants = await renderVariants(oriented, 1, limits); + const thumbnail = byName(variants, 'thumbnail'); + expect([thumbnail.width, thumbnail.height]).toEqual([170, 256]); + }); + + it('composites transparent pixels onto white', async () => { + const transparent = await sharp({ + create: { + width: 10, + height: 10, + channels: 4, + background: { r: 0, g: 0, b: 0, alpha: 0 }, + }, + }) + .png() + .toBuffer(); + const variants = await renderVariants(transparent, 1, limits); + const thumbnail = byName(variants, 'thumbnail'); + const { data } = await sharp(thumbnail.data) + .raw() + .toBuffer({ resolveWithObject: true }); + expect([...data.subarray(0, 3)]).toEqual([255, 255, 255]); + }); + + it('is deterministic for the same input and version', async () => { + const first = await renderVariants(sample, 1, limits); + const second = await renderVariants(sample, 1, limits); + for (const name of ['thumbnail', 'crop', 'optimized'] as const) + expect(byName(first, name).sha256).toBe( + byName(second, name).sha256, + ); + }); + + it('produces a different v2 thumbnail', async () => { + const v1 = byName(await renderVariants(sample, 1, limits), 'thumbnail'); + const v2 = byName(await renderVariants(sample, 2, limits), 'thumbnail'); + expect(v1.sha256).not.toBe(v2.sha256); + expect([v2.width, v2.height]).toEqual([256, 160]); + }); +}); diff --git a/app/src/worker/pipelines.ts b/app/src/worker/pipelines.ts new file mode 100644 index 0000000..2795369 --- /dev/null +++ b/app/src/worker/pipelines.ts @@ -0,0 +1,142 @@ +import { createHash } from 'node:crypto'; +import sharp from 'sharp'; +import { UnsupportedPipelineError } from '../jobs/errors.js'; +import { fitInside } from '../jobs/fit-inside.js'; + +// Same policy as the API decoder: one libvips worker, no operation cache. +sharp.cache(false); +sharp.concurrency(1); + +type SharpImage = ReturnType; + +export const SUPPORTED_PIPELINE_VERSIONS = [1, 2] as const; + +export type RenderedVariant = { + name: 'thumbnail' | 'crop' | 'optimized'; + ext: 'jpg' | 'webp'; + contentType: 'image/jpeg' | 'image/webp'; + data: Buffer; + width: number; + height: number; + sha256: string; +}; + +export type RenderLimits = { + maxPixels: number; + timeoutSeconds: number; +}; + +function input(data: Buffer, limits: RenderLimits): SharpImage { + return sharp(data, { + failOn: 'warning', + limitInputPixels: limits.maxPixels, + sequentialRead: true, + }) + .timeout({ seconds: limits.timeoutSeconds }) + .rotate(); +} + +function jpegV1(image: SharpImage): SharpImage { + return image.jpeg({ quality: 80 }); +} + +function jpegV2(image: SharpImage): SharpImage { + return image.jpeg({ quality: 90, progressive: true, mozjpeg: true }); +} + +function rendered( + name: RenderedVariant['name'], + ext: RenderedVariant['ext'], + contentType: RenderedVariant['contentType'], + data: Buffer, + width: number, + height: number, +): RenderedVariant { + return { + name, + ext, + contentType, + data, + width, + height, + sha256: createHash('sha256').update(data).digest('hex'), + }; +} + +async function output( + image: SharpImage, + name: RenderedVariant['name'], + ext: RenderedVariant['ext'], + contentType: RenderedVariant['contentType'], +): Promise { + const { data, info } = await image.toBuffer({ resolveWithObject: true }); + return rendered(name, ext, contentType, data, info.width, info.height); +} + +/** + * Renders the thumbnail, crop and optimized variants from one decoded source. + * The input dimensions are EXIF-oriented: `.rotate()` applies the orientation + * and `metadata().autoOrient` describes the result, so 300x200 orientation 6 + * becomes 200x300 before fitting. + */ +export async function renderVariants( + data: Buffer, + pipelineVersion: number, + limits: RenderLimits, +): Promise { + if ( + !(SUPPORTED_PIPELINE_VERSIONS as readonly number[]).includes( + pipelineVersion, + ) + ) + throw new UnsupportedPipelineError(pipelineVersion); + + const metadata = await sharp(data, { + failOn: 'warning', + limitInputPixels: limits.maxPixels, + sequentialRead: true, + }).metadata(); + const width = metadata.autoOrient?.width ?? metadata.width; + const height = metadata.autoOrient?.height ?? metadata.height; + const [thumbnailWidth, thumbnailHeight] = fitInside(width, height, 256); + const [optimizedWidth, optimizedHeight] = fitInside(width, height, 1280); + const v2 = pipelineVersion >= 2; + + const thumbnail = v2 ? jpegV2 : jpegV1; + const thumbnailOutput = await output( + thumbnail( + input(data, limits) + .flatten({ background: '#ffffff' }) + .resize(thumbnailWidth, thumbnailHeight, { fit: 'fill' }), + ), + 'thumbnail', + 'jpg', + 'image/jpeg', + ); + + const cropOutput = await output( + thumbnail( + input(data, limits) + .flatten({ background: '#ffffff' }) + .resize(128, 128, { fit: 'cover', position: 'centre' }), + ), + 'crop', + 'jpg', + 'image/jpeg', + ); + + const optimizedOutput = await output( + input(data, limits) + .resize(optimizedWidth, optimizedHeight, { fit: 'fill' }) + .webp( + v2 + ? { quality: 82, effort: 5, smartSubsample: true } + : { quality: 75, effort: 4 }, + ), + 'optimized', + 'webp', + 'image/webp', + ); + + return [thumbnailOutput, cropOutput, optimizedOutput]; +} diff --git a/app/src/worker/staging-cleaner.service.ts b/app/src/worker/staging-cleaner.service.ts new file mode 100644 index 0000000..e1ce13c --- /dev/null +++ b/app/src/worker/staging-cleaner.service.ts @@ -0,0 +1,84 @@ +import { + Injectable, + Logger, + OnModuleDestroy, + OnModuleInit, +} from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import type { AppConfig } from '../config/app-config.schema.js'; +import { JobsRepository } from '../jobs/jobs.repository.js'; +import { MetricsService } from '../metrics/metrics.service.js'; +import { ObjectStorage } from '../storage/object-storage.js'; + +const CLEANUP_BATCH = 100; + +/** + * Removes staging objects after the worker committed the final copy, and after + * the documented retention for failed jobs. A ready row waits one minute so a + * just-acknowledged job is never raced by the cleaner. + */ +@Injectable() +export class StagingCleaner implements OnModuleInit, OnModuleDestroy { + private readonly logger = new Logger(StagingCleaner.name); + private timer: NodeJS.Timeout | undefined; + private running = false; + + constructor( + private readonly config: ConfigService, + private readonly jobs: JobsRepository, + private readonly storage: ObjectStorage, + private readonly metrics: MetricsService, + ) {} + + onModuleInit(): void { + const interval = this.config.getOrThrow('STAGING_CLEANUP_INTERVAL_MS'); + if (interval <= 0) return; + this.timer = setInterval(() => { + if (this.running) return; + this.running = true; + this.cleanupOnce() + .catch((error: unknown) => { + this.metrics.increment('worker.staging.cleanup_error'); + this.logger.warn( + `Staging cleanup failed: ${error instanceof Error ? error.message : 'unknown error'}`, + ); + }) + .finally(() => { + this.running = false; + }); + }, interval); + this.timer.unref(); + } + + onModuleDestroy(): void { + clearInterval(this.timer); + this.timer = undefined; + } + + async cleanupOnce(): Promise { + const candidates = await this.jobs.stagingCleanupCandidates( + this.config.getOrThrow('STAGING_RETENTION_SECONDS'), + CLEANUP_BATCH, + ); + let deleted = 0; + for (const candidate of candidates) { + try { + await this.storage.delete(candidate.staging_key); + if ( + await this.jobs.clearStaging( + candidate.id, + candidate.staging_key, + ) + ) + deleted += 1; + } catch (error) { + this.metrics.increment('worker.staging.cleanup_error'); + this.logger.warn( + `Staging cleanup of ${candidate.id} failed: ${error instanceof Error ? error.message : 'unknown error'}`, + ); + } + } + if (deleted > 0) this.metrics.add('worker.staging.deleted', deleted); + return deleted; + } +} diff --git a/app/src/worker/worker.module.ts b/app/src/worker/worker.module.ts new file mode 100644 index 0000000..2ec87d9 --- /dev/null +++ b/app/src/worker/worker.module.ts @@ -0,0 +1,23 @@ +import { Module } from '@nestjs/common'; +import { AppConfigModule } from '../config/app-config.module.js'; +import { DatabaseModule } from '../database/database.module.js'; +import { JobsModule } from '../jobs/jobs.module.js'; +import { MetricsModule } from '../metrics/metrics.module.js'; +import { RedisModule } from '../redis/redis.module.js'; +import { StorageModule } from '../storage/storage.module.js'; +import { JobProcessor } from './job-processor.service.js'; +import { LeaseReaper } from './lease-reaper.service.js'; +import { StagingCleaner } from './staging-cleaner.service.js'; + +@Module({ + imports: [ + AppConfigModule, + DatabaseModule, + RedisModule, + MetricsModule, + StorageModule, + JobsModule, + ], + providers: [JobProcessor, LeaseReaper, StagingCleaner], +}) +export class WorkerModule {} diff --git a/compose.yaml b/compose.yaml index b7209bc..4e07106 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,4 +1,8 @@ -# Стенд lab5 (база lab4): клиент -> HAProxy -> API x N -> PostgreSQL + Redis + MinIO. +# Стенд lab5 (база lab4): клиент -> HAProxy -> API x N -> PostgreSQL + Redis + MinIO +# + RabbitMQ; отдельный worker читает задания из quorum-очереди и публикует +# оригинал и варианты в конечный S3-prefix. API отвечает 202 после COMMIT +# (staging-объект + attachment + job + outbox), поэтому брокер не входит в +# /health/ready и его недоступность не останавливает приём загрузок. # Одна команда запуска: make stand-up (docker compose --env-file .env up -d --build --wait) # # Проект запинен именем, чтобы имена контейнеров и labels были предсказуемы для @@ -80,6 +84,31 @@ services: timeout: 3s retries: 20 + broker: + image: rabbitmq:4.1.8-alpine@sha256:153322e3974e6365c50e523e5eadd5a7fae231f0e8c58f67c03bb3e02056e34e + # Имя узла rabbit@broker должно оставаться стабильным: при смене + # hostname данные в broker-data превращаются в сирот. + hostname: broker + secrets: + - source: rabbitmq_conf + target: /etc/rabbitmq/conf.d/20-credentials.conf + volumes: + - broker-data:/var/lib/rabbitmq + # Портов нет: брокер доступен только сервисам compose-сети. + deploy: + resources: + limits: + cpus: '${BROKER_CPUS:-0.5}' + memory: ${BROKER_MEMORY:-512m} + healthcheck: + test: ['CMD', 'rabbitmq-diagnostics', '-q', 'check_port_connectivity'] + interval: 5s + timeout: 5s + retries: 30 + start_period: 20s + stop_grace_period: 30s + restart: unless-stopped + s3: image: minio/minio:RELEASE.2025-09-07T16-13-09Z@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e command: ['server', '/data', '--address', ':9000'] @@ -117,6 +146,8 @@ services: - minio_root_password - s3_api_access_key_id - s3_api_secret_access_key + - s3_worker_access_key_id + - s3_worker_secret_access_key depends_on: s3: { condition: service_healthy } restart: 'no' @@ -172,7 +203,16 @@ services: # libvips держит декодируемое изображение в памяти, а не во временном # файле: /tmp — tmpfs 16 MiB, и изображение ~40 Мп туда не помещается. VIPS_DISC_THRESHOLD: ${VIPS_DISC_THRESHOLD:-1g} - secrets: [s3_api_access_key_id, s3_api_secret_access_key] + FAULT_INJECTION_ENABLED: ${FAULT_INJECTION_ENABLED:-0} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_RETRY_BASE_MS: ${JOB_RETRY_BASE_MS:-1000} + JOB_RETRY_MAX_MS: ${JOB_RETRY_MAX_MS:-30000} + # Outbox закрывает недоступность брокера: API принимает загрузку и + # отвечает 202 без публикации, relay доставит задание после восстановления. + secrets: + - s3_api_access_key_id + - s3_api_secret_access_key + - broker_url # Корневая ФС только для чтения: локального файла оригинала на API нет. read_only: true tmpfs: ['/tmp:size=16m'] @@ -201,6 +241,64 @@ services: timeout: 3s retries: 20 + # Worker — отдельный процесс без HTTP API. Он читает задания из quorum- + # очереди, держит lease с heartbeat, пишет варианты и коммитит результат в + # PostgreSQL. Остановка worker не блокирует приём загрузок API. + worker: + build: + context: . + dockerfile: Dockerfile + args: + BUILD_ID: ${BUILD_ID:-unknown} + image: ${API_IMAGE:-mtusi-messenger:lab5} + command: ['node', 'dist/worker.main.js'] + env_file: [.env] + environment: + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_worker_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_worker_secret_access_key + VIPS_DISC_THRESHOLD: ${VIPS_DISC_THRESHOLD:-1g} + FAULT_INJECTION_ENABLED: ${FAULT_INJECTION_ENABLED:-0} + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-30} + JOB_RETRY_BASE_MS: ${JOB_RETRY_BASE_MS:-1000} + JOB_RETRY_MAX_MS: ${JOB_RETRY_MAX_MS:-30000} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + WORKER_HEALTH_PORT: ${WORKER_HEALTH_PORT:-8081} + secrets: + - s3_worker_access_key_id + - s3_worker_secret_access_key + - broker_url + read_only: true + tmpfs: ['/tmp:size=16m'] + deploy: + replicas: ${WORKER_REPLICAS:-1} + resources: + limits: + cpus: '${WORKER_CPUS:-1.5}' + memory: ${WORKER_MEMORY:-1280m} + stop_grace_period: 40s + restart: unless-stopped + depends_on: + migrate: { condition: service_completed_successfully } + postgres: { condition: service_healthy } + redis: { condition: service_healthy } + broker: { condition: service_healthy } + s3-init: { condition: service_completed_successfully } + healthcheck: + test: + [ + 'CMD', + 'node', + '-e', + "fetch('http://127.0.0.1:'+ (process.env.WORKER_HEALTH_PORT||8081) + '/health').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))", + ] + interval: 5s + timeout: 3s + retries: 20 + start_period: 20s + haproxy: image: haproxy:3.2-alpine ports: @@ -223,8 +321,13 @@ secrets: minio_root_password: { file: ./secrets/minio_root_password } s3_api_access_key_id: { file: ./secrets/s3_api_access_key_id } s3_api_secret_access_key: { file: ./secrets/s3_api_secret_access_key } + s3_worker_access_key_id: { file: ./secrets/s3_worker_access_key_id } + s3_worker_secret_access_key: { file: ./secrets/s3_worker_secret_access_key } + broker_url: { file: ./secrets/broker_url } + rabbitmq_conf: { file: ./secrets/rabbitmq.conf } volumes: postgres-data: redis-data: s3-data: + broker-data: diff --git a/infra/minio/bootstrap.sh b/infra/minio/bootstrap.sh index 17baad9..cd3c4ce 100755 --- a/infra/minio/bootstrap.sh +++ b/infra/minio/bootstrap.sh @@ -7,6 +7,8 @@ IFS= read -r root_user < /run/secrets/minio_root_user || [ -n "$root_user" ] IFS= read -r root_password < /run/secrets/minio_root_password || [ -n "$root_password" ] IFS= read -r api_key < /run/secrets/s3_api_access_key_id || [ -n "$api_key" ] IFS= read -r api_secret < /run/secrets/s3_api_secret_access_key || [ -n "$api_secret" ] +IFS= read -r worker_key < /run/secrets/s3_worker_access_key_id || [ -n "$worker_key" ] +IFS= read -r worker_secret < /run/secrets/s3_worker_secret_access_key || [ -n "$worker_secret" ] bucket="${S3_BUCKET:?}" export MC_CONFIG_DIR=/tmp/.mc @@ -15,14 +17,42 @@ export MC_HOST_stand="http://${root_user}:${root_password}@s3:9000" mc mb --ignore-existing "stand/$bucket" mc anonymous set none "stand/$bucket" -printf '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject","s3:DeleteObject"],"Resource":["arn:aws:s3:::%s/attachments/*"]},{"Effect":"Allow","Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::%s"]}]}' "$bucket" "$bucket" > /tmp/messenger-api-policy.json -mc admin policy create stand messenger-api /tmp/messenger-api-policy.json +# The API owns the staging prefix only: it writes the accepted upload there and +# can remove it after the worker commits the result. The final prefix is read +# for variant/original downloads and legacy lab4 originals, but never written. +printf '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject","s3:DeleteObject"],"Resource":["arn:aws:s3:::%s/staging/*"]},{"Effect":"Allow","Action":["s3:GetObject","s3:DeleteObject"],"Resource":["arn:aws:s3:::%s/attachments/*"]},{"Effect":"Allow","Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::%s"]}]}' "$bucket" "$bucket" "$bucket" > /tmp/messenger-api-policy.json -mc admin user add stand "$api_key" "$api_secret" -info=$(mc admin user info stand "$api_key" --json) -case "$info" in -*'"messenger-api"'*) ;; -*) mc admin policy attach stand messenger-api --user "$api_key" ;; -esac +# The worker reads staging, writes the final original and variants, and removes +# staging after a safe commit. It never deletes the final prefix. +printf '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject","s3:DeleteObject"],"Resource":["arn:aws:s3:::%s/staging/*"]},{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject"],"Resource":["arn:aws:s3:::%s/attachments/*"]},{"Effect":"Allow","Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::%s"]}]}' "$bucket" "$bucket" "$bucket" > /tmp/messenger-worker-policy.json -echo "bootstrap ok: bucket=$bucket policy=messenger-api" +ensure_user() { + name=$1 + secret=$2 + if ! mc admin user info stand "$name" >/dev/null 2>&1; then + mc admin user add stand "$name" "$secret" + fi +} + +# `mc admin policy create` does not overwrite and a policy attached to a user +# cannot be removed. Detach, remove and recreate makes the script converge on +# the current policy even when the stand volume predates lab5. +install_policy() { + name=$1 + file=$2 + user=$3 + if mc admin policy info stand "$name" >/dev/null 2>&1; then + mc admin policy detach stand "$name" --user "$user" >/dev/null 2>&1 || true + mc admin policy remove stand "$name" >/dev/null 2>&1 || true + fi + mc admin policy create stand "$name" "$file" + mc admin policy attach stand "$name" --user "$user" +} + +ensure_user "$api_key" "$api_secret" +ensure_user "$worker_key" "$worker_secret" + +install_policy messenger-api /tmp/messenger-api-policy.json "$api_key" +install_policy messenger-worker /tmp/messenger-worker-policy.json "$worker_key" + +echo "bootstrap ok: bucket=$bucket policies=messenger-api,messenger-worker" diff --git a/scripts/init-env.sh b/scripts/init-env.sh index f0ce71c..8b9557d 100755 --- a/scripts/init-env.sh +++ b/scripts/init-env.sh @@ -38,3 +38,28 @@ write_secret minio_root_user "root-$(openssl rand -hex 6)" write_secret minio_root_password "$(openssl rand -hex 24)" write_secret s3_api_access_key_id "api-$(openssl rand -hex 8)" write_secret s3_api_secret_access_key "$(openssl rand -hex 24)" +write_secret s3_worker_access_key_id "worker-$(openssl rand -hex 8)" +write_secret s3_worker_secret_access_key "$(openssl rand -hex 24)" + +# Broker password lives in one place and is rendered into both files, so the +# URL used by the app and the RabbitMQ config cannot drift apart. +if [ -s secrets/broker_url ] && [ -s secrets/rabbitmq.conf ]; then + echo "kept secrets/broker_url secrets/rabbitmq.conf" +else + bp=$(openssl rand -hex 24) + (umask 133; printf '%s' "amqp://messenger:${bp}@broker:5672/messenger" > secrets/broker_url) + chmod 0644 secrets/broker_url + ( + umask 133 + { + printf 'default_vhost = messenger\n' + printf 'default_user = messenger\n' + printf 'default_pass = %s\n' "$bp" + printf 'default_permissions.configure = .*\n' + printf 'default_permissions.read = .*\n' + printf 'default_permissions.write = .*\n' + } > secrets/rabbitmq.conf + ) + chmod 0644 secrets/rabbitmq.conf + echo "created secrets/broker_url secrets/rabbitmq.conf" +fi diff --git a/scripts/restart.sh b/scripts/restart.sh index 36e7451..8653b0b 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -4,5 +4,5 @@ set -euo pipefail compose=(docker compose --env-file .env) -"${compose[@]}" restart postgres redis s3 api haproxy -"${compose[@]}" up -d --no-recreate --no-deps --wait postgres redis s3 api haproxy +"${compose[@]}" restart postgres redis s3 broker api worker haproxy +"${compose[@]}" up -d --no-recreate --no-deps --wait postgres redis s3 broker api worker haproxy