forked from mtusi2026/mtusi2026lab5
lab5: брокер RabbitMQ, worker и асинхронная обработка
This commit is contained in:
+24
-3
@@ -81,11 +81,32 @@ DIRECT_UPLOAD_MAX_PENDING_PER_USER=10
|
||||
# ATTACHMENT_SWEEP_INTERVAL_MS задаются в compose.yaml (environment) с умолчаниями
|
||||
# и переопределяются из shell только для экспериментов.
|
||||
|
||||
# lab5: очередь и асинхронная обработка изображений. Пока не реализовано —
|
||||
# приложение эти переменные не читает. URL брокера берётся из файла секрета,
|
||||
# байты и credentials в сообщения не попадают.
|
||||
# lab5: очередь и асинхронная обработка изображений. URL брокера берётся из
|
||||
# файла секрета (scripts/init-env.sh), байты и credentials в сообщения не
|
||||
# попадают: только job_id/attachment_id/pipeline_version/attempt.
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
BROKER_DELIVERY_LIMIT=10
|
||||
# Сколько заданий worker обрабатывает параллельно; prefetch равен этому числу.
|
||||
WORKER_CONCURRENCY=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
JOB_LEASE_SECONDS=30
|
||||
JOB_RETRY_BASE_MS=1000
|
||||
JOB_RETRY_MAX_MS=30000
|
||||
PIPELINE_VERSION=1
|
||||
OUTBOX_POLL_INTERVAL_MS=500
|
||||
LEASE_REAPER_INTERVAL_MS=2000
|
||||
# 0 выключает фоновую уборку staging (ручной запуск остаётся).
|
||||
STAGING_CLEANUP_INTERVAL_MS=60000
|
||||
# Сколько staging-объект неуспешной загрузки живёт до удаления.
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
WORKER_HEALTH_PORT=8081
|
||||
# 1 включает чтение fault:<point> из Redis; в обычном стенде — 0.
|
||||
FAULT_INJECTION_ENABLED=0
|
||||
WORKER_REPLICAS=1
|
||||
WORKER_CPUS=1.5
|
||||
# Бюджет worker: база ≈250 MiB + WORKER_CONCURRENCY × (10 MiB исходник +
|
||||
# ≈340 MiB худший декод 40 Мп) ≈ 950 MiB при concurrency 2; лимит 1280m.
|
||||
WORKER_MEMORY=1280m
|
||||
BROKER_CPUS=0.5
|
||||
BROKER_MEMORY=512m
|
||||
|
||||
Generated
+10
@@ -17,6 +17,7 @@
|
||||
"@nestjs/platform-express": "^12.0.1",
|
||||
"@nestjs/platform-fastify": "^12.0.3",
|
||||
"@smithy/node-http-handler": "^4.12.1",
|
||||
"amqplib": "^2.2.1",
|
||||
"argon2": "^0.45.1",
|
||||
"class-transformer": "^0.5.1",
|
||||
"class-validator": "^0.15.1",
|
||||
@@ -3806,6 +3807,15 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/amqplib": {
|
||||
"version": "2.2.1",
|
||||
"resolved": "https://registry.npmjs.org/amqplib/-/amqplib-2.2.1.tgz",
|
||||
"integrity": "sha512-J3JjAydC2vJ/AgkgVAE0kVH5M4RNfm51zUaspHpwSai3W35lD3hKryvYY/xqMnYcNfR66NYL06Eik+Adn293dQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/ansi-escapes": {
|
||||
"version": "4.3.2",
|
||||
"resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz",
|
||||
|
||||
+4
-2
@@ -16,7 +16,7 @@
|
||||
"start:prod": "node dist/main",
|
||||
"lint": "oxlint src/ test/",
|
||||
"test": "vitest run",
|
||||
"test:unit": "vitest run --include 'src/**/*.spec.ts'",
|
||||
"test:unit": "vitest run src/",
|
||||
"test:watch": "vitest",
|
||||
"test:cov": "vitest run --coverage",
|
||||
"test:debug": "vitest --inspect-brk --no-file-parallelism",
|
||||
@@ -24,7 +24,8 @@
|
||||
"test:e2e": "vitest run --config ./vitest.config.e2e.ts",
|
||||
"test:dependencies": "E2E_DEPENDENCY_FAULTS=1 vitest run --config ./vitest.config.e2e.ts dependency-failures",
|
||||
"attachments:sweep": "tsx src/attachments/attachments-sweep.cli.ts",
|
||||
"idempotency:gc": "tsx src/messages/idempotency/idempotency-gc.cli.ts"
|
||||
"idempotency:gc": "tsx src/messages/idempotency/idempotency-gc.cli.ts",
|
||||
"jobs": "node dist/jobs/jobs.cli.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "^3.1156.0",
|
||||
@@ -35,6 +36,7 @@
|
||||
"@nestjs/platform-express": "^12.0.1",
|
||||
"@nestjs/platform-fastify": "^12.0.3",
|
||||
"@smithy/node-http-handler": "^4.12.1",
|
||||
"amqplib": "^2.2.1",
|
||||
"argon2": "^0.45.1",
|
||||
"class-transformer": "^0.5.1",
|
||||
"class-validator": "^0.15.1",
|
||||
|
||||
@@ -62,7 +62,9 @@ export class AttachmentSweeper implements OnModuleInit, OnModuleDestroy {
|
||||
let deleted = 0;
|
||||
for (const item of claimed) {
|
||||
try {
|
||||
await this.storage.delete(item.object_key);
|
||||
// New uploads live in staging; lab4 rows have no staging key and
|
||||
// their final object is the only copy.
|
||||
await this.storage.delete(item.staging_key ?? item.object_key);
|
||||
await this.repository.deleteClaimed(item.id);
|
||||
deleted += 1;
|
||||
this.metrics.increment('attachments.sweep.deleted');
|
||||
|
||||
@@ -1,3 +1,17 @@
|
||||
export type AttachmentStatus = 'queued' | 'processing' | 'ready' | 'failed';
|
||||
export type VariantName = 'thumbnail' | 'crop' | 'optimized';
|
||||
|
||||
export const VARIANT_ORDER: VariantName[] = ['thumbnail', 'crop', 'optimized'];
|
||||
|
||||
export type Variant = {
|
||||
name: VariantName;
|
||||
content_type: 'image/jpeg' | 'image/png' | 'image/webp';
|
||||
size_bytes: number;
|
||||
width: number;
|
||||
height: number;
|
||||
sha256: string;
|
||||
};
|
||||
|
||||
export type Attachment = {
|
||||
id: string;
|
||||
chat_id: string;
|
||||
@@ -6,9 +20,9 @@ export type Attachment = {
|
||||
content_type: 'image/png' | 'image/jpeg';
|
||||
size_bytes: number;
|
||||
sha256: string;
|
||||
status: 'ready';
|
||||
variants: never[];
|
||||
error_code: null;
|
||||
status: AttachmentStatus;
|
||||
variants: Variant[];
|
||||
error_code: string | null;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
};
|
||||
@@ -20,18 +34,64 @@ export type AttachmentRow = {
|
||||
filename: string;
|
||||
content_type: 'image/png' | 'image/jpeg';
|
||||
upload_mode: 'proxy' | 'direct';
|
||||
status: 'pending' | 'ready' | 'deleting' | 'rejected';
|
||||
status:
|
||||
| 'pending'
|
||||
| 'queued'
|
||||
| 'processing'
|
||||
| 'ready'
|
||||
| 'failed'
|
||||
| 'deleting'
|
||||
| 'rejected';
|
||||
object_key: string;
|
||||
staging_key: string | null;
|
||||
size_bytes: string | number | null;
|
||||
sha256: string | null;
|
||||
width: number | null;
|
||||
height: number | null;
|
||||
error_code: string | null;
|
||||
ready_pipeline_version: number | null;
|
||||
expires_at: Date | null;
|
||||
created_at: Date;
|
||||
updated_at: Date;
|
||||
};
|
||||
|
||||
export function toAttachment(row: AttachmentRow): Attachment {
|
||||
export type VariantRow = {
|
||||
attachment_id: string;
|
||||
pipeline_version: number;
|
||||
variant: VariantName;
|
||||
object_key: string;
|
||||
content_type: 'image/jpeg' | 'image/png' | 'image/webp';
|
||||
size_bytes: string | number;
|
||||
width: number;
|
||||
height: number;
|
||||
sha256: string;
|
||||
job_id: string;
|
||||
created_at: Date;
|
||||
};
|
||||
|
||||
function toVariant(row: VariantRow): Variant {
|
||||
return {
|
||||
name: row.variant,
|
||||
content_type: row.content_type,
|
||||
size_bytes: Number(row.size_bytes),
|
||||
width: row.width,
|
||||
height: row.height,
|
||||
sha256: row.sha256,
|
||||
};
|
||||
}
|
||||
|
||||
export function toAttachment(
|
||||
row: AttachmentRow,
|
||||
variants: VariantRow[] = [],
|
||||
): Attachment {
|
||||
const ordered =
|
||||
row.status === 'ready'
|
||||
? [...variants].sort(
|
||||
(left, right) =>
|
||||
VARIANT_ORDER.indexOf(left.variant) -
|
||||
VARIANT_ORDER.indexOf(right.variant),
|
||||
)
|
||||
: [];
|
||||
return {
|
||||
id: row.id,
|
||||
chat_id: row.chat_id,
|
||||
@@ -40,9 +100,9 @@ export function toAttachment(row: AttachmentRow): Attachment {
|
||||
content_type: row.content_type,
|
||||
size_bytes: Number(row.size_bytes),
|
||||
sha256: row.sha256 as string,
|
||||
status: 'ready',
|
||||
variants: [],
|
||||
error_code: null,
|
||||
status: row.status as AttachmentStatus,
|
||||
variants: ordered.map(toVariant),
|
||||
error_code: row.status === 'failed' ? row.error_code : null,
|
||||
created_at: row.created_at.toISOString(),
|
||||
updated_at: row.updated_at.toISOString(),
|
||||
};
|
||||
|
||||
@@ -40,7 +40,7 @@ export class AttachmentsController {
|
||||
}
|
||||
|
||||
@Post('chats/:chatId/attachments')
|
||||
@HttpCode(201)
|
||||
@HttpCode(202)
|
||||
upload(
|
||||
@Param('chatId', new ParseUUIDPipe()) chatId: string,
|
||||
@CurrentUser() user: User,
|
||||
@@ -96,16 +96,13 @@ export class AttachmentsController {
|
||||
@Res({ passthrough: true }) reply: FastifyReply,
|
||||
@Query('variant') variant?: string,
|
||||
) {
|
||||
const { row, stream } = await this.attachments.openContent(
|
||||
attachmentId,
|
||||
user,
|
||||
variant,
|
||||
);
|
||||
const { row, contentType, sizeBytes, stream } =
|
||||
await this.attachments.openContent(attachmentId, user, variant);
|
||||
reply.header('X-Content-Type-Options', 'nosniff');
|
||||
reply.header('Cache-Control', 'private, no-store');
|
||||
return new StreamableFile(stream, {
|
||||
type: row.content_type,
|
||||
length: Number(row.size_bytes),
|
||||
type: contentType,
|
||||
length: sizeBytes,
|
||||
disposition: contentDisposition(row.filename),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -5,6 +5,8 @@ import { UsersModule } from '../users/users.module.js';
|
||||
import { RedisModule } from '../redis/redis.module.js';
|
||||
import { MetricsModule } from '../metrics/metrics.module.js';
|
||||
import { StorageModule } from '../storage/storage.module.js';
|
||||
import { JobsModule } from '../jobs/jobs.module.js';
|
||||
import { OutboxRelay } from '../jobs/outbox-relay.service.js';
|
||||
import { AttachmentSweeper } from './attachment-sweeper.service.js';
|
||||
import { AttachmentsController } from './attachments.controller.js';
|
||||
import { AttachmentsRepository } from './attachments.repository.js';
|
||||
@@ -21,6 +23,7 @@ import { AttachmentsService } from './attachments.service.js';
|
||||
RedisModule,
|
||||
MetricsModule,
|
||||
StorageModule,
|
||||
JobsModule,
|
||||
],
|
||||
controllers: [AttachmentsController],
|
||||
providers: [
|
||||
@@ -30,6 +33,9 @@ import { AttachmentsService } from './attachments.service.js';
|
||||
ImageDecoder,
|
||||
UploadSlots,
|
||||
AttachmentSweeper,
|
||||
// The relay runs only in the API process; the worker never imports
|
||||
// AttachmentsModule.
|
||||
OutboxRelay,
|
||||
],
|
||||
exports: [AttachmentsRepository],
|
||||
})
|
||||
|
||||
@@ -10,18 +10,12 @@ export type InsertPending = {
|
||||
contentType: string;
|
||||
uploadMode: 'proxy' | 'direct';
|
||||
objectKey: string;
|
||||
stagingKey: string | null;
|
||||
sizeBytes: number | null;
|
||||
sha256: string | null;
|
||||
ttlMs: number;
|
||||
};
|
||||
|
||||
export type ReadyFields = {
|
||||
sizeBytes: number;
|
||||
sha256: string;
|
||||
width: number;
|
||||
height: number;
|
||||
};
|
||||
|
||||
const VISIBLE_MEMBER = `JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = $2`;
|
||||
|
||||
@Injectable()
|
||||
@@ -33,9 +27,9 @@ export class AttachmentsRepository {
|
||||
'attachments.insert_pending',
|
||||
`INSERT INTO attachments
|
||||
(id, chat_id, owner_id, filename, content_type, upload_mode, status,
|
||||
object_key, size_bytes, sha256, expires_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, 'pending', $7, $8, $9,
|
||||
now() + make_interval(secs => $10::double precision / 1000.0))`,
|
||||
object_key, staging_key, size_bytes, sha256, expires_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, 'pending', $7, $8, $9, $10,
|
||||
now() + make_interval(secs => $11::double precision / 1000.0))`,
|
||||
[
|
||||
input.id,
|
||||
input.chatId,
|
||||
@@ -44,6 +38,7 @@ export class AttachmentsRepository {
|
||||
input.contentType,
|
||||
input.uploadMode,
|
||||
input.objectKey,
|
||||
input.stagingKey,
|
||||
input.sizeBytes,
|
||||
input.sha256,
|
||||
input.ttlMs,
|
||||
@@ -51,22 +46,6 @@ export class AttachmentsRepository {
|
||||
);
|
||||
}
|
||||
|
||||
async markReady(
|
||||
id: string,
|
||||
fields: ReadyFields,
|
||||
): Promise<AttachmentRow | undefined> {
|
||||
const result = await this.database.query<AttachmentRow>(
|
||||
'attachments.mark_ready',
|
||||
`UPDATE attachments
|
||||
SET status = 'ready', size_bytes = $2, sha256 = $3, width = $4,
|
||||
height = $5, expires_at = NULL, updated_at = now()
|
||||
WHERE id = $1 AND status = 'pending'
|
||||
RETURNING *`,
|
||||
[id, fields.sizeBytes, fields.sha256, fields.width, fields.height],
|
||||
);
|
||||
return result.rows[0];
|
||||
}
|
||||
|
||||
async deletePending(id: string): Promise<void> {
|
||||
await this.database.query(
|
||||
'attachments.delete_pending',
|
||||
@@ -116,7 +95,7 @@ export class AttachmentsRepository {
|
||||
const result = await this.database.query<AttachmentRow>(
|
||||
'attachments.find_visible',
|
||||
`SELECT a.* FROM attachments a ${VISIBLE_MEMBER}
|
||||
WHERE a.id = $1 AND a.status = 'ready'`,
|
||||
WHERE a.id = $1 AND a.status IN ('queued','processing','ready','failed')`,
|
||||
[id, userId],
|
||||
);
|
||||
return result.rows[0];
|
||||
@@ -130,7 +109,7 @@ export class AttachmentsRepository {
|
||||
'attachments.find_for_complete',
|
||||
`SELECT a.* FROM attachments a ${VISIBLE_MEMBER}
|
||||
WHERE a.id = $1 AND a.upload_mode = 'direct' AND a.owner_id = $2
|
||||
AND a.status IN ('pending', 'ready')`,
|
||||
AND a.status IN ('pending','queued','processing','ready','failed')`,
|
||||
[id, userId],
|
||||
);
|
||||
return result.rows[0];
|
||||
@@ -147,23 +126,38 @@ export class AttachmentsRepository {
|
||||
return Number(result.rows[0].count);
|
||||
}
|
||||
|
||||
async allLinkable(chatId: string, ids: string[]): Promise<boolean> {
|
||||
const result = await this.database.query<{ count: string }>(
|
||||
'attachments.all_linkable',
|
||||
`SELECT count(*) AS count FROM attachments
|
||||
WHERE chat_id = $1 AND status = 'ready' AND id = ANY($2::uuid[])`,
|
||||
async linkState(
|
||||
chatId: string,
|
||||
ids: string[],
|
||||
): Promise<'ok' | 'failed' | 'missing'> {
|
||||
const result = await this.database.query<{
|
||||
linkable: string;
|
||||
failed: string;
|
||||
}>(
|
||||
'attachments.link_state',
|
||||
`SELECT count(*) FILTER (WHERE status IN ('queued','processing','ready')) AS linkable,
|
||||
count(*) FILTER (WHERE status = 'failed') AS failed
|
||||
FROM attachments
|
||||
WHERE chat_id = $1 AND id = ANY($2::uuid[])`,
|
||||
[chatId, ids],
|
||||
);
|
||||
return Number(result.rows[0].count) === ids.length;
|
||||
const linkable = Number(result.rows[0]?.linkable ?? 0);
|
||||
const failed = Number(result.rows[0]?.failed ?? 0);
|
||||
if (linkable === ids.length) return 'ok';
|
||||
if (linkable + failed === ids.length) return 'failed';
|
||||
return 'missing';
|
||||
}
|
||||
|
||||
async claimExpired(
|
||||
graceSeconds: number,
|
||||
limit: number,
|
||||
): Promise<{ id: string; object_key: string }[]> {
|
||||
): Promise<
|
||||
{ id: string; object_key: string; staging_key: string | null }[]
|
||||
> {
|
||||
const result = await this.database.query<{
|
||||
id: string;
|
||||
object_key: string;
|
||||
staging_key: string | null;
|
||||
}>(
|
||||
'attachments.claim_expired',
|
||||
`UPDATE attachments SET status = 'deleting', updated_at = now()
|
||||
@@ -175,7 +169,7 @@ export class AttachmentsRepository {
|
||||
ORDER BY expires_at NULLS FIRST
|
||||
LIMIT $2
|
||||
FOR UPDATE SKIP LOCKED)
|
||||
RETURNING id, object_key`,
|
||||
RETURNING id, object_key, staging_key`,
|
||||
[graceSeconds, limit],
|
||||
);
|
||||
return result.rows;
|
||||
|
||||
@@ -15,11 +15,17 @@ const settings: Record<string, number> = {
|
||||
MAX_UPLOAD_BYTES: 10_485_760,
|
||||
MAX_IMAGE_PIXELS: 40_000_000,
|
||||
UPLOAD_TIMEOUT_MS: 10_000,
|
||||
PIPELINE_VERSION: 1,
|
||||
JOB_MAX_ATTEMPTS: 5,
|
||||
};
|
||||
|
||||
function setup(repository: Record<string, unknown>) {
|
||||
function setup(
|
||||
repository: Record<string, unknown>,
|
||||
jobs: Record<string, unknown> = {},
|
||||
) {
|
||||
const storage = {
|
||||
objectKey: (id: string) => `attachments/${id}/original`,
|
||||
stagingKey: (id: string) => `staging/${id}/original`,
|
||||
putStream: vi.fn(async (_key: string, body: Readable) => {
|
||||
for await (const _chunk of body) {
|
||||
// drain like S3 does
|
||||
@@ -34,6 +40,9 @@ function setup(repository: Record<string, unknown>) {
|
||||
deleteClaimed: vi.fn(),
|
||||
...repository,
|
||||
} as never,
|
||||
{ acceptUpload: vi.fn(), ...jobs } as never,
|
||||
{ kick: vi.fn() } as never,
|
||||
{ point: vi.fn() } as never,
|
||||
storage as never,
|
||||
{ getForUser: vi.fn() } as never,
|
||||
{ increment: vi.fn(), add: vi.fn() } as never,
|
||||
@@ -63,49 +72,58 @@ const readyRow = (id: string): AttachmentRow => ({
|
||||
upload_mode: 'proxy',
|
||||
status: 'ready',
|
||||
object_key: `attachments/${id}/original`,
|
||||
staging_key: `staging/${id}/original`,
|
||||
size_bytes: png.length,
|
||||
sha256: 'a'.repeat(64),
|
||||
width: 320,
|
||||
height: 200,
|
||||
error_code: null,
|
||||
ready_pipeline_version: 1,
|
||||
expires_at: null,
|
||||
created_at: new Date(),
|
||||
updated_at: new Date(),
|
||||
});
|
||||
|
||||
describe('AttachmentsService compensation', () => {
|
||||
it('keeps the object and answers 201 when ready was committed but its reply was lost', async () => {
|
||||
const { storage, upload } = setup({
|
||||
markReady: vi.fn().mockRejectedValue(lostReply()),
|
||||
claimForCompensation: vi.fn().mockResolvedValue(false),
|
||||
findById: vi.fn((id: string) => Promise.resolve(readyRow(id))),
|
||||
});
|
||||
it('keeps the object and answers 202 when the job commit reply was lost', async () => {
|
||||
const { storage, upload } = setup(
|
||||
{
|
||||
claimForCompensation: vi.fn().mockResolvedValue(false),
|
||||
findById: vi.fn((id: string) => Promise.resolve(readyRow(id))),
|
||||
},
|
||||
{ acceptUpload: vi.fn().mockRejectedValue(lostReply()) },
|
||||
);
|
||||
const attachment = await upload();
|
||||
expect(attachment.status).toBe('ready');
|
||||
expect(storage.delete).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('deletes nothing while PostgreSQL is unreachable', async () => {
|
||||
const { storage, upload } = setup({
|
||||
markReady: vi.fn().mockRejectedValue(lostReply()),
|
||||
claimForCompensation: vi.fn().mockRejectedValue(lostReply()),
|
||||
findById: vi.fn().mockRejectedValue(lostReply()),
|
||||
});
|
||||
const { storage, upload } = setup(
|
||||
{
|
||||
claimForCompensation: vi.fn().mockRejectedValue(lostReply()),
|
||||
findById: vi.fn().mockRejectedValue(lostReply()),
|
||||
},
|
||||
{ acceptUpload: vi.fn().mockRejectedValue(lostReply()) },
|
||||
);
|
||||
await expect(upload()).rejects.toThrow();
|
||||
expect(storage.delete).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('deletes the object only after claiming the pending row', async () => {
|
||||
const order: string[] = [];
|
||||
const { storage, upload } = setup({
|
||||
markReady: vi.fn().mockRejectedValue(lostReply()),
|
||||
claimForCompensation: vi.fn(async () => {
|
||||
order.push('claim');
|
||||
return true;
|
||||
}),
|
||||
deleteClaimed: vi.fn(async () => {
|
||||
order.push('delete-row');
|
||||
}),
|
||||
});
|
||||
const { storage, upload } = setup(
|
||||
{
|
||||
claimForCompensation: vi.fn(async () => {
|
||||
order.push('claim');
|
||||
return true;
|
||||
}),
|
||||
deleteClaimed: vi.fn(async () => {
|
||||
order.push('delete-row');
|
||||
}),
|
||||
},
|
||||
{ acceptUpload: vi.fn().mockRejectedValue(lostReply()) },
|
||||
);
|
||||
storage.delete.mockImplementation(async () => {
|
||||
order.push('delete-object');
|
||||
});
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import {
|
||||
BadRequestException,
|
||||
ConflictException,
|
||||
HttpException,
|
||||
Injectable,
|
||||
InternalServerErrorException,
|
||||
@@ -14,6 +15,9 @@ import { createHash, randomUUID } from 'node:crypto';
|
||||
import { pipeline, Readable, Transform } from 'node:stream';
|
||||
import type { AppConfig } from '../config/app-config.schema.js';
|
||||
import { ChatsService } from '../chats/chats.service.js';
|
||||
import { FaultInjector } from '../jobs/fault-injector.js';
|
||||
import { JobsRepository } from '../jobs/jobs.repository.js';
|
||||
import { OutboxRelay } from '../jobs/outbox-relay.service.js';
|
||||
import { MetricsService } from '../metrics/metrics.service.js';
|
||||
import { ObjectStorage } from '../storage/object-storage.js';
|
||||
import type { User } from '../users/user.js';
|
||||
@@ -53,10 +57,15 @@ export class AttachmentsService {
|
||||
private readonly maxUploadBytes: number;
|
||||
private readonly maxPixels: number;
|
||||
private readonly uploadTimeoutMs: number;
|
||||
private readonly pipelineVersion: number;
|
||||
private readonly maxAttempts: number;
|
||||
|
||||
constructor(
|
||||
config: ConfigService<AppConfig>,
|
||||
private readonly repository: AttachmentsRepository,
|
||||
private readonly jobs: JobsRepository,
|
||||
private readonly relay: OutboxRelay,
|
||||
private readonly faults: FaultInjector,
|
||||
private readonly storage: ObjectStorage,
|
||||
private readonly chats: ChatsService,
|
||||
private readonly metrics: MetricsService,
|
||||
@@ -66,6 +75,8 @@ export class AttachmentsService {
|
||||
this.maxUploadBytes = config.getOrThrow('MAX_UPLOAD_BYTES');
|
||||
this.maxPixels = config.getOrThrow('MAX_IMAGE_PIXELS');
|
||||
this.uploadTimeoutMs = config.getOrThrow('UPLOAD_TIMEOUT_MS');
|
||||
this.pipelineVersion = config.getOrThrow('PIPELINE_VERSION');
|
||||
this.maxAttempts = config.getOrThrow('JOB_MAX_ATTEMPTS');
|
||||
}
|
||||
|
||||
async uploadProxy(
|
||||
@@ -162,7 +173,7 @@ export class AttachmentsService {
|
||||
);
|
||||
|
||||
id = randomUUID();
|
||||
key = this.storage.objectKey(id);
|
||||
key = this.storage.stagingKey(id);
|
||||
await this.repository.insertPending({
|
||||
id,
|
||||
chatId,
|
||||
@@ -170,7 +181,8 @@ export class AttachmentsService {
|
||||
filename,
|
||||
contentType: declared,
|
||||
uploadMode: 'proxy',
|
||||
objectKey: key,
|
||||
objectKey: this.storage.objectKey(id),
|
||||
stagingKey: key,
|
||||
sizeBytes: null,
|
||||
sha256: null,
|
||||
ttlMs: this.uploadTimeoutMs,
|
||||
@@ -251,17 +263,27 @@ export class AttachmentsService {
|
||||
contentLength,
|
||||
);
|
||||
stored = true;
|
||||
const row = await this.repository.markReady(id, {
|
||||
sizeBytes: total,
|
||||
sha256: sha256.digest('hex'),
|
||||
width: header.width,
|
||||
height: header.height,
|
||||
});
|
||||
const row = await this.jobs.acceptUpload(
|
||||
id,
|
||||
{
|
||||
sizeBytes: total,
|
||||
sha256: sha256.digest('hex'),
|
||||
width: header.width,
|
||||
height: header.height,
|
||||
},
|
||||
this.pipelineVersion,
|
||||
this.maxAttempts,
|
||||
);
|
||||
if (row === undefined) {
|
||||
claimed = true;
|
||||
throw new ServiceUnavailableException();
|
||||
}
|
||||
this.metrics.increment('attachments.upload.created');
|
||||
// Durable: staging object + attachment + job + outbox are committed.
|
||||
// The fault point deliberately sits between COMMIT and publish; the
|
||||
// relay recovers the outbox row after a crash.
|
||||
await this.faults.point('api.after_commit_before_publish');
|
||||
this.relay.kick();
|
||||
this.metrics.increment('attachments.upload.queued');
|
||||
return toAttachment(row);
|
||||
} catch (error) {
|
||||
readable.destroy();
|
||||
@@ -272,14 +294,22 @@ export class AttachmentsService {
|
||||
const outcome = await this.compensate(id, key, {
|
||||
keepRow: allSent && !stored,
|
||||
});
|
||||
// markReady failed on our side, but the row is no longer
|
||||
// acceptUpload failed on our side, but the row is no longer
|
||||
// pending: the commit may have succeeded with its reply lost.
|
||||
// If so, the upload is complete and 201 is the truth.
|
||||
// If so, the upload is accepted and 202 is the truth.
|
||||
if (outcome === 'not_pending' && stored) {
|
||||
const row = await this.repository
|
||||
.findById(id)
|
||||
.catch(() => undefined);
|
||||
if (row?.status === 'ready') {
|
||||
if (
|
||||
row &&
|
||||
[
|
||||
'queued',
|
||||
'processing',
|
||||
'ready',
|
||||
'failed',
|
||||
].includes(row.status)
|
||||
) {
|
||||
this.metrics.increment(
|
||||
'attachments.upload.recovered',
|
||||
);
|
||||
@@ -337,22 +367,54 @@ export class AttachmentsService {
|
||||
async getMetadata(id: string, user: User): Promise<Attachment> {
|
||||
const row = await this.repository.findVisible(id, user.id);
|
||||
if (!row) throw new NotFoundException();
|
||||
return toAttachment(row);
|
||||
const variants =
|
||||
row.status === 'ready' && row.ready_pipeline_version !== null
|
||||
? await this.jobs.variantsFor(
|
||||
row.id,
|
||||
row.ready_pipeline_version,
|
||||
)
|
||||
: [];
|
||||
return toAttachment(row, variants);
|
||||
}
|
||||
|
||||
async openContent(
|
||||
id: string,
|
||||
user: User,
|
||||
variant: string | undefined,
|
||||
): Promise<{ row: AttachmentRow; stream: Readable }> {
|
||||
if (variant !== undefined && variant !== 'original') {
|
||||
if (['thumbnail', 'crop', 'optimized'].includes(variant))
|
||||
throw new NotFoundException();
|
||||
): Promise<{
|
||||
row: AttachmentRow;
|
||||
contentType: string;
|
||||
sizeBytes: number;
|
||||
stream: Readable;
|
||||
}> {
|
||||
if (
|
||||
variant !== undefined &&
|
||||
!['original', 'thumbnail', 'crop', 'optimized'].includes(variant)
|
||||
)
|
||||
throw new BadRequestException('variant is invalid');
|
||||
}
|
||||
const row = await this.repository.findVisible(id, user.id);
|
||||
if (!row) throw new NotFoundException();
|
||||
const source = await this.storage.getStream(row.object_key);
|
||||
if (row.status !== 'ready')
|
||||
throw new ConflictException('Attachment is not ready');
|
||||
|
||||
let key = row.object_key;
|
||||
let contentType: string = row.content_type;
|
||||
let sizeBytes = Number(row.size_bytes);
|
||||
if (variant !== undefined && variant !== 'original') {
|
||||
if (row.ready_pipeline_version === null)
|
||||
throw new NotFoundException();
|
||||
const variants = await this.jobs.variantsFor(
|
||||
row.id,
|
||||
row.ready_pipeline_version,
|
||||
);
|
||||
const match = variants.find((item) => item.variant === variant);
|
||||
if (!match) throw new NotFoundException();
|
||||
key = match.object_key;
|
||||
contentType = match.content_type;
|
||||
sizeBytes = Number(match.size_bytes);
|
||||
}
|
||||
|
||||
const source = await this.storage.getStream(key);
|
||||
if (source === null) {
|
||||
this.metrics.increment('attachments.object_missing');
|
||||
this.logger.error(
|
||||
@@ -368,6 +430,6 @@ export class AttachmentsService {
|
||||
},
|
||||
});
|
||||
pipeline(source, counter, () => undefined);
|
||||
return { row, stream: counter };
|
||||
return { row, contentType, sizeBytes, stream: counter };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,6 +10,8 @@ import { ConfigService } from '@nestjs/config';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import type { AppConfig } from '../config/app-config.schema.js';
|
||||
import { ChatsService } from '../chats/chats.service.js';
|
||||
import { JobsRepository } from '../jobs/jobs.repository.js';
|
||||
import { OutboxRelay } from '../jobs/outbox-relay.service.js';
|
||||
import { MetricsService } from '../metrics/metrics.service.js';
|
||||
import {
|
||||
ObjectStorage,
|
||||
@@ -50,10 +52,14 @@ export class DirectUploadsService {
|
||||
private readonly uploadTimeoutMs: number;
|
||||
private readonly ttlSeconds: number;
|
||||
private readonly maxPending: number;
|
||||
private readonly pipelineVersion: number;
|
||||
private readonly maxAttempts: number;
|
||||
|
||||
constructor(
|
||||
config: ConfigService<AppConfig>,
|
||||
private readonly repository: AttachmentsRepository,
|
||||
private readonly jobs: JobsRepository,
|
||||
private readonly relay: OutboxRelay,
|
||||
private readonly storage: ObjectStorage,
|
||||
private readonly chats: ChatsService,
|
||||
private readonly metrics: MetricsService,
|
||||
@@ -67,6 +73,8 @@ export class DirectUploadsService {
|
||||
this.maxPending = config.getOrThrow(
|
||||
'DIRECT_UPLOAD_MAX_PENDING_PER_USER',
|
||||
);
|
||||
this.pipelineVersion = config.getOrThrow('PIPELINE_VERSION');
|
||||
this.maxAttempts = config.getOrThrow('JOB_MAX_ATTEMPTS');
|
||||
}
|
||||
|
||||
async init(
|
||||
@@ -95,7 +103,7 @@ export class DirectUploadsService {
|
||||
);
|
||||
|
||||
const id = randomUUID();
|
||||
const key = this.storage.objectKey(id);
|
||||
const stagingKey = this.storage.stagingKey(id);
|
||||
await this.repository.insertPending({
|
||||
id,
|
||||
chatId,
|
||||
@@ -103,7 +111,8 @@ export class DirectUploadsService {
|
||||
filename,
|
||||
contentType: dto.content_type,
|
||||
uploadMode: 'direct',
|
||||
objectKey: key,
|
||||
objectKey: this.storage.objectKey(id),
|
||||
stagingKey,
|
||||
sizeBytes: dto.size_bytes,
|
||||
sha256: dto.sha256,
|
||||
ttlMs: this.ttlSeconds * 1000,
|
||||
@@ -111,7 +120,7 @@ export class DirectUploadsService {
|
||||
const checksum = Buffer.from(dto.sha256, 'hex').toString('base64');
|
||||
let url: string;
|
||||
try {
|
||||
url = await this.storage.presignPut(key, {
|
||||
url = await this.storage.presignPut(stagingKey, {
|
||||
contentType: dto.content_type,
|
||||
contentLength: dto.size_bytes,
|
||||
checksumSha256Base64: checksum,
|
||||
@@ -142,21 +151,35 @@ export class DirectUploadsService {
|
||||
user.id,
|
||||
);
|
||||
if (!row) throw new NotFoundException();
|
||||
if (row.status === 'ready') return toAttachment(row);
|
||||
return this.slots.run(user.id, () => this.verifyAndMarkReady(row));
|
||||
if (row.status !== 'pending') return this.describe(row);
|
||||
return this.slots.run(user.id, () => this.verifyAndAccept(row));
|
||||
}
|
||||
|
||||
private async verifyAndMarkReady(row: AttachmentRow): Promise<Attachment> {
|
||||
private async describe(row: AttachmentRow): Promise<Attachment> {
|
||||
const variants =
|
||||
row.status === 'ready' && row.ready_pipeline_version !== null
|
||||
? await this.jobs.variantsFor(
|
||||
row.id,
|
||||
row.ready_pipeline_version,
|
||||
)
|
||||
: [];
|
||||
return toAttachment(row, variants);
|
||||
}
|
||||
|
||||
private async verifyAndAccept(row: AttachmentRow): Promise<Attachment> {
|
||||
const size = Number(row.size_bytes);
|
||||
const stagingKey = row.staging_key;
|
||||
if (stagingKey === null)
|
||||
throw new ConflictException('Upload has not been received');
|
||||
const expectedChecksum = Buffer.from(
|
||||
row.sha256 as string,
|
||||
'hex',
|
||||
).toString('base64');
|
||||
const head = await this.storage.head(row.object_key);
|
||||
const head = await this.storage.head(stagingKey);
|
||||
if (head === null)
|
||||
throw new ConflictException('Upload has not been received');
|
||||
if (head.size !== size || head.checksumSha256 !== expectedChecksum) {
|
||||
await this.reject(row);
|
||||
await this.reject(row, stagingKey);
|
||||
throw new ConflictException(
|
||||
'Uploaded object does not match the declaration',
|
||||
);
|
||||
@@ -164,37 +187,44 @@ export class DirectUploadsService {
|
||||
|
||||
let header: ImageHeader;
|
||||
try {
|
||||
header = await this.verifyObject(row.object_key, size);
|
||||
header = await this.verifyObject(stagingKey, size);
|
||||
} catch (error) {
|
||||
if (error instanceof ImageRejectedError) {
|
||||
await this.reject(row);
|
||||
await this.reject(row, stagingKey);
|
||||
throw imageRejection(error);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (header.contentType !== row.content_type) {
|
||||
await this.reject(row);
|
||||
await this.reject(row, stagingKey);
|
||||
throw new UnsupportedMediaTypeException(
|
||||
'Image content does not match content_type',
|
||||
);
|
||||
}
|
||||
const ready = await this.repository.markReady(row.id, {
|
||||
sizeBytes: size,
|
||||
sha256: row.sha256 as string,
|
||||
width: header.width,
|
||||
height: header.height,
|
||||
});
|
||||
if (!ready) {
|
||||
const accepted = await this.jobs.acceptUpload(
|
||||
row.id,
|
||||
{
|
||||
sizeBytes: size,
|
||||
sha256: row.sha256 as string,
|
||||
width: header.width,
|
||||
height: header.height,
|
||||
},
|
||||
this.pipelineVersion,
|
||||
this.maxAttempts,
|
||||
);
|
||||
if (!accepted) {
|
||||
// A parallel complete of the same upload won: answer like a repeat.
|
||||
const current = await this.repository.findForComplete(
|
||||
row.id,
|
||||
row.owner_id,
|
||||
);
|
||||
if (current?.status === 'ready') return toAttachment(current);
|
||||
if (current && current.status !== 'pending')
|
||||
return this.describe(current);
|
||||
throw new ConflictException('Upload is no longer pending');
|
||||
}
|
||||
this.relay.kick();
|
||||
this.metrics.increment('attachments.direct.completed');
|
||||
return toAttachment(ready);
|
||||
return toAttachment(accepted);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -247,12 +277,15 @@ export class DirectUploadsService {
|
||||
* the signed URL may still be used: a later re-PUT would otherwise leave an
|
||||
* object that no row refers to. The sweeper removes both after expiry + grace.
|
||||
*/
|
||||
private async reject(row: AttachmentRow): Promise<void> {
|
||||
// Row first: a row that is not pending any more (ready after a parallel
|
||||
private async reject(
|
||||
row: AttachmentRow,
|
||||
stagingKey: string,
|
||||
): Promise<void> {
|
||||
// Row first: a row that is not pending any more (accepted after a parallel
|
||||
// complete, or claimed by the sweeper) keeps its object.
|
||||
if (!(await this.repository.rejectPending(row.id))) return;
|
||||
try {
|
||||
await this.storage.delete(row.object_key);
|
||||
await this.storage.delete(stagingKey);
|
||||
} catch {
|
||||
// The sweeper deletes the object again after expiry.
|
||||
}
|
||||
|
||||
@@ -55,7 +55,7 @@ describe('ImageDecoder', () => {
|
||||
|
||||
it('accepts the heaviest valid input: 40 MP RGBA 16-bit', async () => {
|
||||
await decoder.verify(await maxPixelsRgba16Png());
|
||||
});
|
||||
}, 30_000);
|
||||
|
||||
it('drops an aborted waiter from the queue without running its decode', async () => {
|
||||
const busy = decoder.verify(noisePng(4_000_000));
|
||||
|
||||
@@ -19,7 +19,7 @@ sharp.concurrency(1);
|
||||
// client 415. libvips spills large images to a temp file unless
|
||||
// VIPS_DISC_THRESHOLD (compose.yaml) keeps them in memory; a full /tmp or an
|
||||
// allocation failure ends up here.
|
||||
const SERVER_FAULT =
|
||||
export const SERVER_FAULT =
|
||||
/no space left|out of memory|unable to allocate|cannot allocate|write failed|unable to open|tmp/iu;
|
||||
|
||||
/**
|
||||
|
||||
@@ -35,6 +35,7 @@ export const appConfigSchema = z
|
||||
SESSION_TTL_SECONDS: numeric('SESSION_TTL_SECONDS', 2, undefined, 3600),
|
||||
DATABASE_URL: requiredString('DATABASE_URL'),
|
||||
REDIS_URL: requiredString('REDIS_URL'),
|
||||
BROKER_URL: requiredString('BROKER_URL'),
|
||||
DATABASE_POOL_MAX: numeric('DATABASE_POOL_MAX', 1, 100, 10),
|
||||
DATABASE_CONNECTION_TIMEOUT_MS: numeric(
|
||||
'DATABASE_CONNECTION_TIMEOUT_MS',
|
||||
@@ -123,6 +124,41 @@ export const appConfigSchema = z
|
||||
1000,
|
||||
10,
|
||||
),
|
||||
WORKER_CONCURRENCY: numeric('WORKER_CONCURRENCY', 1, 16, 2),
|
||||
JOB_MAX_ATTEMPTS: numeric('JOB_MAX_ATTEMPTS', 1, 20, 5),
|
||||
JOB_TIMEOUT_SECONDS: numeric('JOB_TIMEOUT_SECONDS', 1, 600, 60),
|
||||
JOB_LEASE_SECONDS: numeric('JOB_LEASE_SECONDS', 3, 600, 30),
|
||||
JOB_RETRY_BASE_MS: numeric('JOB_RETRY_BASE_MS', 10, undefined, 1000),
|
||||
JOB_RETRY_MAX_MS: numeric('JOB_RETRY_MAX_MS', 10, undefined, 30000),
|
||||
PIPELINE_VERSION: numeric('PIPELINE_VERSION', 1, 2, 1),
|
||||
BROKER_DELIVERY_LIMIT: numeric('BROKER_DELIVERY_LIMIT', 1, 100, 10),
|
||||
OUTBOX_POLL_INTERVAL_MS: numeric(
|
||||
'OUTBOX_POLL_INTERVAL_MS',
|
||||
50,
|
||||
undefined,
|
||||
500,
|
||||
),
|
||||
LEASE_REAPER_INTERVAL_MS: numeric(
|
||||
'LEASE_REAPER_INTERVAL_MS',
|
||||
100,
|
||||
undefined,
|
||||
2000,
|
||||
),
|
||||
// 0 отключает фоновую уборку staging (ручной запуск остаётся).
|
||||
STAGING_CLEANUP_INTERVAL_MS: numeric(
|
||||
'STAGING_CLEANUP_INTERVAL_MS',
|
||||
0,
|
||||
undefined,
|
||||
60000,
|
||||
),
|
||||
STAGING_RETENTION_SECONDS: numeric(
|
||||
'STAGING_RETENTION_SECONDS',
|
||||
60,
|
||||
undefined,
|
||||
86400,
|
||||
),
|
||||
WORKER_HEALTH_PORT: numeric('WORKER_HEALTH_PORT', 1, 65535, 8081),
|
||||
FAULT_INJECTION_ENABLED: numeric('FAULT_INJECTION_ENABLED', 0, 1, 0),
|
||||
})
|
||||
.strip();
|
||||
|
||||
@@ -144,6 +180,7 @@ function resolveSecret(
|
||||
key:
|
||||
| 'DATABASE_URL'
|
||||
| 'REDIS_URL'
|
||||
| 'BROKER_URL'
|
||||
| 'S3_ACCESS_KEY_ID'
|
||||
| 'S3_SECRET_ACCESS_KEY',
|
||||
): unknown {
|
||||
@@ -170,6 +207,7 @@ export function parseAppConfig(raw: Record<string, unknown>): AppConfig {
|
||||
...raw,
|
||||
DATABASE_URL: resolveSecret(raw, 'DATABASE_URL'),
|
||||
REDIS_URL: resolveSecret(raw, 'REDIS_URL'),
|
||||
BROKER_URL: resolveSecret(raw, 'BROKER_URL'),
|
||||
S3_ACCESS_KEY_ID: resolveSecret(raw, 'S3_ACCESS_KEY_ID'),
|
||||
S3_SECRET_ACCESS_KEY: resolveSecret(raw, 'S3_SECRET_ACCESS_KEY'),
|
||||
});
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_status_check;
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_status_check CHECK (status IN
|
||||
('pending','queued','processing','ready','failed','deleting','rejected'));
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_ready_complete;
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_accepted_complete CHECK (
|
||||
status NOT IN ('queued','processing','ready','failed') OR
|
||||
(size_bytes IS NOT NULL AND sha256 IS NOT NULL AND width IS NOT NULL AND height IS NOT NULL));
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_unfinished_expiry;
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_unfinished_expiry CHECK (
|
||||
status NOT IN ('pending','rejected','deleting') OR expires_at IS NOT NULL);
|
||||
ALTER TABLE attachments
|
||||
ADD COLUMN staging_key text UNIQUE,
|
||||
ADD COLUMN error_code text CHECK (error_code ~ '^[a-z_]{1,64}$'),
|
||||
ADD COLUMN ready_pipeline_version integer CHECK (ready_pipeline_version >= 1);
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_failed_code
|
||||
CHECK (status <> 'failed' OR error_code IS NOT NULL);
|
||||
DROP INDEX attachments_unfinished_idx;
|
||||
CREATE INDEX attachments_unfinished_idx ON attachments (expires_at)
|
||||
WHERE status IN ('pending','rejected','deleting');
|
||||
CREATE INDEX attachments_staging_idx ON attachments (updated_at)
|
||||
WHERE staging_key IS NOT NULL AND status IN ('ready','failed');
|
||||
|
||||
-- Rows accepted by lab4 already have their final original and complete metadata.
|
||||
UPDATE attachments SET ready_pipeline_version = 1
|
||||
WHERE status = 'ready' AND ready_pipeline_version IS NULL;
|
||||
|
||||
CREATE TABLE attachment_jobs (
|
||||
job_id uuid PRIMARY KEY,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
|
||||
pipeline_version integer NOT NULL CHECK (pipeline_version >= 1),
|
||||
status text NOT NULL CHECK (status IN ('queued','processing','succeeded','failed','superseded')),
|
||||
attempt integer NOT NULL DEFAULT 1 CHECK (attempt >= 1),
|
||||
max_attempts integer NOT NULL CHECK (max_attempts >= 1),
|
||||
next_attempt_at timestamptz(3) NOT NULL DEFAULT now(),
|
||||
lease_owner text, lease_token uuid, lease_until timestamptz(3),
|
||||
last_error_code text,
|
||||
created_at timestamptz(3) NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz(3) NOT NULL DEFAULT now(),
|
||||
finished_at timestamptz(3),
|
||||
CONSTRAINT attachment_jobs_effect_key UNIQUE (attachment_id, pipeline_version),
|
||||
CONSTRAINT attachment_jobs_lease CHECK ((status = 'processing') =
|
||||
(lease_owner IS NOT NULL AND lease_token IS NOT NULL AND lease_until IS NOT NULL))
|
||||
);
|
||||
CREATE INDEX attachment_jobs_lease_idx ON attachment_jobs (lease_until) WHERE status = 'processing';
|
||||
CREATE INDEX attachment_jobs_failed_idx ON attachment_jobs (updated_at) WHERE status = 'failed';
|
||||
|
||||
CREATE TABLE attachment_variants (
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
|
||||
pipeline_version integer NOT NULL CHECK (pipeline_version >= 1),
|
||||
variant text NOT NULL CHECK (variant IN ('thumbnail','crop','optimized')),
|
||||
object_key text NOT NULL UNIQUE,
|
||||
content_type text NOT NULL CHECK (content_type IN ('image/jpeg','image/png','image/webp')),
|
||||
size_bytes bigint NOT NULL CHECK (size_bytes >= 1),
|
||||
width integer NOT NULL CHECK (width >= 1),
|
||||
height integer NOT NULL CHECK (height >= 1),
|
||||
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
job_id uuid NOT NULL REFERENCES attachment_jobs(job_id) ON DELETE CASCADE,
|
||||
created_at timestamptz(3) NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (attachment_id, pipeline_version, variant)
|
||||
);
|
||||
|
||||
CREATE TABLE job_outbox (
|
||||
id bigserial PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES attachment_jobs(job_id) ON DELETE CASCADE,
|
||||
payload jsonb NOT NULL,
|
||||
available_at timestamptz(3) NOT NULL DEFAULT now(),
|
||||
published_at timestamptz(3),
|
||||
created_at timestamptz(3) NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX job_outbox_pending_idx ON job_outbox (available_at, id) WHERE published_at IS NULL;
|
||||
|
||||
CREATE TABLE job_events (
|
||||
id bigserial PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES attachment_jobs(job_id) ON DELETE CASCADE,
|
||||
event text NOT NULL CHECK (event IN ('created','claimed','retry_scheduled','lease_expired',
|
||||
'succeeded','failed','superseded','replayed','duplicate_skipped','staging_deleted')),
|
||||
attempt integer NOT NULL,
|
||||
detail jsonb NOT NULL DEFAULT '{}'::jsonb,
|
||||
created_at timestamptz(3) NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX job_events_job_idx ON job_events (job_id, id);
|
||||
@@ -0,0 +1,19 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { retryDelayMs } from './backoff.js';
|
||||
|
||||
describe('retryDelayMs', () => {
|
||||
it('uses equal jitter', () => {
|
||||
expect(retryDelayMs(1, 1000, 30000, () => 0)).toBe(500);
|
||||
expect(retryDelayMs(1, 1000, 30000, () => 1)).toBe(1000);
|
||||
expect(retryDelayMs(10, 1000, 30000, () => 1)).toBe(30000);
|
||||
});
|
||||
|
||||
it('never exceeds the cap and grows monotonically at random 0.5', () => {
|
||||
const values = Array.from({ length: 6 }, (_, index) =>
|
||||
retryDelayMs(index + 1, 1000, 30000, () => 0.5),
|
||||
);
|
||||
for (const value of values) expect(value).toBeLessThanOrEqual(30000);
|
||||
for (let index = 1; index < values.length; index += 1)
|
||||
expect(values[index]).toBeGreaterThanOrEqual(values[index - 1]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,14 @@
|
||||
/**
|
||||
* Equal jitter: half of the exponential delay is fixed, half is random. The
|
||||
* result never exceeds `capMs`, so `max_attempts` bounds both attempts and
|
||||
* total wait time.
|
||||
*/
|
||||
export function retryDelayMs(
|
||||
attempt: number,
|
||||
baseMs: number,
|
||||
capMs: number,
|
||||
random: () => number = Math.random,
|
||||
): number {
|
||||
const exponential = Math.min(capMs, baseMs * 2 ** (attempt - 1));
|
||||
return Math.floor(exponential * (0.5 + 0.5 * random()));
|
||||
}
|
||||
@@ -0,0 +1,259 @@
|
||||
import {
|
||||
connect as amqpConnect,
|
||||
type Channel,
|
||||
type ChannelModel,
|
||||
type ConfirmChannel,
|
||||
type ConsumeMessage,
|
||||
} from 'amqplib';
|
||||
import {
|
||||
Injectable,
|
||||
Logger,
|
||||
OnModuleDestroy,
|
||||
ServiceUnavailableException,
|
||||
} from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import type { AppConfig } from '../config/app-config.schema.js';
|
||||
import { MetricsService } from '../metrics/metrics.service.js';
|
||||
import {
|
||||
assertTopology,
|
||||
JOB_EXCHANGE,
|
||||
JOB_QUEUE,
|
||||
JOB_ROUTING_KEY,
|
||||
} from './broker.topology.js';
|
||||
import type { JobMessage } from './job-message.js';
|
||||
|
||||
export class BrokerUnavailableError extends ServiceUnavailableException {
|
||||
constructor() {
|
||||
super('Broker unavailable');
|
||||
}
|
||||
}
|
||||
|
||||
export type ConsumeHandler = (
|
||||
message: ConsumeMessage,
|
||||
channel: Channel,
|
||||
) => Promise<void>;
|
||||
|
||||
const MAX_RECONNECT_DELAY_MS = 10_000;
|
||||
|
||||
@Injectable()
|
||||
export class BrokerClient implements OnModuleDestroy {
|
||||
private readonly logger = new Logger(BrokerClient.name);
|
||||
private readonly url: string;
|
||||
private readonly deliveryLimit: number;
|
||||
private connection: ChannelModel | undefined;
|
||||
private confirmChannel: ConfirmChannel | undefined;
|
||||
private consumeChannel: Channel | undefined;
|
||||
private consumerTag: string | undefined;
|
||||
private prefetch = 0;
|
||||
private handler: ConsumeHandler | undefined;
|
||||
private connecting: Promise<void> | undefined;
|
||||
private reconnectTimer: NodeJS.Timeout | undefined;
|
||||
private reconnectDelayMs = 1000;
|
||||
private closing = false;
|
||||
|
||||
constructor(
|
||||
config: ConfigService<AppConfig>,
|
||||
private readonly metrics: MetricsService,
|
||||
) {
|
||||
this.url = config.getOrThrow('BROKER_URL');
|
||||
this.deliveryLimit = config.getOrThrow('BROKER_DELIVERY_LIMIT');
|
||||
}
|
||||
|
||||
async publish(message: JobMessage | Buffer): Promise<void> {
|
||||
const channel = await this.ensureConnected();
|
||||
const content = Buffer.isBuffer(message)
|
||||
? message
|
||||
: Buffer.from(JSON.stringify(message));
|
||||
const options = Buffer.isBuffer(message)
|
||||
? { persistent: true, contentType: 'application/json' }
|
||||
: {
|
||||
persistent: true,
|
||||
contentType: 'application/json',
|
||||
messageId: message.job_id,
|
||||
type: message.type,
|
||||
headers: { 'x-attempt': message.attempt },
|
||||
};
|
||||
const { promise, resolve, reject } = Promise.withResolvers<void>();
|
||||
try {
|
||||
channel.publish(
|
||||
JOB_EXCHANGE,
|
||||
JOB_ROUTING_KEY,
|
||||
content,
|
||||
options,
|
||||
(error) => {
|
||||
if (error) {
|
||||
this.metrics.increment('broker.publish_failed');
|
||||
reject(new BrokerUnavailableError());
|
||||
return;
|
||||
}
|
||||
this.metrics.increment('broker.published');
|
||||
resolve();
|
||||
},
|
||||
);
|
||||
} catch {
|
||||
this.metrics.increment('broker.publish_failed');
|
||||
reject(new BrokerUnavailableError());
|
||||
}
|
||||
return promise;
|
||||
}
|
||||
|
||||
consume(prefetch: number, handler: ConsumeHandler): void {
|
||||
this.prefetch = prefetch;
|
||||
this.handler = handler;
|
||||
this.ensureConnected()
|
||||
.then(() => this.subscribe())
|
||||
.catch(() => this.scheduleReconnect());
|
||||
}
|
||||
|
||||
isConsuming(): boolean {
|
||||
return this.consumerTag !== undefined;
|
||||
}
|
||||
|
||||
async queueDepth(
|
||||
queue: string,
|
||||
): Promise<{ messages: number; consumers: number }> {
|
||||
const channel = await this.ensureConnected();
|
||||
const result = await channel.checkQueue(queue);
|
||||
return {
|
||||
messages: result.messageCount,
|
||||
consumers: result.consumerCount,
|
||||
};
|
||||
}
|
||||
|
||||
async stopConsuming(): Promise<void> {
|
||||
const channel = this.consumeChannel;
|
||||
const tag = this.consumerTag;
|
||||
this.consumerTag = undefined;
|
||||
this.consumeChannel = undefined;
|
||||
this.handler = undefined;
|
||||
if (channel && tag) {
|
||||
try {
|
||||
await channel.cancel(tag);
|
||||
} catch {
|
||||
// The channel is already closing; there is nothing to cancel.
|
||||
}
|
||||
}
|
||||
if (channel) {
|
||||
try {
|
||||
await channel.close();
|
||||
} catch {
|
||||
// Closing an already closed channel is harmless.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async onModuleDestroy(): Promise<void> {
|
||||
this.closing = true;
|
||||
clearTimeout(this.reconnectTimer);
|
||||
this.reconnectTimer = undefined;
|
||||
await this.stopConsuming();
|
||||
const connection = this.connection;
|
||||
this.connection = undefined;
|
||||
this.confirmChannel = undefined;
|
||||
if (connection) {
|
||||
try {
|
||||
await connection.close();
|
||||
} catch {
|
||||
// Shutdown is best effort; channels are already unusable.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async ensureConnected(): Promise<ConfirmChannel> {
|
||||
if (this.closing) throw new BrokerUnavailableError();
|
||||
if (this.confirmChannel) return this.confirmChannel;
|
||||
if (!this.connecting) {
|
||||
this.connecting = this.connect()
|
||||
.then(() => undefined)
|
||||
.finally(() => {
|
||||
this.connecting = undefined;
|
||||
});
|
||||
}
|
||||
await this.connecting;
|
||||
if (!this.confirmChannel) throw new BrokerUnavailableError();
|
||||
return this.confirmChannel;
|
||||
}
|
||||
|
||||
private async connect(): Promise<void> {
|
||||
const connection = await amqpConnect(this.url);
|
||||
connection.on('error', (error: Error) => {
|
||||
this.metrics.increment('broker.error');
|
||||
this.logger.warn(`Broker connection error: ${error.message}`);
|
||||
});
|
||||
connection.on('close', () => {
|
||||
if (this.connection !== connection) return;
|
||||
this.handleClose();
|
||||
});
|
||||
const confirm = await connection.createConfirmChannel();
|
||||
await assertTopology(confirm, this.deliveryLimit);
|
||||
this.connection = connection;
|
||||
this.confirmChannel = confirm;
|
||||
this.reconnectDelayMs = 1000;
|
||||
clearTimeout(this.reconnectTimer);
|
||||
this.reconnectTimer = undefined;
|
||||
this.metrics.increment('broker.connected');
|
||||
if (this.handler) await this.subscribe();
|
||||
}
|
||||
|
||||
private async subscribe(): Promise<void> {
|
||||
const connection = this.connection;
|
||||
const handler = this.handler;
|
||||
if (!connection || !handler || this.consumerTag) return;
|
||||
const channel = await connection.createChannel();
|
||||
channel.on('error', (error: Error) => {
|
||||
this.metrics.increment('broker.consumer_error');
|
||||
this.logger.warn(`Broker consumer channel error: ${error.message}`);
|
||||
});
|
||||
channel.on('close', () => {
|
||||
if (this.consumeChannel !== channel) return;
|
||||
this.consumeChannel = undefined;
|
||||
this.consumerTag = undefined;
|
||||
});
|
||||
await channel.prefetch(this.prefetch);
|
||||
await assertTopology(channel, this.deliveryLimit);
|
||||
const consumer = await channel.consume(
|
||||
JOB_QUEUE,
|
||||
(message) => {
|
||||
if (!message) return;
|
||||
handler(message, channel).catch((error: unknown) => {
|
||||
this.metrics.increment('broker.handler_error');
|
||||
this.logger.error(
|
||||
`Job handler crashed: ${error instanceof Error ? error.message : 'unknown error'}`,
|
||||
);
|
||||
try {
|
||||
channel.nack(message, false, true);
|
||||
} catch {
|
||||
// The channel is gone; the broker redelivers after the lease.
|
||||
}
|
||||
});
|
||||
},
|
||||
{ noAck: false },
|
||||
);
|
||||
this.consumeChannel = channel;
|
||||
this.consumerTag = consumer.consumerTag;
|
||||
}
|
||||
|
||||
private handleClose(): void {
|
||||
this.metrics.increment('broker.closed');
|
||||
this.connection = undefined;
|
||||
this.confirmChannel = undefined;
|
||||
this.consumeChannel = undefined;
|
||||
this.consumerTag = undefined;
|
||||
if (this.closing) return;
|
||||
this.scheduleReconnect();
|
||||
}
|
||||
|
||||
private scheduleReconnect(): void {
|
||||
if (this.closing || this.reconnectTimer) return;
|
||||
const delay = this.reconnectDelayMs;
|
||||
this.reconnectDelayMs = Math.min(
|
||||
this.reconnectDelayMs * 2,
|
||||
MAX_RECONNECT_DELAY_MS,
|
||||
);
|
||||
this.metrics.increment('broker.reconnect_scheduled');
|
||||
this.reconnectTimer = setTimeout(() => {
|
||||
this.reconnectTimer = undefined;
|
||||
this.ensureConnected().catch(() => this.scheduleReconnect());
|
||||
}, delay);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import type { Channel } from 'amqplib';
|
||||
|
||||
export const JOB_EXCHANGE = 'attachments.jobs';
|
||||
export const JOB_ROUTING_KEY = 'attachment.process.v1';
|
||||
export const JOB_QUEUE = 'attachments.process';
|
||||
export const DEAD_LETTER_EXCHANGE = 'attachments.dlx';
|
||||
export const DEAD_LETTER_QUEUE = 'attachments.process.dlq';
|
||||
|
||||
/**
|
||||
* Idempotent declaration used by both the producer and the consumer on every
|
||||
* (re)connect: a stand started from an empty volume and a reconnect after a
|
||||
* broker restart converge to the same durable topology.
|
||||
*/
|
||||
export async function assertTopology(
|
||||
channel: Channel,
|
||||
deliveryLimit: number,
|
||||
): Promise<void> {
|
||||
await channel.assertExchange(DEAD_LETTER_EXCHANGE, 'fanout', {
|
||||
durable: true,
|
||||
});
|
||||
await channel.assertQueue(DEAD_LETTER_QUEUE, {
|
||||
durable: true,
|
||||
arguments: { 'x-queue-type': 'quorum' },
|
||||
});
|
||||
await channel.bindQueue(DEAD_LETTER_QUEUE, DEAD_LETTER_EXCHANGE, '');
|
||||
|
||||
await channel.assertExchange(JOB_EXCHANGE, 'direct', { durable: true });
|
||||
await channel.assertQueue(JOB_QUEUE, {
|
||||
durable: true,
|
||||
arguments: {
|
||||
'x-queue-type': 'quorum',
|
||||
'x-dead-letter-exchange': DEAD_LETTER_EXCHANGE,
|
||||
'x-delivery-limit': deliveryLimit,
|
||||
},
|
||||
});
|
||||
await channel.bindQueue(JOB_QUEUE, JOB_EXCHANGE, JOB_ROUTING_KEY);
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { ObjectStorageUnavailableError } from '../storage/object-storage.js';
|
||||
import { classifyError, JobTimeoutError } from './errors.js';
|
||||
|
||||
describe('classifyError', () => {
|
||||
it('classifies unsupported sharp input as permanent invalid_image', () => {
|
||||
expect(
|
||||
classifyError(
|
||||
new Error('Input buffer contains unsupported image format'),
|
||||
),
|
||||
).toEqual({ kind: 'permanent', code: 'invalid_image' });
|
||||
});
|
||||
|
||||
it('classifies the pixel limit as permanent image_too_large', () => {
|
||||
expect(
|
||||
classifyError(new Error('Input image exceeds pixel limit')),
|
||||
).toEqual({ kind: 'permanent', code: 'image_too_large' });
|
||||
});
|
||||
|
||||
it('classifies object storage outages as transient', () => {
|
||||
expect(classifyError(new ObjectStorageUnavailableError())).toEqual({
|
||||
kind: 'transient',
|
||||
code: 'processing_failed',
|
||||
});
|
||||
});
|
||||
|
||||
it('classifies a job timeout as transient processing_timeout', () => {
|
||||
expect(classifyError(new JobTimeoutError(10))).toEqual({
|
||||
kind: 'transient',
|
||||
code: 'processing_timeout',
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,65 @@
|
||||
import { SERVER_FAULT } from '../attachments/image/image-decoder.js';
|
||||
import { InjectedFaultError } from './fault-injector.js';
|
||||
|
||||
export class SourceMissingError extends Error {
|
||||
constructor(message = 'Source object is missing') {
|
||||
super(message);
|
||||
this.name = 'SourceMissingError';
|
||||
}
|
||||
}
|
||||
|
||||
export class SourceMismatchError extends Error {
|
||||
constructor(message = 'Source object does not match its metadata') {
|
||||
super(message);
|
||||
this.name = 'SourceMismatchError';
|
||||
}
|
||||
}
|
||||
|
||||
export class UnsupportedPipelineError extends Error {
|
||||
constructor(version: number) {
|
||||
super(`Unsupported pipeline version: ${version}`);
|
||||
this.name = 'UnsupportedPipelineError';
|
||||
}
|
||||
}
|
||||
|
||||
export class JobTimeoutError extends Error {
|
||||
constructor(seconds: number) {
|
||||
super(`Job exceeded ${seconds}s deadline`);
|
||||
this.name = 'JobTimeoutError';
|
||||
}
|
||||
}
|
||||
|
||||
export type ErrorClass = {
|
||||
kind: 'permanent' | 'transient';
|
||||
code: string;
|
||||
};
|
||||
|
||||
const SHARP_INPUT =
|
||||
/unsupported image format|input buffer|input file|bad seek|premature end|vipsjpeg|vippng|corrupt|truncated|invalid (?:image|pixel)|not a valid image|exceeded.*limit/iu;
|
||||
|
||||
/**
|
||||
* Maps a processing failure to the contract's safe error class. Permanent
|
||||
* errors are never retried; transient ones are retried until max_attempts,
|
||||
* after which the job is failed as `retry_exhausted`.
|
||||
*/
|
||||
export function classifyError(error: unknown): ErrorClass {
|
||||
if (error instanceof InjectedFaultError)
|
||||
return { kind: error.kind, code: 'injected_fault' };
|
||||
if (error instanceof SourceMissingError)
|
||||
return { kind: 'permanent', code: 'source_missing' };
|
||||
if (error instanceof SourceMismatchError)
|
||||
return { kind: 'permanent', code: 'source_mismatch' };
|
||||
if (error instanceof UnsupportedPipelineError)
|
||||
return { kind: 'permanent', code: 'unsupported_pipeline' };
|
||||
if (error instanceof JobTimeoutError)
|
||||
return { kind: 'transient', code: 'processing_timeout' };
|
||||
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
if (/pixel limit/iu.test(message))
|
||||
return { kind: 'permanent', code: 'image_too_large' };
|
||||
if (SERVER_FAULT.test(message))
|
||||
return { kind: 'transient', code: 'processing_failed' };
|
||||
if (SHARP_INPUT.test(message))
|
||||
return { kind: 'permanent', code: 'invalid_image' };
|
||||
return { kind: 'transient', code: 'processing_failed' };
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import type { AppConfig } from '../config/app-config.schema.js';
|
||||
import { MetricsService } from '../metrics/metrics.service.js';
|
||||
import { RedisClient } from '../redis/redis.client.js';
|
||||
|
||||
export type FaultPoint =
|
||||
| 'api.after_commit_before_publish'
|
||||
| 'relay.after_publish_before_mark'
|
||||
| 'worker.before_side_effect'
|
||||
| 'worker.after_side_effect'
|
||||
| 'worker.after_commit_before_ack';
|
||||
|
||||
export type FaultTarget = {
|
||||
attachmentId: string;
|
||||
pipelineVersion: number;
|
||||
};
|
||||
|
||||
export type FaultKind = 'permanent' | 'transient';
|
||||
|
||||
export class InjectedFaultError extends Error {
|
||||
constructor(readonly kind: FaultKind) {
|
||||
super(`Injected ${kind} fault`);
|
||||
this.name = 'InjectedFaultError';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Active only with FAULT_INJECTION_ENABLED=1. The switch is deliberately
|
||||
* checked before every Redis call so a normal stand never depends on Redis for
|
||||
* job correctness.
|
||||
*
|
||||
* `transient` stays until the key is deleted; every other value is consumed by
|
||||
* the first point that observes it. Worker points are addressed to one
|
||||
* attachment and pipeline version; the untargeted key is a fallback.
|
||||
*/
|
||||
@Injectable()
|
||||
export class FaultInjector {
|
||||
private readonly logger = new Logger(FaultInjector.name);
|
||||
private readonly enabled: boolean;
|
||||
|
||||
constructor(
|
||||
config: ConfigService<AppConfig>,
|
||||
private readonly redis: RedisClient,
|
||||
private readonly metrics: MetricsService,
|
||||
) {
|
||||
this.enabled = config.getOrThrow('FAULT_INJECTION_ENABLED') === 1;
|
||||
}
|
||||
|
||||
async point(
|
||||
name: FaultPoint,
|
||||
target?: FaultTarget,
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
if (!this.enabled) return;
|
||||
const keys = target
|
||||
? [
|
||||
`fault:${name}:${target.attachmentId}:${target.pipelineVersion}`,
|
||||
`fault:${name}`,
|
||||
]
|
||||
: [`fault:${name}`];
|
||||
for (const key of keys) {
|
||||
let value: string | null;
|
||||
let deleted: number;
|
||||
try {
|
||||
value = await this.redis.get(key);
|
||||
if (value === null) continue;
|
||||
if (value === 'transient') {
|
||||
this.metrics.increment('faults.triggered');
|
||||
throw new InjectedFaultError('transient');
|
||||
}
|
||||
deleted = await this.redis.del(key);
|
||||
} catch (error) {
|
||||
if (error instanceof InjectedFaultError) throw error;
|
||||
this.logger.warn(
|
||||
`Fault point ${name} could not read Redis: ${error instanceof Error ? error.message : 'unknown error'}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (deleted !== 1) continue;
|
||||
// The action is outside the Redis try/catch: an abort or an injected
|
||||
// fault must reach the job processor, not be logged as an outage.
|
||||
await this.act(value, signal);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
private async act(value: string, signal?: AbortSignal): Promise<void> {
|
||||
if (value === 'crash') {
|
||||
this.metrics.increment('faults.triggered');
|
||||
// PID 1 in a container namespace may not be killable by SIGKILL;
|
||||
// exit 137 gives the same observable crash (no graceful shutdown).
|
||||
process.kill(process.pid, 'SIGKILL');
|
||||
process.exit(137);
|
||||
}
|
||||
if (value === 'permanent') {
|
||||
this.metrics.increment('faults.triggered');
|
||||
throw new InjectedFaultError('permanent');
|
||||
}
|
||||
if (value.startsWith('delay:')) {
|
||||
const milliseconds = Number(value.slice('delay:'.length));
|
||||
if (!Number.isFinite(milliseconds) || milliseconds < 0) return;
|
||||
this.metrics.increment('faults.triggered');
|
||||
await this.sleep(milliseconds, signal);
|
||||
}
|
||||
}
|
||||
|
||||
private sleep(milliseconds: number, signal?: AbortSignal): Promise<void> {
|
||||
if (signal?.aborted)
|
||||
return Promise.reject(
|
||||
signal.reason instanceof Error
|
||||
? signal.reason
|
||||
: new Error('Aborted'),
|
||||
);
|
||||
const { promise, resolve, reject } = Promise.withResolvers<void>();
|
||||
const timer = setTimeout(() => {
|
||||
signal?.removeEventListener('abort', cancel);
|
||||
resolve();
|
||||
}, milliseconds);
|
||||
const cancel = (): void => {
|
||||
clearTimeout(timer);
|
||||
reject(
|
||||
signal?.reason instanceof Error
|
||||
? signal.reason
|
||||
: new Error('Aborted'),
|
||||
);
|
||||
};
|
||||
signal?.addEventListener('abort', cancel, { once: true });
|
||||
return promise;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { fitInside } from './fit-inside.js';
|
||||
|
||||
describe('fitInside', () => {
|
||||
it.each([
|
||||
[320, 200, 256, [256, 160]],
|
||||
[1000, 335, 256, [256, 85]],
|
||||
[100, 50, 256, [100, 50]],
|
||||
[10000, 1, 256, [256, 1]],
|
||||
[1, 10000, 256, [1, 256]],
|
||||
])('fits %ix%i into %i as %j', (width, height, box, expected) => {
|
||||
expect(fitInside(width, height, box)).toEqual(expected);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,15 @@
|
||||
/**
|
||||
* Fits `width`x`height` into a `box`x`box` square without upscaling.
|
||||
* Integer arithmetic keeps the result exact for the contract's fixtures:
|
||||
* 320x200 -> 256x160, 1000x335 -> 256x85, 10000x1 -> 256x1.
|
||||
*/
|
||||
export function fitInside(
|
||||
width: number,
|
||||
height: number,
|
||||
box: number,
|
||||
): [number, number] {
|
||||
if (width <= box && height <= box) return [width, height];
|
||||
if (width >= height)
|
||||
return [box, Math.max(1, Math.floor((height * box) / width))];
|
||||
return [Math.max(1, Math.floor((width * box) / height)), box];
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { parseJobMessage } from './job-message.js';
|
||||
|
||||
const contract = JSON.parse(
|
||||
readFileSync(
|
||||
new URL('../../../contracts/jobs.schema.json', import.meta.url),
|
||||
'utf8',
|
||||
),
|
||||
) as { examples: Record<string, unknown>[] };
|
||||
const example = contract.examples[0];
|
||||
|
||||
describe('parseJobMessage', () => {
|
||||
it('accepts the contract example', () => {
|
||||
expect(parseJobMessage(Buffer.from(JSON.stringify(example)))).toEqual(
|
||||
example,
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects an extra property', () => {
|
||||
expect(
|
||||
parseJobMessage(
|
||||
Buffer.from(JSON.stringify({ ...example, extra: true })),
|
||||
),
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it('rejects attempt 0', () => {
|
||||
expect(
|
||||
parseJobMessage(
|
||||
Buffer.from(JSON.stringify({ ...example, attempt: 0 })),
|
||||
),
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it('rejects a wrong type', () => {
|
||||
expect(
|
||||
parseJobMessage(
|
||||
Buffer.from(JSON.stringify({ ...example, type: 'other' })),
|
||||
),
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it('rejects non-JSON', () => {
|
||||
expect(parseJobMessage(Buffer.from('not-json'))).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,53 @@
|
||||
import { z } from 'zod';
|
||||
|
||||
export const JOB_TYPE = 'attachment.process.v1';
|
||||
|
||||
/**
|
||||
* Mirrors contracts/jobs.schema.json. The broker message carries identifiers
|
||||
* and scheduling data only: no bytes, URLs, file names or credentials.
|
||||
*/
|
||||
export const jobMessageSchema = z.strictObject({
|
||||
schema_version: z.literal(1),
|
||||
job_id: z.uuid(),
|
||||
type: z.literal(JOB_TYPE),
|
||||
attachment_id: z.uuid(),
|
||||
pipeline_version: z.number().int().min(1),
|
||||
attempt: z.number().int().min(1),
|
||||
created_at: z.iso.datetime(),
|
||||
traceparent: z
|
||||
.string()
|
||||
.regex(/^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$/u)
|
||||
.optional(),
|
||||
tracestate: z.string().max(512).optional(),
|
||||
});
|
||||
|
||||
export type JobMessage = z.infer<typeof jobMessageSchema>;
|
||||
|
||||
export function buildJobMessage(job: {
|
||||
job_id: string;
|
||||
attachment_id: string;
|
||||
pipeline_version: number;
|
||||
attempt: number;
|
||||
created_at: Date;
|
||||
}): JobMessage {
|
||||
return {
|
||||
schema_version: 1,
|
||||
job_id: job.job_id,
|
||||
type: JOB_TYPE,
|
||||
attachment_id: job.attachment_id,
|
||||
pipeline_version: job.pipeline_version,
|
||||
attempt: job.attempt,
|
||||
created_at: job.created_at.toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
export function parseJobMessage(content: Buffer): JobMessage | undefined {
|
||||
let value: unknown;
|
||||
try {
|
||||
value = JSON.parse(content.toString('utf8'));
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
const parsed = jobMessageSchema.safeParse(value);
|
||||
return parsed.success ? parsed.data : undefined;
|
||||
}
|
||||
@@ -0,0 +1,234 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { NestFactory } from '@nestjs/core';
|
||||
import type { AppConfig } from '../config/app-config.schema.js';
|
||||
import { AppConfigModule } from '../config/app-config.module.js';
|
||||
import { DatabaseModule } from '../database/database.module.js';
|
||||
import { MetricsModule } from '../metrics/metrics.module.js';
|
||||
import { RedisModule } from '../redis/redis.module.js';
|
||||
import { ObjectStorage } from '../storage/object-storage.js';
|
||||
import { StorageModule } from '../storage/storage.module.js';
|
||||
import { DEAD_LETTER_QUEUE, JOB_QUEUE } from './broker.topology.js';
|
||||
import { BrokerClient } from './broker.client.js';
|
||||
import { JobsModule } from './jobs.module.js';
|
||||
import { JobsRepository } from './jobs.repository.js';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
AppConfigModule,
|
||||
DatabaseModule,
|
||||
RedisModule,
|
||||
MetricsModule,
|
||||
StorageModule,
|
||||
JobsModule,
|
||||
],
|
||||
})
|
||||
class JobsCliModule {}
|
||||
|
||||
const SUPPORTED_PIPELINES = [1, 2];
|
||||
|
||||
function option(args: string[], name: string): string | undefined {
|
||||
const index = args.indexOf(name);
|
||||
if (index === -1) return undefined;
|
||||
return args[index + 1];
|
||||
}
|
||||
|
||||
async function readStdin(): Promise<string> {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of process.stdin)
|
||||
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||
return Buffer.concat(chunks).toString('utf8');
|
||||
}
|
||||
|
||||
async function s3State(
|
||||
storage: ObjectStorage,
|
||||
key: string,
|
||||
): Promise<{
|
||||
exists: boolean;
|
||||
etag: string | null;
|
||||
last_modified: string | null;
|
||||
}> {
|
||||
const head = await storage.head(key);
|
||||
return head
|
||||
? {
|
||||
exists: true,
|
||||
etag: head.etag ?? null,
|
||||
last_modified: head.lastModified?.toISOString() ?? null,
|
||||
}
|
||||
: { exists: false, etag: null, last_modified: null };
|
||||
}
|
||||
|
||||
async function main(): Promise<number> {
|
||||
const args = process.argv.slice(2);
|
||||
const command = args[0];
|
||||
const app = await NestFactory.createApplicationContext(JobsCliModule, {
|
||||
logger: false,
|
||||
});
|
||||
try {
|
||||
const repository = app.get(JobsRepository);
|
||||
const storage = app.get(ObjectStorage);
|
||||
const broker = app.get(BrokerClient);
|
||||
const config = app.get(ConfigService<AppConfig>);
|
||||
|
||||
if (command === 'status') {
|
||||
const attachmentId = args[1];
|
||||
if (!attachmentId) return usage();
|
||||
const status = await repository.statusOf(attachmentId);
|
||||
if (!status) {
|
||||
console.log(JSON.stringify({ error: 'attachment_not_found' }));
|
||||
return 1;
|
||||
}
|
||||
const variants = [];
|
||||
for (const variant of status.variants) {
|
||||
variants.push({
|
||||
pipeline_version: variant.pipeline_version,
|
||||
variant: variant.variant,
|
||||
sha256: variant.sha256,
|
||||
size_bytes: Number(variant.size_bytes),
|
||||
width: variant.width,
|
||||
height: variant.height,
|
||||
object_key: variant.object_key,
|
||||
s3: await s3State(storage, variant.object_key),
|
||||
});
|
||||
}
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
attachment: {
|
||||
id: status.attachment.id,
|
||||
status: status.attachment.status,
|
||||
error_code: status.attachment.error_code,
|
||||
ready_pipeline_version:
|
||||
status.attachment.ready_pipeline_version,
|
||||
staging_present: status.attachment.staging_key !== null,
|
||||
},
|
||||
jobs: status.jobs.map((job) => ({
|
||||
job_id: job.job_id,
|
||||
pipeline_version: job.pipeline_version,
|
||||
status: job.status,
|
||||
attempt: job.attempt,
|
||||
max_attempts: job.max_attempts,
|
||||
last_error_code: job.last_error_code,
|
||||
})),
|
||||
variants,
|
||||
original: await s3State(
|
||||
storage,
|
||||
status.attachment.object_key,
|
||||
),
|
||||
events: status.events.map((event) => ({
|
||||
job_id: event.job_id,
|
||||
event: event.event,
|
||||
attempt: event.attempt,
|
||||
detail: event.detail,
|
||||
created_at: event.created_at.toISOString(),
|
||||
})),
|
||||
outbox: status.outbox,
|
||||
}),
|
||||
);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (command === 'find-by-chat') {
|
||||
const chatId = args[1];
|
||||
if (!chatId) return usage();
|
||||
console.log(JSON.stringify(await repository.findByChat(chatId)));
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (command === 'failed') {
|
||||
console.log(JSON.stringify(await repository.failedJobs()));
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (command === 'replay') {
|
||||
const attachmentId = args[1];
|
||||
const reason = option(args, '--reason') ?? 'operator replay';
|
||||
const operator =
|
||||
option(args, '--operator') ?? process.env.USER ?? 'operator';
|
||||
if (!attachmentId) return usage();
|
||||
const candidate = await repository.replayCandidate(attachmentId);
|
||||
let sourceAvailable = true;
|
||||
if (candidate && candidate.staging_key === null) {
|
||||
const original = await storage.head(
|
||||
storage.objectKey(attachmentId),
|
||||
);
|
||||
sourceAvailable = original !== null;
|
||||
}
|
||||
const result = await repository.replay(attachmentId, {
|
||||
reason,
|
||||
operator,
|
||||
maxAttempts: config.getOrThrow('JOB_MAX_ATTEMPTS'),
|
||||
sourceAvailable,
|
||||
});
|
||||
console.log(JSON.stringify(result));
|
||||
return result.kind === 'replayed' ? 0 : 3;
|
||||
}
|
||||
|
||||
if (command === 'reprocess') {
|
||||
const attachmentId = args[1];
|
||||
const rawVersion = option(args, '--pipeline-version');
|
||||
const version = Number(rawVersion);
|
||||
if (
|
||||
!attachmentId ||
|
||||
rawVersion === undefined ||
|
||||
!SUPPORTED_PIPELINES.includes(version)
|
||||
)
|
||||
return usage();
|
||||
const result = await repository.reprocess(attachmentId, version, {
|
||||
reason: option(args, '--reason') ?? 'reprocess',
|
||||
operator:
|
||||
option(args, '--operator') ??
|
||||
process.env.USER ??
|
||||
'operator',
|
||||
maxAttempts: config.getOrThrow('JOB_MAX_ATTEMPTS'),
|
||||
});
|
||||
console.log(JSON.stringify(result));
|
||||
return result.kind === 'created' ? 0 : 3;
|
||||
}
|
||||
|
||||
if (command === 'publish-raw') {
|
||||
if (config.getOrThrow('FAULT_INJECTION_ENABLED') !== 1) {
|
||||
console.log(
|
||||
JSON.stringify({ refused: 'fault_injection_disabled' }),
|
||||
);
|
||||
return 3;
|
||||
}
|
||||
const text = await readStdin();
|
||||
const lines = text.split(/\r?\n/u).filter((line) => line !== '');
|
||||
for (const line of lines) await broker.publish(Buffer.from(line));
|
||||
console.log(JSON.stringify({ published: lines.length }));
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (command === 'queues') {
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
process: await broker.queueDepth(JOB_QUEUE),
|
||||
dlq: await broker.queueDepth(DEAD_LETTER_QUEUE),
|
||||
}),
|
||||
);
|
||||
return 0;
|
||||
}
|
||||
|
||||
return usage();
|
||||
} finally {
|
||||
await app.close();
|
||||
}
|
||||
}
|
||||
|
||||
function usage(): number {
|
||||
console.error(
|
||||
'usage: jobs.cli.js status <id> | find-by-chat <chat> | failed | replay <id> --reason <text> [--operator <name>] | reprocess <id> --pipeline-version <n> [--reason <text>] | publish-raw | queues',
|
||||
);
|
||||
return 2;
|
||||
}
|
||||
|
||||
main()
|
||||
.then((code) => process.exit(code))
|
||||
.catch((error: unknown) => {
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
error: error instanceof Error ? error.message : 'unknown error',
|
||||
}),
|
||||
);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,14 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { DatabaseModule } from '../database/database.module.js';
|
||||
import { MetricsModule } from '../metrics/metrics.module.js';
|
||||
import { RedisModule } from '../redis/redis.module.js';
|
||||
import { BrokerClient } from './broker.client.js';
|
||||
import { FaultInjector } from './fault-injector.js';
|
||||
import { JobsRepository } from './jobs.repository.js';
|
||||
|
||||
@Module({
|
||||
imports: [DatabaseModule, RedisModule, MetricsModule],
|
||||
providers: [BrokerClient, JobsRepository, FaultInjector],
|
||||
exports: [BrokerClient, JobsRepository, FaultInjector],
|
||||
})
|
||||
export class JobsModule {}
|
||||
@@ -0,0 +1,910 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import type { PoolClient } from 'pg';
|
||||
import type { AttachmentRow, VariantRow } from '../attachments/attachment.js';
|
||||
import { DatabaseClient } from '../database/database.client.js';
|
||||
import { buildJobMessage, type JobMessage } from './job-message.js';
|
||||
|
||||
export type JobStatus =
|
||||
'queued' | 'processing' | 'succeeded' | 'failed' | 'superseded';
|
||||
|
||||
export type AttachmentJobRow = {
|
||||
job_id: string;
|
||||
attachment_id: string;
|
||||
pipeline_version: number;
|
||||
status: JobStatus;
|
||||
attempt: number;
|
||||
max_attempts: number;
|
||||
next_attempt_at: Date;
|
||||
lease_owner: string | null;
|
||||
lease_token: string | null;
|
||||
lease_until: Date | null;
|
||||
last_error_code: string | null;
|
||||
created_at: Date;
|
||||
updated_at: Date;
|
||||
finished_at: Date | null;
|
||||
};
|
||||
|
||||
export type VariantWrite = {
|
||||
name: 'thumbnail' | 'crop' | 'optimized';
|
||||
objectKey: string;
|
||||
contentType: 'image/jpeg' | 'image/png' | 'image/webp';
|
||||
sizeBytes: number;
|
||||
width: number;
|
||||
height: number;
|
||||
sha256: string;
|
||||
};
|
||||
|
||||
export type ClaimResult =
|
||||
| { kind: 'skip'; reason: string }
|
||||
| {
|
||||
kind: 'claimed';
|
||||
job: AttachmentJobRow;
|
||||
attachment: AttachmentRow;
|
||||
leaseToken: string;
|
||||
};
|
||||
|
||||
export type CommitFailureResult = 'retry' | 'failed' | 'lease_lost';
|
||||
export type CommitSuccessResult = 'succeeded' | 'superseded' | 'lease_lost';
|
||||
|
||||
export type ReplayResult =
|
||||
| { kind: 'replayed'; job_id: string; attempt: number }
|
||||
| { kind: 'refused'; reason: string; job_id?: string };
|
||||
|
||||
export type ReprocessResult =
|
||||
| { kind: 'created'; job_id: string }
|
||||
| { kind: 'refused'; reason: string; job_id?: string };
|
||||
|
||||
type AttachmentJobExtra = AttachmentJobRow & {
|
||||
attachment_status: AttachmentRow['status'];
|
||||
ready_pipeline_version: number | null;
|
||||
};
|
||||
|
||||
@Injectable()
|
||||
export class JobsRepository {
|
||||
constructor(private readonly database: DatabaseClient) {}
|
||||
|
||||
async acceptUpload(
|
||||
attachmentId: string,
|
||||
fields: {
|
||||
sizeBytes: number;
|
||||
sha256: string;
|
||||
width: number;
|
||||
height: number;
|
||||
},
|
||||
pipelineVersion: number,
|
||||
maxAttempts: number,
|
||||
): Promise<AttachmentRow | undefined> {
|
||||
return this.database.withTransaction(async (client) => {
|
||||
const accepted = await client.query<AttachmentRow>(
|
||||
`UPDATE attachments
|
||||
SET status = 'queued', size_bytes = $2, sha256 = $3,
|
||||
width = $4, height = $5, expires_at = NULL, updated_at = now()
|
||||
WHERE id = $1 AND status = 'pending'
|
||||
RETURNING *`,
|
||||
[
|
||||
attachmentId,
|
||||
fields.sizeBytes,
|
||||
fields.sha256,
|
||||
fields.width,
|
||||
fields.height,
|
||||
],
|
||||
);
|
||||
const attachment = accepted.rows[0];
|
||||
if (!attachment) return undefined;
|
||||
|
||||
const jobId = randomUUID();
|
||||
const created = await client.query<AttachmentJobRow>(
|
||||
`INSERT INTO attachment_jobs
|
||||
(job_id, attachment_id, pipeline_version, status, attempt,
|
||||
max_attempts, next_attempt_at)
|
||||
VALUES ($1, $2, $3, 'queued', 1, $4, now())
|
||||
RETURNING *`,
|
||||
[jobId, attachmentId, pipelineVersion, maxAttempts],
|
||||
);
|
||||
const job = created.rows[0];
|
||||
await client.query(
|
||||
`INSERT INTO job_outbox (job_id, payload, available_at)
|
||||
VALUES ($1, $2::jsonb, now())`,
|
||||
[
|
||||
job.job_id,
|
||||
JSON.stringify(
|
||||
buildJobMessage({
|
||||
...job,
|
||||
created_at: job.created_at,
|
||||
}),
|
||||
),
|
||||
],
|
||||
);
|
||||
await this.event(client, job.job_id, 'created', 1, {});
|
||||
return attachment;
|
||||
});
|
||||
}
|
||||
|
||||
async claim(
|
||||
message: JobMessage,
|
||||
workerId: string,
|
||||
leaseSeconds: number,
|
||||
): Promise<ClaimResult> {
|
||||
return this.database.withTransaction(async (client) => {
|
||||
const found = await client.query<AttachmentJobExtra>(
|
||||
`SELECT j.*, a.status AS attachment_status,
|
||||
a.ready_pipeline_version
|
||||
FROM attachment_jobs j
|
||||
JOIN attachments a ON a.id = j.attachment_id
|
||||
WHERE j.job_id = $1
|
||||
FOR UPDATE OF j, a`,
|
||||
[message.job_id],
|
||||
);
|
||||
const job = found.rows[0];
|
||||
if (!job) return { kind: 'skip', reason: 'unknown_job' };
|
||||
|
||||
if (
|
||||
job.attachment_id !== message.attachment_id ||
|
||||
job.pipeline_version !== message.pipeline_version
|
||||
)
|
||||
return this.skip(client, message, 'mismatch');
|
||||
|
||||
if (job.status === 'succeeded')
|
||||
return this.skip(client, message, 'already_succeeded');
|
||||
if (job.status === 'failed')
|
||||
return this.skip(client, message, 'already_failed');
|
||||
if (job.status === 'superseded')
|
||||
return this.skip(client, message, 'superseded');
|
||||
if (job.status === 'processing')
|
||||
return this.skip(client, message, 'in_progress');
|
||||
if (job.attempt !== message.attempt)
|
||||
return this.skip(client, message, 'stale_attempt');
|
||||
if (new Date(job.next_attempt_at).getTime() > Date.now())
|
||||
return this.skip(client, message, 'not_due');
|
||||
|
||||
if (
|
||||
job.ready_pipeline_version !== null &&
|
||||
job.ready_pipeline_version > job.pipeline_version
|
||||
) {
|
||||
await client.query(
|
||||
`UPDATE attachment_jobs
|
||||
SET status = 'superseded', updated_at = now(), finished_at = now()
|
||||
WHERE job_id = $1`,
|
||||
[job.job_id],
|
||||
);
|
||||
await this.event(
|
||||
client,
|
||||
job.job_id,
|
||||
'superseded',
|
||||
job.attempt,
|
||||
{
|
||||
ready_pipeline_version: job.ready_pipeline_version,
|
||||
},
|
||||
);
|
||||
return this.skip(client, message, 'superseded');
|
||||
}
|
||||
|
||||
const leaseToken = randomUUID();
|
||||
const claimed = await client.query<AttachmentJobRow>(
|
||||
`UPDATE attachment_jobs
|
||||
SET status = 'processing', lease_owner = $2, lease_token = $3,
|
||||
lease_until = now() + make_interval(secs => $4::double precision),
|
||||
updated_at = now()
|
||||
WHERE job_id = $1
|
||||
RETURNING *`,
|
||||
[job.job_id, workerId, leaseToken, leaseSeconds],
|
||||
);
|
||||
const attachment = await client.query<AttachmentRow>(
|
||||
`SELECT * FROM attachments WHERE id = $1`,
|
||||
[job.attachment_id],
|
||||
);
|
||||
await client.query(
|
||||
`UPDATE attachments SET status = 'processing', updated_at = now()
|
||||
WHERE id = $1 AND status = 'queued'`,
|
||||
[job.attachment_id],
|
||||
);
|
||||
await this.event(
|
||||
client,
|
||||
job.job_id,
|
||||
'claimed',
|
||||
claimed.rows[0].attempt,
|
||||
{ lease_owner: workerId },
|
||||
);
|
||||
return {
|
||||
kind: 'claimed',
|
||||
job: claimed.rows[0],
|
||||
attachment: attachment.rows[0],
|
||||
leaseToken,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async extendLease(
|
||||
jobId: string,
|
||||
token: string,
|
||||
leaseSeconds: number,
|
||||
): Promise<boolean> {
|
||||
const result = await this.database.query(
|
||||
'jobs.extend_lease',
|
||||
`UPDATE attachment_jobs
|
||||
SET lease_until = now() + make_interval(secs => $3::double precision),
|
||||
updated_at = now()
|
||||
WHERE job_id = $1 AND lease_token = $2 AND status = 'processing'`,
|
||||
[jobId, token, leaseSeconds],
|
||||
);
|
||||
return result.rowCount === 1;
|
||||
}
|
||||
|
||||
async commitSuccess(
|
||||
jobId: string,
|
||||
token: string,
|
||||
variants: VariantWrite[],
|
||||
): Promise<CommitSuccessResult> {
|
||||
return this.database.withTransaction(async (client) => {
|
||||
const succeeded = await client.query<AttachmentJobRow>(
|
||||
`UPDATE attachment_jobs
|
||||
SET status = 'succeeded', lease_owner = NULL, lease_token = NULL,
|
||||
lease_until = NULL, updated_at = now(), finished_at = now()
|
||||
WHERE job_id = $1 AND lease_token = $2 AND status = 'processing'
|
||||
RETURNING *`,
|
||||
[jobId, token],
|
||||
);
|
||||
const job = succeeded.rows[0];
|
||||
if (!job) return 'lease_lost';
|
||||
|
||||
const attachment = await client.query<AttachmentRow>(
|
||||
`SELECT * FROM attachments WHERE id = $1 FOR UPDATE`,
|
||||
[job.attachment_id],
|
||||
);
|
||||
const row = attachment.rows[0];
|
||||
if (
|
||||
row.ready_pipeline_version !== null &&
|
||||
row.ready_pipeline_version > job.pipeline_version
|
||||
) {
|
||||
await client.query(
|
||||
`UPDATE attachment_jobs
|
||||
SET status = 'superseded', updated_at = now()
|
||||
WHERE job_id = $1`,
|
||||
[job.job_id],
|
||||
);
|
||||
await this.event(
|
||||
client,
|
||||
job.job_id,
|
||||
'superseded',
|
||||
job.attempt,
|
||||
{
|
||||
ready_pipeline_version: row.ready_pipeline_version,
|
||||
},
|
||||
);
|
||||
return 'superseded';
|
||||
}
|
||||
|
||||
for (const variant of variants) {
|
||||
await client.query(
|
||||
`INSERT INTO attachment_variants
|
||||
(attachment_id, pipeline_version, variant, object_key,
|
||||
content_type, size_bytes, width, height, sha256, job_id)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
|
||||
ON CONFLICT (attachment_id, pipeline_version, variant)
|
||||
DO UPDATE SET object_key = EXCLUDED.object_key,
|
||||
content_type = EXCLUDED.content_type,
|
||||
size_bytes = EXCLUDED.size_bytes,
|
||||
width = EXCLUDED.width,
|
||||
height = EXCLUDED.height,
|
||||
sha256 = EXCLUDED.sha256,
|
||||
job_id = EXCLUDED.job_id`,
|
||||
[
|
||||
job.attachment_id,
|
||||
job.pipeline_version,
|
||||
variant.name,
|
||||
variant.objectKey,
|
||||
variant.contentType,
|
||||
variant.sizeBytes,
|
||||
variant.width,
|
||||
variant.height,
|
||||
variant.sha256,
|
||||
job.job_id,
|
||||
],
|
||||
);
|
||||
}
|
||||
await client.query(
|
||||
`UPDATE attachments
|
||||
SET status = 'ready', ready_pipeline_version = $2,
|
||||
error_code = NULL, updated_at = now()
|
||||
WHERE id = $1`,
|
||||
[job.attachment_id, job.pipeline_version],
|
||||
);
|
||||
await this.event(client, job.job_id, 'succeeded', job.attempt, {});
|
||||
return 'succeeded';
|
||||
});
|
||||
}
|
||||
|
||||
async commitFailure(
|
||||
jobId: string,
|
||||
token: string,
|
||||
failure: { kind: 'permanent' | 'transient'; code: string },
|
||||
delayMs: number,
|
||||
): Promise<CommitFailureResult> {
|
||||
return this.database.withTransaction(async (client) => {
|
||||
const found = await client.query<AttachmentJobRow>(
|
||||
`SELECT * FROM attachment_jobs
|
||||
WHERE job_id = $1 AND lease_token = $2 AND status = 'processing'
|
||||
FOR UPDATE`,
|
||||
[jobId, token],
|
||||
);
|
||||
const job = found.rows[0];
|
||||
if (!job) return 'lease_lost';
|
||||
const finalCode =
|
||||
failure.kind === 'permanent' || job.attempt >= job.max_attempts
|
||||
? failure.kind === 'transient'
|
||||
? 'retry_exhausted'
|
||||
: failure.code
|
||||
: failure.code;
|
||||
|
||||
if (
|
||||
failure.kind === 'permanent' ||
|
||||
job.attempt >= job.max_attempts
|
||||
) {
|
||||
await client.query(
|
||||
`UPDATE attachment_jobs
|
||||
SET status = 'failed', last_error_code = $2,
|
||||
lease_owner = NULL, lease_token = NULL, lease_until = NULL,
|
||||
updated_at = now(), finished_at = now()
|
||||
WHERE job_id = $1`,
|
||||
[job.job_id, finalCode],
|
||||
);
|
||||
await client.query(
|
||||
`UPDATE attachments
|
||||
SET status = 'failed', error_code = $2, updated_at = now()
|
||||
WHERE id = $1 AND status <> 'ready'`,
|
||||
[job.attachment_id, finalCode],
|
||||
);
|
||||
await this.event(client, job.job_id, 'failed', job.attempt, {
|
||||
code: finalCode,
|
||||
cause: failure.code,
|
||||
});
|
||||
return 'failed';
|
||||
}
|
||||
|
||||
const nextAttempt = job.attempt + 1;
|
||||
await client.query(
|
||||
`UPDATE attachment_jobs
|
||||
SET status = 'queued', attempt = $2,
|
||||
next_attempt_at = now() + make_interval(secs => $3::double precision),
|
||||
lease_owner = NULL, lease_token = NULL, lease_until = NULL,
|
||||
last_error_code = $4, updated_at = now()
|
||||
WHERE job_id = $1`,
|
||||
[job.job_id, nextAttempt, delayMs / 1000, failure.code],
|
||||
);
|
||||
await client.query(
|
||||
`UPDATE attachments SET status = 'queued', updated_at = now()
|
||||
WHERE id = $1 AND status = 'processing'`,
|
||||
[job.attachment_id],
|
||||
);
|
||||
await client.query(
|
||||
`INSERT INTO job_outbox (job_id, payload, available_at)
|
||||
VALUES ($1, $2::jsonb,
|
||||
now() + make_interval(secs => $3::double precision))`,
|
||||
[
|
||||
job.job_id,
|
||||
JSON.stringify(
|
||||
buildJobMessage({ ...job, attempt: nextAttempt }),
|
||||
),
|
||||
delayMs / 1000,
|
||||
],
|
||||
);
|
||||
await this.event(
|
||||
client,
|
||||
job.job_id,
|
||||
'retry_scheduled',
|
||||
nextAttempt,
|
||||
{
|
||||
code: failure.code,
|
||||
delay_ms: delayMs,
|
||||
},
|
||||
);
|
||||
return 'retry';
|
||||
});
|
||||
}
|
||||
|
||||
async reapExpiredLeases(
|
||||
limit: number,
|
||||
delayFor: (attempt: number) => number,
|
||||
): Promise<number> {
|
||||
return this.database.withTransaction(async (client) => {
|
||||
const expired = await client.query<AttachmentJobExtra>(
|
||||
`SELECT j.*, a.status AS attachment_status,
|
||||
a.ready_pipeline_version
|
||||
FROM attachment_jobs j
|
||||
JOIN attachments a ON a.id = j.attachment_id
|
||||
WHERE j.status = 'processing' AND j.lease_until < now()
|
||||
ORDER BY j.lease_until
|
||||
LIMIT $1
|
||||
FOR UPDATE OF j, a`,
|
||||
[limit],
|
||||
);
|
||||
for (const job of expired.rows) {
|
||||
if (job.attempt >= job.max_attempts) {
|
||||
await client.query(
|
||||
`UPDATE attachment_jobs
|
||||
SET status = 'failed', last_error_code = 'retry_exhausted',
|
||||
lease_owner = NULL, lease_token = NULL,
|
||||
lease_until = NULL, updated_at = now(),
|
||||
finished_at = now()
|
||||
WHERE job_id = $1`,
|
||||
[job.job_id],
|
||||
);
|
||||
await client.query(
|
||||
`UPDATE attachments
|
||||
SET status = 'failed', error_code = 'retry_exhausted',
|
||||
updated_at = now()
|
||||
WHERE id = $1 AND status <> 'ready'`,
|
||||
[job.attachment_id],
|
||||
);
|
||||
await this.event(
|
||||
client,
|
||||
job.job_id,
|
||||
'lease_expired',
|
||||
job.attempt,
|
||||
{
|
||||
lease_owner: job.lease_owner,
|
||||
code: 'retry_exhausted',
|
||||
},
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const nextAttempt = job.attempt + 1;
|
||||
const delayMs = delayFor(job.attempt);
|
||||
await client.query(
|
||||
`UPDATE attachment_jobs
|
||||
SET status = 'queued', attempt = $2,
|
||||
next_attempt_at = now() + make_interval(secs => $3::double precision),
|
||||
lease_owner = NULL, lease_token = NULL,
|
||||
lease_until = NULL, last_error_code = 'lease_expired',
|
||||
updated_at = now()
|
||||
WHERE job_id = $1`,
|
||||
[job.job_id, nextAttempt, delayMs / 1000],
|
||||
);
|
||||
await client.query(
|
||||
`UPDATE attachments SET status = 'queued', updated_at = now()
|
||||
WHERE id = $1 AND status = 'processing'`,
|
||||
[job.attachment_id],
|
||||
);
|
||||
await client.query(
|
||||
`INSERT INTO job_outbox (job_id, payload, available_at)
|
||||
VALUES ($1, $2::jsonb,
|
||||
now() + make_interval(secs => $3::double precision))`,
|
||||
[
|
||||
job.job_id,
|
||||
JSON.stringify(
|
||||
buildJobMessage({ ...job, attempt: nextAttempt }),
|
||||
),
|
||||
delayMs / 1000,
|
||||
],
|
||||
);
|
||||
await this.event(
|
||||
client,
|
||||
job.job_id,
|
||||
'lease_expired',
|
||||
job.attempt,
|
||||
{ lease_owner: job.lease_owner },
|
||||
);
|
||||
}
|
||||
return expired.rows.length;
|
||||
});
|
||||
}
|
||||
|
||||
async publishPending(
|
||||
limit: number,
|
||||
publish: (message: JobMessage) => Promise<void>,
|
||||
afterPublish: () => Promise<void>,
|
||||
): Promise<number> {
|
||||
return this.database.withTransaction(async (client) => {
|
||||
const pending = await client.query<{
|
||||
id: string;
|
||||
payload: JobMessage;
|
||||
}>(
|
||||
`SELECT id, payload FROM job_outbox
|
||||
WHERE published_at IS NULL AND available_at <= now()
|
||||
ORDER BY available_at, id
|
||||
LIMIT $1
|
||||
FOR UPDATE SKIP LOCKED`,
|
||||
[limit],
|
||||
);
|
||||
const published: string[] = [];
|
||||
let failure: unknown;
|
||||
for (const row of pending.rows) {
|
||||
try {
|
||||
await publish(row.payload);
|
||||
published.push(row.id);
|
||||
} catch (error) {
|
||||
failure = error;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (published.length === 0) {
|
||||
if (failure) throw failure;
|
||||
return 0;
|
||||
}
|
||||
await afterPublish();
|
||||
await client.query(
|
||||
`UPDATE job_outbox SET published_at = now() WHERE id = ANY($1::bigint[])`,
|
||||
[published],
|
||||
);
|
||||
if (failure) throw failure;
|
||||
return published.length;
|
||||
});
|
||||
}
|
||||
|
||||
async variantsFor(
|
||||
attachmentId: string,
|
||||
pipelineVersion: number,
|
||||
): Promise<VariantRow[]> {
|
||||
const result = await this.database.query<VariantRow>(
|
||||
'jobs.variants',
|
||||
`SELECT * FROM attachment_variants
|
||||
WHERE attachment_id = $1 AND pipeline_version = $2
|
||||
ORDER BY array_position(ARRAY['thumbnail','crop','optimized']::text[], variant)`,
|
||||
[attachmentId, pipelineVersion],
|
||||
);
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
async stagingCleanupCandidates(
|
||||
retentionSeconds: number,
|
||||
limit: number,
|
||||
): Promise<{ id: string; staging_key: string }[]> {
|
||||
const result = await this.database.query<{
|
||||
id: string;
|
||||
staging_key: string;
|
||||
}>(
|
||||
'jobs.staging_candidates',
|
||||
`SELECT id, staging_key FROM attachments
|
||||
WHERE staging_key IS NOT NULL
|
||||
AND ((status = 'ready' AND updated_at < now() - interval '1 minute')
|
||||
OR (status = 'failed' AND updated_at < now() - make_interval(secs => $1::double precision)))
|
||||
ORDER BY updated_at
|
||||
LIMIT $2`,
|
||||
[retentionSeconds, limit],
|
||||
);
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
async clearStaging(id: string, key: string): Promise<boolean> {
|
||||
return this.database.withTransaction(async (client) => {
|
||||
const cleared = await client.query(
|
||||
`UPDATE attachments SET staging_key = NULL, updated_at = now()
|
||||
WHERE id = $1 AND staging_key = $2 AND status IN ('ready','failed')`,
|
||||
[id, key],
|
||||
);
|
||||
if (cleared.rowCount !== 1) return false;
|
||||
await client.query(
|
||||
`INSERT INTO job_events (job_id, event, attempt, detail)
|
||||
SELECT job_id, 'staging_deleted', attempt, $2::jsonb
|
||||
FROM attachment_jobs
|
||||
WHERE attachment_id = $1
|
||||
ORDER BY created_at DESC
|
||||
LIMIT 1`,
|
||||
[id, JSON.stringify({ staging_key: key })],
|
||||
);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
async statusOf(attachmentId: string): Promise<
|
||||
| {
|
||||
attachment: AttachmentRow;
|
||||
jobs: AttachmentJobRow[];
|
||||
variants: VariantRow[];
|
||||
events: {
|
||||
job_id: string;
|
||||
event: string;
|
||||
attempt: number;
|
||||
detail: unknown;
|
||||
created_at: Date;
|
||||
}[];
|
||||
outbox: { pending: number; published: number };
|
||||
}
|
||||
| undefined
|
||||
> {
|
||||
const attachment = await this.database.query<AttachmentRow>(
|
||||
'jobs.status_attachment',
|
||||
`SELECT * FROM attachments WHERE id = $1`,
|
||||
[attachmentId],
|
||||
);
|
||||
if (!attachment.rows[0]) return undefined;
|
||||
const jobs = await this.database.query<AttachmentJobRow>(
|
||||
'jobs.status_jobs',
|
||||
`SELECT * FROM attachment_jobs WHERE attachment_id = $1 ORDER BY created_at`,
|
||||
[attachmentId],
|
||||
);
|
||||
const variants = await this.database.query<VariantRow>(
|
||||
'jobs.status_variants',
|
||||
`SELECT * FROM attachment_variants WHERE attachment_id = $1
|
||||
ORDER BY pipeline_version,
|
||||
array_position(ARRAY['thumbnail','crop','optimized']::text[], variant)`,
|
||||
[attachmentId],
|
||||
);
|
||||
const events = await this.database.query<{
|
||||
job_id: string;
|
||||
event: string;
|
||||
attempt: number;
|
||||
detail: unknown;
|
||||
created_at: Date;
|
||||
}>(
|
||||
'jobs.status_events',
|
||||
`SELECT e.job_id, e.event, e.attempt, e.detail, e.created_at
|
||||
FROM job_events e
|
||||
JOIN attachment_jobs j ON j.job_id = e.job_id
|
||||
WHERE j.attachment_id = $1
|
||||
ORDER BY e.id`,
|
||||
[attachmentId],
|
||||
);
|
||||
const outbox = await this.database.query<{
|
||||
pending: string;
|
||||
published: string;
|
||||
}>(
|
||||
'jobs.status_outbox',
|
||||
`SELECT count(*) FILTER (WHERE o.published_at IS NULL) AS pending,
|
||||
count(*) FILTER (WHERE o.published_at IS NOT NULL) AS published
|
||||
FROM job_outbox o
|
||||
JOIN attachment_jobs j ON j.job_id = o.job_id
|
||||
WHERE j.attachment_id = $1`,
|
||||
[attachmentId],
|
||||
);
|
||||
return {
|
||||
attachment: attachment.rows[0],
|
||||
jobs: jobs.rows,
|
||||
variants: variants.rows,
|
||||
events: events.rows,
|
||||
outbox: {
|
||||
pending: Number(outbox.rows[0]?.pending ?? 0),
|
||||
published: Number(outbox.rows[0]?.published ?? 0),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async findByChat(
|
||||
chatId: string,
|
||||
): Promise<{ id: string; status: string; created_at: Date }[]> {
|
||||
const result = await this.database.query<{
|
||||
id: string;
|
||||
status: string;
|
||||
created_at: Date;
|
||||
}>(
|
||||
'jobs.find_by_chat',
|
||||
`SELECT id, status, created_at FROM attachments
|
||||
WHERE chat_id = $1 ORDER BY created_at`,
|
||||
[chatId],
|
||||
);
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
async failedJobs(): Promise<
|
||||
{
|
||||
attachment_id: string;
|
||||
job_id: string;
|
||||
pipeline_version: number;
|
||||
attempt: number;
|
||||
last_error_code: string | null;
|
||||
updated_at: Date;
|
||||
}[]
|
||||
> {
|
||||
const result = await this.database.query<{
|
||||
attachment_id: string;
|
||||
job_id: string;
|
||||
pipeline_version: number;
|
||||
attempt: number;
|
||||
last_error_code: string | null;
|
||||
updated_at: Date;
|
||||
}>(
|
||||
'jobs.failed',
|
||||
`SELECT attachment_id, job_id, pipeline_version, attempt,
|
||||
last_error_code, updated_at
|
||||
FROM attachment_jobs WHERE status = 'failed'
|
||||
ORDER BY updated_at`,
|
||||
);
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
async replayCandidate(attachmentId: string): Promise<
|
||||
| {
|
||||
job_id: string;
|
||||
pipeline_version: number;
|
||||
staging_key: string | null;
|
||||
}
|
||||
| undefined
|
||||
> {
|
||||
const result = await this.database.query<{
|
||||
job_id: string;
|
||||
pipeline_version: number;
|
||||
staging_key: string | null;
|
||||
}>(
|
||||
'jobs.replay_candidate',
|
||||
`SELECT j.job_id, j.pipeline_version, a.staging_key
|
||||
FROM attachment_jobs j
|
||||
JOIN attachments a ON a.id = j.attachment_id
|
||||
WHERE j.attachment_id = $1 AND j.status = 'failed'
|
||||
ORDER BY j.updated_at DESC
|
||||
LIMIT 1`,
|
||||
[attachmentId],
|
||||
);
|
||||
return result.rows[0];
|
||||
}
|
||||
|
||||
async replay(
|
||||
attachmentId: string,
|
||||
options: {
|
||||
reason: string;
|
||||
operator: string;
|
||||
maxAttempts: number;
|
||||
sourceAvailable: boolean;
|
||||
},
|
||||
): Promise<ReplayResult> {
|
||||
return this.database.withTransaction(async (client) => {
|
||||
const found = await client.query<
|
||||
AttachmentJobRow & { staging_key: string | null }
|
||||
>(
|
||||
`SELECT j.*, a.staging_key
|
||||
FROM attachment_jobs j
|
||||
JOIN attachments a ON a.id = j.attachment_id
|
||||
WHERE j.attachment_id = $1 AND j.status = 'failed'
|
||||
ORDER BY j.updated_at DESC
|
||||
LIMIT 1
|
||||
FOR UPDATE OF j, a`,
|
||||
[attachmentId],
|
||||
);
|
||||
const job = found.rows[0];
|
||||
if (!job) return { kind: 'refused', reason: 'no_failed_job' };
|
||||
|
||||
const attachment = await client.query<AttachmentRow>(
|
||||
`SELECT * FROM attachments WHERE id = $1`,
|
||||
[attachmentId],
|
||||
);
|
||||
const row = attachment.rows[0];
|
||||
if (
|
||||
row.status === 'ready' &&
|
||||
row.ready_pipeline_version !== null &&
|
||||
row.ready_pipeline_version >= job.pipeline_version
|
||||
)
|
||||
return {
|
||||
kind: 'refused',
|
||||
reason: 'already_ready',
|
||||
job_id: job.job_id,
|
||||
};
|
||||
|
||||
const active = await client.query(
|
||||
`SELECT 1 FROM attachment_jobs
|
||||
WHERE attachment_id = $1 AND status IN ('queued','processing')
|
||||
LIMIT 1`,
|
||||
[attachmentId],
|
||||
);
|
||||
if (active.rowCount !== 0)
|
||||
return {
|
||||
kind: 'refused',
|
||||
reason: 'job_active',
|
||||
job_id: job.job_id,
|
||||
};
|
||||
|
||||
if (row.staging_key === null && !options.sourceAvailable)
|
||||
return {
|
||||
kind: 'refused',
|
||||
reason: 'source_expired',
|
||||
job_id: job.job_id,
|
||||
};
|
||||
|
||||
const attempt = job.attempt + 1;
|
||||
await client.query(
|
||||
`UPDATE attachment_jobs
|
||||
SET status = 'queued', attempt = $2,
|
||||
max_attempts = $2::integer + $3::integer - 1,
|
||||
next_attempt_at = now(), lease_owner = NULL,
|
||||
lease_token = NULL, lease_until = NULL,
|
||||
updated_at = now(), finished_at = NULL
|
||||
WHERE job_id = $1`,
|
||||
[job.job_id, attempt, options.maxAttempts],
|
||||
);
|
||||
await client.query(
|
||||
`UPDATE attachments SET status = 'queued', error_code = NULL,
|
||||
updated_at = now()
|
||||
WHERE id = $1 AND status = 'failed'`,
|
||||
[attachmentId],
|
||||
);
|
||||
await client.query(
|
||||
`INSERT INTO job_outbox (job_id, payload, available_at)
|
||||
VALUES ($1, $2::jsonb, now())`,
|
||||
[
|
||||
job.job_id,
|
||||
JSON.stringify(buildJobMessage({ ...job, attempt })),
|
||||
],
|
||||
);
|
||||
await this.event(client, job.job_id, 'replayed', attempt, {
|
||||
operator: options.operator,
|
||||
reason: options.reason,
|
||||
previous_error: job.last_error_code,
|
||||
});
|
||||
return { kind: 'replayed', job_id: job.job_id, attempt };
|
||||
});
|
||||
}
|
||||
|
||||
async reprocess(
|
||||
attachmentId: string,
|
||||
pipelineVersion: number,
|
||||
options: { reason: string; operator: string; maxAttempts: number },
|
||||
): Promise<ReprocessResult> {
|
||||
return this.database.withTransaction(async (client) => {
|
||||
const attachment = await client.query<AttachmentRow>(
|
||||
`SELECT * FROM attachments WHERE id = $1 FOR UPDATE`,
|
||||
[attachmentId],
|
||||
);
|
||||
const row = attachment.rows[0];
|
||||
if (
|
||||
!row ||
|
||||
!['queued', 'processing', 'ready', 'failed'].includes(
|
||||
row.status,
|
||||
)
|
||||
)
|
||||
return { kind: 'refused', reason: 'not_accepted' };
|
||||
|
||||
const jobId = randomUUID();
|
||||
const created = await client.query<AttachmentJobRow>(
|
||||
`INSERT INTO attachment_jobs
|
||||
(job_id, attachment_id, pipeline_version, status, attempt,
|
||||
max_attempts, next_attempt_at)
|
||||
VALUES ($1, $2, $3, 'queued', 1, $4, now())
|
||||
ON CONFLICT (attachment_id, pipeline_version) DO NOTHING
|
||||
RETURNING *`,
|
||||
[jobId, attachmentId, pipelineVersion, options.maxAttempts],
|
||||
);
|
||||
const job = created.rows[0];
|
||||
if (!job) {
|
||||
const existing = await client.query<{ job_id: string }>(
|
||||
`SELECT job_id FROM attachment_jobs
|
||||
WHERE attachment_id = $1 AND pipeline_version = $2`,
|
||||
[attachmentId, pipelineVersion],
|
||||
);
|
||||
return {
|
||||
kind: 'refused',
|
||||
reason: 'job_exists',
|
||||
job_id: existing.rows[0]?.job_id,
|
||||
};
|
||||
}
|
||||
await client.query(
|
||||
`INSERT INTO job_outbox (job_id, payload, available_at)
|
||||
VALUES ($1, $2::jsonb, now())`,
|
||||
[job.job_id, JSON.stringify(buildJobMessage(job))],
|
||||
);
|
||||
await this.event(client, job.job_id, 'created', 1, {
|
||||
reason: options.reason,
|
||||
operator: options.operator,
|
||||
});
|
||||
return { kind: 'created', job_id: job.job_id };
|
||||
});
|
||||
}
|
||||
|
||||
private async skip(
|
||||
client: PoolClient,
|
||||
message: JobMessage,
|
||||
reason: string,
|
||||
): Promise<ClaimResult> {
|
||||
await this.event(
|
||||
client,
|
||||
message.job_id,
|
||||
'duplicate_skipped',
|
||||
message.attempt,
|
||||
{ reason, message_attempt: message.attempt },
|
||||
);
|
||||
return { kind: 'skip', reason };
|
||||
}
|
||||
|
||||
private async event(
|
||||
client: PoolClient,
|
||||
jobId: string,
|
||||
event: string,
|
||||
attempt: number,
|
||||
detail: Record<string, unknown>,
|
||||
): Promise<void> {
|
||||
await client.query(
|
||||
`INSERT INTO job_events (job_id, event, attempt, detail)
|
||||
VALUES ($1, $2, $3, $4::jsonb)`,
|
||||
[jobId, event, attempt, JSON.stringify(detail)],
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
import {
|
||||
Injectable,
|
||||
Logger,
|
||||
OnApplicationBootstrap,
|
||||
OnModuleDestroy,
|
||||
} from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import type { AppConfig } from '../config/app-config.schema.js';
|
||||
import { MetricsService } from '../metrics/metrics.service.js';
|
||||
import { BrokerClient } from './broker.client.js';
|
||||
import { FaultInjector } from './fault-injector.js';
|
||||
import { JobsRepository } from './jobs.repository.js';
|
||||
|
||||
const BATCH_SIZE = 100;
|
||||
|
||||
/**
|
||||
* Runs only in the API process. `publishPending` holds one transaction around
|
||||
* the broker confirms and the `published_at` update; a crash before the mark
|
||||
* rolls the transaction back and republishes the same message after restart.
|
||||
* That duplicate is expected: the worker fences it on job status.
|
||||
*/
|
||||
@Injectable()
|
||||
export class OutboxRelay implements OnApplicationBootstrap, OnModuleDestroy {
|
||||
private readonly logger = new Logger(OutboxRelay.name);
|
||||
private timer: NodeJS.Timeout | undefined;
|
||||
private running = false;
|
||||
private again = false;
|
||||
private current: Promise<void> | undefined;
|
||||
private failing = false;
|
||||
|
||||
constructor(
|
||||
private readonly config: ConfigService<AppConfig>,
|
||||
private readonly repository: JobsRepository,
|
||||
private readonly broker: BrokerClient,
|
||||
private readonly faults: FaultInjector,
|
||||
private readonly metrics: MetricsService,
|
||||
) {}
|
||||
|
||||
onApplicationBootstrap(): void {
|
||||
const interval = this.config.getOrThrow('OUTBOX_POLL_INTERVAL_MS');
|
||||
this.timer = setInterval(() => this.kick(), interval);
|
||||
this.timer.unref();
|
||||
this.kick();
|
||||
}
|
||||
|
||||
async onModuleDestroy(): Promise<void> {
|
||||
clearInterval(this.timer);
|
||||
this.timer = undefined;
|
||||
await this.current;
|
||||
}
|
||||
|
||||
kick(): void {
|
||||
if (this.running) {
|
||||
this.again = true;
|
||||
return;
|
||||
}
|
||||
this.running = true;
|
||||
this.current = this.drain()
|
||||
.catch((error: unknown) => {
|
||||
this.metrics.increment('outbox.publish_failed');
|
||||
if (!this.failing) {
|
||||
this.failing = true;
|
||||
this.logger.warn(
|
||||
`Outbox publish failed: ${error instanceof Error ? error.message : 'unknown error'}`,
|
||||
);
|
||||
}
|
||||
})
|
||||
.finally(() => {
|
||||
this.running = false;
|
||||
this.current = undefined;
|
||||
if (this.again) {
|
||||
this.again = false;
|
||||
this.kick();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private async drain(): Promise<void> {
|
||||
const published = await this.repository.publishPending(
|
||||
BATCH_SIZE,
|
||||
(message) => this.broker.publish(message),
|
||||
() => this.faults.point('relay.after_publish_before_mark'),
|
||||
);
|
||||
if (published > 0) {
|
||||
this.metrics.add('outbox.published', published);
|
||||
if (this.failing) {
|
||||
this.failing = false;
|
||||
this.logger.log('Outbox publishing recovered');
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -60,7 +60,8 @@ WITH claim AS (
|
||||
SELECT 1 FROM chat_members WHERE chat_id = $7::uuid AND user_id = $1
|
||||
)
|
||||
AND (SELECT count(*) FROM attachments a
|
||||
WHERE a.chat_id = $7::uuid AND a.status = 'ready'
|
||||
WHERE a.chat_id = $7::uuid
|
||||
AND a.status IN ('queued','processing','ready')
|
||||
AND a.id = ANY($8::uuid[])) = cardinality($8::uuid[])
|
||||
ON CONFLICT DO NOTHING
|
||||
RETURNING response_body
|
||||
|
||||
@@ -97,12 +97,15 @@ export class MessagesService {
|
||||
// чужую заявку сняли между нашими двумя запросами (503).
|
||||
if (!(await this.idempotency.isMember(chatId, currentUser.id)))
|
||||
throw new NotFoundException();
|
||||
// Unknown, foreign-chat or unfinished attachment: not found.
|
||||
if (
|
||||
ids.length > 0 &&
|
||||
!(await this.attachments.allLinkable(chatId, ids))
|
||||
)
|
||||
throw new NotFoundException();
|
||||
// Unknown or foreign-chat ids are 404; an attachment whose
|
||||
// processing failed is a conflict, not a missing object.
|
||||
const linkState =
|
||||
ids.length > 0
|
||||
? await this.attachments.linkState(chatId, ids)
|
||||
: 'ok';
|
||||
if (linkState === 'failed')
|
||||
throw new ConflictException('Attachment processing failed');
|
||||
if (linkState === 'missing') throw new NotFoundException();
|
||||
this.metrics.increment('idempotency.claim_lost');
|
||||
throw new ServiceUnavailableException(
|
||||
'Idempotency key could not be claimed',
|
||||
|
||||
@@ -48,9 +48,11 @@ export class ObjectStorage {
|
||||
private readonly bucket: string;
|
||||
private readonly internal: S3Client;
|
||||
private readonly presign: S3Client;
|
||||
private readonly requestTimeoutMs: number;
|
||||
|
||||
constructor(config: ConfigService<AppConfig>) {
|
||||
this.bucket = config.getOrThrow('S3_BUCKET');
|
||||
this.requestTimeoutMs = config.getOrThrow('S3_REQUEST_TIMEOUT_MS');
|
||||
const common = {
|
||||
region: config.getOrThrow('S3_REGION'),
|
||||
forcePathStyle: true,
|
||||
@@ -89,12 +91,25 @@ export class ObjectStorage {
|
||||
return `attachments/${attachmentId}/original`;
|
||||
}
|
||||
|
||||
stagingKey(attachmentId: string): string {
|
||||
return `staging/${attachmentId}/original`;
|
||||
}
|
||||
|
||||
variantKey(
|
||||
attachmentId: string,
|
||||
pipelineVersion: number,
|
||||
variant: 'thumbnail' | 'crop' | 'optimized',
|
||||
extension: 'jpg' | 'webp',
|
||||
): string {
|
||||
return `attachments/${attachmentId}/v${pipelineVersion}/${variant}.${extension}`;
|
||||
}
|
||||
|
||||
async putStream(
|
||||
key: string,
|
||||
body: Readable,
|
||||
contentLength: number,
|
||||
contentType: string,
|
||||
signal: AbortSignal,
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await this.internal.send(
|
||||
@@ -107,13 +122,31 @@ export class ObjectStorage {
|
||||
}),
|
||||
// The upload deadline (UPLOAD_TIMEOUT_MS, via `signal`) bounds the
|
||||
// PUT: it lasts as long as the client sends, not S3_REQUEST_TIMEOUT_MS.
|
||||
{ abortSignal: signal, requestTimeout: 0 },
|
||||
// A worker buffer has no client deadline, so it uses the SDK timeout.
|
||||
signal
|
||||
? { abortSignal: signal, requestTimeout: 0 }
|
||||
: { requestTimeout: this.requestTimeoutMs },
|
||||
);
|
||||
} catch (error) {
|
||||
throw this.fail('put', error);
|
||||
}
|
||||
}
|
||||
|
||||
async putBuffer(
|
||||
key: string,
|
||||
data: Buffer,
|
||||
contentType: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
await this.putStream(
|
||||
key,
|
||||
Readable.from([data]),
|
||||
data.length,
|
||||
contentType,
|
||||
signal,
|
||||
);
|
||||
}
|
||||
|
||||
async getStream(
|
||||
key: string,
|
||||
signal?: AbortSignal,
|
||||
@@ -130,9 +163,12 @@ export class ObjectStorage {
|
||||
}
|
||||
}
|
||||
|
||||
async head(
|
||||
key: string,
|
||||
): Promise<{ size: number; checksumSha256?: string } | null> {
|
||||
async head(key: string): Promise<{
|
||||
size: number;
|
||||
checksumSha256?: string;
|
||||
etag?: string;
|
||||
lastModified?: Date;
|
||||
} | null> {
|
||||
try {
|
||||
const result = await this.internal.send(
|
||||
new HeadObjectCommand({
|
||||
@@ -144,6 +180,8 @@ export class ObjectStorage {
|
||||
return {
|
||||
size: Number(result.ContentLength ?? 0),
|
||||
checksumSha256: result.ChecksumSHA256,
|
||||
etag: result.ETag,
|
||||
lastModified: result.LastModified,
|
||||
};
|
||||
} catch (error) {
|
||||
if (isMissingKey(error)) return null;
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { NestFactory } from '@nestjs/core';
|
||||
import http from 'node:http';
|
||||
import type { AppConfig } from './config/app-config.schema.js';
|
||||
import { BrokerClient } from './jobs/broker.client.js';
|
||||
import { JobProcessor } from './worker/job-processor.service.js';
|
||||
import { WorkerModule } from './worker/worker.module.js';
|
||||
|
||||
const DRAIN_MS = 30_000;
|
||||
|
||||
async function bootstrap(): Promise<void> {
|
||||
const app = await NestFactory.createApplicationContext(WorkerModule, {
|
||||
logger: ['error', 'warn', 'log'],
|
||||
});
|
||||
const config = app.get(ConfigService<AppConfig>);
|
||||
const broker = app.get(BrokerClient);
|
||||
const processor = app.get(JobProcessor);
|
||||
const logger = new Logger('worker');
|
||||
|
||||
const server = http.createServer((request, response) => {
|
||||
if (request.method === 'GET' && request.url === '/health') {
|
||||
const consuming = broker.isConsuming();
|
||||
response.writeHead(consuming ? 200 : 503, {
|
||||
'content-type': 'application/json',
|
||||
});
|
||||
response.end(
|
||||
JSON.stringify({
|
||||
status: consuming ? 'ok' : 'not_consuming',
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
response.writeHead(404, { 'content-type': 'application/json' });
|
||||
response.end(JSON.stringify({ status: 'not_found' }));
|
||||
});
|
||||
await new Promise<void>((resolve) =>
|
||||
server.listen(
|
||||
config.getOrThrow('WORKER_HEALTH_PORT'),
|
||||
'127.0.0.1',
|
||||
resolve,
|
||||
),
|
||||
);
|
||||
logger.log(
|
||||
`worker ready: id=${process.env.INSTANCE_ID ?? process.env.HOSTNAME ?? 'worker'}`,
|
||||
);
|
||||
|
||||
let shuttingDown = false;
|
||||
const shutdown = async (signal: string): Promise<void> => {
|
||||
if (shuttingDown) return;
|
||||
shuttingDown = true;
|
||||
logger.log(
|
||||
`${signal}: stopping consumption and draining in-flight jobs`,
|
||||
);
|
||||
await broker.stopConsuming();
|
||||
await processor.drain(DRAIN_MS);
|
||||
await new Promise<void>((resolve) => server.close(() => resolve()));
|
||||
await app.close();
|
||||
process.exit(0);
|
||||
};
|
||||
for (const signal of ['SIGTERM', 'SIGINT'] as const)
|
||||
process.once(signal, () => {
|
||||
void shutdown(signal);
|
||||
});
|
||||
}
|
||||
|
||||
await bootstrap();
|
||||
@@ -0,0 +1,405 @@
|
||||
import { Injectable, Logger, OnApplicationBootstrap } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import type { Channel, ConsumeMessage } from 'amqplib';
|
||||
import { createHash } from 'node:crypto';
|
||||
import os from 'node:os';
|
||||
import type { AttachmentRow } from '../attachments/attachment.js';
|
||||
import type { AppConfig } from '../config/app-config.schema.js';
|
||||
import { ObjectStorage } from '../storage/object-storage.js';
|
||||
import { retryDelayMs } from '../jobs/backoff.js';
|
||||
import { BrokerClient } from '../jobs/broker.client.js';
|
||||
import {
|
||||
classifyError,
|
||||
JobTimeoutError,
|
||||
SourceMismatchError,
|
||||
SourceMissingError,
|
||||
type ErrorClass,
|
||||
} from '../jobs/errors.js';
|
||||
import { FaultInjector } from '../jobs/fault-injector.js';
|
||||
import { parseJobMessage } from '../jobs/job-message.js';
|
||||
import {
|
||||
JobsRepository,
|
||||
type ClaimResult,
|
||||
type VariantWrite,
|
||||
} from '../jobs/jobs.repository.js';
|
||||
import { MetricsService } from '../metrics/metrics.service.js';
|
||||
import { renderVariants, type RenderedVariant } from './pipelines.js';
|
||||
|
||||
const SHUTDOWN_DRAIN_MS = 30_000;
|
||||
|
||||
function throwIfAborted(signal: AbortSignal): void {
|
||||
if (!signal.aborted) return;
|
||||
throw signal.reason instanceof Error ? signal.reason : new Error('Aborted');
|
||||
}
|
||||
|
||||
function sha256(data: Buffer): string {
|
||||
return createHash('sha256').update(data).digest('hex');
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class JobProcessor implements OnApplicationBootstrap {
|
||||
private readonly logger = new Logger(JobProcessor.name);
|
||||
private readonly workerId = process.env.INSTANCE_ID ?? os.hostname();
|
||||
private readonly maxPixels: number;
|
||||
private readonly timeoutSeconds: number;
|
||||
private readonly leaseSeconds: number;
|
||||
private readonly retryBaseMs: number;
|
||||
private readonly retryMaxMs: number;
|
||||
private readonly maxUploadBytes: number;
|
||||
private active = 0;
|
||||
private readonly idleWaiters: (() => void)[] = [];
|
||||
|
||||
constructor(
|
||||
private readonly config: ConfigService<AppConfig>,
|
||||
private readonly broker: BrokerClient,
|
||||
private readonly jobs: JobsRepository,
|
||||
private readonly faults: FaultInjector,
|
||||
private readonly storage: ObjectStorage,
|
||||
private readonly metrics: MetricsService,
|
||||
) {
|
||||
this.maxPixels = config.getOrThrow('MAX_IMAGE_PIXELS');
|
||||
this.timeoutSeconds = config.getOrThrow('JOB_TIMEOUT_SECONDS');
|
||||
this.leaseSeconds = config.getOrThrow('JOB_LEASE_SECONDS');
|
||||
this.retryBaseMs = config.getOrThrow('JOB_RETRY_BASE_MS');
|
||||
this.retryMaxMs = config.getOrThrow('JOB_RETRY_MAX_MS');
|
||||
this.maxUploadBytes = config.getOrThrow('MAX_UPLOAD_BYTES');
|
||||
}
|
||||
|
||||
onApplicationBootstrap(): void {
|
||||
this.broker.consume(
|
||||
this.config.getOrThrow('WORKER_CONCURRENCY'),
|
||||
(message, channel) => this.handle(message, channel),
|
||||
);
|
||||
}
|
||||
|
||||
async drain(timeoutMs = SHUTDOWN_DRAIN_MS): Promise<void> {
|
||||
if (this.active === 0) return;
|
||||
const { promise, resolve } = Promise.withResolvers<void>();
|
||||
this.idleWaiters.push(resolve);
|
||||
const timer = setTimeout(resolve, timeoutMs);
|
||||
await promise;
|
||||
clearTimeout(timer);
|
||||
}
|
||||
|
||||
private async handle(
|
||||
message: ConsumeMessage,
|
||||
channel: Channel,
|
||||
): Promise<void> {
|
||||
const parsed = parseJobMessage(message.content);
|
||||
if (!parsed) {
|
||||
this.metrics.increment('worker.messages.poison');
|
||||
this.safeNack(channel, message, false);
|
||||
return;
|
||||
}
|
||||
|
||||
let claim: ClaimResult;
|
||||
try {
|
||||
claim = await this.jobs.claim(
|
||||
parsed,
|
||||
this.workerId,
|
||||
this.leaseSeconds,
|
||||
);
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
`Claim failed: ${error instanceof Error ? error.message : 'unknown error'}`,
|
||||
);
|
||||
this.safeNack(channel, message, true);
|
||||
return;
|
||||
}
|
||||
if (claim.kind === 'skip') {
|
||||
if (claim.reason === 'mismatch') {
|
||||
this.safeNack(channel, message, false);
|
||||
return;
|
||||
}
|
||||
this.metrics.increment(`worker.jobs.skipped.${claim.reason}`);
|
||||
this.safeAck(channel, message);
|
||||
return;
|
||||
}
|
||||
|
||||
this.begin();
|
||||
try {
|
||||
await this.process(
|
||||
claim.job,
|
||||
claim.attachment,
|
||||
claim.leaseToken,
|
||||
message,
|
||||
channel,
|
||||
);
|
||||
} finally {
|
||||
this.end();
|
||||
}
|
||||
}
|
||||
|
||||
private async process(
|
||||
job: {
|
||||
job_id: string;
|
||||
attachment_id: string;
|
||||
pipeline_version: number;
|
||||
attempt: number;
|
||||
},
|
||||
attachment: AttachmentRow,
|
||||
leaseToken: string,
|
||||
message: ConsumeMessage,
|
||||
channel: Channel,
|
||||
): Promise<void> {
|
||||
this.metrics.increment('worker.jobs.claimed');
|
||||
const controller = new AbortController();
|
||||
const target = {
|
||||
attachmentId: job.attachment_id,
|
||||
pipelineVersion: job.pipeline_version,
|
||||
};
|
||||
const timeout = setTimeout(
|
||||
() => controller.abort(new JobTimeoutError(this.timeoutSeconds)),
|
||||
this.timeoutSeconds * 1000,
|
||||
);
|
||||
const heartbeat = setInterval(
|
||||
() => {
|
||||
this.jobs
|
||||
.extendLease(job.job_id, leaseToken, this.leaseSeconds)
|
||||
.then((extended) => {
|
||||
if (!extended && !controller.signal.aborted)
|
||||
controller.abort(new Error('Lease lost'));
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
this.logger.warn(
|
||||
`Lease heartbeat failed: ${error instanceof Error ? error.message : 'unknown error'}`,
|
||||
);
|
||||
});
|
||||
},
|
||||
Math.max(1000, (this.leaseSeconds * 1000) / 3),
|
||||
);
|
||||
let committed = false;
|
||||
|
||||
try {
|
||||
await this.faults.point(
|
||||
'worker.before_side_effect',
|
||||
target,
|
||||
controller.signal,
|
||||
);
|
||||
throwIfAborted(controller.signal);
|
||||
const source = await this.ensureFinalOriginal(
|
||||
attachment,
|
||||
controller.signal,
|
||||
);
|
||||
throwIfAborted(controller.signal);
|
||||
const variants = await renderVariants(
|
||||
source,
|
||||
job.pipeline_version,
|
||||
{
|
||||
maxPixels: this.maxPixels,
|
||||
timeoutSeconds: this.timeoutSeconds,
|
||||
},
|
||||
);
|
||||
throwIfAborted(controller.signal);
|
||||
for (const variant of variants) {
|
||||
await this.storage.putBuffer(
|
||||
this.variantKey(job, variant),
|
||||
variant.data,
|
||||
variant.contentType,
|
||||
controller.signal,
|
||||
);
|
||||
}
|
||||
await this.faults.point(
|
||||
'worker.after_side_effect',
|
||||
target,
|
||||
controller.signal,
|
||||
);
|
||||
throwIfAborted(controller.signal);
|
||||
|
||||
const result = await this.jobs.commitSuccess(
|
||||
job.job_id,
|
||||
leaseToken,
|
||||
variants.map((variant) => this.variantWrite(job, variant)),
|
||||
);
|
||||
committed = true;
|
||||
this.metrics.increment(`worker.jobs.${result}`);
|
||||
await this.faults.point(
|
||||
'worker.after_commit_before_ack',
|
||||
target,
|
||||
controller.signal,
|
||||
);
|
||||
this.safeAck(channel, message);
|
||||
if (result === 'succeeded') await this.cleanupStaging(attachment);
|
||||
} catch (error) {
|
||||
if (committed) {
|
||||
// The durable effect is already committed; a failure at the ack
|
||||
// fault point or a lost reply must not turn into a retry.
|
||||
this.safeAck(channel, message);
|
||||
return;
|
||||
}
|
||||
const failure = this.classify(error, controller.signal);
|
||||
const delay = retryDelayMs(
|
||||
job.attempt,
|
||||
this.retryBaseMs,
|
||||
this.retryMaxMs,
|
||||
);
|
||||
try {
|
||||
const outcome = await this.jobs.commitFailure(
|
||||
job.job_id,
|
||||
leaseToken,
|
||||
failure,
|
||||
delay,
|
||||
);
|
||||
this.metrics.increment(`worker.jobs.${outcome}`);
|
||||
this.safeAck(channel, message);
|
||||
} catch (commitError) {
|
||||
this.metrics.increment('worker.jobs.commit_failed');
|
||||
this.logger.warn(
|
||||
`Failure commit failed: ${commitError instanceof Error ? commitError.message : 'unknown error'}`,
|
||||
);
|
||||
setTimeout(() => this.safeNack(channel, message, true), 1000);
|
||||
}
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
clearInterval(heartbeat);
|
||||
}
|
||||
}
|
||||
|
||||
private classify(error: unknown, signal: AbortSignal): ErrorClass {
|
||||
if (signal.aborted && signal.reason instanceof JobTimeoutError)
|
||||
return classifyError(signal.reason);
|
||||
return classifyError(error);
|
||||
}
|
||||
|
||||
private async ensureFinalOriginal(
|
||||
attachment: AttachmentRow,
|
||||
signal: AbortSignal,
|
||||
): Promise<Buffer> {
|
||||
const expectedSize = Number(attachment.size_bytes);
|
||||
const head = await this.storage.head(attachment.object_key);
|
||||
if (head !== null && head.size === expectedSize) {
|
||||
const data = await this.readObject(attachment.object_key, signal);
|
||||
this.verifySource(attachment, data);
|
||||
return data;
|
||||
}
|
||||
|
||||
const stagingKey = attachment.staging_key;
|
||||
if (stagingKey === null) throw new SourceMissingError();
|
||||
const data = await this.readObject(stagingKey, signal);
|
||||
this.verifySource(attachment, data);
|
||||
await this.storage.putBuffer(
|
||||
attachment.object_key,
|
||||
data,
|
||||
attachment.content_type,
|
||||
signal,
|
||||
);
|
||||
this.metrics.increment('worker.original.published');
|
||||
return data;
|
||||
}
|
||||
|
||||
private async readObject(
|
||||
key: string,
|
||||
signal: AbortSignal,
|
||||
): Promise<Buffer> {
|
||||
const stream = await this.storage.getStream(key, signal);
|
||||
if (stream === null) throw new SourceMissingError();
|
||||
const chunks: Buffer[] = [];
|
||||
let total = 0;
|
||||
const destroy = (): void => {
|
||||
stream.destroy(
|
||||
signal.reason instanceof Error
|
||||
? signal.reason
|
||||
: new Error('Aborted'),
|
||||
);
|
||||
};
|
||||
signal.addEventListener('abort', destroy, { once: true });
|
||||
try {
|
||||
for await (const chunk of stream as AsyncIterable<Buffer>) {
|
||||
total += chunk.length;
|
||||
if (total > this.maxUploadBytes)
|
||||
throw new SourceMismatchError(
|
||||
'Source exceeds MAX_UPLOAD_BYTES',
|
||||
);
|
||||
chunks.push(chunk);
|
||||
}
|
||||
} catch (error) {
|
||||
if (signal.aborted)
|
||||
throw signal.reason instanceof Error ? signal.reason : error;
|
||||
throw error;
|
||||
} finally {
|
||||
signal.removeEventListener('abort', destroy);
|
||||
stream.destroy();
|
||||
}
|
||||
return Buffer.concat(chunks, total);
|
||||
}
|
||||
|
||||
private verifySource(attachment: AttachmentRow, data: Buffer): void {
|
||||
if (
|
||||
data.length !== Number(attachment.size_bytes) ||
|
||||
sha256(data) !== attachment.sha256
|
||||
)
|
||||
throw new SourceMismatchError();
|
||||
}
|
||||
|
||||
private variantKey(
|
||||
job: { attachment_id: string; pipeline_version: number },
|
||||
variant: RenderedVariant,
|
||||
): string {
|
||||
return this.storage.variantKey(
|
||||
job.attachment_id,
|
||||
job.pipeline_version,
|
||||
variant.name,
|
||||
variant.ext,
|
||||
);
|
||||
}
|
||||
|
||||
private variantWrite(
|
||||
job: { attachment_id: string; pipeline_version: number },
|
||||
variant: RenderedVariant,
|
||||
): VariantWrite {
|
||||
return {
|
||||
name: variant.name,
|
||||
objectKey: this.variantKey(job, variant),
|
||||
contentType: variant.contentType,
|
||||
sizeBytes: variant.data.length,
|
||||
width: variant.width,
|
||||
height: variant.height,
|
||||
sha256: variant.sha256,
|
||||
};
|
||||
}
|
||||
|
||||
private async cleanupStaging(attachment: AttachmentRow): Promise<void> {
|
||||
const key = attachment.staging_key;
|
||||
if (key === null) return;
|
||||
try {
|
||||
await this.storage.delete(key);
|
||||
await this.jobs.clearStaging(attachment.id, key);
|
||||
this.metrics.increment('worker.staging.deleted');
|
||||
} catch (error) {
|
||||
this.metrics.increment('worker.staging.error');
|
||||
this.logger.warn(
|
||||
`Staging cleanup for ${attachment.id} deferred: ${error instanceof Error ? error.message : 'unknown error'}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private safeAck(channel: Channel, message: ConsumeMessage): void {
|
||||
try {
|
||||
channel.ack(message);
|
||||
} catch {
|
||||
// A closed channel redelivers; the DB fence makes the duplicate a no-op.
|
||||
}
|
||||
}
|
||||
|
||||
private safeNack(
|
||||
channel: Channel,
|
||||
message: ConsumeMessage,
|
||||
requeue: boolean,
|
||||
): void {
|
||||
try {
|
||||
channel.nack(message, false, requeue);
|
||||
} catch {
|
||||
// The broker redelivers after the delivery limit or lease recovery.
|
||||
}
|
||||
}
|
||||
|
||||
private begin(): void {
|
||||
this.active += 1;
|
||||
}
|
||||
|
||||
private end(): void {
|
||||
this.active -= 1;
|
||||
if (this.active !== 0) return;
|
||||
for (const resolve of this.idleWaiters.splice(0)) resolve();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
import {
|
||||
Injectable,
|
||||
Logger,
|
||||
OnModuleDestroy,
|
||||
OnModuleInit,
|
||||
} from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import type { AppConfig } from '../config/app-config.schema.js';
|
||||
import { retryDelayMs } from '../jobs/backoff.js';
|
||||
import { JobsRepository } from '../jobs/jobs.repository.js';
|
||||
import { MetricsService } from '../metrics/metrics.service.js';
|
||||
|
||||
const REAP_BATCH = 20;
|
||||
|
||||
/**
|
||||
* The reaper is the backstop for a worker that was killed mid-job: the lease
|
||||
* expires and the job returns to `queued` without waiting for the broker's
|
||||
* delivery limit.
|
||||
*/
|
||||
@Injectable()
|
||||
export class LeaseReaper implements OnModuleInit, OnModuleDestroy {
|
||||
private readonly logger = new Logger(LeaseReaper.name);
|
||||
private timer: NodeJS.Timeout | undefined;
|
||||
private running = false;
|
||||
|
||||
constructor(
|
||||
private readonly config: ConfigService<AppConfig>,
|
||||
private readonly jobs: JobsRepository,
|
||||
private readonly metrics: MetricsService,
|
||||
) {}
|
||||
|
||||
onModuleInit(): void {
|
||||
const interval = this.config.getOrThrow('LEASE_REAPER_INTERVAL_MS');
|
||||
const base = this.config.getOrThrow('JOB_RETRY_BASE_MS');
|
||||
const cap = this.config.getOrThrow('JOB_RETRY_MAX_MS');
|
||||
this.timer = setInterval(() => {
|
||||
if (this.running) return;
|
||||
this.running = true;
|
||||
this.jobs
|
||||
.reapExpiredLeases(REAP_BATCH, (attempt) =>
|
||||
retryDelayMs(attempt, base, cap),
|
||||
)
|
||||
.then((count) => {
|
||||
if (count > 0)
|
||||
this.metrics.add('worker.leases.reaped', count);
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
this.metrics.increment('worker.leases.reap_error');
|
||||
this.logger.warn(
|
||||
`Lease reaper failed: ${error instanceof Error ? error.message : 'unknown error'}`,
|
||||
);
|
||||
})
|
||||
.finally(() => {
|
||||
this.running = false;
|
||||
});
|
||||
}, interval);
|
||||
this.timer.unref();
|
||||
}
|
||||
|
||||
onModuleDestroy(): void {
|
||||
clearInterval(this.timer);
|
||||
this.timer = undefined;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import sharp from 'sharp';
|
||||
import { renderVariants, type RenderedVariant } from './pipelines.js';
|
||||
|
||||
const sample = readFileSync(
|
||||
new URL('../../../tests/fixtures/sample.png', import.meta.url),
|
||||
);
|
||||
const limits = { maxPixels: 40_000_000, timeoutSeconds: 30 };
|
||||
|
||||
function byName(
|
||||
variants: RenderedVariant[],
|
||||
name: 'thumbnail' | 'crop' | 'optimized',
|
||||
): RenderedVariant {
|
||||
const variant = variants.find((item) => item.name === name);
|
||||
if (!variant) throw new Error(`missing ${name}`);
|
||||
return variant;
|
||||
}
|
||||
|
||||
describe('renderVariants', () => {
|
||||
it('renders the v1 contract geometry and formats', async () => {
|
||||
const variants = await renderVariants(sample, 1, limits);
|
||||
const thumbnail = byName(variants, 'thumbnail');
|
||||
const crop = byName(variants, 'crop');
|
||||
const optimized = byName(variants, 'optimized');
|
||||
|
||||
expect([thumbnail.width, thumbnail.height]).toEqual([256, 160]);
|
||||
expect(thumbnail.contentType).toBe('image/jpeg');
|
||||
expect((await sharp(thumbnail.data).metadata()).format).toBe('jpeg');
|
||||
|
||||
expect([crop.width, crop.height]).toEqual([128, 128]);
|
||||
expect((await sharp(crop.data).metadata()).format).toBe('jpeg');
|
||||
|
||||
expect([optimized.width, optimized.height]).toEqual([320, 200]);
|
||||
expect(optimized.contentType).toBe('image/webp');
|
||||
expect((await sharp(optimized.data).metadata()).format).toBe('webp');
|
||||
});
|
||||
|
||||
it('applies EXIF orientation before fitting', async () => {
|
||||
const oriented = await sharp({
|
||||
create: {
|
||||
width: 300,
|
||||
height: 200,
|
||||
channels: 3,
|
||||
background: { r: 20, g: 80, b: 140 },
|
||||
},
|
||||
})
|
||||
.jpeg()
|
||||
.withMetadata({ orientation: 6 })
|
||||
.toBuffer();
|
||||
const variants = await renderVariants(oriented, 1, limits);
|
||||
const thumbnail = byName(variants, 'thumbnail');
|
||||
expect([thumbnail.width, thumbnail.height]).toEqual([170, 256]);
|
||||
});
|
||||
|
||||
it('composites transparent pixels onto white', async () => {
|
||||
const transparent = await sharp({
|
||||
create: {
|
||||
width: 10,
|
||||
height: 10,
|
||||
channels: 4,
|
||||
background: { r: 0, g: 0, b: 0, alpha: 0 },
|
||||
},
|
||||
})
|
||||
.png()
|
||||
.toBuffer();
|
||||
const variants = await renderVariants(transparent, 1, limits);
|
||||
const thumbnail = byName(variants, 'thumbnail');
|
||||
const { data } = await sharp(thumbnail.data)
|
||||
.raw()
|
||||
.toBuffer({ resolveWithObject: true });
|
||||
expect([...data.subarray(0, 3)]).toEqual([255, 255, 255]);
|
||||
});
|
||||
|
||||
it('is deterministic for the same input and version', async () => {
|
||||
const first = await renderVariants(sample, 1, limits);
|
||||
const second = await renderVariants(sample, 1, limits);
|
||||
for (const name of ['thumbnail', 'crop', 'optimized'] as const)
|
||||
expect(byName(first, name).sha256).toBe(
|
||||
byName(second, name).sha256,
|
||||
);
|
||||
});
|
||||
|
||||
it('produces a different v2 thumbnail', async () => {
|
||||
const v1 = byName(await renderVariants(sample, 1, limits), 'thumbnail');
|
||||
const v2 = byName(await renderVariants(sample, 2, limits), 'thumbnail');
|
||||
expect(v1.sha256).not.toBe(v2.sha256);
|
||||
expect([v2.width, v2.height]).toEqual([256, 160]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,142 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import sharp from 'sharp';
|
||||
import { UnsupportedPipelineError } from '../jobs/errors.js';
|
||||
import { fitInside } from '../jobs/fit-inside.js';
|
||||
|
||||
// Same policy as the API decoder: one libvips worker, no operation cache.
|
||||
sharp.cache(false);
|
||||
sharp.concurrency(1);
|
||||
|
||||
type SharpImage = ReturnType<typeof sharp>;
|
||||
|
||||
export const SUPPORTED_PIPELINE_VERSIONS = [1, 2] as const;
|
||||
|
||||
export type RenderedVariant = {
|
||||
name: 'thumbnail' | 'crop' | 'optimized';
|
||||
ext: 'jpg' | 'webp';
|
||||
contentType: 'image/jpeg' | 'image/webp';
|
||||
data: Buffer;
|
||||
width: number;
|
||||
height: number;
|
||||
sha256: string;
|
||||
};
|
||||
|
||||
export type RenderLimits = {
|
||||
maxPixels: number;
|
||||
timeoutSeconds: number;
|
||||
};
|
||||
|
||||
function input(data: Buffer, limits: RenderLimits): SharpImage {
|
||||
return sharp(data, {
|
||||
failOn: 'warning',
|
||||
limitInputPixels: limits.maxPixels,
|
||||
sequentialRead: true,
|
||||
})
|
||||
.timeout({ seconds: limits.timeoutSeconds })
|
||||
.rotate();
|
||||
}
|
||||
|
||||
function jpegV1(image: SharpImage): SharpImage {
|
||||
return image.jpeg({ quality: 80 });
|
||||
}
|
||||
|
||||
function jpegV2(image: SharpImage): SharpImage {
|
||||
return image.jpeg({ quality: 90, progressive: true, mozjpeg: true });
|
||||
}
|
||||
|
||||
function rendered(
|
||||
name: RenderedVariant['name'],
|
||||
ext: RenderedVariant['ext'],
|
||||
contentType: RenderedVariant['contentType'],
|
||||
data: Buffer,
|
||||
width: number,
|
||||
height: number,
|
||||
): RenderedVariant {
|
||||
return {
|
||||
name,
|
||||
ext,
|
||||
contentType,
|
||||
data,
|
||||
width,
|
||||
height,
|
||||
sha256: createHash('sha256').update(data).digest('hex'),
|
||||
};
|
||||
}
|
||||
|
||||
async function output(
|
||||
image: SharpImage,
|
||||
name: RenderedVariant['name'],
|
||||
ext: RenderedVariant['ext'],
|
||||
contentType: RenderedVariant['contentType'],
|
||||
): Promise<RenderedVariant> {
|
||||
const { data, info } = await image.toBuffer({ resolveWithObject: true });
|
||||
return rendered(name, ext, contentType, data, info.width, info.height);
|
||||
}
|
||||
|
||||
/**
|
||||
* Renders the thumbnail, crop and optimized variants from one decoded source.
|
||||
* The input dimensions are EXIF-oriented: `.rotate()` applies the orientation
|
||||
* and `metadata().autoOrient` describes the result, so 300x200 orientation 6
|
||||
* becomes 200x300 before fitting.
|
||||
*/
|
||||
export async function renderVariants(
|
||||
data: Buffer,
|
||||
pipelineVersion: number,
|
||||
limits: RenderLimits,
|
||||
): Promise<RenderedVariant[]> {
|
||||
if (
|
||||
!(SUPPORTED_PIPELINE_VERSIONS as readonly number[]).includes(
|
||||
pipelineVersion,
|
||||
)
|
||||
)
|
||||
throw new UnsupportedPipelineError(pipelineVersion);
|
||||
|
||||
const metadata = await sharp(data, {
|
||||
failOn: 'warning',
|
||||
limitInputPixels: limits.maxPixels,
|
||||
sequentialRead: true,
|
||||
}).metadata();
|
||||
const width = metadata.autoOrient?.width ?? metadata.width;
|
||||
const height = metadata.autoOrient?.height ?? metadata.height;
|
||||
const [thumbnailWidth, thumbnailHeight] = fitInside(width, height, 256);
|
||||
const [optimizedWidth, optimizedHeight] = fitInside(width, height, 1280);
|
||||
const v2 = pipelineVersion >= 2;
|
||||
|
||||
const thumbnail = v2 ? jpegV2 : jpegV1;
|
||||
const thumbnailOutput = await output(
|
||||
thumbnail(
|
||||
input(data, limits)
|
||||
.flatten({ background: '#ffffff' })
|
||||
.resize(thumbnailWidth, thumbnailHeight, { fit: 'fill' }),
|
||||
),
|
||||
'thumbnail',
|
||||
'jpg',
|
||||
'image/jpeg',
|
||||
);
|
||||
|
||||
const cropOutput = await output(
|
||||
thumbnail(
|
||||
input(data, limits)
|
||||
.flatten({ background: '#ffffff' })
|
||||
.resize(128, 128, { fit: 'cover', position: 'centre' }),
|
||||
),
|
||||
'crop',
|
||||
'jpg',
|
||||
'image/jpeg',
|
||||
);
|
||||
|
||||
const optimizedOutput = await output(
|
||||
input(data, limits)
|
||||
.resize(optimizedWidth, optimizedHeight, { fit: 'fill' })
|
||||
.webp(
|
||||
v2
|
||||
? { quality: 82, effort: 5, smartSubsample: true }
|
||||
: { quality: 75, effort: 4 },
|
||||
),
|
||||
'optimized',
|
||||
'webp',
|
||||
'image/webp',
|
||||
);
|
||||
|
||||
return [thumbnailOutput, cropOutput, optimizedOutput];
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
import {
|
||||
Injectable,
|
||||
Logger,
|
||||
OnModuleDestroy,
|
||||
OnModuleInit,
|
||||
} from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import type { AppConfig } from '../config/app-config.schema.js';
|
||||
import { JobsRepository } from '../jobs/jobs.repository.js';
|
||||
import { MetricsService } from '../metrics/metrics.service.js';
|
||||
import { ObjectStorage } from '../storage/object-storage.js';
|
||||
|
||||
const CLEANUP_BATCH = 100;
|
||||
|
||||
/**
|
||||
* Removes staging objects after the worker committed the final copy, and after
|
||||
* the documented retention for failed jobs. A ready row waits one minute so a
|
||||
* just-acknowledged job is never raced by the cleaner.
|
||||
*/
|
||||
@Injectable()
|
||||
export class StagingCleaner implements OnModuleInit, OnModuleDestroy {
|
||||
private readonly logger = new Logger(StagingCleaner.name);
|
||||
private timer: NodeJS.Timeout | undefined;
|
||||
private running = false;
|
||||
|
||||
constructor(
|
||||
private readonly config: ConfigService<AppConfig>,
|
||||
private readonly jobs: JobsRepository,
|
||||
private readonly storage: ObjectStorage,
|
||||
private readonly metrics: MetricsService,
|
||||
) {}
|
||||
|
||||
onModuleInit(): void {
|
||||
const interval = this.config.getOrThrow('STAGING_CLEANUP_INTERVAL_MS');
|
||||
if (interval <= 0) return;
|
||||
this.timer = setInterval(() => {
|
||||
if (this.running) return;
|
||||
this.running = true;
|
||||
this.cleanupOnce()
|
||||
.catch((error: unknown) => {
|
||||
this.metrics.increment('worker.staging.cleanup_error');
|
||||
this.logger.warn(
|
||||
`Staging cleanup failed: ${error instanceof Error ? error.message : 'unknown error'}`,
|
||||
);
|
||||
})
|
||||
.finally(() => {
|
||||
this.running = false;
|
||||
});
|
||||
}, interval);
|
||||
this.timer.unref();
|
||||
}
|
||||
|
||||
onModuleDestroy(): void {
|
||||
clearInterval(this.timer);
|
||||
this.timer = undefined;
|
||||
}
|
||||
|
||||
async cleanupOnce(): Promise<number> {
|
||||
const candidates = await this.jobs.stagingCleanupCandidates(
|
||||
this.config.getOrThrow('STAGING_RETENTION_SECONDS'),
|
||||
CLEANUP_BATCH,
|
||||
);
|
||||
let deleted = 0;
|
||||
for (const candidate of candidates) {
|
||||
try {
|
||||
await this.storage.delete(candidate.staging_key);
|
||||
if (
|
||||
await this.jobs.clearStaging(
|
||||
candidate.id,
|
||||
candidate.staging_key,
|
||||
)
|
||||
)
|
||||
deleted += 1;
|
||||
} catch (error) {
|
||||
this.metrics.increment('worker.staging.cleanup_error');
|
||||
this.logger.warn(
|
||||
`Staging cleanup of ${candidate.id} failed: ${error instanceof Error ? error.message : 'unknown error'}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (deleted > 0) this.metrics.add('worker.staging.deleted', deleted);
|
||||
return deleted;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { AppConfigModule } from '../config/app-config.module.js';
|
||||
import { DatabaseModule } from '../database/database.module.js';
|
||||
import { JobsModule } from '../jobs/jobs.module.js';
|
||||
import { MetricsModule } from '../metrics/metrics.module.js';
|
||||
import { RedisModule } from '../redis/redis.module.js';
|
||||
import { StorageModule } from '../storage/storage.module.js';
|
||||
import { JobProcessor } from './job-processor.service.js';
|
||||
import { LeaseReaper } from './lease-reaper.service.js';
|
||||
import { StagingCleaner } from './staging-cleaner.service.js';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
AppConfigModule,
|
||||
DatabaseModule,
|
||||
RedisModule,
|
||||
MetricsModule,
|
||||
StorageModule,
|
||||
JobsModule,
|
||||
],
|
||||
providers: [JobProcessor, LeaseReaper, StagingCleaner],
|
||||
})
|
||||
export class WorkerModule {}
|
||||
+105
-2
@@ -1,4 +1,8 @@
|
||||
# Стенд lab5 (база lab4): клиент -> HAProxy -> API x N -> PostgreSQL + Redis + MinIO.
|
||||
# Стенд lab5 (база lab4): клиент -> HAProxy -> API x N -> PostgreSQL + Redis + MinIO
|
||||
# + RabbitMQ; отдельный worker читает задания из quorum-очереди и публикует
|
||||
# оригинал и варианты в конечный S3-prefix. API отвечает 202 после COMMIT
|
||||
# (staging-объект + attachment + job + outbox), поэтому брокер не входит в
|
||||
# /health/ready и его недоступность не останавливает приём загрузок.
|
||||
# Одна команда запуска: make stand-up (docker compose --env-file .env up -d --build --wait)
|
||||
#
|
||||
# Проект запинен именем, чтобы имена контейнеров и labels были предсказуемы для
|
||||
@@ -80,6 +84,31 @@ services:
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
|
||||
broker:
|
||||
image: rabbitmq:4.1.8-alpine@sha256:153322e3974e6365c50e523e5eadd5a7fae231f0e8c58f67c03bb3e02056e34e
|
||||
# Имя узла rabbit@broker должно оставаться стабильным: при смене
|
||||
# hostname данные в broker-data превращаются в сирот.
|
||||
hostname: broker
|
||||
secrets:
|
||||
- source: rabbitmq_conf
|
||||
target: /etc/rabbitmq/conf.d/20-credentials.conf
|
||||
volumes:
|
||||
- broker-data:/var/lib/rabbitmq
|
||||
# Портов нет: брокер доступен только сервисам compose-сети.
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '${BROKER_CPUS:-0.5}'
|
||||
memory: ${BROKER_MEMORY:-512m}
|
||||
healthcheck:
|
||||
test: ['CMD', 'rabbitmq-diagnostics', '-q', 'check_port_connectivity']
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
start_period: 20s
|
||||
stop_grace_period: 30s
|
||||
restart: unless-stopped
|
||||
|
||||
s3:
|
||||
image: minio/minio:RELEASE.2025-09-07T16-13-09Z@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
|
||||
command: ['server', '/data', '--address', ':9000']
|
||||
@@ -117,6 +146,8 @@ services:
|
||||
- minio_root_password
|
||||
- s3_api_access_key_id
|
||||
- s3_api_secret_access_key
|
||||
- s3_worker_access_key_id
|
||||
- s3_worker_secret_access_key
|
||||
depends_on:
|
||||
s3: { condition: service_healthy }
|
||||
restart: 'no'
|
||||
@@ -172,7 +203,16 @@ services:
|
||||
# libvips держит декодируемое изображение в памяти, а не во временном
|
||||
# файле: /tmp — tmpfs 16 MiB, и изображение ~40 Мп туда не помещается.
|
||||
VIPS_DISC_THRESHOLD: ${VIPS_DISC_THRESHOLD:-1g}
|
||||
secrets: [s3_api_access_key_id, s3_api_secret_access_key]
|
||||
FAULT_INJECTION_ENABLED: ${FAULT_INJECTION_ENABLED:-0}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_RETRY_BASE_MS: ${JOB_RETRY_BASE_MS:-1000}
|
||||
JOB_RETRY_MAX_MS: ${JOB_RETRY_MAX_MS:-30000}
|
||||
# Outbox закрывает недоступность брокера: API принимает загрузку и
|
||||
# отвечает 202 без публикации, relay доставит задание после восстановления.
|
||||
secrets:
|
||||
- s3_api_access_key_id
|
||||
- s3_api_secret_access_key
|
||||
- broker_url
|
||||
# Корневая ФС только для чтения: локального файла оригинала на API нет.
|
||||
read_only: true
|
||||
tmpfs: ['/tmp:size=16m']
|
||||
@@ -201,6 +241,64 @@ services:
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
|
||||
# Worker — отдельный процесс без HTTP API. Он читает задания из quorum-
|
||||
# очереди, держит lease с heartbeat, пишет варианты и коммитит результат в
|
||||
# PostgreSQL. Остановка worker не блокирует приём загрузок API.
|
||||
worker:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
args:
|
||||
BUILD_ID: ${BUILD_ID:-unknown}
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
command: ['node', 'dist/worker.main.js']
|
||||
env_file: [.env]
|
||||
environment:
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_worker_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_worker_secret_access_key
|
||||
VIPS_DISC_THRESHOLD: ${VIPS_DISC_THRESHOLD:-1g}
|
||||
FAULT_INJECTION_ENABLED: ${FAULT_INJECTION_ENABLED:-0}
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-30}
|
||||
JOB_RETRY_BASE_MS: ${JOB_RETRY_BASE_MS:-1000}
|
||||
JOB_RETRY_MAX_MS: ${JOB_RETRY_MAX_MS:-30000}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
WORKER_HEALTH_PORT: ${WORKER_HEALTH_PORT:-8081}
|
||||
secrets:
|
||||
- s3_worker_access_key_id
|
||||
- s3_worker_secret_access_key
|
||||
- broker_url
|
||||
read_only: true
|
||||
tmpfs: ['/tmp:size=16m']
|
||||
deploy:
|
||||
replicas: ${WORKER_REPLICAS:-1}
|
||||
resources:
|
||||
limits:
|
||||
cpus: '${WORKER_CPUS:-1.5}'
|
||||
memory: ${WORKER_MEMORY:-1280m}
|
||||
stop_grace_period: 40s
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
migrate: { condition: service_completed_successfully }
|
||||
postgres: { condition: service_healthy }
|
||||
redis: { condition: service_healthy }
|
||||
broker: { condition: service_healthy }
|
||||
s3-init: { condition: service_completed_successfully }
|
||||
healthcheck:
|
||||
test:
|
||||
[
|
||||
'CMD',
|
||||
'node',
|
||||
'-e',
|
||||
"fetch('http://127.0.0.1:'+ (process.env.WORKER_HEALTH_PORT||8081) + '/health').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))",
|
||||
]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
start_period: 20s
|
||||
|
||||
haproxy:
|
||||
image: haproxy:3.2-alpine
|
||||
ports:
|
||||
@@ -223,8 +321,13 @@ secrets:
|
||||
minio_root_password: { file: ./secrets/minio_root_password }
|
||||
s3_api_access_key_id: { file: ./secrets/s3_api_access_key_id }
|
||||
s3_api_secret_access_key: { file: ./secrets/s3_api_secret_access_key }
|
||||
s3_worker_access_key_id: { file: ./secrets/s3_worker_access_key_id }
|
||||
s3_worker_secret_access_key: { file: ./secrets/s3_worker_secret_access_key }
|
||||
broker_url: { file: ./secrets/broker_url }
|
||||
rabbitmq_conf: { file: ./secrets/rabbitmq.conf }
|
||||
|
||||
volumes:
|
||||
postgres-data:
|
||||
redis-data:
|
||||
s3-data:
|
||||
broker-data:
|
||||
|
||||
@@ -7,6 +7,8 @@ IFS= read -r root_user < /run/secrets/minio_root_user || [ -n "$root_user" ]
|
||||
IFS= read -r root_password < /run/secrets/minio_root_password || [ -n "$root_password" ]
|
||||
IFS= read -r api_key < /run/secrets/s3_api_access_key_id || [ -n "$api_key" ]
|
||||
IFS= read -r api_secret < /run/secrets/s3_api_secret_access_key || [ -n "$api_secret" ]
|
||||
IFS= read -r worker_key < /run/secrets/s3_worker_access_key_id || [ -n "$worker_key" ]
|
||||
IFS= read -r worker_secret < /run/secrets/s3_worker_secret_access_key || [ -n "$worker_secret" ]
|
||||
bucket="${S3_BUCKET:?}"
|
||||
|
||||
export MC_CONFIG_DIR=/tmp/.mc
|
||||
@@ -15,14 +17,42 @@ export MC_HOST_stand="http://${root_user}:${root_password}@s3:9000"
|
||||
mc mb --ignore-existing "stand/$bucket"
|
||||
mc anonymous set none "stand/$bucket"
|
||||
|
||||
printf '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject","s3:DeleteObject"],"Resource":["arn:aws:s3:::%s/attachments/*"]},{"Effect":"Allow","Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::%s"]}]}' "$bucket" "$bucket" > /tmp/messenger-api-policy.json
|
||||
mc admin policy create stand messenger-api /tmp/messenger-api-policy.json
|
||||
# The API owns the staging prefix only: it writes the accepted upload there and
|
||||
# can remove it after the worker commits the result. The final prefix is read
|
||||
# for variant/original downloads and legacy lab4 originals, but never written.
|
||||
printf '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject","s3:DeleteObject"],"Resource":["arn:aws:s3:::%s/staging/*"]},{"Effect":"Allow","Action":["s3:GetObject","s3:DeleteObject"],"Resource":["arn:aws:s3:::%s/attachments/*"]},{"Effect":"Allow","Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::%s"]}]}' "$bucket" "$bucket" "$bucket" > /tmp/messenger-api-policy.json
|
||||
|
||||
mc admin user add stand "$api_key" "$api_secret"
|
||||
info=$(mc admin user info stand "$api_key" --json)
|
||||
case "$info" in
|
||||
*'"messenger-api"'*) ;;
|
||||
*) mc admin policy attach stand messenger-api --user "$api_key" ;;
|
||||
esac
|
||||
# The worker reads staging, writes the final original and variants, and removes
|
||||
# staging after a safe commit. It never deletes the final prefix.
|
||||
printf '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject","s3:DeleteObject"],"Resource":["arn:aws:s3:::%s/staging/*"]},{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject"],"Resource":["arn:aws:s3:::%s/attachments/*"]},{"Effect":"Allow","Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::%s"]}]}' "$bucket" "$bucket" "$bucket" > /tmp/messenger-worker-policy.json
|
||||
|
||||
echo "bootstrap ok: bucket=$bucket policy=messenger-api"
|
||||
ensure_user() {
|
||||
name=$1
|
||||
secret=$2
|
||||
if ! mc admin user info stand "$name" >/dev/null 2>&1; then
|
||||
mc admin user add stand "$name" "$secret"
|
||||
fi
|
||||
}
|
||||
|
||||
# `mc admin policy create` does not overwrite and a policy attached to a user
|
||||
# cannot be removed. Detach, remove and recreate makes the script converge on
|
||||
# the current policy even when the stand volume predates lab5.
|
||||
install_policy() {
|
||||
name=$1
|
||||
file=$2
|
||||
user=$3
|
||||
if mc admin policy info stand "$name" >/dev/null 2>&1; then
|
||||
mc admin policy detach stand "$name" --user "$user" >/dev/null 2>&1 || true
|
||||
mc admin policy remove stand "$name" >/dev/null 2>&1 || true
|
||||
fi
|
||||
mc admin policy create stand "$name" "$file"
|
||||
mc admin policy attach stand "$name" --user "$user"
|
||||
}
|
||||
|
||||
ensure_user "$api_key" "$api_secret"
|
||||
ensure_user "$worker_key" "$worker_secret"
|
||||
|
||||
install_policy messenger-api /tmp/messenger-api-policy.json "$api_key"
|
||||
install_policy messenger-worker /tmp/messenger-worker-policy.json "$worker_key"
|
||||
|
||||
echo "bootstrap ok: bucket=$bucket policies=messenger-api,messenger-worker"
|
||||
|
||||
@@ -38,3 +38,28 @@ write_secret minio_root_user "root-$(openssl rand -hex 6)"
|
||||
write_secret minio_root_password "$(openssl rand -hex 24)"
|
||||
write_secret s3_api_access_key_id "api-$(openssl rand -hex 8)"
|
||||
write_secret s3_api_secret_access_key "$(openssl rand -hex 24)"
|
||||
write_secret s3_worker_access_key_id "worker-$(openssl rand -hex 8)"
|
||||
write_secret s3_worker_secret_access_key "$(openssl rand -hex 24)"
|
||||
|
||||
# Broker password lives in one place and is rendered into both files, so the
|
||||
# URL used by the app and the RabbitMQ config cannot drift apart.
|
||||
if [ -s secrets/broker_url ] && [ -s secrets/rabbitmq.conf ]; then
|
||||
echo "kept secrets/broker_url secrets/rabbitmq.conf"
|
||||
else
|
||||
bp=$(openssl rand -hex 24)
|
||||
(umask 133; printf '%s' "amqp://messenger:${bp}@broker:5672/messenger" > secrets/broker_url)
|
||||
chmod 0644 secrets/broker_url
|
||||
(
|
||||
umask 133
|
||||
{
|
||||
printf 'default_vhost = messenger\n'
|
||||
printf 'default_user = messenger\n'
|
||||
printf 'default_pass = %s\n' "$bp"
|
||||
printf 'default_permissions.configure = .*\n'
|
||||
printf 'default_permissions.read = .*\n'
|
||||
printf 'default_permissions.write = .*\n'
|
||||
} > secrets/rabbitmq.conf
|
||||
)
|
||||
chmod 0644 secrets/rabbitmq.conf
|
||||
echo "created secrets/broker_url secrets/rabbitmq.conf"
|
||||
fi
|
||||
|
||||
+2
-2
@@ -4,5 +4,5 @@
|
||||
set -euo pipefail
|
||||
|
||||
compose=(docker compose --env-file .env)
|
||||
"${compose[@]}" restart postgres redis s3 api haproxy
|
||||
"${compose[@]}" up -d --no-recreate --no-deps --wait postgres redis s3 api haproxy
|
||||
"${compose[@]}" restart postgres redis s3 broker api worker haproxy
|
||||
"${compose[@]}" up -d --no-recreate --no-deps --wait postgres redis s3 broker api worker haproxy
|
||||
|
||||
Reference in New Issue
Block a user