forked from mtusi2026/mtusi2026lab6
chore: port lab5 solution as lab6 base
This commit is contained in:
@@ -0,0 +1,25 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { AttachmentsModule } from './attachments/attachments.module';
|
||||
import { AuthModule } from './auth/auth.module';
|
||||
import { CacheModule } from './cache/cache.module';
|
||||
import { ChatsModule } from './chats/chats.module';
|
||||
import { DbModule } from './db/db.module';
|
||||
import { HealthModule } from './health/health.module';
|
||||
import { MessagesModule } from './messages/messages.module';
|
||||
import { StorageModule } from './storage/storage.module';
|
||||
import { UsersModule } from './users/users.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
DbModule,
|
||||
CacheModule,
|
||||
StorageModule,
|
||||
AuthModule,
|
||||
HealthModule,
|
||||
UsersModule,
|
||||
ChatsModule,
|
||||
MessagesModule,
|
||||
AttachmentsModule,
|
||||
],
|
||||
})
|
||||
export class AppModule {}
|
||||
@@ -0,0 +1,79 @@
|
||||
import {
|
||||
Controller,
|
||||
Get,
|
||||
HttpCode,
|
||||
Param,
|
||||
Post,
|
||||
Query,
|
||||
Req,
|
||||
Res,
|
||||
UseGuards,
|
||||
} from '@nestjs/common';
|
||||
import { FastifyReply, FastifyRequest } from 'fastify';
|
||||
import { Readable } from 'stream';
|
||||
import { AuthGuard } from '../auth/auth.guard';
|
||||
import { CurrentUser } from '../auth/current-user.decorator';
|
||||
import { User } from '../common/domain';
|
||||
import { AttachmentsService } from './attachments.service';
|
||||
import {
|
||||
AttachmentResponse,
|
||||
toAttachmentResponse,
|
||||
} from './dto/attachment.response';
|
||||
|
||||
@Controller()
|
||||
@UseGuards(AuthGuard)
|
||||
export class AttachmentsController {
|
||||
constructor(private readonly attachments: AttachmentsService) {}
|
||||
|
||||
@Post('api/v1/chats/:chatId/attachments')
|
||||
@HttpCode(202)
|
||||
async upload(
|
||||
@Param('chatId') chatId: string,
|
||||
@Query('filename') filename: unknown,
|
||||
@Req() request: FastifyRequest,
|
||||
@CurrentUser() user: User,
|
||||
): Promise<AttachmentResponse> {
|
||||
const attachment = await this.attachments.upload({
|
||||
chatId,
|
||||
ownerId: user.id,
|
||||
filename,
|
||||
contentType: request.uploadContentType,
|
||||
contentLength: request.headers['content-length'],
|
||||
body: request.body as Readable | undefined,
|
||||
});
|
||||
return toAttachmentResponse(attachment);
|
||||
}
|
||||
|
||||
@Get('api/v1/attachments/:attachmentId')
|
||||
async metadata(
|
||||
@Param('attachmentId') attachmentId: string,
|
||||
@CurrentUser() user: User,
|
||||
): Promise<AttachmentResponse> {
|
||||
return toAttachmentResponse(
|
||||
await this.attachments.getVisible(attachmentId, user.id),
|
||||
);
|
||||
}
|
||||
|
||||
@Get('api/v1/attachments/:attachmentId/content')
|
||||
async content(
|
||||
@Param('attachmentId') attachmentId: string,
|
||||
@Query('variant') variant: unknown,
|
||||
@CurrentUser() user: User,
|
||||
@Res() reply: FastifyReply,
|
||||
): Promise<void> {
|
||||
const download = await this.attachments.openContent(
|
||||
attachmentId,
|
||||
user.id,
|
||||
variant,
|
||||
);
|
||||
await reply
|
||||
.header('content-type', download.contentType)
|
||||
.header('content-length', download.sizeBytes)
|
||||
.header('x-content-type-options', 'nosniff')
|
||||
.header(
|
||||
'content-disposition',
|
||||
`inline; filename*=UTF-8''${encodeURIComponent(download.filename)}`,
|
||||
)
|
||||
.send(download.content);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { AttachmentsController } from './attachments.controller';
|
||||
import { AttachmentsService } from './attachments.service';
|
||||
|
||||
@Module({
|
||||
controllers: [AttachmentsController],
|
||||
providers: [AttachmentsService],
|
||||
})
|
||||
export class AttachmentsModule {}
|
||||
@@ -0,0 +1,244 @@
|
||||
import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
|
||||
import { Attachment, AttachmentVariant, Prisma } from '@prisma/client';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { mkdtemp, readdir, rm, stat } from 'fs/promises';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { Readable } from 'stream';
|
||||
import { assertChatMember } from '../chats/chat-access';
|
||||
import { codePointLength } from '../common/dto/code-point-length.validator';
|
||||
import { ApiError } from '../common/filters/api-error';
|
||||
import { validate as isUuid } from '../common/uuid';
|
||||
import { isDatabaseUnavailable, PrismaService } from '../db/prisma.service';
|
||||
import { PIPELINE_VERSION } from '../jobs/job-message';
|
||||
import { enqueueJob } from '../jobs/job-store';
|
||||
import { S3Service } from '../storage/s3.service';
|
||||
import { assertValidImage, isSupportedContentType } from './image-validation';
|
||||
import { receiveBody } from './upload-body';
|
||||
|
||||
const TEMP_PREFIX = 'messenger-upload-';
|
||||
const STALE_TEMP_MS = 60 * 60 * 1000;
|
||||
const MAX_FILENAME_CODE_POINTS = 255;
|
||||
const FORBIDDEN_FILENAME = /[/\\\p{Cc}\p{Zl}\p{Zp}]/u;
|
||||
const KNOWN_VARIANTS = new Set(['original', 'thumbnail', 'crop', 'optimized']);
|
||||
|
||||
export type AttachmentWithVariants = Attachment & { variants: AttachmentVariant[] };
|
||||
|
||||
export interface DownloadableContent {
|
||||
filename: string;
|
||||
contentType: string;
|
||||
sizeBytes: number;
|
||||
content: Readable;
|
||||
}
|
||||
|
||||
export interface UploadInput {
|
||||
chatId: string;
|
||||
ownerId: string;
|
||||
filename: unknown;
|
||||
contentType: string | undefined;
|
||||
contentLength: string | undefined;
|
||||
body: Readable | undefined;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class AttachmentsService implements OnModuleInit {
|
||||
private readonly logger = new Logger('AttachmentsService');
|
||||
private readonly maxBytes = Number(process.env.MAX_UPLOAD_BYTES ?? 10485760);
|
||||
private readonly maxPixels = Number(process.env.MAX_IMAGE_PIXELS ?? 40000000);
|
||||
private readonly maxConcurrent = Number(process.env.MAX_CONCURRENT_UPLOADS ?? 16);
|
||||
private readonly bodyTimeoutMs = Number(process.env.UPLOAD_BODY_TIMEOUT_MS ?? 60000);
|
||||
private readonly tempRoot = process.env.UPLOAD_TMP_DIR ?? tmpdir();
|
||||
private activeUploads = 0;
|
||||
|
||||
constructor(
|
||||
private readonly prisma: PrismaService,
|
||||
private readonly s3: S3Service,
|
||||
) {}
|
||||
|
||||
async onModuleInit(): Promise<void> {
|
||||
await this.removeStaleTempDirs();
|
||||
}
|
||||
|
||||
async upload(input: UploadInput): Promise<AttachmentWithVariants> {
|
||||
await assertChatMember(this.prisma, input.chatId, input.ownerId);
|
||||
const filename = parseFilename(input.filename);
|
||||
const contentType = parseContentType(input.contentType);
|
||||
if (Number(input.contentLength) > this.maxBytes) {
|
||||
throw new ApiError('payload_too_large', 'attachment exceeds size limit');
|
||||
}
|
||||
if (this.activeUploads >= this.maxConcurrent) {
|
||||
throw new ApiError('unavailable', 'too many concurrent uploads');
|
||||
}
|
||||
this.activeUploads++;
|
||||
const dir = await mkdtemp(join(this.tempRoot, TEMP_PREFIX));
|
||||
try {
|
||||
return await this.store(input, filename, contentType, join(dir, 'body'));
|
||||
} finally {
|
||||
this.activeUploads--;
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async getVisible(
|
||||
attachmentId: string,
|
||||
userId: string,
|
||||
): Promise<AttachmentWithVariants> {
|
||||
if (!isUuid(attachmentId)) {
|
||||
throw new ApiError('invalid_request', 'attachmentId must be a valid UUID');
|
||||
}
|
||||
const row = await this.prisma.attachment.findFirst({
|
||||
where: {
|
||||
id: attachmentId,
|
||||
chat: { members: { some: { userId } } },
|
||||
},
|
||||
include: { variants: { where: { pipelineVersion: PIPELINE_VERSION } } },
|
||||
});
|
||||
if (!row) {
|
||||
throw new ApiError('not_found', 'attachment not found');
|
||||
}
|
||||
return row;
|
||||
}
|
||||
|
||||
async openContent(
|
||||
attachmentId: string,
|
||||
userId: string,
|
||||
variant: unknown,
|
||||
): Promise<DownloadableContent> {
|
||||
const attachment = await this.getVisible(attachmentId, userId);
|
||||
const name = parseVariant(variant);
|
||||
if (attachment.status !== 'ready') {
|
||||
throw new ApiError('conflict', `attachment is ${attachment.status}`);
|
||||
}
|
||||
if (name === 'original') {
|
||||
return {
|
||||
filename: attachment.filename,
|
||||
contentType: attachment.contentType,
|
||||
sizeBytes: attachment.sizeBytes,
|
||||
content: await this.s3.getStream(attachment.objectKey),
|
||||
};
|
||||
}
|
||||
const stored = attachment.variants.find((v) => v.name === name);
|
||||
if (!stored) {
|
||||
throw new ApiError('not_found', 'attachment variant not found');
|
||||
}
|
||||
return {
|
||||
filename: attachment.filename,
|
||||
contentType: stored.contentType,
|
||||
sizeBytes: stored.sizeBytes,
|
||||
content: await this.s3.getStream(stored.objectKey),
|
||||
};
|
||||
}
|
||||
|
||||
private async store(
|
||||
input: UploadInput,
|
||||
filename: string,
|
||||
contentType: string,
|
||||
path: string,
|
||||
): Promise<AttachmentWithVariants> {
|
||||
if (!input.body) {
|
||||
throw new ApiError('invalid_request', 'request body must not be empty');
|
||||
}
|
||||
const { size, sha256 } = await receiveBody(
|
||||
input.body,
|
||||
path,
|
||||
this.maxBytes,
|
||||
this.bodyTimeoutMs,
|
||||
);
|
||||
if (size === 0) {
|
||||
throw new ApiError('invalid_request', 'request body must not be empty');
|
||||
}
|
||||
await assertValidImage(path, size, contentType, this.maxPixels);
|
||||
const id = randomUUID();
|
||||
const stagingKey = `staging/${id}`;
|
||||
await this.s3.putFile({ key: stagingKey, path, size, contentType, sha256 });
|
||||
const now = new Date();
|
||||
try {
|
||||
return await this.prisma.$transaction(async (tx) => {
|
||||
const attachment = await tx.attachment.create({
|
||||
data: {
|
||||
id,
|
||||
chatId: input.chatId,
|
||||
ownerId: input.ownerId,
|
||||
filename,
|
||||
contentType,
|
||||
sizeBytes: size,
|
||||
sha256,
|
||||
objectKey: `attachments/${id}/original`,
|
||||
stagingKey,
|
||||
status: 'queued',
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
},
|
||||
});
|
||||
await enqueueJob(tx, id, now);
|
||||
return { ...attachment, variants: [] };
|
||||
});
|
||||
} catch (error) {
|
||||
if (isDefiniteRejection(error)) {
|
||||
await this.s3.delete(stagingKey).catch(() => {
|
||||
this.logger.warn(`compensation failed for ${stagingKey}`);
|
||||
});
|
||||
}
|
||||
if (error instanceof Prisma.PrismaClientUnknownRequestError) {
|
||||
this.logger.error(`metadata insert failed for ${stagingKey}`);
|
||||
throw new ApiError('unavailable', 'database unavailable');
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async removeStaleTempDirs(): Promise<void> {
|
||||
const root = this.tempRoot;
|
||||
const names = await readdir(root);
|
||||
for (const name of names.filter((n) => n.startsWith(TEMP_PREFIX))) {
|
||||
const path = join(root, name);
|
||||
const info = await stat(path).catch(() => null);
|
||||
if (info && Date.now() - info.mtimeMs > STALE_TEMP_MS) {
|
||||
await rm(path, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function parseVariant(raw: unknown): string {
|
||||
if (raw === undefined) {
|
||||
return 'original';
|
||||
}
|
||||
if (typeof raw === 'string' && KNOWN_VARIANTS.has(raw)) {
|
||||
return raw;
|
||||
}
|
||||
throw new ApiError('invalid_request', 'unknown attachment variant');
|
||||
}
|
||||
|
||||
function isDefiniteRejection(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof Prisma.PrismaClientKnownRequestError &&
|
||||
!isDatabaseUnavailable(error)
|
||||
);
|
||||
}
|
||||
|
||||
function parseFilename(raw: unknown): string {
|
||||
if (
|
||||
typeof raw !== 'string' ||
|
||||
raw.trim() === '' ||
|
||||
codePointLength(raw) > MAX_FILENAME_CODE_POINTS ||
|
||||
FORBIDDEN_FILENAME.test(raw)
|
||||
) {
|
||||
throw new ApiError(
|
||||
'invalid_request',
|
||||
'filename must be 1-255 characters without path separators or control characters',
|
||||
);
|
||||
}
|
||||
return raw;
|
||||
}
|
||||
|
||||
function parseContentType(raw: string | undefined): string {
|
||||
const contentType = raw?.split(';')[0].trim().toLowerCase() ?? '';
|
||||
if (!isSupportedContentType(contentType)) {
|
||||
throw new ApiError(
|
||||
'unsupported_media_type',
|
||||
'Content-Type must be image/png or image/jpeg',
|
||||
);
|
||||
}
|
||||
return contentType;
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
import { Attachment, AttachmentVariant } from '@prisma/client';
|
||||
|
||||
const VARIANT_ORDER = ['thumbnail', 'crop', 'optimized'];
|
||||
|
||||
export interface VariantResponse {
|
||||
name: string;
|
||||
content_type: string;
|
||||
size_bytes: number;
|
||||
width: number;
|
||||
height: number;
|
||||
sha256: string;
|
||||
}
|
||||
|
||||
export interface AttachmentResponse {
|
||||
id: string;
|
||||
chat_id: string;
|
||||
owner_id: string;
|
||||
filename: string;
|
||||
content_type: string;
|
||||
size_bytes: number;
|
||||
sha256: string;
|
||||
status: string;
|
||||
variants: VariantResponse[];
|
||||
error_code: string | null;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
}
|
||||
|
||||
export function toAttachmentResponse(
|
||||
row: Attachment & { variants: AttachmentVariant[] },
|
||||
): AttachmentResponse {
|
||||
const variants = row.status === 'ready' ? [...row.variants] : [];
|
||||
variants.sort((a, b) => VARIANT_ORDER.indexOf(a.name) - VARIANT_ORDER.indexOf(b.name));
|
||||
return {
|
||||
id: row.id,
|
||||
chat_id: row.chatId,
|
||||
owner_id: row.ownerId,
|
||||
filename: row.filename,
|
||||
content_type: row.contentType,
|
||||
size_bytes: row.sizeBytes,
|
||||
sha256: row.sha256,
|
||||
status: row.status,
|
||||
variants: variants.map((variant) => ({
|
||||
name: variant.name,
|
||||
content_type: variant.contentType,
|
||||
size_bytes: variant.sizeBytes,
|
||||
width: variant.width,
|
||||
height: variant.height,
|
||||
sha256: variant.sha256,
|
||||
})),
|
||||
error_code: row.errorCode,
|
||||
created_at: row.createdAt.toISOString(),
|
||||
updated_at: row.updatedAt.toISOString(),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
import { open } from 'fs/promises';
|
||||
import sharp from 'sharp';
|
||||
import { ApiError } from '../common/filters/api-error';
|
||||
|
||||
sharp.cache(false);
|
||||
sharp.concurrency(1);
|
||||
|
||||
const PNG_SIGNATURE = Buffer.from('89504e470d0a1a0a', 'hex');
|
||||
const JPEG_SIGNATURE = Buffer.from('ffd8ff', 'hex');
|
||||
const PNG_END_CHUNK = Buffer.from('0000000049454e44ae426082', 'hex');
|
||||
const PNG_TAIL_BYTES = 64 * 1024;
|
||||
const MAX_CONCURRENT_DECODES = 2;
|
||||
|
||||
const FORMAT_BY_CONTENT_TYPE: Record<string, string> = {
|
||||
'image/png': 'png',
|
||||
'image/jpeg': 'jpeg',
|
||||
};
|
||||
|
||||
let activeDecodes = 0;
|
||||
const decodeQueue: (() => void)[] = [];
|
||||
|
||||
export function isSupportedContentType(contentType: string): boolean {
|
||||
return contentType in FORMAT_BY_CONTENT_TYPE;
|
||||
}
|
||||
|
||||
export async function assertValidImage(
|
||||
path: string,
|
||||
size: number,
|
||||
contentType: string,
|
||||
maxPixels: number,
|
||||
): Promise<void> {
|
||||
const invalid = new ApiError(
|
||||
'unsupported_media_type',
|
||||
'file is not a valid image of the declared type',
|
||||
);
|
||||
const format = FORMAT_BY_CONTENT_TYPE[contentType];
|
||||
if (!(await hasExpectedFraming(path, size, format))) {
|
||||
throw invalid;
|
||||
}
|
||||
const { width, height, format: actual } = await sharp(path, {
|
||||
limitInputPixels: false,
|
||||
})
|
||||
.metadata()
|
||||
.catch(() => {
|
||||
throw invalid;
|
||||
});
|
||||
if (!width || !height || actual !== format) {
|
||||
throw invalid;
|
||||
}
|
||||
if (width * height > maxPixels) {
|
||||
throw new ApiError('payload_too_large', 'image exceeds pixel limit');
|
||||
}
|
||||
await acquireDecodeSlot();
|
||||
try {
|
||||
await sharp(path, {
|
||||
failOn: 'warning',
|
||||
sequentialRead: true,
|
||||
limitInputPixels: maxPixels,
|
||||
}).stats();
|
||||
} catch {
|
||||
throw invalid;
|
||||
} finally {
|
||||
releaseDecodeSlot();
|
||||
}
|
||||
}
|
||||
|
||||
async function hasExpectedFraming(
|
||||
path: string,
|
||||
size: number,
|
||||
format: string,
|
||||
): Promise<boolean> {
|
||||
const file = await open(path, 'r');
|
||||
try {
|
||||
const head = Buffer.alloc(PNG_SIGNATURE.length);
|
||||
await file.read(head, 0, head.length, 0);
|
||||
if (format === 'jpeg') {
|
||||
return head.subarray(0, JPEG_SIGNATURE.length).equals(JPEG_SIGNATURE);
|
||||
}
|
||||
if (!head.equals(PNG_SIGNATURE)) {
|
||||
return false;
|
||||
}
|
||||
const tail = Buffer.alloc(Math.min(size, PNG_TAIL_BYTES));
|
||||
await file.read(tail, 0, tail.length, size - tail.length);
|
||||
return tail.includes(PNG_END_CHUNK);
|
||||
} finally {
|
||||
await file.close();
|
||||
}
|
||||
}
|
||||
|
||||
function acquireDecodeSlot(): Promise<void> {
|
||||
if (activeDecodes < MAX_CONCURRENT_DECODES) {
|
||||
activeDecodes++;
|
||||
return Promise.resolve();
|
||||
}
|
||||
return new Promise((resolve) => decodeQueue.push(resolve));
|
||||
}
|
||||
|
||||
function releaseDecodeSlot(): void {
|
||||
const next = decodeQueue.shift();
|
||||
if (next) {
|
||||
next();
|
||||
} else {
|
||||
activeDecodes--;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
import { PrismaClient } from '@prisma/client';
|
||||
import { S3Service } from '../storage/s3.service';
|
||||
|
||||
const MAX_REPORTED_KEYS = 20;
|
||||
const DEFAULT_PREFIXES = ['attachments/', 'staging/'];
|
||||
|
||||
type ReconcilePrisma = Pick<PrismaClient, 'attachment' | 'attachmentVariant'>;
|
||||
|
||||
export interface ReconcileOptions {
|
||||
apply: boolean;
|
||||
graceMs: number;
|
||||
stagingRetentionMs?: number;
|
||||
prefix?: string;
|
||||
now?: Date;
|
||||
pageSize?: number;
|
||||
}
|
||||
|
||||
export interface ReconcileReport {
|
||||
mode: 'apply' | 'dry-run';
|
||||
grace_seconds: number;
|
||||
staging_expired: number;
|
||||
scanned: number;
|
||||
fresh_skipped: number;
|
||||
referenced: number;
|
||||
orphans: number;
|
||||
deleted: number;
|
||||
failed: number;
|
||||
orphan_keys: string[];
|
||||
duration_ms: number;
|
||||
}
|
||||
|
||||
export async function reconcile(
|
||||
prisma: ReconcilePrisma,
|
||||
s3: S3Service,
|
||||
options: ReconcileOptions,
|
||||
): Promise<ReconcileReport> {
|
||||
const started = Date.now();
|
||||
const cutoff = (options.now ?? new Date()).getTime() - options.graceMs;
|
||||
const report: ReconcileReport = {
|
||||
mode: options.apply ? 'apply' : 'dry-run',
|
||||
grace_seconds: options.graceMs / 1000,
|
||||
staging_expired: 0,
|
||||
scanned: 0,
|
||||
fresh_skipped: 0,
|
||||
referenced: 0,
|
||||
orphans: 0,
|
||||
deleted: 0,
|
||||
failed: 0,
|
||||
orphan_keys: [],
|
||||
duration_ms: 0,
|
||||
};
|
||||
if (options.stagingRetentionMs !== undefined) {
|
||||
report.staging_expired = await expireFailedStaging(
|
||||
prisma,
|
||||
s3,
|
||||
(options.now ?? new Date()).getTime() - options.stagingRetentionMs,
|
||||
options.apply,
|
||||
options.prefix,
|
||||
);
|
||||
}
|
||||
for (const prefix of options.prefix ? [options.prefix] : DEFAULT_PREFIXES) {
|
||||
await reconcilePrefix(prisma, s3, prefix, cutoff, options, report);
|
||||
}
|
||||
report.duration_ms = Date.now() - started;
|
||||
return report;
|
||||
}
|
||||
|
||||
async function expireFailedStaging(
|
||||
prisma: ReconcilePrisma,
|
||||
s3: S3Service,
|
||||
before: number,
|
||||
apply: boolean,
|
||||
prefix?: string,
|
||||
): Promise<number> {
|
||||
const rows = await prisma.attachment.findMany({
|
||||
where: {
|
||||
stagingKey: { not: null, startsWith: prefix },
|
||||
OR: [{ status: 'ready' }, { status: 'failed', updatedAt: { lt: new Date(before) } }],
|
||||
},
|
||||
select: { id: true, stagingKey: true },
|
||||
take: 500,
|
||||
});
|
||||
if (!apply) {
|
||||
return rows.length;
|
||||
}
|
||||
let expired = 0;
|
||||
for (const row of rows) {
|
||||
const released = await prisma.attachment.updateMany({
|
||||
where: {
|
||||
id: row.id,
|
||||
stagingKey: row.stagingKey,
|
||||
OR: [{ status: 'ready' }, { status: 'failed', updatedAt: { lt: new Date(before) } }],
|
||||
},
|
||||
data: { stagingKey: null },
|
||||
});
|
||||
if (released.count === 0) {
|
||||
continue;
|
||||
}
|
||||
await s3.delete(row.stagingKey!).then(
|
||||
() => expired++,
|
||||
() => undefined,
|
||||
);
|
||||
}
|
||||
return expired;
|
||||
}
|
||||
|
||||
function attachmentIdOf(key: string): string | null {
|
||||
const match = /^attachments\/([0-9a-f-]{36})(\/|$)/.exec(key);
|
||||
return match ? match[1] : null;
|
||||
}
|
||||
|
||||
async function reconcilePrefix(
|
||||
prisma: ReconcilePrisma,
|
||||
s3: S3Service,
|
||||
prefix: string,
|
||||
cutoff: number,
|
||||
options: ReconcileOptions,
|
||||
report: ReconcileReport,
|
||||
): Promise<void> {
|
||||
let token: string | undefined;
|
||||
do {
|
||||
const page = await s3.listObjects(prefix, options.pageSize ?? 500, token);
|
||||
token = page.nextToken;
|
||||
report.scanned += page.objects.length;
|
||||
const old = page.objects.filter((object) => object.lastModified.getTime() < cutoff);
|
||||
report.fresh_skipped += page.objects.length - old.length;
|
||||
const keys = old.map((object) => object.key);
|
||||
const rows = await prisma.attachment.findMany({
|
||||
where: { OR: [{ objectKey: { in: keys } }, { stagingKey: { in: keys } }] },
|
||||
select: { objectKey: true, stagingKey: true },
|
||||
});
|
||||
const variants = await prisma.attachmentVariant.findMany({
|
||||
where: { objectKey: { in: keys } },
|
||||
select: { objectKey: true },
|
||||
});
|
||||
const ids = [...new Set(keys.map(attachmentIdOf).filter((id): id is string => id !== null))];
|
||||
const owners = ids.length
|
||||
? await prisma.attachment.findMany({ where: { id: { in: ids } }, select: { id: true } })
|
||||
: [];
|
||||
const owned = new Set(owners.map((row) => row.id));
|
||||
const known = new Set([
|
||||
...rows.flatMap((row) => [row.objectKey, row.stagingKey]),
|
||||
...variants.map((row) => row.objectKey),
|
||||
]);
|
||||
for (const { key } of old) {
|
||||
const owner = attachmentIdOf(key);
|
||||
if (known.has(key) || (owner !== null && owned.has(owner))) {
|
||||
report.referenced++;
|
||||
continue;
|
||||
}
|
||||
report.orphans++;
|
||||
if (report.orphan_keys.length < MAX_REPORTED_KEYS) {
|
||||
report.orphan_keys.push(key);
|
||||
}
|
||||
if (options.apply) {
|
||||
await s3.delete(key).then(
|
||||
() => report.deleted++,
|
||||
() => report.failed++,
|
||||
);
|
||||
}
|
||||
}
|
||||
} while (token);
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
import { createHash } from 'crypto';
|
||||
import { FastifyInstance, FastifyRequest } from 'fastify';
|
||||
import { createWriteStream } from 'fs';
|
||||
import { Readable } from 'stream';
|
||||
import { ApiError } from '../common/filters/api-error';
|
||||
|
||||
const UPLOAD_ROUTE = '/api/v1/chats/:chatId/attachments';
|
||||
const PASS_THROUGH_TYPE = 'application/x-messenger-upload';
|
||||
|
||||
declare module 'fastify' {
|
||||
interface FastifyRequest {
|
||||
uploadContentType?: string;
|
||||
}
|
||||
}
|
||||
|
||||
export function registerUploadBody(instance: FastifyInstance): void {
|
||||
instance.addContentTypeParser(PASS_THROUGH_TYPE, (_request, payload, done) => {
|
||||
done(null, payload);
|
||||
});
|
||||
instance.addHook('onRequest', (request: FastifyRequest, _reply, done) => {
|
||||
if (request.method === 'POST' && request.routeOptions.url === UPLOAD_ROUTE) {
|
||||
request.uploadContentType = request.headers['content-type'];
|
||||
request.headers['content-type'] = PASS_THROUGH_TYPE;
|
||||
}
|
||||
done();
|
||||
});
|
||||
}
|
||||
|
||||
export interface ReceivedBody {
|
||||
size: number;
|
||||
sha256: string;
|
||||
}
|
||||
|
||||
export function receiveBody(
|
||||
body: Readable,
|
||||
path: string,
|
||||
maxBytes: number,
|
||||
timeoutMs: number,
|
||||
): Promise<ReceivedBody> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const hash = createHash('sha256');
|
||||
const out = createWriteStream(path, { flags: 'wx', mode: 0o600 });
|
||||
let size = 0;
|
||||
let settled = false;
|
||||
|
||||
const timer = setTimeout(() => body.destroy(), timeoutMs);
|
||||
const settle = (done: () => void): void => {
|
||||
if (settled) {
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
body.off('data', onData);
|
||||
body.off('end', onEnd);
|
||||
body.off('close', onClose);
|
||||
done();
|
||||
};
|
||||
const fail = (error: ApiError): void => {
|
||||
settle(() => {
|
||||
out.destroy();
|
||||
reject(error);
|
||||
});
|
||||
};
|
||||
const onData = (chunk: Buffer): void => {
|
||||
size += chunk.length;
|
||||
if (size > maxBytes) {
|
||||
fail(new ApiError('payload_too_large', 'attachment exceeds size limit'));
|
||||
return;
|
||||
}
|
||||
hash.update(chunk);
|
||||
if (!out.write(chunk)) {
|
||||
body.pause();
|
||||
out.once('drain', () => {
|
||||
if (!settled) {
|
||||
body.resume();
|
||||
}
|
||||
});
|
||||
}
|
||||
};
|
||||
const onEnd = (): void => {
|
||||
settle(() => {
|
||||
out.end(() =>
|
||||
resolve({ size, sha256: hash.digest('hex') }),
|
||||
);
|
||||
});
|
||||
};
|
||||
const onClose = (): void => {
|
||||
fail(new ApiError('invalid_request', 'upload was interrupted'));
|
||||
};
|
||||
|
||||
out.on('error', () => {
|
||||
fail(new ApiError('unavailable', 'temporary storage unavailable'));
|
||||
});
|
||||
body.on('data', onData);
|
||||
body.on('end', onEnd);
|
||||
body.on('close', onClose);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
import {
|
||||
Controller,
|
||||
Delete,
|
||||
Headers,
|
||||
HttpCode,
|
||||
Post,
|
||||
Req,
|
||||
UseGuards,
|
||||
} from '@nestjs/common';
|
||||
import { FastifyRequest } from 'fastify';
|
||||
import { ApiError } from '../common/filters/api-error';
|
||||
import { toUserResponse, UserResponse } from '../users/dto/user.response';
|
||||
import { AuthGuard } from './auth.guard';
|
||||
import { AuthService } from './auth.service';
|
||||
|
||||
interface SessionResponse {
|
||||
token: string;
|
||||
token_type: 'Bearer';
|
||||
expires_at: string;
|
||||
user: UserResponse;
|
||||
}
|
||||
|
||||
@Controller('api/v1/auth/sessions')
|
||||
export class AuthController {
|
||||
constructor(private readonly auth: AuthService) {}
|
||||
|
||||
@Post()
|
||||
@HttpCode(201)
|
||||
async create(
|
||||
@Headers('authorization') authorization: string | undefined,
|
||||
@Req() request: FastifyRequest,
|
||||
): Promise<SessionResponse> {
|
||||
if (request.body !== undefined) {
|
||||
throw new ApiError('invalid_request', 'request body is not allowed');
|
||||
}
|
||||
const session = await this.auth.login(authorization);
|
||||
return {
|
||||
token: session.token,
|
||||
token_type: 'Bearer',
|
||||
expires_at: session.expiresAt.toISOString(),
|
||||
user: toUserResponse(session.user),
|
||||
};
|
||||
}
|
||||
|
||||
@Delete('current')
|
||||
@HttpCode(204)
|
||||
@UseGuards(AuthGuard)
|
||||
async revoke(@Req() request: FastifyRequest): Promise<void> {
|
||||
await this.auth.revoke(request.sessionTokenHash!);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
import { CanActivate, ExecutionContext, Injectable } from '@nestjs/common';
|
||||
import { FastifyRequest } from 'fastify';
|
||||
import { ApiError } from '../common/filters/api-error';
|
||||
import { AuthService } from './auth.service';
|
||||
|
||||
const BEARER_CHALLENGE = 'Bearer realm="messenger"';
|
||||
|
||||
@Injectable()
|
||||
export class AuthGuard implements CanActivate {
|
||||
constructor(private readonly auth: AuthService) {}
|
||||
|
||||
async canActivate(context: ExecutionContext): Promise<boolean> {
|
||||
const request = context.switchToHttp().getRequest<FastifyRequest>();
|
||||
const match = /^Bearer (\S+)$/i.exec(request.headers.authorization ?? '');
|
||||
const session = match ? await this.auth.resolve(match[1]) : null;
|
||||
if (!session) {
|
||||
throw new ApiError('unauthorized', 'missing or invalid session', {
|
||||
'WWW-Authenticate': BEARER_CHALLENGE,
|
||||
});
|
||||
}
|
||||
request.currentUser = session.user;
|
||||
request.sessionTokenHash = session.tokenHash;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
import { Global, Module } from '@nestjs/common';
|
||||
import { AuthController } from './auth.controller';
|
||||
import { AuthGuard } from './auth.guard';
|
||||
import { AuthService } from './auth.service';
|
||||
|
||||
@Global()
|
||||
@Module({
|
||||
controllers: [AuthController],
|
||||
providers: [AuthService, AuthGuard],
|
||||
exports: [AuthService, AuthGuard],
|
||||
})
|
||||
export class AuthModule {}
|
||||
@@ -0,0 +1,155 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { createHash, randomBytes } from 'crypto';
|
||||
import { CacheService } from '../cache/cache.service';
|
||||
import { codePointLength } from '../common/dto/code-point-length.validator';
|
||||
import { User } from '../common/domain';
|
||||
import { ApiError } from '../common/filters/api-error';
|
||||
import { PrismaService } from '../db/prisma.service';
|
||||
import { toUser } from '../users/users.service';
|
||||
import { verifyPassword } from './password';
|
||||
|
||||
const BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"';
|
||||
const USERNAME_PATTERN = /^[a-z0-9_]{3,32}$/;
|
||||
const CACHE_TTL_MS = 60_000;
|
||||
const REVOKED = 'revoked';
|
||||
|
||||
interface CachedSession {
|
||||
user: User;
|
||||
expiresAt: number;
|
||||
}
|
||||
|
||||
export interface CreatedSession {
|
||||
token: string;
|
||||
expiresAt: Date;
|
||||
user: User;
|
||||
}
|
||||
|
||||
export interface ResolvedSession {
|
||||
tokenHash: string;
|
||||
user: User;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class AuthService {
|
||||
private readonly ttlSeconds = readSessionTtl();
|
||||
|
||||
constructor(
|
||||
private readonly prisma: PrismaService,
|
||||
private readonly cache: CacheService,
|
||||
) {}
|
||||
|
||||
async login(authorization: string | undefined): Promise<CreatedSession> {
|
||||
const credentials = parseBasic(authorization);
|
||||
if (!credentials) {
|
||||
throw invalidCredentials();
|
||||
}
|
||||
const row = await this.prisma.user.findUnique({
|
||||
where: { username: credentials.username },
|
||||
});
|
||||
const valid = await verifyPassword(
|
||||
row?.passwordHash ?? null,
|
||||
credentials.password,
|
||||
);
|
||||
if (!row || !valid) {
|
||||
throw invalidCredentials();
|
||||
}
|
||||
const token = randomBytes(32).toString('base64url');
|
||||
const expiresAt = new Date(Date.now() + this.ttlSeconds * 1000);
|
||||
await this.prisma.session.create({
|
||||
data: { tokenHash: hashToken(token), userId: row.id, expiresAt },
|
||||
});
|
||||
return { token, expiresAt, user: toUser(row) };
|
||||
}
|
||||
|
||||
async resolve(token: string): Promise<ResolvedSession | null> {
|
||||
const tokenHash = hashToken(token);
|
||||
const key = cacheKey(tokenHash);
|
||||
const cached = await this.cache.get(key);
|
||||
if (cached === REVOKED) {
|
||||
return null;
|
||||
}
|
||||
if (cached) {
|
||||
const entry = JSON.parse(cached) as CachedSession;
|
||||
if (entry.expiresAt > Date.now()) {
|
||||
return { tokenHash, user: entry.user };
|
||||
}
|
||||
}
|
||||
const session = await this.prisma.session.findUnique({
|
||||
where: { tokenHash },
|
||||
include: { user: true },
|
||||
});
|
||||
const remainingMs = session
|
||||
? session.expiresAt.getTime() - Date.now()
|
||||
: 0;
|
||||
if (!session || remainingMs <= 0) {
|
||||
return null;
|
||||
}
|
||||
const user = toUser(session.user);
|
||||
const entry: CachedSession = {
|
||||
user,
|
||||
expiresAt: session.expiresAt.getTime(),
|
||||
};
|
||||
await this.cache.set(
|
||||
key,
|
||||
JSON.stringify(entry),
|
||||
Math.min(remainingMs, CACHE_TTL_MS),
|
||||
true,
|
||||
);
|
||||
return { tokenHash, user };
|
||||
}
|
||||
|
||||
async revoke(tokenHash: string): Promise<void> {
|
||||
const marked = await this.cache.setStrict(
|
||||
cacheKey(tokenHash),
|
||||
REVOKED,
|
||||
CACHE_TTL_MS,
|
||||
);
|
||||
if (!marked) {
|
||||
throw new ApiError('unavailable', 'session cache unavailable');
|
||||
}
|
||||
await this.prisma.session.deleteMany({ where: { tokenHash } });
|
||||
}
|
||||
}
|
||||
|
||||
function cacheKey(tokenHash: string): string {
|
||||
return `session:v1:${tokenHash}`;
|
||||
}
|
||||
|
||||
function parseBasic(
|
||||
header: string | undefined,
|
||||
): { username: string; password: string } | null {
|
||||
const match = /^Basic ([A-Za-z0-9+/]+={0,2})$/i.exec(header ?? '');
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
const decoded = Buffer.from(match[1], 'base64').toString('utf8');
|
||||
const separator = decoded.indexOf(':');
|
||||
if (separator < 0) {
|
||||
return null;
|
||||
}
|
||||
const username = decoded.slice(0, separator);
|
||||
const password = decoded.slice(separator + 1);
|
||||
const length = codePointLength(password);
|
||||
if (!USERNAME_PATTERN.test(username) || length < 12 || length > 128) {
|
||||
return null;
|
||||
}
|
||||
return { username, password };
|
||||
}
|
||||
|
||||
function hashToken(token: string): string {
|
||||
return createHash('sha256').update(token).digest('hex');
|
||||
}
|
||||
|
||||
function invalidCredentials(): ApiError {
|
||||
return new ApiError('unauthorized', 'invalid credentials', {
|
||||
'WWW-Authenticate': BASIC_CHALLENGE,
|
||||
});
|
||||
}
|
||||
|
||||
function readSessionTtl(): number {
|
||||
const ttl = Number(process.env.SESSION_TTL_SECONDS ?? 3600);
|
||||
if (!Number.isInteger(ttl) || ttl <= 0) {
|
||||
throw new Error('SESSION_TTL_SECONDS must be a positive integer');
|
||||
}
|
||||
return ttl;
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import { createParamDecorator, ExecutionContext } from '@nestjs/common';
|
||||
import { FastifyRequest } from 'fastify';
|
||||
import { User } from '../common/domain';
|
||||
|
||||
declare module 'fastify' {
|
||||
interface FastifyRequest {
|
||||
currentUser?: User;
|
||||
sessionTokenHash?: string;
|
||||
}
|
||||
}
|
||||
|
||||
export const CurrentUser = createParamDecorator(
|
||||
(_data: unknown, ctx: ExecutionContext): User => {
|
||||
const request = ctx.switchToHttp().getRequest<FastifyRequest>();
|
||||
if (!request.currentUser) {
|
||||
throw new Error('CurrentUser used without AuthGuard');
|
||||
}
|
||||
return request.currentUser;
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,17 @@
|
||||
import { argon2id, hash, verify } from 'argon2';
|
||||
|
||||
const OPTIONS = { type: argon2id, memoryCost: 19456, timeCost: 2, parallelism: 1 };
|
||||
|
||||
const dummyHash = hash('timing-equalizer-password', OPTIONS);
|
||||
|
||||
export function hashPassword(password: string): Promise<string> {
|
||||
return hash(password, OPTIONS);
|
||||
}
|
||||
|
||||
export async function verifyPassword(
|
||||
passwordHash: string | null,
|
||||
password: string,
|
||||
): Promise<boolean> {
|
||||
const valid = await verify(passwordHash ?? (await dummyHash), password);
|
||||
return passwordHash !== null && valid;
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
import { Logger } from '@nestjs/common';
|
||||
import amqp, { Channel, Connection } from 'amqplib';
|
||||
import { readFileSync } from 'fs';
|
||||
|
||||
export const JOB_QUEUE = 'attachment.jobs';
|
||||
export const DEAD_LETTER_QUEUE = 'attachment.jobs.dlq';
|
||||
const DELIVERY_LIMIT = 20;
|
||||
const RECONNECT_MAX_MS = 10000;
|
||||
|
||||
export function brokerUrl(): string {
|
||||
const file = process.env.BROKER_URL_FILE;
|
||||
const url = file ? readFileSync(file, 'utf8').trim() : process.env.BROKER_URL;
|
||||
if (!url) {
|
||||
throw new Error('BROKER_URL_FILE or BROKER_URL must be set');
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
export async function assertTopology(channel: Channel): Promise<void> {
|
||||
await channel.assertQueue(DEAD_LETTER_QUEUE, {
|
||||
durable: true,
|
||||
arguments: { 'x-queue-type': 'quorum' },
|
||||
});
|
||||
await channel.assertQueue(JOB_QUEUE, {
|
||||
durable: true,
|
||||
arguments: {
|
||||
'x-queue-type': 'quorum',
|
||||
'x-delivery-limit': DELIVERY_LIMIT,
|
||||
'x-dead-letter-exchange': '',
|
||||
'x-dead-letter-routing-key': DEAD_LETTER_QUEUE,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export async function connectWithRetry(
|
||||
logger: Logger,
|
||||
stopping: () => boolean,
|
||||
): Promise<Connection | null> {
|
||||
let delay = 500;
|
||||
while (!stopping()) {
|
||||
try {
|
||||
const connection = await amqp.connect(brokerUrl(), { timeout: 5000 });
|
||||
logger.log('connected to broker');
|
||||
return connection;
|
||||
} catch (error) {
|
||||
const reason = error instanceof Error ? error.message.split(' ')[0] : 'unknown';
|
||||
logger.warn(`broker unavailable (${reason}), retry in ${delay} ms`);
|
||||
await sleep(delay);
|
||||
delay = Math.min(RECONNECT_MAX_MS, delay * 2);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
Vendored
+9
@@ -0,0 +1,9 @@
|
||||
import { Global, Module } from '@nestjs/common';
|
||||
import { CacheService } from './cache.service';
|
||||
|
||||
@Global()
|
||||
@Module({
|
||||
providers: [CacheService],
|
||||
exports: [CacheService],
|
||||
})
|
||||
export class CacheModule {}
|
||||
Vendored
+98
@@ -0,0 +1,98 @@
|
||||
import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common';
|
||||
import { readFileSync } from 'fs';
|
||||
import Redis from 'ioredis';
|
||||
|
||||
@Injectable()
|
||||
export class CacheService implements OnModuleDestroy {
|
||||
private readonly logger = new Logger('CacheService');
|
||||
private readonly client: Redis | null;
|
||||
private trusted = false;
|
||||
|
||||
constructor() {
|
||||
const url = readCacheUrl();
|
||||
this.client = url
|
||||
? new Redis(url, {
|
||||
enableOfflineQueue: false,
|
||||
maxRetriesPerRequest: 0,
|
||||
connectTimeout: 1000,
|
||||
commandTimeout: 200,
|
||||
})
|
||||
: null;
|
||||
this.client?.on('error', (error) =>
|
||||
this.logger.warn(`redis unavailable: ${error.message}`),
|
||||
);
|
||||
this.client?.on('close', () => {
|
||||
this.trusted = false;
|
||||
});
|
||||
this.client?.on('ready', () => {
|
||||
this.client!.flushdb().then(
|
||||
() => {
|
||||
this.trusted = true;
|
||||
},
|
||||
() => this.distrust(),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
async get(key: string): Promise<string | null> {
|
||||
if (!this.client || !this.trusted) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return await this.client.get(key);
|
||||
} catch {
|
||||
this.distrust();
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async set(
|
||||
key: string,
|
||||
value: string,
|
||||
ttlMs: number,
|
||||
onlyIfAbsent = false,
|
||||
): Promise<void> {
|
||||
if (!this.client || !this.trusted) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
if (onlyIfAbsent) {
|
||||
await this.client.set(key, value, 'PX', ttlMs, 'NX');
|
||||
} else {
|
||||
await this.client.set(key, value, 'PX', ttlMs);
|
||||
}
|
||||
} catch {
|
||||
this.distrust();
|
||||
}
|
||||
}
|
||||
|
||||
async setStrict(key: string, value: string, ttlMs: number): Promise<boolean> {
|
||||
if (!this.client) {
|
||||
return true;
|
||||
}
|
||||
if (!this.trusted) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
await this.client.set(key, value, 'PX', ttlMs);
|
||||
return true;
|
||||
} catch {
|
||||
this.distrust();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
onModuleDestroy(): void {
|
||||
this.client?.disconnect();
|
||||
}
|
||||
|
||||
private distrust(): void {
|
||||
this.trusted = false;
|
||||
this.client?.disconnect(true);
|
||||
}
|
||||
}
|
||||
|
||||
function readCacheUrl(): string | undefined {
|
||||
const file = process.env.CACHE_URL_FILE;
|
||||
return file ? readFileSync(file, 'utf8').trim() : process.env.CACHE_URL;
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
import { ApiError } from '../common/filters/api-error';
|
||||
import { validate as isUuid } from '../common/uuid';
|
||||
import { PrismaService } from '../db/prisma.service';
|
||||
|
||||
export async function assertChatMember(
|
||||
prisma: PrismaService,
|
||||
chatId: string,
|
||||
userId: string,
|
||||
): Promise<void> {
|
||||
if (!isUuid(chatId)) {
|
||||
throw new ApiError('invalid_request', 'chatId must be a valid UUID');
|
||||
}
|
||||
const member = await prisma.chatMember.findUnique({
|
||||
where: { chatId_userId: { chatId, userId } },
|
||||
});
|
||||
if (!member) {
|
||||
throw new ApiError('not_found', 'chat not found');
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
import {
|
||||
Body,
|
||||
Controller,
|
||||
Get,
|
||||
HttpCode,
|
||||
Param,
|
||||
Post,
|
||||
Query,
|
||||
UseGuards,
|
||||
} from '@nestjs/common';
|
||||
import { AuthGuard } from '../auth/auth.guard';
|
||||
import { CurrentUser } from '../auth/current-user.decorator';
|
||||
import { parseLimit, rawCursorParam } from '../common/dto/pagination-query.dto';
|
||||
import { User } from '../common/domain';
|
||||
import { ChatsService } from './chats.service';
|
||||
import {
|
||||
ChatPageResponse,
|
||||
ChatResponse,
|
||||
toChatResponse,
|
||||
} from './dto/chat.response';
|
||||
import { CreateChatDto } from './dto/create-chat.dto';
|
||||
|
||||
@Controller('api/v1/chats')
|
||||
@UseGuards(AuthGuard)
|
||||
export class ChatsController {
|
||||
constructor(private readonly chats: ChatsService) {}
|
||||
|
||||
@Post()
|
||||
@HttpCode(201)
|
||||
async create(
|
||||
@Body() dto: CreateChatDto,
|
||||
@CurrentUser() user: User,
|
||||
): Promise<ChatResponse> {
|
||||
const chat = await this.chats.create(dto.title, dto.member_ids, user.id);
|
||||
return toChatResponse(chat);
|
||||
}
|
||||
|
||||
@Get()
|
||||
async list(
|
||||
@CurrentUser() user: User,
|
||||
@Query('limit') limitRaw: unknown,
|
||||
@Query('cursor') cursorRaw: unknown,
|
||||
): Promise<ChatPageResponse> {
|
||||
const limit = parseLimit(limitRaw);
|
||||
const cursor = rawCursorParam(cursorRaw);
|
||||
return this.chats.listForMember(user.id, limit, cursor);
|
||||
}
|
||||
|
||||
@Get(':chatId')
|
||||
async getOne(
|
||||
@Param('chatId') chatId: string,
|
||||
@CurrentUser() user: User,
|
||||
): Promise<ChatResponse> {
|
||||
const chat = await this.chats.getForMember(chatId, user.id);
|
||||
return toChatResponse(chat);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { ChatsController } from './chats.controller';
|
||||
import { ChatsService } from './chats.service';
|
||||
|
||||
@Module({
|
||||
controllers: [ChatsController],
|
||||
providers: [ChatsService],
|
||||
exports: [ChatsService],
|
||||
})
|
||||
export class ChatsModule {}
|
||||
@@ -0,0 +1,131 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { Prisma } from '@prisma/client';
|
||||
import { decodeChatCursor, encodeChatCursor } from '../common/cursor';
|
||||
import { Chat } from '../common/domain';
|
||||
import { ApiError } from '../common/filters/api-error';
|
||||
import { validate as isUuid } from '../common/uuid';
|
||||
import { PrismaService, isForeignKeyViolation } from '../db/prisma.service';
|
||||
import { ChatPageResponse, toChatResponse } from './dto/chat.response';
|
||||
|
||||
interface ChatRow {
|
||||
id: string;
|
||||
title: string;
|
||||
createdAt: Date;
|
||||
memberIds: string[];
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class ChatsService {
|
||||
constructor(private readonly prisma: PrismaService) {}
|
||||
|
||||
async create(
|
||||
title: string,
|
||||
rawMemberIds: string[],
|
||||
callerId: string,
|
||||
): Promise<Chat> {
|
||||
const invalid = rawMemberIds.find((id) => !isUuid(id));
|
||||
if (invalid) {
|
||||
throw new ApiError('invalid_request', `unknown member_id: ${invalid}`);
|
||||
}
|
||||
const memberIds = rawMemberIds.map((id) => id.toLowerCase());
|
||||
if (new Set(memberIds).size !== memberIds.length) {
|
||||
throw new ApiError(
|
||||
'invalid_request',
|
||||
"All member_ids's elements must be unique",
|
||||
);
|
||||
}
|
||||
if (!memberIds.includes(callerId)) {
|
||||
throw new ApiError(
|
||||
'invalid_request',
|
||||
'member_ids must include the caller',
|
||||
);
|
||||
}
|
||||
try {
|
||||
const chat = await this.prisma.$transaction(async (tx) => {
|
||||
const created = await tx.chat.create({
|
||||
data: { title: title.trim() },
|
||||
});
|
||||
await tx.chatMember.createMany({
|
||||
data: memberIds.map((userId) => ({
|
||||
chatId: created.id,
|
||||
userId,
|
||||
})),
|
||||
});
|
||||
return created;
|
||||
});
|
||||
return toChat({ ...chat, memberIds });
|
||||
} catch (error) {
|
||||
if (isForeignKeyViolation(error)) {
|
||||
throw new ApiError(
|
||||
'invalid_request',
|
||||
'member_ids contain an unknown user',
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async getForMember(chatId: string, callerId: string): Promise<Chat> {
|
||||
if (!isUuid(chatId)) {
|
||||
throw new ApiError('invalid_request', 'chatId must be a valid UUID');
|
||||
}
|
||||
const chat = await this.prisma.chat.findFirst({
|
||||
where: { id: chatId, members: { some: { userId: callerId } } },
|
||||
include: { members: { select: { userId: true } } },
|
||||
});
|
||||
if (!chat) {
|
||||
throw new ApiError('not_found', 'chat not found');
|
||||
}
|
||||
return toChat({
|
||||
...chat,
|
||||
memberIds: chat.members.map((m) => m.userId),
|
||||
});
|
||||
}
|
||||
|
||||
async listForMember(
|
||||
callerId: string,
|
||||
limit: number,
|
||||
rawCursor: string | null,
|
||||
): Promise<ChatPageResponse> {
|
||||
let after = Prisma.empty;
|
||||
if (rawCursor !== null) {
|
||||
const cursor = decodeChatCursor(rawCursor);
|
||||
const member = cursor
|
||||
? await this.prisma.chatMember.findUnique({
|
||||
where: {
|
||||
chatId_userId: { chatId: cursor.id, userId: callerId },
|
||||
},
|
||||
include: { chat: { select: { createdAt: true } } },
|
||||
})
|
||||
: null;
|
||||
if (!member) {
|
||||
throw new ApiError('invalid_request', 'invalid cursor');
|
||||
}
|
||||
after = Prisma.sql`AND (c.created_at, c.id) > (${member.chat.createdAt}::timestamptz, ${member.chatId}::uuid)`;
|
||||
}
|
||||
const rows = await this.prisma.$queryRaw<ChatRow[]>`
|
||||
SELECT c.id, c.title, c.created_at AS "createdAt",
|
||||
ARRAY(SELECT m.user_id::text FROM chat_members m WHERE m.chat_id = c.id) AS "memberIds"
|
||||
FROM chat_members cm
|
||||
JOIN chats c ON c.id = cm.chat_id
|
||||
WHERE cm.user_id = ${callerId}::uuid ${after}
|
||||
ORDER BY c.created_at, c.id
|
||||
LIMIT ${limit + 1}`;
|
||||
const items = rows.slice(0, limit).map(toChat);
|
||||
const last = items[items.length - 1];
|
||||
return {
|
||||
items: items.map(toChatResponse),
|
||||
next_cursor:
|
||||
rows.length > limit ? encodeChatCursor({ id: last.id }) : null,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function toChat(row: ChatRow): Chat {
|
||||
return {
|
||||
id: row.id,
|
||||
title: row.title,
|
||||
memberIds: row.memberIds,
|
||||
createdAt: row.createdAt.toISOString(),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import { Chat } from '../../common/domain';
|
||||
|
||||
export interface ChatResponse {
|
||||
id: string;
|
||||
title: string;
|
||||
member_ids: string[];
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
export interface ChatPageResponse {
|
||||
items: ChatResponse[];
|
||||
next_cursor: string | null;
|
||||
}
|
||||
|
||||
export function toChatResponse(chat: Chat): ChatResponse {
|
||||
return {
|
||||
id: chat.id,
|
||||
title: chat.title,
|
||||
member_ids: chat.memberIds,
|
||||
created_at: chat.createdAt,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import {
|
||||
ArrayMaxSize,
|
||||
ArrayMinSize,
|
||||
ArrayUnique,
|
||||
IsArray,
|
||||
IsString,
|
||||
} from 'class-validator';
|
||||
import { CodePointLength } from '../../common/dto/code-point-length.validator';
|
||||
import { NotBlank } from '../../common/dto/not-blank.validator';
|
||||
|
||||
export class CreateChatDto {
|
||||
@NotBlank()
|
||||
@CodePointLength(1, 100)
|
||||
title!: string;
|
||||
|
||||
@IsArray()
|
||||
@ArrayMinSize(1)
|
||||
@ArrayMaxSize(100)
|
||||
@ArrayUnique()
|
||||
@IsString({ each: true })
|
||||
member_ids!: string[];
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
import { validate as isUuid } from './uuid';
|
||||
|
||||
export interface ChatCursor {
|
||||
id: string;
|
||||
}
|
||||
|
||||
export interface MessageCursor {
|
||||
chatId: string;
|
||||
id: string;
|
||||
}
|
||||
|
||||
export function encodeChatCursor(cursor: ChatCursor): string {
|
||||
return encode(cursor);
|
||||
}
|
||||
|
||||
export function decodeChatCursor(raw: string): ChatCursor | null {
|
||||
const decoded = decode(raw);
|
||||
return decoded && isUuidField(decoded.id) ? { id: decoded.id } : null;
|
||||
}
|
||||
|
||||
export function encodeMessageCursor(cursor: MessageCursor): string {
|
||||
return encode(cursor);
|
||||
}
|
||||
|
||||
export function decodeMessageCursor(raw: string): MessageCursor | null {
|
||||
const decoded = decode(raw);
|
||||
return decoded && isUuidField(decoded.chatId) && isUuidField(decoded.id)
|
||||
? { chatId: decoded.chatId, id: decoded.id }
|
||||
: null;
|
||||
}
|
||||
|
||||
function isUuidField(value: unknown): value is string {
|
||||
return typeof value === 'string' && isUuid(value);
|
||||
}
|
||||
|
||||
function encode(value: object): string {
|
||||
return Buffer.from(JSON.stringify(value), 'utf8').toString('base64url');
|
||||
}
|
||||
|
||||
function decode(raw: string): Record<string, unknown> | null {
|
||||
try {
|
||||
const parsed = JSON.parse(
|
||||
Buffer.from(raw, 'base64url').toString('utf8'),
|
||||
);
|
||||
return typeof parsed === 'object' && parsed !== null ? parsed : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
export interface User {
|
||||
id: string;
|
||||
username: string;
|
||||
displayName: string;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
export interface Chat {
|
||||
id: string;
|
||||
title: string;
|
||||
memberIds: string[];
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
export interface Message {
|
||||
id: string;
|
||||
chatId: string;
|
||||
senderId: string;
|
||||
text: string;
|
||||
createdAt: string;
|
||||
attachmentIds: string[];
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import { registerDecorator, ValidationOptions } from 'class-validator';
|
||||
|
||||
export function codePointLength(value: string): number {
|
||||
return [...value].length;
|
||||
}
|
||||
|
||||
export function CodePointLength(
|
||||
min: number,
|
||||
max: number,
|
||||
options?: ValidationOptions,
|
||||
) {
|
||||
return function (object: object, propertyName: string) {
|
||||
registerDecorator({
|
||||
name: 'codePointLength',
|
||||
target: object.constructor,
|
||||
propertyName,
|
||||
constraints: [min, max],
|
||||
options,
|
||||
validator: {
|
||||
validate(value: unknown): boolean {
|
||||
if (typeof value !== 'string') {
|
||||
return false;
|
||||
}
|
||||
const length = codePointLength(value);
|
||||
return length >= min && length <= max;
|
||||
},
|
||||
defaultMessage(): string {
|
||||
return `${propertyName} must be between ${min} and ${max} Unicode code points`;
|
||||
},
|
||||
},
|
||||
});
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import { registerDecorator, ValidationOptions } from 'class-validator';
|
||||
|
||||
export function NotBlank(options?: ValidationOptions) {
|
||||
return function (object: object, propertyName: string) {
|
||||
registerDecorator({
|
||||
name: 'notBlank',
|
||||
target: object.constructor,
|
||||
propertyName,
|
||||
options,
|
||||
validator: {
|
||||
validate(value: unknown): boolean {
|
||||
return typeof value === 'string' && value.trim().length > 0;
|
||||
},
|
||||
defaultMessage(): string {
|
||||
return `${propertyName} must not be empty or whitespace-only`;
|
||||
},
|
||||
},
|
||||
});
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import { ApiError } from '../filters/api-error';
|
||||
|
||||
const DEFAULT_LIMIT = 50;
|
||||
const MIN_LIMIT = 1;
|
||||
const MAX_LIMIT = 100;
|
||||
|
||||
export function parseLimit(raw: unknown): number {
|
||||
if (raw === undefined) {
|
||||
return DEFAULT_LIMIT;
|
||||
}
|
||||
if (typeof raw !== 'string' || !/^\d+$/.test(raw)) {
|
||||
throw new ApiError('invalid_request', 'limit must be an integer');
|
||||
}
|
||||
const limit = Number(raw);
|
||||
if (limit < MIN_LIMIT || limit > MAX_LIMIT) {
|
||||
throw new ApiError(
|
||||
'invalid_request',
|
||||
`limit must be between ${MIN_LIMIT} and ${MAX_LIMIT}`,
|
||||
);
|
||||
}
|
||||
return limit;
|
||||
}
|
||||
|
||||
export function rawCursorParam(raw: unknown): string | null {
|
||||
if (raw === undefined) {
|
||||
return null;
|
||||
}
|
||||
if (typeof raw !== 'string' || raw.length === 0) {
|
||||
throw new ApiError(
|
||||
'invalid_request',
|
||||
'cursor must be a non-empty string',
|
||||
);
|
||||
}
|
||||
return raw;
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
import { HttpException, HttpStatus } from '@nestjs/common';
|
||||
|
||||
export type ErrorCode =
|
||||
| 'invalid_request'
|
||||
| 'unauthorized'
|
||||
| 'not_found'
|
||||
| 'conflict'
|
||||
| 'payload_too_large'
|
||||
| 'unsupported_media_type'
|
||||
| 'rate_limited'
|
||||
| 'unavailable'
|
||||
| 'internal_error';
|
||||
|
||||
const STATUS_BY_CODE: Record<ErrorCode, number> = {
|
||||
invalid_request: HttpStatus.BAD_REQUEST,
|
||||
unauthorized: HttpStatus.UNAUTHORIZED,
|
||||
not_found: HttpStatus.NOT_FOUND,
|
||||
conflict: HttpStatus.CONFLICT,
|
||||
payload_too_large: HttpStatus.PAYLOAD_TOO_LARGE,
|
||||
unsupported_media_type: HttpStatus.UNSUPPORTED_MEDIA_TYPE,
|
||||
rate_limited: HttpStatus.TOO_MANY_REQUESTS,
|
||||
unavailable: HttpStatus.SERVICE_UNAVAILABLE,
|
||||
internal_error: HttpStatus.INTERNAL_SERVER_ERROR,
|
||||
};
|
||||
|
||||
export class ApiError extends HttpException {
|
||||
constructor(
|
||||
public readonly code: ErrorCode,
|
||||
message: string,
|
||||
public readonly headers: Record<string, string> = {},
|
||||
) {
|
||||
super(message, STATUS_BY_CODE[code]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
import {
|
||||
ArgumentsHost,
|
||||
Catch,
|
||||
ExceptionFilter,
|
||||
HttpException,
|
||||
HttpStatus,
|
||||
Logger,
|
||||
} from '@nestjs/common';
|
||||
import { FastifyReply, FastifyRequest } from 'fastify';
|
||||
import { isDatabaseUnavailable } from '../../db/prisma.service';
|
||||
import { ApiError, ErrorCode } from './api-error';
|
||||
|
||||
const CODE_BY_STATUS: Partial<Record<number, ErrorCode>> = {
|
||||
[HttpStatus.BAD_REQUEST]: 'invalid_request',
|
||||
[HttpStatus.UNAUTHORIZED]: 'unauthorized',
|
||||
[HttpStatus.NOT_FOUND]: 'not_found',
|
||||
[HttpStatus.CONFLICT]: 'conflict',
|
||||
[HttpStatus.PAYLOAD_TOO_LARGE]: 'payload_too_large',
|
||||
[HttpStatus.UNSUPPORTED_MEDIA_TYPE]: 'unsupported_media_type',
|
||||
[HttpStatus.TOO_MANY_REQUESTS]: 'rate_limited',
|
||||
[HttpStatus.SERVICE_UNAVAILABLE]: 'unavailable',
|
||||
};
|
||||
|
||||
interface Resolved {
|
||||
status: number;
|
||||
code: ErrorCode;
|
||||
message: string;
|
||||
}
|
||||
|
||||
@Catch()
|
||||
export class HttpExceptionFilter implements ExceptionFilter {
|
||||
private readonly logger = new Logger('HttpExceptionFilter');
|
||||
|
||||
catch(exception: unknown, host: ArgumentsHost): void {
|
||||
const ctx = host.switchToHttp();
|
||||
const reply = ctx.getResponse<FastifyReply>();
|
||||
const request = ctx.getRequest<FastifyRequest>();
|
||||
const requestId = request.requestId;
|
||||
|
||||
const { status, code, message } = this.resolve(exception);
|
||||
if (status >= 500) {
|
||||
this.logger.error(
|
||||
`${request.method} ${request.url} -> ${status}: ${message}`,
|
||||
);
|
||||
}
|
||||
|
||||
if (exception instanceof ApiError) {
|
||||
reply.headers(exception.headers);
|
||||
}
|
||||
reply
|
||||
.code(status)
|
||||
.header('x-request-id', requestId)
|
||||
.send({ error: { code, message, request_id: requestId } });
|
||||
}
|
||||
|
||||
private resolve(exception: unknown): Resolved {
|
||||
if (exception instanceof ApiError) {
|
||||
return {
|
||||
status: exception.getStatus(),
|
||||
code: exception.code,
|
||||
message: exception.message,
|
||||
};
|
||||
}
|
||||
if (exception instanceof HttpException) {
|
||||
const status = exception.getStatus();
|
||||
const code = CODE_BY_STATUS[status] ?? 'internal_error';
|
||||
const message =
|
||||
extractMessage(exception.getResponse()) ?? exception.message;
|
||||
return { status, code, message };
|
||||
}
|
||||
const fastifyStatus = (exception as { statusCode?: unknown })
|
||||
?.statusCode;
|
||||
if (typeof fastifyStatus === 'number' && CODE_BY_STATUS[fastifyStatus]) {
|
||||
const message =
|
||||
exception instanceof Error ? exception.message : 'Invalid request';
|
||||
return {
|
||||
status: fastifyStatus,
|
||||
code: CODE_BY_STATUS[fastifyStatus]!,
|
||||
message,
|
||||
};
|
||||
}
|
||||
if (isDatabaseUnavailable(exception)) {
|
||||
return {
|
||||
status: 503,
|
||||
code: 'unavailable',
|
||||
message: 'database unavailable',
|
||||
};
|
||||
}
|
||||
return {
|
||||
status: 500,
|
||||
code: 'internal_error',
|
||||
message: 'Internal server error',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function extractMessage(response: unknown): string | null {
|
||||
if (typeof response === 'string') {
|
||||
return response;
|
||||
}
|
||||
if (
|
||||
typeof response === 'object' &&
|
||||
response !== null &&
|
||||
'message' in response
|
||||
) {
|
||||
const value = (response as { message: unknown }).message;
|
||||
if (typeof value === 'string') {
|
||||
return value;
|
||||
}
|
||||
if (Array.isArray(value) && value.every((v) => typeof v === 'string')) {
|
||||
return value.join('; ');
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
import { randomUUID } from 'crypto';
|
||||
import { FastifyInstance, FastifyRequest } from 'fastify';
|
||||
import { hostname } from 'os';
|
||||
|
||||
const REQUEST_ID_HEADER = 'x-request-id';
|
||||
const INSTANCE_ID_HEADER = 'x-instance-id';
|
||||
|
||||
export const INSTANCE_ID = process.env.INSTANCE_ID ?? hostname();
|
||||
|
||||
declare module 'fastify' {
|
||||
interface FastifyRequest {
|
||||
requestId: string;
|
||||
}
|
||||
}
|
||||
|
||||
export function registerRequestId(instance: FastifyInstance): void {
|
||||
instance.addHook('onRequest', (request: FastifyRequest, reply, done) => {
|
||||
request.requestId = randomUUID();
|
||||
reply.header(REQUEST_ID_HEADER, request.requestId);
|
||||
reply.header(INSTANCE_ID_HEADER, INSTANCE_ID);
|
||||
done();
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
const UUID_PATTERN =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
||||
|
||||
export function validate(value: string): boolean {
|
||||
return UUID_PATTERN.test(value);
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
import { ValidationPipe } from '@nestjs/common';
|
||||
import { NestFastifyApplication } from '@nestjs/platform-fastify';
|
||||
import { registerUploadBody } from './attachments/upload-body';
|
||||
import { HttpExceptionFilter } from './common/filters/http-exception.filter';
|
||||
import { registerRequestId } from './common/interceptors/request-id';
|
||||
|
||||
export function configureApp(app: NestFastifyApplication): void {
|
||||
const instance = app.getHttpAdapter().getInstance();
|
||||
registerRequestId(instance);
|
||||
registerUploadBody(instance);
|
||||
|
||||
app.useGlobalPipes(
|
||||
new ValidationPipe({
|
||||
whitelist: true,
|
||||
forbidNonWhitelisted: true,
|
||||
transform: true,
|
||||
forbidUnknownValues: true,
|
||||
}),
|
||||
);
|
||||
app.useGlobalFilters(new HttpExceptionFilter());
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
import { Global, Module } from '@nestjs/common';
|
||||
import { PrismaService } from './prisma.service';
|
||||
|
||||
@Global()
|
||||
@Module({
|
||||
providers: [PrismaService],
|
||||
exports: [PrismaService],
|
||||
})
|
||||
export class DbModule {}
|
||||
@@ -0,0 +1,49 @@
|
||||
import { Injectable, OnModuleDestroy } from '@nestjs/common';
|
||||
import { Prisma, PrismaClient } from '@prisma/client';
|
||||
import { readFileSync } from 'fs';
|
||||
|
||||
const UNAVAILABLE_CODES = new Set(['P1001', 'P1002', 'P1008', 'P1017', 'P2024']);
|
||||
|
||||
@Injectable()
|
||||
export class PrismaService extends PrismaClient implements OnModuleDestroy {
|
||||
constructor() {
|
||||
super({ datasourceUrl: readDatabaseUrl() });
|
||||
}
|
||||
|
||||
async onModuleDestroy(): Promise<void> {
|
||||
await this.$disconnect();
|
||||
}
|
||||
}
|
||||
|
||||
function readDatabaseUrl(): string {
|
||||
const file = process.env.DATABASE_URL_FILE;
|
||||
const url = file ? readFileSync(file, 'utf8').trim() : process.env.DATABASE_URL;
|
||||
if (!url) {
|
||||
throw new Error('DATABASE_URL_FILE or DATABASE_URL must be set');
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
export function isUniqueViolation(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof Prisma.PrismaClientKnownRequestError &&
|
||||
error.code === 'P2002'
|
||||
);
|
||||
}
|
||||
|
||||
export function isForeignKeyViolation(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof Prisma.PrismaClientKnownRequestError &&
|
||||
error.code === 'P2003'
|
||||
);
|
||||
}
|
||||
|
||||
export function isDatabaseUnavailable(error: unknown): boolean {
|
||||
if (error instanceof Prisma.PrismaClientInitializationError) {
|
||||
return true;
|
||||
}
|
||||
return (
|
||||
error instanceof Prisma.PrismaClientKnownRequestError &&
|
||||
UNAVAILABLE_CODES.has(error.code)
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,351 @@
|
||||
const ALICE_ID = '11111111-1111-4111-8111-111111111111';
|
||||
const BOB_ID = '22222222-2222-4222-8222-222222222222';
|
||||
const CHAT_ID = '33333333-3333-4333-8333-333333333333';
|
||||
const MESSAGE_ID = '44444444-4444-4444-8444-444444444444';
|
||||
const CURSOR =
|
||||
'eyJjaGF0SWQiOiIzMzMzMzMzMy0zMzMzLTQzMzMtODMzMy0zMzMzMzMzMzMzMzMiLCJpZCI6IjQ0NDQ0NDQ0LTQ0NDQtNDQ0NC04NDQ0LTQ0NDQ0NDQ0NDQ0NCJ9';
|
||||
|
||||
const ALICE = {
|
||||
id: ALICE_ID,
|
||||
username: 'alice',
|
||||
display_name: 'Алиса',
|
||||
created_at: '2026-09-01T12:00:00.000Z',
|
||||
};
|
||||
|
||||
const CHAT = {
|
||||
id: CHAT_ID,
|
||||
title: 'Проект по ВНП',
|
||||
member_ids: [ALICE_ID, BOB_ID],
|
||||
created_at: '2026-09-01T12:01:00.000Z',
|
||||
};
|
||||
|
||||
const MESSAGE = {
|
||||
id: MESSAGE_ID,
|
||||
chat_id: CHAT_ID,
|
||||
sender_id: ALICE_ID,
|
||||
text: 'Привет, Боб! 👋',
|
||||
created_at: '2026-09-01T12:02:00.000Z',
|
||||
attachment_ids: [],
|
||||
};
|
||||
|
||||
export interface Example {
|
||||
summary: string;
|
||||
value: unknown;
|
||||
}
|
||||
|
||||
export interface OperationExamples {
|
||||
request?: Record<string, Example>;
|
||||
responses: Record<string, Record<string, Example>>;
|
||||
}
|
||||
|
||||
function example(summary: string, value: unknown): Example {
|
||||
return { summary, value };
|
||||
}
|
||||
|
||||
function error(code: string, message: string): unknown {
|
||||
return {
|
||||
error: {
|
||||
code,
|
||||
message,
|
||||
request_id: '9b2c6f1e-5d4a-4c3b-8a2f-1e0d9c8b7a6f',
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const UNAUTHORIZED = {
|
||||
missing: example(
|
||||
'Нет/неверный/истёкший/отозванный Bearer-токен',
|
||||
error('unauthorized', 'missing or invalid session'),
|
||||
),
|
||||
};
|
||||
|
||||
const UNAVAILABLE = {
|
||||
database: example(
|
||||
'PostgreSQL недоступен',
|
||||
error('unavailable', 'database unavailable'),
|
||||
),
|
||||
};
|
||||
|
||||
const CHAT_NOT_FOUND = {
|
||||
notFound: example(
|
||||
'Чата нет или вызывающий не участник (не 403)',
|
||||
error('not_found', 'chat not found'),
|
||||
),
|
||||
};
|
||||
|
||||
const INVALID_CHAT_ID = example(
|
||||
'chat_id не UUID',
|
||||
error('invalid_request', 'chatId must be a valid UUID'),
|
||||
);
|
||||
|
||||
export const GUIDE = `
|
||||
Учебный мессенджер, лабораторная 6: PostgreSQL, сессии, Redis-кэш сессий, вложения в S3 и их фоновая обработка.
|
||||
Документ — обязательный контракт \`contracts/openapi.json\`, дополненный примерами запросов и ответов.
|
||||
|
||||
**Как пройти сценарий в Swagger UI**
|
||||
|
||||
1. \`POST /api/v1/users\` — зарегистрируйте пользователей (примеры alice, bob, eve в списке Examples). Повторная регистрация того же username — 409, меняйте username.
|
||||
2. **Authorize → basicAuth**: username и password из шага 1 → \`POST /api/v1/auth/sessions\` → скопируйте \`token\`.
|
||||
3. **Authorize → bearerAuth**: вставьте token (без слова Bearer). Basic отправляется только на вход, остальные запросы идут с Bearer.
|
||||
4. \`GET /api/v1/users/me\` — свой id; id второго участника — из ответа его регистрации.
|
||||
5. \`POST /api/v1/chats\` с \`member_ids\` = [свой id, id собеседника] → \`chat_id\`.
|
||||
6. \`POST /api/v1/chats/{chat_id}/messages\` с заголовком \`Idempotency-Key\` (любая строка из \`A-Za-z0-9._:-\`, до 128 символов). Повтор с тем же ключом и телом вернёт то же сообщение, с другим телом — 409. Затем \`GET …/messages?limit=1\` и следующая страница с \`cursor\` = \`next_cursor\`.
|
||||
7. \`POST /api/v1/chats/{chat_id}/attachments?filename=photo.png\` с телом PNG/JPEG → 202 \`queued\`. \`GET /api/v1/attachments/{id}\` повторять, пока \`status\` не станет \`ready\` (до этого \`/content\` отвечает 409), затем \`/content?variant=thumbnail\`, \`crop\` или \`optimized\`.
|
||||
8. Чтобы действовать от другого пользователя: Logout в Authorize, вход им и новый token.
|
||||
9. \`DELETE /api/v1/auth/sessions/current\` — logout; тот же token дальше даёт 401.
|
||||
|
||||
Сессия живёт \`SESSION_TTL_SECONDS\` (по умолчанию 3600 с). Пароль — 12–128 символов Unicode, двоеточие допустимо.
|
||||
`;
|
||||
|
||||
export const OPERATION_EXAMPLES: Record<string, OperationExamples> = {
|
||||
'GET /health/live': {
|
||||
responses: {
|
||||
'200': { ok: example('Процесс жив', { status: 'ok' }) },
|
||||
},
|
||||
},
|
||||
'GET /health/ready': {
|
||||
responses: {
|
||||
'200': {
|
||||
ok: example('PostgreSQL доступен (Redis не критичен)', {
|
||||
status: 'ok',
|
||||
}),
|
||||
},
|
||||
'503': UNAVAILABLE,
|
||||
},
|
||||
},
|
||||
'POST /api/v1/users': {
|
||||
request: {
|
||||
alice: example('Алиса', {
|
||||
username: 'alice',
|
||||
display_name: 'Алиса',
|
||||
password: 'Alice-password-123',
|
||||
}),
|
||||
bob: example('Боб', {
|
||||
username: 'bob',
|
||||
display_name: 'Боб',
|
||||
password: 'Bob-password-123',
|
||||
}),
|
||||
eve: example('Ева (посторонняя)', {
|
||||
username: 'eve',
|
||||
display_name: 'Ева',
|
||||
password: 'Eve-password-123',
|
||||
}),
|
||||
colon: example('Пароль с двоеточием и эмодзи', {
|
||||
username: 'carol',
|
||||
display_name: 'Кэрол 🌸',
|
||||
password: 'пароль:с:двоеточием🔑',
|
||||
}),
|
||||
},
|
||||
responses: {
|
||||
'201': { created: example('Пользователь создан', ALICE) },
|
||||
'400': {
|
||||
username: example(
|
||||
'username не соответствует [a-z0-9_]{3,32}',
|
||||
error('invalid_request', 'username must match [a-z0-9_]{3,32}'),
|
||||
),
|
||||
password: example(
|
||||
'password короче 12 символов',
|
||||
error(
|
||||
'invalid_request',
|
||||
'password must be between 12 and 128 Unicode code points',
|
||||
),
|
||||
),
|
||||
extra: example(
|
||||
'Лишнее поле в JSON',
|
||||
error('invalid_request', 'property extra should not exist'),
|
||||
),
|
||||
},
|
||||
'409': {
|
||||
taken: example(
|
||||
'username уже занят',
|
||||
error('conflict', 'username is already taken'),
|
||||
),
|
||||
},
|
||||
'503': UNAVAILABLE,
|
||||
},
|
||||
},
|
||||
'POST /api/v1/auth/sessions': {
|
||||
responses: {
|
||||
'201': {
|
||||
created: example('Сессия создана', {
|
||||
token: 'q8Hn3vXl0ZbJ7mWcY2sRkT5uPaE9dFgVhN1oLxI4w6A',
|
||||
token_type: 'Bearer',
|
||||
expires_at: '2026-09-01T13:00:00.000Z',
|
||||
user: ALICE,
|
||||
}),
|
||||
},
|
||||
'401': {
|
||||
invalid: example(
|
||||
'Неверный пароль, неизвестный пользователь или нет Basic — ответ одинаковый',
|
||||
error('unauthorized', 'invalid credentials'),
|
||||
),
|
||||
},
|
||||
'503': UNAVAILABLE,
|
||||
},
|
||||
},
|
||||
'DELETE /api/v1/auth/sessions/current': {
|
||||
responses: {
|
||||
'401': UNAUTHORIZED,
|
||||
'503': {
|
||||
...UNAVAILABLE,
|
||||
cache: example(
|
||||
'Redis недоступен — отзыв не выполнен, повторите позже',
|
||||
error('unavailable', 'session cache unavailable'),
|
||||
),
|
||||
},
|
||||
},
|
||||
},
|
||||
'GET /api/v1/users/me': {
|
||||
responses: {
|
||||
'200': { me: example('Текущий пользователь из сессии', ALICE) },
|
||||
'401': UNAUTHORIZED,
|
||||
'503': UNAVAILABLE,
|
||||
},
|
||||
},
|
||||
'POST /api/v1/chats': {
|
||||
request: {
|
||||
pair: example('Чат Алисы и Боба', {
|
||||
title: CHAT.title,
|
||||
member_ids: CHAT.member_ids,
|
||||
}),
|
||||
solo: example('Чат только с собой', {
|
||||
title: 'Заметки',
|
||||
member_ids: [ALICE_ID],
|
||||
}),
|
||||
},
|
||||
responses: {
|
||||
'201': { created: example('Чат создан атомарно', CHAT) },
|
||||
'400': {
|
||||
noCaller: example(
|
||||
'Вызывающий не в member_ids',
|
||||
error('invalid_request', 'member_ids must include the caller'),
|
||||
),
|
||||
unknown: example(
|
||||
'Неизвестный участник — транзакция откатывается',
|
||||
error('invalid_request', 'member_ids contain an unknown user'),
|
||||
),
|
||||
duplicate: example(
|
||||
'Дубликаты в member_ids',
|
||||
error(
|
||||
'invalid_request',
|
||||
"All member_ids's elements must be unique",
|
||||
),
|
||||
),
|
||||
},
|
||||
'401': UNAUTHORIZED,
|
||||
'503': UNAVAILABLE,
|
||||
},
|
||||
},
|
||||
'GET /api/v1/chats': {
|
||||
responses: {
|
||||
'200': {
|
||||
page: example('Страница чатов с продолжением', {
|
||||
items: [CHAT],
|
||||
next_cursor: 'eyJpZCI6IjMzMzMzMzMzLTMzMzMtNDMzMy04MzMzLTMzMzMzMzMzMzMzMyJ9',
|
||||
}),
|
||||
last: example('Последняя страница', {
|
||||
items: [CHAT],
|
||||
next_cursor: null,
|
||||
}),
|
||||
},
|
||||
'400': {
|
||||
limit: example(
|
||||
'limit вне 1–100',
|
||||
error('invalid_request', 'limit must be between 1 and 100'),
|
||||
),
|
||||
cursor: example(
|
||||
'Испорченный или чужой cursor',
|
||||
error('invalid_request', 'invalid cursor'),
|
||||
),
|
||||
},
|
||||
'401': UNAUTHORIZED,
|
||||
'503': UNAVAILABLE,
|
||||
},
|
||||
},
|
||||
'GET /api/v1/chats/{chat_id}': {
|
||||
responses: {
|
||||
'200': { chat: example('Чат, где вызывающий — участник', CHAT) },
|
||||
'400': { chatId: INVALID_CHAT_ID },
|
||||
'401': UNAUTHORIZED,
|
||||
'404': CHAT_NOT_FOUND,
|
||||
'503': UNAVAILABLE,
|
||||
},
|
||||
},
|
||||
'POST /api/v1/chats/{chat_id}/messages': {
|
||||
request: {
|
||||
text: example('Обычное сообщение', { text: MESSAGE.text }),
|
||||
long: example('Максимум — 4000 символов Unicode', {
|
||||
text: 'Длинный текст…',
|
||||
}),
|
||||
},
|
||||
responses: {
|
||||
'201': {
|
||||
created: example('Отправитель берётся из сессии', MESSAGE),
|
||||
},
|
||||
'400': {
|
||||
blank: example(
|
||||
'Пустой или пробельный текст',
|
||||
error(
|
||||
'invalid_request',
|
||||
'text must not be empty or whitespace-only',
|
||||
),
|
||||
),
|
||||
sender: example(
|
||||
'sender_id в теле не принимается',
|
||||
error('invalid_request', 'property sender_id should not exist'),
|
||||
),
|
||||
chatId: INVALID_CHAT_ID,
|
||||
key: example(
|
||||
'Нет Idempotency-Key или неверный формат',
|
||||
error(
|
||||
'invalid_request',
|
||||
'Idempotency-Key header must match [A-Za-z0-9._:-]{1,128}',
|
||||
),
|
||||
),
|
||||
},
|
||||
'401': UNAUTHORIZED,
|
||||
'404': CHAT_NOT_FOUND,
|
||||
'409': {
|
||||
keyReused: example(
|
||||
'Тот же Idempotency-Key с другим телом',
|
||||
error(
|
||||
'conflict',
|
||||
'Idempotency-Key was already used with a different request',
|
||||
),
|
||||
),
|
||||
},
|
||||
'503': UNAVAILABLE,
|
||||
},
|
||||
},
|
||||
'GET /api/v1/chats/{chat_id}/messages': {
|
||||
responses: {
|
||||
'200': {
|
||||
page: example('Страница истории (limit=1) с cursor', {
|
||||
items: [MESSAGE],
|
||||
next_cursor: CURSOR,
|
||||
}),
|
||||
last: example('Последняя страница', {
|
||||
items: [MESSAGE],
|
||||
next_cursor: null,
|
||||
}),
|
||||
},
|
||||
'400': {
|
||||
otherChat: example(
|
||||
'cursor от другого чата',
|
||||
error('invalid_request', 'cursor belongs to a different chat'),
|
||||
),
|
||||
cursor: example(
|
||||
'Испорченный cursor или cursor несуществующего сообщения',
|
||||
error('invalid_request', 'invalid cursor'),
|
||||
),
|
||||
limit: example(
|
||||
'limit вне 1–100',
|
||||
error('invalid_request', 'limit must be between 1 and 100'),
|
||||
),
|
||||
},
|
||||
'401': UNAUTHORIZED,
|
||||
'404': CHAT_NOT_FOUND,
|
||||
'503': UNAVAILABLE,
|
||||
},
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,43 @@
|
||||
import { NestFastifyApplication } from '@nestjs/platform-fastify';
|
||||
import { OpenAPIObject, SwaggerModule } from '@nestjs/swagger';
|
||||
import { readFileSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { GUIDE, OPERATION_EXAMPLES } from './examples';
|
||||
|
||||
const CONTRACT_PATH = join(process.cwd(), 'contracts', 'openapi.json');
|
||||
|
||||
export function setupSwagger(app: NestFastifyApplication): void {
|
||||
SwaggerModule.setup('docs', app, loadDocument(), {
|
||||
customSiteTitle: 'Messenger lab6 API',
|
||||
swaggerOptions: {
|
||||
persistAuthorization: true,
|
||||
displayRequestDuration: true,
|
||||
tryItOutEnabled: true,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export function loadDocument(): OpenAPIObject {
|
||||
const document = JSON.parse(readFileSync(CONTRACT_PATH, 'utf8'));
|
||||
document.info.description = GUIDE;
|
||||
document.servers = [{ url: '/', description: 'Этот экземпляр API' }];
|
||||
for (const [key, examples] of Object.entries(OPERATION_EXAMPLES)) {
|
||||
const [method, path] = key.split(' ');
|
||||
const operation = document.paths[path]?.[method.toLowerCase()];
|
||||
if (!operation) {
|
||||
throw new Error(`examples for unknown operation ${key}`);
|
||||
}
|
||||
if (examples.request) {
|
||||
operation.requestBody.content['application/json'].examples =
|
||||
examples.request;
|
||||
}
|
||||
for (const [status, named] of Object.entries(examples.responses)) {
|
||||
const content = operation.responses[status]?.content;
|
||||
if (!content) {
|
||||
throw new Error(`examples for unknown response ${key} ${status}`);
|
||||
}
|
||||
content['application/json'].examples = named;
|
||||
}
|
||||
}
|
||||
return document;
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
import { Controller, Get } from '@nestjs/common';
|
||||
import { ApiError } from '../common/filters/api-error';
|
||||
import { PrismaService } from '../db/prisma.service';
|
||||
import { S3Service } from '../storage/s3.service';
|
||||
import { isDraining } from './shutdown';
|
||||
|
||||
@Controller('health')
|
||||
export class HealthController {
|
||||
constructor(
|
||||
private readonly prisma: PrismaService,
|
||||
private readonly s3: S3Service,
|
||||
) {}
|
||||
|
||||
@Get('live')
|
||||
live(): { status: 'ok' } {
|
||||
return { status: 'ok' };
|
||||
}
|
||||
|
||||
@Get('core')
|
||||
async core(): Promise<{ status: 'ok' }> {
|
||||
await this.assertCoreReady();
|
||||
return { status: 'ok' };
|
||||
}
|
||||
|
||||
@Get('ready')
|
||||
async ready(): Promise<{ status: 'ok' }> {
|
||||
await this.assertCoreReady();
|
||||
if (!(await this.s3.isAvailable())) {
|
||||
throw new ApiError('unavailable', 'object storage unavailable');
|
||||
}
|
||||
return { status: 'ok' };
|
||||
}
|
||||
|
||||
private async assertCoreReady(): Promise<void> {
|
||||
if (isDraining()) {
|
||||
throw new ApiError('unavailable', 'shutting down');
|
||||
}
|
||||
try {
|
||||
await this.prisma.$queryRaw`SELECT 1`;
|
||||
} catch {
|
||||
throw new ApiError('unavailable', 'database unavailable');
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { HealthController } from './health.controller';
|
||||
|
||||
@Module({
|
||||
controllers: [HealthController],
|
||||
})
|
||||
export class HealthModule {}
|
||||
@@ -0,0 +1,26 @@
|
||||
import { INestApplication, Logger } from '@nestjs/common';
|
||||
|
||||
let draining = false;
|
||||
|
||||
export function isDraining(): boolean {
|
||||
return draining;
|
||||
}
|
||||
|
||||
export function enableGracefulShutdown(app: INestApplication): void {
|
||||
const delayMs = Number(process.env.SHUTDOWN_DELAY_MS ?? 3000);
|
||||
const shutdown = (signal: string): void => {
|
||||
if (draining) {
|
||||
return;
|
||||
}
|
||||
draining = true;
|
||||
Logger.log(`${signal}: readiness → 503, closing in ${delayMs} ms`, 'Shutdown');
|
||||
setTimeout(() => {
|
||||
app.close().then(
|
||||
() => process.exit(0),
|
||||
() => process.exit(1),
|
||||
);
|
||||
}, delayMs);
|
||||
};
|
||||
process.once('SIGTERM', () => shutdown('SIGTERM'));
|
||||
process.once('SIGINT', () => shutdown('SIGINT'));
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
import { validate as isUuid } from '../common/uuid';
|
||||
|
||||
export const PIPELINE_VERSION = 1;
|
||||
export const JOB_TYPE = 'attachment.process.v1';
|
||||
|
||||
export interface JobMessage {
|
||||
schema_version: 1;
|
||||
job_id: string;
|
||||
type: typeof JOB_TYPE;
|
||||
attachment_id: string;
|
||||
pipeline_version: number;
|
||||
attempt: number;
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
const FIELDS = [
|
||||
'schema_version',
|
||||
'job_id',
|
||||
'type',
|
||||
'attachment_id',
|
||||
'pipeline_version',
|
||||
'attempt',
|
||||
'created_at',
|
||||
];
|
||||
|
||||
export function jobMessage(
|
||||
job: { id: string; attachmentId: string; pipelineVersion: number; attempt: number },
|
||||
createdAt: Date,
|
||||
): JobMessage {
|
||||
return {
|
||||
schema_version: 1,
|
||||
job_id: job.id,
|
||||
type: JOB_TYPE,
|
||||
attachment_id: job.attachmentId,
|
||||
pipeline_version: job.pipelineVersion,
|
||||
attempt: job.attempt,
|
||||
created_at: createdAt.toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
export function parseJobMessage(raw: Buffer): JobMessage | null {
|
||||
let value: any;
|
||||
try {
|
||||
value = JSON.parse(raw.toString('utf8'));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (
|
||||
typeof value !== 'object' ||
|
||||
value === null ||
|
||||
Array.isArray(value) ||
|
||||
Object.keys(value).some((key) => !FIELDS.includes(key)) ||
|
||||
value.schema_version !== 1 ||
|
||||
value.type !== JOB_TYPE ||
|
||||
typeof value.job_id !== 'string' ||
|
||||
!isUuid(value.job_id) ||
|
||||
typeof value.attachment_id !== 'string' ||
|
||||
!isUuid(value.attachment_id) ||
|
||||
!Number.isInteger(value.pipeline_version) ||
|
||||
value.pipeline_version < 1 ||
|
||||
!Number.isInteger(value.attempt) ||
|
||||
value.attempt < 1 ||
|
||||
typeof value.created_at !== 'string' ||
|
||||
Number.isNaN(Date.parse(value.created_at))
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return value as JobMessage;
|
||||
}
|
||||
@@ -0,0 +1,257 @@
|
||||
import { AttachmentJob, Prisma, PrismaClient } from '@prisma/client';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { jobMessage, PIPELINE_VERSION } from './job-message';
|
||||
|
||||
type Tx = Prisma.TransactionClient;
|
||||
|
||||
export const RETRY_EXHAUSTED = 'retry_exhausted';
|
||||
|
||||
export function maxAttempts(): number {
|
||||
return Number(process.env.JOB_MAX_ATTEMPTS ?? 5);
|
||||
}
|
||||
|
||||
export function backoffMs(attempt: number, random = Math.random): number {
|
||||
const base = Number(process.env.JOB_RETRY_BASE_MS ?? 1000);
|
||||
const cap = Number(process.env.JOB_RETRY_MAX_MS ?? 30000);
|
||||
const delay = Math.min(cap, base * 2 ** (attempt - 1));
|
||||
return Math.round(delay / 2 + random() * (delay / 2));
|
||||
}
|
||||
|
||||
export async function enqueueJob(
|
||||
tx: Tx,
|
||||
attachmentId: string,
|
||||
now: Date,
|
||||
): Promise<AttachmentJob> {
|
||||
const job = await tx.attachmentJob.create({
|
||||
data: {
|
||||
id: randomUUID(),
|
||||
attachmentId,
|
||||
pipelineVersion: PIPELINE_VERSION,
|
||||
status: 'queued',
|
||||
attempt: 1,
|
||||
maxAttempts: maxAttempts(),
|
||||
nextAttemptAt: now,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
},
|
||||
});
|
||||
await tx.jobEvent.create({
|
||||
data: { jobId: job.id, attempt: 1, event: 'enqueued', createdAt: now },
|
||||
});
|
||||
await publishLater(tx, job, now);
|
||||
return job;
|
||||
}
|
||||
|
||||
export async function scheduleRetry(
|
||||
tx: Tx,
|
||||
job: AttachmentJob,
|
||||
reason: string,
|
||||
now: Date,
|
||||
event = 'retry_scheduled',
|
||||
): Promise<AttachmentJob> {
|
||||
const delay = backoffMs(job.attempt);
|
||||
const next = await tx.attachmentJob.update({
|
||||
where: { id: job.id },
|
||||
data: {
|
||||
status: 'queued',
|
||||
attempt: job.attempt + 1,
|
||||
nextAttemptAt: new Date(now.getTime() + delay),
|
||||
leaseOwner: null,
|
||||
leaseUntil: null,
|
||||
lastError: reason,
|
||||
},
|
||||
});
|
||||
await tx.attachment.update({
|
||||
where: { id: job.attachmentId },
|
||||
data: { status: 'queued' },
|
||||
});
|
||||
await tx.jobEvent.create({
|
||||
data: {
|
||||
jobId: job.id,
|
||||
attempt: job.attempt,
|
||||
event,
|
||||
detail: `${reason}; next attempt ${next.attempt} in ${delay} ms`,
|
||||
createdAt: now,
|
||||
},
|
||||
});
|
||||
await publishLater(tx, next, next.nextAttemptAt);
|
||||
return next;
|
||||
}
|
||||
|
||||
export async function markFailed(
|
||||
tx: Tx,
|
||||
job: AttachmentJob,
|
||||
errorCode: string,
|
||||
reason: string,
|
||||
now: Date,
|
||||
): Promise<void> {
|
||||
await tx.attachmentJob.update({
|
||||
where: { id: job.id },
|
||||
data: {
|
||||
status: 'failed',
|
||||
errorCode,
|
||||
lastError: reason,
|
||||
leaseOwner: null,
|
||||
leaseUntil: null,
|
||||
finishedAt: now,
|
||||
},
|
||||
});
|
||||
await tx.attachment.update({
|
||||
where: { id: job.attachmentId },
|
||||
data: { status: 'failed', errorCode },
|
||||
});
|
||||
await tx.jobEvent.create({
|
||||
data: {
|
||||
jobId: job.id,
|
||||
attempt: job.attempt,
|
||||
event: 'failed',
|
||||
detail: `${errorCode}: ${reason}`,
|
||||
createdAt: now,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export async function retryOrFail(
|
||||
tx: Tx,
|
||||
job: AttachmentJob,
|
||||
reason: string,
|
||||
now: Date,
|
||||
event?: string,
|
||||
): Promise<'retry' | 'failed'> {
|
||||
if (job.attempt >= job.maxAttempts) {
|
||||
await markFailed(tx, job, RETRY_EXHAUSTED, reason, now);
|
||||
return 'failed';
|
||||
}
|
||||
await scheduleRetry(tx, job, reason, now, event);
|
||||
return 'retry';
|
||||
}
|
||||
|
||||
export async function recoverExpiredLeases(
|
||||
prisma: PrismaClient,
|
||||
now = new Date(),
|
||||
limit = 100,
|
||||
): Promise<number> {
|
||||
const expired = await prisma.attachmentJob.findMany({
|
||||
where: { status: 'processing', leaseUntil: { lt: now } },
|
||||
orderBy: { leaseUntil: 'asc' },
|
||||
take: limit,
|
||||
});
|
||||
let recovered = 0;
|
||||
for (const candidate of expired) {
|
||||
await prisma.$transaction(async (tx) => {
|
||||
const taken = await tx.attachmentJob.updateMany({
|
||||
where: {
|
||||
id: candidate.id,
|
||||
status: 'processing',
|
||||
leaseOwner: candidate.leaseOwner,
|
||||
leaseUntil: { lt: now },
|
||||
},
|
||||
data: { leaseOwner: null, leaseUntil: null, status: 'queued' },
|
||||
});
|
||||
if (taken.count === 0) {
|
||||
return;
|
||||
}
|
||||
await retryOrFail(tx, candidate, 'lease expired', now, 'lease_expired');
|
||||
recovered++;
|
||||
});
|
||||
}
|
||||
return recovered;
|
||||
}
|
||||
|
||||
export async function republishStale(
|
||||
prisma: PrismaClient,
|
||||
staleMs: number,
|
||||
now = new Date(),
|
||||
limit = 100,
|
||||
): Promise<number> {
|
||||
const cutoff = new Date(now.getTime() - staleMs);
|
||||
const stale = await prisma.$queryRaw<AttachmentJob[]>`
|
||||
SELECT j.id, j.attachment_id AS "attachmentId", j.pipeline_version AS "pipelineVersion", j.attempt
|
||||
FROM attachment_jobs j
|
||||
WHERE j.status = 'queued'
|
||||
AND j.next_attempt_at < ${cutoff}
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM outbox o
|
||||
WHERE o.job_id = j.id AND (o.published_at IS NULL OR o.published_at > ${cutoff})
|
||||
)
|
||||
ORDER BY j.next_attempt_at
|
||||
LIMIT ${limit}`;
|
||||
for (const job of stale) {
|
||||
await prisma.$transaction(async (tx) => {
|
||||
await tx.jobEvent.create({
|
||||
data: { jobId: job.id, attempt: job.attempt, event: 'republished', createdAt: now },
|
||||
});
|
||||
await publishLater(tx, job, now);
|
||||
});
|
||||
}
|
||||
return stale.length;
|
||||
}
|
||||
|
||||
export type ReplayResult =
|
||||
| 'replayed'
|
||||
| 'already_ready'
|
||||
| 'not_failed'
|
||||
| 'source_missing'
|
||||
| 'not_found';
|
||||
|
||||
export async function replayJob(
|
||||
prisma: PrismaClient,
|
||||
jobId: string,
|
||||
actor: string,
|
||||
now = new Date(),
|
||||
): Promise<ReplayResult> {
|
||||
return prisma.$transaction(async (tx) => {
|
||||
const job = await tx.attachmentJob.findUnique({ where: { id: jobId } });
|
||||
if (!job) {
|
||||
return 'not_found';
|
||||
}
|
||||
if (job.status === 'ready') {
|
||||
return 'already_ready';
|
||||
}
|
||||
if (job.status !== 'failed') {
|
||||
return 'not_failed';
|
||||
}
|
||||
const source = await tx.attachment.updateMany({
|
||||
where: { id: job.attachmentId, status: 'failed', stagingKey: { not: null } },
|
||||
data: { status: 'queued', errorCode: null },
|
||||
});
|
||||
if (source.count === 0) {
|
||||
return 'source_missing';
|
||||
}
|
||||
const taken = await tx.attachmentJob.updateMany({
|
||||
where: { id: jobId, status: 'failed' },
|
||||
data: {
|
||||
status: 'queued',
|
||||
attempt: 1,
|
||||
errorCode: null,
|
||||
nextAttemptAt: now,
|
||||
finishedAt: null,
|
||||
},
|
||||
});
|
||||
if (taken.count === 0) {
|
||||
throw new Error('job changed during replay');
|
||||
}
|
||||
await tx.jobEvent.create({
|
||||
data: {
|
||||
jobId,
|
||||
attempt: 1,
|
||||
event: 'replayed',
|
||||
detail: `by ${actor}; previous ${job.errorCode} after attempt ${job.attempt}`,
|
||||
createdAt: now,
|
||||
},
|
||||
});
|
||||
await publishLater(tx, { ...job, attempt: 1 }, now);
|
||||
return 'replayed';
|
||||
});
|
||||
}
|
||||
|
||||
async function publishLater(tx: Tx, job: AttachmentJob, availableAt: Date): Promise<void> {
|
||||
await tx.outboxEntry.create({
|
||||
data: {
|
||||
jobId: job.id,
|
||||
payload: jobMessage(job, availableAt) as unknown as Prisma.InputJsonObject,
|
||||
availableAt,
|
||||
createdAt: new Date(),
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { Attachment, AttachmentJob, PrismaClient } from '@prisma/client';
|
||||
import { createHash, randomUUID } from 'crypto';
|
||||
import {
|
||||
ObjectMissingError,
|
||||
ObjectTooLargeError,
|
||||
S3Service,
|
||||
} from '../storage/s3.service';
|
||||
import { JobMessage, PIPELINE_VERSION } from './job-message';
|
||||
import { markFailed, retryOrFail } from './job-store';
|
||||
import {
|
||||
InvalidImageError,
|
||||
RenderedVariant,
|
||||
RenderTimeoutError,
|
||||
renderVariants,
|
||||
} from './variants';
|
||||
|
||||
export type Outcome = 'ready' | 'retry' | 'failed' | 'skipped';
|
||||
|
||||
export interface ProcessorOptions {
|
||||
timeoutMs: number;
|
||||
leaseMs: number;
|
||||
maxBytes: number;
|
||||
maxPixels: number;
|
||||
}
|
||||
|
||||
export function processorOptions(): ProcessorOptions {
|
||||
const timeoutMs = Number(process.env.JOB_TIMEOUT_SECONDS ?? 60) * 1000;
|
||||
return {
|
||||
timeoutMs,
|
||||
leaseMs: Number(process.env.JOB_LEASE_SECONDS ?? timeoutMs / 1000 + 30) * 1000,
|
||||
maxBytes: Number(process.env.MAX_UPLOAD_BYTES ?? 10485760),
|
||||
maxPixels: Number(process.env.MAX_IMAGE_PIXELS ?? 40000000),
|
||||
};
|
||||
}
|
||||
|
||||
export interface ProcessorHooks {
|
||||
afterStore?: () => Promise<void>;
|
||||
afterCommit?: () => Promise<void>;
|
||||
}
|
||||
|
||||
class PermanentError extends Error {
|
||||
constructor(
|
||||
readonly code: string,
|
||||
reason: string,
|
||||
) {
|
||||
super(reason);
|
||||
}
|
||||
}
|
||||
|
||||
class TimeoutError extends Error {}
|
||||
|
||||
export class JobProcessor {
|
||||
private readonly logger = new Logger('JobProcessor');
|
||||
|
||||
constructor(
|
||||
private readonly prisma: PrismaClient,
|
||||
private readonly s3: S3Service,
|
||||
private readonly options: ProcessorOptions = processorOptions(),
|
||||
private readonly hooks: ProcessorHooks = {},
|
||||
) {}
|
||||
|
||||
async handle(message: JobMessage): Promise<Outcome> {
|
||||
const job = await this.prisma.attachmentJob.findUnique({
|
||||
where: { id: message.job_id },
|
||||
include: { attachment: true },
|
||||
});
|
||||
if (
|
||||
!job ||
|
||||
job.attachmentId !== message.attachment_id ||
|
||||
job.pipelineVersion !== message.pipeline_version ||
|
||||
job.pipelineVersion !== PIPELINE_VERSION
|
||||
) {
|
||||
this.logger.warn(`job ${message.job_id}: unknown or mismatched, dropped`);
|
||||
return 'skipped';
|
||||
}
|
||||
const lease = await this.claim(job);
|
||||
if (!lease) {
|
||||
if (job.status === 'ready') {
|
||||
await this.removeStaging(job.attachment);
|
||||
}
|
||||
this.logger.log(`job ${job.id}: ${job.status}, not due or leased elsewhere, skipped`);
|
||||
return 'skipped';
|
||||
}
|
||||
let variants: RenderedVariant[];
|
||||
try {
|
||||
variants = await this.withTimeout(this.produce(job.attachment));
|
||||
} catch (error) {
|
||||
return this.recordFailure(lease, error);
|
||||
}
|
||||
await this.hooks.afterStore?.();
|
||||
return this.commit(lease, job.attachment, variants);
|
||||
}
|
||||
|
||||
private async claim(job: AttachmentJob): Promise<AttachmentJob | null> {
|
||||
const now = new Date();
|
||||
const owner = randomUUID();
|
||||
return this.prisma.$transaction(async (tx) => {
|
||||
const taken = await tx.attachmentJob.updateMany({
|
||||
where: { id: job.id, status: 'queued', nextAttemptAt: { lte: now } },
|
||||
data: {
|
||||
status: 'processing',
|
||||
leaseOwner: owner,
|
||||
leaseUntil: new Date(now.getTime() + this.options.leaseMs),
|
||||
},
|
||||
});
|
||||
if (taken.count === 0) {
|
||||
return null;
|
||||
}
|
||||
await tx.attachment.update({
|
||||
where: { id: job.attachmentId },
|
||||
data: { status: 'processing' },
|
||||
});
|
||||
const leased = await tx.attachmentJob.findUniqueOrThrow({ where: { id: job.id } });
|
||||
await tx.jobEvent.create({
|
||||
data: { jobId: job.id, attempt: leased.attempt, event: 'processing', createdAt: now },
|
||||
});
|
||||
return leased;
|
||||
});
|
||||
}
|
||||
|
||||
private async produce(attachment: Attachment): Promise<RenderedVariant[]> {
|
||||
if (!attachment.stagingKey) {
|
||||
throw new PermanentError('staging_missing', 'staging object was already removed');
|
||||
}
|
||||
let original: Buffer;
|
||||
try {
|
||||
original = await this.s3.getBuffer(attachment.stagingKey, this.options.maxBytes);
|
||||
} catch (error) {
|
||||
if (error instanceof ObjectMissingError) {
|
||||
throw new PermanentError('staging_missing', 'staging object not found');
|
||||
}
|
||||
if (error instanceof ObjectTooLargeError) {
|
||||
throw new PermanentError('invalid_image', 'staging object exceeds size limit');
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
const sha256 = createHash('sha256').update(original).digest('hex');
|
||||
if (original.length !== attachment.sizeBytes || sha256 !== attachment.sha256) {
|
||||
throw new PermanentError('checksum_mismatch', 'staging object differs from accepted upload');
|
||||
}
|
||||
let variants: RenderedVariant[];
|
||||
try {
|
||||
variants = await renderVariants(
|
||||
original,
|
||||
this.options.maxPixels,
|
||||
Math.ceil(this.options.timeoutMs / 1000),
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof InvalidImageError) {
|
||||
throw new PermanentError('invalid_image', 'image cannot be decoded');
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
await this.s3.putBuffer(attachment.objectKey, original, attachment.contentType, sha256);
|
||||
for (const variant of variants) {
|
||||
await this.s3.putBuffer(
|
||||
variantKey(attachment.id, variant),
|
||||
variant.body,
|
||||
variant.contentType,
|
||||
variant.sha256,
|
||||
);
|
||||
}
|
||||
return variants;
|
||||
}
|
||||
|
||||
private async commit(
|
||||
lease: AttachmentJob,
|
||||
attachment: Attachment,
|
||||
variants: RenderedVariant[],
|
||||
): Promise<Outcome> {
|
||||
const now = new Date();
|
||||
const committed = await this.prisma.$transaction(async (tx) => {
|
||||
const owned = await tx.attachmentJob.updateMany({
|
||||
where: { id: lease.id, status: 'processing', leaseOwner: lease.leaseOwner },
|
||||
data: {
|
||||
status: 'ready',
|
||||
leaseOwner: null,
|
||||
leaseUntil: null,
|
||||
lastError: null,
|
||||
finishedAt: now,
|
||||
},
|
||||
});
|
||||
if (owned.count === 0) {
|
||||
return false;
|
||||
}
|
||||
await tx.attachmentVariant.createMany({
|
||||
data: variants.map((variant) => ({
|
||||
attachmentId: attachment.id,
|
||||
pipelineVersion: PIPELINE_VERSION,
|
||||
name: variant.name,
|
||||
contentType: variant.contentType,
|
||||
objectKey: variantKey(attachment.id, variant),
|
||||
sizeBytes: variant.body.length,
|
||||
width: variant.width,
|
||||
height: variant.height,
|
||||
sha256: variant.sha256,
|
||||
createdAt: now,
|
||||
})),
|
||||
skipDuplicates: true,
|
||||
});
|
||||
await tx.attachment.update({
|
||||
where: { id: attachment.id },
|
||||
data: { status: 'ready', errorCode: null },
|
||||
});
|
||||
await tx.jobEvent.create({
|
||||
data: { jobId: lease.id, attempt: lease.attempt, event: 'ready', createdAt: now },
|
||||
});
|
||||
return true;
|
||||
});
|
||||
if (!committed) {
|
||||
this.logger.warn(`job ${lease.id}: lease lost before commit, result left to owner`);
|
||||
return 'skipped';
|
||||
}
|
||||
await this.hooks.afterCommit?.();
|
||||
await this.removeStaging(attachment);
|
||||
return 'ready';
|
||||
}
|
||||
|
||||
private async recordFailure(lease: AttachmentJob, error: unknown): Promise<Outcome> {
|
||||
const now = new Date();
|
||||
const reason =
|
||||
error instanceof PermanentError
|
||||
? error.message
|
||||
: error instanceof TimeoutError || error instanceof RenderTimeoutError
|
||||
? 'processing timed out'
|
||||
: `temporary failure: ${error instanceof Error ? error.name : 'unknown'}`;
|
||||
this.logger.warn(`job ${lease.id} attempt ${lease.attempt}: ${reason}`);
|
||||
return this.prisma.$transaction(async (tx) => {
|
||||
const owned = await tx.attachmentJob.count({
|
||||
where: { id: lease.id, status: 'processing', leaseOwner: lease.leaseOwner },
|
||||
});
|
||||
if (owned === 0) {
|
||||
return 'skipped' as const;
|
||||
}
|
||||
if (error instanceof PermanentError) {
|
||||
await markFailed(tx, lease, error.code, reason, now);
|
||||
return 'failed' as const;
|
||||
}
|
||||
return retryOrFail(tx, lease, reason, now);
|
||||
});
|
||||
}
|
||||
|
||||
private async removeStaging(attachment: Attachment): Promise<void> {
|
||||
if (!attachment.stagingKey) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await this.s3.delete(attachment.stagingKey);
|
||||
await this.prisma.attachment.update({
|
||||
where: { id: attachment.id },
|
||||
data: { stagingKey: null },
|
||||
});
|
||||
} catch {
|
||||
this.logger.warn(`staging cleanup postponed for ${attachment.id}`);
|
||||
}
|
||||
}
|
||||
|
||||
private withTimeout<T>(work: Promise<T>): Promise<T> {
|
||||
let timer: NodeJS.Timeout;
|
||||
const deadline = new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(() => reject(new TimeoutError()), this.options.timeoutMs);
|
||||
});
|
||||
return Promise.race([work, deadline]).finally(() => clearTimeout(timer));
|
||||
}
|
||||
}
|
||||
|
||||
export function variantKey(
|
||||
attachmentId: string,
|
||||
variant: Pick<RenderedVariant, 'name' | 'extension'>,
|
||||
): string {
|
||||
return `attachments/${attachmentId}/v${PIPELINE_VERSION}/${variant.name}.${variant.extension}`;
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
import { createHash } from 'crypto';
|
||||
import sharp, { Sharp } from 'sharp';
|
||||
|
||||
sharp.cache(false);
|
||||
sharp.concurrency(1);
|
||||
|
||||
const WHITE = { r: 255, g: 255, b: 255 };
|
||||
const JPEG_QUALITY = 82;
|
||||
const WEBP_QUALITY = 80;
|
||||
|
||||
export interface RenderedVariant {
|
||||
name: 'thumbnail' | 'crop' | 'optimized';
|
||||
contentType: string;
|
||||
extension: string;
|
||||
body: Buffer;
|
||||
width: number;
|
||||
height: number;
|
||||
sha256: string;
|
||||
}
|
||||
|
||||
export class InvalidImageError extends Error {
|
||||
constructor(reason: string) {
|
||||
super(reason);
|
||||
this.name = 'InvalidImageError';
|
||||
}
|
||||
}
|
||||
|
||||
export class RenderTimeoutError extends Error {
|
||||
constructor() {
|
||||
super('image processing exceeded its time limit');
|
||||
this.name = 'RenderTimeoutError';
|
||||
}
|
||||
}
|
||||
|
||||
export function fitInside(
|
||||
width: number,
|
||||
height: number,
|
||||
box: number,
|
||||
): { width: number; height: number } {
|
||||
if (width <= box && height <= box) {
|
||||
return { width, height };
|
||||
}
|
||||
if (width >= height) {
|
||||
return { width: box, height: Math.max(1, Math.floor((height * box) / width)) };
|
||||
}
|
||||
return { width: Math.max(1, Math.floor((width * box) / height)), height: box };
|
||||
}
|
||||
|
||||
export async function renderVariants(
|
||||
original: Buffer,
|
||||
maxPixels: number,
|
||||
timeoutSeconds = 0,
|
||||
): Promise<RenderedVariant[]> {
|
||||
const open = (): Sharp =>
|
||||
sharp(original, { failOn: 'error', limitInputPixels: maxPixels })
|
||||
.timeout({ seconds: timeoutSeconds })
|
||||
.rotate();
|
||||
try {
|
||||
const meta = await sharp(original, { limitInputPixels: maxPixels }).metadata();
|
||||
if (!meta.width || !meta.height) {
|
||||
throw new InvalidImageError('image has no dimensions');
|
||||
}
|
||||
const swapped = (meta.orientation ?? 1) >= 5;
|
||||
const width = swapped ? meta.height : meta.width;
|
||||
const height = swapped ? meta.width : meta.height;
|
||||
const thumb = fitInside(width, height, 256);
|
||||
const large = fitInside(width, height, 1280);
|
||||
const thumbnail = open()
|
||||
.resize(thumb.width, thumb.height, { fit: 'fill' })
|
||||
.flatten({ background: WHITE })
|
||||
.jpeg({ quality: JPEG_QUALITY });
|
||||
const crop = open()
|
||||
.resize(128, 128, { fit: 'cover', position: 'centre' })
|
||||
.flatten({ background: WHITE })
|
||||
.jpeg({ quality: JPEG_QUALITY });
|
||||
const optimized = open()
|
||||
.resize(large.width, large.height, { fit: 'fill' })
|
||||
.webp({ quality: WEBP_QUALITY, effort: 4 });
|
||||
return [
|
||||
await finish('thumbnail', 'image/jpeg', 'jpg', thumbnail),
|
||||
await finish('crop', 'image/jpeg', 'jpg', crop),
|
||||
await finish('optimized', 'image/webp', 'webp', optimized),
|
||||
];
|
||||
} catch (error) {
|
||||
if (error instanceof InvalidImageError) {
|
||||
throw error;
|
||||
}
|
||||
if (error instanceof Error && /timeout/i.test(error.message)) {
|
||||
throw new RenderTimeoutError();
|
||||
}
|
||||
throw new InvalidImageError(error instanceof Error ? error.message : 'decode failed');
|
||||
}
|
||||
}
|
||||
|
||||
async function finish(
|
||||
name: RenderedVariant['name'],
|
||||
contentType: string,
|
||||
extension: string,
|
||||
pipeline: Sharp,
|
||||
): Promise<RenderedVariant> {
|
||||
const { data, info } = await pipeline.toBuffer({ resolveWithObject: true });
|
||||
return {
|
||||
name,
|
||||
contentType,
|
||||
extension,
|
||||
body: data,
|
||||
width: info.width,
|
||||
height: info.height,
|
||||
sha256: createHash('sha256').update(data).digest('hex'),
|
||||
};
|
||||
}
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { NestFactory } from '@nestjs/core';
|
||||
import {
|
||||
FastifyAdapter,
|
||||
NestFastifyApplication,
|
||||
} from '@nestjs/platform-fastify';
|
||||
import { AppModule } from './app.module';
|
||||
import { configureApp } from './configure-app';
|
||||
import { setupSwagger } from './docs/swagger';
|
||||
import { enableGracefulShutdown } from './health/shutdown';
|
||||
|
||||
async function bootstrap(): Promise<void> {
|
||||
const app = await NestFactory.create<NestFastifyApplication>(
|
||||
AppModule,
|
||||
new FastifyAdapter(),
|
||||
{
|
||||
bufferLogs: true,
|
||||
},
|
||||
);
|
||||
|
||||
configureApp(app);
|
||||
setupSwagger(app);
|
||||
enableGracefulShutdown(app);
|
||||
|
||||
const host = process.env.APP_HOST ?? '0.0.0.0';
|
||||
const port = Number(process.env.APP_PORT ?? 8080);
|
||||
await app.listen(port, host);
|
||||
Logger.log(`Listening on http://${host}:${port}`, 'Bootstrap');
|
||||
}
|
||||
|
||||
bootstrap().catch((error) => {
|
||||
Logger.error('Fatal error during bootstrap', error);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,41 @@
|
||||
import {
|
||||
ArrayMaxSize,
|
||||
IsArray,
|
||||
IsString,
|
||||
Validate,
|
||||
ValidateIf,
|
||||
ValidatorConstraint,
|
||||
ValidatorConstraintInterface,
|
||||
} from 'class-validator';
|
||||
import { CodePointLength } from '../../common/dto/code-point-length.validator';
|
||||
import { validate as isUuid } from '../../common/uuid';
|
||||
|
||||
@ValidatorConstraint({ name: 'uniqueUuidList' })
|
||||
class UniqueUuidList implements ValidatorConstraintInterface {
|
||||
validate(value: unknown): boolean {
|
||||
if (
|
||||
!Array.isArray(value) ||
|
||||
!value.every((id) => typeof id === 'string' && isUuid(id))
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
const normalized = (value as string[]).map((id) => id.toLowerCase());
|
||||
return new Set(normalized).size === normalized.length;
|
||||
}
|
||||
|
||||
defaultMessage(): string {
|
||||
return 'attachment_ids must contain unique UUIDs';
|
||||
}
|
||||
}
|
||||
|
||||
export class CreateMessageDto {
|
||||
@IsString()
|
||||
@CodePointLength(0, 4000)
|
||||
text!: string;
|
||||
|
||||
@ValidateIf((dto: CreateMessageDto) => dto.attachment_ids !== undefined)
|
||||
@IsArray()
|
||||
@ArrayMaxSize(10)
|
||||
@Validate(UniqueUuidList)
|
||||
attachment_ids?: string[];
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import { Message } from '../../common/domain';
|
||||
|
||||
export interface MessageResponse {
|
||||
id: string;
|
||||
chat_id: string;
|
||||
sender_id: string;
|
||||
text: string;
|
||||
created_at: string;
|
||||
attachment_ids: string[];
|
||||
}
|
||||
|
||||
export interface MessagePageResponse {
|
||||
items: MessageResponse[];
|
||||
next_cursor: string | null;
|
||||
}
|
||||
|
||||
export function toMessageResponse(message: Message): MessageResponse {
|
||||
return {
|
||||
id: message.id,
|
||||
chat_id: message.chatId,
|
||||
sender_id: message.senderId,
|
||||
text: message.text,
|
||||
created_at: message.createdAt,
|
||||
attachment_ids: message.attachmentIds,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
import {
|
||||
Body,
|
||||
Controller,
|
||||
Get,
|
||||
Headers,
|
||||
HttpCode,
|
||||
Param,
|
||||
Post,
|
||||
Query,
|
||||
UseGuards,
|
||||
} from '@nestjs/common';
|
||||
import { AuthGuard } from '../auth/auth.guard';
|
||||
import { CurrentUser } from '../auth/current-user.decorator';
|
||||
import { parseLimit, rawCursorParam } from '../common/dto/pagination-query.dto';
|
||||
import { User } from '../common/domain';
|
||||
import { ApiError } from '../common/filters/api-error';
|
||||
import { CreateMessageDto } from './dto/create-message.dto';
|
||||
import {
|
||||
MessagePageResponse,
|
||||
MessageResponse,
|
||||
toMessageResponse,
|
||||
} from './dto/message.response';
|
||||
import { MessagesService } from './messages.service';
|
||||
|
||||
@Controller('api/v1/chats/:chatId/messages')
|
||||
@UseGuards(AuthGuard)
|
||||
export class MessagesController {
|
||||
constructor(private readonly messages: MessagesService) {}
|
||||
|
||||
@Post()
|
||||
@HttpCode(201)
|
||||
async create(
|
||||
@Param('chatId') chatId: string,
|
||||
@Body() dto: CreateMessageDto,
|
||||
@Headers('idempotency-key') keyRaw: unknown,
|
||||
@CurrentUser() user: User,
|
||||
): Promise<MessageResponse> {
|
||||
const key = parseIdempotencyKey(keyRaw);
|
||||
const message = await this.messages.create(
|
||||
chatId,
|
||||
user.id,
|
||||
dto.text,
|
||||
(dto.attachment_ids ?? []).map((id) => id.toLowerCase()),
|
||||
key,
|
||||
);
|
||||
return toMessageResponse(message);
|
||||
}
|
||||
|
||||
@Get()
|
||||
async list(
|
||||
@Param('chatId') chatId: string,
|
||||
@CurrentUser() user: User,
|
||||
@Query('limit') limitRaw: unknown,
|
||||
@Query('cursor') cursorRaw: unknown,
|
||||
): Promise<MessagePageResponse> {
|
||||
const limit = parseLimit(limitRaw);
|
||||
const cursor = rawCursorParam(cursorRaw);
|
||||
return this.messages.list(chatId, user.id, limit, cursor);
|
||||
}
|
||||
}
|
||||
|
||||
function parseIdempotencyKey(raw: unknown): string {
|
||||
if (typeof raw !== 'string' || !/^[A-Za-z0-9._:-]{1,128}$/.test(raw)) {
|
||||
throw new ApiError(
|
||||
'invalid_request',
|
||||
'Idempotency-Key header must match [A-Za-z0-9._:-]{1,128}',
|
||||
);
|
||||
}
|
||||
return raw;
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { MessagesController } from './messages.controller';
|
||||
import { MessagesService } from './messages.service';
|
||||
|
||||
@Module({
|
||||
controllers: [MessagesController],
|
||||
providers: [MessagesService],
|
||||
})
|
||||
export class MessagesModule {}
|
||||
@@ -0,0 +1,198 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { Prisma } from '@prisma/client';
|
||||
import { createHash } from 'crypto';
|
||||
import { assertChatMember } from '../chats/chat-access';
|
||||
import { codePointLength } from '../common/dto/code-point-length.validator';
|
||||
import { decodeMessageCursor, encodeMessageCursor } from '../common/cursor';
|
||||
import { Message } from '../common/domain';
|
||||
import { ApiError } from '../common/filters/api-error';
|
||||
import { PrismaService } from '../db/prisma.service';
|
||||
import { MessagePageResponse, toMessageResponse } from './dto/message.response';
|
||||
|
||||
interface MessageRow {
|
||||
id: string;
|
||||
chatId: string;
|
||||
senderId: string;
|
||||
text: string;
|
||||
createdAt: Date;
|
||||
attachmentIds: string[];
|
||||
}
|
||||
|
||||
const ATTACHMENT_LINKS = {
|
||||
attachments: {
|
||||
orderBy: { position: 'asc' },
|
||||
select: { attachmentId: true },
|
||||
},
|
||||
} satisfies Prisma.MessageInclude;
|
||||
|
||||
type MessageWithLinks = Prisma.MessageGetPayload<{
|
||||
include: typeof ATTACHMENT_LINKS;
|
||||
}>;
|
||||
|
||||
class KeyAlreadyUsed extends Error {}
|
||||
|
||||
@Injectable()
|
||||
export class MessagesService {
|
||||
private readonly keyTtlMs =
|
||||
Number(process.env.IDEMPOTENCY_TTL_SECONDS ?? 86400) * 1000;
|
||||
|
||||
constructor(private readonly prisma: PrismaService) {}
|
||||
|
||||
async create(
|
||||
chatId: string,
|
||||
callerId: string,
|
||||
text: string,
|
||||
attachmentIds: string[],
|
||||
key: string,
|
||||
): Promise<Message> {
|
||||
await this.assertMember(chatId, callerId);
|
||||
if (codePointLength(text.trim()) === 0 && attachmentIds.length === 0) {
|
||||
throw new ApiError(
|
||||
'invalid_request',
|
||||
'text must not be empty or whitespace-only without attachments',
|
||||
);
|
||||
}
|
||||
await this.assertAttachmentsInChat(chatId, attachmentIds);
|
||||
const scope = { userId: callerId, chatId, key };
|
||||
const requestHash = createHash('sha256')
|
||||
.update(JSON.stringify({ text, attachment_ids: attachmentIds }))
|
||||
.digest('hex');
|
||||
try {
|
||||
return await this.prisma.$transaction(async (tx) => {
|
||||
await tx.idempotencyKey.deleteMany({
|
||||
where: {
|
||||
...scope,
|
||||
createdAt: { lt: new Date(Date.now() - this.keyTtlMs) },
|
||||
},
|
||||
});
|
||||
const row = await tx.message.create({
|
||||
data: {
|
||||
chatId,
|
||||
senderId: callerId,
|
||||
text,
|
||||
attachments: {
|
||||
create: attachmentIds.map((attachmentId, position) => ({
|
||||
attachmentId,
|
||||
position,
|
||||
})),
|
||||
},
|
||||
},
|
||||
include: ATTACHMENT_LINKS,
|
||||
});
|
||||
const { count } = await tx.idempotencyKey.createMany({
|
||||
data: [{ ...scope, requestHash, messageId: row.id }],
|
||||
skipDuplicates: true,
|
||||
});
|
||||
if (count === 0) {
|
||||
throw new KeyAlreadyUsed();
|
||||
}
|
||||
return linkedMessage(row);
|
||||
});
|
||||
} catch (error) {
|
||||
if (!(error instanceof KeyAlreadyUsed)) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
const stored = await this.prisma.idempotencyKey.findUnique({
|
||||
where: { userId_chatId_key: scope },
|
||||
include: { message: { include: ATTACHMENT_LINKS } },
|
||||
});
|
||||
if (!stored || stored.requestHash !== requestHash) {
|
||||
throw new ApiError(
|
||||
'conflict',
|
||||
'Idempotency-Key was already used with a different request',
|
||||
);
|
||||
}
|
||||
return linkedMessage(stored.message);
|
||||
}
|
||||
|
||||
async list(
|
||||
chatId: string,
|
||||
callerId: string,
|
||||
limit: number,
|
||||
rawCursor: string | null,
|
||||
): Promise<MessagePageResponse> {
|
||||
await this.assertMember(chatId, callerId);
|
||||
let after = Prisma.empty;
|
||||
if (rawCursor !== null) {
|
||||
const cursor = decodeMessageCursor(rawCursor);
|
||||
if (cursor && cursor.chatId !== chatId) {
|
||||
throw new ApiError(
|
||||
'invalid_request',
|
||||
'cursor belongs to a different chat',
|
||||
);
|
||||
}
|
||||
const anchor = cursor
|
||||
? await this.prisma.message.findFirst({
|
||||
where: { id: cursor.id, chatId },
|
||||
select: { id: true, createdAt: true },
|
||||
})
|
||||
: null;
|
||||
if (!anchor) {
|
||||
throw new ApiError('invalid_request', 'invalid cursor');
|
||||
}
|
||||
after = Prisma.sql`AND (created_at, id) > (${anchor.createdAt}::timestamptz, ${anchor.id}::uuid)`;
|
||||
}
|
||||
const rows = await this.prisma.$queryRaw<MessageRow[]>`
|
||||
SELECT id, chat_id AS "chatId", sender_id AS "senderId", text, created_at AS "createdAt",
|
||||
ARRAY(
|
||||
SELECT attachment_id::text FROM message_attachments
|
||||
WHERE message_id = messages.id
|
||||
ORDER BY position
|
||||
) AS "attachmentIds"
|
||||
FROM messages
|
||||
WHERE chat_id = ${chatId}::uuid ${after}
|
||||
ORDER BY created_at, id
|
||||
LIMIT ${limit + 1}`;
|
||||
const items = rows.slice(0, limit).map(toMessage);
|
||||
const last = items[items.length - 1];
|
||||
return {
|
||||
items: items.map(toMessageResponse),
|
||||
next_cursor:
|
||||
rows.length > limit
|
||||
? encodeMessageCursor({ chatId, id: last.id })
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
private assertMember(chatId: string, callerId: string): Promise<void> {
|
||||
return assertChatMember(this.prisma, chatId, callerId);
|
||||
}
|
||||
|
||||
private async assertAttachmentsInChat(
|
||||
chatId: string,
|
||||
attachmentIds: string[],
|
||||
): Promise<void> {
|
||||
if (attachmentIds.length === 0) {
|
||||
return;
|
||||
}
|
||||
const found = await this.prisma.attachment.findMany({
|
||||
where: { id: { in: attachmentIds }, chatId },
|
||||
select: { status: true },
|
||||
});
|
||||
if (found.length !== attachmentIds.length) {
|
||||
throw new ApiError('not_found', 'attachment not found');
|
||||
}
|
||||
if (found.some((row) => row.status === 'failed')) {
|
||||
throw new ApiError('conflict', 'failed attachment cannot be attached');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function linkedMessage(row: MessageWithLinks): Message {
|
||||
return toMessage({
|
||||
...row,
|
||||
attachmentIds: row.attachments.map((link) => link.attachmentId),
|
||||
});
|
||||
}
|
||||
|
||||
function toMessage(row: MessageRow): Message {
|
||||
return {
|
||||
id: row.id,
|
||||
chatId: row.chatId,
|
||||
senderId: row.senderId,
|
||||
text: row.text,
|
||||
createdAt: row.createdAt.toISOString(),
|
||||
attachmentIds: row.attachmentIds,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { reconcile } from './attachments/reconciler';
|
||||
import { PrismaService } from './db/prisma.service';
|
||||
import { S3Service } from './storage/s3.service';
|
||||
|
||||
const DEFAULT_GRACE_SECONDS = 3600;
|
||||
const DEFAULT_INTERVAL_SECONDS = 300;
|
||||
const DEFAULT_STAGING_RETENTION_SECONDS = 86400;
|
||||
|
||||
function option(name: string, fallback: number): number {
|
||||
const prefix = `--${name}=`;
|
||||
const raw = process.argv.find((arg) => arg.startsWith(prefix));
|
||||
const value = raw ? Number(raw.slice(prefix.length)) : fallback;
|
||||
if (!Number.isInteger(value) || value < 0) {
|
||||
throw new Error(`--${name} must be a non-negative integer`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
process.once('SIGTERM', () => process.exit(0));
|
||||
process.once('SIGINT', () => process.exit(0));
|
||||
const apply = process.argv.includes('--apply');
|
||||
const loop = process.argv.includes('--loop');
|
||||
const graceSeconds = option('grace-seconds', DEFAULT_GRACE_SECONDS);
|
||||
const intervalSeconds = option('interval-seconds', DEFAULT_INTERVAL_SECONDS);
|
||||
const stagingRetentionSeconds = option(
|
||||
'staging-retention-seconds',
|
||||
Number(process.env.STAGING_RETENTION_SECONDS ?? DEFAULT_STAGING_RETENTION_SECONDS),
|
||||
);
|
||||
const prisma = new PrismaService();
|
||||
const s3 = new S3Service();
|
||||
do {
|
||||
try {
|
||||
const report = await reconcile(prisma, s3, {
|
||||
apply,
|
||||
graceMs: graceSeconds * 1000,
|
||||
stagingRetentionMs: stagingRetentionSeconds * 1000,
|
||||
});
|
||||
process.stdout.write(`${JSON.stringify(report)}\n`);
|
||||
} catch (error) {
|
||||
const reason = error instanceof Error ? error.name : 'unknown error';
|
||||
Logger.error(`reconcile pass aborted: ${reason}`, 'Reconciler');
|
||||
process.exitCode = loop ? 0 : 1;
|
||||
}
|
||||
if (loop) {
|
||||
await new Promise((resolve) => setTimeout(resolve, intervalSeconds * 1000));
|
||||
}
|
||||
} while (loop);
|
||||
await prisma.$disconnect();
|
||||
s3.onModuleDestroy();
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
Logger.error(error instanceof Error ? error.message : 'fatal error', 'Reconciler');
|
||||
process.exit(1);
|
||||
});
|
||||
+129
@@ -0,0 +1,129 @@
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { ConfirmChannel } from 'amqplib';
|
||||
import { assertTopology, connectWithRetry, JOB_QUEUE, sleep } from './broker/broker';
|
||||
import { PrismaService } from './db/prisma.service';
|
||||
import { JobMessage } from './jobs/job-message';
|
||||
import { recoverExpiredLeases, republishStale } from './jobs/job-store';
|
||||
|
||||
const BATCH_SIZE = 100;
|
||||
const POLL_MS = Number(process.env.RELAY_POLL_MS ?? 500);
|
||||
const SWEEP_MS = Number(process.env.RELAY_SWEEP_MS ?? 5000);
|
||||
const STALE_MS = Number(process.env.JOB_STALE_SECONDS ?? 300) * 1000;
|
||||
const FAULT = process.env.RELAY_FAULT;
|
||||
|
||||
const logger = new Logger('OutboxRelay');
|
||||
const prisma = new PrismaService();
|
||||
let stopping = false;
|
||||
|
||||
async function publishBatch(channel: ConfirmChannel): Promise<number> {
|
||||
return prisma.$transaction(
|
||||
async (tx) => {
|
||||
const rows = await tx.$queryRaw<{ id: bigint; payload: JobMessage }[]>`
|
||||
SELECT id, payload FROM outbox
|
||||
WHERE published_at IS NULL AND available_at <= ${new Date()}
|
||||
ORDER BY id
|
||||
LIMIT ${BATCH_SIZE}
|
||||
FOR UPDATE SKIP LOCKED`;
|
||||
if (rows.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
for (const { payload } of rows) {
|
||||
channel.sendToQueue(JOB_QUEUE, Buffer.from(JSON.stringify(payload)), {
|
||||
persistent: true,
|
||||
contentType: 'application/json',
|
||||
messageId: payload.job_id,
|
||||
type: payload.type,
|
||||
});
|
||||
}
|
||||
await channel.waitForConfirms();
|
||||
logger.log(`published ${rows.length} job message(s), broker confirmed`);
|
||||
if (FAULT === 'exit-after-confirm') {
|
||||
logger.warn('fault injection: exit after broker confirm, before outbox is marked');
|
||||
process.exit(137);
|
||||
}
|
||||
await tx.outboxEntry.updateMany({
|
||||
where: { id: { in: rows.map((row) => row.id) } },
|
||||
data: { publishedAt: new Date(), publishAttempts: { increment: 1 } },
|
||||
});
|
||||
return rows.length;
|
||||
},
|
||||
{ timeout: 15000 },
|
||||
);
|
||||
}
|
||||
|
||||
async function sweep(): Promise<void> {
|
||||
const recovered = await recoverExpiredLeases(prisma);
|
||||
const republished = await republishStale(prisma, STALE_MS);
|
||||
if (recovered + republished > 0) {
|
||||
logger.log(`lease expired: ${recovered}, stale queued republished: ${republished}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function run(): Promise<void> {
|
||||
let lastSweep = 0;
|
||||
while (!stopping) {
|
||||
const connection = await connectWithRetry(logger, () => stopping);
|
||||
if (!connection) {
|
||||
break;
|
||||
}
|
||||
let closed = false;
|
||||
connection.on('close', () => {
|
||||
closed = true;
|
||||
});
|
||||
connection.on('error', () => {
|
||||
closed = true;
|
||||
});
|
||||
try {
|
||||
const channel = await connection.createConfirmChannel();
|
||||
channel.on('error', () => {
|
||||
closed = true;
|
||||
});
|
||||
channel.on('close', () => {
|
||||
closed = true;
|
||||
});
|
||||
await assertTopology(channel);
|
||||
while (!stopping && !closed) {
|
||||
let published = 0;
|
||||
try {
|
||||
published = await publishBatch(channel);
|
||||
if (Date.now() - lastSweep >= SWEEP_MS) {
|
||||
await sweep();
|
||||
lastSweep = Date.now();
|
||||
}
|
||||
} catch (error) {
|
||||
const reason = error instanceof Error ? error.name : 'unknown error';
|
||||
logger.error(`relay pass failed: ${reason}`);
|
||||
await sleep(1000);
|
||||
}
|
||||
if (published < BATCH_SIZE) {
|
||||
await sleep(POLL_MS);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
const reason = error instanceof Error ? error.name : 'unknown error';
|
||||
logger.error(`broker channel failed: ${reason}`);
|
||||
}
|
||||
await connection.close().catch(() => undefined);
|
||||
if (!stopping) {
|
||||
logger.warn('broker connection lost, reconnecting');
|
||||
await sleep(1000);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const signal of ['SIGTERM', 'SIGINT'] as const) {
|
||||
process.once(signal, () => {
|
||||
stopping = true;
|
||||
});
|
||||
}
|
||||
|
||||
run()
|
||||
.then(async () => {
|
||||
await prisma.$disconnect();
|
||||
logger.log('stopped');
|
||||
process.exit(0);
|
||||
})
|
||||
.catch((error) => {
|
||||
logger.error(error instanceof Error ? error.message : 'fatal error');
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,54 @@
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { validate as isUuid } from './common/uuid';
|
||||
import { PrismaService } from './db/prisma.service';
|
||||
import { PIPELINE_VERSION } from './jobs/job-message';
|
||||
import { replayJob } from './jobs/job-store';
|
||||
|
||||
const USAGE = 'usage: replay.js --list | <job_id|attachment_id>...';
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const args = process.argv.slice(2);
|
||||
if (args.length === 0 || (args[0] !== '--list' && args.some((arg) => !isUuid(arg)))) {
|
||||
throw new Error(USAGE);
|
||||
}
|
||||
const prisma = new PrismaService();
|
||||
try {
|
||||
if (args[0] === '--list') {
|
||||
const failed = await prisma.attachmentJob.findMany({
|
||||
where: { status: 'failed' },
|
||||
orderBy: { finishedAt: 'asc' },
|
||||
take: 100,
|
||||
});
|
||||
for (const job of failed) {
|
||||
process.stdout.write(
|
||||
`${JSON.stringify({
|
||||
job_id: job.id,
|
||||
attachment_id: job.attachmentId,
|
||||
error_code: job.errorCode,
|
||||
attempt: job.attempt,
|
||||
last_error: job.lastError,
|
||||
finished_at: job.finishedAt?.toISOString(),
|
||||
})}\n`,
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
const actor = process.env.OPERATOR ?? 'operator';
|
||||
for (const id of args) {
|
||||
const job = await prisma.attachmentJob.findFirst({
|
||||
where: {
|
||||
OR: [{ id }, { attachmentId: id, pipelineVersion: PIPELINE_VERSION }],
|
||||
},
|
||||
});
|
||||
const result = job ? await replayJob(prisma, job.id, actor) : 'not_found';
|
||||
process.stdout.write(`${JSON.stringify({ id, job_id: job?.id ?? null, result })}\n`);
|
||||
}
|
||||
} finally {
|
||||
await prisma.$disconnect();
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
Logger.error(error instanceof Error ? error.message : 'fatal error', 'Replay');
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,245 @@
|
||||
import {
|
||||
DeleteObjectCommand,
|
||||
GetObjectCommand,
|
||||
HeadBucketCommand,
|
||||
ListObjectsV2Command,
|
||||
PutObjectCommand,
|
||||
S3Client,
|
||||
} from '@aws-sdk/client-s3';
|
||||
import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common';
|
||||
import { createReadStream, readFileSync } from 'fs';
|
||||
import { Readable } from 'stream';
|
||||
import { ApiError } from '../common/filters/api-error';
|
||||
|
||||
const CONNECT_TIMEOUT_MS = 1000;
|
||||
const IDLE_TIMEOUT_MS = 5000;
|
||||
const OPERATION_TIMEOUT_MS = 20000;
|
||||
const HEALTH_TIMEOUT_MS = 1000;
|
||||
|
||||
export interface ObjectPage {
|
||||
objects: { key: string; lastModified: Date }[];
|
||||
nextToken?: string;
|
||||
}
|
||||
|
||||
export interface StoredFile {
|
||||
key: string;
|
||||
path: string;
|
||||
size: number;
|
||||
contentType: string;
|
||||
sha256: string;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class S3Service implements OnModuleDestroy {
|
||||
private readonly logger = new Logger('S3Service');
|
||||
private readonly client: S3Client;
|
||||
private readonly streamClient: S3Client;
|
||||
private readonly bucket = process.env.S3_BUCKET ?? 'messenger-attachments';
|
||||
|
||||
constructor() {
|
||||
const config = {
|
||||
endpoint: process.env.S3_ENDPOINT ?? 'http://s3:9000',
|
||||
region: process.env.S3_REGION ?? 'us-east-1',
|
||||
forcePathStyle: true,
|
||||
maxAttempts: 1,
|
||||
credentials: {
|
||||
accessKeyId: readSecret('S3_ACCESS_KEY_ID'),
|
||||
secretAccessKey: readSecret('S3_SECRET_ACCESS_KEY'),
|
||||
},
|
||||
};
|
||||
this.client = new S3Client({
|
||||
...config,
|
||||
requestHandler: {
|
||||
connectionTimeout: CONNECT_TIMEOUT_MS,
|
||||
requestTimeout: IDLE_TIMEOUT_MS,
|
||||
},
|
||||
});
|
||||
this.streamClient = new S3Client({
|
||||
...config,
|
||||
requestHandler: { connectionTimeout: CONNECT_TIMEOUT_MS },
|
||||
});
|
||||
}
|
||||
|
||||
async putFile(file: StoredFile): Promise<void> {
|
||||
await this.guard(`put ${file.key}`, () =>
|
||||
this.client.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: this.bucket,
|
||||
Key: file.key,
|
||||
Body: createReadStream(file.path),
|
||||
ContentLength: file.size,
|
||||
ContentType: file.contentType,
|
||||
ChecksumSHA256: Buffer.from(file.sha256, 'hex').toString('base64'),
|
||||
}),
|
||||
{ abortSignal: AbortSignal.timeout(OPERATION_TIMEOUT_MS) },
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
async putBuffer(
|
||||
key: string,
|
||||
body: Buffer,
|
||||
contentType: string,
|
||||
sha256: string,
|
||||
): Promise<void> {
|
||||
await this.guard(`put ${key}`, () =>
|
||||
this.client.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: this.bucket,
|
||||
Key: key,
|
||||
Body: body,
|
||||
ContentLength: body.length,
|
||||
ContentType: contentType,
|
||||
ChecksumSHA256: Buffer.from(sha256, 'hex').toString('base64'),
|
||||
}),
|
||||
{ abortSignal: AbortSignal.timeout(OPERATION_TIMEOUT_MS) },
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
async getBuffer(key: string, maxBytes: number): Promise<Buffer> {
|
||||
let body: Readable;
|
||||
try {
|
||||
const response = await this.client.send(
|
||||
new GetObjectCommand({ Bucket: this.bucket, Key: key }),
|
||||
{ abortSignal: AbortSignal.timeout(OPERATION_TIMEOUT_MS) },
|
||||
);
|
||||
if ((response.ContentLength ?? 0) > maxBytes) {
|
||||
throw new ObjectTooLargeError(key);
|
||||
}
|
||||
body = response.Body as Readable;
|
||||
} catch (error) {
|
||||
if (error instanceof ObjectTooLargeError) {
|
||||
throw error;
|
||||
}
|
||||
const name = error instanceof Error ? error.name : 'unknown error';
|
||||
this.logger.error(`get ${key} failed: ${name}`);
|
||||
if (name === 'NoSuchKey') {
|
||||
throw new ObjectMissingError(key);
|
||||
}
|
||||
throw new ApiError('unavailable', 'object storage unavailable');
|
||||
}
|
||||
const chunks: Buffer[] = [];
|
||||
let size = 0;
|
||||
try {
|
||||
for await (const chunk of body) {
|
||||
size += chunk.length;
|
||||
if (size > maxBytes) {
|
||||
body.destroy();
|
||||
throw new ObjectTooLargeError(key);
|
||||
}
|
||||
chunks.push(chunk);
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof ObjectTooLargeError) {
|
||||
throw error;
|
||||
}
|
||||
throw new ApiError('unavailable', 'object storage unavailable');
|
||||
}
|
||||
return Buffer.concat(chunks);
|
||||
}
|
||||
|
||||
async getStream(key: string): Promise<Readable> {
|
||||
const deadline = new AbortController();
|
||||
const timer = setTimeout(() => deadline.abort(), IDLE_TIMEOUT_MS);
|
||||
try {
|
||||
const { Body } = await this.streamClient.send(
|
||||
new GetObjectCommand({ Bucket: this.bucket, Key: key }),
|
||||
{ abortSignal: deadline.signal },
|
||||
);
|
||||
return Body as Readable;
|
||||
} catch (error) {
|
||||
const name = error instanceof Error ? error.name : 'unknown error';
|
||||
this.logger.error(`get ${key} failed: ${name}`);
|
||||
if (name === 'NoSuchKey') {
|
||||
throw new Error(`object ${key} is missing from storage`);
|
||||
}
|
||||
throw new ApiError('unavailable', 'object storage unavailable');
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
async isAvailable(): Promise<boolean> {
|
||||
try {
|
||||
await this.client.send(new HeadBucketCommand({ Bucket: this.bucket }), {
|
||||
abortSignal: AbortSignal.timeout(HEALTH_TIMEOUT_MS),
|
||||
});
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async listObjects(
|
||||
prefix: string,
|
||||
pageSize: number,
|
||||
token?: string,
|
||||
): Promise<ObjectPage> {
|
||||
const page = await this.guard(`list ${prefix}`, () =>
|
||||
this.client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: this.bucket,
|
||||
Prefix: prefix,
|
||||
MaxKeys: pageSize,
|
||||
ContinuationToken: token,
|
||||
}),
|
||||
{ abortSignal: AbortSignal.timeout(OPERATION_TIMEOUT_MS) },
|
||||
),
|
||||
);
|
||||
return {
|
||||
objects: (page.Contents ?? []).map((object) => ({
|
||||
key: object.Key!,
|
||||
lastModified: object.LastModified!,
|
||||
})),
|
||||
nextToken: page.NextContinuationToken,
|
||||
};
|
||||
}
|
||||
|
||||
async delete(key: string): Promise<void> {
|
||||
await this.guard(`delete ${key}`, () =>
|
||||
this.client.send(
|
||||
new DeleteObjectCommand({ Bucket: this.bucket, Key: key }),
|
||||
{ abortSignal: AbortSignal.timeout(OPERATION_TIMEOUT_MS) },
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
onModuleDestroy(): void {
|
||||
this.client.destroy();
|
||||
this.streamClient.destroy();
|
||||
}
|
||||
|
||||
private async guard<T>(action: string, run: () => Promise<T>): Promise<T> {
|
||||
try {
|
||||
return await run();
|
||||
} catch (error) {
|
||||
const reason = error instanceof Error ? error.name : 'unknown error';
|
||||
this.logger.error(`${action} failed: ${reason}`);
|
||||
throw new ApiError('unavailable', 'object storage unavailable');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class ObjectMissingError extends Error {
|
||||
constructor(key: string) {
|
||||
super(`object ${key} is missing from storage`);
|
||||
this.name = 'ObjectMissingError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ObjectTooLargeError extends Error {
|
||||
constructor(key: string) {
|
||||
super(`object ${key} exceeds size limit`);
|
||||
this.name = 'ObjectTooLargeError';
|
||||
}
|
||||
}
|
||||
|
||||
function readSecret(name: string): string {
|
||||
const file = process.env[`${name}_FILE`];
|
||||
const value = file ? readFileSync(file, 'utf8').trim() : process.env[name];
|
||||
if (!value) {
|
||||
throw new Error(`${name}_FILE or ${name} must be set`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
import { Global, Module } from '@nestjs/common';
|
||||
import { S3Service } from './s3.service';
|
||||
|
||||
@Global()
|
||||
@Module({
|
||||
providers: [S3Service],
|
||||
exports: [S3Service],
|
||||
})
|
||||
export class StorageModule {}
|
||||
@@ -0,0 +1,18 @@
|
||||
import { Matches } from 'class-validator';
|
||||
import { CodePointLength } from '../../common/dto/code-point-length.validator';
|
||||
import { NotBlank } from '../../common/dto/not-blank.validator';
|
||||
|
||||
export class CreateUserDto {
|
||||
@Matches(/^[a-z0-9_]{3,32}$/, {
|
||||
message: 'username must match [a-z0-9_]{3,32}',
|
||||
})
|
||||
username!: string;
|
||||
|
||||
@NotBlank()
|
||||
@CodePointLength(1, 100)
|
||||
display_name!: string;
|
||||
|
||||
@NotBlank()
|
||||
@CodePointLength(12, 128)
|
||||
password!: string;
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import { User } from '../../common/domain';
|
||||
|
||||
export interface UserResponse {
|
||||
id: string;
|
||||
username: string;
|
||||
display_name: string;
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
export function toUserResponse(user: User): UserResponse {
|
||||
return {
|
||||
id: user.id,
|
||||
username: user.username,
|
||||
display_name: user.displayName,
|
||||
created_at: user.createdAt,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
import {
|
||||
Body,
|
||||
Controller,
|
||||
Get,
|
||||
HttpCode,
|
||||
Post,
|
||||
UseGuards,
|
||||
} from '@nestjs/common';
|
||||
import { AuthGuard } from '../auth/auth.guard';
|
||||
import { CurrentUser } from '../auth/current-user.decorator';
|
||||
import { User } from '../common/domain';
|
||||
import { CreateUserDto } from './dto/create-user.dto';
|
||||
import { toUserResponse, UserResponse } from './dto/user.response';
|
||||
import { UsersService } from './users.service';
|
||||
|
||||
@Controller('api/v1/users')
|
||||
export class UsersController {
|
||||
constructor(private readonly users: UsersService) {}
|
||||
|
||||
@Post()
|
||||
@HttpCode(201)
|
||||
async create(@Body() dto: CreateUserDto): Promise<UserResponse> {
|
||||
const user = await this.users.register(
|
||||
dto.username,
|
||||
dto.display_name,
|
||||
dto.password,
|
||||
);
|
||||
return toUserResponse(user);
|
||||
}
|
||||
|
||||
@Get('me')
|
||||
@UseGuards(AuthGuard)
|
||||
me(@CurrentUser() user: User): UserResponse {
|
||||
return toUserResponse(user);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { UsersController } from './users.controller';
|
||||
import { UsersService } from './users.service';
|
||||
|
||||
@Module({
|
||||
controllers: [UsersController],
|
||||
providers: [UsersService],
|
||||
exports: [UsersService],
|
||||
})
|
||||
export class UsersModule {}
|
||||
@@ -0,0 +1,48 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { hashPassword } from '../auth/password';
|
||||
import { ApiError } from '../common/filters/api-error';
|
||||
import { User } from '../common/domain';
|
||||
import { PrismaService, isUniqueViolation } from '../db/prisma.service';
|
||||
|
||||
@Injectable()
|
||||
export class UsersService {
|
||||
constructor(private readonly prisma: PrismaService) {}
|
||||
|
||||
async register(
|
||||
username: string,
|
||||
displayName: string,
|
||||
password: string,
|
||||
): Promise<User> {
|
||||
const passwordHash = await hashPassword(password);
|
||||
try {
|
||||
const row = await this.prisma.user.create({
|
||||
data: { username, displayName: displayName.trim(), passwordHash },
|
||||
});
|
||||
return toUser(row);
|
||||
} catch (error) {
|
||||
if (isUniqueViolation(error)) {
|
||||
throw new ApiError('conflict', 'username is already taken');
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async findById(id: string): Promise<User | null> {
|
||||
const row = await this.prisma.user.findUnique({ where: { id } });
|
||||
return row ? toUser(row) : null;
|
||||
}
|
||||
}
|
||||
|
||||
export function toUser(row: {
|
||||
id: string;
|
||||
username: string;
|
||||
displayName: string;
|
||||
createdAt: Date;
|
||||
}): User {
|
||||
return {
|
||||
id: row.id,
|
||||
username: row.username,
|
||||
displayName: row.displayName,
|
||||
createdAt: row.createdAt.toISOString(),
|
||||
};
|
||||
}
|
||||
+137
@@ -0,0 +1,137 @@
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { Channel, ConsumeMessage } from 'amqplib';
|
||||
import { assertTopology, connectWithRetry, JOB_QUEUE, sleep } from './broker/broker';
|
||||
import { PrismaService } from './db/prisma.service';
|
||||
import { parseJobMessage } from './jobs/job-message';
|
||||
import { JobProcessor, ProcessorHooks, processorOptions } from './jobs/processor';
|
||||
import { S3Service } from './storage/s3.service';
|
||||
|
||||
const CONCURRENCY = Number(process.env.WORKER_CONCURRENCY ?? 2);
|
||||
const INFRA_RETRY_MS = Number(process.env.WORKER_INFRA_RETRY_MS ?? 5000);
|
||||
const DRAIN_TIMEOUT_MS = Number(process.env.WORKER_DRAIN_TIMEOUT_MS ?? 25000);
|
||||
const FAULT = process.env.WORKER_FAULT;
|
||||
|
||||
const logger = new Logger('Worker');
|
||||
const prisma = new PrismaService();
|
||||
const s3 = new S3Service();
|
||||
const processor = new JobProcessor(prisma, s3, processorOptions(), faultHooks());
|
||||
let stopping = false;
|
||||
let inFlight = 0;
|
||||
|
||||
function faultHooks(): ProcessorHooks {
|
||||
const crash = (point: string) => async () => {
|
||||
logger.warn(`fault injection: worker killed ${point}`);
|
||||
process.exit(137);
|
||||
};
|
||||
if (FAULT === 'exit-after-store') {
|
||||
return { afterStore: crash('after storing objects, before commit and ack') };
|
||||
}
|
||||
if (FAULT === 'exit-after-commit') {
|
||||
return { afterCommit: crash('after commit, before ack') };
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
async function handle(channel: Channel, delivery: ConsumeMessage): Promise<void> {
|
||||
inFlight++;
|
||||
try {
|
||||
const message = parseJobMessage(delivery.content);
|
||||
if (!message) {
|
||||
logger.warn('malformed job message moved to dead letter queue');
|
||||
channel.nack(delivery, false, false);
|
||||
return;
|
||||
}
|
||||
const outcome = await processor.handle(message);
|
||||
logger.log(
|
||||
`job ${message.job_id} attempt ${message.attempt}` +
|
||||
`${delivery.fields.redelivered ? ' (redelivered)' : ''}: ${outcome}`,
|
||||
);
|
||||
channel.ack(delivery);
|
||||
} catch (error) {
|
||||
const reason = error instanceof Error ? error.name : 'unknown error';
|
||||
logger.error(`job handling interrupted (${reason}), returning message after ${INFRA_RETRY_MS} ms`);
|
||||
await sleep(INFRA_RETRY_MS);
|
||||
try {
|
||||
channel.nack(delivery, false, true);
|
||||
} catch {
|
||||
logger.warn('channel closed, broker will redeliver unacked message');
|
||||
}
|
||||
} finally {
|
||||
inFlight--;
|
||||
}
|
||||
}
|
||||
|
||||
async function run(): Promise<void> {
|
||||
while (!stopping) {
|
||||
const connection = await connectWithRetry(logger, () => stopping);
|
||||
if (!connection) {
|
||||
break;
|
||||
}
|
||||
const closed = new Promise<void>((resolve) => {
|
||||
connection.on('close', () => resolve());
|
||||
connection.on('error', () => resolve());
|
||||
});
|
||||
try {
|
||||
const channel = await connection.createChannel();
|
||||
await assertTopology(channel);
|
||||
await channel.prefetch(CONCURRENCY);
|
||||
const { consumerTag } = await channel.consume(
|
||||
JOB_QUEUE,
|
||||
(delivery) => {
|
||||
if (delivery) {
|
||||
void handle(channel, delivery);
|
||||
}
|
||||
},
|
||||
{ noAck: false },
|
||||
);
|
||||
logger.log(`consuming ${JOB_QUEUE} with prefetch ${CONCURRENCY}`);
|
||||
const stop = new Promise<void>((resolve) => {
|
||||
const timer = setInterval(() => {
|
||||
if (stopping) {
|
||||
clearInterval(timer);
|
||||
resolve();
|
||||
}
|
||||
}, 200);
|
||||
void closed.then(() => {
|
||||
clearInterval(timer);
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
await stop;
|
||||
if (stopping) {
|
||||
await channel.cancel(consumerTag).catch(() => undefined);
|
||||
const deadline = Date.now() + DRAIN_TIMEOUT_MS;
|
||||
while (inFlight > 0 && Date.now() < deadline) {
|
||||
await sleep(100);
|
||||
}
|
||||
logger.log(`drained, ${inFlight} job(s) left unacked for redelivery`);
|
||||
}
|
||||
} catch (error) {
|
||||
const reason = error instanceof Error ? error.name : 'unknown error';
|
||||
logger.error(`broker channel failed: ${reason}`);
|
||||
}
|
||||
await connection.close().catch(() => undefined);
|
||||
if (!stopping) {
|
||||
logger.warn('broker connection lost, reconnecting');
|
||||
await sleep(1000);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const signal of ['SIGTERM', 'SIGINT'] as const) {
|
||||
process.once(signal, () => {
|
||||
stopping = true;
|
||||
});
|
||||
}
|
||||
|
||||
run()
|
||||
.then(async () => {
|
||||
await prisma.$disconnect();
|
||||
s3.onModuleDestroy();
|
||||
logger.log('stopped');
|
||||
process.exit(0);
|
||||
})
|
||||
.catch((error) => {
|
||||
logger.error(error instanceof Error ? error.message : 'fatal error');
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user