chore: port lab5 solution as lab6 base

This commit is contained in:
2026-11-18 18:40:47 +03:00
parent 470d9d6f1e
commit 98d5787545
105 changed files with 21898 additions and 0 deletions
+25
View File
@@ -0,0 +1,25 @@
import { Module } from '@nestjs/common';
import { AttachmentsModule } from './attachments/attachments.module';
import { AuthModule } from './auth/auth.module';
import { CacheModule } from './cache/cache.module';
import { ChatsModule } from './chats/chats.module';
import { DbModule } from './db/db.module';
import { HealthModule } from './health/health.module';
import { MessagesModule } from './messages/messages.module';
import { StorageModule } from './storage/storage.module';
import { UsersModule } from './users/users.module';
@Module({
imports: [
DbModule,
CacheModule,
StorageModule,
AuthModule,
HealthModule,
UsersModule,
ChatsModule,
MessagesModule,
AttachmentsModule,
],
})
export class AppModule {}
+79
View File
@@ -0,0 +1,79 @@
import {
Controller,
Get,
HttpCode,
Param,
Post,
Query,
Req,
Res,
UseGuards,
} from '@nestjs/common';
import { FastifyReply, FastifyRequest } from 'fastify';
import { Readable } from 'stream';
import { AuthGuard } from '../auth/auth.guard';
import { CurrentUser } from '../auth/current-user.decorator';
import { User } from '../common/domain';
import { AttachmentsService } from './attachments.service';
import {
AttachmentResponse,
toAttachmentResponse,
} from './dto/attachment.response';
@Controller()
@UseGuards(AuthGuard)
export class AttachmentsController {
constructor(private readonly attachments: AttachmentsService) {}
@Post('api/v1/chats/:chatId/attachments')
@HttpCode(202)
async upload(
@Param('chatId') chatId: string,
@Query('filename') filename: unknown,
@Req() request: FastifyRequest,
@CurrentUser() user: User,
): Promise<AttachmentResponse> {
const attachment = await this.attachments.upload({
chatId,
ownerId: user.id,
filename,
contentType: request.uploadContentType,
contentLength: request.headers['content-length'],
body: request.body as Readable | undefined,
});
return toAttachmentResponse(attachment);
}
@Get('api/v1/attachments/:attachmentId')
async metadata(
@Param('attachmentId') attachmentId: string,
@CurrentUser() user: User,
): Promise<AttachmentResponse> {
return toAttachmentResponse(
await this.attachments.getVisible(attachmentId, user.id),
);
}
@Get('api/v1/attachments/:attachmentId/content')
async content(
@Param('attachmentId') attachmentId: string,
@Query('variant') variant: unknown,
@CurrentUser() user: User,
@Res() reply: FastifyReply,
): Promise<void> {
const download = await this.attachments.openContent(
attachmentId,
user.id,
variant,
);
await reply
.header('content-type', download.contentType)
.header('content-length', download.sizeBytes)
.header('x-content-type-options', 'nosniff')
.header(
'content-disposition',
`inline; filename*=UTF-8''${encodeURIComponent(download.filename)}`,
)
.send(download.content);
}
}
+9
View File
@@ -0,0 +1,9 @@
import { Module } from '@nestjs/common';
import { AttachmentsController } from './attachments.controller';
import { AttachmentsService } from './attachments.service';
@Module({
controllers: [AttachmentsController],
providers: [AttachmentsService],
})
export class AttachmentsModule {}
+244
View File
@@ -0,0 +1,244 @@
import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
import { Attachment, AttachmentVariant, Prisma } from '@prisma/client';
import { randomUUID } from 'crypto';
import { mkdtemp, readdir, rm, stat } from 'fs/promises';
import { tmpdir } from 'os';
import { join } from 'path';
import { Readable } from 'stream';
import { assertChatMember } from '../chats/chat-access';
import { codePointLength } from '../common/dto/code-point-length.validator';
import { ApiError } from '../common/filters/api-error';
import { validate as isUuid } from '../common/uuid';
import { isDatabaseUnavailable, PrismaService } from '../db/prisma.service';
import { PIPELINE_VERSION } from '../jobs/job-message';
import { enqueueJob } from '../jobs/job-store';
import { S3Service } from '../storage/s3.service';
import { assertValidImage, isSupportedContentType } from './image-validation';
import { receiveBody } from './upload-body';
const TEMP_PREFIX = 'messenger-upload-';
const STALE_TEMP_MS = 60 * 60 * 1000;
const MAX_FILENAME_CODE_POINTS = 255;
const FORBIDDEN_FILENAME = /[/\\\p{Cc}\p{Zl}\p{Zp}]/u;
const KNOWN_VARIANTS = new Set(['original', 'thumbnail', 'crop', 'optimized']);
export type AttachmentWithVariants = Attachment & { variants: AttachmentVariant[] };
export interface DownloadableContent {
filename: string;
contentType: string;
sizeBytes: number;
content: Readable;
}
export interface UploadInput {
chatId: string;
ownerId: string;
filename: unknown;
contentType: string | undefined;
contentLength: string | undefined;
body: Readable | undefined;
}
@Injectable()
export class AttachmentsService implements OnModuleInit {
private readonly logger = new Logger('AttachmentsService');
private readonly maxBytes = Number(process.env.MAX_UPLOAD_BYTES ?? 10485760);
private readonly maxPixels = Number(process.env.MAX_IMAGE_PIXELS ?? 40000000);
private readonly maxConcurrent = Number(process.env.MAX_CONCURRENT_UPLOADS ?? 16);
private readonly bodyTimeoutMs = Number(process.env.UPLOAD_BODY_TIMEOUT_MS ?? 60000);
private readonly tempRoot = process.env.UPLOAD_TMP_DIR ?? tmpdir();
private activeUploads = 0;
constructor(
private readonly prisma: PrismaService,
private readonly s3: S3Service,
) {}
async onModuleInit(): Promise<void> {
await this.removeStaleTempDirs();
}
async upload(input: UploadInput): Promise<AttachmentWithVariants> {
await assertChatMember(this.prisma, input.chatId, input.ownerId);
const filename = parseFilename(input.filename);
const contentType = parseContentType(input.contentType);
if (Number(input.contentLength) > this.maxBytes) {
throw new ApiError('payload_too_large', 'attachment exceeds size limit');
}
if (this.activeUploads >= this.maxConcurrent) {
throw new ApiError('unavailable', 'too many concurrent uploads');
}
this.activeUploads++;
const dir = await mkdtemp(join(this.tempRoot, TEMP_PREFIX));
try {
return await this.store(input, filename, contentType, join(dir, 'body'));
} finally {
this.activeUploads--;
await rm(dir, { recursive: true, force: true });
}
}
async getVisible(
attachmentId: string,
userId: string,
): Promise<AttachmentWithVariants> {
if (!isUuid(attachmentId)) {
throw new ApiError('invalid_request', 'attachmentId must be a valid UUID');
}
const row = await this.prisma.attachment.findFirst({
where: {
id: attachmentId,
chat: { members: { some: { userId } } },
},
include: { variants: { where: { pipelineVersion: PIPELINE_VERSION } } },
});
if (!row) {
throw new ApiError('not_found', 'attachment not found');
}
return row;
}
async openContent(
attachmentId: string,
userId: string,
variant: unknown,
): Promise<DownloadableContent> {
const attachment = await this.getVisible(attachmentId, userId);
const name = parseVariant(variant);
if (attachment.status !== 'ready') {
throw new ApiError('conflict', `attachment is ${attachment.status}`);
}
if (name === 'original') {
return {
filename: attachment.filename,
contentType: attachment.contentType,
sizeBytes: attachment.sizeBytes,
content: await this.s3.getStream(attachment.objectKey),
};
}
const stored = attachment.variants.find((v) => v.name === name);
if (!stored) {
throw new ApiError('not_found', 'attachment variant not found');
}
return {
filename: attachment.filename,
contentType: stored.contentType,
sizeBytes: stored.sizeBytes,
content: await this.s3.getStream(stored.objectKey),
};
}
private async store(
input: UploadInput,
filename: string,
contentType: string,
path: string,
): Promise<AttachmentWithVariants> {
if (!input.body) {
throw new ApiError('invalid_request', 'request body must not be empty');
}
const { size, sha256 } = await receiveBody(
input.body,
path,
this.maxBytes,
this.bodyTimeoutMs,
);
if (size === 0) {
throw new ApiError('invalid_request', 'request body must not be empty');
}
await assertValidImage(path, size, contentType, this.maxPixels);
const id = randomUUID();
const stagingKey = `staging/${id}`;
await this.s3.putFile({ key: stagingKey, path, size, contentType, sha256 });
const now = new Date();
try {
return await this.prisma.$transaction(async (tx) => {
const attachment = await tx.attachment.create({
data: {
id,
chatId: input.chatId,
ownerId: input.ownerId,
filename,
contentType,
sizeBytes: size,
sha256,
objectKey: `attachments/${id}/original`,
stagingKey,
status: 'queued',
createdAt: now,
updatedAt: now,
},
});
await enqueueJob(tx, id, now);
return { ...attachment, variants: [] };
});
} catch (error) {
if (isDefiniteRejection(error)) {
await this.s3.delete(stagingKey).catch(() => {
this.logger.warn(`compensation failed for ${stagingKey}`);
});
}
if (error instanceof Prisma.PrismaClientUnknownRequestError) {
this.logger.error(`metadata insert failed for ${stagingKey}`);
throw new ApiError('unavailable', 'database unavailable');
}
throw error;
}
}
private async removeStaleTempDirs(): Promise<void> {
const root = this.tempRoot;
const names = await readdir(root);
for (const name of names.filter((n) => n.startsWith(TEMP_PREFIX))) {
const path = join(root, name);
const info = await stat(path).catch(() => null);
if (info && Date.now() - info.mtimeMs > STALE_TEMP_MS) {
await rm(path, { recursive: true, force: true });
}
}
}
}
function parseVariant(raw: unknown): string {
if (raw === undefined) {
return 'original';
}
if (typeof raw === 'string' && KNOWN_VARIANTS.has(raw)) {
return raw;
}
throw new ApiError('invalid_request', 'unknown attachment variant');
}
function isDefiniteRejection(error: unknown): boolean {
return (
error instanceof Prisma.PrismaClientKnownRequestError &&
!isDatabaseUnavailable(error)
);
}
function parseFilename(raw: unknown): string {
if (
typeof raw !== 'string' ||
raw.trim() === '' ||
codePointLength(raw) > MAX_FILENAME_CODE_POINTS ||
FORBIDDEN_FILENAME.test(raw)
) {
throw new ApiError(
'invalid_request',
'filename must be 1-255 characters without path separators or control characters',
);
}
return raw;
}
function parseContentType(raw: string | undefined): string {
const contentType = raw?.split(';')[0].trim().toLowerCase() ?? '';
if (!isSupportedContentType(contentType)) {
throw new ApiError(
'unsupported_media_type',
'Content-Type must be image/png or image/jpeg',
);
}
return contentType;
}
@@ -0,0 +1,55 @@
import { Attachment, AttachmentVariant } from '@prisma/client';
const VARIANT_ORDER = ['thumbnail', 'crop', 'optimized'];
export interface VariantResponse {
name: string;
content_type: string;
size_bytes: number;
width: number;
height: number;
sha256: string;
}
export interface AttachmentResponse {
id: string;
chat_id: string;
owner_id: string;
filename: string;
content_type: string;
size_bytes: number;
sha256: string;
status: string;
variants: VariantResponse[];
error_code: string | null;
created_at: string;
updated_at: string;
}
export function toAttachmentResponse(
row: Attachment & { variants: AttachmentVariant[] },
): AttachmentResponse {
const variants = row.status === 'ready' ? [...row.variants] : [];
variants.sort((a, b) => VARIANT_ORDER.indexOf(a.name) - VARIANT_ORDER.indexOf(b.name));
return {
id: row.id,
chat_id: row.chatId,
owner_id: row.ownerId,
filename: row.filename,
content_type: row.contentType,
size_bytes: row.sizeBytes,
sha256: row.sha256,
status: row.status,
variants: variants.map((variant) => ({
name: variant.name,
content_type: variant.contentType,
size_bytes: variant.sizeBytes,
width: variant.width,
height: variant.height,
sha256: variant.sha256,
})),
error_code: row.errorCode,
created_at: row.createdAt.toISOString(),
updated_at: row.updatedAt.toISOString(),
};
}
+105
View File
@@ -0,0 +1,105 @@
import { open } from 'fs/promises';
import sharp from 'sharp';
import { ApiError } from '../common/filters/api-error';
sharp.cache(false);
sharp.concurrency(1);
const PNG_SIGNATURE = Buffer.from('89504e470d0a1a0a', 'hex');
const JPEG_SIGNATURE = Buffer.from('ffd8ff', 'hex');
const PNG_END_CHUNK = Buffer.from('0000000049454e44ae426082', 'hex');
const PNG_TAIL_BYTES = 64 * 1024;
const MAX_CONCURRENT_DECODES = 2;
const FORMAT_BY_CONTENT_TYPE: Record<string, string> = {
'image/png': 'png',
'image/jpeg': 'jpeg',
};
let activeDecodes = 0;
const decodeQueue: (() => void)[] = [];
export function isSupportedContentType(contentType: string): boolean {
return contentType in FORMAT_BY_CONTENT_TYPE;
}
export async function assertValidImage(
path: string,
size: number,
contentType: string,
maxPixels: number,
): Promise<void> {
const invalid = new ApiError(
'unsupported_media_type',
'file is not a valid image of the declared type',
);
const format = FORMAT_BY_CONTENT_TYPE[contentType];
if (!(await hasExpectedFraming(path, size, format))) {
throw invalid;
}
const { width, height, format: actual } = await sharp(path, {
limitInputPixels: false,
})
.metadata()
.catch(() => {
throw invalid;
});
if (!width || !height || actual !== format) {
throw invalid;
}
if (width * height > maxPixels) {
throw new ApiError('payload_too_large', 'image exceeds pixel limit');
}
await acquireDecodeSlot();
try {
await sharp(path, {
failOn: 'warning',
sequentialRead: true,
limitInputPixels: maxPixels,
}).stats();
} catch {
throw invalid;
} finally {
releaseDecodeSlot();
}
}
async function hasExpectedFraming(
path: string,
size: number,
format: string,
): Promise<boolean> {
const file = await open(path, 'r');
try {
const head = Buffer.alloc(PNG_SIGNATURE.length);
await file.read(head, 0, head.length, 0);
if (format === 'jpeg') {
return head.subarray(0, JPEG_SIGNATURE.length).equals(JPEG_SIGNATURE);
}
if (!head.equals(PNG_SIGNATURE)) {
return false;
}
const tail = Buffer.alloc(Math.min(size, PNG_TAIL_BYTES));
await file.read(tail, 0, tail.length, size - tail.length);
return tail.includes(PNG_END_CHUNK);
} finally {
await file.close();
}
}
function acquireDecodeSlot(): Promise<void> {
if (activeDecodes < MAX_CONCURRENT_DECODES) {
activeDecodes++;
return Promise.resolve();
}
return new Promise((resolve) => decodeQueue.push(resolve));
}
function releaseDecodeSlot(): void {
const next = decodeQueue.shift();
if (next) {
next();
} else {
activeDecodes--;
}
}
+163
View File
@@ -0,0 +1,163 @@
import { PrismaClient } from '@prisma/client';
import { S3Service } from '../storage/s3.service';
const MAX_REPORTED_KEYS = 20;
const DEFAULT_PREFIXES = ['attachments/', 'staging/'];
type ReconcilePrisma = Pick<PrismaClient, 'attachment' | 'attachmentVariant'>;
export interface ReconcileOptions {
apply: boolean;
graceMs: number;
stagingRetentionMs?: number;
prefix?: string;
now?: Date;
pageSize?: number;
}
export interface ReconcileReport {
mode: 'apply' | 'dry-run';
grace_seconds: number;
staging_expired: number;
scanned: number;
fresh_skipped: number;
referenced: number;
orphans: number;
deleted: number;
failed: number;
orphan_keys: string[];
duration_ms: number;
}
export async function reconcile(
prisma: ReconcilePrisma,
s3: S3Service,
options: ReconcileOptions,
): Promise<ReconcileReport> {
const started = Date.now();
const cutoff = (options.now ?? new Date()).getTime() - options.graceMs;
const report: ReconcileReport = {
mode: options.apply ? 'apply' : 'dry-run',
grace_seconds: options.graceMs / 1000,
staging_expired: 0,
scanned: 0,
fresh_skipped: 0,
referenced: 0,
orphans: 0,
deleted: 0,
failed: 0,
orphan_keys: [],
duration_ms: 0,
};
if (options.stagingRetentionMs !== undefined) {
report.staging_expired = await expireFailedStaging(
prisma,
s3,
(options.now ?? new Date()).getTime() - options.stagingRetentionMs,
options.apply,
options.prefix,
);
}
for (const prefix of options.prefix ? [options.prefix] : DEFAULT_PREFIXES) {
await reconcilePrefix(prisma, s3, prefix, cutoff, options, report);
}
report.duration_ms = Date.now() - started;
return report;
}
async function expireFailedStaging(
prisma: ReconcilePrisma,
s3: S3Service,
before: number,
apply: boolean,
prefix?: string,
): Promise<number> {
const rows = await prisma.attachment.findMany({
where: {
stagingKey: { not: null, startsWith: prefix },
OR: [{ status: 'ready' }, { status: 'failed', updatedAt: { lt: new Date(before) } }],
},
select: { id: true, stagingKey: true },
take: 500,
});
if (!apply) {
return rows.length;
}
let expired = 0;
for (const row of rows) {
const released = await prisma.attachment.updateMany({
where: {
id: row.id,
stagingKey: row.stagingKey,
OR: [{ status: 'ready' }, { status: 'failed', updatedAt: { lt: new Date(before) } }],
},
data: { stagingKey: null },
});
if (released.count === 0) {
continue;
}
await s3.delete(row.stagingKey!).then(
() => expired++,
() => undefined,
);
}
return expired;
}
function attachmentIdOf(key: string): string | null {
const match = /^attachments\/([0-9a-f-]{36})(\/|$)/.exec(key);
return match ? match[1] : null;
}
async function reconcilePrefix(
prisma: ReconcilePrisma,
s3: S3Service,
prefix: string,
cutoff: number,
options: ReconcileOptions,
report: ReconcileReport,
): Promise<void> {
let token: string | undefined;
do {
const page = await s3.listObjects(prefix, options.pageSize ?? 500, token);
token = page.nextToken;
report.scanned += page.objects.length;
const old = page.objects.filter((object) => object.lastModified.getTime() < cutoff);
report.fresh_skipped += page.objects.length - old.length;
const keys = old.map((object) => object.key);
const rows = await prisma.attachment.findMany({
where: { OR: [{ objectKey: { in: keys } }, { stagingKey: { in: keys } }] },
select: { objectKey: true, stagingKey: true },
});
const variants = await prisma.attachmentVariant.findMany({
where: { objectKey: { in: keys } },
select: { objectKey: true },
});
const ids = [...new Set(keys.map(attachmentIdOf).filter((id): id is string => id !== null))];
const owners = ids.length
? await prisma.attachment.findMany({ where: { id: { in: ids } }, select: { id: true } })
: [];
const owned = new Set(owners.map((row) => row.id));
const known = new Set([
...rows.flatMap((row) => [row.objectKey, row.stagingKey]),
...variants.map((row) => row.objectKey),
]);
for (const { key } of old) {
const owner = attachmentIdOf(key);
if (known.has(key) || (owner !== null && owned.has(owner))) {
report.referenced++;
continue;
}
report.orphans++;
if (report.orphan_keys.length < MAX_REPORTED_KEYS) {
report.orphan_keys.push(key);
}
if (options.apply) {
await s3.delete(key).then(
() => report.deleted++,
() => report.failed++,
);
}
}
} while (token);
}
+98
View File
@@ -0,0 +1,98 @@
import { createHash } from 'crypto';
import { FastifyInstance, FastifyRequest } from 'fastify';
import { createWriteStream } from 'fs';
import { Readable } from 'stream';
import { ApiError } from '../common/filters/api-error';
const UPLOAD_ROUTE = '/api/v1/chats/:chatId/attachments';
const PASS_THROUGH_TYPE = 'application/x-messenger-upload';
declare module 'fastify' {
interface FastifyRequest {
uploadContentType?: string;
}
}
export function registerUploadBody(instance: FastifyInstance): void {
instance.addContentTypeParser(PASS_THROUGH_TYPE, (_request, payload, done) => {
done(null, payload);
});
instance.addHook('onRequest', (request: FastifyRequest, _reply, done) => {
if (request.method === 'POST' && request.routeOptions.url === UPLOAD_ROUTE) {
request.uploadContentType = request.headers['content-type'];
request.headers['content-type'] = PASS_THROUGH_TYPE;
}
done();
});
}
export interface ReceivedBody {
size: number;
sha256: string;
}
export function receiveBody(
body: Readable,
path: string,
maxBytes: number,
timeoutMs: number,
): Promise<ReceivedBody> {
return new Promise((resolve, reject) => {
const hash = createHash('sha256');
const out = createWriteStream(path, { flags: 'wx', mode: 0o600 });
let size = 0;
let settled = false;
const timer = setTimeout(() => body.destroy(), timeoutMs);
const settle = (done: () => void): void => {
if (settled) {
return;
}
settled = true;
clearTimeout(timer);
body.off('data', onData);
body.off('end', onEnd);
body.off('close', onClose);
done();
};
const fail = (error: ApiError): void => {
settle(() => {
out.destroy();
reject(error);
});
};
const onData = (chunk: Buffer): void => {
size += chunk.length;
if (size > maxBytes) {
fail(new ApiError('payload_too_large', 'attachment exceeds size limit'));
return;
}
hash.update(chunk);
if (!out.write(chunk)) {
body.pause();
out.once('drain', () => {
if (!settled) {
body.resume();
}
});
}
};
const onEnd = (): void => {
settle(() => {
out.end(() =>
resolve({ size, sha256: hash.digest('hex') }),
);
});
};
const onClose = (): void => {
fail(new ApiError('invalid_request', 'upload was interrupted'));
};
out.on('error', () => {
fail(new ApiError('unavailable', 'temporary storage unavailable'));
});
body.on('data', onData);
body.on('end', onEnd);
body.on('close', onClose);
});
}
+51
View File
@@ -0,0 +1,51 @@
import {
Controller,
Delete,
Headers,
HttpCode,
Post,
Req,
UseGuards,
} from '@nestjs/common';
import { FastifyRequest } from 'fastify';
import { ApiError } from '../common/filters/api-error';
import { toUserResponse, UserResponse } from '../users/dto/user.response';
import { AuthGuard } from './auth.guard';
import { AuthService } from './auth.service';
interface SessionResponse {
token: string;
token_type: 'Bearer';
expires_at: string;
user: UserResponse;
}
@Controller('api/v1/auth/sessions')
export class AuthController {
constructor(private readonly auth: AuthService) {}
@Post()
@HttpCode(201)
async create(
@Headers('authorization') authorization: string | undefined,
@Req() request: FastifyRequest,
): Promise<SessionResponse> {
if (request.body !== undefined) {
throw new ApiError('invalid_request', 'request body is not allowed');
}
const session = await this.auth.login(authorization);
return {
token: session.token,
token_type: 'Bearer',
expires_at: session.expiresAt.toISOString(),
user: toUserResponse(session.user),
};
}
@Delete('current')
@HttpCode(204)
@UseGuards(AuthGuard)
async revoke(@Req() request: FastifyRequest): Promise<void> {
await this.auth.revoke(request.sessionTokenHash!);
}
}
+25
View File
@@ -0,0 +1,25 @@
import { CanActivate, ExecutionContext, Injectable } from '@nestjs/common';
import { FastifyRequest } from 'fastify';
import { ApiError } from '../common/filters/api-error';
import { AuthService } from './auth.service';
const BEARER_CHALLENGE = 'Bearer realm="messenger"';
@Injectable()
export class AuthGuard implements CanActivate {
constructor(private readonly auth: AuthService) {}
async canActivate(context: ExecutionContext): Promise<boolean> {
const request = context.switchToHttp().getRequest<FastifyRequest>();
const match = /^Bearer (\S+)$/i.exec(request.headers.authorization ?? '');
const session = match ? await this.auth.resolve(match[1]) : null;
if (!session) {
throw new ApiError('unauthorized', 'missing or invalid session', {
'WWW-Authenticate': BEARER_CHALLENGE,
});
}
request.currentUser = session.user;
request.sessionTokenHash = session.tokenHash;
return true;
}
}
+12
View File
@@ -0,0 +1,12 @@
import { Global, Module } from '@nestjs/common';
import { AuthController } from './auth.controller';
import { AuthGuard } from './auth.guard';
import { AuthService } from './auth.service';
@Global()
@Module({
controllers: [AuthController],
providers: [AuthService, AuthGuard],
exports: [AuthService, AuthGuard],
})
export class AuthModule {}
+155
View File
@@ -0,0 +1,155 @@
import { Injectable } from '@nestjs/common';
import { createHash, randomBytes } from 'crypto';
import { CacheService } from '../cache/cache.service';
import { codePointLength } from '../common/dto/code-point-length.validator';
import { User } from '../common/domain';
import { ApiError } from '../common/filters/api-error';
import { PrismaService } from '../db/prisma.service';
import { toUser } from '../users/users.service';
import { verifyPassword } from './password';
const BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"';
const USERNAME_PATTERN = /^[a-z0-9_]{3,32}$/;
const CACHE_TTL_MS = 60_000;
const REVOKED = 'revoked';
interface CachedSession {
user: User;
expiresAt: number;
}
export interface CreatedSession {
token: string;
expiresAt: Date;
user: User;
}
export interface ResolvedSession {
tokenHash: string;
user: User;
}
@Injectable()
export class AuthService {
private readonly ttlSeconds = readSessionTtl();
constructor(
private readonly prisma: PrismaService,
private readonly cache: CacheService,
) {}
async login(authorization: string | undefined): Promise<CreatedSession> {
const credentials = parseBasic(authorization);
if (!credentials) {
throw invalidCredentials();
}
const row = await this.prisma.user.findUnique({
where: { username: credentials.username },
});
const valid = await verifyPassword(
row?.passwordHash ?? null,
credentials.password,
);
if (!row || !valid) {
throw invalidCredentials();
}
const token = randomBytes(32).toString('base64url');
const expiresAt = new Date(Date.now() + this.ttlSeconds * 1000);
await this.prisma.session.create({
data: { tokenHash: hashToken(token), userId: row.id, expiresAt },
});
return { token, expiresAt, user: toUser(row) };
}
async resolve(token: string): Promise<ResolvedSession | null> {
const tokenHash = hashToken(token);
const key = cacheKey(tokenHash);
const cached = await this.cache.get(key);
if (cached === REVOKED) {
return null;
}
if (cached) {
const entry = JSON.parse(cached) as CachedSession;
if (entry.expiresAt > Date.now()) {
return { tokenHash, user: entry.user };
}
}
const session = await this.prisma.session.findUnique({
where: { tokenHash },
include: { user: true },
});
const remainingMs = session
? session.expiresAt.getTime() - Date.now()
: 0;
if (!session || remainingMs <= 0) {
return null;
}
const user = toUser(session.user);
const entry: CachedSession = {
user,
expiresAt: session.expiresAt.getTime(),
};
await this.cache.set(
key,
JSON.stringify(entry),
Math.min(remainingMs, CACHE_TTL_MS),
true,
);
return { tokenHash, user };
}
async revoke(tokenHash: string): Promise<void> {
const marked = await this.cache.setStrict(
cacheKey(tokenHash),
REVOKED,
CACHE_TTL_MS,
);
if (!marked) {
throw new ApiError('unavailable', 'session cache unavailable');
}
await this.prisma.session.deleteMany({ where: { tokenHash } });
}
}
function cacheKey(tokenHash: string): string {
return `session:v1:${tokenHash}`;
}
function parseBasic(
header: string | undefined,
): { username: string; password: string } | null {
const match = /^Basic ([A-Za-z0-9+/]+={0,2})$/i.exec(header ?? '');
if (!match) {
return null;
}
const decoded = Buffer.from(match[1], 'base64').toString('utf8');
const separator = decoded.indexOf(':');
if (separator < 0) {
return null;
}
const username = decoded.slice(0, separator);
const password = decoded.slice(separator + 1);
const length = codePointLength(password);
if (!USERNAME_PATTERN.test(username) || length < 12 || length > 128) {
return null;
}
return { username, password };
}
function hashToken(token: string): string {
return createHash('sha256').update(token).digest('hex');
}
function invalidCredentials(): ApiError {
return new ApiError('unauthorized', 'invalid credentials', {
'WWW-Authenticate': BASIC_CHALLENGE,
});
}
function readSessionTtl(): number {
const ttl = Number(process.env.SESSION_TTL_SECONDS ?? 3600);
if (!Number.isInteger(ttl) || ttl <= 0) {
throw new Error('SESSION_TTL_SECONDS must be a positive integer');
}
return ttl;
}
+20
View File
@@ -0,0 +1,20 @@
import { createParamDecorator, ExecutionContext } from '@nestjs/common';
import { FastifyRequest } from 'fastify';
import { User } from '../common/domain';
declare module 'fastify' {
interface FastifyRequest {
currentUser?: User;
sessionTokenHash?: string;
}
}
export const CurrentUser = createParamDecorator(
(_data: unknown, ctx: ExecutionContext): User => {
const request = ctx.switchToHttp().getRequest<FastifyRequest>();
if (!request.currentUser) {
throw new Error('CurrentUser used without AuthGuard');
}
return request.currentUser;
},
);
+17
View File
@@ -0,0 +1,17 @@
import { argon2id, hash, verify } from 'argon2';
const OPTIONS = { type: argon2id, memoryCost: 19456, timeCost: 2, parallelism: 1 };
const dummyHash = hash('timing-equalizer-password', OPTIONS);
export function hashPassword(password: string): Promise<string> {
return hash(password, OPTIONS);
}
export async function verifyPassword(
passwordHash: string | null,
password: string,
): Promise<boolean> {
const valid = await verify(passwordHash ?? (await dummyHash), password);
return passwordHash !== null && valid;
}
+57
View File
@@ -0,0 +1,57 @@
import { Logger } from '@nestjs/common';
import amqp, { Channel, Connection } from 'amqplib';
import { readFileSync } from 'fs';
export const JOB_QUEUE = 'attachment.jobs';
export const DEAD_LETTER_QUEUE = 'attachment.jobs.dlq';
const DELIVERY_LIMIT = 20;
const RECONNECT_MAX_MS = 10000;
export function brokerUrl(): string {
const file = process.env.BROKER_URL_FILE;
const url = file ? readFileSync(file, 'utf8').trim() : process.env.BROKER_URL;
if (!url) {
throw new Error('BROKER_URL_FILE or BROKER_URL must be set');
}
return url;
}
export async function assertTopology(channel: Channel): Promise<void> {
await channel.assertQueue(DEAD_LETTER_QUEUE, {
durable: true,
arguments: { 'x-queue-type': 'quorum' },
});
await channel.assertQueue(JOB_QUEUE, {
durable: true,
arguments: {
'x-queue-type': 'quorum',
'x-delivery-limit': DELIVERY_LIMIT,
'x-dead-letter-exchange': '',
'x-dead-letter-routing-key': DEAD_LETTER_QUEUE,
},
});
}
export async function connectWithRetry(
logger: Logger,
stopping: () => boolean,
): Promise<Connection | null> {
let delay = 500;
while (!stopping()) {
try {
const connection = await amqp.connect(brokerUrl(), { timeout: 5000 });
logger.log('connected to broker');
return connection;
} catch (error) {
const reason = error instanceof Error ? error.message.split(' ')[0] : 'unknown';
logger.warn(`broker unavailable (${reason}), retry in ${delay} ms`);
await sleep(delay);
delay = Math.min(RECONNECT_MAX_MS, delay * 2);
}
}
return null;
}
export function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
+9
View File
@@ -0,0 +1,9 @@
import { Global, Module } from '@nestjs/common';
import { CacheService } from './cache.service';
@Global()
@Module({
providers: [CacheService],
exports: [CacheService],
})
export class CacheModule {}
+98
View File
@@ -0,0 +1,98 @@
import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common';
import { readFileSync } from 'fs';
import Redis from 'ioredis';
@Injectable()
export class CacheService implements OnModuleDestroy {
private readonly logger = new Logger('CacheService');
private readonly client: Redis | null;
private trusted = false;
constructor() {
const url = readCacheUrl();
this.client = url
? new Redis(url, {
enableOfflineQueue: false,
maxRetriesPerRequest: 0,
connectTimeout: 1000,
commandTimeout: 200,
})
: null;
this.client?.on('error', (error) =>
this.logger.warn(`redis unavailable: ${error.message}`),
);
this.client?.on('close', () => {
this.trusted = false;
});
this.client?.on('ready', () => {
this.client!.flushdb().then(
() => {
this.trusted = true;
},
() => this.distrust(),
);
});
}
async get(key: string): Promise<string | null> {
if (!this.client || !this.trusted) {
return null;
}
try {
return await this.client.get(key);
} catch {
this.distrust();
return null;
}
}
async set(
key: string,
value: string,
ttlMs: number,
onlyIfAbsent = false,
): Promise<void> {
if (!this.client || !this.trusted) {
return;
}
try {
if (onlyIfAbsent) {
await this.client.set(key, value, 'PX', ttlMs, 'NX');
} else {
await this.client.set(key, value, 'PX', ttlMs);
}
} catch {
this.distrust();
}
}
async setStrict(key: string, value: string, ttlMs: number): Promise<boolean> {
if (!this.client) {
return true;
}
if (!this.trusted) {
return false;
}
try {
await this.client.set(key, value, 'PX', ttlMs);
return true;
} catch {
this.distrust();
return false;
}
}
onModuleDestroy(): void {
this.client?.disconnect();
}
private distrust(): void {
this.trusted = false;
this.client?.disconnect(true);
}
}
function readCacheUrl(): string | undefined {
const file = process.env.CACHE_URL_FILE;
return file ? readFileSync(file, 'utf8').trim() : process.env.CACHE_URL;
}
+19
View File
@@ -0,0 +1,19 @@
import { ApiError } from '../common/filters/api-error';
import { validate as isUuid } from '../common/uuid';
import { PrismaService } from '../db/prisma.service';
export async function assertChatMember(
prisma: PrismaService,
chatId: string,
userId: string,
): Promise<void> {
if (!isUuid(chatId)) {
throw new ApiError('invalid_request', 'chatId must be a valid UUID');
}
const member = await prisma.chatMember.findUnique({
where: { chatId_userId: { chatId, userId } },
});
if (!member) {
throw new ApiError('not_found', 'chat not found');
}
}
+57
View File
@@ -0,0 +1,57 @@
import {
Body,
Controller,
Get,
HttpCode,
Param,
Post,
Query,
UseGuards,
} from '@nestjs/common';
import { AuthGuard } from '../auth/auth.guard';
import { CurrentUser } from '../auth/current-user.decorator';
import { parseLimit, rawCursorParam } from '../common/dto/pagination-query.dto';
import { User } from '../common/domain';
import { ChatsService } from './chats.service';
import {
ChatPageResponse,
ChatResponse,
toChatResponse,
} from './dto/chat.response';
import { CreateChatDto } from './dto/create-chat.dto';
@Controller('api/v1/chats')
@UseGuards(AuthGuard)
export class ChatsController {
constructor(private readonly chats: ChatsService) {}
@Post()
@HttpCode(201)
async create(
@Body() dto: CreateChatDto,
@CurrentUser() user: User,
): Promise<ChatResponse> {
const chat = await this.chats.create(dto.title, dto.member_ids, user.id);
return toChatResponse(chat);
}
@Get()
async list(
@CurrentUser() user: User,
@Query('limit') limitRaw: unknown,
@Query('cursor') cursorRaw: unknown,
): Promise<ChatPageResponse> {
const limit = parseLimit(limitRaw);
const cursor = rawCursorParam(cursorRaw);
return this.chats.listForMember(user.id, limit, cursor);
}
@Get(':chatId')
async getOne(
@Param('chatId') chatId: string,
@CurrentUser() user: User,
): Promise<ChatResponse> {
const chat = await this.chats.getForMember(chatId, user.id);
return toChatResponse(chat);
}
}
+10
View File
@@ -0,0 +1,10 @@
import { Module } from '@nestjs/common';
import { ChatsController } from './chats.controller';
import { ChatsService } from './chats.service';
@Module({
controllers: [ChatsController],
providers: [ChatsService],
exports: [ChatsService],
})
export class ChatsModule {}
+131
View File
@@ -0,0 +1,131 @@
import { Injectable } from '@nestjs/common';
import { Prisma } from '@prisma/client';
import { decodeChatCursor, encodeChatCursor } from '../common/cursor';
import { Chat } from '../common/domain';
import { ApiError } from '../common/filters/api-error';
import { validate as isUuid } from '../common/uuid';
import { PrismaService, isForeignKeyViolation } from '../db/prisma.service';
import { ChatPageResponse, toChatResponse } from './dto/chat.response';
interface ChatRow {
id: string;
title: string;
createdAt: Date;
memberIds: string[];
}
@Injectable()
export class ChatsService {
constructor(private readonly prisma: PrismaService) {}
async create(
title: string,
rawMemberIds: string[],
callerId: string,
): Promise<Chat> {
const invalid = rawMemberIds.find((id) => !isUuid(id));
if (invalid) {
throw new ApiError('invalid_request', `unknown member_id: ${invalid}`);
}
const memberIds = rawMemberIds.map((id) => id.toLowerCase());
if (new Set(memberIds).size !== memberIds.length) {
throw new ApiError(
'invalid_request',
"All member_ids's elements must be unique",
);
}
if (!memberIds.includes(callerId)) {
throw new ApiError(
'invalid_request',
'member_ids must include the caller',
);
}
try {
const chat = await this.prisma.$transaction(async (tx) => {
const created = await tx.chat.create({
data: { title: title.trim() },
});
await tx.chatMember.createMany({
data: memberIds.map((userId) => ({
chatId: created.id,
userId,
})),
});
return created;
});
return toChat({ ...chat, memberIds });
} catch (error) {
if (isForeignKeyViolation(error)) {
throw new ApiError(
'invalid_request',
'member_ids contain an unknown user',
);
}
throw error;
}
}
async getForMember(chatId: string, callerId: string): Promise<Chat> {
if (!isUuid(chatId)) {
throw new ApiError('invalid_request', 'chatId must be a valid UUID');
}
const chat = await this.prisma.chat.findFirst({
where: { id: chatId, members: { some: { userId: callerId } } },
include: { members: { select: { userId: true } } },
});
if (!chat) {
throw new ApiError('not_found', 'chat not found');
}
return toChat({
...chat,
memberIds: chat.members.map((m) => m.userId),
});
}
async listForMember(
callerId: string,
limit: number,
rawCursor: string | null,
): Promise<ChatPageResponse> {
let after = Prisma.empty;
if (rawCursor !== null) {
const cursor = decodeChatCursor(rawCursor);
const member = cursor
? await this.prisma.chatMember.findUnique({
where: {
chatId_userId: { chatId: cursor.id, userId: callerId },
},
include: { chat: { select: { createdAt: true } } },
})
: null;
if (!member) {
throw new ApiError('invalid_request', 'invalid cursor');
}
after = Prisma.sql`AND (c.created_at, c.id) > (${member.chat.createdAt}::timestamptz, ${member.chatId}::uuid)`;
}
const rows = await this.prisma.$queryRaw<ChatRow[]>`
SELECT c.id, c.title, c.created_at AS "createdAt",
ARRAY(SELECT m.user_id::text FROM chat_members m WHERE m.chat_id = c.id) AS "memberIds"
FROM chat_members cm
JOIN chats c ON c.id = cm.chat_id
WHERE cm.user_id = ${callerId}::uuid ${after}
ORDER BY c.created_at, c.id
LIMIT ${limit + 1}`;
const items = rows.slice(0, limit).map(toChat);
const last = items[items.length - 1];
return {
items: items.map(toChatResponse),
next_cursor:
rows.length > limit ? encodeChatCursor({ id: last.id }) : null,
};
}
}
function toChat(row: ChatRow): Chat {
return {
id: row.id,
title: row.title,
memberIds: row.memberIds,
createdAt: row.createdAt.toISOString(),
};
}
+22
View File
@@ -0,0 +1,22 @@
import { Chat } from '../../common/domain';
export interface ChatResponse {
id: string;
title: string;
member_ids: string[];
created_at: string;
}
export interface ChatPageResponse {
items: ChatResponse[];
next_cursor: string | null;
}
export function toChatResponse(chat: Chat): ChatResponse {
return {
id: chat.id,
title: chat.title,
member_ids: chat.memberIds,
created_at: chat.createdAt,
};
}
+22
View File
@@ -0,0 +1,22 @@
import {
ArrayMaxSize,
ArrayMinSize,
ArrayUnique,
IsArray,
IsString,
} from 'class-validator';
import { CodePointLength } from '../../common/dto/code-point-length.validator';
import { NotBlank } from '../../common/dto/not-blank.validator';
export class CreateChatDto {
@NotBlank()
@CodePointLength(1, 100)
title!: string;
@IsArray()
@ArrayMinSize(1)
@ArrayMaxSize(100)
@ArrayUnique()
@IsString({ each: true })
member_ids!: string[];
}
+49
View File
@@ -0,0 +1,49 @@
import { validate as isUuid } from './uuid';
export interface ChatCursor {
id: string;
}
export interface MessageCursor {
chatId: string;
id: string;
}
export function encodeChatCursor(cursor: ChatCursor): string {
return encode(cursor);
}
export function decodeChatCursor(raw: string): ChatCursor | null {
const decoded = decode(raw);
return decoded && isUuidField(decoded.id) ? { id: decoded.id } : null;
}
export function encodeMessageCursor(cursor: MessageCursor): string {
return encode(cursor);
}
export function decodeMessageCursor(raw: string): MessageCursor | null {
const decoded = decode(raw);
return decoded && isUuidField(decoded.chatId) && isUuidField(decoded.id)
? { chatId: decoded.chatId, id: decoded.id }
: null;
}
function isUuidField(value: unknown): value is string {
return typeof value === 'string' && isUuid(value);
}
function encode(value: object): string {
return Buffer.from(JSON.stringify(value), 'utf8').toString('base64url');
}
function decode(raw: string): Record<string, unknown> | null {
try {
const parsed = JSON.parse(
Buffer.from(raw, 'base64url').toString('utf8'),
);
return typeof parsed === 'object' && parsed !== null ? parsed : null;
} catch {
return null;
}
}
+22
View File
@@ -0,0 +1,22 @@
export interface User {
id: string;
username: string;
displayName: string;
createdAt: string;
}
export interface Chat {
id: string;
title: string;
memberIds: string[];
createdAt: string;
}
export interface Message {
id: string;
chatId: string;
senderId: string;
text: string;
createdAt: string;
attachmentIds: string[];
}
@@ -0,0 +1,33 @@
import { registerDecorator, ValidationOptions } from 'class-validator';
export function codePointLength(value: string): number {
return [...value].length;
}
export function CodePointLength(
min: number,
max: number,
options?: ValidationOptions,
) {
return function (object: object, propertyName: string) {
registerDecorator({
name: 'codePointLength',
target: object.constructor,
propertyName,
constraints: [min, max],
options,
validator: {
validate(value: unknown): boolean {
if (typeof value !== 'string') {
return false;
}
const length = codePointLength(value);
return length >= min && length <= max;
},
defaultMessage(): string {
return `${propertyName} must be between ${min} and ${max} Unicode code points`;
},
},
});
};
}
+20
View File
@@ -0,0 +1,20 @@
import { registerDecorator, ValidationOptions } from 'class-validator';
export function NotBlank(options?: ValidationOptions) {
return function (object: object, propertyName: string) {
registerDecorator({
name: 'notBlank',
target: object.constructor,
propertyName,
options,
validator: {
validate(value: unknown): boolean {
return typeof value === 'string' && value.trim().length > 0;
},
defaultMessage(): string {
return `${propertyName} must not be empty or whitespace-only`;
},
},
});
};
}
+35
View File
@@ -0,0 +1,35 @@
import { ApiError } from '../filters/api-error';
const DEFAULT_LIMIT = 50;
const MIN_LIMIT = 1;
const MAX_LIMIT = 100;
export function parseLimit(raw: unknown): number {
if (raw === undefined) {
return DEFAULT_LIMIT;
}
if (typeof raw !== 'string' || !/^\d+$/.test(raw)) {
throw new ApiError('invalid_request', 'limit must be an integer');
}
const limit = Number(raw);
if (limit < MIN_LIMIT || limit > MAX_LIMIT) {
throw new ApiError(
'invalid_request',
`limit must be between ${MIN_LIMIT} and ${MAX_LIMIT}`,
);
}
return limit;
}
export function rawCursorParam(raw: unknown): string | null {
if (raw === undefined) {
return null;
}
if (typeof raw !== 'string' || raw.length === 0) {
throw new ApiError(
'invalid_request',
'cursor must be a non-empty string',
);
}
return raw;
}
+34
View File
@@ -0,0 +1,34 @@
import { HttpException, HttpStatus } from '@nestjs/common';
export type ErrorCode =
| 'invalid_request'
| 'unauthorized'
| 'not_found'
| 'conflict'
| 'payload_too_large'
| 'unsupported_media_type'
| 'rate_limited'
| 'unavailable'
| 'internal_error';
const STATUS_BY_CODE: Record<ErrorCode, number> = {
invalid_request: HttpStatus.BAD_REQUEST,
unauthorized: HttpStatus.UNAUTHORIZED,
not_found: HttpStatus.NOT_FOUND,
conflict: HttpStatus.CONFLICT,
payload_too_large: HttpStatus.PAYLOAD_TOO_LARGE,
unsupported_media_type: HttpStatus.UNSUPPORTED_MEDIA_TYPE,
rate_limited: HttpStatus.TOO_MANY_REQUESTS,
unavailable: HttpStatus.SERVICE_UNAVAILABLE,
internal_error: HttpStatus.INTERNAL_SERVER_ERROR,
};
export class ApiError extends HttpException {
constructor(
public readonly code: ErrorCode,
message: string,
public readonly headers: Record<string, string> = {},
) {
super(message, STATUS_BY_CODE[code]);
}
}
+115
View File
@@ -0,0 +1,115 @@
import {
ArgumentsHost,
Catch,
ExceptionFilter,
HttpException,
HttpStatus,
Logger,
} from '@nestjs/common';
import { FastifyReply, FastifyRequest } from 'fastify';
import { isDatabaseUnavailable } from '../../db/prisma.service';
import { ApiError, ErrorCode } from './api-error';
const CODE_BY_STATUS: Partial<Record<number, ErrorCode>> = {
[HttpStatus.BAD_REQUEST]: 'invalid_request',
[HttpStatus.UNAUTHORIZED]: 'unauthorized',
[HttpStatus.NOT_FOUND]: 'not_found',
[HttpStatus.CONFLICT]: 'conflict',
[HttpStatus.PAYLOAD_TOO_LARGE]: 'payload_too_large',
[HttpStatus.UNSUPPORTED_MEDIA_TYPE]: 'unsupported_media_type',
[HttpStatus.TOO_MANY_REQUESTS]: 'rate_limited',
[HttpStatus.SERVICE_UNAVAILABLE]: 'unavailable',
};
interface Resolved {
status: number;
code: ErrorCode;
message: string;
}
@Catch()
export class HttpExceptionFilter implements ExceptionFilter {
private readonly logger = new Logger('HttpExceptionFilter');
catch(exception: unknown, host: ArgumentsHost): void {
const ctx = host.switchToHttp();
const reply = ctx.getResponse<FastifyReply>();
const request = ctx.getRequest<FastifyRequest>();
const requestId = request.requestId;
const { status, code, message } = this.resolve(exception);
if (status >= 500) {
this.logger.error(
`${request.method} ${request.url} -> ${status}: ${message}`,
);
}
if (exception instanceof ApiError) {
reply.headers(exception.headers);
}
reply
.code(status)
.header('x-request-id', requestId)
.send({ error: { code, message, request_id: requestId } });
}
private resolve(exception: unknown): Resolved {
if (exception instanceof ApiError) {
return {
status: exception.getStatus(),
code: exception.code,
message: exception.message,
};
}
if (exception instanceof HttpException) {
const status = exception.getStatus();
const code = CODE_BY_STATUS[status] ?? 'internal_error';
const message =
extractMessage(exception.getResponse()) ?? exception.message;
return { status, code, message };
}
const fastifyStatus = (exception as { statusCode?: unknown })
?.statusCode;
if (typeof fastifyStatus === 'number' && CODE_BY_STATUS[fastifyStatus]) {
const message =
exception instanceof Error ? exception.message : 'Invalid request';
return {
status: fastifyStatus,
code: CODE_BY_STATUS[fastifyStatus]!,
message,
};
}
if (isDatabaseUnavailable(exception)) {
return {
status: 503,
code: 'unavailable',
message: 'database unavailable',
};
}
return {
status: 500,
code: 'internal_error',
message: 'Internal server error',
};
}
}
function extractMessage(response: unknown): string | null {
if (typeof response === 'string') {
return response;
}
if (
typeof response === 'object' &&
response !== null &&
'message' in response
) {
const value = (response as { message: unknown }).message;
if (typeof value === 'string') {
return value;
}
if (Array.isArray(value) && value.every((v) => typeof v === 'string')) {
return value.join('; ');
}
}
return null;
}
+23
View File
@@ -0,0 +1,23 @@
import { randomUUID } from 'crypto';
import { FastifyInstance, FastifyRequest } from 'fastify';
import { hostname } from 'os';
const REQUEST_ID_HEADER = 'x-request-id';
const INSTANCE_ID_HEADER = 'x-instance-id';
export const INSTANCE_ID = process.env.INSTANCE_ID ?? hostname();
declare module 'fastify' {
interface FastifyRequest {
requestId: string;
}
}
export function registerRequestId(instance: FastifyInstance): void {
instance.addHook('onRequest', (request: FastifyRequest, reply, done) => {
request.requestId = randomUUID();
reply.header(REQUEST_ID_HEADER, request.requestId);
reply.header(INSTANCE_ID_HEADER, INSTANCE_ID);
done();
});
}
+6
View File
@@ -0,0 +1,6 @@
const UUID_PATTERN =
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
export function validate(value: string): boolean {
return UUID_PATTERN.test(value);
}
+21
View File
@@ -0,0 +1,21 @@
import { ValidationPipe } from '@nestjs/common';
import { NestFastifyApplication } from '@nestjs/platform-fastify';
import { registerUploadBody } from './attachments/upload-body';
import { HttpExceptionFilter } from './common/filters/http-exception.filter';
import { registerRequestId } from './common/interceptors/request-id';
export function configureApp(app: NestFastifyApplication): void {
const instance = app.getHttpAdapter().getInstance();
registerRequestId(instance);
registerUploadBody(instance);
app.useGlobalPipes(
new ValidationPipe({
whitelist: true,
forbidNonWhitelisted: true,
transform: true,
forbidUnknownValues: true,
}),
);
app.useGlobalFilters(new HttpExceptionFilter());
}
+9
View File
@@ -0,0 +1,9 @@
import { Global, Module } from '@nestjs/common';
import { PrismaService } from './prisma.service';
@Global()
@Module({
providers: [PrismaService],
exports: [PrismaService],
})
export class DbModule {}
+49
View File
@@ -0,0 +1,49 @@
import { Injectable, OnModuleDestroy } from '@nestjs/common';
import { Prisma, PrismaClient } from '@prisma/client';
import { readFileSync } from 'fs';
const UNAVAILABLE_CODES = new Set(['P1001', 'P1002', 'P1008', 'P1017', 'P2024']);
@Injectable()
export class PrismaService extends PrismaClient implements OnModuleDestroy {
constructor() {
super({ datasourceUrl: readDatabaseUrl() });
}
async onModuleDestroy(): Promise<void> {
await this.$disconnect();
}
}
function readDatabaseUrl(): string {
const file = process.env.DATABASE_URL_FILE;
const url = file ? readFileSync(file, 'utf8').trim() : process.env.DATABASE_URL;
if (!url) {
throw new Error('DATABASE_URL_FILE or DATABASE_URL must be set');
}
return url;
}
export function isUniqueViolation(error: unknown): boolean {
return (
error instanceof Prisma.PrismaClientKnownRequestError &&
error.code === 'P2002'
);
}
export function isForeignKeyViolation(error: unknown): boolean {
return (
error instanceof Prisma.PrismaClientKnownRequestError &&
error.code === 'P2003'
);
}
export function isDatabaseUnavailable(error: unknown): boolean {
if (error instanceof Prisma.PrismaClientInitializationError) {
return true;
}
return (
error instanceof Prisma.PrismaClientKnownRequestError &&
UNAVAILABLE_CODES.has(error.code)
);
}
+351
View File
@@ -0,0 +1,351 @@
const ALICE_ID = '11111111-1111-4111-8111-111111111111';
const BOB_ID = '22222222-2222-4222-8222-222222222222';
const CHAT_ID = '33333333-3333-4333-8333-333333333333';
const MESSAGE_ID = '44444444-4444-4444-8444-444444444444';
const CURSOR =
'eyJjaGF0SWQiOiIzMzMzMzMzMy0zMzMzLTQzMzMtODMzMy0zMzMzMzMzMzMzMzMiLCJpZCI6IjQ0NDQ0NDQ0LTQ0NDQtNDQ0NC04NDQ0LTQ0NDQ0NDQ0NDQ0NCJ9';
const ALICE = {
id: ALICE_ID,
username: 'alice',
display_name: 'Алиса',
created_at: '2026-09-01T12:00:00.000Z',
};
const CHAT = {
id: CHAT_ID,
title: 'Проект по ВНП',
member_ids: [ALICE_ID, BOB_ID],
created_at: '2026-09-01T12:01:00.000Z',
};
const MESSAGE = {
id: MESSAGE_ID,
chat_id: CHAT_ID,
sender_id: ALICE_ID,
text: 'Привет, Боб! 👋',
created_at: '2026-09-01T12:02:00.000Z',
attachment_ids: [],
};
export interface Example {
summary: string;
value: unknown;
}
export interface OperationExamples {
request?: Record<string, Example>;
responses: Record<string, Record<string, Example>>;
}
function example(summary: string, value: unknown): Example {
return { summary, value };
}
function error(code: string, message: string): unknown {
return {
error: {
code,
message,
request_id: '9b2c6f1e-5d4a-4c3b-8a2f-1e0d9c8b7a6f',
},
};
}
const UNAUTHORIZED = {
missing: example(
'Нет/неверный/истёкший/отозванный Bearer-токен',
error('unauthorized', 'missing or invalid session'),
),
};
const UNAVAILABLE = {
database: example(
'PostgreSQL недоступен',
error('unavailable', 'database unavailable'),
),
};
const CHAT_NOT_FOUND = {
notFound: example(
'Чата нет или вызывающий не участник (не 403)',
error('not_found', 'chat not found'),
),
};
const INVALID_CHAT_ID = example(
'chat_id не UUID',
error('invalid_request', 'chatId must be a valid UUID'),
);
export const GUIDE = `
Учебный мессенджер, лабораторная 6: PostgreSQL, сессии, Redis-кэш сессий, вложения в S3 и их фоновая обработка.
Документ — обязательный контракт \`contracts/openapi.json\`, дополненный примерами запросов и ответов.
**Как пройти сценарий в Swagger UI**
1. \`POST /api/v1/users\` — зарегистрируйте пользователей (примеры alice, bob, eve в списке Examples). Повторная регистрация того же username — 409, меняйте username.
2. **Authorize → basicAuth**: username и password из шага 1 → \`POST /api/v1/auth/sessions\` → скопируйте \`token\`.
3. **Authorize → bearerAuth**: вставьте token (без слова Bearer). Basic отправляется только на вход, остальные запросы идут с Bearer.
4. \`GET /api/v1/users/me\` — свой id; id второго участника — из ответа его регистрации.
5. \`POST /api/v1/chats\` с \`member_ids\` = [свой id, id собеседника] → \`chat_id\`.
6. \`POST /api/v1/chats/{chat_id}/messages\` с заголовком \`Idempotency-Key\` (любая строка из \`A-Za-z0-9._:-\`, до 128 символов). Повтор с тем же ключом и телом вернёт то же сообщение, с другим телом — 409. Затем \`GET …/messages?limit=1\` и следующая страница с \`cursor\` = \`next_cursor\`.
7. \`POST /api/v1/chats/{chat_id}/attachments?filename=photo.png\` с телом PNG/JPEG → 202 \`queued\`. \`GET /api/v1/attachments/{id}\` повторять, пока \`status\` не станет \`ready\` (до этого \`/content\` отвечает 409), затем \`/content?variant=thumbnail\`, \`crop\` или \`optimized\`.
8. Чтобы действовать от другого пользователя: Logout в Authorize, вход им и новый token.
9. \`DELETE /api/v1/auth/sessions/current\` — logout; тот же token дальше даёт 401.
Сессия живёт \`SESSION_TTL_SECONDS\` (по умолчанию 3600 с). Пароль — 12–128 символов Unicode, двоеточие допустимо.
`;
export const OPERATION_EXAMPLES: Record<string, OperationExamples> = {
'GET /health/live': {
responses: {
'200': { ok: example('Процесс жив', { status: 'ok' }) },
},
},
'GET /health/ready': {
responses: {
'200': {
ok: example('PostgreSQL доступен (Redis не критичен)', {
status: 'ok',
}),
},
'503': UNAVAILABLE,
},
},
'POST /api/v1/users': {
request: {
alice: example('Алиса', {
username: 'alice',
display_name: 'Алиса',
password: 'Alice-password-123',
}),
bob: example('Боб', {
username: 'bob',
display_name: 'Боб',
password: 'Bob-password-123',
}),
eve: example('Ева (посторонняя)', {
username: 'eve',
display_name: 'Ева',
password: 'Eve-password-123',
}),
colon: example('Пароль с двоеточием и эмодзи', {
username: 'carol',
display_name: 'Кэрол 🌸',
password: 'пароль:с:двоеточием🔑',
}),
},
responses: {
'201': { created: example('Пользователь создан', ALICE) },
'400': {
username: example(
'username не соответствует [a-z0-9_]{3,32}',
error('invalid_request', 'username must match [a-z0-9_]{3,32}'),
),
password: example(
'password короче 12 символов',
error(
'invalid_request',
'password must be between 12 and 128 Unicode code points',
),
),
extra: example(
'Лишнее поле в JSON',
error('invalid_request', 'property extra should not exist'),
),
},
'409': {
taken: example(
'username уже занят',
error('conflict', 'username is already taken'),
),
},
'503': UNAVAILABLE,
},
},
'POST /api/v1/auth/sessions': {
responses: {
'201': {
created: example('Сессия создана', {
token: 'q8Hn3vXl0ZbJ7mWcY2sRkT5uPaE9dFgVhN1oLxI4w6A',
token_type: 'Bearer',
expires_at: '2026-09-01T13:00:00.000Z',
user: ALICE,
}),
},
'401': {
invalid: example(
'Неверный пароль, неизвестный пользователь или нет Basic — ответ одинаковый',
error('unauthorized', 'invalid credentials'),
),
},
'503': UNAVAILABLE,
},
},
'DELETE /api/v1/auth/sessions/current': {
responses: {
'401': UNAUTHORIZED,
'503': {
...UNAVAILABLE,
cache: example(
'Redis недоступен — отзыв не выполнен, повторите позже',
error('unavailable', 'session cache unavailable'),
),
},
},
},
'GET /api/v1/users/me': {
responses: {
'200': { me: example('Текущий пользователь из сессии', ALICE) },
'401': UNAUTHORIZED,
'503': UNAVAILABLE,
},
},
'POST /api/v1/chats': {
request: {
pair: example('Чат Алисы и Боба', {
title: CHAT.title,
member_ids: CHAT.member_ids,
}),
solo: example('Чат только с собой', {
title: 'Заметки',
member_ids: [ALICE_ID],
}),
},
responses: {
'201': { created: example('Чат создан атомарно', CHAT) },
'400': {
noCaller: example(
'Вызывающий не в member_ids',
error('invalid_request', 'member_ids must include the caller'),
),
unknown: example(
'Неизвестный участник — транзакция откатывается',
error('invalid_request', 'member_ids contain an unknown user'),
),
duplicate: example(
'Дубликаты в member_ids',
error(
'invalid_request',
"All member_ids's elements must be unique",
),
),
},
'401': UNAUTHORIZED,
'503': UNAVAILABLE,
},
},
'GET /api/v1/chats': {
responses: {
'200': {
page: example('Страница чатов с продолжением', {
items: [CHAT],
next_cursor: 'eyJpZCI6IjMzMzMzMzMzLTMzMzMtNDMzMy04MzMzLTMzMzMzMzMzMzMzMyJ9',
}),
last: example('Последняя страница', {
items: [CHAT],
next_cursor: null,
}),
},
'400': {
limit: example(
'limit вне 1–100',
error('invalid_request', 'limit must be between 1 and 100'),
),
cursor: example(
'Испорченный или чужой cursor',
error('invalid_request', 'invalid cursor'),
),
},
'401': UNAUTHORIZED,
'503': UNAVAILABLE,
},
},
'GET /api/v1/chats/{chat_id}': {
responses: {
'200': { chat: example('Чат, где вызывающий — участник', CHAT) },
'400': { chatId: INVALID_CHAT_ID },
'401': UNAUTHORIZED,
'404': CHAT_NOT_FOUND,
'503': UNAVAILABLE,
},
},
'POST /api/v1/chats/{chat_id}/messages': {
request: {
text: example('Обычное сообщение', { text: MESSAGE.text }),
long: example('Максимум — 4000 символов Unicode', {
text: 'Длинный текст…',
}),
},
responses: {
'201': {
created: example('Отправитель берётся из сессии', MESSAGE),
},
'400': {
blank: example(
'Пустой или пробельный текст',
error(
'invalid_request',
'text must not be empty or whitespace-only',
),
),
sender: example(
'sender_id в теле не принимается',
error('invalid_request', 'property sender_id should not exist'),
),
chatId: INVALID_CHAT_ID,
key: example(
'Нет Idempotency-Key или неверный формат',
error(
'invalid_request',
'Idempotency-Key header must match [A-Za-z0-9._:-]{1,128}',
),
),
},
'401': UNAUTHORIZED,
'404': CHAT_NOT_FOUND,
'409': {
keyReused: example(
'Тот же Idempotency-Key с другим телом',
error(
'conflict',
'Idempotency-Key was already used with a different request',
),
),
},
'503': UNAVAILABLE,
},
},
'GET /api/v1/chats/{chat_id}/messages': {
responses: {
'200': {
page: example('Страница истории (limit=1) с cursor', {
items: [MESSAGE],
next_cursor: CURSOR,
}),
last: example('Последняя страница', {
items: [MESSAGE],
next_cursor: null,
}),
},
'400': {
otherChat: example(
'cursor от другого чата',
error('invalid_request', 'cursor belongs to a different chat'),
),
cursor: example(
'Испорченный cursor или cursor несуществующего сообщения',
error('invalid_request', 'invalid cursor'),
),
limit: example(
'limit вне 1–100',
error('invalid_request', 'limit must be between 1 and 100'),
),
},
'401': UNAUTHORIZED,
'404': CHAT_NOT_FOUND,
'503': UNAVAILABLE,
},
},
};
+43
View File
@@ -0,0 +1,43 @@
import { NestFastifyApplication } from '@nestjs/platform-fastify';
import { OpenAPIObject, SwaggerModule } from '@nestjs/swagger';
import { readFileSync } from 'fs';
import { join } from 'path';
import { GUIDE, OPERATION_EXAMPLES } from './examples';
const CONTRACT_PATH = join(process.cwd(), 'contracts', 'openapi.json');
export function setupSwagger(app: NestFastifyApplication): void {
SwaggerModule.setup('docs', app, loadDocument(), {
customSiteTitle: 'Messenger lab6 API',
swaggerOptions: {
persistAuthorization: true,
displayRequestDuration: true,
tryItOutEnabled: true,
},
});
}
export function loadDocument(): OpenAPIObject {
const document = JSON.parse(readFileSync(CONTRACT_PATH, 'utf8'));
document.info.description = GUIDE;
document.servers = [{ url: '/', description: 'Этот экземпляр API' }];
for (const [key, examples] of Object.entries(OPERATION_EXAMPLES)) {
const [method, path] = key.split(' ');
const operation = document.paths[path]?.[method.toLowerCase()];
if (!operation) {
throw new Error(`examples for unknown operation ${key}`);
}
if (examples.request) {
operation.requestBody.content['application/json'].examples =
examples.request;
}
for (const [status, named] of Object.entries(examples.responses)) {
const content = operation.responses[status]?.content;
if (!content) {
throw new Error(`examples for unknown response ${key} ${status}`);
}
content['application/json'].examples = named;
}
}
return document;
}
+44
View File
@@ -0,0 +1,44 @@
import { Controller, Get } from '@nestjs/common';
import { ApiError } from '../common/filters/api-error';
import { PrismaService } from '../db/prisma.service';
import { S3Service } from '../storage/s3.service';
import { isDraining } from './shutdown';
@Controller('health')
export class HealthController {
constructor(
private readonly prisma: PrismaService,
private readonly s3: S3Service,
) {}
@Get('live')
live(): { status: 'ok' } {
return { status: 'ok' };
}
@Get('core')
async core(): Promise<{ status: 'ok' }> {
await this.assertCoreReady();
return { status: 'ok' };
}
@Get('ready')
async ready(): Promise<{ status: 'ok' }> {
await this.assertCoreReady();
if (!(await this.s3.isAvailable())) {
throw new ApiError('unavailable', 'object storage unavailable');
}
return { status: 'ok' };
}
private async assertCoreReady(): Promise<void> {
if (isDraining()) {
throw new ApiError('unavailable', 'shutting down');
}
try {
await this.prisma.$queryRaw`SELECT 1`;
} catch {
throw new ApiError('unavailable', 'database unavailable');
}
}
}
+7
View File
@@ -0,0 +1,7 @@
import { Module } from '@nestjs/common';
import { HealthController } from './health.controller';
@Module({
controllers: [HealthController],
})
export class HealthModule {}
+26
View File
@@ -0,0 +1,26 @@
import { INestApplication, Logger } from '@nestjs/common';
let draining = false;
export function isDraining(): boolean {
return draining;
}
export function enableGracefulShutdown(app: INestApplication): void {
const delayMs = Number(process.env.SHUTDOWN_DELAY_MS ?? 3000);
const shutdown = (signal: string): void => {
if (draining) {
return;
}
draining = true;
Logger.log(`${signal}: readiness → 503, closing in ${delayMs} ms`, 'Shutdown');
setTimeout(() => {
app.close().then(
() => process.exit(0),
() => process.exit(1),
);
}, delayMs);
};
process.once('SIGTERM', () => shutdown('SIGTERM'));
process.once('SIGINT', () => shutdown('SIGINT'));
}
+69
View File
@@ -0,0 +1,69 @@
import { validate as isUuid } from '../common/uuid';
export const PIPELINE_VERSION = 1;
export const JOB_TYPE = 'attachment.process.v1';
export interface JobMessage {
schema_version: 1;
job_id: string;
type: typeof JOB_TYPE;
attachment_id: string;
pipeline_version: number;
attempt: number;
created_at: string;
}
const FIELDS = [
'schema_version',
'job_id',
'type',
'attachment_id',
'pipeline_version',
'attempt',
'created_at',
];
export function jobMessage(
job: { id: string; attachmentId: string; pipelineVersion: number; attempt: number },
createdAt: Date,
): JobMessage {
return {
schema_version: 1,
job_id: job.id,
type: JOB_TYPE,
attachment_id: job.attachmentId,
pipeline_version: job.pipelineVersion,
attempt: job.attempt,
created_at: createdAt.toISOString(),
};
}
export function parseJobMessage(raw: Buffer): JobMessage | null {
let value: any;
try {
value = JSON.parse(raw.toString('utf8'));
} catch {
return null;
}
if (
typeof value !== 'object' ||
value === null ||
Array.isArray(value) ||
Object.keys(value).some((key) => !FIELDS.includes(key)) ||
value.schema_version !== 1 ||
value.type !== JOB_TYPE ||
typeof value.job_id !== 'string' ||
!isUuid(value.job_id) ||
typeof value.attachment_id !== 'string' ||
!isUuid(value.attachment_id) ||
!Number.isInteger(value.pipeline_version) ||
value.pipeline_version < 1 ||
!Number.isInteger(value.attempt) ||
value.attempt < 1 ||
typeof value.created_at !== 'string' ||
Number.isNaN(Date.parse(value.created_at))
) {
return null;
}
return value as JobMessage;
}
+257
View File
@@ -0,0 +1,257 @@
import { AttachmentJob, Prisma, PrismaClient } from '@prisma/client';
import { randomUUID } from 'crypto';
import { jobMessage, PIPELINE_VERSION } from './job-message';
type Tx = Prisma.TransactionClient;
export const RETRY_EXHAUSTED = 'retry_exhausted';
export function maxAttempts(): number {
return Number(process.env.JOB_MAX_ATTEMPTS ?? 5);
}
export function backoffMs(attempt: number, random = Math.random): number {
const base = Number(process.env.JOB_RETRY_BASE_MS ?? 1000);
const cap = Number(process.env.JOB_RETRY_MAX_MS ?? 30000);
const delay = Math.min(cap, base * 2 ** (attempt - 1));
return Math.round(delay / 2 + random() * (delay / 2));
}
export async function enqueueJob(
tx: Tx,
attachmentId: string,
now: Date,
): Promise<AttachmentJob> {
const job = await tx.attachmentJob.create({
data: {
id: randomUUID(),
attachmentId,
pipelineVersion: PIPELINE_VERSION,
status: 'queued',
attempt: 1,
maxAttempts: maxAttempts(),
nextAttemptAt: now,
createdAt: now,
updatedAt: now,
},
});
await tx.jobEvent.create({
data: { jobId: job.id, attempt: 1, event: 'enqueued', createdAt: now },
});
await publishLater(tx, job, now);
return job;
}
export async function scheduleRetry(
tx: Tx,
job: AttachmentJob,
reason: string,
now: Date,
event = 'retry_scheduled',
): Promise<AttachmentJob> {
const delay = backoffMs(job.attempt);
const next = await tx.attachmentJob.update({
where: { id: job.id },
data: {
status: 'queued',
attempt: job.attempt + 1,
nextAttemptAt: new Date(now.getTime() + delay),
leaseOwner: null,
leaseUntil: null,
lastError: reason,
},
});
await tx.attachment.update({
where: { id: job.attachmentId },
data: { status: 'queued' },
});
await tx.jobEvent.create({
data: {
jobId: job.id,
attempt: job.attempt,
event,
detail: `${reason}; next attempt ${next.attempt} in ${delay} ms`,
createdAt: now,
},
});
await publishLater(tx, next, next.nextAttemptAt);
return next;
}
export async function markFailed(
tx: Tx,
job: AttachmentJob,
errorCode: string,
reason: string,
now: Date,
): Promise<void> {
await tx.attachmentJob.update({
where: { id: job.id },
data: {
status: 'failed',
errorCode,
lastError: reason,
leaseOwner: null,
leaseUntil: null,
finishedAt: now,
},
});
await tx.attachment.update({
where: { id: job.attachmentId },
data: { status: 'failed', errorCode },
});
await tx.jobEvent.create({
data: {
jobId: job.id,
attempt: job.attempt,
event: 'failed',
detail: `${errorCode}: ${reason}`,
createdAt: now,
},
});
}
export async function retryOrFail(
tx: Tx,
job: AttachmentJob,
reason: string,
now: Date,
event?: string,
): Promise<'retry' | 'failed'> {
if (job.attempt >= job.maxAttempts) {
await markFailed(tx, job, RETRY_EXHAUSTED, reason, now);
return 'failed';
}
await scheduleRetry(tx, job, reason, now, event);
return 'retry';
}
export async function recoverExpiredLeases(
prisma: PrismaClient,
now = new Date(),
limit = 100,
): Promise<number> {
const expired = await prisma.attachmentJob.findMany({
where: { status: 'processing', leaseUntil: { lt: now } },
orderBy: { leaseUntil: 'asc' },
take: limit,
});
let recovered = 0;
for (const candidate of expired) {
await prisma.$transaction(async (tx) => {
const taken = await tx.attachmentJob.updateMany({
where: {
id: candidate.id,
status: 'processing',
leaseOwner: candidate.leaseOwner,
leaseUntil: { lt: now },
},
data: { leaseOwner: null, leaseUntil: null, status: 'queued' },
});
if (taken.count === 0) {
return;
}
await retryOrFail(tx, candidate, 'lease expired', now, 'lease_expired');
recovered++;
});
}
return recovered;
}
export async function republishStale(
prisma: PrismaClient,
staleMs: number,
now = new Date(),
limit = 100,
): Promise<number> {
const cutoff = new Date(now.getTime() - staleMs);
const stale = await prisma.$queryRaw<AttachmentJob[]>`
SELECT j.id, j.attachment_id AS "attachmentId", j.pipeline_version AS "pipelineVersion", j.attempt
FROM attachment_jobs j
WHERE j.status = 'queued'
AND j.next_attempt_at < ${cutoff}
AND NOT EXISTS (
SELECT 1 FROM outbox o
WHERE o.job_id = j.id AND (o.published_at IS NULL OR o.published_at > ${cutoff})
)
ORDER BY j.next_attempt_at
LIMIT ${limit}`;
for (const job of stale) {
await prisma.$transaction(async (tx) => {
await tx.jobEvent.create({
data: { jobId: job.id, attempt: job.attempt, event: 'republished', createdAt: now },
});
await publishLater(tx, job, now);
});
}
return stale.length;
}
export type ReplayResult =
| 'replayed'
| 'already_ready'
| 'not_failed'
| 'source_missing'
| 'not_found';
export async function replayJob(
prisma: PrismaClient,
jobId: string,
actor: string,
now = new Date(),
): Promise<ReplayResult> {
return prisma.$transaction(async (tx) => {
const job = await tx.attachmentJob.findUnique({ where: { id: jobId } });
if (!job) {
return 'not_found';
}
if (job.status === 'ready') {
return 'already_ready';
}
if (job.status !== 'failed') {
return 'not_failed';
}
const source = await tx.attachment.updateMany({
where: { id: job.attachmentId, status: 'failed', stagingKey: { not: null } },
data: { status: 'queued', errorCode: null },
});
if (source.count === 0) {
return 'source_missing';
}
const taken = await tx.attachmentJob.updateMany({
where: { id: jobId, status: 'failed' },
data: {
status: 'queued',
attempt: 1,
errorCode: null,
nextAttemptAt: now,
finishedAt: null,
},
});
if (taken.count === 0) {
throw new Error('job changed during replay');
}
await tx.jobEvent.create({
data: {
jobId,
attempt: 1,
event: 'replayed',
detail: `by ${actor}; previous ${job.errorCode} after attempt ${job.attempt}`,
createdAt: now,
},
});
await publishLater(tx, { ...job, attempt: 1 }, now);
return 'replayed';
});
}
async function publishLater(tx: Tx, job: AttachmentJob, availableAt: Date): Promise<void> {
await tx.outboxEntry.create({
data: {
jobId: job.id,
payload: jobMessage(job, availableAt) as unknown as Prisma.InputJsonObject,
availableAt,
createdAt: new Date(),
},
});
}
+273
View File
@@ -0,0 +1,273 @@
import { Logger } from '@nestjs/common';
import { Attachment, AttachmentJob, PrismaClient } from '@prisma/client';
import { createHash, randomUUID } from 'crypto';
import {
ObjectMissingError,
ObjectTooLargeError,
S3Service,
} from '../storage/s3.service';
import { JobMessage, PIPELINE_VERSION } from './job-message';
import { markFailed, retryOrFail } from './job-store';
import {
InvalidImageError,
RenderedVariant,
RenderTimeoutError,
renderVariants,
} from './variants';
export type Outcome = 'ready' | 'retry' | 'failed' | 'skipped';
export interface ProcessorOptions {
timeoutMs: number;
leaseMs: number;
maxBytes: number;
maxPixels: number;
}
export function processorOptions(): ProcessorOptions {
const timeoutMs = Number(process.env.JOB_TIMEOUT_SECONDS ?? 60) * 1000;
return {
timeoutMs,
leaseMs: Number(process.env.JOB_LEASE_SECONDS ?? timeoutMs / 1000 + 30) * 1000,
maxBytes: Number(process.env.MAX_UPLOAD_BYTES ?? 10485760),
maxPixels: Number(process.env.MAX_IMAGE_PIXELS ?? 40000000),
};
}
export interface ProcessorHooks {
afterStore?: () => Promise<void>;
afterCommit?: () => Promise<void>;
}
class PermanentError extends Error {
constructor(
readonly code: string,
reason: string,
) {
super(reason);
}
}
class TimeoutError extends Error {}
export class JobProcessor {
private readonly logger = new Logger('JobProcessor');
constructor(
private readonly prisma: PrismaClient,
private readonly s3: S3Service,
private readonly options: ProcessorOptions = processorOptions(),
private readonly hooks: ProcessorHooks = {},
) {}
async handle(message: JobMessage): Promise<Outcome> {
const job = await this.prisma.attachmentJob.findUnique({
where: { id: message.job_id },
include: { attachment: true },
});
if (
!job ||
job.attachmentId !== message.attachment_id ||
job.pipelineVersion !== message.pipeline_version ||
job.pipelineVersion !== PIPELINE_VERSION
) {
this.logger.warn(`job ${message.job_id}: unknown or mismatched, dropped`);
return 'skipped';
}
const lease = await this.claim(job);
if (!lease) {
if (job.status === 'ready') {
await this.removeStaging(job.attachment);
}
this.logger.log(`job ${job.id}: ${job.status}, not due or leased elsewhere, skipped`);
return 'skipped';
}
let variants: RenderedVariant[];
try {
variants = await this.withTimeout(this.produce(job.attachment));
} catch (error) {
return this.recordFailure(lease, error);
}
await this.hooks.afterStore?.();
return this.commit(lease, job.attachment, variants);
}
private async claim(job: AttachmentJob): Promise<AttachmentJob | null> {
const now = new Date();
const owner = randomUUID();
return this.prisma.$transaction(async (tx) => {
const taken = await tx.attachmentJob.updateMany({
where: { id: job.id, status: 'queued', nextAttemptAt: { lte: now } },
data: {
status: 'processing',
leaseOwner: owner,
leaseUntil: new Date(now.getTime() + this.options.leaseMs),
},
});
if (taken.count === 0) {
return null;
}
await tx.attachment.update({
where: { id: job.attachmentId },
data: { status: 'processing' },
});
const leased = await tx.attachmentJob.findUniqueOrThrow({ where: { id: job.id } });
await tx.jobEvent.create({
data: { jobId: job.id, attempt: leased.attempt, event: 'processing', createdAt: now },
});
return leased;
});
}
private async produce(attachment: Attachment): Promise<RenderedVariant[]> {
if (!attachment.stagingKey) {
throw new PermanentError('staging_missing', 'staging object was already removed');
}
let original: Buffer;
try {
original = await this.s3.getBuffer(attachment.stagingKey, this.options.maxBytes);
} catch (error) {
if (error instanceof ObjectMissingError) {
throw new PermanentError('staging_missing', 'staging object not found');
}
if (error instanceof ObjectTooLargeError) {
throw new PermanentError('invalid_image', 'staging object exceeds size limit');
}
throw error;
}
const sha256 = createHash('sha256').update(original).digest('hex');
if (original.length !== attachment.sizeBytes || sha256 !== attachment.sha256) {
throw new PermanentError('checksum_mismatch', 'staging object differs from accepted upload');
}
let variants: RenderedVariant[];
try {
variants = await renderVariants(
original,
this.options.maxPixels,
Math.ceil(this.options.timeoutMs / 1000),
);
} catch (error) {
if (error instanceof InvalidImageError) {
throw new PermanentError('invalid_image', 'image cannot be decoded');
}
throw error;
}
await this.s3.putBuffer(attachment.objectKey, original, attachment.contentType, sha256);
for (const variant of variants) {
await this.s3.putBuffer(
variantKey(attachment.id, variant),
variant.body,
variant.contentType,
variant.sha256,
);
}
return variants;
}
private async commit(
lease: AttachmentJob,
attachment: Attachment,
variants: RenderedVariant[],
): Promise<Outcome> {
const now = new Date();
const committed = await this.prisma.$transaction(async (tx) => {
const owned = await tx.attachmentJob.updateMany({
where: { id: lease.id, status: 'processing', leaseOwner: lease.leaseOwner },
data: {
status: 'ready',
leaseOwner: null,
leaseUntil: null,
lastError: null,
finishedAt: now,
},
});
if (owned.count === 0) {
return false;
}
await tx.attachmentVariant.createMany({
data: variants.map((variant) => ({
attachmentId: attachment.id,
pipelineVersion: PIPELINE_VERSION,
name: variant.name,
contentType: variant.contentType,
objectKey: variantKey(attachment.id, variant),
sizeBytes: variant.body.length,
width: variant.width,
height: variant.height,
sha256: variant.sha256,
createdAt: now,
})),
skipDuplicates: true,
});
await tx.attachment.update({
where: { id: attachment.id },
data: { status: 'ready', errorCode: null },
});
await tx.jobEvent.create({
data: { jobId: lease.id, attempt: lease.attempt, event: 'ready', createdAt: now },
});
return true;
});
if (!committed) {
this.logger.warn(`job ${lease.id}: lease lost before commit, result left to owner`);
return 'skipped';
}
await this.hooks.afterCommit?.();
await this.removeStaging(attachment);
return 'ready';
}
private async recordFailure(lease: AttachmentJob, error: unknown): Promise<Outcome> {
const now = new Date();
const reason =
error instanceof PermanentError
? error.message
: error instanceof TimeoutError || error instanceof RenderTimeoutError
? 'processing timed out'
: `temporary failure: ${error instanceof Error ? error.name : 'unknown'}`;
this.logger.warn(`job ${lease.id} attempt ${lease.attempt}: ${reason}`);
return this.prisma.$transaction(async (tx) => {
const owned = await tx.attachmentJob.count({
where: { id: lease.id, status: 'processing', leaseOwner: lease.leaseOwner },
});
if (owned === 0) {
return 'skipped' as const;
}
if (error instanceof PermanentError) {
await markFailed(tx, lease, error.code, reason, now);
return 'failed' as const;
}
return retryOrFail(tx, lease, reason, now);
});
}
private async removeStaging(attachment: Attachment): Promise<void> {
if (!attachment.stagingKey) {
return;
}
try {
await this.s3.delete(attachment.stagingKey);
await this.prisma.attachment.update({
where: { id: attachment.id },
data: { stagingKey: null },
});
} catch {
this.logger.warn(`staging cleanup postponed for ${attachment.id}`);
}
}
private withTimeout<T>(work: Promise<T>): Promise<T> {
let timer: NodeJS.Timeout;
const deadline = new Promise<never>((_, reject) => {
timer = setTimeout(() => reject(new TimeoutError()), this.options.timeoutMs);
});
return Promise.race([work, deadline]).finally(() => clearTimeout(timer));
}
}
export function variantKey(
attachmentId: string,
variant: Pick<RenderedVariant, 'name' | 'extension'>,
): string {
return `attachments/${attachmentId}/v${PIPELINE_VERSION}/${variant.name}.${variant.extension}`;
}
+111
View File
@@ -0,0 +1,111 @@
import { createHash } from 'crypto';
import sharp, { Sharp } from 'sharp';
sharp.cache(false);
sharp.concurrency(1);
const WHITE = { r: 255, g: 255, b: 255 };
const JPEG_QUALITY = 82;
const WEBP_QUALITY = 80;
export interface RenderedVariant {
name: 'thumbnail' | 'crop' | 'optimized';
contentType: string;
extension: string;
body: Buffer;
width: number;
height: number;
sha256: string;
}
export class InvalidImageError extends Error {
constructor(reason: string) {
super(reason);
this.name = 'InvalidImageError';
}
}
export class RenderTimeoutError extends Error {
constructor() {
super('image processing exceeded its time limit');
this.name = 'RenderTimeoutError';
}
}
export function fitInside(
width: number,
height: number,
box: number,
): { width: number; height: number } {
if (width <= box && height <= box) {
return { width, height };
}
if (width >= height) {
return { width: box, height: Math.max(1, Math.floor((height * box) / width)) };
}
return { width: Math.max(1, Math.floor((width * box) / height)), height: box };
}
export async function renderVariants(
original: Buffer,
maxPixels: number,
timeoutSeconds = 0,
): Promise<RenderedVariant[]> {
const open = (): Sharp =>
sharp(original, { failOn: 'error', limitInputPixels: maxPixels })
.timeout({ seconds: timeoutSeconds })
.rotate();
try {
const meta = await sharp(original, { limitInputPixels: maxPixels }).metadata();
if (!meta.width || !meta.height) {
throw new InvalidImageError('image has no dimensions');
}
const swapped = (meta.orientation ?? 1) >= 5;
const width = swapped ? meta.height : meta.width;
const height = swapped ? meta.width : meta.height;
const thumb = fitInside(width, height, 256);
const large = fitInside(width, height, 1280);
const thumbnail = open()
.resize(thumb.width, thumb.height, { fit: 'fill' })
.flatten({ background: WHITE })
.jpeg({ quality: JPEG_QUALITY });
const crop = open()
.resize(128, 128, { fit: 'cover', position: 'centre' })
.flatten({ background: WHITE })
.jpeg({ quality: JPEG_QUALITY });
const optimized = open()
.resize(large.width, large.height, { fit: 'fill' })
.webp({ quality: WEBP_QUALITY, effort: 4 });
return [
await finish('thumbnail', 'image/jpeg', 'jpg', thumbnail),
await finish('crop', 'image/jpeg', 'jpg', crop),
await finish('optimized', 'image/webp', 'webp', optimized),
];
} catch (error) {
if (error instanceof InvalidImageError) {
throw error;
}
if (error instanceof Error && /timeout/i.test(error.message)) {
throw new RenderTimeoutError();
}
throw new InvalidImageError(error instanceof Error ? error.message : 'decode failed');
}
}
async function finish(
name: RenderedVariant['name'],
contentType: string,
extension: string,
pipeline: Sharp,
): Promise<RenderedVariant> {
const { data, info } = await pipeline.toBuffer({ resolveWithObject: true });
return {
name,
contentType,
extension,
body: data,
width: info.width,
height: info.height,
sha256: createHash('sha256').update(data).digest('hex'),
};
}
+34
View File
@@ -0,0 +1,34 @@
import { Logger } from '@nestjs/common';
import { NestFactory } from '@nestjs/core';
import {
FastifyAdapter,
NestFastifyApplication,
} from '@nestjs/platform-fastify';
import { AppModule } from './app.module';
import { configureApp } from './configure-app';
import { setupSwagger } from './docs/swagger';
import { enableGracefulShutdown } from './health/shutdown';
async function bootstrap(): Promise<void> {
const app = await NestFactory.create<NestFastifyApplication>(
AppModule,
new FastifyAdapter(),
{
bufferLogs: true,
},
);
configureApp(app);
setupSwagger(app);
enableGracefulShutdown(app);
const host = process.env.APP_HOST ?? '0.0.0.0';
const port = Number(process.env.APP_PORT ?? 8080);
await app.listen(port, host);
Logger.log(`Listening on http://${host}:${port}`, 'Bootstrap');
}
bootstrap().catch((error) => {
Logger.error('Fatal error during bootstrap', error);
process.exit(1);
});
+41
View File
@@ -0,0 +1,41 @@
import {
ArrayMaxSize,
IsArray,
IsString,
Validate,
ValidateIf,
ValidatorConstraint,
ValidatorConstraintInterface,
} from 'class-validator';
import { CodePointLength } from '../../common/dto/code-point-length.validator';
import { validate as isUuid } from '../../common/uuid';
@ValidatorConstraint({ name: 'uniqueUuidList' })
class UniqueUuidList implements ValidatorConstraintInterface {
validate(value: unknown): boolean {
if (
!Array.isArray(value) ||
!value.every((id) => typeof id === 'string' && isUuid(id))
) {
return false;
}
const normalized = (value as string[]).map((id) => id.toLowerCase());
return new Set(normalized).size === normalized.length;
}
defaultMessage(): string {
return 'attachment_ids must contain unique UUIDs';
}
}
export class CreateMessageDto {
@IsString()
@CodePointLength(0, 4000)
text!: string;
@ValidateIf((dto: CreateMessageDto) => dto.attachment_ids !== undefined)
@IsArray()
@ArrayMaxSize(10)
@Validate(UniqueUuidList)
attachment_ids?: string[];
}
+26
View File
@@ -0,0 +1,26 @@
import { Message } from '../../common/domain';
export interface MessageResponse {
id: string;
chat_id: string;
sender_id: string;
text: string;
created_at: string;
attachment_ids: string[];
}
export interface MessagePageResponse {
items: MessageResponse[];
next_cursor: string | null;
}
export function toMessageResponse(message: Message): MessageResponse {
return {
id: message.id,
chat_id: message.chatId,
sender_id: message.senderId,
text: message.text,
created_at: message.createdAt,
attachment_ids: message.attachmentIds,
};
}
+70
View File
@@ -0,0 +1,70 @@
import {
Body,
Controller,
Get,
Headers,
HttpCode,
Param,
Post,
Query,
UseGuards,
} from '@nestjs/common';
import { AuthGuard } from '../auth/auth.guard';
import { CurrentUser } from '../auth/current-user.decorator';
import { parseLimit, rawCursorParam } from '../common/dto/pagination-query.dto';
import { User } from '../common/domain';
import { ApiError } from '../common/filters/api-error';
import { CreateMessageDto } from './dto/create-message.dto';
import {
MessagePageResponse,
MessageResponse,
toMessageResponse,
} from './dto/message.response';
import { MessagesService } from './messages.service';
@Controller('api/v1/chats/:chatId/messages')
@UseGuards(AuthGuard)
export class MessagesController {
constructor(private readonly messages: MessagesService) {}
@Post()
@HttpCode(201)
async create(
@Param('chatId') chatId: string,
@Body() dto: CreateMessageDto,
@Headers('idempotency-key') keyRaw: unknown,
@CurrentUser() user: User,
): Promise<MessageResponse> {
const key = parseIdempotencyKey(keyRaw);
const message = await this.messages.create(
chatId,
user.id,
dto.text,
(dto.attachment_ids ?? []).map((id) => id.toLowerCase()),
key,
);
return toMessageResponse(message);
}
@Get()
async list(
@Param('chatId') chatId: string,
@CurrentUser() user: User,
@Query('limit') limitRaw: unknown,
@Query('cursor') cursorRaw: unknown,
): Promise<MessagePageResponse> {
const limit = parseLimit(limitRaw);
const cursor = rawCursorParam(cursorRaw);
return this.messages.list(chatId, user.id, limit, cursor);
}
}
function parseIdempotencyKey(raw: unknown): string {
if (typeof raw !== 'string' || !/^[A-Za-z0-9._:-]{1,128}$/.test(raw)) {
throw new ApiError(
'invalid_request',
'Idempotency-Key header must match [A-Za-z0-9._:-]{1,128}',
);
}
return raw;
}
+9
View File
@@ -0,0 +1,9 @@
import { Module } from '@nestjs/common';
import { MessagesController } from './messages.controller';
import { MessagesService } from './messages.service';
@Module({
controllers: [MessagesController],
providers: [MessagesService],
})
export class MessagesModule {}
+198
View File
@@ -0,0 +1,198 @@
import { Injectable } from '@nestjs/common';
import { Prisma } from '@prisma/client';
import { createHash } from 'crypto';
import { assertChatMember } from '../chats/chat-access';
import { codePointLength } from '../common/dto/code-point-length.validator';
import { decodeMessageCursor, encodeMessageCursor } from '../common/cursor';
import { Message } from '../common/domain';
import { ApiError } from '../common/filters/api-error';
import { PrismaService } from '../db/prisma.service';
import { MessagePageResponse, toMessageResponse } from './dto/message.response';
interface MessageRow {
id: string;
chatId: string;
senderId: string;
text: string;
createdAt: Date;
attachmentIds: string[];
}
const ATTACHMENT_LINKS = {
attachments: {
orderBy: { position: 'asc' },
select: { attachmentId: true },
},
} satisfies Prisma.MessageInclude;
type MessageWithLinks = Prisma.MessageGetPayload<{
include: typeof ATTACHMENT_LINKS;
}>;
class KeyAlreadyUsed extends Error {}
@Injectable()
export class MessagesService {
private readonly keyTtlMs =
Number(process.env.IDEMPOTENCY_TTL_SECONDS ?? 86400) * 1000;
constructor(private readonly prisma: PrismaService) {}
async create(
chatId: string,
callerId: string,
text: string,
attachmentIds: string[],
key: string,
): Promise<Message> {
await this.assertMember(chatId, callerId);
if (codePointLength(text.trim()) === 0 && attachmentIds.length === 0) {
throw new ApiError(
'invalid_request',
'text must not be empty or whitespace-only without attachments',
);
}
await this.assertAttachmentsInChat(chatId, attachmentIds);
const scope = { userId: callerId, chatId, key };
const requestHash = createHash('sha256')
.update(JSON.stringify({ text, attachment_ids: attachmentIds }))
.digest('hex');
try {
return await this.prisma.$transaction(async (tx) => {
await tx.idempotencyKey.deleteMany({
where: {
...scope,
createdAt: { lt: new Date(Date.now() - this.keyTtlMs) },
},
});
const row = await tx.message.create({
data: {
chatId,
senderId: callerId,
text,
attachments: {
create: attachmentIds.map((attachmentId, position) => ({
attachmentId,
position,
})),
},
},
include: ATTACHMENT_LINKS,
});
const { count } = await tx.idempotencyKey.createMany({
data: [{ ...scope, requestHash, messageId: row.id }],
skipDuplicates: true,
});
if (count === 0) {
throw new KeyAlreadyUsed();
}
return linkedMessage(row);
});
} catch (error) {
if (!(error instanceof KeyAlreadyUsed)) {
throw error;
}
}
const stored = await this.prisma.idempotencyKey.findUnique({
where: { userId_chatId_key: scope },
include: { message: { include: ATTACHMENT_LINKS } },
});
if (!stored || stored.requestHash !== requestHash) {
throw new ApiError(
'conflict',
'Idempotency-Key was already used with a different request',
);
}
return linkedMessage(stored.message);
}
async list(
chatId: string,
callerId: string,
limit: number,
rawCursor: string | null,
): Promise<MessagePageResponse> {
await this.assertMember(chatId, callerId);
let after = Prisma.empty;
if (rawCursor !== null) {
const cursor = decodeMessageCursor(rawCursor);
if (cursor && cursor.chatId !== chatId) {
throw new ApiError(
'invalid_request',
'cursor belongs to a different chat',
);
}
const anchor = cursor
? await this.prisma.message.findFirst({
where: { id: cursor.id, chatId },
select: { id: true, createdAt: true },
})
: null;
if (!anchor) {
throw new ApiError('invalid_request', 'invalid cursor');
}
after = Prisma.sql`AND (created_at, id) > (${anchor.createdAt}::timestamptz, ${anchor.id}::uuid)`;
}
const rows = await this.prisma.$queryRaw<MessageRow[]>`
SELECT id, chat_id AS "chatId", sender_id AS "senderId", text, created_at AS "createdAt",
ARRAY(
SELECT attachment_id::text FROM message_attachments
WHERE message_id = messages.id
ORDER BY position
) AS "attachmentIds"
FROM messages
WHERE chat_id = ${chatId}::uuid ${after}
ORDER BY created_at, id
LIMIT ${limit + 1}`;
const items = rows.slice(0, limit).map(toMessage);
const last = items[items.length - 1];
return {
items: items.map(toMessageResponse),
next_cursor:
rows.length > limit
? encodeMessageCursor({ chatId, id: last.id })
: null,
};
}
private assertMember(chatId: string, callerId: string): Promise<void> {
return assertChatMember(this.prisma, chatId, callerId);
}
private async assertAttachmentsInChat(
chatId: string,
attachmentIds: string[],
): Promise<void> {
if (attachmentIds.length === 0) {
return;
}
const found = await this.prisma.attachment.findMany({
where: { id: { in: attachmentIds }, chatId },
select: { status: true },
});
if (found.length !== attachmentIds.length) {
throw new ApiError('not_found', 'attachment not found');
}
if (found.some((row) => row.status === 'failed')) {
throw new ApiError('conflict', 'failed attachment cannot be attached');
}
}
}
function linkedMessage(row: MessageWithLinks): Message {
return toMessage({
...row,
attachmentIds: row.attachments.map((link) => link.attachmentId),
});
}
function toMessage(row: MessageRow): Message {
return {
id: row.id,
chatId: row.chatId,
senderId: row.senderId,
text: row.text,
createdAt: row.createdAt.toISOString(),
attachmentIds: row.attachmentIds,
};
}
+57
View File
@@ -0,0 +1,57 @@
import { Logger } from '@nestjs/common';
import { reconcile } from './attachments/reconciler';
import { PrismaService } from './db/prisma.service';
import { S3Service } from './storage/s3.service';
const DEFAULT_GRACE_SECONDS = 3600;
const DEFAULT_INTERVAL_SECONDS = 300;
const DEFAULT_STAGING_RETENTION_SECONDS = 86400;
function option(name: string, fallback: number): number {
const prefix = `--${name}=`;
const raw = process.argv.find((arg) => arg.startsWith(prefix));
const value = raw ? Number(raw.slice(prefix.length)) : fallback;
if (!Number.isInteger(value) || value < 0) {
throw new Error(`--${name} must be a non-negative integer`);
}
return value;
}
async function main(): Promise<void> {
process.once('SIGTERM', () => process.exit(0));
process.once('SIGINT', () => process.exit(0));
const apply = process.argv.includes('--apply');
const loop = process.argv.includes('--loop');
const graceSeconds = option('grace-seconds', DEFAULT_GRACE_SECONDS);
const intervalSeconds = option('interval-seconds', DEFAULT_INTERVAL_SECONDS);
const stagingRetentionSeconds = option(
'staging-retention-seconds',
Number(process.env.STAGING_RETENTION_SECONDS ?? DEFAULT_STAGING_RETENTION_SECONDS),
);
const prisma = new PrismaService();
const s3 = new S3Service();
do {
try {
const report = await reconcile(prisma, s3, {
apply,
graceMs: graceSeconds * 1000,
stagingRetentionMs: stagingRetentionSeconds * 1000,
});
process.stdout.write(`${JSON.stringify(report)}\n`);
} catch (error) {
const reason = error instanceof Error ? error.name : 'unknown error';
Logger.error(`reconcile pass aborted: ${reason}`, 'Reconciler');
process.exitCode = loop ? 0 : 1;
}
if (loop) {
await new Promise((resolve) => setTimeout(resolve, intervalSeconds * 1000));
}
} while (loop);
await prisma.$disconnect();
s3.onModuleDestroy();
}
main().catch((error) => {
Logger.error(error instanceof Error ? error.message : 'fatal error', 'Reconciler');
process.exit(1);
});
+129
View File
@@ -0,0 +1,129 @@
import { Logger } from '@nestjs/common';
import { ConfirmChannel } from 'amqplib';
import { assertTopology, connectWithRetry, JOB_QUEUE, sleep } from './broker/broker';
import { PrismaService } from './db/prisma.service';
import { JobMessage } from './jobs/job-message';
import { recoverExpiredLeases, republishStale } from './jobs/job-store';
const BATCH_SIZE = 100;
const POLL_MS = Number(process.env.RELAY_POLL_MS ?? 500);
const SWEEP_MS = Number(process.env.RELAY_SWEEP_MS ?? 5000);
const STALE_MS = Number(process.env.JOB_STALE_SECONDS ?? 300) * 1000;
const FAULT = process.env.RELAY_FAULT;
const logger = new Logger('OutboxRelay');
const prisma = new PrismaService();
let stopping = false;
async function publishBatch(channel: ConfirmChannel): Promise<number> {
return prisma.$transaction(
async (tx) => {
const rows = await tx.$queryRaw<{ id: bigint; payload: JobMessage }[]>`
SELECT id, payload FROM outbox
WHERE published_at IS NULL AND available_at <= ${new Date()}
ORDER BY id
LIMIT ${BATCH_SIZE}
FOR UPDATE SKIP LOCKED`;
if (rows.length === 0) {
return 0;
}
for (const { payload } of rows) {
channel.sendToQueue(JOB_QUEUE, Buffer.from(JSON.stringify(payload)), {
persistent: true,
contentType: 'application/json',
messageId: payload.job_id,
type: payload.type,
});
}
await channel.waitForConfirms();
logger.log(`published ${rows.length} job message(s), broker confirmed`);
if (FAULT === 'exit-after-confirm') {
logger.warn('fault injection: exit after broker confirm, before outbox is marked');
process.exit(137);
}
await tx.outboxEntry.updateMany({
where: { id: { in: rows.map((row) => row.id) } },
data: { publishedAt: new Date(), publishAttempts: { increment: 1 } },
});
return rows.length;
},
{ timeout: 15000 },
);
}
async function sweep(): Promise<void> {
const recovered = await recoverExpiredLeases(prisma);
const republished = await republishStale(prisma, STALE_MS);
if (recovered + republished > 0) {
logger.log(`lease expired: ${recovered}, stale queued republished: ${republished}`);
}
}
async function run(): Promise<void> {
let lastSweep = 0;
while (!stopping) {
const connection = await connectWithRetry(logger, () => stopping);
if (!connection) {
break;
}
let closed = false;
connection.on('close', () => {
closed = true;
});
connection.on('error', () => {
closed = true;
});
try {
const channel = await connection.createConfirmChannel();
channel.on('error', () => {
closed = true;
});
channel.on('close', () => {
closed = true;
});
await assertTopology(channel);
while (!stopping && !closed) {
let published = 0;
try {
published = await publishBatch(channel);
if (Date.now() - lastSweep >= SWEEP_MS) {
await sweep();
lastSweep = Date.now();
}
} catch (error) {
const reason = error instanceof Error ? error.name : 'unknown error';
logger.error(`relay pass failed: ${reason}`);
await sleep(1000);
}
if (published < BATCH_SIZE) {
await sleep(POLL_MS);
}
}
} catch (error) {
const reason = error instanceof Error ? error.name : 'unknown error';
logger.error(`broker channel failed: ${reason}`);
}
await connection.close().catch(() => undefined);
if (!stopping) {
logger.warn('broker connection lost, reconnecting');
await sleep(1000);
}
}
}
for (const signal of ['SIGTERM', 'SIGINT'] as const) {
process.once(signal, () => {
stopping = true;
});
}
run()
.then(async () => {
await prisma.$disconnect();
logger.log('stopped');
process.exit(0);
})
.catch((error) => {
logger.error(error instanceof Error ? error.message : 'fatal error');
process.exit(1);
});
+54
View File
@@ -0,0 +1,54 @@
import { Logger } from '@nestjs/common';
import { validate as isUuid } from './common/uuid';
import { PrismaService } from './db/prisma.service';
import { PIPELINE_VERSION } from './jobs/job-message';
import { replayJob } from './jobs/job-store';
const USAGE = 'usage: replay.js --list | <job_id|attachment_id>...';
async function main(): Promise<void> {
const args = process.argv.slice(2);
if (args.length === 0 || (args[0] !== '--list' && args.some((arg) => !isUuid(arg)))) {
throw new Error(USAGE);
}
const prisma = new PrismaService();
try {
if (args[0] === '--list') {
const failed = await prisma.attachmentJob.findMany({
where: { status: 'failed' },
orderBy: { finishedAt: 'asc' },
take: 100,
});
for (const job of failed) {
process.stdout.write(
`${JSON.stringify({
job_id: job.id,
attachment_id: job.attachmentId,
error_code: job.errorCode,
attempt: job.attempt,
last_error: job.lastError,
finished_at: job.finishedAt?.toISOString(),
})}\n`,
);
}
return;
}
const actor = process.env.OPERATOR ?? 'operator';
for (const id of args) {
const job = await prisma.attachmentJob.findFirst({
where: {
OR: [{ id }, { attachmentId: id, pipelineVersion: PIPELINE_VERSION }],
},
});
const result = job ? await replayJob(prisma, job.id, actor) : 'not_found';
process.stdout.write(`${JSON.stringify({ id, job_id: job?.id ?? null, result })}\n`);
}
} finally {
await prisma.$disconnect();
}
}
main().catch((error) => {
Logger.error(error instanceof Error ? error.message : 'fatal error', 'Replay');
process.exit(1);
});
+245
View File
@@ -0,0 +1,245 @@
import {
DeleteObjectCommand,
GetObjectCommand,
HeadBucketCommand,
ListObjectsV2Command,
PutObjectCommand,
S3Client,
} from '@aws-sdk/client-s3';
import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common';
import { createReadStream, readFileSync } from 'fs';
import { Readable } from 'stream';
import { ApiError } from '../common/filters/api-error';
const CONNECT_TIMEOUT_MS = 1000;
const IDLE_TIMEOUT_MS = 5000;
const OPERATION_TIMEOUT_MS = 20000;
const HEALTH_TIMEOUT_MS = 1000;
export interface ObjectPage {
objects: { key: string; lastModified: Date }[];
nextToken?: string;
}
export interface StoredFile {
key: string;
path: string;
size: number;
contentType: string;
sha256: string;
}
@Injectable()
export class S3Service implements OnModuleDestroy {
private readonly logger = new Logger('S3Service');
private readonly client: S3Client;
private readonly streamClient: S3Client;
private readonly bucket = process.env.S3_BUCKET ?? 'messenger-attachments';
constructor() {
const config = {
endpoint: process.env.S3_ENDPOINT ?? 'http://s3:9000',
region: process.env.S3_REGION ?? 'us-east-1',
forcePathStyle: true,
maxAttempts: 1,
credentials: {
accessKeyId: readSecret('S3_ACCESS_KEY_ID'),
secretAccessKey: readSecret('S3_SECRET_ACCESS_KEY'),
},
};
this.client = new S3Client({
...config,
requestHandler: {
connectionTimeout: CONNECT_TIMEOUT_MS,
requestTimeout: IDLE_TIMEOUT_MS,
},
});
this.streamClient = new S3Client({
...config,
requestHandler: { connectionTimeout: CONNECT_TIMEOUT_MS },
});
}
async putFile(file: StoredFile): Promise<void> {
await this.guard(`put ${file.key}`, () =>
this.client.send(
new PutObjectCommand({
Bucket: this.bucket,
Key: file.key,
Body: createReadStream(file.path),
ContentLength: file.size,
ContentType: file.contentType,
ChecksumSHA256: Buffer.from(file.sha256, 'hex').toString('base64'),
}),
{ abortSignal: AbortSignal.timeout(OPERATION_TIMEOUT_MS) },
),
);
}
async putBuffer(
key: string,
body: Buffer,
contentType: string,
sha256: string,
): Promise<void> {
await this.guard(`put ${key}`, () =>
this.client.send(
new PutObjectCommand({
Bucket: this.bucket,
Key: key,
Body: body,
ContentLength: body.length,
ContentType: contentType,
ChecksumSHA256: Buffer.from(sha256, 'hex').toString('base64'),
}),
{ abortSignal: AbortSignal.timeout(OPERATION_TIMEOUT_MS) },
),
);
}
async getBuffer(key: string, maxBytes: number): Promise<Buffer> {
let body: Readable;
try {
const response = await this.client.send(
new GetObjectCommand({ Bucket: this.bucket, Key: key }),
{ abortSignal: AbortSignal.timeout(OPERATION_TIMEOUT_MS) },
);
if ((response.ContentLength ?? 0) > maxBytes) {
throw new ObjectTooLargeError(key);
}
body = response.Body as Readable;
} catch (error) {
if (error instanceof ObjectTooLargeError) {
throw error;
}
const name = error instanceof Error ? error.name : 'unknown error';
this.logger.error(`get ${key} failed: ${name}`);
if (name === 'NoSuchKey') {
throw new ObjectMissingError(key);
}
throw new ApiError('unavailable', 'object storage unavailable');
}
const chunks: Buffer[] = [];
let size = 0;
try {
for await (const chunk of body) {
size += chunk.length;
if (size > maxBytes) {
body.destroy();
throw new ObjectTooLargeError(key);
}
chunks.push(chunk);
}
} catch (error) {
if (error instanceof ObjectTooLargeError) {
throw error;
}
throw new ApiError('unavailable', 'object storage unavailable');
}
return Buffer.concat(chunks);
}
async getStream(key: string): Promise<Readable> {
const deadline = new AbortController();
const timer = setTimeout(() => deadline.abort(), IDLE_TIMEOUT_MS);
try {
const { Body } = await this.streamClient.send(
new GetObjectCommand({ Bucket: this.bucket, Key: key }),
{ abortSignal: deadline.signal },
);
return Body as Readable;
} catch (error) {
const name = error instanceof Error ? error.name : 'unknown error';
this.logger.error(`get ${key} failed: ${name}`);
if (name === 'NoSuchKey') {
throw new Error(`object ${key} is missing from storage`);
}
throw new ApiError('unavailable', 'object storage unavailable');
} finally {
clearTimeout(timer);
}
}
async isAvailable(): Promise<boolean> {
try {
await this.client.send(new HeadBucketCommand({ Bucket: this.bucket }), {
abortSignal: AbortSignal.timeout(HEALTH_TIMEOUT_MS),
});
return true;
} catch {
return false;
}
}
async listObjects(
prefix: string,
pageSize: number,
token?: string,
): Promise<ObjectPage> {
const page = await this.guard(`list ${prefix}`, () =>
this.client.send(
new ListObjectsV2Command({
Bucket: this.bucket,
Prefix: prefix,
MaxKeys: pageSize,
ContinuationToken: token,
}),
{ abortSignal: AbortSignal.timeout(OPERATION_TIMEOUT_MS) },
),
);
return {
objects: (page.Contents ?? []).map((object) => ({
key: object.Key!,
lastModified: object.LastModified!,
})),
nextToken: page.NextContinuationToken,
};
}
async delete(key: string): Promise<void> {
await this.guard(`delete ${key}`, () =>
this.client.send(
new DeleteObjectCommand({ Bucket: this.bucket, Key: key }),
{ abortSignal: AbortSignal.timeout(OPERATION_TIMEOUT_MS) },
),
);
}
onModuleDestroy(): void {
this.client.destroy();
this.streamClient.destroy();
}
private async guard<T>(action: string, run: () => Promise<T>): Promise<T> {
try {
return await run();
} catch (error) {
const reason = error instanceof Error ? error.name : 'unknown error';
this.logger.error(`${action} failed: ${reason}`);
throw new ApiError('unavailable', 'object storage unavailable');
}
}
}
export class ObjectMissingError extends Error {
constructor(key: string) {
super(`object ${key} is missing from storage`);
this.name = 'ObjectMissingError';
}
}
export class ObjectTooLargeError extends Error {
constructor(key: string) {
super(`object ${key} exceeds size limit`);
this.name = 'ObjectTooLargeError';
}
}
function readSecret(name: string): string {
const file = process.env[`${name}_FILE`];
const value = file ? readFileSync(file, 'utf8').trim() : process.env[name];
if (!value) {
throw new Error(`${name}_FILE or ${name} must be set`);
}
return value;
}
+9
View File
@@ -0,0 +1,9 @@
import { Global, Module } from '@nestjs/common';
import { S3Service } from './s3.service';
@Global()
@Module({
providers: [S3Service],
exports: [S3Service],
})
export class StorageModule {}
+18
View File
@@ -0,0 +1,18 @@
import { Matches } from 'class-validator';
import { CodePointLength } from '../../common/dto/code-point-length.validator';
import { NotBlank } from '../../common/dto/not-blank.validator';
export class CreateUserDto {
@Matches(/^[a-z0-9_]{3,32}$/, {
message: 'username must match [a-z0-9_]{3,32}',
})
username!: string;
@NotBlank()
@CodePointLength(1, 100)
display_name!: string;
@NotBlank()
@CodePointLength(12, 128)
password!: string;
}
+17
View File
@@ -0,0 +1,17 @@
import { User } from '../../common/domain';
export interface UserResponse {
id: string;
username: string;
display_name: string;
created_at: string;
}
export function toUserResponse(user: User): UserResponse {
return {
id: user.id,
username: user.username,
display_name: user.displayName,
created_at: user.createdAt,
};
}
+36
View File
@@ -0,0 +1,36 @@
import {
Body,
Controller,
Get,
HttpCode,
Post,
UseGuards,
} from '@nestjs/common';
import { AuthGuard } from '../auth/auth.guard';
import { CurrentUser } from '../auth/current-user.decorator';
import { User } from '../common/domain';
import { CreateUserDto } from './dto/create-user.dto';
import { toUserResponse, UserResponse } from './dto/user.response';
import { UsersService } from './users.service';
@Controller('api/v1/users')
export class UsersController {
constructor(private readonly users: UsersService) {}
@Post()
@HttpCode(201)
async create(@Body() dto: CreateUserDto): Promise<UserResponse> {
const user = await this.users.register(
dto.username,
dto.display_name,
dto.password,
);
return toUserResponse(user);
}
@Get('me')
@UseGuards(AuthGuard)
me(@CurrentUser() user: User): UserResponse {
return toUserResponse(user);
}
}
+10
View File
@@ -0,0 +1,10 @@
import { Module } from '@nestjs/common';
import { UsersController } from './users.controller';
import { UsersService } from './users.service';
@Module({
controllers: [UsersController],
providers: [UsersService],
exports: [UsersService],
})
export class UsersModule {}
+48
View File
@@ -0,0 +1,48 @@
import { Injectable } from '@nestjs/common';
import { hashPassword } from '../auth/password';
import { ApiError } from '../common/filters/api-error';
import { User } from '../common/domain';
import { PrismaService, isUniqueViolation } from '../db/prisma.service';
@Injectable()
export class UsersService {
constructor(private readonly prisma: PrismaService) {}
async register(
username: string,
displayName: string,
password: string,
): Promise<User> {
const passwordHash = await hashPassword(password);
try {
const row = await this.prisma.user.create({
data: { username, displayName: displayName.trim(), passwordHash },
});
return toUser(row);
} catch (error) {
if (isUniqueViolation(error)) {
throw new ApiError('conflict', 'username is already taken');
}
throw error;
}
}
async findById(id: string): Promise<User | null> {
const row = await this.prisma.user.findUnique({ where: { id } });
return row ? toUser(row) : null;
}
}
export function toUser(row: {
id: string;
username: string;
displayName: string;
createdAt: Date;
}): User {
return {
id: row.id,
username: row.username,
displayName: row.displayName,
createdAt: row.createdAt.toISOString(),
};
}
+137
View File
@@ -0,0 +1,137 @@
import { Logger } from '@nestjs/common';
import { Channel, ConsumeMessage } from 'amqplib';
import { assertTopology, connectWithRetry, JOB_QUEUE, sleep } from './broker/broker';
import { PrismaService } from './db/prisma.service';
import { parseJobMessage } from './jobs/job-message';
import { JobProcessor, ProcessorHooks, processorOptions } from './jobs/processor';
import { S3Service } from './storage/s3.service';
const CONCURRENCY = Number(process.env.WORKER_CONCURRENCY ?? 2);
const INFRA_RETRY_MS = Number(process.env.WORKER_INFRA_RETRY_MS ?? 5000);
const DRAIN_TIMEOUT_MS = Number(process.env.WORKER_DRAIN_TIMEOUT_MS ?? 25000);
const FAULT = process.env.WORKER_FAULT;
const logger = new Logger('Worker');
const prisma = new PrismaService();
const s3 = new S3Service();
const processor = new JobProcessor(prisma, s3, processorOptions(), faultHooks());
let stopping = false;
let inFlight = 0;
function faultHooks(): ProcessorHooks {
const crash = (point: string) => async () => {
logger.warn(`fault injection: worker killed ${point}`);
process.exit(137);
};
if (FAULT === 'exit-after-store') {
return { afterStore: crash('after storing objects, before commit and ack') };
}
if (FAULT === 'exit-after-commit') {
return { afterCommit: crash('after commit, before ack') };
}
return {};
}
async function handle(channel: Channel, delivery: ConsumeMessage): Promise<void> {
inFlight++;
try {
const message = parseJobMessage(delivery.content);
if (!message) {
logger.warn('malformed job message moved to dead letter queue');
channel.nack(delivery, false, false);
return;
}
const outcome = await processor.handle(message);
logger.log(
`job ${message.job_id} attempt ${message.attempt}` +
`${delivery.fields.redelivered ? ' (redelivered)' : ''}: ${outcome}`,
);
channel.ack(delivery);
} catch (error) {
const reason = error instanceof Error ? error.name : 'unknown error';
logger.error(`job handling interrupted (${reason}), returning message after ${INFRA_RETRY_MS} ms`);
await sleep(INFRA_RETRY_MS);
try {
channel.nack(delivery, false, true);
} catch {
logger.warn('channel closed, broker will redeliver unacked message');
}
} finally {
inFlight--;
}
}
async function run(): Promise<void> {
while (!stopping) {
const connection = await connectWithRetry(logger, () => stopping);
if (!connection) {
break;
}
const closed = new Promise<void>((resolve) => {
connection.on('close', () => resolve());
connection.on('error', () => resolve());
});
try {
const channel = await connection.createChannel();
await assertTopology(channel);
await channel.prefetch(CONCURRENCY);
const { consumerTag } = await channel.consume(
JOB_QUEUE,
(delivery) => {
if (delivery) {
void handle(channel, delivery);
}
},
{ noAck: false },
);
logger.log(`consuming ${JOB_QUEUE} with prefetch ${CONCURRENCY}`);
const stop = new Promise<void>((resolve) => {
const timer = setInterval(() => {
if (stopping) {
clearInterval(timer);
resolve();
}
}, 200);
void closed.then(() => {
clearInterval(timer);
resolve();
});
});
await stop;
if (stopping) {
await channel.cancel(consumerTag).catch(() => undefined);
const deadline = Date.now() + DRAIN_TIMEOUT_MS;
while (inFlight > 0 && Date.now() < deadline) {
await sleep(100);
}
logger.log(`drained, ${inFlight} job(s) left unacked for redelivery`);
}
} catch (error) {
const reason = error instanceof Error ? error.name : 'unknown error';
logger.error(`broker channel failed: ${reason}`);
}
await connection.close().catch(() => undefined);
if (!stopping) {
logger.warn('broker connection lost, reconnecting');
await sleep(1000);
}
}
}
for (const signal of ['SIGTERM', 'SIGINT'] as const) {
process.once(signal, () => {
stopping = true;
});
}
run()
.then(async () => {
await prisma.$disconnect();
s3.onModuleDestroy();
logger.log('stopped');
process.exit(0);
})
.catch((error) => {
logger.error(error instanceof Error ? error.message : 'fatal error');
process.exit(1);
});