lab4: вложения (proxy upload, download, уборщик), attachment_ids в сообщениях

This commit is contained in:
2026-10-22 22:28:54 +03:00
parent ec4e9fb608
commit 6fcc623447
23 changed files with 959 additions and 106 deletions
+1
View File
@@ -23,6 +23,7 @@
"graph": "npm run build && DATABASE_URL=postgresql://graph-only.invalid/messenger REDIS_URL=redis://graph-only.invalid:6379 tsx scripts/graph-modules.ts",
"test:e2e": "vitest run --config ./vitest.config.e2e.ts",
"test:dependencies": "E2E_DEPENDENCY_FAULTS=1 vitest run --config ./vitest.config.e2e.ts dependency-failures",
"attachments:sweep": "tsx src/attachments/attachments-sweep.cli.ts",
"idempotency:gc": "tsx src/messages/idempotency/idempotency-gc.cli.ts"
},
"dependencies": {
+2
View File
@@ -5,6 +5,7 @@ import { RedisModule } from './redis/redis.module.js';
import { AuthModule } from './auth/auth.module.js';
import { ChatsModule } from './chats/chats.module.js';
import { HealthModule } from './health/health.module.js';
import { AttachmentsModule } from './attachments/attachments.module.js';
import { MessagesModule } from './messages/messages.module.js';
import { MetricsModule } from './metrics/metrics.module.js';
import { UsersModule } from './users/users.module.js';
@@ -21,6 +22,7 @@ import { ApiExceptionFilter } from './common/errors/api-exception.filter.js';
HealthModule,
ChatsModule,
MessagesModule,
AttachmentsModule,
MetricsModule,
],
providers: [
@@ -0,0 +1,78 @@
import {
Injectable,
Logger,
OnModuleDestroy,
OnModuleInit,
} from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import type { AppConfig } from '../config/app-config.schema.js';
import { MetricsService } from '../metrics/metrics.service.js';
import { ObjectStorage } from '../storage/object-storage.js';
import { AttachmentsRepository } from './attachments.repository.js';
const SWEEP_BATCH = 100;
/**
* Removes unfinished uploads (pending past expiry + grace) and rows whose
* deletion was claimed but never finished. A live upload is never touched:
* its expiry is a hard request deadline and the grace period is far longer.
*/
@Injectable()
export class AttachmentSweeper implements OnModuleInit, OnModuleDestroy {
private readonly logger = new Logger(AttachmentSweeper.name);
private timer: NodeJS.Timeout | undefined;
private running = false;
constructor(
private readonly config: ConfigService<AppConfig>,
private readonly repository: AttachmentsRepository,
private readonly storage: ObjectStorage,
private readonly metrics: MetricsService,
) {}
onModuleInit(): void {
const interval = this.config.getOrThrow('ATTACHMENT_SWEEP_INTERVAL_MS');
if (interval <= 0) return;
this.timer = setInterval(() => {
if (this.running) return;
this.running = true;
this.sweepOnce()
.catch((error: unknown) => {
this.logger.warn(
`Sweep failed: ${error instanceof Error ? error.message : 'unknown error'}`,
);
})
.finally(() => {
this.running = false;
});
}, interval);
this.timer.unref();
}
onModuleDestroy(): void {
if (this.timer) clearInterval(this.timer);
this.timer = undefined;
}
async sweepOnce(): Promise<{ claimed: number; deleted: number }> {
const claimed = await this.repository.claimExpired(
this.config.getOrThrow('ATTACHMENT_SWEEP_GRACE_SECONDS'),
SWEEP_BATCH,
);
let deleted = 0;
for (const item of claimed) {
try {
await this.storage.delete(item.object_key);
await this.repository.deleteClaimed(item.id);
deleted += 1;
this.metrics.increment('attachments.sweep.deleted');
} catch (error) {
this.metrics.increment('attachments.sweep.error');
this.logger.warn(
`Sweep of attachment ${item.id} failed: ${error instanceof Error ? error.message : 'unknown error'}`,
);
}
}
return { claimed: claimed.length, deleted };
}
}
+49
View File
@@ -0,0 +1,49 @@
export type Attachment = {
id: string;
chat_id: string;
owner_id: string;
filename: string;
content_type: 'image/png' | 'image/jpeg';
size_bytes: number;
sha256: string;
status: 'ready';
variants: never[];
error_code: null;
created_at: string;
updated_at: string;
};
export type AttachmentRow = {
id: string;
chat_id: string;
owner_id: string;
filename: string;
content_type: 'image/png' | 'image/jpeg';
upload_mode: 'proxy' | 'direct';
status: 'pending' | 'ready' | 'deleting';
object_key: string;
size_bytes: string | number | null;
sha256: string | null;
width: number | null;
height: number | null;
expires_at: Date | null;
created_at: Date;
updated_at: Date;
};
export function toAttachment(row: AttachmentRow): Attachment {
return {
id: row.id,
chat_id: row.chat_id,
owner_id: row.owner_id,
filename: row.filename,
content_type: row.content_type,
size_bytes: Number(row.size_bytes),
sha256: row.sha256 as string,
status: 'ready',
variants: [],
error_code: null,
created_at: row.created_at.toISOString(),
updated_at: row.updated_at.toISOString(),
};
}
@@ -0,0 +1,42 @@
import { fileURLToPath } from 'node:url';
import { parseAppConfig } from '../config/app-config.schema.js';
import { DatabaseClient } from '../database/database.client.js';
import { MetricsService } from '../metrics/metrics.service.js';
import { ObjectStorage } from '../storage/object-storage.js';
import { AttachmentSweeper } from './attachment-sweeper.service.js';
import { AttachmentsRepository } from './attachments.repository.js';
// Manual operator pass over unfinished uploads (the API also sweeps in the
// background). Never touches rows before expiry + ATTACHMENT_SWEEP_GRACE_SECONDS.
// in the stand: docker compose exec api node dist/attachments/attachments-sweep.cli.js
// on the host: npm run attachments:sweep (needs reachable DATABASE_URL and S3)
async function main(): Promise<void> {
const config = parseAppConfig(process.env as Record<string, unknown>);
const configService = {
getOrThrow: (key: keyof typeof config) => config[key],
} as never;
const metrics = new MetricsService();
const database = new DatabaseClient(configService, metrics);
try {
const sweeper = new AttachmentSweeper(
configService,
new AttachmentsRepository(database),
new ObjectStorage(configService),
metrics,
);
let total = 0;
for (;;) {
const { claimed, deleted } = await sweeper.sweepOnce();
total += deleted;
// A pass that claimed rows but could not delete any would loop forever.
if (claimed === 0 || deleted === 0) break;
}
process.stdout.write(
`Swept ${total} attachment(s) older than ${config.ATTACHMENT_SWEEP_GRACE_SECONDS} seconds past expiry\n`,
);
} finally {
await database.onModuleDestroy();
}
}
if (process.argv[1] === fileURLToPath(import.meta.url)) await main();
@@ -0,0 +1,89 @@
import {
Controller,
Get,
Headers,
HttpCode,
Param,
ParseUUIDPipe,
Post,
Query,
Req,
Res,
StreamableFile,
UseGuards,
} from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import type { FastifyReply, FastifyRequest } from 'fastify';
import type { Readable } from 'node:stream';
import { CurrentUser } from '../common/auth/current-user.decorator.js';
import { CurrentUserGuard } from '../common/auth/current-user.guard.js';
import type { AppConfig } from '../config/app-config.schema.js';
import type { User } from '../users/user.js';
import { AttachmentsService } from './attachments.service.js';
import { contentDisposition } from './filename.js';
@Controller('api/v1')
@UseGuards(CurrentUserGuard)
export class AttachmentsController {
private readonly maxUploadBytes: number;
constructor(
private readonly attachments: AttachmentsService,
config: ConfigService<AppConfig>,
) {
this.maxUploadBytes = config.getOrThrow('MAX_UPLOAD_BYTES');
}
@Post('chats/:chatId/attachments')
@HttpCode(201)
upload(
@Param('chatId', new ParseUUIDPipe()) chatId: string,
@CurrentUser() user: User,
@Query('filename') filename: unknown,
@Headers('content-type') contentType: string | undefined,
@Headers('content-length') contentLength: string | undefined,
@Req() request: FastifyRequest,
@Res({ passthrough: true }) reply: FastifyReply,
) {
// A gigantic body is not worth draining: close the connection after the answer.
if (Number(contentLength) > 2 * this.maxUploadBytes)
reply.header('Connection', 'close');
return this.attachments.uploadProxy(
chatId,
user,
filename,
contentType,
contentLength,
request.body as Readable | undefined,
);
}
@Get('attachments/:attachmentId')
metadata(
@Param('attachmentId', new ParseUUIDPipe()) attachmentId: string,
@CurrentUser() user: User,
) {
return this.attachments.getMetadata(attachmentId, user);
}
@Get('attachments/:attachmentId/content')
async content(
@Param('attachmentId', new ParseUUIDPipe()) attachmentId: string,
@CurrentUser() user: User,
@Res({ passthrough: true }) reply: FastifyReply,
@Query('variant') variant?: string,
) {
const { row, stream } = await this.attachments.openContent(
attachmentId,
user,
variant,
);
reply.header('X-Content-Type-Options', 'nosniff');
reply.header('Cache-Control', 'private, no-store');
return new StreamableFile(stream, {
type: row.content_type,
length: Number(row.size_bytes),
disposition: contentDisposition(row.filename),
});
}
}
+26
View File
@@ -0,0 +1,26 @@
import { Module } from '@nestjs/common';
import { DatabaseModule } from '../database/database.module.js';
import { ChatsModule } from '../chats/chats.module.js';
import { UsersModule } from '../users/users.module.js';
import { RedisModule } from '../redis/redis.module.js';
import { MetricsModule } from '../metrics/metrics.module.js';
import { StorageModule } from '../storage/storage.module.js';
import { AttachmentSweeper } from './attachment-sweeper.service.js';
import { AttachmentsController } from './attachments.controller.js';
import { AttachmentsRepository } from './attachments.repository.js';
import { AttachmentsService } from './attachments.service.js';
@Module({
imports: [
DatabaseModule,
ChatsModule,
UsersModule,
RedisModule,
MetricsModule,
StorageModule,
],
controllers: [AttachmentsController],
providers: [AttachmentsRepository, AttachmentsService, AttachmentSweeper],
exports: [AttachmentsRepository],
})
export class AttachmentsModule {}
@@ -0,0 +1,156 @@
import { Injectable } from '@nestjs/common';
import { DatabaseClient } from '../database/database.client.js';
import type { AttachmentRow } from './attachment.js';
export type InsertPending = {
id: string;
chatId: string;
ownerId: string;
filename: string;
contentType: string;
uploadMode: 'proxy' | 'direct';
objectKey: string;
sizeBytes: number | null;
sha256: string | null;
ttlMs: number;
};
export type ReadyFields = {
sizeBytes: number;
sha256: string;
width: number;
height: number;
};
const VISIBLE_MEMBER = `JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = $2`;
@Injectable()
export class AttachmentsRepository {
constructor(private readonly database: DatabaseClient) {}
async insertPending(input: InsertPending): Promise<void> {
await this.database.query(
'attachments.insert_pending',
`INSERT INTO attachments
(id, chat_id, owner_id, filename, content_type, upload_mode, status,
object_key, size_bytes, sha256, expires_at)
VALUES ($1, $2, $3, $4, $5, $6, 'pending', $7, $8, $9,
now() + make_interval(secs => $10::double precision / 1000.0))`,
[
input.id,
input.chatId,
input.ownerId,
input.filename,
input.contentType,
input.uploadMode,
input.objectKey,
input.sizeBytes,
input.sha256,
input.ttlMs,
],
);
}
async markReady(
id: string,
fields: ReadyFields,
): Promise<AttachmentRow | undefined> {
const result = await this.database.query<AttachmentRow>(
'attachments.mark_ready',
`UPDATE attachments
SET status = 'ready', size_bytes = $2, sha256 = $3, width = $4,
height = $5, expires_at = NULL, updated_at = now()
WHERE id = $1 AND status = 'pending'
RETURNING *`,
[id, fields.sizeBytes, fields.sha256, fields.width, fields.height],
);
return result.rows[0];
}
async deletePending(id: string): Promise<void> {
await this.database.query(
'attachments.delete_pending',
`DELETE FROM attachments WHERE id = $1 AND status = 'pending'`,
[id],
);
}
async findVisible(
id: string,
userId: string,
): Promise<AttachmentRow | undefined> {
const result = await this.database.query<AttachmentRow>(
'attachments.find_visible',
`SELECT a.* FROM attachments a ${VISIBLE_MEMBER}
WHERE a.id = $1 AND a.status = 'ready'`,
[id, userId],
);
return result.rows[0];
}
async findForComplete(
id: string,
userId: string,
): Promise<AttachmentRow | undefined> {
const result = await this.database.query<AttachmentRow>(
'attachments.find_for_complete',
`SELECT a.* FROM attachments a ${VISIBLE_MEMBER}
WHERE a.id = $1 AND a.upload_mode = 'direct' AND a.owner_id = $2
AND a.status IN ('pending', 'ready')`,
[id, userId],
);
return result.rows[0];
}
async countPendingDirect(ownerId: string): Promise<number> {
const result = await this.database.query<{ count: string }>(
'attachments.count_pending_direct',
`SELECT count(*) AS count FROM attachments
WHERE owner_id = $1 AND status = 'pending' AND upload_mode = 'direct'`,
[ownerId],
);
return Number(result.rows[0].count);
}
async allLinkable(chatId: string, ids: string[]): Promise<boolean> {
const result = await this.database.query<{ count: string }>(
'attachments.all_linkable',
`SELECT count(*) AS count FROM attachments
WHERE chat_id = $1 AND status = 'ready' AND id = ANY($2::uuid[])`,
[chatId, ids],
);
return Number(result.rows[0].count) === ids.length;
}
async claimExpired(
graceSeconds: number,
limit: number,
): Promise<{ id: string; object_key: string }[]> {
const result = await this.database.query<{
id: string;
object_key: string;
}>(
'attachments.claim_expired',
`UPDATE attachments SET status = 'deleting', updated_at = now()
WHERE id IN (
SELECT id FROM attachments
WHERE (status = 'pending'
AND expires_at < now() - make_interval(secs => $1::double precision))
OR (status = 'deleting' AND updated_at < now() - interval '1 minute')
ORDER BY expires_at NULLS FIRST
LIMIT $2
FOR UPDATE SKIP LOCKED)
RETURNING id, object_key`,
[graceSeconds, limit],
);
return result.rows;
}
async deleteClaimed(id: string): Promise<void> {
await this.database.query(
'attachments.delete_claimed',
`DELETE FROM attachments WHERE id = $1 AND status = 'deleting'`,
[id],
);
}
}
+325
View File
@@ -0,0 +1,325 @@
import {
BadRequestException,
HttpException,
Injectable,
InternalServerErrorException,
Logger,
NotFoundException,
PayloadTooLargeException,
ServiceUnavailableException,
UnsupportedMediaTypeException,
} from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { createHash, randomUUID } from 'node:crypto';
import { pipeline, Readable, Transform } from 'node:stream';
import type { AppConfig } from '../config/app-config.schema.js';
import { ChatsService } from '../chats/chats.service.js';
import { MetricsService } from '../metrics/metrics.service.js';
import { ObjectStorage } from '../storage/object-storage.js';
import type { User } from '../users/user.js';
import {
toAttachment,
type Attachment,
type AttachmentRow,
} from './attachment.js';
import { validateFilename } from './filename.js';
import {
IMAGE_HEADER_MAX_BYTES,
ImageHeader,
ImageInspector,
ImageRejectedError,
} from './image/image-inspector.js';
import { AttachmentsRepository } from './attachments.repository.js';
const MEDIA_TYPES = new Set(['image/png', 'image/jpeg']);
export function mediaType(header: unknown): string | undefined {
if (typeof header !== 'string') return undefined;
return header.split(';', 1)[0].trim().toLowerCase();
}
export function imageRejection(error: ImageRejectedError): HttpException {
return error.reason === 'too_many_pixels'
? new PayloadTooLargeException(error.message)
: new UnsupportedMediaTypeException(error.message);
}
class UploadInterruptedError extends Error {}
@Injectable()
export class AttachmentsService {
private readonly logger = new Logger(AttachmentsService.name);
private inFlight = 0;
private readonly maxUploadBytes: number;
private readonly maxPixels: number;
private readonly uploadTimeoutMs: number;
private readonly maxInFlight: number;
constructor(
config: ConfigService<AppConfig>,
private readonly repository: AttachmentsRepository,
private readonly storage: ObjectStorage,
private readonly chats: ChatsService,
private readonly metrics: MetricsService,
) {
this.maxUploadBytes = config.getOrThrow('MAX_UPLOAD_BYTES');
this.maxPixels = config.getOrThrow('MAX_IMAGE_PIXELS');
this.uploadTimeoutMs = config.getOrThrow('UPLOAD_TIMEOUT_MS');
this.maxInFlight = config.getOrThrow('UPLOAD_MAX_IN_FLIGHT');
}
async uploadProxy(
chatId: string,
user: User,
rawFilename: unknown,
contentTypeHeader: unknown,
contentLengthHeader: unknown,
body: Readable | undefined,
): Promise<Attachment> {
const filename = validateFilename(rawFilename);
const declared = mediaType(contentTypeHeader);
if (declared === undefined || !MEDIA_TYPES.has(declared))
throw new UnsupportedMediaTypeException(
'Content-Type must be image/png or image/jpeg',
);
if (
typeof contentLengthHeader !== 'string' ||
!/^\d{1,15}$/u.test(contentLengthHeader)
)
throw new BadRequestException('Content-Length is required');
const contentLength = Number(contentLengthHeader);
if (contentLength === 0 || body === undefined)
throw new UnsupportedMediaTypeException('Image body is empty');
if (contentLength > this.maxUploadBytes)
throw new PayloadTooLargeException('Image is too large');
if (this.inFlight >= this.maxInFlight)
throw new HttpException(
{ message: 'Too many concurrent uploads', retryAfter: 1 },
429,
);
this.inFlight += 1;
try {
await this.chats.getForUser(chatId, user);
return await this.receive(
chatId,
user,
filename,
declared,
contentLength,
body,
);
} finally {
this.inFlight -= 1;
}
}
private async receive(
chatId: string,
user: User,
filename: string,
declared: string,
contentLength: number,
body: Readable,
): Promise<Attachment> {
const controller = new AbortController();
let timedOut = false;
const timer = setTimeout(() => {
timedOut = true;
body.destroy(new Error('upload timeout'));
controller.abort();
}, this.uploadTimeoutMs);
const iterator = body[Symbol.asyncIterator]() as AsyncIterator<Buffer>;
const next = async (): Promise<IteratorResult<Buffer>> => {
try {
return await iterator.next();
} catch {
throw new UploadInterruptedError('Upload was interrupted');
}
};
const inspector = new ImageInspector(this.maxPixels);
const head: Buffer[] = [];
let id: string | undefined;
let key = '';
let rejection: unknown;
try {
// Header phase: nothing exists in the database or S3 yet.
let headBytes = 0;
try {
while (inspector.header === undefined) {
const step = await next();
if (step.done) {
inspector.finish();
throw new ImageRejectedError(
'unsupported_media_type',
'Image header is missing',
);
}
head.push(step.value);
headBytes += step.value.length;
inspector.push(step.value);
if (
inspector.header === undefined &&
headBytes > IMAGE_HEADER_MAX_BYTES
)
throw new ImageRejectedError(
'unsupported_media_type',
'Image header is missing',
);
}
} catch (error) {
throw this.mapUploadError(error, timedOut);
}
const header = inspector.header as ImageHeader;
if (header.contentType !== declared)
throw new UnsupportedMediaTypeException(
'Image content does not match Content-Type',
);
id = randomUUID();
key = this.storage.objectKey(id);
await this.repository.insertPending({
id,
chatId,
ownerId: user.id,
filename,
contentType: declared,
uploadMode: 'proxy',
objectKey: key,
sizeBytes: null,
sha256: null,
ttlMs: this.uploadTimeoutMs,
});
const sha256 = createHash('sha256');
let total = 0;
const account = (chunk: Buffer, inspect: boolean): void => {
total += chunk.length;
if (total > contentLength)
throw new UploadInterruptedError(
'Body exceeds Content-Length',
);
if (inspect) inspector.push(chunk);
sha256.update(chunk);
this.metrics.add('attachments.proxy.bytes_in', chunk.length);
// Validate before the last bytes reach S3: it cannot commit an
// object whose body is incomplete.
if (total === contentLength) inspector.finish();
};
async function* generate(): AsyncGenerator<Buffer> {
for (const chunk of head) {
account(chunk, false);
yield chunk;
}
for (;;) {
const step = await next();
if (step.done) break;
account(step.value, true);
yield step.value;
}
if (total < contentLength)
throw new UploadInterruptedError('Upload was interrupted');
}
const readable = Readable.from(
(async function* () {
try {
yield* generate();
} catch (error) {
rejection = error;
throw error;
}
})(),
);
let claimed = false;
try {
await this.storage.putStream(
key,
readable,
contentLength,
declared,
controller.signal,
);
this.metrics.add(
'attachments.proxy.bytes_to_s3',
contentLength,
);
const row = await this.repository.markReady(id, {
sizeBytes: total,
sha256: sha256.digest('hex'),
width: header.width,
height: header.height,
});
if (row === undefined) {
claimed = true;
throw new ServiceUnavailableException();
}
this.metrics.increment('attachments.upload.created');
return toAttachment(row);
} catch (error) {
if (!claimed) await this.compensate(id, key);
throw this.mapUploadError(rejection ?? error, timedOut);
}
} finally {
clearTimeout(timer);
}
}
private mapUploadError(error: unknown, timedOut: boolean): unknown {
if (error instanceof ImageRejectedError) return imageRejection(error);
if (error instanceof UploadInterruptedError || timedOut)
return new BadRequestException('Upload was interrupted');
return error;
}
private async compensate(id: string, key: string): Promise<void> {
try {
await this.storage.delete(key);
await this.repository.deletePending(id);
this.metrics.increment('attachments.compensated');
} catch (error) {
this.metrics.increment('attachments.compensation_deferred');
this.logger.warn(
`Compensation of attachment ${id} deferred: ${error instanceof Error ? error.message : 'unknown error'}`,
);
}
}
async getMetadata(id: string, user: User): Promise<Attachment> {
const row = await this.repository.findVisible(id, user.id);
if (!row) throw new NotFoundException();
return toAttachment(row);
}
async openContent(
id: string,
user: User,
variant: string | undefined,
): Promise<{ row: AttachmentRow; stream: Readable }> {
if (variant !== undefined && variant !== 'original') {
if (['thumbnail', 'crop', 'optimized'].includes(variant))
throw new NotFoundException();
throw new BadRequestException('variant is invalid');
}
const row = await this.repository.findVisible(id, user.id);
if (!row) throw new NotFoundException();
const source = await this.storage.getStream(row.object_key);
if (source === null) {
this.metrics.increment('attachments.object_missing');
this.logger.error(
`Object for ready attachment ${row.id} is missing in storage`,
);
throw new InternalServerErrorException();
}
const metrics = this.metrics;
const counter = new Transform({
transform(chunk: Buffer, _encoding, callback) {
metrics.add('attachments.download.bytes_out', chunk.length);
callback(null, chunk);
},
});
pipeline(source, counter, () => undefined);
return { row, stream: counter };
}
}
+34
View File
@@ -0,0 +1,34 @@
import { BadRequestException } from '@nestjs/common';
// oxlint-disable-next-line no-control-regex
const FORBIDDEN =
/[/\\\u0000-\u001f\u007f-\u009f\u200e\u200f\u202a-\u202e\u2066-\u2069]/u;
/** The name is stored as-is and used for display only, never in paths or keys. */
export function validateFilename(value: unknown): string {
if (typeof value !== 'string') throw invalid();
const length = [...value].length;
if (
length < 1 ||
length > 255 ||
/^\s*$/u.test(value) ||
value === '.' ||
value === '..' ||
FORBIDDEN.test(value)
)
throw invalid();
return value;
}
function invalid(): BadRequestException {
return new BadRequestException('filename is invalid');
}
export function contentDisposition(filename: string): string {
const fallback = filename.replace(/[^\x20-\x7e]|["\\]/gu, '_');
const encoded = encodeURIComponent(filename).replace(
/['()*]/gu,
(character) => `%${character.charCodeAt(0).toString(16).toUpperCase()}`,
);
return `inline; filename="${fallback}"; filename*=UTF-8''${encoded}`;
}
+3 -18
View File
@@ -82,24 +82,9 @@ export const appConfigSchema = z
undefined,
15000,
),
MAX_UPLOAD_BYTES: numeric(
'MAX_UPLOAD_BYTES',
1,
10485760,
10485760,
),
MAX_IMAGE_PIXELS: numeric(
'MAX_IMAGE_PIXELS',
1,
40000000,
40000000,
),
UPLOAD_TIMEOUT_MS: numeric(
'UPLOAD_TIMEOUT_MS',
1000,
undefined,
25000,
),
MAX_UPLOAD_BYTES: numeric('MAX_UPLOAD_BYTES', 1, 10485760, 10485760),
MAX_IMAGE_PIXELS: numeric('MAX_IMAGE_PIXELS', 1, 40000000, 40000000),
UPLOAD_TIMEOUT_MS: numeric('UPLOAD_TIMEOUT_MS', 1000, undefined, 25000),
UPLOAD_MAX_IN_FLIGHT: numeric('UPLOAD_MAX_IN_FLIGHT', 1, 1024, 32),
// 0 отключает фоновый проход уборщика (остаётся ручной CLI).
ATTACHMENT_SWEEP_INTERVAL_MS: numeric(
+5 -2
View File
@@ -36,12 +36,15 @@ export class DatabaseClient implements OnModuleDestroy {
connectionString: this.config.getOrThrow('DATABASE_URL'),
max: 1,
idleTimeoutMillis: 0,
connectionTimeoutMillis: this.config.getOrThrow('HEALTH_TIMEOUT_MS'),
connectionTimeoutMillis:
this.config.getOrThrow('HEALTH_TIMEOUT_MS'),
query_timeout: this.config.getOrThrow('HEALTH_TIMEOUT_MS'),
});
this.healthPool.on('error', (error: Error) => {
this.metrics.increment('database.health_pool_error');
this.logger.warn(`PostgreSQL health connection error: ${error.message}`);
this.logger.warn(
`PostgreSQL health connection error: ${error.message}`,
);
});
}
+15 -1
View File
@@ -16,6 +16,18 @@ async function bootstrap(): Promise<void> {
AppModule,
new FastifyAdapter({ logger: true }),
);
// Raw image bodies reach the controller as an unread stream: the upload is
// bounded by our own checks, not by Fastify's bodyLimit.
app.getHttpAdapter()
.getInstance()
.addContentTypeParser(
['image/png', 'image/jpeg'],
(
_request: unknown,
payload: unknown,
done: (error: Error | null, body?: unknown) => void,
) => done(null, payload),
);
app.useGlobalInterceptors(
new RequestIdInterceptor(
app.get(HttpAdapterHost),
@@ -50,7 +62,9 @@ async function bootstrap(): Promise<void> {
shuttingDown = true;
health.setDraining();
if (drainDelayMs > 0)
await new Promise<void>((resolve) => setTimeout(resolve, drainDelayMs));
await new Promise<void>((resolve) =>
setTimeout(resolve, drainDelayMs),
);
await app.close();
process.exit(0);
};
+17 -2
View File
@@ -1,8 +1,23 @@
import { IsString } from 'class-validator';
import {
ArrayMaxSize,
ArrayUnique,
IsArray,
IsString,
IsUUID,
ValidateIf,
} from 'class-validator';
import { CodePointLength } from '../../common/validation/code-point-length.js';
export class CreateMessageDto {
@IsString()
@CodePointLength(1, 4000)
@CodePointLength(0, 4000)
text!: string;
// Absent means []; null is a client error.
@ValidateIf((_, value) => value !== undefined)
@IsArray()
@ArrayMaxSize(10)
@ArrayUnique()
@IsUUID('all', { each: true })
attachment_ids: string[] = [];
}
@@ -6,12 +6,17 @@ describe('IdempotencyRepository expired-key cleanup', () => {
const query = vi.fn().mockResolvedValue({ rowCount: 0, rows: [] });
const repository = new IdempotencyRepository({ query } as never);
await repository.forget('user', '/api/v1/chats/chat/messages', 'key', 86400);
await repository.forget(
'user',
'/api/v1/chats/chat/messages',
'key',
86400,
);
expect(query).toHaveBeenCalledWith(
'idempotency.forget',
expect.stringContaining(
'created_at < now() - make_interval(secs => $4::double precision)',
'created_at < now() - make_interval(secs => $4::double precision)',
),
['user', '/api/v1/chats/chat/messages', 'key', 86400],
);
@@ -22,30 +22,30 @@ const PURGE_BATCHES = 10;
export class IdempotencyRepository {
constructor(private readonly database: DatabaseClient) {}
/**
* Reads the stored result and re-checks access in the same round trip: a
* replay must still prove that the caller may read this chat.
*/
async read(
userId: string,
path: string,
key: string,
chatId: string,
ttlSeconds: number,
): Promise<IdempotencyRecord | undefined> {
const result = await this.database.query<IdempotencyRecord>(
'idempotency.read',
`SELECT k.request_fingerprint, k.status_code, k.response_body, k.created_at,
/**
* Reads the stored result and re-checks access in the same round trip: a
* replay must still prove that the caller may read this chat.
*/
async read(
userId: string,
path: string,
key: string,
chatId: string,
ttlSeconds: number,
): Promise<IdempotencyRecord | undefined> {
const result = await this.database.query<IdempotencyRecord>(
'idempotency.read',
`SELECT k.request_fingerprint, k.status_code, k.response_body, k.created_at,
k.created_at < now() - make_interval(secs => $4::double precision) AS expired,
EXISTS (
SELECT 1 FROM chat_members WHERE chat_id = $5 AND user_id = $1
) AS is_member
FROM idempotency_keys k
WHERE k.user_id = $1 AND k.method = 'POST' AND k.path = $2 AND k.key = $3`,
[userId, path, key, ttlSeconds, chatId],
);
return result.rows[0];
}
[userId, path, key, ttlSeconds, chatId],
);
return result.rows[0];
}
async forget(
userId: string,
+11 -3
View File
@@ -1,5 +1,4 @@
import { createHash } from 'node:crypto';
import type { CreateMessageDto } from '../dto/create-message.dto.js';
// Contract: ASCII [A-Za-z0-9._:-]{1,128}. Anything else (including a duplicated
// header, which Fastify joins with ", ") is rejected as a client error.
@@ -11,8 +10,17 @@ export function messagesPath(chatId: string): string {
// The fingerprint is derived from the validated payload with a fixed field
// order, so neither JSON key order nor an absent optional field can change it.
export function idempotencyFingerprint(input: CreateMessageDto): string {
// Text-only messages keep the lab3 fingerprint; the order of attachment ids
// is significant because it is the order of attachments in the message.
export function idempotencyFingerprint(
text: string,
attachmentIds: string[],
): string {
return createHash('sha256')
.update(JSON.stringify({ text: input.text }))
.update(
attachmentIds.length === 0
? JSON.stringify({ text })
: JSON.stringify({ text, attachment_ids: attachmentIds }),
)
.digest('hex');
}
+1
View File
@@ -4,4 +4,5 @@ export type Message = {
sender_id: string;
text: string;
created_at: string;
attachment_ids: string[];
};
+2
View File
@@ -8,6 +8,7 @@ import { MessagesCacheModule } from './cache/messages-cache.module.js';
import { IdempotencyModule } from './idempotency/idempotency.module.js';
import { UsersModule } from '../users/users.module.js';
import { RedisModule } from '../redis/redis.module.js';
import { AttachmentsModule } from '../attachments/attachments.module.js';
import { MetricsModule } from '../metrics/metrics.module.js';
@Module({
@@ -19,6 +20,7 @@ import { MetricsModule } from '../metrics/metrics.module.js';
UsersModule,
RedisModule,
MetricsModule,
AttachmentsModule,
],
controllers: [MessagesController],
providers: [MessagesRepository, MessagesService],
+5 -26
View File
@@ -1,5 +1,4 @@
import { describe, expect, it, vi } from 'vitest';
import type { Cursor } from '../common/http/cursor.js';
import type { Message } from './message.js';
import { MessagesRepository } from './messages.repository.js';
@@ -9,35 +8,15 @@ const message = (id: string, created_at: string): Message => ({
sender_id: 'sender',
text: id,
created_at,
attachment_ids: [],
});
describe('MessagesRepository', () => {
it('uses named SQL history and a keyset cursor', async () => {
const query = vi.fn().mockResolvedValue({ rows: [], rowCount: 0 });
const repository = new MessagesRepository({
query,
withTransaction: vi.fn(),
} as never);
const cursor: Cursor = {
scope: 'messages:chat',
created_at: '2026-01-01T00:00:01.000Z',
id: '1',
};
await repository.findPage('chat', 10, cursor);
expect(query).toHaveBeenCalledWith(
'messages.history',
expect.stringContaining('(created_at, id) >'),
expect.arrayContaining(['chat', cursor.created_at, cursor.id, 11]),
);
});
it('returns rows in response shape', async () => {
const query = vi
.fn()
.mockResolvedValue({
rows: [message('1', '2026-01-01T00:00:00.000Z')],
rowCount: 1,
});
const query = vi.fn().mockResolvedValue({
rows: [message('1', '2026-01-01T00:00:00.000Z')],
rowCount: 1,
});
const repository = new MessagesRepository({ query } as never);
await expect(repository.findPage('chat', 1)).resolves.toEqual({
items: [message('1', '2026-01-01T00:00:00.000Z')],
+19 -3
View File
@@ -10,6 +10,7 @@ type MessageRow = {
sender_id: string;
text: string;
created_at: Date | string;
attachment_ids?: string[] | null;
};
export type CreateOnceRequest = {
@@ -19,6 +20,7 @@ export type CreateOnceRequest = {
key: string;
fingerprint: string;
text: string;
attachmentIds: string[];
};
function toMessage(row: MessageRow): Message {
@@ -28,6 +30,7 @@ function toMessage(row: MessageRow): Message {
sender_id: row.sender_id,
text: row.text,
created_at: new Date(row.created_at).toISOString(),
attachment_ids: row.attachment_ids ?? [],
};
}
@@ -50,11 +53,15 @@ WITH claim AS (
'sender_id', $1::uuid,
'text', $6::text,
'created_at', to_char(date_trunc('milliseconds', now()) AT TIME ZONE 'UTC',
'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')),
'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'),
'attachment_ids', to_jsonb($8::uuid[])),
$5::uuid
WHERE EXISTS (
SELECT 1 FROM chat_members WHERE chat_id = $7::uuid AND user_id = $1
)
AND (SELECT count(*) FROM attachments a
WHERE a.chat_id = $7::uuid AND a.status = 'ready'
AND a.id = ANY($8::uuid[])) = cardinality($8::uuid[])
ON CONFLICT DO NOTHING
RETURNING response_body
), inserted AS (
@@ -62,6 +69,10 @@ WITH claim AS (
SELECT $5::uuid, $7::uuid, $1::uuid, $6::text, date_trunc('milliseconds', now())
FROM claim
RETURNING id
), linked AS (
INSERT INTO message_attachments (message_id, chat_id, attachment_id, ordinal)
SELECT $5::uuid, $7::uuid, t.id, t.ord
FROM inserted, unnest($8::uuid[]) WITH ORDINALITY AS t(id, ord)
), bumped AS (
UPDATE chats SET message_version = message_version + 1
WHERE id = $7::uuid AND EXISTS (SELECT 1 FROM inserted)
@@ -88,6 +99,7 @@ export class MessagesRepository {
randomUUID(),
request.text,
request.chatId,
request.attachmentIds,
],
);
return result.rows[0]?.response_body;
@@ -112,12 +124,16 @@ export class MessagesRepository {
let predicate = '';
if (cursor) {
values.push(cursor.created_at, cursor.id);
predicate = 'AND (created_at, id) > ($2::timestamptz, $3::uuid)';
predicate =
'AND (m.created_at, m.id) > ($2::timestamptz, $3::uuid)';
}
values.push(limit + 1);
const result = await this.database.query<MessageRow>(
'messages.history',
`SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = $1 ${predicate} ORDER BY created_at, id LIMIT $${values.length}`,
`SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
COALESCE((SELECT json_agg(ma.attachment_id ORDER BY ma.ordinal)
FROM message_attachments ma WHERE ma.message_id = m.id), '[]'::json) AS attachment_ids
FROM messages m WHERE m.chat_id = $1 ${predicate} ORDER BY m.created_at, m.id LIMIT $${values.length}`,
values,
);
return {
+31 -25
View File
@@ -28,9 +28,10 @@ describe('MessagesService idempotency expiry race', () => {
sender_id: 'user',
text: 'hello',
created_at: '2026-01-01T00:00:00.000Z',
attachment_ids: [],
};
let row: StoredRow | undefined = {
request_fingerprint: idempotencyFingerprint({ text: 'hello' }),
let row: StoredRow | undefined = {
request_fingerprint: idempotencyFingerprint('hello', []),
response_body: {
...newMessage,
id: 'message-old',
@@ -58,9 +59,9 @@ describe('MessagesService idempotency expiry race', () => {
}
if (row) return undefined;
messageWrites += 1;
row = {
request_fingerprint: idempotencyFingerprint({ text: 'hello' }),
response_body: newMessage,
row = {
request_fingerprint: idempotencyFingerprint('hello', []),
response_body: newMessage,
created_at: new Date('2026-01-01T00:00:00.000Z'),
expired: false,
is_member: true,
@@ -69,25 +70,29 @@ describe('MessagesService idempotency expiry race', () => {
return newMessage;
}),
};
const idempotency = {
read: vi.fn(async () => {
readCalls += 1;
if (readCalls === 2) bothInitialReads.resolve();
if (readCalls <= 2) await bothInitialReads.promise;
return row;
}),
forget: vi.fn(async (...args: unknown[]) => {
forgetCalls += 1;
if (args.length !== 4) {
row = undefined;
return;
}
if (forgetCalls === 2) await replacementCreated.promise;
if (!row || row.created_at >= new Date('2026-01-01T00:00:00.000Z')) return;
row = undefined;
}),
isMember: vi.fn().mockResolvedValue(true),
};
const idempotency = {
read: vi.fn(async () => {
readCalls += 1;
if (readCalls === 2) bothInitialReads.resolve();
if (readCalls <= 2) await bothInitialReads.promise;
return row;
}),
forget: vi.fn(async (...args: unknown[]) => {
forgetCalls += 1;
if (args.length !== 4) {
row = undefined;
return;
}
if (forgetCalls === 2) await replacementCreated.promise;
if (
!row ||
row.created_at >= new Date('2026-01-01T00:00:00.000Z')
)
return;
row = undefined;
}),
isMember: vi.fn().mockResolvedValue(true),
};
const makeService = () =>
new MessagesService(
messagesRepository as never,
@@ -96,8 +101,9 @@ describe('MessagesService idempotency expiry race', () => {
idempotency as never,
{ getOrThrow: vi.fn().mockReturnValue(60) } as never,
{ increment: vi.fn() } as never,
{} as never,
);
const input = { text: 'hello' } as never;
const input = { text: 'hello', attachment_ids: [] } as never;
const user = { id: 'user' } as never;
const results = await Promise.all([
+23 -6
View File
@@ -12,6 +12,7 @@ import type { Page } from '../common/http/page.js';
import type { AppConfig } from '../config/app-config.schema.js';
import type { User } from '../users/user.js';
import { ChatsService } from '../chats/chats.service.js';
import { AttachmentsRepository } from '../attachments/attachments.repository.js';
import { MetricsService } from '../metrics/metrics.service.js';
import type { CreateMessageDto } from './dto/create-message.dto.js';
import type { Message } from './message.js';
@@ -42,6 +43,7 @@ export class MessagesService {
private readonly idempotency: IdempotencyRepository,
private readonly config: ConfigService<AppConfig>,
private readonly metrics: MetricsService,
private readonly attachments: AttachmentsRepository,
) {}
async create(
@@ -53,14 +55,18 @@ export class MessagesService {
// Ключ и тело проверяются до обращения к БД: клиентская ошибка не должна
// создавать заявку и не должна зависеть от состояния хранилища.
if (typeof rawKey !== 'string' || !IDEMPOTENCY_KEY_PATTERN.test(rawKey))
throw new BadRequestException(
'Idempotency-Key header is required',
);
if ([...input.text].every((character) => /\s/u.test(character)))
throw new BadRequestException('Idempotency-Key header is required');
const ids = input.attachment_ids.map((id) => id.toLowerCase());
if (new Set(ids).size !== ids.length)
throw new BadRequestException('attachment_ids must be unique');
if (
ids.length === 0 &&
[...input.text].every((character) => /\s/u.test(character))
)
throw new BadRequestException('Message text cannot be blank');
const path = messagesPath(chatId);
const fingerprint = idempotencyFingerprint(input);
const fingerprint = idempotencyFingerprint(input.text, ids);
const ttl = this.config.getOrThrow('IDEMPOTENCY_TTL_SECONDS');
for (let attempt = 1; attempt <= MAX_CLAIM_ATTEMPTS; attempt += 1) {
const created = await this.messagesRepository.createOnce({
@@ -70,6 +76,7 @@ export class MessagesService {
key: rawKey,
fingerprint,
text: input.text,
attachmentIds: ids,
});
if (created) {
this.metrics.increment('idempotency.claim');
@@ -90,6 +97,12 @@ export class MessagesService {
// чужую заявку сняли между нашими двумя запросами (503).
if (!(await this.idempotency.isMember(chatId, currentUser.id)))
throw new NotFoundException();
// Unknown, foreign-chat or unfinished attachment: not found.
if (
ids.length > 0 &&
!(await this.attachments.allLinkable(chatId, ids))
)
throw new NotFoundException();
this.metrics.increment('idempotency.claim_lost');
throw new ServiceUnavailableException(
'Idempotency key could not be claimed',
@@ -113,7 +126,11 @@ export class MessagesService {
}
if (!stored.is_member) throw new NotFoundException();
this.metrics.increment('idempotency.replay');
return stored.response_body;
// Bodies stored before attachments existed carry no attachment_ids.
return {
...stored.response_body,
attachment_ids: stored.response_body.attachment_ids ?? [],
};
}
throw new ServiceUnavailableException(
'Idempotency key could not be claimed',