forked from mtusi2026/mtusi2026lab4
lab4: вложения (proxy upload, download, уборщик), attachment_ids в сообщениях
This commit is contained in:
@@ -23,6 +23,7 @@
|
||||
"graph": "npm run build && DATABASE_URL=postgresql://graph-only.invalid/messenger REDIS_URL=redis://graph-only.invalid:6379 tsx scripts/graph-modules.ts",
|
||||
"test:e2e": "vitest run --config ./vitest.config.e2e.ts",
|
||||
"test:dependencies": "E2E_DEPENDENCY_FAULTS=1 vitest run --config ./vitest.config.e2e.ts dependency-failures",
|
||||
"attachments:sweep": "tsx src/attachments/attachments-sweep.cli.ts",
|
||||
"idempotency:gc": "tsx src/messages/idempotency/idempotency-gc.cli.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
|
||||
@@ -5,6 +5,7 @@ import { RedisModule } from './redis/redis.module.js';
|
||||
import { AuthModule } from './auth/auth.module.js';
|
||||
import { ChatsModule } from './chats/chats.module.js';
|
||||
import { HealthModule } from './health/health.module.js';
|
||||
import { AttachmentsModule } from './attachments/attachments.module.js';
|
||||
import { MessagesModule } from './messages/messages.module.js';
|
||||
import { MetricsModule } from './metrics/metrics.module.js';
|
||||
import { UsersModule } from './users/users.module.js';
|
||||
@@ -21,6 +22,7 @@ import { ApiExceptionFilter } from './common/errors/api-exception.filter.js';
|
||||
HealthModule,
|
||||
ChatsModule,
|
||||
MessagesModule,
|
||||
AttachmentsModule,
|
||||
MetricsModule,
|
||||
],
|
||||
providers: [
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
import {
|
||||
Injectable,
|
||||
Logger,
|
||||
OnModuleDestroy,
|
||||
OnModuleInit,
|
||||
} from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import type { AppConfig } from '../config/app-config.schema.js';
|
||||
import { MetricsService } from '../metrics/metrics.service.js';
|
||||
import { ObjectStorage } from '../storage/object-storage.js';
|
||||
import { AttachmentsRepository } from './attachments.repository.js';
|
||||
|
||||
const SWEEP_BATCH = 100;
|
||||
|
||||
/**
|
||||
* Removes unfinished uploads (pending past expiry + grace) and rows whose
|
||||
* deletion was claimed but never finished. A live upload is never touched:
|
||||
* its expiry is a hard request deadline and the grace period is far longer.
|
||||
*/
|
||||
@Injectable()
|
||||
export class AttachmentSweeper implements OnModuleInit, OnModuleDestroy {
|
||||
private readonly logger = new Logger(AttachmentSweeper.name);
|
||||
private timer: NodeJS.Timeout | undefined;
|
||||
private running = false;
|
||||
|
||||
constructor(
|
||||
private readonly config: ConfigService<AppConfig>,
|
||||
private readonly repository: AttachmentsRepository,
|
||||
private readonly storage: ObjectStorage,
|
||||
private readonly metrics: MetricsService,
|
||||
) {}
|
||||
|
||||
onModuleInit(): void {
|
||||
const interval = this.config.getOrThrow('ATTACHMENT_SWEEP_INTERVAL_MS');
|
||||
if (interval <= 0) return;
|
||||
this.timer = setInterval(() => {
|
||||
if (this.running) return;
|
||||
this.running = true;
|
||||
this.sweepOnce()
|
||||
.catch((error: unknown) => {
|
||||
this.logger.warn(
|
||||
`Sweep failed: ${error instanceof Error ? error.message : 'unknown error'}`,
|
||||
);
|
||||
})
|
||||
.finally(() => {
|
||||
this.running = false;
|
||||
});
|
||||
}, interval);
|
||||
this.timer.unref();
|
||||
}
|
||||
|
||||
onModuleDestroy(): void {
|
||||
if (this.timer) clearInterval(this.timer);
|
||||
this.timer = undefined;
|
||||
}
|
||||
|
||||
async sweepOnce(): Promise<{ claimed: number; deleted: number }> {
|
||||
const claimed = await this.repository.claimExpired(
|
||||
this.config.getOrThrow('ATTACHMENT_SWEEP_GRACE_SECONDS'),
|
||||
SWEEP_BATCH,
|
||||
);
|
||||
let deleted = 0;
|
||||
for (const item of claimed) {
|
||||
try {
|
||||
await this.storage.delete(item.object_key);
|
||||
await this.repository.deleteClaimed(item.id);
|
||||
deleted += 1;
|
||||
this.metrics.increment('attachments.sweep.deleted');
|
||||
} catch (error) {
|
||||
this.metrics.increment('attachments.sweep.error');
|
||||
this.logger.warn(
|
||||
`Sweep of attachment ${item.id} failed: ${error instanceof Error ? error.message : 'unknown error'}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
return { claimed: claimed.length, deleted };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
export type Attachment = {
|
||||
id: string;
|
||||
chat_id: string;
|
||||
owner_id: string;
|
||||
filename: string;
|
||||
content_type: 'image/png' | 'image/jpeg';
|
||||
size_bytes: number;
|
||||
sha256: string;
|
||||
status: 'ready';
|
||||
variants: never[];
|
||||
error_code: null;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
};
|
||||
|
||||
export type AttachmentRow = {
|
||||
id: string;
|
||||
chat_id: string;
|
||||
owner_id: string;
|
||||
filename: string;
|
||||
content_type: 'image/png' | 'image/jpeg';
|
||||
upload_mode: 'proxy' | 'direct';
|
||||
status: 'pending' | 'ready' | 'deleting';
|
||||
object_key: string;
|
||||
size_bytes: string | number | null;
|
||||
sha256: string | null;
|
||||
width: number | null;
|
||||
height: number | null;
|
||||
expires_at: Date | null;
|
||||
created_at: Date;
|
||||
updated_at: Date;
|
||||
};
|
||||
|
||||
export function toAttachment(row: AttachmentRow): Attachment {
|
||||
return {
|
||||
id: row.id,
|
||||
chat_id: row.chat_id,
|
||||
owner_id: row.owner_id,
|
||||
filename: row.filename,
|
||||
content_type: row.content_type,
|
||||
size_bytes: Number(row.size_bytes),
|
||||
sha256: row.sha256 as string,
|
||||
status: 'ready',
|
||||
variants: [],
|
||||
error_code: null,
|
||||
created_at: row.created_at.toISOString(),
|
||||
updated_at: row.updated_at.toISOString(),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { parseAppConfig } from '../config/app-config.schema.js';
|
||||
import { DatabaseClient } from '../database/database.client.js';
|
||||
import { MetricsService } from '../metrics/metrics.service.js';
|
||||
import { ObjectStorage } from '../storage/object-storage.js';
|
||||
import { AttachmentSweeper } from './attachment-sweeper.service.js';
|
||||
import { AttachmentsRepository } from './attachments.repository.js';
|
||||
|
||||
// Manual operator pass over unfinished uploads (the API also sweeps in the
|
||||
// background). Never touches rows before expiry + ATTACHMENT_SWEEP_GRACE_SECONDS.
|
||||
// in the stand: docker compose exec api node dist/attachments/attachments-sweep.cli.js
|
||||
// on the host: npm run attachments:sweep (needs reachable DATABASE_URL and S3)
|
||||
async function main(): Promise<void> {
|
||||
const config = parseAppConfig(process.env as Record<string, unknown>);
|
||||
const configService = {
|
||||
getOrThrow: (key: keyof typeof config) => config[key],
|
||||
} as never;
|
||||
const metrics = new MetricsService();
|
||||
const database = new DatabaseClient(configService, metrics);
|
||||
try {
|
||||
const sweeper = new AttachmentSweeper(
|
||||
configService,
|
||||
new AttachmentsRepository(database),
|
||||
new ObjectStorage(configService),
|
||||
metrics,
|
||||
);
|
||||
let total = 0;
|
||||
for (;;) {
|
||||
const { claimed, deleted } = await sweeper.sweepOnce();
|
||||
total += deleted;
|
||||
// A pass that claimed rows but could not delete any would loop forever.
|
||||
if (claimed === 0 || deleted === 0) break;
|
||||
}
|
||||
process.stdout.write(
|
||||
`Swept ${total} attachment(s) older than ${config.ATTACHMENT_SWEEP_GRACE_SECONDS} seconds past expiry\n`,
|
||||
);
|
||||
} finally {
|
||||
await database.onModuleDestroy();
|
||||
}
|
||||
}
|
||||
|
||||
if (process.argv[1] === fileURLToPath(import.meta.url)) await main();
|
||||
@@ -0,0 +1,89 @@
|
||||
import {
|
||||
Controller,
|
||||
Get,
|
||||
Headers,
|
||||
HttpCode,
|
||||
Param,
|
||||
ParseUUIDPipe,
|
||||
Post,
|
||||
Query,
|
||||
Req,
|
||||
Res,
|
||||
StreamableFile,
|
||||
UseGuards,
|
||||
} from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import type { FastifyReply, FastifyRequest } from 'fastify';
|
||||
import type { Readable } from 'node:stream';
|
||||
import { CurrentUser } from '../common/auth/current-user.decorator.js';
|
||||
import { CurrentUserGuard } from '../common/auth/current-user.guard.js';
|
||||
import type { AppConfig } from '../config/app-config.schema.js';
|
||||
import type { User } from '../users/user.js';
|
||||
import { AttachmentsService } from './attachments.service.js';
|
||||
import { contentDisposition } from './filename.js';
|
||||
|
||||
@Controller('api/v1')
|
||||
@UseGuards(CurrentUserGuard)
|
||||
export class AttachmentsController {
|
||||
private readonly maxUploadBytes: number;
|
||||
|
||||
constructor(
|
||||
private readonly attachments: AttachmentsService,
|
||||
config: ConfigService<AppConfig>,
|
||||
) {
|
||||
this.maxUploadBytes = config.getOrThrow('MAX_UPLOAD_BYTES');
|
||||
}
|
||||
|
||||
@Post('chats/:chatId/attachments')
|
||||
@HttpCode(201)
|
||||
upload(
|
||||
@Param('chatId', new ParseUUIDPipe()) chatId: string,
|
||||
@CurrentUser() user: User,
|
||||
@Query('filename') filename: unknown,
|
||||
@Headers('content-type') contentType: string | undefined,
|
||||
@Headers('content-length') contentLength: string | undefined,
|
||||
@Req() request: FastifyRequest,
|
||||
@Res({ passthrough: true }) reply: FastifyReply,
|
||||
) {
|
||||
// A gigantic body is not worth draining: close the connection after the answer.
|
||||
if (Number(contentLength) > 2 * this.maxUploadBytes)
|
||||
reply.header('Connection', 'close');
|
||||
return this.attachments.uploadProxy(
|
||||
chatId,
|
||||
user,
|
||||
filename,
|
||||
contentType,
|
||||
contentLength,
|
||||
request.body as Readable | undefined,
|
||||
);
|
||||
}
|
||||
|
||||
@Get('attachments/:attachmentId')
|
||||
metadata(
|
||||
@Param('attachmentId', new ParseUUIDPipe()) attachmentId: string,
|
||||
@CurrentUser() user: User,
|
||||
) {
|
||||
return this.attachments.getMetadata(attachmentId, user);
|
||||
}
|
||||
|
||||
@Get('attachments/:attachmentId/content')
|
||||
async content(
|
||||
@Param('attachmentId', new ParseUUIDPipe()) attachmentId: string,
|
||||
@CurrentUser() user: User,
|
||||
@Res({ passthrough: true }) reply: FastifyReply,
|
||||
@Query('variant') variant?: string,
|
||||
) {
|
||||
const { row, stream } = await this.attachments.openContent(
|
||||
attachmentId,
|
||||
user,
|
||||
variant,
|
||||
);
|
||||
reply.header('X-Content-Type-Options', 'nosniff');
|
||||
reply.header('Cache-Control', 'private, no-store');
|
||||
return new StreamableFile(stream, {
|
||||
type: row.content_type,
|
||||
length: Number(row.size_bytes),
|
||||
disposition: contentDisposition(row.filename),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { DatabaseModule } from '../database/database.module.js';
|
||||
import { ChatsModule } from '../chats/chats.module.js';
|
||||
import { UsersModule } from '../users/users.module.js';
|
||||
import { RedisModule } from '../redis/redis.module.js';
|
||||
import { MetricsModule } from '../metrics/metrics.module.js';
|
||||
import { StorageModule } from '../storage/storage.module.js';
|
||||
import { AttachmentSweeper } from './attachment-sweeper.service.js';
|
||||
import { AttachmentsController } from './attachments.controller.js';
|
||||
import { AttachmentsRepository } from './attachments.repository.js';
|
||||
import { AttachmentsService } from './attachments.service.js';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
DatabaseModule,
|
||||
ChatsModule,
|
||||
UsersModule,
|
||||
RedisModule,
|
||||
MetricsModule,
|
||||
StorageModule,
|
||||
],
|
||||
controllers: [AttachmentsController],
|
||||
providers: [AttachmentsRepository, AttachmentsService, AttachmentSweeper],
|
||||
exports: [AttachmentsRepository],
|
||||
})
|
||||
export class AttachmentsModule {}
|
||||
@@ -0,0 +1,156 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { DatabaseClient } from '../database/database.client.js';
|
||||
import type { AttachmentRow } from './attachment.js';
|
||||
|
||||
export type InsertPending = {
|
||||
id: string;
|
||||
chatId: string;
|
||||
ownerId: string;
|
||||
filename: string;
|
||||
contentType: string;
|
||||
uploadMode: 'proxy' | 'direct';
|
||||
objectKey: string;
|
||||
sizeBytes: number | null;
|
||||
sha256: string | null;
|
||||
ttlMs: number;
|
||||
};
|
||||
|
||||
export type ReadyFields = {
|
||||
sizeBytes: number;
|
||||
sha256: string;
|
||||
width: number;
|
||||
height: number;
|
||||
};
|
||||
|
||||
const VISIBLE_MEMBER = `JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = $2`;
|
||||
|
||||
@Injectable()
|
||||
export class AttachmentsRepository {
|
||||
constructor(private readonly database: DatabaseClient) {}
|
||||
|
||||
async insertPending(input: InsertPending): Promise<void> {
|
||||
await this.database.query(
|
||||
'attachments.insert_pending',
|
||||
`INSERT INTO attachments
|
||||
(id, chat_id, owner_id, filename, content_type, upload_mode, status,
|
||||
object_key, size_bytes, sha256, expires_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, 'pending', $7, $8, $9,
|
||||
now() + make_interval(secs => $10::double precision / 1000.0))`,
|
||||
[
|
||||
input.id,
|
||||
input.chatId,
|
||||
input.ownerId,
|
||||
input.filename,
|
||||
input.contentType,
|
||||
input.uploadMode,
|
||||
input.objectKey,
|
||||
input.sizeBytes,
|
||||
input.sha256,
|
||||
input.ttlMs,
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
async markReady(
|
||||
id: string,
|
||||
fields: ReadyFields,
|
||||
): Promise<AttachmentRow | undefined> {
|
||||
const result = await this.database.query<AttachmentRow>(
|
||||
'attachments.mark_ready',
|
||||
`UPDATE attachments
|
||||
SET status = 'ready', size_bytes = $2, sha256 = $3, width = $4,
|
||||
height = $5, expires_at = NULL, updated_at = now()
|
||||
WHERE id = $1 AND status = 'pending'
|
||||
RETURNING *`,
|
||||
[id, fields.sizeBytes, fields.sha256, fields.width, fields.height],
|
||||
);
|
||||
return result.rows[0];
|
||||
}
|
||||
|
||||
async deletePending(id: string): Promise<void> {
|
||||
await this.database.query(
|
||||
'attachments.delete_pending',
|
||||
`DELETE FROM attachments WHERE id = $1 AND status = 'pending'`,
|
||||
[id],
|
||||
);
|
||||
}
|
||||
|
||||
async findVisible(
|
||||
id: string,
|
||||
userId: string,
|
||||
): Promise<AttachmentRow | undefined> {
|
||||
const result = await this.database.query<AttachmentRow>(
|
||||
'attachments.find_visible',
|
||||
`SELECT a.* FROM attachments a ${VISIBLE_MEMBER}
|
||||
WHERE a.id = $1 AND a.status = 'ready'`,
|
||||
[id, userId],
|
||||
);
|
||||
return result.rows[0];
|
||||
}
|
||||
|
||||
async findForComplete(
|
||||
id: string,
|
||||
userId: string,
|
||||
): Promise<AttachmentRow | undefined> {
|
||||
const result = await this.database.query<AttachmentRow>(
|
||||
'attachments.find_for_complete',
|
||||
`SELECT a.* FROM attachments a ${VISIBLE_MEMBER}
|
||||
WHERE a.id = $1 AND a.upload_mode = 'direct' AND a.owner_id = $2
|
||||
AND a.status IN ('pending', 'ready')`,
|
||||
[id, userId],
|
||||
);
|
||||
return result.rows[0];
|
||||
}
|
||||
|
||||
async countPendingDirect(ownerId: string): Promise<number> {
|
||||
const result = await this.database.query<{ count: string }>(
|
||||
'attachments.count_pending_direct',
|
||||
`SELECT count(*) AS count FROM attachments
|
||||
WHERE owner_id = $1 AND status = 'pending' AND upload_mode = 'direct'`,
|
||||
[ownerId],
|
||||
);
|
||||
return Number(result.rows[0].count);
|
||||
}
|
||||
|
||||
async allLinkable(chatId: string, ids: string[]): Promise<boolean> {
|
||||
const result = await this.database.query<{ count: string }>(
|
||||
'attachments.all_linkable',
|
||||
`SELECT count(*) AS count FROM attachments
|
||||
WHERE chat_id = $1 AND status = 'ready' AND id = ANY($2::uuid[])`,
|
||||
[chatId, ids],
|
||||
);
|
||||
return Number(result.rows[0].count) === ids.length;
|
||||
}
|
||||
|
||||
async claimExpired(
|
||||
graceSeconds: number,
|
||||
limit: number,
|
||||
): Promise<{ id: string; object_key: string }[]> {
|
||||
const result = await this.database.query<{
|
||||
id: string;
|
||||
object_key: string;
|
||||
}>(
|
||||
'attachments.claim_expired',
|
||||
`UPDATE attachments SET status = 'deleting', updated_at = now()
|
||||
WHERE id IN (
|
||||
SELECT id FROM attachments
|
||||
WHERE (status = 'pending'
|
||||
AND expires_at < now() - make_interval(secs => $1::double precision))
|
||||
OR (status = 'deleting' AND updated_at < now() - interval '1 minute')
|
||||
ORDER BY expires_at NULLS FIRST
|
||||
LIMIT $2
|
||||
FOR UPDATE SKIP LOCKED)
|
||||
RETURNING id, object_key`,
|
||||
[graceSeconds, limit],
|
||||
);
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
async deleteClaimed(id: string): Promise<void> {
|
||||
await this.database.query(
|
||||
'attachments.delete_claimed',
|
||||
`DELETE FROM attachments WHERE id = $1 AND status = 'deleting'`,
|
||||
[id],
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,325 @@
|
||||
import {
|
||||
BadRequestException,
|
||||
HttpException,
|
||||
Injectable,
|
||||
InternalServerErrorException,
|
||||
Logger,
|
||||
NotFoundException,
|
||||
PayloadTooLargeException,
|
||||
ServiceUnavailableException,
|
||||
UnsupportedMediaTypeException,
|
||||
} from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { pipeline, Readable, Transform } from 'node:stream';
|
||||
import type { AppConfig } from '../config/app-config.schema.js';
|
||||
import { ChatsService } from '../chats/chats.service.js';
|
||||
import { MetricsService } from '../metrics/metrics.service.js';
|
||||
import { ObjectStorage } from '../storage/object-storage.js';
|
||||
import type { User } from '../users/user.js';
|
||||
import {
|
||||
toAttachment,
|
||||
type Attachment,
|
||||
type AttachmentRow,
|
||||
} from './attachment.js';
|
||||
import { validateFilename } from './filename.js';
|
||||
import {
|
||||
IMAGE_HEADER_MAX_BYTES,
|
||||
ImageHeader,
|
||||
ImageInspector,
|
||||
ImageRejectedError,
|
||||
} from './image/image-inspector.js';
|
||||
import { AttachmentsRepository } from './attachments.repository.js';
|
||||
|
||||
const MEDIA_TYPES = new Set(['image/png', 'image/jpeg']);
|
||||
|
||||
export function mediaType(header: unknown): string | undefined {
|
||||
if (typeof header !== 'string') return undefined;
|
||||
return header.split(';', 1)[0].trim().toLowerCase();
|
||||
}
|
||||
|
||||
export function imageRejection(error: ImageRejectedError): HttpException {
|
||||
return error.reason === 'too_many_pixels'
|
||||
? new PayloadTooLargeException(error.message)
|
||||
: new UnsupportedMediaTypeException(error.message);
|
||||
}
|
||||
|
||||
class UploadInterruptedError extends Error {}
|
||||
|
||||
@Injectable()
|
||||
export class AttachmentsService {
|
||||
private readonly logger = new Logger(AttachmentsService.name);
|
||||
private inFlight = 0;
|
||||
private readonly maxUploadBytes: number;
|
||||
private readonly maxPixels: number;
|
||||
private readonly uploadTimeoutMs: number;
|
||||
private readonly maxInFlight: number;
|
||||
|
||||
constructor(
|
||||
config: ConfigService<AppConfig>,
|
||||
private readonly repository: AttachmentsRepository,
|
||||
private readonly storage: ObjectStorage,
|
||||
private readonly chats: ChatsService,
|
||||
private readonly metrics: MetricsService,
|
||||
) {
|
||||
this.maxUploadBytes = config.getOrThrow('MAX_UPLOAD_BYTES');
|
||||
this.maxPixels = config.getOrThrow('MAX_IMAGE_PIXELS');
|
||||
this.uploadTimeoutMs = config.getOrThrow('UPLOAD_TIMEOUT_MS');
|
||||
this.maxInFlight = config.getOrThrow('UPLOAD_MAX_IN_FLIGHT');
|
||||
}
|
||||
|
||||
async uploadProxy(
|
||||
chatId: string,
|
||||
user: User,
|
||||
rawFilename: unknown,
|
||||
contentTypeHeader: unknown,
|
||||
contentLengthHeader: unknown,
|
||||
body: Readable | undefined,
|
||||
): Promise<Attachment> {
|
||||
const filename = validateFilename(rawFilename);
|
||||
const declared = mediaType(contentTypeHeader);
|
||||
if (declared === undefined || !MEDIA_TYPES.has(declared))
|
||||
throw new UnsupportedMediaTypeException(
|
||||
'Content-Type must be image/png or image/jpeg',
|
||||
);
|
||||
if (
|
||||
typeof contentLengthHeader !== 'string' ||
|
||||
!/^\d{1,15}$/u.test(contentLengthHeader)
|
||||
)
|
||||
throw new BadRequestException('Content-Length is required');
|
||||
const contentLength = Number(contentLengthHeader);
|
||||
if (contentLength === 0 || body === undefined)
|
||||
throw new UnsupportedMediaTypeException('Image body is empty');
|
||||
if (contentLength > this.maxUploadBytes)
|
||||
throw new PayloadTooLargeException('Image is too large');
|
||||
|
||||
if (this.inFlight >= this.maxInFlight)
|
||||
throw new HttpException(
|
||||
{ message: 'Too many concurrent uploads', retryAfter: 1 },
|
||||
429,
|
||||
);
|
||||
this.inFlight += 1;
|
||||
try {
|
||||
await this.chats.getForUser(chatId, user);
|
||||
return await this.receive(
|
||||
chatId,
|
||||
user,
|
||||
filename,
|
||||
declared,
|
||||
contentLength,
|
||||
body,
|
||||
);
|
||||
} finally {
|
||||
this.inFlight -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
private async receive(
|
||||
chatId: string,
|
||||
user: User,
|
||||
filename: string,
|
||||
declared: string,
|
||||
contentLength: number,
|
||||
body: Readable,
|
||||
): Promise<Attachment> {
|
||||
const controller = new AbortController();
|
||||
let timedOut = false;
|
||||
const timer = setTimeout(() => {
|
||||
timedOut = true;
|
||||
body.destroy(new Error('upload timeout'));
|
||||
controller.abort();
|
||||
}, this.uploadTimeoutMs);
|
||||
const iterator = body[Symbol.asyncIterator]() as AsyncIterator<Buffer>;
|
||||
const next = async (): Promise<IteratorResult<Buffer>> => {
|
||||
try {
|
||||
return await iterator.next();
|
||||
} catch {
|
||||
throw new UploadInterruptedError('Upload was interrupted');
|
||||
}
|
||||
};
|
||||
|
||||
const inspector = new ImageInspector(this.maxPixels);
|
||||
const head: Buffer[] = [];
|
||||
let id: string | undefined;
|
||||
let key = '';
|
||||
let rejection: unknown;
|
||||
try {
|
||||
// Header phase: nothing exists in the database or S3 yet.
|
||||
let headBytes = 0;
|
||||
try {
|
||||
while (inspector.header === undefined) {
|
||||
const step = await next();
|
||||
if (step.done) {
|
||||
inspector.finish();
|
||||
throw new ImageRejectedError(
|
||||
'unsupported_media_type',
|
||||
'Image header is missing',
|
||||
);
|
||||
}
|
||||
head.push(step.value);
|
||||
headBytes += step.value.length;
|
||||
inspector.push(step.value);
|
||||
if (
|
||||
inspector.header === undefined &&
|
||||
headBytes > IMAGE_HEADER_MAX_BYTES
|
||||
)
|
||||
throw new ImageRejectedError(
|
||||
'unsupported_media_type',
|
||||
'Image header is missing',
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
throw this.mapUploadError(error, timedOut);
|
||||
}
|
||||
const header = inspector.header as ImageHeader;
|
||||
if (header.contentType !== declared)
|
||||
throw new UnsupportedMediaTypeException(
|
||||
'Image content does not match Content-Type',
|
||||
);
|
||||
|
||||
id = randomUUID();
|
||||
key = this.storage.objectKey(id);
|
||||
await this.repository.insertPending({
|
||||
id,
|
||||
chatId,
|
||||
ownerId: user.id,
|
||||
filename,
|
||||
contentType: declared,
|
||||
uploadMode: 'proxy',
|
||||
objectKey: key,
|
||||
sizeBytes: null,
|
||||
sha256: null,
|
||||
ttlMs: this.uploadTimeoutMs,
|
||||
});
|
||||
|
||||
const sha256 = createHash('sha256');
|
||||
let total = 0;
|
||||
const account = (chunk: Buffer, inspect: boolean): void => {
|
||||
total += chunk.length;
|
||||
if (total > contentLength)
|
||||
throw new UploadInterruptedError(
|
||||
'Body exceeds Content-Length',
|
||||
);
|
||||
if (inspect) inspector.push(chunk);
|
||||
sha256.update(chunk);
|
||||
this.metrics.add('attachments.proxy.bytes_in', chunk.length);
|
||||
// Validate before the last bytes reach S3: it cannot commit an
|
||||
// object whose body is incomplete.
|
||||
if (total === contentLength) inspector.finish();
|
||||
};
|
||||
async function* generate(): AsyncGenerator<Buffer> {
|
||||
for (const chunk of head) {
|
||||
account(chunk, false);
|
||||
yield chunk;
|
||||
}
|
||||
for (;;) {
|
||||
const step = await next();
|
||||
if (step.done) break;
|
||||
account(step.value, true);
|
||||
yield step.value;
|
||||
}
|
||||
if (total < contentLength)
|
||||
throw new UploadInterruptedError('Upload was interrupted');
|
||||
}
|
||||
const readable = Readable.from(
|
||||
(async function* () {
|
||||
try {
|
||||
yield* generate();
|
||||
} catch (error) {
|
||||
rejection = error;
|
||||
throw error;
|
||||
}
|
||||
})(),
|
||||
);
|
||||
|
||||
let claimed = false;
|
||||
try {
|
||||
await this.storage.putStream(
|
||||
key,
|
||||
readable,
|
||||
contentLength,
|
||||
declared,
|
||||
controller.signal,
|
||||
);
|
||||
this.metrics.add(
|
||||
'attachments.proxy.bytes_to_s3',
|
||||
contentLength,
|
||||
);
|
||||
const row = await this.repository.markReady(id, {
|
||||
sizeBytes: total,
|
||||
sha256: sha256.digest('hex'),
|
||||
width: header.width,
|
||||
height: header.height,
|
||||
});
|
||||
if (row === undefined) {
|
||||
claimed = true;
|
||||
throw new ServiceUnavailableException();
|
||||
}
|
||||
this.metrics.increment('attachments.upload.created');
|
||||
return toAttachment(row);
|
||||
} catch (error) {
|
||||
if (!claimed) await this.compensate(id, key);
|
||||
throw this.mapUploadError(rejection ?? error, timedOut);
|
||||
}
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
private mapUploadError(error: unknown, timedOut: boolean): unknown {
|
||||
if (error instanceof ImageRejectedError) return imageRejection(error);
|
||||
if (error instanceof UploadInterruptedError || timedOut)
|
||||
return new BadRequestException('Upload was interrupted');
|
||||
return error;
|
||||
}
|
||||
|
||||
private async compensate(id: string, key: string): Promise<void> {
|
||||
try {
|
||||
await this.storage.delete(key);
|
||||
await this.repository.deletePending(id);
|
||||
this.metrics.increment('attachments.compensated');
|
||||
} catch (error) {
|
||||
this.metrics.increment('attachments.compensation_deferred');
|
||||
this.logger.warn(
|
||||
`Compensation of attachment ${id} deferred: ${error instanceof Error ? error.message : 'unknown error'}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async getMetadata(id: string, user: User): Promise<Attachment> {
|
||||
const row = await this.repository.findVisible(id, user.id);
|
||||
if (!row) throw new NotFoundException();
|
||||
return toAttachment(row);
|
||||
}
|
||||
|
||||
async openContent(
|
||||
id: string,
|
||||
user: User,
|
||||
variant: string | undefined,
|
||||
): Promise<{ row: AttachmentRow; stream: Readable }> {
|
||||
if (variant !== undefined && variant !== 'original') {
|
||||
if (['thumbnail', 'crop', 'optimized'].includes(variant))
|
||||
throw new NotFoundException();
|
||||
throw new BadRequestException('variant is invalid');
|
||||
}
|
||||
const row = await this.repository.findVisible(id, user.id);
|
||||
if (!row) throw new NotFoundException();
|
||||
const source = await this.storage.getStream(row.object_key);
|
||||
if (source === null) {
|
||||
this.metrics.increment('attachments.object_missing');
|
||||
this.logger.error(
|
||||
`Object for ready attachment ${row.id} is missing in storage`,
|
||||
);
|
||||
throw new InternalServerErrorException();
|
||||
}
|
||||
const metrics = this.metrics;
|
||||
const counter = new Transform({
|
||||
transform(chunk: Buffer, _encoding, callback) {
|
||||
metrics.add('attachments.download.bytes_out', chunk.length);
|
||||
callback(null, chunk);
|
||||
},
|
||||
});
|
||||
pipeline(source, counter, () => undefined);
|
||||
return { row, stream: counter };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
|
||||
// oxlint-disable-next-line no-control-regex
|
||||
const FORBIDDEN =
|
||||
/[/\\\u0000-\u001f\u007f-\u009f\u200e\u200f\u202a-\u202e\u2066-\u2069]/u;
|
||||
|
||||
/** The name is stored as-is and used for display only, never in paths or keys. */
|
||||
export function validateFilename(value: unknown): string {
|
||||
if (typeof value !== 'string') throw invalid();
|
||||
const length = [...value].length;
|
||||
if (
|
||||
length < 1 ||
|
||||
length > 255 ||
|
||||
/^\s*$/u.test(value) ||
|
||||
value === '.' ||
|
||||
value === '..' ||
|
||||
FORBIDDEN.test(value)
|
||||
)
|
||||
throw invalid();
|
||||
return value;
|
||||
}
|
||||
|
||||
function invalid(): BadRequestException {
|
||||
return new BadRequestException('filename is invalid');
|
||||
}
|
||||
|
||||
export function contentDisposition(filename: string): string {
|
||||
const fallback = filename.replace(/[^\x20-\x7e]|["\\]/gu, '_');
|
||||
const encoded = encodeURIComponent(filename).replace(
|
||||
/['()*]/gu,
|
||||
(character) => `%${character.charCodeAt(0).toString(16).toUpperCase()}`,
|
||||
);
|
||||
return `inline; filename="${fallback}"; filename*=UTF-8''${encoded}`;
|
||||
}
|
||||
@@ -82,24 +82,9 @@ export const appConfigSchema = z
|
||||
undefined,
|
||||
15000,
|
||||
),
|
||||
MAX_UPLOAD_BYTES: numeric(
|
||||
'MAX_UPLOAD_BYTES',
|
||||
1,
|
||||
10485760,
|
||||
10485760,
|
||||
),
|
||||
MAX_IMAGE_PIXELS: numeric(
|
||||
'MAX_IMAGE_PIXELS',
|
||||
1,
|
||||
40000000,
|
||||
40000000,
|
||||
),
|
||||
UPLOAD_TIMEOUT_MS: numeric(
|
||||
'UPLOAD_TIMEOUT_MS',
|
||||
1000,
|
||||
undefined,
|
||||
25000,
|
||||
),
|
||||
MAX_UPLOAD_BYTES: numeric('MAX_UPLOAD_BYTES', 1, 10485760, 10485760),
|
||||
MAX_IMAGE_PIXELS: numeric('MAX_IMAGE_PIXELS', 1, 40000000, 40000000),
|
||||
UPLOAD_TIMEOUT_MS: numeric('UPLOAD_TIMEOUT_MS', 1000, undefined, 25000),
|
||||
UPLOAD_MAX_IN_FLIGHT: numeric('UPLOAD_MAX_IN_FLIGHT', 1, 1024, 32),
|
||||
// 0 отключает фоновый проход уборщика (остаётся ручной CLI).
|
||||
ATTACHMENT_SWEEP_INTERVAL_MS: numeric(
|
||||
|
||||
@@ -36,12 +36,15 @@ export class DatabaseClient implements OnModuleDestroy {
|
||||
connectionString: this.config.getOrThrow('DATABASE_URL'),
|
||||
max: 1,
|
||||
idleTimeoutMillis: 0,
|
||||
connectionTimeoutMillis: this.config.getOrThrow('HEALTH_TIMEOUT_MS'),
|
||||
connectionTimeoutMillis:
|
||||
this.config.getOrThrow('HEALTH_TIMEOUT_MS'),
|
||||
query_timeout: this.config.getOrThrow('HEALTH_TIMEOUT_MS'),
|
||||
});
|
||||
this.healthPool.on('error', (error: Error) => {
|
||||
this.metrics.increment('database.health_pool_error');
|
||||
this.logger.warn(`PostgreSQL health connection error: ${error.message}`);
|
||||
this.logger.warn(
|
||||
`PostgreSQL health connection error: ${error.message}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
+15
-1
@@ -16,6 +16,18 @@ async function bootstrap(): Promise<void> {
|
||||
AppModule,
|
||||
new FastifyAdapter({ logger: true }),
|
||||
);
|
||||
// Raw image bodies reach the controller as an unread stream: the upload is
|
||||
// bounded by our own checks, not by Fastify's bodyLimit.
|
||||
app.getHttpAdapter()
|
||||
.getInstance()
|
||||
.addContentTypeParser(
|
||||
['image/png', 'image/jpeg'],
|
||||
(
|
||||
_request: unknown,
|
||||
payload: unknown,
|
||||
done: (error: Error | null, body?: unknown) => void,
|
||||
) => done(null, payload),
|
||||
);
|
||||
app.useGlobalInterceptors(
|
||||
new RequestIdInterceptor(
|
||||
app.get(HttpAdapterHost),
|
||||
@@ -50,7 +62,9 @@ async function bootstrap(): Promise<void> {
|
||||
shuttingDown = true;
|
||||
health.setDraining();
|
||||
if (drainDelayMs > 0)
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, drainDelayMs));
|
||||
await new Promise<void>((resolve) =>
|
||||
setTimeout(resolve, drainDelayMs),
|
||||
);
|
||||
await app.close();
|
||||
process.exit(0);
|
||||
};
|
||||
|
||||
@@ -1,8 +1,23 @@
|
||||
import { IsString } from 'class-validator';
|
||||
import {
|
||||
ArrayMaxSize,
|
||||
ArrayUnique,
|
||||
IsArray,
|
||||
IsString,
|
||||
IsUUID,
|
||||
ValidateIf,
|
||||
} from 'class-validator';
|
||||
import { CodePointLength } from '../../common/validation/code-point-length.js';
|
||||
|
||||
export class CreateMessageDto {
|
||||
@IsString()
|
||||
@CodePointLength(1, 4000)
|
||||
@CodePointLength(0, 4000)
|
||||
text!: string;
|
||||
|
||||
// Absent means []; null is a client error.
|
||||
@ValidateIf((_, value) => value !== undefined)
|
||||
@IsArray()
|
||||
@ArrayMaxSize(10)
|
||||
@ArrayUnique()
|
||||
@IsUUID('all', { each: true })
|
||||
attachment_ids: string[] = [];
|
||||
}
|
||||
|
||||
@@ -6,12 +6,17 @@ describe('IdempotencyRepository expired-key cleanup', () => {
|
||||
const query = vi.fn().mockResolvedValue({ rowCount: 0, rows: [] });
|
||||
const repository = new IdempotencyRepository({ query } as never);
|
||||
|
||||
await repository.forget('user', '/api/v1/chats/chat/messages', 'key', 86400);
|
||||
await repository.forget(
|
||||
'user',
|
||||
'/api/v1/chats/chat/messages',
|
||||
'key',
|
||||
86400,
|
||||
);
|
||||
|
||||
expect(query).toHaveBeenCalledWith(
|
||||
'idempotency.forget',
|
||||
expect.stringContaining(
|
||||
'created_at < now() - make_interval(secs => $4::double precision)',
|
||||
'created_at < now() - make_interval(secs => $4::double precision)',
|
||||
),
|
||||
['user', '/api/v1/chats/chat/messages', 'key', 86400],
|
||||
);
|
||||
|
||||
@@ -22,30 +22,30 @@ const PURGE_BATCHES = 10;
|
||||
export class IdempotencyRepository {
|
||||
constructor(private readonly database: DatabaseClient) {}
|
||||
|
||||
/**
|
||||
* Reads the stored result and re-checks access in the same round trip: a
|
||||
* replay must still prove that the caller may read this chat.
|
||||
*/
|
||||
async read(
|
||||
userId: string,
|
||||
path: string,
|
||||
key: string,
|
||||
chatId: string,
|
||||
ttlSeconds: number,
|
||||
): Promise<IdempotencyRecord | undefined> {
|
||||
const result = await this.database.query<IdempotencyRecord>(
|
||||
'idempotency.read',
|
||||
`SELECT k.request_fingerprint, k.status_code, k.response_body, k.created_at,
|
||||
/**
|
||||
* Reads the stored result and re-checks access in the same round trip: a
|
||||
* replay must still prove that the caller may read this chat.
|
||||
*/
|
||||
async read(
|
||||
userId: string,
|
||||
path: string,
|
||||
key: string,
|
||||
chatId: string,
|
||||
ttlSeconds: number,
|
||||
): Promise<IdempotencyRecord | undefined> {
|
||||
const result = await this.database.query<IdempotencyRecord>(
|
||||
'idempotency.read',
|
||||
`SELECT k.request_fingerprint, k.status_code, k.response_body, k.created_at,
|
||||
k.created_at < now() - make_interval(secs => $4::double precision) AS expired,
|
||||
EXISTS (
|
||||
SELECT 1 FROM chat_members WHERE chat_id = $5 AND user_id = $1
|
||||
) AS is_member
|
||||
FROM idempotency_keys k
|
||||
WHERE k.user_id = $1 AND k.method = 'POST' AND k.path = $2 AND k.key = $3`,
|
||||
[userId, path, key, ttlSeconds, chatId],
|
||||
);
|
||||
return result.rows[0];
|
||||
}
|
||||
[userId, path, key, ttlSeconds, chatId],
|
||||
);
|
||||
return result.rows[0];
|
||||
}
|
||||
|
||||
async forget(
|
||||
userId: string,
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import type { CreateMessageDto } from '../dto/create-message.dto.js';
|
||||
|
||||
// Contract: ASCII [A-Za-z0-9._:-]{1,128}. Anything else (including a duplicated
|
||||
// header, which Fastify joins with ", ") is rejected as a client error.
|
||||
@@ -11,8 +10,17 @@ export function messagesPath(chatId: string): string {
|
||||
|
||||
// The fingerprint is derived from the validated payload with a fixed field
|
||||
// order, so neither JSON key order nor an absent optional field can change it.
|
||||
export function idempotencyFingerprint(input: CreateMessageDto): string {
|
||||
// Text-only messages keep the lab3 fingerprint; the order of attachment ids
|
||||
// is significant because it is the order of attachments in the message.
|
||||
export function idempotencyFingerprint(
|
||||
text: string,
|
||||
attachmentIds: string[],
|
||||
): string {
|
||||
return createHash('sha256')
|
||||
.update(JSON.stringify({ text: input.text }))
|
||||
.update(
|
||||
attachmentIds.length === 0
|
||||
? JSON.stringify({ text })
|
||||
: JSON.stringify({ text, attachment_ids: attachmentIds }),
|
||||
)
|
||||
.digest('hex');
|
||||
}
|
||||
|
||||
@@ -4,4 +4,5 @@ export type Message = {
|
||||
sender_id: string;
|
||||
text: string;
|
||||
created_at: string;
|
||||
attachment_ids: string[];
|
||||
};
|
||||
|
||||
@@ -8,6 +8,7 @@ import { MessagesCacheModule } from './cache/messages-cache.module.js';
|
||||
import { IdempotencyModule } from './idempotency/idempotency.module.js';
|
||||
import { UsersModule } from '../users/users.module.js';
|
||||
import { RedisModule } from '../redis/redis.module.js';
|
||||
import { AttachmentsModule } from '../attachments/attachments.module.js';
|
||||
import { MetricsModule } from '../metrics/metrics.module.js';
|
||||
|
||||
@Module({
|
||||
@@ -19,6 +20,7 @@ import { MetricsModule } from '../metrics/metrics.module.js';
|
||||
UsersModule,
|
||||
RedisModule,
|
||||
MetricsModule,
|
||||
AttachmentsModule,
|
||||
],
|
||||
controllers: [MessagesController],
|
||||
providers: [MessagesRepository, MessagesService],
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
import type { Cursor } from '../common/http/cursor.js';
|
||||
import type { Message } from './message.js';
|
||||
import { MessagesRepository } from './messages.repository.js';
|
||||
|
||||
@@ -9,35 +8,15 @@ const message = (id: string, created_at: string): Message => ({
|
||||
sender_id: 'sender',
|
||||
text: id,
|
||||
created_at,
|
||||
attachment_ids: [],
|
||||
});
|
||||
|
||||
describe('MessagesRepository', () => {
|
||||
it('uses named SQL history and a keyset cursor', async () => {
|
||||
const query = vi.fn().mockResolvedValue({ rows: [], rowCount: 0 });
|
||||
const repository = new MessagesRepository({
|
||||
query,
|
||||
withTransaction: vi.fn(),
|
||||
} as never);
|
||||
const cursor: Cursor = {
|
||||
scope: 'messages:chat',
|
||||
created_at: '2026-01-01T00:00:01.000Z',
|
||||
id: '1',
|
||||
};
|
||||
await repository.findPage('chat', 10, cursor);
|
||||
expect(query).toHaveBeenCalledWith(
|
||||
'messages.history',
|
||||
expect.stringContaining('(created_at, id) >'),
|
||||
expect.arrayContaining(['chat', cursor.created_at, cursor.id, 11]),
|
||||
);
|
||||
});
|
||||
|
||||
it('returns rows in response shape', async () => {
|
||||
const query = vi
|
||||
.fn()
|
||||
.mockResolvedValue({
|
||||
rows: [message('1', '2026-01-01T00:00:00.000Z')],
|
||||
rowCount: 1,
|
||||
});
|
||||
const query = vi.fn().mockResolvedValue({
|
||||
rows: [message('1', '2026-01-01T00:00:00.000Z')],
|
||||
rowCount: 1,
|
||||
});
|
||||
const repository = new MessagesRepository({ query } as never);
|
||||
await expect(repository.findPage('chat', 1)).resolves.toEqual({
|
||||
items: [message('1', '2026-01-01T00:00:00.000Z')],
|
||||
|
||||
@@ -10,6 +10,7 @@ type MessageRow = {
|
||||
sender_id: string;
|
||||
text: string;
|
||||
created_at: Date | string;
|
||||
attachment_ids?: string[] | null;
|
||||
};
|
||||
|
||||
export type CreateOnceRequest = {
|
||||
@@ -19,6 +20,7 @@ export type CreateOnceRequest = {
|
||||
key: string;
|
||||
fingerprint: string;
|
||||
text: string;
|
||||
attachmentIds: string[];
|
||||
};
|
||||
|
||||
function toMessage(row: MessageRow): Message {
|
||||
@@ -28,6 +30,7 @@ function toMessage(row: MessageRow): Message {
|
||||
sender_id: row.sender_id,
|
||||
text: row.text,
|
||||
created_at: new Date(row.created_at).toISOString(),
|
||||
attachment_ids: row.attachment_ids ?? [],
|
||||
};
|
||||
}
|
||||
|
||||
@@ -50,11 +53,15 @@ WITH claim AS (
|
||||
'sender_id', $1::uuid,
|
||||
'text', $6::text,
|
||||
'created_at', to_char(date_trunc('milliseconds', now()) AT TIME ZONE 'UTC',
|
||||
'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')),
|
||||
'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'),
|
||||
'attachment_ids', to_jsonb($8::uuid[])),
|
||||
$5::uuid
|
||||
WHERE EXISTS (
|
||||
SELECT 1 FROM chat_members WHERE chat_id = $7::uuid AND user_id = $1
|
||||
)
|
||||
AND (SELECT count(*) FROM attachments a
|
||||
WHERE a.chat_id = $7::uuid AND a.status = 'ready'
|
||||
AND a.id = ANY($8::uuid[])) = cardinality($8::uuid[])
|
||||
ON CONFLICT DO NOTHING
|
||||
RETURNING response_body
|
||||
), inserted AS (
|
||||
@@ -62,6 +69,10 @@ WITH claim AS (
|
||||
SELECT $5::uuid, $7::uuid, $1::uuid, $6::text, date_trunc('milliseconds', now())
|
||||
FROM claim
|
||||
RETURNING id
|
||||
), linked AS (
|
||||
INSERT INTO message_attachments (message_id, chat_id, attachment_id, ordinal)
|
||||
SELECT $5::uuid, $7::uuid, t.id, t.ord
|
||||
FROM inserted, unnest($8::uuid[]) WITH ORDINALITY AS t(id, ord)
|
||||
), bumped AS (
|
||||
UPDATE chats SET message_version = message_version + 1
|
||||
WHERE id = $7::uuid AND EXISTS (SELECT 1 FROM inserted)
|
||||
@@ -88,6 +99,7 @@ export class MessagesRepository {
|
||||
randomUUID(),
|
||||
request.text,
|
||||
request.chatId,
|
||||
request.attachmentIds,
|
||||
],
|
||||
);
|
||||
return result.rows[0]?.response_body;
|
||||
@@ -112,12 +124,16 @@ export class MessagesRepository {
|
||||
let predicate = '';
|
||||
if (cursor) {
|
||||
values.push(cursor.created_at, cursor.id);
|
||||
predicate = 'AND (created_at, id) > ($2::timestamptz, $3::uuid)';
|
||||
predicate =
|
||||
'AND (m.created_at, m.id) > ($2::timestamptz, $3::uuid)';
|
||||
}
|
||||
values.push(limit + 1);
|
||||
const result = await this.database.query<MessageRow>(
|
||||
'messages.history',
|
||||
`SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = $1 ${predicate} ORDER BY created_at, id LIMIT $${values.length}`,
|
||||
`SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
|
||||
COALESCE((SELECT json_agg(ma.attachment_id ORDER BY ma.ordinal)
|
||||
FROM message_attachments ma WHERE ma.message_id = m.id), '[]'::json) AS attachment_ids
|
||||
FROM messages m WHERE m.chat_id = $1 ${predicate} ORDER BY m.created_at, m.id LIMIT $${values.length}`,
|
||||
values,
|
||||
);
|
||||
return {
|
||||
|
||||
@@ -28,9 +28,10 @@ describe('MessagesService idempotency expiry race', () => {
|
||||
sender_id: 'user',
|
||||
text: 'hello',
|
||||
created_at: '2026-01-01T00:00:00.000Z',
|
||||
attachment_ids: [],
|
||||
};
|
||||
let row: StoredRow | undefined = {
|
||||
request_fingerprint: idempotencyFingerprint({ text: 'hello' }),
|
||||
let row: StoredRow | undefined = {
|
||||
request_fingerprint: idempotencyFingerprint('hello', []),
|
||||
response_body: {
|
||||
...newMessage,
|
||||
id: 'message-old',
|
||||
@@ -58,9 +59,9 @@ describe('MessagesService idempotency expiry race', () => {
|
||||
}
|
||||
if (row) return undefined;
|
||||
messageWrites += 1;
|
||||
row = {
|
||||
request_fingerprint: idempotencyFingerprint({ text: 'hello' }),
|
||||
response_body: newMessage,
|
||||
row = {
|
||||
request_fingerprint: idempotencyFingerprint('hello', []),
|
||||
response_body: newMessage,
|
||||
created_at: new Date('2026-01-01T00:00:00.000Z'),
|
||||
expired: false,
|
||||
is_member: true,
|
||||
@@ -69,25 +70,29 @@ describe('MessagesService idempotency expiry race', () => {
|
||||
return newMessage;
|
||||
}),
|
||||
};
|
||||
const idempotency = {
|
||||
read: vi.fn(async () => {
|
||||
readCalls += 1;
|
||||
if (readCalls === 2) bothInitialReads.resolve();
|
||||
if (readCalls <= 2) await bothInitialReads.promise;
|
||||
return row;
|
||||
}),
|
||||
forget: vi.fn(async (...args: unknown[]) => {
|
||||
forgetCalls += 1;
|
||||
if (args.length !== 4) {
|
||||
row = undefined;
|
||||
return;
|
||||
}
|
||||
if (forgetCalls === 2) await replacementCreated.promise;
|
||||
if (!row || row.created_at >= new Date('2026-01-01T00:00:00.000Z')) return;
|
||||
row = undefined;
|
||||
}),
|
||||
isMember: vi.fn().mockResolvedValue(true),
|
||||
};
|
||||
const idempotency = {
|
||||
read: vi.fn(async () => {
|
||||
readCalls += 1;
|
||||
if (readCalls === 2) bothInitialReads.resolve();
|
||||
if (readCalls <= 2) await bothInitialReads.promise;
|
||||
return row;
|
||||
}),
|
||||
forget: vi.fn(async (...args: unknown[]) => {
|
||||
forgetCalls += 1;
|
||||
if (args.length !== 4) {
|
||||
row = undefined;
|
||||
return;
|
||||
}
|
||||
if (forgetCalls === 2) await replacementCreated.promise;
|
||||
if (
|
||||
!row ||
|
||||
row.created_at >= new Date('2026-01-01T00:00:00.000Z')
|
||||
)
|
||||
return;
|
||||
row = undefined;
|
||||
}),
|
||||
isMember: vi.fn().mockResolvedValue(true),
|
||||
};
|
||||
const makeService = () =>
|
||||
new MessagesService(
|
||||
messagesRepository as never,
|
||||
@@ -96,8 +101,9 @@ describe('MessagesService idempotency expiry race', () => {
|
||||
idempotency as never,
|
||||
{ getOrThrow: vi.fn().mockReturnValue(60) } as never,
|
||||
{ increment: vi.fn() } as never,
|
||||
{} as never,
|
||||
);
|
||||
const input = { text: 'hello' } as never;
|
||||
const input = { text: 'hello', attachment_ids: [] } as never;
|
||||
const user = { id: 'user' } as never;
|
||||
|
||||
const results = await Promise.all([
|
||||
|
||||
@@ -12,6 +12,7 @@ import type { Page } from '../common/http/page.js';
|
||||
import type { AppConfig } from '../config/app-config.schema.js';
|
||||
import type { User } from '../users/user.js';
|
||||
import { ChatsService } from '../chats/chats.service.js';
|
||||
import { AttachmentsRepository } from '../attachments/attachments.repository.js';
|
||||
import { MetricsService } from '../metrics/metrics.service.js';
|
||||
import type { CreateMessageDto } from './dto/create-message.dto.js';
|
||||
import type { Message } from './message.js';
|
||||
@@ -42,6 +43,7 @@ export class MessagesService {
|
||||
private readonly idempotency: IdempotencyRepository,
|
||||
private readonly config: ConfigService<AppConfig>,
|
||||
private readonly metrics: MetricsService,
|
||||
private readonly attachments: AttachmentsRepository,
|
||||
) {}
|
||||
|
||||
async create(
|
||||
@@ -53,14 +55,18 @@ export class MessagesService {
|
||||
// Ключ и тело проверяются до обращения к БД: клиентская ошибка не должна
|
||||
// создавать заявку и не должна зависеть от состояния хранилища.
|
||||
if (typeof rawKey !== 'string' || !IDEMPOTENCY_KEY_PATTERN.test(rawKey))
|
||||
throw new BadRequestException(
|
||||
'Idempotency-Key header is required',
|
||||
);
|
||||
if ([...input.text].every((character) => /\s/u.test(character)))
|
||||
throw new BadRequestException('Idempotency-Key header is required');
|
||||
const ids = input.attachment_ids.map((id) => id.toLowerCase());
|
||||
if (new Set(ids).size !== ids.length)
|
||||
throw new BadRequestException('attachment_ids must be unique');
|
||||
if (
|
||||
ids.length === 0 &&
|
||||
[...input.text].every((character) => /\s/u.test(character))
|
||||
)
|
||||
throw new BadRequestException('Message text cannot be blank');
|
||||
|
||||
const path = messagesPath(chatId);
|
||||
const fingerprint = idempotencyFingerprint(input);
|
||||
const fingerprint = idempotencyFingerprint(input.text, ids);
|
||||
const ttl = this.config.getOrThrow('IDEMPOTENCY_TTL_SECONDS');
|
||||
for (let attempt = 1; attempt <= MAX_CLAIM_ATTEMPTS; attempt += 1) {
|
||||
const created = await this.messagesRepository.createOnce({
|
||||
@@ -70,6 +76,7 @@ export class MessagesService {
|
||||
key: rawKey,
|
||||
fingerprint,
|
||||
text: input.text,
|
||||
attachmentIds: ids,
|
||||
});
|
||||
if (created) {
|
||||
this.metrics.increment('idempotency.claim');
|
||||
@@ -90,6 +97,12 @@ export class MessagesService {
|
||||
// чужую заявку сняли между нашими двумя запросами (503).
|
||||
if (!(await this.idempotency.isMember(chatId, currentUser.id)))
|
||||
throw new NotFoundException();
|
||||
// Unknown, foreign-chat or unfinished attachment: not found.
|
||||
if (
|
||||
ids.length > 0 &&
|
||||
!(await this.attachments.allLinkable(chatId, ids))
|
||||
)
|
||||
throw new NotFoundException();
|
||||
this.metrics.increment('idempotency.claim_lost');
|
||||
throw new ServiceUnavailableException(
|
||||
'Idempotency key could not be claimed',
|
||||
@@ -113,7 +126,11 @@ export class MessagesService {
|
||||
}
|
||||
if (!stored.is_member) throw new NotFoundException();
|
||||
this.metrics.increment('idempotency.replay');
|
||||
return stored.response_body;
|
||||
// Bodies stored before attachments existed carry no attachment_ids.
|
||||
return {
|
||||
...stored.response_body,
|
||||
attachment_ids: stored.response_body.attachment_ids ?? [],
|
||||
};
|
||||
}
|
||||
throw new ServiceUnavailableException(
|
||||
'Idempotency key could not be claimed',
|
||||
|
||||
Reference in New Issue
Block a user