lab6: базовый коммит из lab5 (0790599)

Перенесено дерево lab5 целиком: NestJS-приложение (API + worker,
RabbitMQ, outbox, S3), Dockerfile, compose.yaml, infra/{haproxy,minio,
rabbitmq}, стендовые скрипты, scripts/lab4, scripts/lab5,
evidence/lab4, evidence/lab5 и contracts/extensions.openapi.json.

Задание lab6 оставлено своим: README, REPORT, COURSE, lab.json,
contracts/{openapi,infrastructure,jobs.schema,attachment-states,
examples,observability}, tests, compose.example.yaml, infra/otel,
.github/pull_request_template.md.

Адаптации при переносе:
- docker-проект mtusi-lab5 -> mtusi-lab6 и тег образа
  mtusi-messenger:lab6 (compose.yaml, scripts/stop-one.sh, .env.example,
  infra/README.md), чтобы тома и контейнеры не пересекались с lab5;
- Makefile: цели lab5 сохранены целиком (skeleton lab6 не отличается
  от skeleton lab5);
- .env.example: переменные приложения и стенда lab5 + блок lab6
  (OTEL_*) из skeleton.
This commit is contained in:
2026-11-09 16:44:21 +03:00
parent 470d9d6f1e
commit 6889e77ab4
203 changed files with 27707 additions and 13 deletions
+105
View File
@@ -0,0 +1,105 @@
import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import pg from 'pg';
import { MetricsService } from '../metrics/metrics.service.js';
import type { AppConfig } from '../config/app-config.schema.js';
@Injectable()
export class DatabaseClient implements OnModuleDestroy {
readonly pool: pg.Pool;
private readonly healthPool: pg.Pool;
private readonly logger = new Logger(DatabaseClient.name);
constructor(
private readonly config: ConfigService<AppConfig>,
private readonly metrics: MetricsService,
) {
this.pool = new pg.Pool({
connectionString: this.config.getOrThrow('DATABASE_URL'),
max: this.config.getOrThrow('DATABASE_POOL_MAX'),
connectionTimeoutMillis: this.config.getOrThrow(
'DATABASE_CONNECTION_TIMEOUT_MS',
),
statement_timeout: this.config.getOrThrow(
'DATABASE_STATEMENT_TIMEOUT_MS',
),
// A relay that waits on the broker must not leave the transaction
// idle forever if confirm handling stalls.
options: `-c idle_in_transaction_session_timeout=${this.config.getOrThrow('DATABASE_IDLE_TX_TIMEOUT_MS')}`,
});
// Idle clients emit 'error' on the pool when the database drops; without a
// listener Node treats it as uncaught and terminates the process.
this.pool.on('error', (error: Error) => {
this.metrics.increment('database.pool_error');
this.logger.warn(`PostgreSQL pool error: ${error.message}`);
});
// Проба готовности не должна стоять в очереди за бизнес-запросами: занятый
// общий пул — это перегрузка, а не потеря зависимости.
this.healthPool = new pg.Pool({
connectionString: this.config.getOrThrow('DATABASE_URL'),
max: 1,
idleTimeoutMillis: 0,
connectionTimeoutMillis:
this.config.getOrThrow('HEALTH_TIMEOUT_MS'),
query_timeout: this.config.getOrThrow('HEALTH_TIMEOUT_MS'),
});
this.healthPool.on('error', (error: Error) => {
this.metrics.increment('database.health_pool_error');
this.logger.warn(
`PostgreSQL health connection error: ${error.message}`,
);
});
}
query<T extends pg.QueryResultRow = pg.QueryResultRow>(
label: string,
text: string,
values: readonly unknown[] = [],
): Promise<pg.QueryResult<T>> {
this.metrics.increment(label);
return this.pool.query<T>({ text, values: [...values] });
}
async health(): Promise<void> {
this.metrics.increment('health.database');
await this.healthPool.query('SELECT 1');
}
async withTransaction<T>(
work: (client: pg.PoolClient) => Promise<T>,
): Promise<T> {
const client = await this.pool.connect();
// A server-side idle_in_transaction_session_timeout or a network drop
// can emit 'error' on a checked-out client after pg-pool removed its
// idle listener. Without this listener Node treats it as uncaught.
let fatal: Error | undefined;
const onError = (error: Error): void => {
fatal = error;
};
client.on('error', onError);
try {
await client.query('BEGIN');
const result = await work(client);
await client.query('COMMIT');
return result;
} catch (error) {
try {
await client.query('ROLLBACK');
} catch (rollbackError) {
this.metrics.increment('database.rollback_error');
this.logger.warn(
`Rollback failed: ${rollbackError instanceof Error ? rollbackError.message : 'unknown error'}`,
);
}
// Callers must see the operation error, not the rollback error.
throw error;
} finally {
client.removeListener('error', onError);
client.release(fatal);
}
}
async onModuleDestroy(): Promise<void> {
await Promise.all([this.pool.end(), this.healthPool.end()]);
}
}
+10
View File
@@ -0,0 +1,10 @@
import { Module } from '@nestjs/common';
import { MetricsModule } from '../metrics/metrics.module.js';
import { DatabaseClient } from './database.client.js';
@Module({
imports: [MetricsModule],
providers: [DatabaseClient],
exports: [DatabaseClient],
})
export class DatabaseModule {}
+63
View File
@@ -0,0 +1,63 @@
import { readdir, readFile } from 'node:fs/promises';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { DatabaseClient } from './database.client.js';
import { MetricsService } from '../metrics/metrics.service.js';
import { parseDatabaseConfig } from '../config/app-config.schema.js';
const migrationsDirectory = join(
dirname(fileURLToPath(import.meta.url)),
'migrations',
);
async function migrationFiles(): Promise<string[]> {
const files = await readdir(migrationsDirectory);
return files.filter((file) => /^\d+_.+\.sql$/u.test(file)).sort();
}
export async function runMigrations(database: DatabaseClient): Promise<void> {
await database.query(
'migrations.ensure_table',
`CREATE TABLE IF NOT EXISTS schema_migrations (
version text PRIMARY KEY,
applied_at timestamptz NOT NULL DEFAULT now()
)`,
);
for (const version of await migrationFiles()) {
await database.withTransaction(async (client) => {
const applied = await client.query({
text: 'SELECT version FROM schema_migrations WHERE version = $1',
values: [version],
});
if (applied.rowCount) return;
const sql = await readFile(
join(migrationsDirectory, version),
'utf8',
);
await client.query(sql);
await client.query({
text: 'INSERT INTO schema_migrations (version) VALUES ($1)',
values: [version],
});
});
}
}
async function main(): Promise<void> {
const config = parseDatabaseConfig(process.env as Record<string, unknown>);
const database = new DatabaseClient(
{
getOrThrow: (key: keyof typeof config) => config[key],
} as never,
new MetricsService(),
);
try {
await runMigrations(database);
} finally {
await database.onModuleDestroy();
}
}
if (process.argv[1] === fileURLToPath(import.meta.url)) await main();
@@ -0,0 +1,41 @@
CREATE TABLE IF NOT EXISTS schema_migrations (
version text PRIMARY KEY,
applied_at timestamptz NOT NULL DEFAULT now()
);
CREATE TABLE IF NOT EXISTS users (
id uuid PRIMARY KEY,
username text NOT NULL UNIQUE,
display_name text NOT NULL,
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT now()
);
CREATE TABLE IF NOT EXISTS chats (
id uuid PRIMARY KEY,
title text NOT NULL,
created_at timestamptz NOT NULL DEFAULT now(),
message_version bigint NOT NULL DEFAULT 0
);
CREATE TABLE IF NOT EXISTS chat_members (
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
PRIMARY KEY (chat_id, user_id)
);
CREATE TABLE IF NOT EXISTS messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
sender_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
text text NOT NULL,
created_at timestamptz NOT NULL DEFAULT now(),
CONSTRAINT messages_sender_member_fk
FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id)
ON DELETE RESTRICT
);
CREATE INDEX IF NOT EXISTS messages_chat_created_id_idx
ON messages (chat_id, created_at, id);
CREATE INDEX IF NOT EXISTS chat_members_user_chat_idx
ON chat_members (user_id, chat_id);
@@ -0,0 +1,5 @@
-- HTTP timestamps and keyset cursors use JavaScript millisecond precision.
-- Store the same precision so a returned cursor cannot point before its own row.
ALTER TABLE users ALTER COLUMN created_at TYPE timestamptz(3);
ALTER TABLE chats ALTER COLUMN created_at TYPE timestamptz(3);
ALTER TABLE messages ALTER COLUMN created_at TYPE timestamptz(3);
@@ -0,0 +1,20 @@
-- Shared idempotency ledger for POST /api/v1/chats/{id}/messages.
-- The key scope is (user, method, path, key): the same key from another user or
-- another chat is independent, as the contract requires.
-- claim + response are committed in one transaction, so a committed row always
-- carries status_code and response_body (there is no "poisoned" intermediate state).
CREATE TABLE IF NOT EXISTS idempotency_keys (
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
method text NOT NULL,
path text NOT NULL,
key text NOT NULL,
request_fingerprint text NOT NULL,
status_code integer,
response_body jsonb,
message_id uuid,
created_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (user_id, method, path, key)
);
CREATE INDEX IF NOT EXISTS idempotency_keys_created_at_idx
ON idempotency_keys (created_at);
@@ -0,0 +1,9 @@
-- Instrumentation for the scaling experiments. pg_stat_statements only works
-- when the server is started with shared_preload_libraries=pg_stat_statements
-- (compose.yaml does that); without the preload the CREATE fails with a plain
-- notice, so a database started without the preload still migrates cleanly.
DO $$ BEGIN
CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
EXCEPTION WHEN OTHERS THEN
RAISE NOTICE 'pg_stat_statements unavailable: %', SQLERRM;
END $$;
@@ -0,0 +1,42 @@
ALTER TABLE messages ADD CONSTRAINT messages_id_chat_key UNIQUE (id, chat_id);
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
owner_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255),
content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
upload_mode text NOT NULL CHECK (upload_mode IN ('proxy', 'direct')),
status text NOT NULL CHECK (status IN ('pending', 'ready', 'deleting')),
object_key text NOT NULL UNIQUE,
size_bytes bigint CHECK (size_bytes BETWEEN 1 AND 10485760),
sha256 text CHECK (sha256 ~ '^[a-f0-9]{64}$'),
width integer CHECK (width > 0),
height integer CHECK (height > 0),
expires_at timestamptz(3),
created_at timestamptz(3) NOT NULL DEFAULT now(),
updated_at timestamptz(3) NOT NULL DEFAULT now(),
CONSTRAINT attachments_id_chat_key UNIQUE (id, chat_id),
CONSTRAINT attachments_owner_member_fk FOREIGN KEY (chat_id, owner_id)
REFERENCES chat_members (chat_id, user_id),
CONSTRAINT attachments_ready_complete CHECK (status <> 'ready' OR
(size_bytes IS NOT NULL AND sha256 IS NOT NULL AND width IS NOT NULL AND height IS NOT NULL)),
CONSTRAINT attachments_unfinished_expiry CHECK (status = 'ready' OR expires_at IS NOT NULL)
);
CREATE INDEX attachments_unfinished_idx ON attachments (expires_at) WHERE status <> 'ready';
CREATE INDEX attachments_owner_direct_pending_idx ON attachments (owner_id)
WHERE status = 'pending' AND upload_mode = 'direct';
CREATE TABLE message_attachments (
message_id uuid NOT NULL,
chat_id uuid NOT NULL,
attachment_id uuid NOT NULL,
ordinal smallint NOT NULL CHECK (ordinal BETWEEN 1 AND 10),
PRIMARY KEY (message_id, attachment_id),
CONSTRAINT message_attachments_ordinal_key UNIQUE (message_id, ordinal),
CONSTRAINT message_attachments_message_fk FOREIGN KEY (message_id, chat_id)
REFERENCES messages (id, chat_id) ON DELETE CASCADE,
CONSTRAINT message_attachments_attachment_fk FOREIGN KEY (attachment_id, chat_id)
REFERENCES attachments (id, chat_id)
);
CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id);
@@ -0,0 +1,8 @@
-- Rejected direct uploads keep their row until the signed URL can no longer be
-- used, so an object re-PUT after rejection is still found by the sweeper.
ALTER TABLE attachments DROP CONSTRAINT attachments_status_check;
ALTER TABLE attachments ADD CONSTRAINT attachments_status_check
CHECK (status IN ('pending', 'ready', 'deleting', 'rejected'));
DROP INDEX attachments_owner_direct_pending_idx;
CREATE INDEX attachments_owner_direct_pending_idx ON attachments (owner_id)
WHERE status IN ('pending', 'rejected') AND upload_mode = 'direct';
@@ -0,0 +1,81 @@
ALTER TABLE attachments DROP CONSTRAINT attachments_status_check;
ALTER TABLE attachments ADD CONSTRAINT attachments_status_check CHECK (status IN
('pending','queued','processing','ready','failed','deleting','rejected'));
ALTER TABLE attachments DROP CONSTRAINT attachments_ready_complete;
ALTER TABLE attachments ADD CONSTRAINT attachments_accepted_complete CHECK (
status NOT IN ('queued','processing','ready','failed') OR
(size_bytes IS NOT NULL AND sha256 IS NOT NULL AND width IS NOT NULL AND height IS NOT NULL));
ALTER TABLE attachments DROP CONSTRAINT attachments_unfinished_expiry;
ALTER TABLE attachments ADD CONSTRAINT attachments_unfinished_expiry CHECK (
status NOT IN ('pending','rejected','deleting') OR expires_at IS NOT NULL);
ALTER TABLE attachments
ADD COLUMN staging_key text UNIQUE,
ADD COLUMN error_code text CHECK (error_code ~ '^[a-z_]{1,64}$'),
ADD COLUMN ready_pipeline_version integer CHECK (ready_pipeline_version >= 1);
ALTER TABLE attachments ADD CONSTRAINT attachments_failed_code
CHECK (status <> 'failed' OR error_code IS NOT NULL);
DROP INDEX attachments_unfinished_idx;
CREATE INDEX attachments_unfinished_idx ON attachments (expires_at)
WHERE status IN ('pending','rejected','deleting');
CREATE INDEX attachments_staging_idx ON attachments (updated_at)
WHERE staging_key IS NOT NULL AND status IN ('ready','failed');
-- Rows accepted by lab4 already have their final original and complete metadata.
UPDATE attachments SET ready_pipeline_version = 1
WHERE status = 'ready' AND ready_pipeline_version IS NULL;
CREATE TABLE attachment_jobs (
job_id uuid PRIMARY KEY,
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
pipeline_version integer NOT NULL CHECK (pipeline_version >= 1),
status text NOT NULL CHECK (status IN ('queued','processing','succeeded','failed','superseded')),
attempt integer NOT NULL DEFAULT 1 CHECK (attempt >= 1),
max_attempts integer NOT NULL CHECK (max_attempts >= 1),
next_attempt_at timestamptz(3) NOT NULL DEFAULT now(),
lease_owner text, lease_token uuid, lease_until timestamptz(3),
last_error_code text,
created_at timestamptz(3) NOT NULL DEFAULT now(),
updated_at timestamptz(3) NOT NULL DEFAULT now(),
finished_at timestamptz(3),
CONSTRAINT attachment_jobs_effect_key UNIQUE (attachment_id, pipeline_version),
CONSTRAINT attachment_jobs_lease CHECK ((status = 'processing') =
(lease_owner IS NOT NULL AND lease_token IS NOT NULL AND lease_until IS NOT NULL))
);
CREATE INDEX attachment_jobs_lease_idx ON attachment_jobs (lease_until) WHERE status = 'processing';
CREATE INDEX attachment_jobs_failed_idx ON attachment_jobs (updated_at) WHERE status = 'failed';
CREATE TABLE attachment_variants (
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
pipeline_version integer NOT NULL CHECK (pipeline_version >= 1),
variant text NOT NULL CHECK (variant IN ('thumbnail','crop','optimized')),
object_key text NOT NULL UNIQUE,
content_type text NOT NULL CHECK (content_type IN ('image/jpeg','image/png','image/webp')),
size_bytes bigint NOT NULL CHECK (size_bytes >= 1),
width integer NOT NULL CHECK (width >= 1),
height integer NOT NULL CHECK (height >= 1),
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
job_id uuid NOT NULL REFERENCES attachment_jobs(job_id) ON DELETE CASCADE,
created_at timestamptz(3) NOT NULL DEFAULT now(),
PRIMARY KEY (attachment_id, pipeline_version, variant)
);
CREATE TABLE job_outbox (
id bigserial PRIMARY KEY,
job_id uuid NOT NULL REFERENCES attachment_jobs(job_id) ON DELETE CASCADE,
payload jsonb NOT NULL,
available_at timestamptz(3) NOT NULL DEFAULT now(),
published_at timestamptz(3),
created_at timestamptz(3) NOT NULL DEFAULT now()
);
CREATE INDEX job_outbox_pending_idx ON job_outbox (available_at, id) WHERE published_at IS NULL;
CREATE TABLE job_events (
id bigserial PRIMARY KEY,
job_id uuid NOT NULL REFERENCES attachment_jobs(job_id) ON DELETE CASCADE,
event text NOT NULL CHECK (event IN ('created','claimed','retry_scheduled','lease_expired',
'succeeded','failed','superseded','replayed','duplicate_skipped','staging_deleted')),
attempt integer NOT NULL,
detail jsonb NOT NULL DEFAULT '{}'::jsonb,
created_at timestamptz(3) NOT NULL DEFAULT now()
);
CREATE INDEX job_events_job_idx ON job_events (job_id, id);
@@ -0,0 +1,37 @@
ALTER TABLE attachment_jobs ADD COLUMN last_reconciled_at timestamptz(3);
CREATE INDEX job_outbox_job_idx ON job_outbox (job_id);
-- lab4 rows are ready but predate variants. Requeue them once so the worker
-- materializes the same v1 variants every other ready attachment has. The
-- INSERT ... RETURNING CTE keeps the job, outbox and audit event atomic.
WITH legacy AS (
INSERT INTO attachment_jobs
(job_id, attachment_id, pipeline_version, status, attempt, max_attempts,
next_attempt_at, created_at)
SELECT gen_random_uuid(), a.id, 1, 'queued', 1, 5, now(), now()
FROM attachments a
WHERE a.status = 'ready'
AND NOT EXISTS (
SELECT 1 FROM attachment_variants v WHERE v.attachment_id = a.id)
AND NOT EXISTS (
SELECT 1 FROM attachment_jobs j
WHERE j.attachment_id = a.id AND j.pipeline_version = 1)
RETURNING job_id, attachment_id, pipeline_version, attempt, created_at
), queued AS (
INSERT INTO job_outbox (job_id, payload, available_at)
SELECT job_id,
jsonb_build_object(
'schema_version', 1,
'job_id', job_id,
'type', 'attachment.process.v1',
'attachment_id', attachment_id,
'pipeline_version', pipeline_version,
'attempt', attempt,
'created_at', to_char(created_at AT TIME ZONE 'UTC',
'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')),
now()
FROM legacy
RETURNING job_id
)
INSERT INTO job_events (job_id, event, attempt, detail)
SELECT job_id, 'created', 1, '{"backfill":true}'::jsonb FROM legacy;
@@ -0,0 +1,10 @@
CREATE INDEX attachment_jobs_queued_idx
ON attachment_jobs (next_attempt_at) WHERE status = 'queued';
CREATE INDEX job_events_created_idx ON job_events (created_at);
CREATE INDEX job_outbox_published_idx
ON job_outbox (published_at) WHERE published_at IS NOT NULL;
ALTER TABLE job_events DROP CONSTRAINT job_events_event_check;
ALTER TABLE job_events ADD CONSTRAINT job_events_event_check CHECK (event IN
('created','claimed','retry_scheduled','lease_expired','succeeded','failed',
'superseded','replayed','duplicate_skipped','staging_deleted','reconciled'));