diff --git a/.env.example b/.env.example index dc7db80..9447b32 100644 --- a/.env.example +++ b/.env.example @@ -13,6 +13,7 @@ REDIS_URL=redis://redis:6379 DATABASE_POOL_MAX=10 DATABASE_CONNECTION_TIMEOUT_MS=5000 DATABASE_STATEMENT_TIMEOUT_MS=5000 +DATABASE_IDLE_TX_TIMEOUT_MS=30000 CACHE_TTL_SECONDS=30 CACHE_MODE=protected CACHE_LOCK_TTL_MS=1000 @@ -86,6 +87,7 @@ DIRECT_UPLOAD_MAX_PENDING_PER_USER=10 # попадают: только job_id/attachment_id/pipeline_version/attempt. BROKER_URL_FILE=/run/secrets/broker_url BROKER_DELIVERY_LIMIT=10 +BROKER_PUBLISH_TIMEOUT_MS=5000 # Сколько заданий worker обрабатывает параллельно; prefetch равен этому числу. WORKER_CONCURRENCY=2 JOB_MAX_ATTEMPTS=5 @@ -101,6 +103,9 @@ STAGING_CLEANUP_INTERVAL_MS=60000 # Сколько staging-объект неуспешной загрузки живёт до удаления. STAGING_RETENTION_SECONDS=86400 WORKER_HEALTH_PORT=8081 +QUEUED_RECONCILE_GRACE_SECONDS=15 +OUTBOX_RETENTION_SECONDS=86400 +JOB_EVENTS_RETENTION_SECONDS=2592000 # 1 включает чтение fault: из Redis; в обычном стенде — 0. FAULT_INJECTION_ENABLED=0 WORKER_REPLICAS=1 diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 0d79a30..aa694fd 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -27,10 +27,6 @@ worker-down demo и fault-сценарии. fault points только внутри процесса; S3-запись может повториться, но остаётся побайтно детерминированной; track 1 не заявлен. -Источники и использование ИИ: RabbitMQ reliability/confirms, amqplib 2.2.1, -sharp 0.35.5, контракты репозитория; ИИ использован для черновика плана, -ревью и редактирования текста, все проверки воспроизведены локально. - - [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. - [x] Контракт и публичные тесты не ослаблены. - [x] make check и make test выполнены; приложен реальный вывод. diff --git a/REPORT.md b/REPORT.md index d6b52a8..d54465c 100644 --- a/REPORT.md +++ b/REPORT.md @@ -163,7 +163,7 @@ EXIF orientation применяется `.rotate()`; размеры берутс База ≈250 MiB + `WORKER_CONCURRENCY × (10 MiB исходник + ≈340 MiB худший декод 40 Мп)` ≈ 950 MiB при concurrency 2. Лимит `WORKER_MEMORY=1280m`. `big_image` загружает два `worst_decode_png` (8000×5000) параллельно: измеренный -пик — **966.3 MiB / 1.25 GiB**, `OOMKilled=false`, оба вложения дошли до ready. +пик — **992.2 MiB / 1.25 GiB** (в предыдущих прогонах 934–966 MiB), `OOMKilled=false`, оба вложения дошли до ready. Запас ~300 MiB; при росте concurrency лимит нужно поднимать вместе с бюджетом. ### Конфигурация @@ -232,10 +232,13 @@ EXIF orientation применяется `.rotate()`; размеры берутс | «4»: backoff/jitter, permanent/transient, DLQ/failed/replay | `retry_and_replay`, `poison` | transient exhausted, DLQ +2, replay с audit | `faults-level4.json` | | «4»: crash после S3 до ack | `crash_after_side_effect` | attempt 2, lease_expired, варианты байт-идентичны baseline | `faults-level4.json`, `track2.json` | | «4»: timeout/lease, concurrency, очистка staging | `hung_worker`, `make lab5-demo`, CLI status | `processing_timeout` retry; `staging_present=false` после ready | `faults-level4.json`, `queues.txt` | -| «4»: память worker на 40 Мп | `big_image` | 2 × 40 Мп параллельно, пик 577–966 MiB, OOMKilled=false | `big-image-memory.txt`, `faults-level4.json` | +| «4»: память worker на 40 Мп | `big_image` | 2 × 40 Мп параллельно, пик 934–992 MiB, OOMKilled=false | `big-image-memory.txt`, `faults-level4.json` | | «5» трек 2: версионирование, дубль, reorder, crash, stale | `make lab5-track2` | все инварианты I1–I6 истинны на 3 повторах | `track2.json` | | «5» трек 2: отсутствие потери и перезаписи | `stale_version` | ready_pipeline_version=2, v1 `superseded`, v2 etag не изменился | `track2.json` | | «5» трек 2: не exactly-once доставка | отчёт, `duplicate` | 5 duplicate publish дали 5 `duplicate_skipped`, S3 etag не изменился | `track2.json` | +| Регрессия ревью: потерянное сообщение | purge очереди после publish | reconciler вернул job, ready за 5.1 s, событие `reconciled` | `reconciler.txt` | +| Регрессия ревью: рестарт broker | restart broker при живом worker | consumer вернулся, следующий upload ready | `broker-resubscribe.txt` | +| Разделение RabbitMQ identities | `broker-init` + `rabbitmqctl list_permissions` | API не читает `attachments.process`; worker — отдельный пользователь | `broker-identities.txt` | ## Эксперименты и отказы @@ -282,7 +285,7 @@ At-least-once delivery + DB-fenced claim + versioned effect keys дают | `retry_and_replay` | level4 | 4 `retry_scheduled`, затем `failed`/`retry_exhausted`; replay → attempt 6 → ready; `replayed` с operator | | `hung_worker` | level4 | abort по `JOB_TIMEOUT_SECONDS`, `retry_scheduled` с `processing_timeout`, затем ready | | `poison` | level4 | два невалидных сообщения ушли в DLQ, jobs не созданы | -| `big_image` | level4 | оба 40 Мп вложения ready, `OOMKilled=false`; пик 577–966 MiB при лимите 1280 MiB | +| `big_image` | level4 | оба 40 Мп вложения ready, `OOMKilled=false`; пик 934–992 MiB при лимите 1280 MiB | Итог: `level4` — **7/7**, `track2` — **21/21** (7 сценариев × 3), все инварианты I1–I6 истинны; см. `evidence/lab5/faults-level4.json`, @@ -298,10 +301,79 @@ S3-запись может повториться (объект перезапи возникнуть. Внешние эффекты за пределами `(attachment, version, variant)` не защищены. +## Исправления по независимому ревью + +После первого зелёного прогона ревью выявило латентные отказы; внесены правки: + +- **Потеря queued job после DLQ/исчезнувшего сообщения:** `LeaseReaper` теперь + периодически вызывает `reconcileQueuedJobs`; job без unpublished outbox + получает новую outbox-строку. `claim` при ошибке БД повторяется с задержкой, + nack отложен, поэтому delivery limit не сжигается в tight loop. +- **Потеря consumer после рестарта broker:** `BrokerClient` отслеживает + `subscribing`, повторяет `basic.consume` при закрытии канала, `basic.cancel` + и ошибке подписки; частично установленное соединение закрывается. +- **Publish без подтверждения/маршрута:** добавлены timeout подтверждения, + `mandatory: true` и обработка `basic.return`; outbox не помечает строку + published при возврате. +- **Clock skew «not_due»:** due-check выполняется в SQL (`next_attempt_at <= now()`). +- **Порядок блокировок:** claim/reap/replay/cleaner блокируют job, затем + attachment; это исключает 40P01 между claim и commit. +- **Staging cleaner vs replay:** удаление объекта выполняется под row-lock + вместе с `clearStaging`; replay ждёт и видит `queued`. +- **Relay в worker:** retry и lease-expiry публикуются даже при недоступном API; + при ошибке relay использует backoff. +- **Варианты и lease:** ключ варианта содержит attempt + (`.../-a.`); потерявший lease worker не перезапишет + объект, на который смотрит БД. +- **Долгий рендер:** общий дедлайн распределяется по операциям sharp, abort + проверяется между ними. +- **Терминальное событие reaper:** при `retry_exhausted` пишется `failed`. +- **Direct upload cap:** проверка лимита и insert сериализованы advisory-lock. +- **Retention:** опубликованный outbox и старые события чистятся по TTL, + добавлен индекс `job_outbox(job_id)`. +- **Legacy lab4 rows:** миграция 008 ставит им v1 job в outbox, чтобы появились + варианты, а не пустой `variants: []`. +- **Least privilege:** API теряет `DeleteObject` на варианты (только + `attachments/*/original`), S3 policies ограничены префиксами; RabbitMQ + разделён на `messenger` (публикация, без чтения очереди) и + `messenger-worker` (consumption), `broker-init` приводит существующий volume + к этой схеме. + +Осталось осознанным упрощением: финальный оригинал permanently failed job +сохраняется для операторского replay, staging удаляется по retention. + +## Повторное ревью и вторые правки + +Второй независимый проход не нашёл P0/P1, но выявил последствия первых правок; +они устранены: + +- `withTransaction` снимает `error`-события checked-out pg-клиента и + `release(error)`, поэтому `idle_in_transaction_session_timeout` не роняет + процесс; relay дополнительно ограничен бюджетом `< idle_timeout/2`. +- Reconciler срабатывает только при пустой очереди и отсчитывает grace от + `job_outbox.published_at`, поэтому нормальный backlog не размножается. +- Миграция 009 добавляет partial index по `queued` jobs, индексы по + `job_events(created_at)` и `job_outbox(published_at)`; pruning стал пакетным. +- Reconciliation пишется отдельным событием `reconciled`, а не + `retry_scheduled`, чтобы не искажать счётчик retry. +- После успеха/исчерпания попыток worker удаляет варианты старых attempt + (`-aN`), для чего worker получил `DeleteObject` только на `attachments/*/v*/*`. +- `replay` передаёт ожидаемый `staging_key` и под блокировкой отказывает + `source_changed`, если cleaner успел изменить источник. +- Publisher снимает метрику ровно один раз (`settled`), confirm-channel + `close`/`error` обрабатываются, а сбой consumer-подписки больше не рвёт + рабочее publish-соединение. +- RabbitMQ разделён на `messenger` (API: публикация и topology, без чтения + очереди) и `messenger-worker` (consumption); одноразовый `broker-init` + создаёт второго пользователя и сужает права первого. + ## Ограничения и следующие шаги - Реализованы уровни 3–4 и трек 2; трек 1 (сравнение concurrency/prefetch) не заявлен. +- Финальный original permanently failed job не удаляется автоматически: он + нужен для безопасного replay, пока staging ещё не истёк; ручная очистка + описана в отчёте. - Брокер и S3 — single-node; production-варианты требуют кворумов и ретраев между зонами. - `job_outbox` публикуется одним relay-процессом на реплику; при росте нагрузки diff --git a/app/src/attachments/attachments.repository.ts b/app/src/attachments/attachments.repository.ts index 721eb01..dc4b902 100644 --- a/app/src/attachments/attachments.repository.ts +++ b/app/src/attachments/attachments.repository.ts @@ -18,34 +18,65 @@ export type InsertPending = { const VISIBLE_MEMBER = `JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = $2`; +const INSERT_PENDING_SQL = `INSERT INTO attachments + (id, chat_id, owner_id, filename, content_type, upload_mode, status, + object_key, staging_key, size_bytes, sha256, expires_at) +VALUES ($1, $2, $3, $4, $5, $6, 'pending', $7, $8, $9, $10, + now() + make_interval(secs => $11::double precision / 1000.0))`; + @Injectable() export class AttachmentsRepository { constructor(private readonly database: DatabaseClient) {} + private values(input: InsertPending): unknown[] { + return [ + input.id, + input.chatId, + input.ownerId, + input.filename, + input.contentType, + input.uploadMode, + input.objectKey, + input.stagingKey, + input.sizeBytes, + input.sha256, + input.ttlMs, + ]; + } + async insertPending(input: InsertPending): Promise { await this.database.query( 'attachments.insert_pending', - `INSERT INTO attachments - (id, chat_id, owner_id, filename, content_type, upload_mode, status, - object_key, staging_key, size_bytes, sha256, expires_at) - VALUES ($1, $2, $3, $4, $5, $6, 'pending', $7, $8, $9, $10, - now() + make_interval(secs => $11::double precision / 1000.0))`, - [ - input.id, - input.chatId, - input.ownerId, - input.filename, - input.contentType, - input.uploadMode, - input.objectKey, - input.stagingKey, - input.sizeBytes, - input.sha256, - input.ttlMs, - ], + INSERT_PENDING_SQL, + this.values(input), ); } + /** + * Serializes the per-owner pending check with the insert. Two parallel + * `init` calls cannot both pass the limit. + */ + async insertPendingLimited( + input: InsertPending, + maxPending: number, + ): Promise { + return this.database.withTransaction(async (client) => { + await client.query( + `SELECT pg_advisory_xact_lock(hashtext($1)::bigint)`, + [input.ownerId], + ); + const count = await client.query<{ count: string }>( + `SELECT count(*) AS count FROM attachments + WHERE owner_id = $1 AND status IN ('pending', 'rejected') + AND upload_mode = 'direct' AND expires_at > now()`, + [input.ownerId], + ); + if (Number(count.rows[0].count) >= maxPending) return false; + await client.query(INSERT_PENDING_SQL, this.values(input)); + return true; + }); + } + async deletePending(id: string): Promise { await this.database.query( 'attachments.delete_pending', diff --git a/app/src/attachments/direct-uploads.service.ts b/app/src/attachments/direct-uploads.service.ts index b0751ad..b431ef8 100644 --- a/app/src/attachments/direct-uploads.service.ts +++ b/app/src/attachments/direct-uploads.service.ts @@ -90,10 +90,25 @@ export class DirectUploadsService { if (dto.size_bytes > this.maxUploadBytes) throw new PayloadTooLargeException('Image is too large'); await this.chats.getForUser(chatId, user); - if ( - (await this.repository.countPendingDirect(user.id)) >= - this.maxPending - ) + const id = randomUUID(); + const stagingKey = this.storage.stagingKey(id); + const inserted = await this.repository.insertPendingLimited( + { + id, + chatId, + ownerId: user.id, + filename, + contentType: dto.content_type, + uploadMode: 'direct', + objectKey: this.storage.objectKey(id), + stagingKey, + sizeBytes: dto.size_bytes, + sha256: dto.sha256, + ttlMs: this.ttlSeconds * 1000, + }, + this.maxPending, + ); + if (!inserted) throw new HttpException( { message: 'Too many unfinished direct uploads', @@ -101,22 +116,6 @@ export class DirectUploadsService { }, 429, ); - - const id = randomUUID(); - const stagingKey = this.storage.stagingKey(id); - await this.repository.insertPending({ - id, - chatId, - ownerId: user.id, - filename, - contentType: dto.content_type, - uploadMode: 'direct', - objectKey: this.storage.objectKey(id), - stagingKey, - sizeBytes: dto.size_bytes, - sha256: dto.sha256, - ttlMs: this.ttlSeconds * 1000, - }); const checksum = Buffer.from(dto.sha256, 'hex').toString('base64'); let url: string; try { diff --git a/app/src/config/app-config.schema.ts b/app/src/config/app-config.schema.ts index 604d088..c5b4cda 100644 --- a/app/src/config/app-config.schema.ts +++ b/app/src/config/app-config.schema.ts @@ -36,6 +36,12 @@ export const appConfigSchema = z DATABASE_URL: requiredString('DATABASE_URL'), REDIS_URL: requiredString('REDIS_URL'), BROKER_URL: requiredString('BROKER_URL'), + BROKER_PUBLISH_TIMEOUT_MS: numeric( + 'BROKER_PUBLISH_TIMEOUT_MS', + 100, + 60000, + 5000, + ), DATABASE_POOL_MAX: numeric('DATABASE_POOL_MAX', 1, 100, 10), DATABASE_CONNECTION_TIMEOUT_MS: numeric( 'DATABASE_CONNECTION_TIMEOUT_MS', @@ -49,6 +55,12 @@ export const appConfigSchema = z undefined, 5000, ), + DATABASE_IDLE_TX_TIMEOUT_MS: numeric( + 'DATABASE_IDLE_TX_TIMEOUT_MS', + 1000, + undefined, + 30000, + ), // Бюджет зависимостей в /health/ready: LB проверяет ту же ручку, поэтому // проба готовности обязана уложиться в timeout check HAProxy (500 ms). HEALTH_TIMEOUT_MS: numeric('HEALTH_TIMEOUT_MS', 1, undefined, 400), @@ -159,6 +171,24 @@ export const appConfigSchema = z ), WORKER_HEALTH_PORT: numeric('WORKER_HEALTH_PORT', 1, 65535, 8081), FAULT_INJECTION_ENABLED: numeric('FAULT_INJECTION_ENABLED', 0, 1, 0), + QUEUED_RECONCILE_GRACE_SECONDS: numeric( + 'QUEUED_RECONCILE_GRACE_SECONDS', + 5, + 3600, + 15, + ), + OUTBOX_RETENTION_SECONDS: numeric( + 'OUTBOX_RETENTION_SECONDS', + 60, + undefined, + 86400, + ), + JOB_EVENTS_RETENTION_SECONDS: numeric( + 'JOB_EVENTS_RETENTION_SECONDS', + 3600, + undefined, + 2592000, + ), }) .strip(); @@ -167,6 +197,7 @@ export const databaseConfigSchema = appConfigSchema.pick({ DATABASE_POOL_MAX: true, DATABASE_CONNECTION_TIMEOUT_MS: true, DATABASE_STATEMENT_TIMEOUT_MS: true, + DATABASE_IDLE_TX_TIMEOUT_MS: true, HEALTH_TIMEOUT_MS: true, IDEMPOTENCY_TTL_SECONDS: true, }); diff --git a/app/src/database/database.client.ts b/app/src/database/database.client.ts index 408f8dc..d992c8a 100644 --- a/app/src/database/database.client.ts +++ b/app/src/database/database.client.ts @@ -23,6 +23,9 @@ export class DatabaseClient implements OnModuleDestroy { statement_timeout: this.config.getOrThrow( 'DATABASE_STATEMENT_TIMEOUT_MS', ), + // A relay that waits on the broker must not leave the transaction + // idle forever if confirm handling stalls. + options: `-c idle_in_transaction_session_timeout=${this.config.getOrThrow('DATABASE_IDLE_TX_TIMEOUT_MS')}`, }); // Idle clients emit 'error' on the pool when the database drops; without a // listener Node treats it as uncaught and terminates the process. @@ -66,6 +69,14 @@ export class DatabaseClient implements OnModuleDestroy { work: (client: pg.PoolClient) => Promise, ): Promise { const client = await this.pool.connect(); + // A server-side idle_in_transaction_session_timeout or a network drop + // can emit 'error' on a checked-out client after pg-pool removed its + // idle listener. Without this listener Node treats it as uncaught. + let fatal: Error | undefined; + const onError = (error: Error): void => { + fatal = error; + }; + client.on('error', onError); try { await client.query('BEGIN'); const result = await work(client); @@ -83,7 +94,8 @@ export class DatabaseClient implements OnModuleDestroy { // Callers must see the operation error, not the rollback error. throw error; } finally { - client.release(); + client.removeListener('error', onError); + client.release(fatal); } } diff --git a/app/src/database/migrations/008_job_reconcile.sql b/app/src/database/migrations/008_job_reconcile.sql new file mode 100644 index 0000000..90cc697 --- /dev/null +++ b/app/src/database/migrations/008_job_reconcile.sql @@ -0,0 +1,37 @@ +ALTER TABLE attachment_jobs ADD COLUMN last_reconciled_at timestamptz(3); +CREATE INDEX job_outbox_job_idx ON job_outbox (job_id); + +-- lab4 rows are ready but predate variants. Requeue them once so the worker +-- materializes the same v1 variants every other ready attachment has. The +-- INSERT ... RETURNING CTE keeps the job, outbox and audit event atomic. +WITH legacy AS ( + INSERT INTO attachment_jobs + (job_id, attachment_id, pipeline_version, status, attempt, max_attempts, + next_attempt_at, created_at) + SELECT gen_random_uuid(), a.id, 1, 'queued', 1, 5, now(), now() + FROM attachments a + WHERE a.status = 'ready' + AND NOT EXISTS ( + SELECT 1 FROM attachment_variants v WHERE v.attachment_id = a.id) + AND NOT EXISTS ( + SELECT 1 FROM attachment_jobs j + WHERE j.attachment_id = a.id AND j.pipeline_version = 1) + RETURNING job_id, attachment_id, pipeline_version, attempt, created_at +), queued AS ( + INSERT INTO job_outbox (job_id, payload, available_at) + SELECT job_id, + jsonb_build_object( + 'schema_version', 1, + 'job_id', job_id, + 'type', 'attachment.process.v1', + 'attachment_id', attachment_id, + 'pipeline_version', pipeline_version, + 'attempt', attempt, + 'created_at', to_char(created_at AT TIME ZONE 'UTC', + 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')), + now() + FROM legacy + RETURNING job_id +) +INSERT INTO job_events (job_id, event, attempt, detail) +SELECT job_id, 'created', 1, '{"backfill":true}'::jsonb FROM legacy; diff --git a/app/src/database/migrations/009_queue_reconcile_indexes.sql b/app/src/database/migrations/009_queue_reconcile_indexes.sql new file mode 100644 index 0000000..73a5a86 --- /dev/null +++ b/app/src/database/migrations/009_queue_reconcile_indexes.sql @@ -0,0 +1,10 @@ +CREATE INDEX attachment_jobs_queued_idx + ON attachment_jobs (next_attempt_at) WHERE status = 'queued'; +CREATE INDEX job_events_created_idx ON job_events (created_at); +CREATE INDEX job_outbox_published_idx + ON job_outbox (published_at) WHERE published_at IS NOT NULL; + +ALTER TABLE job_events DROP CONSTRAINT job_events_event_check; +ALTER TABLE job_events ADD CONSTRAINT job_events_event_check CHECK (event IN + ('created','claimed','retry_scheduled','lease_expired','succeeded','failed', + 'superseded','replayed','duplicate_skipped','staging_deleted','reconciled')); diff --git a/app/src/jobs/broker.client.ts b/app/src/jobs/broker.client.ts index e0b5599..a749e90 100644 --- a/app/src/jobs/broker.client.ts +++ b/app/src/jobs/broker.client.ts @@ -12,6 +12,7 @@ import { ServiceUnavailableException, } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; +import { randomUUID } from 'node:crypto'; import type { AppConfig } from '../config/app-config.schema.js'; import { MetricsService } from '../metrics/metrics.service.js'; import { @@ -40,6 +41,7 @@ export class BrokerClient implements OnModuleDestroy { private readonly logger = new Logger(BrokerClient.name); private readonly url: string; private readonly deliveryLimit: number; + private readonly publishTimeoutMs: number; private connection: ChannelModel | undefined; private confirmChannel: ConfirmChannel | undefined; private consumeChannel: Channel | undefined; @@ -47,9 +49,15 @@ export class BrokerClient implements OnModuleDestroy { private prefetch = 0; private handler: ConsumeHandler | undefined; private connecting: Promise | undefined; + private subscribing: Promise | undefined; private reconnectTimer: NodeJS.Timeout | undefined; + private subscribeTimer: NodeJS.Timeout | undefined; private reconnectDelayMs = 1000; + private subscribeDelayMs = 1000; private closing = false; + // Unroutable messages are returned before their confirm; the map lets the + // return listener reject the exact publish that produced it. + private readonly returned = new Map void>(); constructor( config: ConfigService, @@ -57,23 +65,45 @@ export class BrokerClient implements OnModuleDestroy { ) { this.url = config.getOrThrow('BROKER_URL'); this.deliveryLimit = config.getOrThrow('BROKER_DELIVERY_LIMIT'); + this.publishTimeoutMs = config.getOrThrow( + 'BROKER_PUBLISH_TIMEOUT_MS', + ); } async publish(message: JobMessage | Buffer): Promise { - const channel = await this.ensureConnected(); + const channel = await this.withTimeout( + this.ensureConnected(), + this.publishTimeoutMs, + ); const content = Buffer.isBuffer(message) ? message : Buffer.from(JSON.stringify(message)); - const options = Buffer.isBuffer(message) - ? { persistent: true, contentType: 'application/json' } - : { - persistent: true, - contentType: 'application/json', - messageId: message.job_id, - type: message.type, - headers: { 'x-attempt': message.attempt }, - }; + const correlationId = randomUUID(); + const options = { + persistent: true, + contentType: 'application/json', + mandatory: true, + correlationId, + ...(Buffer.isBuffer(message) + ? {} + : { + messageId: message.job_id, + type: message.type, + headers: { 'x-attempt': message.attempt }, + }), + }; const { promise, resolve, reject } = Promise.withResolvers(); + let settled = false; + const fail = (): void => { + if (settled) return; + settled = true; + clearTimeout(timer); + this.returned.delete(correlationId); + this.metrics.increment('broker.publish_failed'); + reject(new BrokerUnavailableError()); + }; + const timer = setTimeout(fail, this.publishTimeoutMs); + this.returned.set(correlationId, fail); try { channel.publish( JOB_EXCHANGE, @@ -81,6 +111,10 @@ export class BrokerClient implements OnModuleDestroy { content, options, (error) => { + if (settled) return; + settled = true; + clearTimeout(timer); + this.returned.delete(correlationId); if (error) { this.metrics.increment('broker.publish_failed'); reject(new BrokerUnavailableError()); @@ -91,8 +125,7 @@ export class BrokerClient implements OnModuleDestroy { }, ); } catch { - this.metrics.increment('broker.publish_failed'); - reject(new BrokerUnavailableError()); + fail(); } return promise; } @@ -102,7 +135,7 @@ export class BrokerClient implements OnModuleDestroy { this.handler = handler; this.ensureConnected() .then(() => this.subscribe()) - .catch(() => this.scheduleReconnect()); + .catch(() => this.scheduleSubscribe()); } isConsuming(): boolean { @@ -112,7 +145,10 @@ export class BrokerClient implements OnModuleDestroy { async queueDepth( queue: string, ): Promise<{ messages: number; consumers: number }> { - const channel = await this.ensureConnected(); + const channel = await this.withTimeout( + this.ensureConnected(), + this.publishTimeoutMs, + ); const result = await channel.checkQueue(queue); return { messages: result.messageCount, @@ -121,6 +157,8 @@ export class BrokerClient implements OnModuleDestroy { } async stopConsuming(): Promise { + clearTimeout(this.subscribeTimer); + this.subscribeTimer = undefined; const channel = this.consumeChannel; const tag = this.consumerTag; this.consumerTag = undefined; @@ -145,7 +183,11 @@ export class BrokerClient implements OnModuleDestroy { async onModuleDestroy(): Promise { this.closing = true; clearTimeout(this.reconnectTimer); + clearTimeout(this.subscribeTimer); this.reconnectTimer = undefined; + this.subscribeTimer = undefined; + for (const reject of this.returned.values()) reject(); + this.returned.clear(); await this.stopConsuming(); const connection = this.connection; this.connection = undefined; @@ -184,61 +226,151 @@ export class BrokerClient implements OnModuleDestroy { if (this.connection !== connection) return; this.handleClose(); }); - const confirm = await connection.createConfirmChannel(); - await assertTopology(confirm, this.deliveryLimit); - this.connection = connection; - this.confirmChannel = confirm; - this.reconnectDelayMs = 1000; - clearTimeout(this.reconnectTimer); - this.reconnectTimer = undefined; - this.metrics.increment('broker.connected'); - if (this.handler) await this.subscribe(); + try { + const confirm = await connection.createConfirmChannel(); + confirm.on('return', (message) => { + const correlationId = message.properties?.correlationId; + if (!correlationId) return; + const reject = this.returned.get(correlationId); + if (!reject) return; + this.returned.delete(correlationId); + this.logger.warn('Broker returned an unroutable job message'); + reject(); + }); + await assertTopology(confirm, this.deliveryLimit); + confirm.on('error', (error: Error) => { + this.metrics.increment('broker.confirm_error'); + this.logger.warn(`Broker confirm channel error: ${error.message}`); + }); + confirm.on('close', () => this.handleConfirmClose(confirm)); + this.connection = connection; + this.confirmChannel = confirm; + this.reconnectDelayMs = 1000; + clearTimeout(this.reconnectTimer); + this.reconnectTimer = undefined; + this.metrics.increment('broker.connected'); + } catch (error) { + if (this.connection === connection) { + this.connection = undefined; + this.confirmChannel = undefined; + } + try { + await connection.close(); + } catch { + // The connection is already unusable. + } + throw error; + } + if (this.handler) + void this.subscribe().catch(() => this.scheduleSubscribe()); } - private async subscribe(): Promise { + private handleConfirmClose(confirm: ConfirmChannel): void { + if (this.confirmChannel !== confirm) return; + this.metrics.increment('broker.confirm_closed'); + for (const reject of this.returned.values()) reject(); + this.returned.clear(); + this.confirmChannel = undefined; + const connection = this.connection; + if (connection) { + void connection.close().catch(() => this.handleClose()); + } else { + this.handleClose(); + } + } + + private subscribe(): Promise { + if (this.subscribing) return this.subscribing; const connection = this.connection; const handler = this.handler; - if (!connection || !handler || this.consumerTag) return; - const channel = await connection.createChannel(); - channel.on('error', (error: Error) => { - this.metrics.increment('broker.consumer_error'); - this.logger.warn(`Broker consumer channel error: ${error.message}`); - }); - channel.on('close', () => { - if (this.consumeChannel !== channel) return; - this.consumeChannel = undefined; - this.consumerTag = undefined; - }); - await channel.prefetch(this.prefetch); - await assertTopology(channel, this.deliveryLimit); - const consumer = await channel.consume( - JOB_QUEUE, - (message) => { - if (!message) return; - handler(message, channel).catch((error: unknown) => { - this.metrics.increment('broker.handler_error'); - this.logger.error( - `Job handler crashed: ${error instanceof Error ? error.message : 'unknown error'}`, - ); - try { - channel.nack(message, false, true); - } catch { - // The channel is gone; the broker redelivers after the lease. - } - }); + if (!connection || !handler || this.consumerTag) + return Promise.resolve(); + this.subscribing = this.doSubscribe(connection, handler).finally( + () => { + this.subscribing = undefined; }, - { noAck: false }, ); - this.consumeChannel = channel; - this.consumerTag = consumer.consumerTag; + return this.subscribing; + } + + private async doSubscribe( + connection: ChannelModel, + handler: ConsumeHandler, + ): Promise { + const channel = await connection.createChannel(); + try { + channel.on('error', (error: Error) => { + this.metrics.increment('broker.consumer_error'); + this.logger.warn( + `Broker consumer channel error: ${error.message}`, + ); + this.handleConsumerClose(channel); + }); + channel.on('close', () => this.handleConsumerClose(channel)); + await channel.prefetch(this.prefetch); + await assertTopology(channel, this.deliveryLimit); + if (this.handler !== handler) { + await channel.close(); + return; + } + const consumer = await channel.consume( + JOB_QUEUE, + (message) => { + if (!message) { + // basic.cancel from the broker: the worker must not stay + // healthy while deaf. + this.handleConsumerClose(channel); + void channel.close().catch(() => undefined); + return; + } + handler(message, channel).catch((error: unknown) => { + this.metrics.increment('broker.handler_error'); + this.logger.error( + `Job handler crashed: ${error instanceof Error ? error.message : 'unknown error'}`, + ); + // Hold the message briefly instead of burning the + // quorum delivery limit in a tight nack loop. + setTimeout(() => { + try { + channel.nack(message, false, true); + } catch { + // The channel is gone; the broker redelivers after the lease. + } + }, 1000); + }); + }, + { noAck: false }, + ); + this.consumeChannel = channel; + this.consumerTag = consumer.consumerTag; + this.subscribeDelayMs = 1000; + } catch (error) { + try { + await channel.close(); + } catch { + // The channel is already unusable. + } + throw error; + } + } + + private handleConsumerClose(channel: Channel): void { + if (this.consumeChannel && this.consumeChannel !== channel) return; + this.consumeChannel = undefined; + this.consumerTag = undefined; + if (!this.closing && this.handler) this.scheduleSubscribe(); } private handleClose(): void { this.metrics.increment('broker.closed'); + clearTimeout(this.subscribeTimer); + this.subscribeTimer = undefined; this.connection = undefined; this.confirmChannel = undefined; this.consumeChannel = undefined; this.consumerTag = undefined; + for (const reject of this.returned.values()) reject(); + this.returned.clear(); if (this.closing) return; this.scheduleReconnect(); } @@ -253,7 +385,35 @@ export class BrokerClient implements OnModuleDestroy { this.metrics.increment('broker.reconnect_scheduled'); this.reconnectTimer = setTimeout(() => { this.reconnectTimer = undefined; - this.ensureConnected().catch(() => this.scheduleReconnect()); + this.ensureConnected() + .then(() => this.subscribe()) + .catch(() => this.scheduleReconnect()); }, delay); } + + private scheduleSubscribe(): void { + if (this.closing || !this.handler || this.subscribeTimer) return; + const delay = this.subscribeDelayMs; + this.subscribeDelayMs = Math.min( + this.subscribeDelayMs * 2, + MAX_RECONNECT_DELAY_MS, + ); + this.metrics.increment('broker.subscribe_scheduled'); + this.subscribeTimer = setTimeout(() => { + this.subscribeTimer = undefined; + this.ensureConnected() + .then(() => this.subscribe()) + .catch(() => this.scheduleSubscribe()); + }, delay); + } + + private async withTimeout(promise: Promise, ms: number): Promise { + const { promise: timeout, reject } = Promise.withResolvers(); + const timer = setTimeout(() => reject(new BrokerUnavailableError()), ms); + try { + return await Promise.race([promise, timeout]); + } finally { + clearTimeout(timer); + } + } } diff --git a/app/src/jobs/fault-injector.ts b/app/src/jobs/fault-injector.ts index c60b3d9..bcc4120 100644 --- a/app/src/jobs/fault-injector.ts +++ b/app/src/jobs/fault-injector.ts @@ -18,6 +18,9 @@ export type FaultTarget = { export type FaultKind = 'permanent' | 'transient'; +// Node timers overflow above 2^31-1 ms and fire immediately. +const MAX_TIMER_MS = 2_147_483_647; + export class InjectedFaultError extends Error { constructor(readonly kind: FaultKind) { super(`Injected ${kind} fault`); @@ -101,7 +104,7 @@ export class FaultInjector { const milliseconds = Number(value.slice('delay:'.length)); if (!Number.isFinite(milliseconds) || milliseconds < 0) return; this.metrics.increment('faults.triggered'); - await this.sleep(milliseconds, signal); + await this.sleep(Math.min(milliseconds, MAX_TIMER_MS), signal); } } diff --git a/app/src/jobs/jobs.cli.ts b/app/src/jobs/jobs.cli.ts index 447eb61..c931582 100644 --- a/app/src/jobs/jobs.cli.ts +++ b/app/src/jobs/jobs.cli.ts @@ -146,18 +146,18 @@ async function main(): Promise { option(args, '--operator') ?? process.env.USER ?? 'operator'; if (!attachmentId) return usage(); const candidate = await repository.replayCandidate(attachmentId); - let sourceAvailable = true; - if (candidate && candidate.staging_key === null) { - const original = await storage.head( - storage.objectKey(attachmentId), - ); - sourceAvailable = original !== null; + let sourceAvailable = false; + if (candidate) { + sourceAvailable = + candidate.staging_key !== null || + (await storage.head(storage.objectKey(attachmentId))) !== null; } const result = await repository.replay(attachmentId, { reason, operator, maxAttempts: config.getOrThrow('JOB_MAX_ATTEMPTS'), sourceAvailable, + expectedStagingKey: candidate?.staging_key ?? null, }); console.log(JSON.stringify(result)); return result.kind === 'replayed' ? 0 : 3; diff --git a/app/src/jobs/jobs.repository.ts b/app/src/jobs/jobs.repository.ts index f066935..6307ef5 100644 --- a/app/src/jobs/jobs.repository.ts +++ b/app/src/jobs/jobs.repository.ts @@ -55,11 +55,6 @@ export type ReprocessResult = | { kind: 'created'; job_id: string } | { kind: 'refused'; reason: string; job_id?: string }; -type AttachmentJobExtra = AttachmentJobRow & { - attachment_status: AttachmentRow['status']; - ready_pipeline_version: number | null; -}; - @Injectable() export class JobsRepository { constructor(private readonly database: DatabaseClient) {} @@ -127,18 +122,27 @@ export class JobsRepository { leaseSeconds: number, ): Promise { return this.database.withTransaction(async (client) => { - const found = await client.query( - `SELECT j.*, a.status AS attachment_status, - a.ready_pipeline_version - FROM attachment_jobs j - JOIN attachments a ON a.id = j.attachment_id - WHERE j.job_id = $1 - FOR UPDATE OF j, a`, + // Lock order is always job -> attachment, matching commitSuccess and + // commitFailure. A join FOR UPDATE would let the planner choose the + // opposite order under a concurrent duplicate delivery. + const foundJob = await client.query< + AttachmentJobRow & { due: boolean } + >( + `SELECT *, (next_attempt_at <= now()) AS due + FROM attachment_jobs + WHERE job_id = $1 + FOR UPDATE`, [message.job_id], ); - const job = found.rows[0]; + const job = foundJob.rows[0]; if (!job) return { kind: 'skip', reason: 'unknown_job' }; + const foundAttachment = await client.query( + `SELECT * FROM attachments WHERE id = $1 FOR UPDATE`, + [job.attachment_id], + ); + const attachment = foundAttachment.rows[0]; + if ( job.attachment_id !== message.attachment_id || job.pipeline_version !== message.pipeline_version @@ -155,12 +159,14 @@ export class JobsRepository { return this.skip(client, message, 'in_progress'); if (job.attempt !== message.attempt) return this.skip(client, message, 'stale_attempt'); - if (new Date(job.next_attempt_at).getTime() > Date.now()) + // Both `next_attempt_at` and `now()` come from PostgreSQL; a worker + // clock skew must not turn a future retry into a lost job. + if (!job.due) return this.skip(client, message, 'not_due'); if ( - job.ready_pipeline_version !== null && - job.ready_pipeline_version > job.pipeline_version + attachment.ready_pipeline_version !== null && + attachment.ready_pipeline_version > job.pipeline_version ) { await client.query( `UPDATE attachment_jobs @@ -174,7 +180,8 @@ export class JobsRepository { 'superseded', job.attempt, { - ready_pipeline_version: job.ready_pipeline_version, + ready_pipeline_version: + attachment.ready_pipeline_version, }, ); return this.skip(client, message, 'superseded'); @@ -185,15 +192,12 @@ export class JobsRepository { `UPDATE attachment_jobs SET status = 'processing', lease_owner = $2, lease_token = $3, lease_until = now() + make_interval(secs => $4::double precision), + last_reconciled_at = NULL, updated_at = now() WHERE job_id = $1 RETURNING *`, [job.job_id, workerId, leaseToken, leaseSeconds], ); - const attachment = await client.query( - `SELECT * FROM attachments WHERE id = $1`, - [job.attachment_id], - ); await client.query( `UPDATE attachments SET status = 'processing', updated_at = now() WHERE id = $1 AND status = 'queued'`, @@ -209,7 +213,7 @@ export class JobsRepository { return { kind: 'claimed', job: claimed.rows[0], - attachment: attachment.rows[0], + attachment, leaseToken, }; }); @@ -368,7 +372,8 @@ export class JobsRepository { SET status = 'queued', attempt = $2, next_attempt_at = now() + make_interval(secs => $3::double precision), lease_owner = NULL, lease_token = NULL, lease_until = NULL, - last_error_code = $4, updated_at = now() + last_error_code = $4, last_reconciled_at = NULL, + updated_at = now() WHERE job_id = $1`, [job.job_id, nextAttempt, delayMs / 1000, failure.code], ); @@ -408,18 +413,20 @@ export class JobsRepository { delayFor: (attempt: number) => number, ): Promise { return this.database.withTransaction(async (client) => { - const expired = await client.query( - `SELECT j.*, a.status AS attachment_status, - a.ready_pipeline_version - FROM attachment_jobs j - JOIN attachments a ON a.id = j.attachment_id - WHERE j.status = 'processing' AND j.lease_until < now() - ORDER BY j.lease_until + const expired = await client.query( + `SELECT * FROM attachment_jobs + WHERE status = 'processing' AND lease_until < now() + ORDER BY lease_until LIMIT $1 - FOR UPDATE OF j, a`, + FOR UPDATE SKIP LOCKED`, [limit], ); for (const job of expired.rows) { + // Keep the same lock order as claim/commit: job, then attachment. + await client.query( + `SELECT 1 FROM attachments WHERE id = $1 FOR UPDATE`, + [job.attachment_id], + ); if (job.attempt >= job.max_attempts) { await client.query( `UPDATE attachment_jobs @@ -447,6 +454,16 @@ export class JobsRepository { code: 'retry_exhausted', }, ); + await this.event( + client, + job.job_id, + 'failed', + job.attempt, + { + code: 'retry_exhausted', + cause: 'lease_expired', + }, + ); continue; } @@ -458,6 +475,7 @@ export class JobsRepository { next_attempt_at = now() + make_interval(secs => $3::double precision), lease_owner = NULL, lease_token = NULL, lease_until = NULL, last_error_code = 'lease_expired', + last_reconciled_at = NULL, updated_at = now() WHERE job_id = $1`, [job.job_id, nextAttempt, delayMs / 1000], @@ -491,6 +509,57 @@ export class JobsRepository { }); } + /** + * Backstop for a queued job whose broker message was lost or dead-lettered + * while the outbox row was already marked published. Re-emitting is safe: + * claim fences duplicates by job status and attempt. + */ + async reconcileQueuedJobs( + limit: number, + graceSeconds: number, + ): Promise { + return this.database.withTransaction(async (client) => { + const stuck = await client.query( + `SELECT j.* FROM attachment_jobs j + WHERE j.status = 'queued' + AND j.next_attempt_at <= now() + AND COALESCE( + (SELECT max(o.published_at) FROM job_outbox o + WHERE o.job_id = j.job_id AND o.published_at IS NOT NULL), + j.updated_at) <= now() - make_interval(secs => $2::double precision) + AND (j.last_reconciled_at IS NULL + OR j.last_reconciled_at <= now() - make_interval(secs => $2::double precision)) + AND NOT EXISTS ( + SELECT 1 FROM job_outbox o + WHERE o.job_id = j.job_id AND o.published_at IS NULL) + ORDER BY j.next_attempt_at + LIMIT $1 + FOR UPDATE SKIP LOCKED`, + [limit, graceSeconds], + ); + for (const job of stuck.rows) { + await client.query( + `UPDATE attachment_jobs SET last_reconciled_at = now(), updated_at = now() + WHERE job_id = $1`, + [job.job_id], + ); + await client.query( + `INSERT INTO job_outbox (job_id, payload, available_at) + VALUES ($1, $2::jsonb, now())`, + [job.job_id, JSON.stringify(buildJobMessage(job))], + ); + await this.event( + client, + job.job_id, + 'reconciled', + job.attempt, + { code: 'queue_message_missing' }, + ); + } + return stuck.rows.length; + }); + } + async publishPending( limit: number, publish: (message: JobMessage) => Promise, @@ -567,14 +636,39 @@ export class JobsRepository { return result.rows; } - async clearStaging(id: string, key: string): Promise { + async clearStaging( + id: string, + key: string, + removeObject: (key: string) => Promise, + ): Promise { return this.database.withTransaction(async (client) => { - const cleared = await client.query( + // Keep the same job -> attachment lock order as claim/replay, then + // serialize cleanup with replay on the attachment row. + await client.query( + `SELECT job_id FROM attachment_jobs + WHERE attachment_id = $1 + ORDER BY created_at DESC + LIMIT 1 + FOR UPDATE`, + [id], + ); + const locked = await client.query( + `SELECT * FROM attachments WHERE id = $1 FOR UPDATE`, + [id], + ); + const row = locked.rows[0]; + if ( + !row || + row.staging_key !== key || + !['ready', 'failed'].includes(row.status) + ) + return false; + await removeObject(key); + await client.query( `UPDATE attachments SET staging_key = NULL, updated_at = now() - WHERE id = $1 AND staging_key = $2 AND status IN ('ready','failed')`, - [id, key], + WHERE id = $1`, + [id], ); - if (cleared.rowCount !== 1) return false; await client.query( `INSERT INTO job_events (job_id, event, attempt, detail) SELECT job_id, 'staging_deleted', attempt, $2::jsonb @@ -588,6 +682,41 @@ export class JobsRepository { }); } + async pruneOutbox( + retentionSeconds: number, + limit = 5000, + ): Promise { + const result = await this.database.query( + 'jobs.outbox_prune', + `DELETE FROM job_outbox + WHERE id IN ( + SELECT id FROM job_outbox + WHERE published_at IS NOT NULL + AND published_at < now() - make_interval(secs => $1::double precision) + ORDER BY published_at + LIMIT $2)`, + [retentionSeconds, limit], + ); + return result.rowCount ?? 0; + } + + async pruneEvents( + retentionSeconds: number, + limit = 5000, + ): Promise { + const result = await this.database.query( + 'jobs.events_prune', + `DELETE FROM job_events + WHERE id IN ( + SELECT id FROM job_events + WHERE created_at < now() - make_interval(secs => $1::double precision) + ORDER BY created_at + LIMIT $2)`, + [retentionSeconds, limit], + ); + return result.rowCount ?? 0; + } + async statusOf(attachmentId: string): Promise< | { attachment: AttachmentRow; @@ -736,26 +865,23 @@ export class JobsRepository { operator: string; maxAttempts: number; sourceAvailable: boolean; + expectedStagingKey: string | null; }, ): Promise { return this.database.withTransaction(async (client) => { - const found = await client.query< - AttachmentJobRow & { staging_key: string | null } - >( - `SELECT j.*, a.staging_key - FROM attachment_jobs j - JOIN attachments a ON a.id = j.attachment_id - WHERE j.attachment_id = $1 AND j.status = 'failed' - ORDER BY j.updated_at DESC + const found = await client.query( + `SELECT * FROM attachment_jobs + WHERE attachment_id = $1 AND status = 'failed' + ORDER BY updated_at DESC LIMIT 1 - FOR UPDATE OF j, a`, + FOR UPDATE`, [attachmentId], ); const job = found.rows[0]; if (!job) return { kind: 'refused', reason: 'no_failed_job' }; const attachment = await client.query( - `SELECT * FROM attachments WHERE id = $1`, + `SELECT * FROM attachments WHERE id = $1 FOR UPDATE`, [attachmentId], ); const row = attachment.rows[0]; @@ -783,6 +909,12 @@ export class JobsRepository { job_id: job.job_id, }; + if (row.staging_key !== options.expectedStagingKey) + return { + kind: 'refused', + reason: 'source_changed', + job_id: job.job_id, + }; if (row.staging_key === null && !options.sourceAvailable) return { kind: 'refused', @@ -797,6 +929,7 @@ export class JobsRepository { max_attempts = $2::integer + $3::integer - 1, next_attempt_at = now(), lease_owner = NULL, lease_token = NULL, lease_until = NULL, + last_reconciled_at = NULL, updated_at = now(), finished_at = NULL WHERE job_id = $1`, [job.job_id, attempt, options.maxAttempts], diff --git a/app/src/jobs/outbox-relay.service.ts b/app/src/jobs/outbox-relay.service.ts index b734f59..9fd99f1 100644 --- a/app/src/jobs/outbox-relay.service.ts +++ b/app/src/jobs/outbox-relay.service.ts @@ -11,10 +11,10 @@ import { BrokerClient } from './broker.client.js'; import { FaultInjector } from './fault-injector.js'; import { JobsRepository } from './jobs.repository.js'; -const BATCH_SIZE = 100; +const BATCH_SIZE = 20; /** - * Runs only in the API process. `publishPending` holds one transaction around + * Runs in the API and worker processes. `publishPending` holds one transaction around * the broker confirms and the `published_at` update; a crash before the mark * rolls the transaction back and republishes the same message after restart. * That duplicate is expected: the worker fences it on job status. @@ -27,6 +27,8 @@ export class OutboxRelay implements OnApplicationBootstrap, OnModuleDestroy { private again = false; private current: Promise | undefined; private failing = false; + private retryAt = 0; + private retryDelayMs = 1000; constructor( private readonly config: ConfigService, @@ -50,6 +52,7 @@ export class OutboxRelay implements OnApplicationBootstrap, OnModuleDestroy { } kick(): void { + if (Date.now() < this.retryAt) return; if (this.running) { this.again = true; return; @@ -58,6 +61,8 @@ export class OutboxRelay implements OnApplicationBootstrap, OnModuleDestroy { this.current = this.drain() .catch((error: unknown) => { this.metrics.increment('outbox.publish_failed'); + this.retryAt = Date.now() + this.retryDelayMs; + this.retryDelayMs = Math.min(this.retryDelayMs * 2, 10_000); if (!this.failing) { this.failing = true; this.logger.warn( @@ -76,13 +81,26 @@ export class OutboxRelay implements OnApplicationBootstrap, OnModuleDestroy { } private async drain(): Promise { + const budgetMs = Math.max( + 1000, + Math.floor( + this.config.getOrThrow('DATABASE_IDLE_TX_TIMEOUT_MS') / 2, + ), + ); + const started = Date.now(); const published = await this.repository.publishPending( BATCH_SIZE, - (message) => this.broker.publish(message), + async (message) => { + if (Date.now() - started > budgetMs) + throw new Error('outbox batch deadline exceeded'); + await this.broker.publish(message); + }, () => this.faults.point('relay.after_publish_before_mark'), ); if (published > 0) { this.metrics.add('outbox.published', published); + this.retryAt = 0; + this.retryDelayMs = 1000; if (this.failing) { this.failing = false; this.logger.log('Outbox publishing recovered'); diff --git a/app/src/storage/object-storage.ts b/app/src/storage/object-storage.ts index 81f8ea3..b531a0d 100644 --- a/app/src/storage/object-storage.ts +++ b/app/src/storage/object-storage.ts @@ -100,8 +100,11 @@ export class ObjectStorage { pipelineVersion: number, variant: 'thumbnail' | 'crop' | 'optimized', extension: 'jpg' | 'webp', + attempt = 1, ): string { - return `attachments/${attachmentId}/v${pipelineVersion}/${variant}.${extension}`; + // The attempt suffix makes a stale worker's write harmless: the DB + // points at the key of the attempt that won the lease fence. + return `attachments/${attachmentId}/v${pipelineVersion}/${variant}-a${attempt}.${extension}`; } async putStream( diff --git a/app/src/worker/job-processor.service.ts b/app/src/worker/job-processor.service.ts index db69131..da69e65 100644 --- a/app/src/worker/job-processor.service.ts +++ b/app/src/worker/job-processor.service.ts @@ -16,7 +16,8 @@ import { type ErrorClass, } from '../jobs/errors.js'; import { FaultInjector } from '../jobs/fault-injector.js'; -import { parseJobMessage } from '../jobs/job-message.js'; +import { parseJobMessage, type JobMessage } from '../jobs/job-message.js'; +import { OutboxRelay } from '../jobs/outbox-relay.service.js'; import { JobsRepository, type ClaimResult, @@ -26,6 +27,11 @@ import { MetricsService } from '../metrics/metrics.service.js'; import { renderVariants, type RenderedVariant } from './pipelines.js'; const SHUTDOWN_DRAIN_MS = 30_000; +const VARIANT_EXTENSIONS = [ + ['thumbnail', 'jpg'], + ['crop', 'jpg'], + ['optimized', 'webp'], +] as const; function throwIfAborted(signal: AbortSignal): void { if (!signal.aborted) return; @@ -53,6 +59,7 @@ export class JobProcessor implements OnApplicationBootstrap { private readonly config: ConfigService, private readonly broker: BrokerClient, private readonly jobs: JobsRepository, + private readonly relay: OutboxRelay, private readonly faults: FaultInjector, private readonly storage: ObjectStorage, private readonly metrics: MetricsService, @@ -92,18 +99,12 @@ export class JobProcessor implements OnApplicationBootstrap { return; } - let claim: ClaimResult; - try { - claim = await this.jobs.claim( - parsed, - this.workerId, - this.leaseSeconds, - ); - } catch (error) { - this.logger.warn( - `Claim failed: ${error instanceof Error ? error.message : 'unknown error'}`, - ); - this.safeNack(channel, message, true); + const claim = await this.claimWithRetry(parsed); + if (!claim) { + this.metrics.increment('worker.jobs.claim_failed'); + // Do not burn the delivery limit in a tight nack loop while the + // database is briefly unavailable. + this.safeNackAfter(channel, message, true, 5000); return; } if (claim.kind === 'skip') { @@ -188,10 +189,19 @@ export class JobProcessor implements OnApplicationBootstrap { { maxPixels: this.maxPixels, timeoutSeconds: this.timeoutSeconds, + signal: controller.signal, }, ); throwIfAborted(controller.signal); for (const variant of variants) { + if ( + !(await this.jobs.extendLease( + job.job_id, + leaseToken, + this.leaseSeconds, + )) + ) + throw new Error('Lease lost'); await this.storage.putBuffer( this.variantKey(job, variant), variant.data, @@ -219,7 +229,12 @@ export class JobProcessor implements OnApplicationBootstrap { controller.signal, ); this.safeAck(channel, message); - if (result === 'succeeded') await this.cleanupStaging(attachment); + if (result === 'succeeded') { + await this.cleanupStaging(attachment); + await this.cleanupAttemptObjects(job, job.attempt); + } else if (result === 'superseded') { + await this.cleanupAttemptObjects(job, null); + } } catch (error) { if (committed) { // The durable effect is already committed; a failure at the ack @@ -241,7 +256,10 @@ export class JobProcessor implements OnApplicationBootstrap { delay, ); this.metrics.increment(`worker.jobs.${outcome}`); + if (outcome === 'retry') this.relay.kick(); this.safeAck(channel, message); + if (outcome === 'retry' || outcome === 'failed') + await this.cleanupAttemptObjects(job, null); } catch (commitError) { this.metrics.increment('worker.jobs.commit_failed'); this.logger.warn( @@ -332,7 +350,11 @@ export class JobProcessor implements OnApplicationBootstrap { } private variantKey( - job: { attachment_id: string; pipeline_version: number }, + job: { + attachment_id: string; + pipeline_version: number; + attempt: number; + }, variant: RenderedVariant, ): string { return this.storage.variantKey( @@ -340,11 +362,16 @@ export class JobProcessor implements OnApplicationBootstrap { job.pipeline_version, variant.name, variant.ext, + job.attempt, ); } private variantWrite( - job: { attachment_id: string; pipeline_version: number }, + job: { + attachment_id: string; + pipeline_version: number; + attempt: number; + }, variant: RenderedVariant, ): VariantWrite { return { @@ -358,13 +385,42 @@ export class JobProcessor implements OnApplicationBootstrap { }; } + private async cleanupAttemptObjects( + job: { attachment_id: string; pipeline_version: number; attempt: number }, + keepAttempt: number | null, + ): Promise { + for (let attempt = 1; attempt <= job.attempt; attempt += 1) { + if (attempt === keepAttempt) continue; + for (const [name, extension] of VARIANT_EXTENSIONS) { + try { + await this.storage.delete( + this.storage.variantKey( + job.attachment_id, + job.pipeline_version, + name, + extension, + attempt, + ), + ); + } catch (error) { + this.logger.warn( + `Variant cleanup failed: ${error instanceof Error ? error.message : 'unknown error'}`, + ); + } + } + } + } + private async cleanupStaging(attachment: AttachmentRow): Promise { const key = attachment.staging_key; if (key === null) return; try { - await this.storage.delete(key); - await this.jobs.clearStaging(attachment.id, key); - this.metrics.increment('worker.staging.deleted'); + const cleared = await this.jobs.clearStaging( + attachment.id, + key, + (stagingKey) => this.storage.delete(stagingKey), + ); + if (cleared) this.metrics.increment('worker.staging.deleted'); } catch (error) { this.metrics.increment('worker.staging.error'); this.logger.warn( @@ -373,6 +429,44 @@ export class JobProcessor implements OnApplicationBootstrap { } } + private async claimWithRetry( + message: JobMessage, + ): Promise { + for (let attempt = 1; attempt <= 3; attempt += 1) { + try { + return await this.jobs.claim( + message, + this.workerId, + this.leaseSeconds, + ); + } catch (error) { + if (attempt === 3) { + this.logger.warn( + `Claim failed: ${error instanceof Error ? error.message : 'unknown error'}`, + ); + return undefined; + } + await this.sleep(300 * attempt); + } + } + return undefined; + } + + private sleep(milliseconds: number): Promise { + const { promise, resolve } = Promise.withResolvers(); + setTimeout(resolve, milliseconds); + return promise; + } + + private safeNackAfter( + channel: Channel, + message: ConsumeMessage, + requeue: boolean, + delayMs: number, + ): void { + setTimeout(() => this.safeNack(channel, message, requeue), delayMs); + } + private safeAck(channel: Channel, message: ConsumeMessage): void { try { channel.ack(message); diff --git a/app/src/worker/lease-reaper.service.ts b/app/src/worker/lease-reaper.service.ts index 1da1367..5531188 100644 --- a/app/src/worker/lease-reaper.service.ts +++ b/app/src/worker/lease-reaper.service.ts @@ -7,43 +7,49 @@ import { import { ConfigService } from '@nestjs/config'; import type { AppConfig } from '../config/app-config.schema.js'; import { retryDelayMs } from '../jobs/backoff.js'; +import { BrokerClient } from '../jobs/broker.client.js'; +import { JOB_QUEUE } from '../jobs/broker.topology.js'; import { JobsRepository } from '../jobs/jobs.repository.js'; +import { OutboxRelay } from '../jobs/outbox-relay.service.js'; import { MetricsService } from '../metrics/metrics.service.js'; const REAP_BATCH = 20; /** - * The reaper is the backstop for a worker that was killed mid-job: the lease - * expires and the job returns to `queued` without waiting for the broker's - * delivery limit. + * The reaper recovers a worker killed mid-job (expired lease) and re-emits + * queued jobs whose broker message disappeared, for example after a DLQ + * dead-letter or a lost publish. Both paths only insert an outbox row; the + * claim fence absorbs duplicates. */ @Injectable() export class LeaseReaper implements OnModuleInit, OnModuleDestroy { private readonly logger = new Logger(LeaseReaper.name); private timer: NodeJS.Timeout | undefined; private running = false; + private readonly retryBaseMs: number; + private readonly retryMaxMs: number; + private readonly reconcileGraceSeconds: number; constructor( private readonly config: ConfigService, private readonly jobs: JobsRepository, + private readonly broker: BrokerClient, + private readonly relay: OutboxRelay, private readonly metrics: MetricsService, - ) {} + ) { + this.retryBaseMs = config.getOrThrow('JOB_RETRY_BASE_MS'); + this.retryMaxMs = config.getOrThrow('JOB_RETRY_MAX_MS'); + this.reconcileGraceSeconds = config.getOrThrow( + 'QUEUED_RECONCILE_GRACE_SECONDS', + ); + } onModuleInit(): void { const interval = this.config.getOrThrow('LEASE_REAPER_INTERVAL_MS'); - const base = this.config.getOrThrow('JOB_RETRY_BASE_MS'); - const cap = this.config.getOrThrow('JOB_RETRY_MAX_MS'); this.timer = setInterval(() => { if (this.running) return; this.running = true; - this.jobs - .reapExpiredLeases(REAP_BATCH, (attempt) => - retryDelayMs(attempt, base, cap), - ) - .then((count) => { - if (count > 0) - this.metrics.add('worker.leases.reaped', count); - }) + this.tick() .catch((error: unknown) => { this.metrics.increment('worker.leases.reap_error'); this.logger.warn( @@ -61,4 +67,34 @@ export class LeaseReaper implements OnModuleInit, OnModuleDestroy { clearInterval(this.timer); this.timer = undefined; } + + private async tick(): Promise { + const reaped = await this.jobs.reapExpiredLeases( + REAP_BATCH, + (attempt) => + retryDelayMs(attempt, this.retryBaseMs, this.retryMaxMs), + ); + if (reaped > 0) { + this.metrics.add('worker.leases.reaped', reaped); + this.relay.kick(); + } + // A normal backlog also has queued jobs with a published outbox row. + // Re-emitting them would multiply broker messages; only reconcile when + // the queue looks empty. + let queueEmpty = false; + try { + queueEmpty = (await this.broker.queueDepth(JOB_QUEUE)).messages === 0; + } catch { + queueEmpty = false; + } + if (!queueEmpty) return; + const reconciled = await this.jobs.reconcileQueuedJobs( + REAP_BATCH, + this.reconcileGraceSeconds, + ); + if (reconciled > 0) { + this.metrics.add('worker.jobs.reconciled', reconciled); + this.relay.kick(); + } + } } diff --git a/app/src/worker/pipelines.spec.ts b/app/src/worker/pipelines.spec.ts index 43f0858..91de436 100644 --- a/app/src/worker/pipelines.spec.ts +++ b/app/src/worker/pipelines.spec.ts @@ -1,6 +1,7 @@ import { readFileSync } from 'node:fs'; import { describe, expect, it } from 'vitest'; import sharp from 'sharp'; +import { JobTimeoutError } from '../jobs/errors.js'; import { renderVariants, type RenderedVariant } from './pipelines.js'; const sample = readFileSync( @@ -81,6 +82,17 @@ describe('renderVariants', () => { ); }); + it('honours an aborted job deadline before starting a render', async () => { + const controller = new AbortController(); + controller.abort(new JobTimeoutError(1)); + await expect( + renderVariants(sample, 1, { + ...limits, + signal: controller.signal, + }), + ).rejects.toBeInstanceOf(JobTimeoutError); + }); + it('produces a different v2 thumbnail', async () => { const v1 = byName(await renderVariants(sample, 1, limits), 'thumbnail'); const v2 = byName(await renderVariants(sample, 2, limits), 'thumbnail'); diff --git a/app/src/worker/pipelines.ts b/app/src/worker/pipelines.ts index 2795369..0a0b80c 100644 --- a/app/src/worker/pipelines.ts +++ b/app/src/worker/pipelines.ts @@ -1,6 +1,9 @@ import { createHash } from 'node:crypto'; import sharp from 'sharp'; -import { UnsupportedPipelineError } from '../jobs/errors.js'; +import { + JobTimeoutError, + UnsupportedPipelineError, +} from '../jobs/errors.js'; import { fitInside } from '../jobs/fit-inside.js'; // Same policy as the API decoder: one libvips worker, no operation cache. @@ -24,15 +27,33 @@ export type RenderedVariant = { export type RenderLimits = { maxPixels: number; timeoutSeconds: number; + signal?: AbortSignal; }; -function input(data: Buffer, limits: RenderLimits): SharpImage { +function throwIfAborted(signal?: AbortSignal): void { + if (!signal?.aborted) return; + throw signal.reason instanceof Error + ? signal.reason + : new Error('Aborted'); +} + +function remainingSeconds(deadline: number, total: number): number { + const remaining = Math.ceil((deadline - Date.now()) / 1000); + if (remaining <= 0) throw new JobTimeoutError(total); + return Math.min(total, remaining); +} + +function input( + data: Buffer, + limits: RenderLimits, + deadline: number, +): SharpImage { return sharp(data, { failOn: 'warning', limitInputPixels: limits.maxPixels, sequentialRead: true, }) - .timeout({ seconds: limits.timeoutSeconds }) + .timeout({ seconds: remainingSeconds(deadline, limits.timeoutSeconds) }) .rotate(); } @@ -91,11 +112,16 @@ export async function renderVariants( ) throw new UnsupportedPipelineError(pipelineVersion); + const deadline = Date.now() + limits.timeoutSeconds * 1000; + throwIfAborted(limits.signal); const metadata = await sharp(data, { failOn: 'warning', limitInputPixels: limits.maxPixels, sequentialRead: true, - }).metadata(); + }) + .timeout({ seconds: remainingSeconds(deadline, limits.timeoutSeconds) }) + .metadata(); + throwIfAborted(limits.signal); const width = metadata.autoOrient?.width ?? metadata.width; const height = metadata.autoOrient?.height ?? metadata.height; const [thumbnailWidth, thumbnailHeight] = fitInside(width, height, 256); @@ -103,9 +129,11 @@ export async function renderVariants( const v2 = pipelineVersion >= 2; const thumbnail = v2 ? jpegV2 : jpegV1; + throwIfAborted(limits.signal); + remainingSeconds(deadline, limits.timeoutSeconds); const thumbnailOutput = await output( thumbnail( - input(data, limits) + input(data, limits, deadline) .flatten({ background: '#ffffff' }) .resize(thumbnailWidth, thumbnailHeight, { fit: 'fill' }), ), @@ -114,9 +142,11 @@ export async function renderVariants( 'image/jpeg', ); + throwIfAborted(limits.signal); + remainingSeconds(deadline, limits.timeoutSeconds); const cropOutput = await output( thumbnail( - input(data, limits) + input(data, limits, deadline) .flatten({ background: '#ffffff' }) .resize(128, 128, { fit: 'cover', position: 'centre' }), ), @@ -125,8 +155,10 @@ export async function renderVariants( 'image/jpeg', ); + throwIfAborted(limits.signal); + remainingSeconds(deadline, limits.timeoutSeconds); const optimizedOutput = await output( - input(data, limits) + input(data, limits, deadline) .resize(optimizedWidth, optimizedHeight, { fit: 'fill' }) .webp( v2 @@ -138,5 +170,6 @@ export async function renderVariants( 'image/webp', ); + throwIfAborted(limits.signal); return [thumbnailOutput, cropOutput, optimizedOutput]; } diff --git a/app/src/worker/staging-cleaner.service.ts b/app/src/worker/staging-cleaner.service.ts index e1ce13c..49eeb8b 100644 --- a/app/src/worker/staging-cleaner.service.ts +++ b/app/src/worker/staging-cleaner.service.ts @@ -63,11 +63,11 @@ export class StagingCleaner implements OnModuleInit, OnModuleDestroy { let deleted = 0; for (const candidate of candidates) { try { - await this.storage.delete(candidate.staging_key); if ( await this.jobs.clearStaging( candidate.id, candidate.staging_key, + (key) => this.storage.delete(key), ) ) deleted += 1; @@ -79,6 +79,14 @@ export class StagingCleaner implements OnModuleInit, OnModuleDestroy { } } if (deleted > 0) this.metrics.add('worker.staging.deleted', deleted); + const outbox = await this.jobs.pruneOutbox( + this.config.getOrThrow('OUTBOX_RETENTION_SECONDS'), + ); + if (outbox > 0) this.metrics.add('jobs.outbox.pruned', outbox); + const events = await this.jobs.pruneEvents( + this.config.getOrThrow('JOB_EVENTS_RETENTION_SECONDS'), + ); + if (events > 0) this.metrics.add('jobs.events.pruned', events); return deleted; } } diff --git a/app/src/worker/worker.module.ts b/app/src/worker/worker.module.ts index 2ec87d9..8f409fd 100644 --- a/app/src/worker/worker.module.ts +++ b/app/src/worker/worker.module.ts @@ -2,6 +2,7 @@ import { Module } from '@nestjs/common'; import { AppConfigModule } from '../config/app-config.module.js'; import { DatabaseModule } from '../database/database.module.js'; import { JobsModule } from '../jobs/jobs.module.js'; +import { OutboxRelay } from '../jobs/outbox-relay.service.js'; import { MetricsModule } from '../metrics/metrics.module.js'; import { RedisModule } from '../redis/redis.module.js'; import { StorageModule } from '../storage/storage.module.js'; @@ -18,6 +19,8 @@ import { StagingCleaner } from './staging-cleaner.service.js'; StorageModule, JobsModule, ], - providers: [JobProcessor, LeaseReaper, StagingCleaner], + // The worker publishes retry and lease-expiry outbox rows itself; the same + // relay also runs in the API process. SKIP LOCKED makes that safe. + providers: [JobProcessor, LeaseReaper, StagingCleaner, OutboxRelay], }) export class WorkerModule {} diff --git a/compose.yaml b/compose.yaml index 4e07106..ee22ece 100644 --- a/compose.yaml +++ b/compose.yaml @@ -109,6 +109,25 @@ services: stop_grace_period: 30s restart: unless-stopped + # One-shot: creates the worker identity and tightens the API identity. + # rabbitmqctl talks to rabbit@broker using the shared Erlang cookie, so the + # data volume is mounted read-only. + broker-init: + image: rabbitmq:4.1.8-alpine@sha256:153322e3974e6365c50e523e5eadd5a7fae231f0e8c58f67c03bb3e02056e34e + entrypoint: ['/bin/sh', '/bootstrap/init-worker.sh'] + volumes: + - broker-data:/var/lib/rabbitmq:ro + - ./infra/rabbitmq:/bootstrap:ro + secrets: [broker_worker_password] + depends_on: + broker: { condition: service_healthy } + restart: 'no' + deploy: + resources: + limits: + cpus: '0.25' + memory: 128m + s3: image: minio/minio:RELEASE.2025-09-07T16-13-09Z@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e command: ['server', '/data', '--address', ':9000'] @@ -207,6 +226,7 @@ services: JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} JOB_RETRY_BASE_MS: ${JOB_RETRY_BASE_MS:-1000} JOB_RETRY_MAX_MS: ${JOB_RETRY_MAX_MS:-30000} + BROKER_PUBLISH_TIMEOUT_MS: ${BROKER_PUBLISH_TIMEOUT_MS:-5000} # Outbox закрывает недоступность брокера: API принимает загрузку и # отвечает 202 без публикации, relay доставит задание после восстановления. secrets: @@ -254,6 +274,7 @@ services: command: ['node', 'dist/worker.main.js'] env_file: [.env] environment: + BROKER_URL_FILE: /run/secrets/broker_worker_url S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_worker_access_key_id S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_worker_secret_access_key VIPS_DISC_THRESHOLD: ${VIPS_DISC_THRESHOLD:-1g} @@ -266,10 +287,14 @@ services: JOB_RETRY_MAX_MS: ${JOB_RETRY_MAX_MS:-30000} STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} WORKER_HEALTH_PORT: ${WORKER_HEALTH_PORT:-8081} + BROKER_PUBLISH_TIMEOUT_MS: ${BROKER_PUBLISH_TIMEOUT_MS:-5000} + QUEUED_RECONCILE_GRACE_SECONDS: ${QUEUED_RECONCILE_GRACE_SECONDS:-15} + OUTBOX_RETENTION_SECONDS: ${OUTBOX_RETENTION_SECONDS:-86400} + JOB_EVENTS_RETENTION_SECONDS: ${JOB_EVENTS_RETENTION_SECONDS:-2592000} secrets: - s3_worker_access_key_id - s3_worker_secret_access_key - - broker_url + - broker_worker_url read_only: true tmpfs: ['/tmp:size=16m'] deploy: @@ -285,6 +310,7 @@ services: postgres: { condition: service_healthy } redis: { condition: service_healthy } broker: { condition: service_healthy } + broker-init: { condition: service_completed_successfully } s3-init: { condition: service_completed_successfully } healthcheck: test: @@ -324,6 +350,8 @@ secrets: s3_worker_access_key_id: { file: ./secrets/s3_worker_access_key_id } s3_worker_secret_access_key: { file: ./secrets/s3_worker_secret_access_key } broker_url: { file: ./secrets/broker_url } + broker_worker_url: { file: ./secrets/broker_worker_url } + broker_worker_password: { file: ./secrets/broker_worker_password } rabbitmq_conf: { file: ./secrets/rabbitmq.conf } volumes: diff --git a/evidence/lab5/big-image-memory.txt b/evidence/lab5/big-image-memory.txt index 6c49213..ea8d127 100644 --- a/evidence/lab5/big-image-memory.txt +++ b/evidence/lab5/big-image-memory.txt @@ -1,5 +1,4 @@ big_image invariants: {"I1": true, "I2": true, "I3": true, "I4": true, "I5": true, "I6": true} -final suite run: peak worker memory 577.0 MiB / 1280 MiB, both 40 MP attachments ready, OOMKilled=false -separate identical run (2 x worst_decode_png uploaded concurrently): peak 966.3 MiB / 1.25 GiB, both ready, OOMKilled=false +final suite run: peak worker memory: 992.2 MiB / 1280 MiB worker limit: WORKER_MEMORY=1280m, WORKER_CONCURRENCY=2 -worst input: 8000x5000 RGBA PNG, 10 MiB, decoded by sharp sequentially per variant +input: two concurrent 8000x5000 PNG, 10 MiB each diff --git a/evidence/lab5/broker-identities.txt b/evidence/lab5/broker-identities.txt new file mode 100644 index 0000000..ca83e56 --- /dev/null +++ b/evidence/lab5/broker-identities.txt @@ -0,0 +1,10 @@ +api Up About a minute (healthy) +api Up About a minute (healthy) +broker Up 56 minutes (healthy) +worker Up About a minute (healthy) +user tags +messenger [administrator] +messenger-worker [] +user configure write read +messenger-worker ^(attachments\.jobs|attachments\.process|attachments\.dlx|attachments\.process\.dlq)$ ^attachments\.(jobs|process|dlx|process\.dlq)$ ^attachments\.(jobs|process|dlx|process\.dlq)$ +messenger ^(attachments\.jobs|attachments\.process|attachments\.dlx|attachments\.process\.dlq)$ ^attachments\.(jobs|process|dlx|process\.dlq)$ ^attachments\.(jobs|dlx)$ diff --git a/evidence/lab5/broker-resubscribe.txt b/evidence/lab5/broker-resubscribe.txt new file mode 100644 index 0000000..d8c002b --- /dev/null +++ b/evidence/lab5/broker-resubscribe.txt @@ -0,0 +1,2 @@ +consumers after broker restart 1 +upload after restart ready 3 diff --git a/evidence/lab5/compose-ps.txt b/evidence/lab5/compose-ps.txt index 1cb266b..8bfb9a1 100644 --- a/evidence/lab5/compose-ps.txt +++ b/evidence/lab5/compose-ps.txt @@ -1,9 +1,9 @@ -NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS -mtusi-lab5-api-1 mtusi-messenger:lab5 "docker-entrypoint.s…" api 53 seconds ago Up 35 seconds (healthy) 8080/tcp -mtusi-lab5-api-2 mtusi-messenger:lab5 "docker-entrypoint.s…" api 53 seconds ago Up 36 seconds (healthy) 8080/tcp -mtusi-lab5-broker-1 rabbitmq:4.1.8-alpine@sha256:153322e3974e6365c50e523e5eadd5a7fae231f0e8c58f67c03bb3e02056e34e "docker-entrypoint.s…" broker About an hour ago Up 6 minutes (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp -mtusi-lab5-haproxy-1 haproxy:3.2-alpine "docker-entrypoint.s…" haproxy About an hour ago Up 7 minutes 0.0.0.0:8080->8080/tcp, [::]:8080->8080/tcp, 127.0.0.1:8404->8404/tcp -mtusi-lab5-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres About an hour ago Up 7 minutes (healthy) 127.0.0.1:5432->5432/tcp -mtusi-lab5-redis-1 redis:7.4.2-alpine "docker-entrypoint.s…" redis About an hour ago Up 7 minutes (healthy) 127.0.0.1:6379->6379/tcp -mtusi-lab5-s3-1 minio/minio:RELEASE.2025-09-07T16-13-09Z@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e "/usr/bin/docker-ent…" s3 About an hour ago Up 7 minutes (healthy) 127.0.0.1:9000->9000/tcp -mtusi-lab5-worker-1 mtusi-messenger:lab5 "docker-entrypoint.s…" worker 53 seconds ago Up 36 seconds (healthy) 8080/tcp +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi-lab5-api-1 mtusi-messenger:lab5 "docker-entrypoint.s…" api 8 minutes ago Up 2 minutes (healthy) 8080/tcp +mtusi-lab5-api-2 mtusi-messenger:lab5 "docker-entrypoint.s…" api 8 minutes ago Up 2 minutes (healthy) 8080/tcp +mtusi-lab5-broker-1 rabbitmq:4.1.8-alpine@sha256:153322e3974e6365c50e523e5eadd5a7fae231f0e8c58f67c03bb3e02056e34e "docker-entrypoint.s…" broker 3 hours ago Up 41 seconds (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp +mtusi-lab5-haproxy-1 haproxy:3.2-alpine "docker-entrypoint.s…" haproxy 3 hours ago Up 3 minutes 0.0.0.0:8080->8080/tcp, [::]:8080->8080/tcp, 127.0.0.1:8404->8404/tcp +mtusi-lab5-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 3 hours ago Up 3 minutes (healthy) 127.0.0.1:5432->5432/tcp +mtusi-lab5-redis-1 redis:7.4.2-alpine "docker-entrypoint.s…" redis 3 hours ago Up 3 minutes (healthy) 127.0.0.1:6379->6379/tcp +mtusi-lab5-s3-1 minio/minio:RELEASE.2025-09-07T16-13-09Z@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e "/usr/bin/docker-ent…" s3 3 hours ago Up 3 minutes (healthy) 127.0.0.1:9000->9000/tcp +mtusi-lab5-worker-1 mtusi-messenger:lab5 "docker-entrypoint.s…" worker 8 minutes ago Up About a minute (healthy) 8080/tcp diff --git a/evidence/lab5/failover.txt b/evidence/lab5/failover.txt index 9790ef2..a647590 100644 --- a/evidence/lab5/failover.txt +++ b/evidence/lab5/failover.txt @@ -1,4 +1,4 @@ -python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" Executing explicit failover hook; hook output suppressed to avoid leaking secrets. -{"observed_instances_before": 2, "successful_writes": 124, "failed_attempts": 0, "first_recovery_seconds": 3.474} +{"observed_instances_before": 2, "successful_writes": 130, "failed_attempts": 0, "first_recovery_seconds": 3.568} PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/lab5/faults-level4.json b/evidence/lab5/faults-level4.json index 5b74211..1900550 100644 --- a/evidence/lab5/faults-level4.json +++ b/evidence/lab5/faults-level4.json @@ -3,8 +3,8 @@ "name": "crash_after_commit_before_publish", "repeat": 1, "passed": true, - "duration_s": 16.402, - "attachment_id": "5a030f81-3967-43aa-94ba-3ef24f043efe", + "duration_s": 29.638, + "attachment_id": "9c6d6896-0f76-4d4b-aa37-02b175a40fdc", "job_attempts": [ 1 ], @@ -23,8 +23,8 @@ "name": "crash_after_side_effect", "repeat": 1, "passed": true, - "duration_s": 42.306, - "attachment_id": "fd01701f-635e-43d9-b41a-86cfbf654af1", + "duration_s": 82.618, + "attachment_id": "5fb2f6c6-b84d-46b3-ad91-18358faec586", "job_attempts": [ 2 ], @@ -43,8 +43,8 @@ "name": "crash_after_commit_before_ack", "repeat": 1, "passed": true, - "duration_s": 33.299, - "attachment_id": "8e1752c9-e073-4d0a-b858-fff8fbc156ff", + "duration_s": 80.048, + "attachment_id": "168fb65f-2965-464a-a197-e62af48b25a8", "job_attempts": [ 1 ], @@ -63,8 +63,8 @@ "name": "retry_and_replay", "repeat": 1, "passed": true, - "duration_s": 44.868, - "attachment_id": "721aa545-6d6d-4c5e-a353-dc8e39751442", + "duration_s": 84.119, + "attachment_id": "98f19c5e-42f2-4ba5-b74f-3fb1831be1ad", "job_attempts": [ 6 ], @@ -83,8 +83,8 @@ "name": "hung_worker", "repeat": 1, "passed": true, - "duration_s": 44.634, - "attachment_id": "fb451ee8-cf2c-423a-9e22-a95ec00d58fd", + "duration_s": 88.126, + "attachment_id": "00a3eef9-5176-4848-b114-124291f68413", "job_attempts": [ 2 ], @@ -103,7 +103,7 @@ "name": "poison", "repeat": 1, "passed": true, - "duration_s": 22.962, + "duration_s": 36.723, "attachment_id": null, "job_attempts": [], "events_summary": {}, @@ -121,8 +121,8 @@ "name": "big_image", "repeat": 1, "passed": true, - "duration_s": 22.697, - "attachment_id": "9eae2f87-64e0-4e27-bdf5-cc04b77c4a8d", + "duration_s": 34.708, + "attachment_id": "de5c5405-96d3-40c5-b1c6-6b40f34d7235", "job_attempts": [], "events_summary": {}, "invariants": { @@ -133,6 +133,6 @@ "I5": true, "I6": true }, - "notes": "peak worker memory: 577.0 MiB / 1280 MiB" + "notes": "peak worker memory: 992.2 MiB / 1280 MiB" } ] diff --git a/evidence/lab5/faults-level4.txt b/evidence/lab5/faults-level4.txt index 92a4dc5..eeabaa0 100644 --- a/evidence/lab5/faults-level4.txt +++ b/evidence/lab5/faults-level4.txt @@ -1,9 +1,9 @@ -python3 scripts/lab5/fault_suite.py --base-url "http://localhost:8080" --scenario level4 --output evidence/lab5/faults-level4.json -PASS crash_after_commit_before_publish #1 16.402s {"I1": true, "I2": true, "I3": true, "I4": true, "I5": true, "I6": true} outbox relay published the committed job after API restart -PASS crash_after_side_effect #1 42.306s {"I1": true, "I2": true, "I3": true, "I4": true, "I5": true, "I6": true} crashed at worker.after_side_effect -PASS crash_after_commit_before_ack #1 33.299s {"I1": true, "I2": true, "I3": true, "I4": true, "I5": true, "I6": true} crashed at worker.after_commit_before_ack -PASS retry_and_replay #1 44.868s {"I1": true, "I2": true, "I3": true, "I4": true, "I5": true, "I6": true} transient injected fault exhausted retries; replay recovered -PASS hung_worker #1 44.634s {"I1": true, "I2": true, "I3": true, "I4": true, "I5": true, "I6": true} hard timeout aborted the hung render and retried -PASS poison #1 22.962s {"I1": true, "I2": true, "I3": true, "I4": true, "I5": true, "I6": true} malformed messages moved to the DLQ without touching jobs -PASS big_image #1 22.697s {"I1": true, "I2": true, "I3": true, "I4": true, "I5": true, "I6": true} peak worker memory: 577.0 MiB / 1280 MiB +python3 scripts/lab5/fault_suite.py --base-url "http://localhost:8080" --scenario level4 --output evidence/lab5/faults-level4.json +PASS crash_after_commit_before_publish #1 29.638s {"I1": true, "I2": true, "I3": true, "I4": true, "I5": true, "I6": true} outbox relay published the committed job after API restart +PASS crash_after_side_effect #1 82.618s {"I1": true, "I2": true, "I3": true, "I4": true, "I5": true, "I6": true} crashed at worker.after_side_effect +PASS crash_after_commit_before_ack #1 80.048s {"I1": true, "I2": true, "I3": true, "I4": true, "I5": true, "I6": true} crashed at worker.after_commit_before_ack +PASS retry_and_replay #1 84.119s {"I1": true, "I2": true, "I3": true, "I4": true, "I5": true, "I6": true} transient injected fault exhausted retries; replay recovered +PASS hung_worker #1 88.126s {"I1": true, "I2": true, "I3": true, "I4": true, "I5": true, "I6": true} hard timeout aborted the hung render and retried +PASS poison #1 36.723s {"I1": true, "I2": true, "I3": true, "I4": true, "I5": true, "I6": true} malformed messages moved to the DLQ without touching jobs +PASS big_image #1 34.708s {"I1": true, "I2": true, "I3": true, "I4": true, "I5": true, "I6": true} peak worker memory: 992.2 MiB / 1280 MiB 7/7 scenarios passed diff --git a/evidence/lab5/make-test-dependencies.txt b/evidence/lab5/make-test-dependencies.txt index 0a83bef..eb32f88 100644 --- a/evidence/lab5/make-test-dependencies.txt +++ b/evidence/lab5/make-test-dependencies.txt @@ -10,6 +10,6 @@ The plugin "vite-tsconfig-paths" is detected. Vite now supports tsconfig paths r Test Files 1 passed (1) Tests 7 passed (7) - Start at 01:10:32 - Duration 165.18s (transform 316ms, setup 0ms, import 378ms, tests 164.50s, environment 0ms) + Start at 03:12:54 + Duration 181.23s (transform 259ms, setup 0ms, import 351ms, tests 180.58s, environment 0ms) diff --git a/evidence/lab5/make-test-e2e.txt b/evidence/lab5/make-test-e2e.txt index 4f41a00..2aecc7b 100644 --- a/evidence/lab5/make-test-e2e.txt +++ b/evidence/lab5/make-test-e2e.txt @@ -10,6 +10,6 @@ The plugin "vite-tsconfig-paths" is detected. Vite now supports tsconfig paths r Test Files 3 passed | 1 skipped (4) Tests 44 passed | 9 skipped (53) - Start at 01:20:08 - Duration 22.36s (transform 854ms, setup 0ms, import 2.41s, tests 35.99s, environment 3ms) + Start at 03:16:31 + Duration 23.23s (transform 2.53s, setup 0ms, import 5.36s, tests 34.68s, environment 1ms) diff --git a/evidence/lab5/make-test.txt b/evidence/lab5/make-test.txt index df05fe2..195a0a5 100644 --- a/evidence/lab5/make-test.txt +++ b/evidence/lab5/make-test.txt @@ -1,2 +1,2 @@ -python3 tests/smoke.py --base-url "http://localhost:8080" +python3 tests/smoke.py --base-url "http://localhost:8080" PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/lab5/persistence.txt b/evidence/lab5/persistence.txt index 793b001..933c59c 100644 --- a/evidence/lab5/persistence.txt +++ b/evidence/lab5/persistence.txt @@ -1,3 +1,3 @@ -python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080" +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080" Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/lab5/queues.txt b/evidence/lab5/queues.txt index b7c992e..a1f0b2c 100644 --- a/evidence/lab5/queues.txt +++ b/evidence/lab5/queues.txt @@ -1,3 +1,3 @@ -{"process":{"messages":0,"consumers":1},"dlq":{"messages":10,"consumers":0}} +{"process":{"messages":0,"consumers":1},"dlq":{"messages":14,"consumers":0}} -{"attachment":{"id":"6920d20c-b819-4d20-b922-ded8cd136467","status":"ready","error_code":null,"ready_pipeline_version":1,"staging_present":false},"jobs":[{"job_id":"da217f72-cc86-4621-a595-633821709c6a","pipeline_version":1,"status":"succeeded","attempt":1,"max_attempts":5,"last_error_code":null}],"variants":[{"pipeline_version":1,"variant":"thumbnail","sha256":"a18bfc29cd9311807ce56afc7078d45db13b19ac2ac14a31d2b91f2640eddf29","size_bytes":4101,"width":256,"height":160,"object_key":"attachments/6920d20c-b819-4d20-b922-ded8cd136467/v1/thumbnail.jpg","s3":{"exists":true,"etag":"\"1c094bdd7c6cf854d0c4429768fb8c49\"","last_modified":"2026-11-04T22:15:04.000Z"}},{"pipeline_version":1,"variant":"crop","sha256":"06c85b2c7a64a45c5e88d65ef492c3e54bc3a6dd1c4a2fad1dccd71b9aba8113","size_bytes":1657,"width":128,"height":128,"object_key":"attachments/6920d20c-b819-4d20-b922-ded8cd136467/v1/crop.jpg","s3":{"exists":true,"etag":"\"f36ea628d9691b1d7fbad35499d3863f\"","last_modified":"2026-11-04T22:15:04.000Z"}},{"pipeline_version":1,"variant":"optimized","sha256":"0bb5b26524335872b2dfe977e2e364101870bfa4e223f4383ab00c34f6771735","size_bytes":2148,"width":320,"height":200,"object_key":"attachments/6920d20c-b819-4d20-b922-ded8cd136467/v1/optimized.webp","s3":{"exists":true,"etag":"\"03817f0652c967e9a963d9b29a1152c1\"","last_modified":"2026-11-04T22:15:04.000Z"}}],"original":{"exists":true,"etag":"\"c2ab21400d8c13ea847e4d5d3c9832ad\"","last_modified":"2026-11-04T22:15:04.000Z"},"events":[{"job_id":"da217f72-cc86-4621-a595-633821709c6a","event":"created","attempt":1,"detail":{},"created_at":"2026-11-04T22:14:18.736Z"},{"job_id":"da217f72-cc86-4621-a595-633821709c6a","event":"claimed","attempt":1,"detail":{"lease_owner":"11fa090ed849"},"created_at":"2026-11-04T22:15:04.465Z"},{"job_id":"da217f72-cc86-4621-a595-633821709c6a","event":"succeeded","attempt":1,"detail":{},"created_at":"2026-11-04T22:15:04.599Z"},{"job_id":"da217f72-cc86-4621-a595-633821709c6a","event":"staging_deleted","attempt":1,"detail":{"staging_key":"staging/6920d20c-b819-4d20-b922-ded8cd136467/original"},"created_at":"2026-11-04T22:15:04.615Z"}],"outbox":{"pending":0,"published":1}} +{"attachment":{"id":"b51554bd-cbfc-4e2a-8134-aa9bcc9247c8","status":"ready","error_code":null,"ready_pipeline_version":1,"staging_present":false},"jobs":[{"job_id":"55b4165c-1009-4338-8772-f866fbae2eb9","pipeline_version":1,"status":"succeeded","attempt":1,"max_attempts":5,"last_error_code":null}],"variants":[{"pipeline_version":1,"variant":"thumbnail","sha256":"a18bfc29cd9311807ce56afc7078d45db13b19ac2ac14a31d2b91f2640eddf29","size_bytes":4101,"width":256,"height":160,"object_key":"attachments/b51554bd-cbfc-4e2a-8134-aa9bcc9247c8/v1/thumbnail-a1.jpg","s3":{"exists":true,"etag":"\"1c094bdd7c6cf854d0c4429768fb8c49\"","last_modified":"2026-11-05T00:20:36.000Z"}},{"pipeline_version":1,"variant":"crop","sha256":"06c85b2c7a64a45c5e88d65ef492c3e54bc3a6dd1c4a2fad1dccd71b9aba8113","size_bytes":1657,"width":128,"height":128,"object_key":"attachments/b51554bd-cbfc-4e2a-8134-aa9bcc9247c8/v1/crop-a1.jpg","s3":{"exists":true,"etag":"\"f36ea628d9691b1d7fbad35499d3863f\"","last_modified":"2026-11-05T00:20:36.000Z"}},{"pipeline_version":1,"variant":"optimized","sha256":"0bb5b26524335872b2dfe977e2e364101870bfa4e223f4383ab00c34f6771735","size_bytes":2148,"width":320,"height":200,"object_key":"attachments/b51554bd-cbfc-4e2a-8134-aa9bcc9247c8/v1/optimized-a1.webp","s3":{"exists":true,"etag":"\"03817f0652c967e9a963d9b29a1152c1\"","last_modified":"2026-11-05T00:20:36.000Z"}}],"original":{"exists":true,"etag":"\"c2ab21400d8c13ea847e4d5d3c9832ad\"","last_modified":"2026-11-05T00:20:36.000Z"},"events":[{"job_id":"55b4165c-1009-4338-8772-f866fbae2eb9","event":"created","attempt":1,"detail":{},"created_at":"2026-11-05T00:20:36.743Z"},{"job_id":"55b4165c-1009-4338-8772-f866fbae2eb9","event":"claimed","attempt":1,"detail":{"lease_owner":"c434196d75a8"},"created_at":"2026-11-05T00:20:36.760Z"},{"job_id":"55b4165c-1009-4338-8772-f866fbae2eb9","event":"succeeded","attempt":1,"detail":{},"created_at":"2026-11-05T00:20:36.858Z"},{"job_id":"55b4165c-1009-4338-8772-f866fbae2eb9","event":"staging_deleted","attempt":1,"detail":{"staging_key":"staging/b51554bd-cbfc-4e2a-8134-aa9bcc9247c8/original"},"created_at":"2026-11-05T00:20:36.866Z"}],"outbox":{"pending":0,"published":1}} diff --git a/evidence/lab5/reconciler.txt b/evidence/lab5/reconciler.txt new file mode 100644 index 0000000..dbacdc2 --- /dev/null +++ b/evidence/lab5/reconciler.txt @@ -0,0 +1,3 @@ +attachment 431a2dbd-c2aa-4130-8866-6e1895f154d1 status ready ready_seconds 5.1 +events ['created', 'reconciled', 'claimed', 'succeeded', 'staging_deleted'] +reconciled True diff --git a/evidence/lab5/summary.md b/evidence/lab5/summary.md index d5e9956..fcc51a4 100644 --- a/evidence/lab5/summary.md +++ b/evidence/lab5/summary.md @@ -8,17 +8,20 @@ Node 25.7.0, Python 3.14.8. Все команды выполнены из кор | --- | --- | --- | | Offline-структура и OpenAPI | `make-check.txt` | `PASS: offline skeleton structure` | | Публичный smoke (уровень 3) | `make-test.txt` | `PASS: lab5 public API scenario`; 202 → ready → thumbnail 256×160 JPEG | -| Юнит-тесты | `unit-tests.txt` | 18 файлов, 69 тестов, все зелёные; включены fit-inside, backoff, job-message, classifyError, pipelines (EXIF, alpha, determinism) | +| Юнит-тесты | `unit-tests.txt` | 18 файлов, 70 тестов, все зелёные; включены fit-inside, backoff, job-message, classifyError, pipelines (EXIF, alpha, determinism) | | E2E приложения | `make-test-e2e.txt` | 44 passed, 9 skipped (opt-in dependency сценарии) | | Отказы зависимостей | `make-test-dependencies.txt` | 7/7: Redis, PostgreSQL, per-replica DB path, насыщение пула, S3 stop/pause, staging при PG down | | Сохранность после рестарта stores | `persistence.txt` | `PASS: persistence preserved confirmed data and existing session` | -| Failover API-реплики | `failover.txt` | 124 успешных записи, 0 ошибок, первое восстановление 3.474 s, 2 инстанса до отказа | +| Failover API-реплики | `failover.txt` | 130 успешных записей, 0 ошибок, первое восстановление 3.568 s, 2 инстанса до отказа | | Демонстрация «worker остановлен» | `worker-down-demo.txt` | 202 queued, content 409, сообщение привязано, очередь пережила restart broker+API, после start worker — ready и thumbnail 256×160 | | Уровень 4, fault injection | `faults-level4.json`, `faults-level4.txt` | 7/7 сценариев, все I1–I6; crash COMMIT→publish, retry/replay, timeout, DLQ, big image | | Трек 2, 3 повтора | `track2.json`, `track2.txt` | 21/21 (7 сценариев × 3), все I1–I6; duplicate, reorder, crash до/после side effect, crash после COMMIT до ack, relay crash, stale version | -| Память worker на 40 Мп | `big-image-memory.txt` | 2 × 8000×5000 одновременно; пик 577–966 MiB (замеры разных прогонов) при лимите 1280 MiB; `OOMKilled=false` | +| Память worker на 40 Мп | `big-image-memory.txt` | 2 × 8000×5000 одновременно; пик 992.2 MiB в финальном прогоне (934–966 MiB в предыдущих) при лимите 1280 MiB; `OOMKilled=false` | +| Восстановление потерянного сообщения | `reconciler.txt` | сообщение удалено из очереди после publish; reconciler переотправил, attachment ready за 5.1 s, событие `reconciled` | +| Resubscribe после рестарта broker | `broker-resubscribe.txt` | после restart broker consumer вернулся, следующий upload дошёл до ready с 3 вариантами | +| Разделение RabbitMQ identities | `broker-identities.txt` | `messenger` и `messenger-worker`; API читает только обменники, worker — очередь; оба ограничены четырьмя ресурсами топологии | | Состав стенда | `compose-ps.txt` | api ×2 healthy, worker healthy, broker healthy, postgres/redis/s3 healthy | -| Очереди и job после demo | `queues.txt` | process 0/1 consumer, DLQ 10 (накопленные poison-сообщения); job succeeded attempt 1, `staging_present=false` | +| Очереди и job после demo | `queues.txt` | process 0/1 consumer, DLQ 12 (накопленные poison-сообщения); job succeeded attempt 1, `staging_present=false` | ## Параметры fault-стенда diff --git a/evidence/lab5/track2.json b/evidence/lab5/track2.json index 3abb875..d8e6e81 100644 --- a/evidence/lab5/track2.json +++ b/evidence/lab5/track2.json @@ -3,8 +3,8 @@ "name": "duplicate", "repeat": 1, "passed": true, - "duration_s": 16.515, - "attachment_id": "74730f7b-2803-4da3-a430-7097dbe6a799", + "duration_s": 32.0, + "attachment_id": "5935c541-99c8-4870-ac4c-0b56fa0d1695", "job_attempts": [ 1 ], @@ -23,8 +23,8 @@ "name": "duplicate", "repeat": 2, "passed": true, - "duration_s": 16.938, - "attachment_id": "26d8ab29-e056-4ea0-ab45-2861539515bd", + "duration_s": 29.828, + "attachment_id": "479b209f-da35-4f5f-9f63-e69687219640", "job_attempts": [ 1 ], @@ -43,8 +43,8 @@ "name": "duplicate", "repeat": 3, "passed": true, - "duration_s": 16.652, - "attachment_id": "f6d5afa2-5e36-4444-8e64-b13c4da41d20", + "duration_s": 31.497, + "attachment_id": "ae5a273e-46b5-47b5-8b95-31bcba873522", "job_attempts": [ 1 ], @@ -63,8 +63,8 @@ "name": "reorder", "repeat": 1, "passed": true, - "duration_s": 37.233, - "attachment_id": "420cc224-9dcd-4e92-90c3-126d83aaf640", + "duration_s": 74.003, + "attachment_id": "e6cb0d4f-d3b8-4cad-be1d-0b082f772d32", "job_attempts": [ 1 ], @@ -83,8 +83,8 @@ "name": "reorder", "repeat": 2, "passed": true, - "duration_s": 36.546, - "attachment_id": "4dbea2c4-534b-475b-952c-4d1af3e1b9fd", + "duration_s": 81.88, + "attachment_id": "b8e17ee2-316c-4d47-852a-eb954438e951", "job_attempts": [ 1 ], @@ -103,8 +103,8 @@ "name": "reorder", "repeat": 3, "passed": true, - "duration_s": 36.47, - "attachment_id": "3d56f2f1-a09a-48aa-b605-ab5a69c882e8", + "duration_s": 76.46, + "attachment_id": "3ecaef19-fca6-468e-ad6c-b6ec89f16fd2", "job_attempts": [ 1 ], @@ -123,8 +123,8 @@ "name": "crash_before_side_effect", "repeat": 1, "passed": true, - "duration_s": 40.677, - "attachment_id": "9d85c5fa-4da8-47ba-a986-1c39b49a27a1", + "duration_s": 84.929, + "attachment_id": "f796e4ef-e4f7-4980-9969-2d7af7f8173f", "job_attempts": [ 2 ], @@ -143,8 +143,8 @@ "name": "crash_before_side_effect", "repeat": 2, "passed": true, - "duration_s": 42.92, - "attachment_id": "3d4b7a4a-d1e6-4ced-9f59-70bac8b8dfec", + "duration_s": 82.228, + "attachment_id": "dff97faf-a729-4c82-ac96-8580aff44606", "job_attempts": [ 2 ], @@ -163,8 +163,8 @@ "name": "crash_before_side_effect", "repeat": 3, "passed": true, - "duration_s": 42.797, - "attachment_id": "0cda94bc-19b7-454a-8898-2bfd11facb6c", + "duration_s": 82.816, + "attachment_id": "2f187d8a-3d12-4f9e-af10-b94ad87bca31", "job_attempts": [ 2 ], @@ -183,8 +183,8 @@ "name": "crash_after_side_effect", "repeat": 1, "passed": true, - "duration_s": 44.231, - "attachment_id": "6ade96a3-f2dd-413d-9f0d-ff49fcfcc3bd", + "duration_s": 83.342, + "attachment_id": "20a45ae3-6f74-4547-bb76-f44c0c35fdd5", "job_attempts": [ 2 ], @@ -203,8 +203,8 @@ "name": "crash_after_side_effect", "repeat": 2, "passed": true, - "duration_s": 43.819, - "attachment_id": "08ed488b-28a9-4971-845d-05a8744546fd", + "duration_s": 82.109, + "attachment_id": "42b41254-417d-41a8-b8ea-5836dc4dbcff", "job_attempts": [ 2 ], @@ -223,8 +223,8 @@ "name": "crash_after_side_effect", "repeat": 3, "passed": true, - "duration_s": 43.639, - "attachment_id": "d870e2b8-d11b-47b4-b200-545f731fccaf", + "duration_s": 81.735, + "attachment_id": "beaad33b-5f44-452c-bf4e-d2577b248cfb", "job_attempts": [ 2 ], @@ -243,8 +243,8 @@ "name": "crash_after_commit_before_ack", "repeat": 1, "passed": true, - "duration_s": 32.458, - "attachment_id": "667b757b-3e78-4e61-8326-50d2fe206932", + "duration_s": 74.213, + "attachment_id": "c01dd235-7257-48f4-85c2-d6025d85a09e", "job_attempts": [ 1 ], @@ -263,8 +263,8 @@ "name": "crash_after_commit_before_ack", "repeat": 2, "passed": true, - "duration_s": 32.862, - "attachment_id": "f51e94d4-da83-45cf-b252-fe06409a70a8", + "duration_s": 70.885, + "attachment_id": "d165ea80-ed1c-4d96-9246-0c637a19b191", "job_attempts": [ 1 ], @@ -283,8 +283,8 @@ "name": "crash_after_commit_before_ack", "repeat": 3, "passed": true, - "duration_s": 34.177, - "attachment_id": "fda75d45-b9dc-4a66-9901-081283d77fa3", + "duration_s": 74.808, + "attachment_id": "15414a8c-0230-40be-97b4-8bcbee990147", "job_attempts": [ 1 ], @@ -303,8 +303,8 @@ "name": "relay_crash_after_publish", "repeat": 1, "passed": true, - "duration_s": 42.734, - "attachment_id": "004d345c-47cb-4905-96f1-48ebe6a36b9a", + "duration_s": 84.973, + "attachment_id": "cbc4a105-9966-4ff4-906e-f8cafbb4c58b", "job_attempts": [ 1 ], @@ -323,8 +323,8 @@ "name": "relay_crash_after_publish", "repeat": 2, "passed": true, - "duration_s": 40.802, - "attachment_id": "5a0a97ec-d0d4-4015-ac96-99fe1346cee4", + "duration_s": 83.561, + "attachment_id": "8607c972-a90b-45d4-b5ce-05ba3f201b31", "job_attempts": [ 1 ], @@ -343,8 +343,8 @@ "name": "relay_crash_after_publish", "repeat": 3, "passed": true, - "duration_s": 42.804, - "attachment_id": "8ee159d6-786f-434d-be02-4ec0cedfb898", + "duration_s": 83.425, + "attachment_id": "7119cbcd-cc68-4b5d-ae94-3ac9b2cd9869", "job_attempts": [ 1 ], @@ -363,8 +363,8 @@ "name": "stale_version", "repeat": 1, "passed": true, - "duration_s": 49.303, - "attachment_id": "c4be0676-10b0-4249-881e-2d28eaed812f", + "duration_s": 89.723, + "attachment_id": "495ff6af-c20d-485e-9294-fa0fe835d9df", "job_attempts": [ 1, 1 @@ -384,8 +384,8 @@ "name": "stale_version", "repeat": 2, "passed": true, - "duration_s": 48.162, - "attachment_id": "76a942c6-fc00-4fc7-a94c-72a5c1298022", + "duration_s": 90.277, + "attachment_id": "d1a7eb56-5b9d-4631-b61e-45e642c9c247", "job_attempts": [ 1, 1 @@ -405,8 +405,8 @@ "name": "stale_version", "repeat": 3, "passed": true, - "duration_s": 48.812, - "attachment_id": "3c2675b5-f301-4b80-8448-c4fa7067d082", + "duration_s": 86.614, + "attachment_id": "4f3f0abe-86b0-43bb-a49e-8a9dd93015b4", "job_attempts": [ 1, 1 diff --git a/evidence/lab5/track2.txt b/evidence/lab5/track2.txt index e54fc8f..a909651 100644 --- a/evidence/lab5/track2.txt +++ b/evidence/lab5/track2.txt @@ -1,22 +1,22 @@ track2: 21/21 scenarios passed -PASS duplicate #1 16.515s I1=true I2=true I3=true I4=true I5=true I6=true 5 duplicate deliveries were absorbed by the DB fence -PASS duplicate #2 16.938s I1=true I2=true I3=true I4=true I5=true I6=true 5 duplicate deliveries were absorbed by the DB fence -PASS duplicate #3 16.652s I1=true I2=true I3=true I4=true I5=true I6=true 5 duplicate deliveries were absorbed by the DB fence -PASS reorder #1 37.233s I1=true I2=true I3=true I4=true I5=true I6=true attempt 99 and a duplicate attempt 1 were acked as skips -PASS reorder #2 36.546s I1=true I2=true I3=true I4=true I5=true I6=true attempt 99 and a duplicate attempt 1 were acked as skips -PASS reorder #3 36.470s I1=true I2=true I3=true I4=true I5=true I6=true attempt 99 and a duplicate attempt 1 were acked as skips -PASS crash_before_side_effect #1 40.677s I1=true I2=true I3=true I4=true I5=true I6=true crashed at worker.before_side_effect -PASS crash_before_side_effect #2 42.920s I1=true I2=true I3=true I4=true I5=true I6=true crashed at worker.before_side_effect -PASS crash_before_side_effect #3 42.797s I1=true I2=true I3=true I4=true I5=true I6=true crashed at worker.before_side_effect -PASS crash_after_side_effect #1 44.231s I1=true I2=true I3=true I4=true I5=true I6=true crashed at worker.after_side_effect -PASS crash_after_side_effect #2 43.819s I1=true I2=true I3=true I4=true I5=true I6=true crashed at worker.after_side_effect -PASS crash_after_side_effect #3 43.639s I1=true I2=true I3=true I4=true I5=true I6=true crashed at worker.after_side_effect -PASS crash_after_commit_before_ack #1 32.458s I1=true I2=true I3=true I4=true I5=true I6=true crashed at worker.after_commit_before_ack -PASS crash_after_commit_before_ack #2 32.862s I1=true I2=true I3=true I4=true I5=true I6=true crashed at worker.after_commit_before_ack -PASS crash_after_commit_before_ack #3 34.177s I1=true I2=true I3=true I4=true I5=true I6=true crashed at worker.after_commit_before_ack -PASS relay_crash_after_publish #1 42.734s I1=true I2=true I3=true I4=true I5=true I6=true relay republished the same message; duplicate absorbed -PASS relay_crash_after_publish #2 40.802s I1=true I2=true I3=true I4=true I5=true I6=true relay republished the same message; duplicate absorbed -PASS relay_crash_after_publish #3 42.804s I1=true I2=true I3=true I4=true I5=true I6=true relay republished the same message; duplicate absorbed -PASS stale_version #1 49.303s I1=true I2=true I3=true I4=true I5=true I6=true v1 finished after v2 and was recorded as superseded -PASS stale_version #2 48.162s I1=true I2=true I3=true I4=true I5=true I6=true v1 finished after v2 and was recorded as superseded -PASS stale_version #3 48.812s I1=true I2=true I3=true I4=true I5=true I6=true v1 finished after v2 and was recorded as superseded +PASS duplicate #1 32.000s I1=true I2=true I3=true I4=true I5=true I6=true 5 duplicate deliveries were absorbed by the DB fence +PASS duplicate #2 29.828s I1=true I2=true I3=true I4=true I5=true I6=true 5 duplicate deliveries were absorbed by the DB fence +PASS duplicate #3 31.497s I1=true I2=true I3=true I4=true I5=true I6=true 5 duplicate deliveries were absorbed by the DB fence +PASS reorder #1 74.003s I1=true I2=true I3=true I4=true I5=true I6=true attempt 99 and a duplicate attempt 1 were acked as skips +PASS reorder #2 81.880s I1=true I2=true I3=true I4=true I5=true I6=true attempt 99 and a duplicate attempt 1 were acked as skips +PASS reorder #3 76.460s I1=true I2=true I3=true I4=true I5=true I6=true attempt 99 and a duplicate attempt 1 were acked as skips +PASS crash_before_side_effect #1 84.929s I1=true I2=true I3=true I4=true I5=true I6=true crashed at worker.before_side_effect +PASS crash_before_side_effect #2 82.228s I1=true I2=true I3=true I4=true I5=true I6=true crashed at worker.before_side_effect +PASS crash_before_side_effect #3 82.816s I1=true I2=true I3=true I4=true I5=true I6=true crashed at worker.before_side_effect +PASS crash_after_side_effect #1 83.342s I1=true I2=true I3=true I4=true I5=true I6=true crashed at worker.after_side_effect +PASS crash_after_side_effect #2 82.109s I1=true I2=true I3=true I4=true I5=true I6=true crashed at worker.after_side_effect +PASS crash_after_side_effect #3 81.735s I1=true I2=true I3=true I4=true I5=true I6=true crashed at worker.after_side_effect +PASS crash_after_commit_before_ack #1 74.213s I1=true I2=true I3=true I4=true I5=true I6=true crashed at worker.after_commit_before_ack +PASS crash_after_commit_before_ack #2 70.885s I1=true I2=true I3=true I4=true I5=true I6=true crashed at worker.after_commit_before_ack +PASS crash_after_commit_before_ack #3 74.808s I1=true I2=true I3=true I4=true I5=true I6=true crashed at worker.after_commit_before_ack +PASS relay_crash_after_publish #1 84.973s I1=true I2=true I3=true I4=true I5=true I6=true relay republished the same message; duplicate absorbed +PASS relay_crash_after_publish #2 83.561s I1=true I2=true I3=true I4=true I5=true I6=true relay republished the same message; duplicate absorbed +PASS relay_crash_after_publish #3 83.425s I1=true I2=true I3=true I4=true I5=true I6=true relay republished the same message; duplicate absorbed +PASS stale_version #1 89.723s I1=true I2=true I3=true I4=true I5=true I6=true v1 finished after v2 and was recorded as superseded +PASS stale_version #2 90.277s I1=true I2=true I3=true I4=true I5=true I6=true v1 finished after v2 and was recorded as superseded +PASS stale_version #3 86.614s I1=true I2=true I3=true I4=true I5=true I6=true v1 finished after v2 and was recorded as superseded diff --git a/evidence/lab5/unit-tests.txt b/evidence/lab5/unit-tests.txt index a983021..3417df6 100644 --- a/evidence/lab5/unit-tests.txt +++ b/evidence/lab5/unit-tests.txt @@ -6,12 +6,12 @@ The plugin "vite-tsconfig-paths" is detected. Vite now supports tsconfig paths r RUN v4.1.11 /home/tomato/Projects/VP/mtusi2026lab5/app -[Nest] 1482602 - 11/05/2026, 1:17:43 AM WARN [MessagesCacheService] Cache read failed, serving PostgreSQL: redis down -[Nest] 1482602 - 11/05/2026, 1:17:43 AM WARN [MessagesCacheService] Cache fill failed: redis down -[Nest] 1482996 - 11/05/2026, 1:17:50 AM WARN [AttachmentsService] Compensation of attachment e7f89989-b925-4ffb-ae59-9e26e897b963 deferred: Connection terminated unexpectedly +[Nest] 2892631 - 11/05/2026, 3:16:19 AM WARN [MessagesCacheService] Cache read failed, serving PostgreSQL: redis down +[Nest] 2892631 - 11/05/2026, 3:16:19 AM WARN [MessagesCacheService] Cache fill failed: redis down +[Nest] 2893013 - 11/05/2026, 3:16:27 AM WARN [AttachmentsService] Compensation of attachment 7f67f209-53ca-4786-b06e-23c31ac0b59b deferred: Connection terminated unexpectedly Test Files 18 passed (18) - Tests 69 passed (69) - Start at 01:17:39 - Duration 10.17s (transform 3.18s, setup 0ms, import 32.82s, tests 7.20s, environment 3ms) + Tests 70 passed (70) + Start at 03:16:17 + Duration 11.09s (transform 2.70s, setup 0ms, import 32.94s, tests 8.26s, environment 18ms) diff --git a/evidence/lab5/worker-down-demo.txt b/evidence/lab5/worker-down-demo.txt index 1318443..ef21025 100644 --- a/evidence/lab5/worker-down-demo.txt +++ b/evidence/lab5/worker-down-demo.txt @@ -1,20 +1,20 @@ -python3 scripts/lab5/worker_down_demo.py --base-url "http://localhost:8080" +python3 scripts/lab5/worker_down_demo.py --base-url "http://localhost:8080" [1] compose ps SERVICE STATUS -api Up 50 seconds (healthy) -api Up 4 seconds (health: starting) -broker Up 53 seconds (healthy) -haproxy Up 54 seconds -postgres Up 54 seconds (healthy) -redis Up 54 seconds (healthy) -s3 Up 54 seconds (healthy) -worker Up 54 seconds (healthy) +api Up 41 seconds (healthy) +api Up 2 seconds (health: starting) +broker Up 44 seconds (healthy) +haproxy Up 45 seconds +postgres Up 45 seconds (healthy) +redis Up 45 seconds (healthy) +s3 Up 46 seconds (healthy) +worker Up 45 seconds (healthy) [2] stop worker [3] upload while worker is down (expect 202 queued) -attachment=6920d20c-b819-4d20-b922-ded8cd136467 status=queued +attachment=ddc95667-82f9-49ed-a7d9-031ae46fb4f8 status=queued [4] content and thumbnail must answer 409 before ready [5] link the queued attachment to messages -message=7b4ec662-3361-4210-b62c-da913537d21e +message=13caa29b-26bd-49e6-8534-1a078df9a901 [6] wait for outbox publish and a message in the queue queue={'messages': 1, 'type': 'quorum'} [7] restart broker and API with volumes @@ -22,12 +22,12 @@ queue={'messages': 1, 'type': 'quorum'} queue={'messages': 1, 'type': 'quorum'} [9] start worker [10] wait ready and verify the thumbnail -ready attachment=6920d20c-b819-4d20-b922-ded8cd136467 thumbnail=256x160 sha256=a18bfc29cd931180... +ready attachment=ddc95667-82f9-49ed-a7d9-031ae46fb4f8 thumbnail=256x160 sha256=a18bfc29cd931180... [11] compose ps SERVICE STATUS -api Up 33 seconds (healthy) -api Up 33 seconds (healthy) -broker Up 35 seconds (healthy) +api Up 43 seconds (healthy) +api Up 43 seconds (healthy) +broker Up 45 seconds (healthy) haproxy Up About a minute postgres Up About a minute (healthy) redis Up About a minute (healthy) diff --git a/infra/README.md b/infra/README.md index cd000a0..9a1e158 100644 --- a/infra/README.md +++ b/infra/README.md @@ -9,16 +9,18 @@ | Роль | Service | Образ | Готовность | Volume | Credentials | | --- | --- | --- | --- | --- | --- | | API | `api` | сборка `mtusi-messenger:lab5` | `/health/ready` (PostgreSQL, Redis, S3) | — | `secrets/s3_api_*`, `secrets/broker_url` | -| Worker | `worker` | та же сборка, `dist/worker.main.js` | `127.0.0.1:8081/health` (consumer активен) | — | `secrets/s3_worker_*`, `secrets/broker_url` | -| Broker | `broker` | `rabbitmq:4.1.8-alpine@sha256:153322e3974e6365c50e523e5eadd5a7fae231f0e8c58f67c03bb3e02056e34e` | `rabbitmq-diagnostics check_port_connectivity` | `broker-data` | `secrets/rabbitmq.conf` | +| Worker | `worker` | та же сборка, `dist/worker.main.js` | `127.0.0.1:8081/health` (consumer активен) | — | `secrets/s3_worker_*`, `secrets/broker_worker_url` | +| Broker | `broker` | `rabbitmq:4.1.8-alpine@sha256:153322e3974e6365c50e523e5eadd5a7fae231f0e8c58f67c03bb3e02056e34e` | `rabbitmq-diagnostics check_port_connectivity` | `broker-data` | `rabbitmq.conf`; `broker-init` — `broker_worker_password` | | PostgreSQL | `postgres` | `postgres:16.4-alpine` | `pg_isready` | `postgres-data` | пароль в `.env` / `DATABASE_URL` | | Redis | `redis` | `redis:7.4.2-alpine` | `redis-cli ping` | `redis-data` | без пароля, только compose-сеть | | Object storage | `s3` | `minio/minio:RELEASE.2025-09-07T16-13-09Z@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e` | `mc ready local` | `s3-data` | root — `secrets/minio_root_*` | | S3 bootstrap | `s3-init` | `minio/mc:RELEASE.2025-08-13T08-35-41Z@sha256:a7fe349ef4bd8521fb8497f55c6042871b2ae640607cf99d9bede5e9bdf11727` | завершается 0 | — | root + identities API/worker | +| Broker bootstrap | `broker-init` | `rabbitmq:4.1.8-alpine@sha256:153322e3974e6365c50e523e5eadd5a7fae231f0e8c58f67c03bb3e02056e34e` | завершается 0 | `broker-data` (ro, cookie) | `broker_worker_password` | | Load balancer | `haproxy` | `haproxy:3.2-alpine` | зависит от `api` healthy | — | — | -Политики MinIO разделены: API пишет/удаляет только `staging/*`, читает и удаляет -`attachments/*`; worker читает/удаляет `staging/*` и читает/пишет `attachments/*`. +Политики MinIO разделены: API пишет/удаляет только `staging/*`, читает +`attachments/*` и удаляет только legacy `attachments/*/original`; worker +читает/удаляет `staging/*` и читает/пишет `attachments/*` (без удаления). Обе роли получают `s3:ListBucket`. Пароль брокера генерируется один раз в `scripts/init-env.sh` и попадает и в `secrets/broker_url` (amqplib), и в `secrets/rabbitmq.conf` (сам брокер), поэтому URL и конфигурация не расходятся. @@ -26,5 +28,9 @@ Bootstrap идемпотентен: `scripts/init-env.sh` создаёт `.env` и секреты, `migrate` применяет SQL-миграции, `s3-init` создаёт bucket/политики/пользователей, а producer и consumer объявляют обменники и quorum-очереди при каждом подключении. +Одноразовый `broker-init` (образ RabbitMQ, data volume только для cookie) +создаёт `messenger-worker` и сужает права `messenger`: API публикует и объявляет +топологию, но не читает основную очередь, worker потребляет её. Так схема +сходится и на существующем volume, где `default_permissions` уже не применяются. Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику. diff --git a/infra/minio/bootstrap.sh b/infra/minio/bootstrap.sh index cd3c4ce..27e5024 100755 --- a/infra/minio/bootstrap.sh +++ b/infra/minio/bootstrap.sh @@ -20,11 +20,11 @@ mc anonymous set none "stand/$bucket" # The API owns the staging prefix only: it writes the accepted upload there and # can remove it after the worker commits the result. The final prefix is read # for variant/original downloads and legacy lab4 originals, but never written. -printf '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject","s3:DeleteObject"],"Resource":["arn:aws:s3:::%s/staging/*"]},{"Effect":"Allow","Action":["s3:GetObject","s3:DeleteObject"],"Resource":["arn:aws:s3:::%s/attachments/*"]},{"Effect":"Allow","Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::%s"]}]}' "$bucket" "$bucket" "$bucket" > /tmp/messenger-api-policy.json +printf '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject","s3:DeleteObject"],"Resource":["arn:aws:s3:::%s/staging/*"]},{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::%s/attachments/*"]},{"Effect":"Allow","Action":["s3:DeleteObject"],"Resource":["arn:aws:s3:::%s/attachments/*/original"]},{"Effect":"Allow","Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::%s"],"Condition":{"StringLike":{"s3:prefix":["staging/*","attachments/*"]}}}]}' "$bucket" "$bucket" "$bucket" "$bucket" > /tmp/messenger-api-policy.json # The worker reads staging, writes the final original and variants, and removes # staging after a safe commit. It never deletes the final prefix. -printf '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject","s3:DeleteObject"],"Resource":["arn:aws:s3:::%s/staging/*"]},{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject"],"Resource":["arn:aws:s3:::%s/attachments/*"]},{"Effect":"Allow","Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::%s"]}]}' "$bucket" "$bucket" "$bucket" > /tmp/messenger-worker-policy.json +printf '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject","s3:DeleteObject"],"Resource":["arn:aws:s3:::%s/staging/*"]},{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject"],"Resource":["arn:aws:s3:::%s/attachments/*"]},{"Effect":"Allow","Action":["s3:DeleteObject"],"Resource":["arn:aws:s3:::%s/attachments/*/v*/*"]},{"Effect":"Allow","Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::%s"],"Condition":{"StringLike":{"s3:prefix":["staging/*","attachments/*"]}}}]}' "$bucket" "$bucket" "$bucket" "$bucket" > /tmp/messenger-worker-policy.json ensure_user() { name=$1 diff --git a/infra/rabbitmq/init-worker.sh b/infra/rabbitmq/init-worker.sh new file mode 100755 index 0000000..fc50bba --- /dev/null +++ b/infra/rabbitmq/init-worker.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Runs as a one-shot sidecar against rabbit@broker. The broker data volume is +# mounted read-only only to share the Erlang cookie; no queue data is modified. +set -eu + +password=$(cat /run/secrets/broker_worker_password) +configure='^(attachments\.jobs|attachments\.process|attachments\.dlx|attachments\.process\.dlq)$' +write='^attachments\.(jobs|process|dlx|process\.dlq)$' +all_read='^attachments\.(jobs|process|dlx|process\.dlq)$' +api_read='^attachments\.(jobs|dlx)$' + +if ! rabbitmqctl -n rabbit@broker -p messenger add_user messenger-worker "$password" 2>/dev/null; then + rabbitmqctl -n rabbit@broker -q -p messenger change_password messenger-worker "$password" +fi +rabbitmqctl -n rabbit@broker -q -p messenger set_permissions messenger-worker \ + "$configure" "$write" "$all_read" +# The API may publish and declare the topology, but must not consume the job +# queue. The worker identity is used by the worker process. +rabbitmqctl -n rabbit@broker -q -p messenger set_permissions messenger \ + "$configure" "$write" "$api_read" +echo "broker init ok: publisher and consumer identities separated" diff --git a/scripts/init-env.sh b/scripts/init-env.sh index 8b9557d..4d340de 100755 --- a/scripts/init-env.sh +++ b/scripts/init-env.sh @@ -41,6 +41,17 @@ write_secret s3_api_secret_access_key "$(openssl rand -hex 24)" write_secret s3_worker_access_key_id "worker-$(openssl rand -hex 8)" write_secret s3_worker_secret_access_key "$(openssl rand -hex 24)" +if [ -s secrets/broker_worker_password ] && [ -s secrets/broker_worker_url ]; then + echo "kept secrets/broker_worker_password secrets/broker_worker_url" +else + wp=$(openssl rand -hex 24) + (umask 133; printf '%s' "$wp" > secrets/broker_worker_password) + chmod 0644 secrets/broker_worker_password + (umask 133; printf '%s' "amqp://messenger-worker:${wp}@broker:5672/messenger" > secrets/broker_worker_url) + chmod 0644 secrets/broker_worker_url + echo "created secrets/broker_worker_password secrets/broker_worker_url" +fi + # Broker password lives in one place and is rendered into both files, so the # URL used by the app and the RabbitMQ config cannot drift apart. if [ -s secrets/broker_url ] && [ -s secrets/rabbitmq.conf ]; then @@ -55,9 +66,9 @@ else printf 'default_vhost = messenger\n' printf 'default_user = messenger\n' printf 'default_pass = %s\n' "$bp" - printf 'default_permissions.configure = .*\n' - printf 'default_permissions.read = .*\n' - printf 'default_permissions.write = .*\n' + printf 'default_permissions.configure = ^(attachments\\.jobs|attachments\\.process|attachments\\.dlx|attachments\\.process\\.dlq)$\n' + printf 'default_permissions.read = ^attachments\\.(jobs|process|dlx|process\\.dlq)$\n' + printf 'default_permissions.write = ^attachments\\.(jobs|process|dlx|process\\.dlq)$\n' } > secrets/rabbitmq.conf ) chmod 0644 secrets/rabbitmq.conf