import { execFile } from 'node:child_process'; import { randomUUID } from 'node:crypto'; import { setTimeout as sleep } from 'node:timers/promises'; import { readFileSync } from 'node:fs'; import net from 'node:net'; import { fileURLToPath } from 'node:url'; import { promisify } from 'node:util'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import type { SessionResponse } from '../src/auth/auth.service.js'; import type { Chat } from '../src/chats/chat.js'; import type { ApiErrorBody } from '../src/common/errors/api-error.js'; import type { Page } from '../src/common/http/page.js'; import type { Message } from '../src/messages/message.js'; import type { User } from '../src/users/user.js'; import { noisePng } from './support/png.js'; // Automated counterpart of the manual Redis/PostgreSQL outage checks. Four // topologies, because the load balancer probes /health/ready (the same route as // the container healthcheck): // 1. Redis outage — shared dependency: every replica turns not-ready, HAProxy // removes all backends and answers 503 itself, while the replica's own // readiness stays a bounded API 503 with request_id. // 2. PostgreSQL outage — same shared-dependency topology, confirmed data and // the session survive the restart. // 3. Per-replica PostgreSQL fault — only the selected replica loses its own // path to the database (temporary pg_hba reject entry for its address plus // connection termination): that slot leaves rotation, the healthy replica // keeps serving, and the fault is always undone. // 4. Pool saturation — a busy shared pool is overload, not a lost dependency: // the replicas stay in rotation because the probe uses its own connection. // // It is opt-in because it needs control over the Docker Compose project: // make test-dependencies // cd app && E2E_DEPENDENCY_FAULTS=1 npm run test:dependencies const enabled = process.env.E2E_DEPENDENCY_FAULTS === '1'; const baseUrl = (process.env.API_BASE_URL ?? 'http://localhost:8080').replace( /\/+$/u, '', ); const composeDir = process.env.E2E_COMPOSE_DIR ?? fileURLToPath(new URL('../..', import.meta.url)); const composeEnvFile = process.env.E2E_COMPOSE_ENV_FILE ?? '.env'; const redisService = process.env.E2E_REDIS_SERVICE ?? 'redis'; const postgresService = process.env.E2E_POSTGRES_SERVICE ?? 'postgres'; const apiService = process.env.E2E_API_SERVICE ?? 'api'; const s3Service = process.env.E2E_S3_SERVICE ?? 's3'; const s3Bucket = process.env.E2E_S3_BUCKET ?? 'messenger-attachments'; const lbStatsUrl = process.env.E2E_LB_STATS_URL ?? 'http://127.0.0.1:8404/stats;csv'; // PostgreSQL superuser of the stand (compose.yaml, .env); psql runs over the // container's local socket, so no password is needed here. const postgresUser = process.env.E2E_POSTGRES_USER ?? 'messenger'; const postgresDatabase = process.env.E2E_POSTGRES_DB ?? 'messenger'; const pgHbaPath = process.env.E2E_PG_HBA_PATH ?? '/var/lib/postgresql/data/pg_hba.conf'; const pgHbaBackup = `${pgHbaPath}.lab3-partial-fault.bak`; const password = 'outage-check:пароль🔐'; const runId = randomUUID().replaceAll('-', '').slice(0, 12); // A request that reaches a stopped database waits for the pool connection // timeout, so every outage step costs up to DATABASE_CONNECTION_TIMEOUT_MS. const outagedRequestBudgetMs = 180_000; const slotBudgetMs = 20_000; const readyReads = 16; const saturatingWrites = 60; // больше, чем replicas × DATABASE_POOL_MAX (2 × 10) const execFileAsync = promisify(execFile); type Result = { body: T; text: string; response: Response }; type ReplicaProbe = { instanceId: string | null; requestId: string | null; status: number; text: string; }; type LbSlot = { addr: string; check: string; name: string; status: string }; async function compose(...args: string[]): Promise { const result = await execFileAsync( 'docker', ['compose', '--env-file', composeEnvFile, ...args], { cwd: composeDir, timeout: 120_000 }, ); return result.stdout.trim(); } async function docker(...args: string[]): Promise { const result = await execFileAsync('docker', args, { cwd: composeDir, timeout: 60_000, }); return result.stdout.trim(); } async function request( path: string, init: RequestInit = {}, ): Promise> { const response = await fetch(`${baseUrl}${path}`, init); const text = await response.text(); // With every backend out of rotation the answer is the LB's own HTML 503, // so a body is not guaranteed to be JSON (it always is for API answers). let body: T | undefined; try { body = text ? (JSON.parse(text) as T) : undefined; } catch { body = undefined; } return { body: body as T, text, response }; } function json( body: unknown, headers: Record = {}, ): RequestInit { return { method: 'POST', headers: { 'content-type': 'application/json', ...headers }, body: JSON.stringify(body), }; } function bearer(token: string): Record { return { authorization: `Bearer ${token}` }; } // POST messages needs a unique Idempotency-Key in lab3+; outage checks only // need the write path, not replay semantics. function messageHeaders(token: string): Record { return { ...bearer(token), 'idempotency-key': randomUUID().replaceAll('-', ''), }; } function basic(username: string, secret: string): string { return `Basic ${Buffer.from(`${username}:${secret}`, 'utf8').toString('base64')}`; } // The dependencies recover asynchronously, so readiness is polled instead of // asserted right after docker compose returns. async function waitForReadiness( expected: number, timeoutMs = 90_000, ): Promise { const deadline = Date.now() + timeoutMs; let last: number | string = 'no response'; for (;;) { try { const probe = await request('/health/ready'); last = probe.response.status; if (probe.response.status === expected) return; } catch (error) { last = error instanceof Error ? error.message : 'request failed'; } if (Date.now() > deadline) throw new Error( `readiness never became ${expected} within ${timeoutMs}ms (last: ${String(last)})`, ); await sleep(250); } } async function waitFor( condition: () => Promise, describe: () => Promise, timeoutMs: number, ): Promise { const deadline = Date.now() + timeoutMs; for (;;) { if (await condition()) return; if (Date.now() > deadline) throw new Error( `condition not reached within ${timeoutMs}ms: ${await describe()}`, ); await sleep(250); } } function expectBoundedError( status: number, text: string, requestId: string | null, ): void { expect(status).toBe(503); const body = JSON.parse(text) as ApiErrorBody; expect(body.error.code).toBe('unavailable'); expect(body.error.message).toBe('Service unavailable'); expect(body.error.request_id).toBe(requestId); // A bounded error must not leak driver details, SQL text or a stack trace. expect(text).not.toMatch( /ECONNREFUSED|ECONNRESET|Connection terminated|pg\.|node_modules|at .*\(.+:\d+:\d+\)/u, ); } function expectBoundedOutage(result: Result): void { expectBoundedError( result.response.status, result.text, result.response.headers.get('x-request-id'), ); } // With every backend out of rotation the last hop is the load balancer itself: // it answers with its own 503 page, so there is no API error body and no // X-Instance-Id — the request never reached a replica. function expectLbOutage(result: Result): void { expect(result.response.status).toBe(503); expect(result.response.headers.get('x-instance-id')).toBeNull(); } // What a client sees while a shared dependency is down: either the bounded 503 // of a replica that is still in rotation (identified by X-Instance-Id), or the // LB's own 503 once every replica has left rotation. Never a success. function expectUnavailableThroughLb(result: Result): void { expect(result.response.status).toBe(503); if (result.response.headers.get('x-instance-id') === null) return; expectBoundedOutage(result); } async function historyOf( chatId: string, token: string, ): Promise>> { return request>( `/api/v1/chats/${chatId}/messages?limit=100`, { headers: bearer(token), }, ); } async function apiContainers(): Promise { const stdout = await compose('ps', '-q', apiService); return stdout .split('\n') .map((line) => line.trim()) .filter((line) => line.length > 0) .sort(); } async function containerIp(container: string): Promise { return docker( 'inspect', '-f', '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}', container, ); } // The replicas publish no ports, so a single replica is probed from inside its // own network namespace (loopback). const probeScript = ` const params = JSON.parse(process.env.LAB3_PROBE); const port = process.env.APP_PORT || 8080; const response = await fetch('http://127.0.0.1:' + port + params.path, { method: params.method || 'GET', headers: params.headers || {}, body: params.body, }); const text = await response.text(); console.log(JSON.stringify({ status: response.status, text, instanceId: response.headers.get('x-instance-id'), requestId: response.headers.get('x-request-id'), })); `; type ProbeInit = { body?: string; headers?: Record; method?: string; }; async function probeReplica( container: string, path: string, init: ProbeInit = {}, ): Promise { const stdout = await docker( 'exec', '-e', `LAB3_PROBE=${JSON.stringify({ path, ...init })}`, container, 'node', '--input-type=module', '-e', probeScript, ); return JSON.parse(stdout) as ReplicaProbe; } async function waitForReplicaReadiness( container: string, expected: number, timeoutMs: number, ): Promise { let last = 'no probe'; await waitFor( async () => { try { const probe = await probeReplica(container, '/health/ready'); last = `${probe.status} ${probe.text}`; return probe.status === expected; } catch (error) { last = error instanceof Error ? error.message : 'probe failed'; return false; } }, async () => last, timeoutMs, ); } function parseCsvLine(line: string): string[] { const values: string[] = []; let current = ''; let quoted = false; for (const character of line) { if (character === '"') quoted = !quoted; else if (character === ',' && !quoted) { values.push(current); current = ''; } else current += character; } values.push(current.replace(/\r$/u, '')); return values; } // Resolved backend slots of the api backend: the unused slots of the // server-template have no address and are ignored. async function lbSlots(): Promise { const response = await fetch(lbStatsUrl); const lines = (await response.text()).trim().split('\n'); const header = parseCsvLine(lines[0].replace(/^#\s*/u, '')).map((name) => name.trim(), ); const column = (name: string): number => header.indexOf(name); return lines .slice(1) .map(parseCsvLine) .filter( (columns) => columns[column('pxname')] === 'api' && /^api\d+$/u.test(columns[column('svname')] ?? '') && (columns[column('addr')] ?? '').length > 0, ) .map((columns) => ({ addr: columns[column('addr')], check: columns[column('check_status')], name: columns[column('svname')], status: columns[column('status')], })); } async function waitForSlot( predicate: (slots: LbSlot[]) => boolean, describe: string, timeoutMs = slotBudgetMs, ): Promise { let last = 'no stats'; await waitFor( async () => { try { const slots = await lbSlots(); last = slots .map( (slot) => `${slot.name}@${slot.addr} ${slot.status}/${slot.check}`, ) .join(', '); return slots.length > 0 && predicate(slots); } catch (error) { last = error instanceof Error ? error.message : 'stats failed'; return false; } }, async () => `${describe} (last: ${last})`, timeoutMs, ); return lbSlots(); } function upSlots(slots: LbSlot[]): LbSlot[] { return slots.filter((slot) => slot.status === 'UP'); } async function psql(query: string): Promise { return dockerExecService( postgresService, 'psql', '-U', postgresUser, '-d', postgresDatabase, '-tA', '-c', query, ); } // docker exec needs a container, while the suite is configured with Compose // service names, so the container of the service is resolved first. async function dockerExecService( service: string, ...args: string[] ): Promise { const container = (await compose('ps', '-q', service)) .split('\n')[0] ?.trim(); if (!container) throw new Error(`no container for service ${service}`); return docker('exec', container, ...args); } // Takes away the replica's own path to PostgreSQL without touching the shared // database: a reject entry for the replica's address becomes the first matching // line of pg_hba.conf, so its new connections fail while the server stays up // for everyone else. The file is rewritten through the existing inode (`cat > // file`) to preserve the owner and mode the server needs to reload it. async function applyReplicaDatabaseReject(address: string): Promise { const script = [ `test -f ${pgHbaBackup} || cat ${pgHbaPath} > ${pgHbaBackup}`, `printf 'host all all ${address}/32 reject\\n' > /tmp/lab3-pg-hba`, `cat ${pgHbaPath} >> /tmp/lab3-pg-hba`, `cat /tmp/lab3-pg-hba > ${pgHbaPath}`, 'rm -f /tmp/lab3-pg-hba', ].join('; '); await dockerExecService(postgresService, 'sh', '-c', script); await psql('select pg_reload_conf()'); await psql( `select count(pg_terminate_backend(pid)) from pg_stat_activity where client_addr = '${address}'`, ); } // Idempotent restore: also runs from afterAll, so an assertion that failed // mid-fault cannot leave the stand rejecting one replica. async function restoreDatabaseAccess(): Promise { await dockerExecService( postgresService, 'sh', '-c', `if [ -f ${pgHbaBackup} ]; then cat ${pgHbaBackup} > ${pgHbaPath}; rm -f ${pgHbaBackup}; fi`, ); await psql('select pg_reload_conf()'); } const samplePng = readFileSync( new URL('../../tests/fixtures/sample.png', import.meta.url), ); function secretFile(name: string): string { return readFileSync(`${composeDir}/secrets/${name}`, 'utf8').trim(); } // Root credentials stay inside the local process and the container's exec // environment; they are never printed. async function objectExists(key: string): Promise { const host = `http://${secretFile('minio_root_user')}:${secretFile('minio_root_password')}@127.0.0.1:9000`; try { await dockerExecService( s3Service, 'env', `MC_HOST_stand=${host}`, 'mc', 'stat', `stand/${s3Bucket}/${key}`, ); return true; } catch { return false; } } async function waitForS3Healthy(): Promise { const container = (await compose('ps', '-q', s3Service)).split('\n')[0]; await waitFor( async () => (await docker( 'inspect', '-f', '{{.State.Health.Status}}', container, ).catch(() => '')) === 'healthy', async () => 's3 is not healthy', 60_000, ); } function uploadPath(chatId: string, filename: string): string { return `/api/v1/chats/${chatId}/attachments?filename=${encodeURIComponent(filename)}`; } async function sweepNow(): Promise { await compose( 'exec', '-T', '-e', 'ATTACHMENT_SWEEP_GRACE_SECONDS=0', apiService, 'node', 'dist/attachments/attachments-sweep.cli.js', ); } async function waitUntilExpired(filter: string): Promise { await waitFor( async () => Number( await psql( `select count(*) from attachments where ${filter} and expires_at < now()`, ), ) > 0, async () => 'pending row did not expire', 60_000, ); } async function rawUpload( data: Buffer, chatId: string, filename: string, token: string, splitAt: number, between: () => Promise, ): Promise { const url = new URL(baseUrl); const socket = net.connect(Number(url.port || 80), url.hostname); await new Promise((resolve, reject) => { socket.once('connect', resolve); socket.once('error', reject); }); const chunks: Buffer[] = []; socket.on('data', (chunk: Buffer) => chunks.push(chunk)); const closed = new Promise((resolve) => socket.on('close', resolve)); socket.write( `POST ${uploadPath(chatId, filename)} HTTP/1.1\r\nHost: ${url.host}\r\n` + `Authorization: Bearer ${token}\r\nContent-Type: image/png\r\n` + `Content-Length: ${data.length}\r\nConnection: close\r\n\r\n`, ); socket.write(data.subarray(0, splitAt)); await between(); socket.write(data.subarray(splitAt)); await closed; return Buffer.concat(chunks).toString('latin1'); } describe.skipIf(!enabled)('dependency outages', () => { let user: User; let token: string; let chat: Chat; let message: Message; let replicas: string[]; beforeAll(async () => { await compose('up', '-d'); await waitForReadiness(200); const username = `outage_${runId}`; const registered = await request( '/api/v1/users', json({ username, display_name: 'Отказ зависимости', password }), ); expect(registered.response.status).toBe(201); user = registered.body; const session = await request( '/api/v1/auth/sessions', { method: 'POST', headers: { authorization: basic(username, password) }, }, ); expect(session.response.status).toBe(201); token = session.body.token; const created = await request( '/api/v1/chats', json( { title: `outage-${runId}`, member_ids: [user.id] }, bearer(token), ), ); expect(created.response.status).toBe(201); chat = created.body; const sent = await request( `/api/v1/chats/${chat.id}/messages`, json({ text: 'Сообщение до отказа' }, messageHeaders(token)), ); expect(sent.response.status).toBe(201); message = sent.body; replicas = await apiContainers(); expect(replicas.length).toBeGreaterThanOrEqual(2); await waitForSlot( (slots) => upSlots(slots).length === replicas.length, 'not every replica is UP', 30_000, ); }, outagedRequestBudgetMs); afterAll(async () => { // Always restore the stand, even when an assertion failed mid-outage: // first the services (pg_hba is only reachable in a running container), // then the temporary per-replica rule, then rotation. await compose('start', redisService); await compose('start', postgresService); await compose('start', s3Service); const s3Container = (await compose('ps', '-q', s3Service)).split( '\n', )[0]; await docker('unpause', s3Container).catch(() => undefined); await waitForReadiness(200); await restoreDatabaseAccess(); const expected = (await apiContainers()).length; await waitForSlot( (slots) => upSlots(slots).length === expected, 'backends did not return to rotation', 30_000, ); }, outagedRequestBudgetMs); it( 'keeps every replica in rotation while the database pool is saturated', async () => { // A row lock on the chat parks every write in the shared pool; the // readiness probe uses its own connection and must keep answering. const lock = psql( `begin; select id from chats where id = '${chat.id}' for update; select pg_sleep(4); commit;`, ); await sleep(300); const statuses = new Set(); let sampling = true; const sampler = (async () => { while (sampling) { for (const slot of await lbSlots()) statuses.add(slot.status); await sleep(200); } })(); const writes = await Promise.all( Array.from({ length: saturatingWrites }, () => request( `/api/v1/chats/${chat.id}/messages`, json({ text: 'saturation' }, messageHeaders(token)), ), ), ); await lock; await sleep(1000); sampling = false; await sampler; expect([...statuses]).toEqual(['UP']); expect( writes .map((write) => write.response.status) .filter((status) => status !== 201), ).toEqual([]); }, outagedRequestBudgetMs, ); it( 'serves bounded 503 for the Redis outage and recovers the session', async () => { await compose('stop', redisService); const replica = replicas[0]; expect((await probeReplica(replica, '/health/live')).status).toBe( 200, ); await waitForReplicaReadiness(replica, 503, 30_000); const readiness = await probeReplica(replica, '/health/ready'); expectBoundedError( readiness.status, readiness.text, readiness.requestId, ); const me = await probeReplica(replica, '/api/v1/users/me', { headers: bearer(token), }); expectBoundedError(me.status, me.text, me.requestId); // A request without a session must stay unauthorized during an outage: // the outage may not turn an anonymous caller into a trusted one. const anonymous = await probeReplica(replica, '/api/v1/chats', { headers: { 'x-user-id': user.id }, }); expect(anonymous.status).toBe(401); expect( (JSON.parse(anonymous.text) as ApiErrorBody).error.code, ).toBe('unauthorized'); expectUnavailableThroughLb( await request('/api/v1/users/me', { headers: bearer(token), }), ); await waitForSlot( // 'UP 1/2' is still in rotation: wait for DOWN itself. (slots) => slots.every((slot) => slot.status === 'DOWN'), 'not every slot left rotation', ); expectLbOutage( await request('/api/v1/users/me', { headers: bearer(token) }), ); await compose('start', redisService); await waitForReadiness(200); await waitForSlot( (slots) => upSlots(slots).length === replicas.length, 'backends did not return to rotation', 30_000, ); // Sessions live in Redis with persistence, so the old token still works. const recovered = await request('/api/v1/users/me', { headers: bearer(token), }); expect(recovered.response.status).toBe(200); expect(recovered.body.id).toBe(user.id); const history = await historyOf(chat.id, token); expect(history.response.status).toBe(200); expect(history.body.items.map((item) => item.id)).toContain( message.id, ); }, outagedRequestBudgetMs, ); it( 'serves bounded 503 for the PostgreSQL outage and keeps confirmed data', async () => { await compose('stop', postgresService); const replica = replicas[0]; expect((await probeReplica(replica, '/health/live')).status).toBe( 200, ); await waitForReplicaReadiness(replica, 503, 30_000); const readiness = await probeReplica(replica, '/health/ready'); expectBoundedError( readiness.status, readiness.text, readiness.requestId, ); // Redis still answers, so these requests reach the database layer and // must fail with the bounded 503, not an unhandled 500. const me = await probeReplica(replica, '/api/v1/users/me', { headers: bearer(token), }); expectBoundedError(me.status, me.text, me.requestId); const chats = await probeReplica(replica, '/api/v1/chats', { headers: bearer(token), }); expectBoundedError(chats.status, chats.text, chats.requestId); const history = await probeReplica( replica, `/api/v1/chats/${chat.id}/messages?limit=100`, { headers: bearer(token) }, ); expectBoundedError(history.status, history.text, history.requestId); const registration = await probeReplica(replica, '/api/v1/users', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ username: `outage_${runId}_late`, display_name: 'Отказ зависимости', password, }), }); expectBoundedError( registration.status, registration.text, registration.requestId, ); expectUnavailableThroughLb( await request('/api/v1/users/me', { headers: bearer(token), }), ); await waitForSlot( // 'UP 1/2' is still in rotation: wait for DOWN itself. (slots) => slots.every((slot) => slot.status === 'DOWN'), 'not every slot left rotation', ); expectLbOutage( await request('/api/v1/users/me', { headers: bearer(token) }), ); await compose('start', postgresService); await waitForReadiness(200); await waitForSlot( (slots) => upSlots(slots).length === replicas.length, 'backends did not return to rotation', 30_000, ); const recovered = await request('/api/v1/users/me', { headers: bearer(token), }); expect(recovered.response.status).toBe(200); expect(recovered.body.id).toBe(user.id); const after = await historyOf(chat.id, token); expect(after.response.status).toBe(200); expect(after.body.items.map((item) => item.id)).toContain( message.id, ); const sent = await request( `/api/v1/chats/${chat.id}/messages`, json({ text: 'Сообщение после отказа' }, messageHeaders(token)), ); expect(sent.response.status).toBe(201); }, outagedRequestBudgetMs, ); it( 'excludes only the replica that lost its own database path', async () => { const selected = replicas[0]; const healthy = replicas[1]; const selectedIp = await containerIp(selected); const selectedAddress = `${selectedIp}:8080`; const selectedInstance = ( await probeReplica(selected, '/health/live') ).instanceId; expect(selectedInstance).not.toBeNull(); const healthyInstance = ( await probeReplica(healthy, '/health/live') ).instanceId; try { await applyReplicaDatabaseReject(selectedIp); const faultAt = Date.now(); await waitForReplicaReadiness(selected, 503, 30_000); expect( (await probeReplica(healthy, '/health/ready')).status, ).toBe(200); // Only the affected slot leaves rotation. While `fall 2` counts the // failed checks the slot passes through "UP 1/2 (L7STS)", so the // wait is for DOWN itself, not for "no longer plain UP". const slots = await waitForSlot( (candidates) => upSlots(candidates).length === replicas.length - 1 && candidates.some( (slot) => slot.addr === selectedAddress && slot.status === 'DOWN', ), `selected slot ${selectedAddress} did not go down`, 30_000, ); // inter 500 ms × fall 2 + timeout check 500 ms + polling step 250 ms. expect(Date.now() - faultAt).toBeLessThan(3_000); expect( slots.find((slot) => slot.addr === selectedAddress)?.status, ).toBe('DOWN'); const served = new Set(); for (let index = 0; index < readyReads; index += 1) { const read = await historyOf(chat.id, token); expect(read.response.status).toBe(200); served.add( read.response.headers.get('x-instance-id') ?? 'unknown', ); } expect(served.has(selectedInstance as string)).toBe(false); expect(served.has(healthyInstance as string)).toBe(true); } finally { await restoreDatabaseAccess(); } await waitForReplicaReadiness(selected, 200, 30_000); await waitForSlot( (slots) => upSlots(slots).length === replicas.length && slots.some( (slot) => slot.addr === selectedAddress && slot.status === 'UP', ), 'restored replica did not return to rotation', 30_000, ); let servedAgain = false; for ( let index = 0; index < readyReads * 2 && !servedAgain; index += 1 ) { const read = await historyOf(chat.id, token); expect(read.response.status).toBe(200); servedAgain = read.response.headers.get('x-instance-id') === selectedInstance; } expect(servedAgain).toBe(true); }, outagedRequestBudgetMs, ); it( 'degrades attachments while S3 is down and sweeps the failed upload afterwards', async () => { const upload = (data: Buffer, name: string) => request( uploadPath(chat.id, name), { method: 'POST', headers: { ...bearer(token), 'content-type': 'image/png', }, body: data as unknown as BodyInit, }, ); const before = await upload(samplePng, `s3before-${runId}.png`); expect(before.response.status).toBe(201); await compose('stop', s3Service); try { const failed = await upload(samplePng, `s3down-${runId}.png`); expectBoundedOutage(failed); const metadata = await request( `/api/v1/attachments/${before.body.id}`, { headers: bearer(token) }, ); expect(metadata.response.status).toBe(200); const content = await request( `/api/v1/attachments/${before.body.id}/content`, { headers: bearer(token) }, ); expectBoundedOutage(content); const text = await request( `/api/v1/chats/${chat.id}/messages`, json({ text: 'S3 down' }, messageHeaders(token)), ); expect(text.response.status).toBe(201); expect((await request('/health/ready')).response.status).toBe( 200, ); // Compensation claimed the row but could not delete the object: // the row stays in `deleting`, tracked for the sweeper. expect( await psql( `select status from attachments where chat_id = '${chat.id}' and filename = 's3down-${runId}.png'`, ), ).toBe('deleting'); } finally { await compose('start', s3Service); } await waitForS3Healthy(); const bytes = await fetch( `${baseUrl}/api/v1/attachments/${before.body.id}/content`, { headers: bearer(token) }, ); expect(bytes.status).toBe(200); expect( Buffer.from(await bytes.arrayBuffer()).equals(samplePng), ).toBe(true); const after = await upload(samplePng, `s3after-${runId}.png`); expect(after.response.status).toBe(201); // A `deleting` row is re-claimed one minute after its last change // (an in-flight compensation is never raced), then swept. const filter = `chat_id = '${chat.id}' and filename = 's3down-${runId}.png'`; await waitFor( async () => { await sweepNow(); return ( (await psql( `select count(*) from attachments where ${filter}`, )) === '0' ); }, async () => 'deleting row was not swept', 90_000, ); }, outagedRequestBudgetMs, ); it( 'bounds requests by S3_REQUEST_TIMEOUT_MS when S3 hangs instead of refusing', async () => { const created = await request<{ id: string }>( uploadPath(chat.id, `hang-${runId}.png`), { method: 'POST', headers: { ...bearer(token), 'content-type': 'image/png' }, body: samplePng as unknown as BodyInit, }, ); expect(created.response.status).toBe(201); // A paused container keeps its TCP port open but never answers: the // failure mode a stopped container (connection refused) does not show. const container = (await compose('ps', '-q', s3Service)).split( '\n', )[0]; await docker('pause', container); try { const started = Date.now(); const content = await request( `/api/v1/attachments/${created.body.id}/content`, { headers: bearer(token) }, ); const elapsed = Date.now() - started; expectBoundedOutage(content); // S3_REQUEST_TIMEOUT_MS is 15 s in the stand; HAProxy gives up at 30 s. expect(elapsed).toBeLessThan(20_000); expect( ( await request( `/api/v1/attachments/${created.body.id}`, { headers: bearer(token), }, ) ).response.status, ).toBe(200); expect((await request('/health/ready')).response.status).toBe( 200, ); } finally { await docker('unpause', container); } const bytes = await fetch( `${baseUrl}/api/v1/attachments/${created.body.id}/content`, { headers: bearer(token) }, ); expect(bytes.status).toBe(200); expect( Buffer.from(await bytes.arrayBuffer()).equals(samplePng), ).toBe(true); }, outagedRequestBudgetMs, ); it( 'keeps the object while PostgreSQL is down before ready and sweeps it afterwards', async () => { const data = noisePng(1024 * 1024); const filename = `pgfault-${runId}.png`; const filter = `chat_id = '${chat.id}' and status = 'pending' and filename = '${filename}'`; let restarted = false; let reply = ''; try { reply = await rawUpload( data, chat.id, filename, token, 64 * 1024, async () => { await waitFor( async () => Number( await psql( `select count(*) from attachments where ${filter}`, ), ) === 1, async () => 'pending row did not appear', 10_000, ); await compose('stop', postgresService); }, ); } finally { await compose('start', postgresService); restarted = true; } expect(restarted).toBe(true); expect(reply.startsWith('HTTP/1.1 503')).toBe(true); await waitForReadiness(200); const key = await psql( `select object_key from attachments where ${filter}`, ); expect(key).toMatch(/^attachments\/[0-9a-f-]{36}\/original$/u); // Without PostgreSQL the API cannot tell a lost commit from a failed // one, so it deletes nothing; the pending row keeps the object tracked. expect(await objectExists(key)).toBe(true); await waitUntilExpired(filter); await sweepNow(); expect( await psql(`select count(*) from attachments where ${filter}`), ).toBe('0'); expect(await objectExists(key)).toBe(false); }, outagedRequestBudgetMs, ); });