forked from mtusi2026/mtusi2026lab3
800 lines
24 KiB
TypeScript
800 lines
24 KiB
TypeScript
import { execFile } from 'node:child_process';
|
||
import { randomUUID } from 'node:crypto';
|
||
import { setTimeout as sleep } from 'node:timers/promises';
|
||
import { fileURLToPath } from 'node:url';
|
||
import { promisify } from 'node:util';
|
||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||
import type { SessionResponse } from '../src/auth/auth.service.js';
|
||
import type { Chat } from '../src/chats/chat.js';
|
||
import type { ApiErrorBody } from '../src/common/errors/api-error.js';
|
||
import type { Page } from '../src/common/http/page.js';
|
||
import type { Message } from '../src/messages/message.js';
|
||
import type { User } from '../src/users/user.js';
|
||
|
||
// Automated counterpart of the manual Redis/PostgreSQL outage checks. Four
|
||
// topologies, because the load balancer probes /health/ready (the same route as
|
||
// the container healthcheck):
|
||
// 1. Redis outage — shared dependency: every replica turns not-ready, HAProxy
|
||
// removes all backends and answers 503 itself, while the replica's own
|
||
// readiness stays a bounded API 503 with request_id.
|
||
// 2. PostgreSQL outage — same shared-dependency topology, confirmed data and
|
||
// the session survive the restart.
|
||
// 3. Per-replica PostgreSQL fault — only the selected replica loses its own
|
||
// path to the database (temporary pg_hba reject entry for its address plus
|
||
// connection termination): that slot leaves rotation, the healthy replica
|
||
// keeps serving, and the fault is always undone.
|
||
// 4. Pool saturation — a busy shared pool is overload, not a lost dependency:
|
||
// the replicas stay in rotation because the probe uses its own connection.
|
||
//
|
||
// It is opt-in because it needs control over the Docker Compose project:
|
||
// make test-dependencies
|
||
// cd app && E2E_DEPENDENCY_FAULTS=1 npm run test:dependencies
|
||
const enabled = process.env.E2E_DEPENDENCY_FAULTS === '1';
|
||
const baseUrl = (process.env.API_BASE_URL ?? 'http://localhost:8080').replace(
|
||
/\/+$/u,
|
||
'',
|
||
);
|
||
const composeDir =
|
||
process.env.E2E_COMPOSE_DIR ??
|
||
fileURLToPath(new URL('../..', import.meta.url));
|
||
const composeEnvFile = process.env.E2E_COMPOSE_ENV_FILE ?? '.env';
|
||
const redisService = process.env.E2E_REDIS_SERVICE ?? 'redis';
|
||
const postgresService = process.env.E2E_POSTGRES_SERVICE ?? 'postgres';
|
||
const apiService = process.env.E2E_API_SERVICE ?? 'api';
|
||
const lbStatsUrl =
|
||
process.env.E2E_LB_STATS_URL ?? 'http://127.0.0.1:8404/stats;csv';
|
||
// PostgreSQL superuser of the stand (compose.yaml, .env); psql runs over the
|
||
// container's local socket, so no password is needed here.
|
||
const postgresUser = process.env.E2E_POSTGRES_USER ?? 'messenger';
|
||
const postgresDatabase = process.env.E2E_POSTGRES_DB ?? 'messenger';
|
||
const pgHbaPath =
|
||
process.env.E2E_PG_HBA_PATH ?? '/var/lib/postgresql/data/pg_hba.conf';
|
||
const pgHbaBackup = `${pgHbaPath}.lab3-partial-fault.bak`;
|
||
const password = 'outage-check:пароль🔐';
|
||
const runId = randomUUID().replaceAll('-', '').slice(0, 12);
|
||
// A request that reaches a stopped database waits for the pool connection
|
||
// timeout, so every outage step costs up to DATABASE_CONNECTION_TIMEOUT_MS.
|
||
const outagedRequestBudgetMs = 180_000;
|
||
const slotBudgetMs = 20_000;
|
||
const readyReads = 16;
|
||
const saturatingWrites = 60; // больше, чем replicas × DATABASE_POOL_MAX (2 × 10)
|
||
|
||
const execFileAsync = promisify(execFile);
|
||
|
||
type Result<T> = { body: T; text: string; response: Response };
|
||
type ReplicaProbe = {
|
||
instanceId: string | null;
|
||
requestId: string | null;
|
||
status: number;
|
||
text: string;
|
||
};
|
||
type LbSlot = { addr: string; check: string; name: string; status: string };
|
||
|
||
async function compose(...args: string[]): Promise<string> {
|
||
const result = await execFileAsync(
|
||
'docker',
|
||
['compose', '--env-file', composeEnvFile, ...args],
|
||
{ cwd: composeDir, timeout: 120_000 },
|
||
);
|
||
return result.stdout.trim();
|
||
}
|
||
|
||
async function docker(...args: string[]): Promise<string> {
|
||
const result = await execFileAsync('docker', args, {
|
||
cwd: composeDir,
|
||
timeout: 60_000,
|
||
});
|
||
return result.stdout.trim();
|
||
}
|
||
|
||
async function request<T = unknown>(
|
||
path: string,
|
||
init: RequestInit = {},
|
||
): Promise<Result<T>> {
|
||
const response = await fetch(`${baseUrl}${path}`, init);
|
||
const text = await response.text();
|
||
// With every backend out of rotation the answer is the LB's own HTML 503,
|
||
// so a body is not guaranteed to be JSON (it always is for API answers).
|
||
let body: T | undefined;
|
||
try {
|
||
body = text ? (JSON.parse(text) as T) : undefined;
|
||
} catch {
|
||
body = undefined;
|
||
}
|
||
return { body: body as T, text, response };
|
||
}
|
||
|
||
function json(
|
||
body: unknown,
|
||
headers: Record<string, string> = {},
|
||
): RequestInit {
|
||
return {
|
||
method: 'POST',
|
||
headers: { 'content-type': 'application/json', ...headers },
|
||
body: JSON.stringify(body),
|
||
};
|
||
}
|
||
|
||
function bearer(token: string): Record<string, string> {
|
||
return { authorization: `Bearer ${token}` };
|
||
}
|
||
|
||
// POST messages needs a unique Idempotency-Key in lab3+; outage checks only
|
||
// need the write path, not replay semantics.
|
||
function messageHeaders(token: string): Record<string, string> {
|
||
return {
|
||
...bearer(token),
|
||
'idempotency-key': randomUUID().replaceAll('-', ''),
|
||
};
|
||
}
|
||
|
||
function basic(username: string, secret: string): string {
|
||
return `Basic ${Buffer.from(`${username}:${secret}`, 'utf8').toString('base64')}`;
|
||
}
|
||
|
||
// The dependencies recover asynchronously, so readiness is polled instead of
|
||
// asserted right after docker compose returns.
|
||
async function waitForReadiness(
|
||
expected: number,
|
||
timeoutMs = 90_000,
|
||
): Promise<void> {
|
||
const deadline = Date.now() + timeoutMs;
|
||
let last: number | string = 'no response';
|
||
for (;;) {
|
||
try {
|
||
const probe = await request('/health/ready');
|
||
last = probe.response.status;
|
||
if (probe.response.status === expected) return;
|
||
} catch (error) {
|
||
last = error instanceof Error ? error.message : 'request failed';
|
||
}
|
||
if (Date.now() > deadline)
|
||
throw new Error(
|
||
`readiness never became ${expected} within ${timeoutMs}ms (last: ${String(last)})`,
|
||
);
|
||
await sleep(250);
|
||
}
|
||
}
|
||
|
||
async function waitFor(
|
||
condition: () => Promise<boolean>,
|
||
describe: () => Promise<string>,
|
||
timeoutMs: number,
|
||
): Promise<void> {
|
||
const deadline = Date.now() + timeoutMs;
|
||
for (;;) {
|
||
if (await condition()) return;
|
||
if (Date.now() > deadline)
|
||
throw new Error(
|
||
`condition not reached within ${timeoutMs}ms: ${await describe()}`,
|
||
);
|
||
await sleep(250);
|
||
}
|
||
}
|
||
|
||
function expectBoundedError(
|
||
status: number,
|
||
text: string,
|
||
requestId: string | null,
|
||
): void {
|
||
expect(status).toBe(503);
|
||
const body = JSON.parse(text) as ApiErrorBody;
|
||
expect(body.error.code).toBe('unavailable');
|
||
expect(body.error.message).toBe('Service unavailable');
|
||
expect(body.error.request_id).toBe(requestId);
|
||
// A bounded error must not leak driver details, SQL text or a stack trace.
|
||
expect(text).not.toMatch(
|
||
/ECONNREFUSED|ECONNRESET|Connection terminated|pg\.|node_modules|at .*\(.+:\d+:\d+\)/u,
|
||
);
|
||
}
|
||
|
||
function expectBoundedOutage(result: Result<ApiErrorBody>): void {
|
||
expectBoundedError(
|
||
result.response.status,
|
||
result.text,
|
||
result.response.headers.get('x-request-id'),
|
||
);
|
||
}
|
||
|
||
// With every backend out of rotation the last hop is the load balancer itself:
|
||
// it answers with its own 503 page, so there is no API error body and no
|
||
// X-Instance-Id — the request never reached a replica.
|
||
function expectLbOutage(result: Result<unknown>): void {
|
||
expect(result.response.status).toBe(503);
|
||
expect(result.response.headers.get('x-instance-id')).toBeNull();
|
||
}
|
||
|
||
// What a client sees while a shared dependency is down: either the bounded 503
|
||
// of a replica that is still in rotation (identified by X-Instance-Id), or the
|
||
// LB's own 503 once every replica has left rotation. Never a success.
|
||
function expectUnavailableThroughLb(result: Result<ApiErrorBody>): void {
|
||
expect(result.response.status).toBe(503);
|
||
if (result.response.headers.get('x-instance-id') === null) return;
|
||
expectBoundedOutage(result);
|
||
}
|
||
|
||
async function historyOf(
|
||
chatId: string,
|
||
token: string,
|
||
): Promise<Result<Page<Message>>> {
|
||
return request<Page<Message>>(
|
||
`/api/v1/chats/${chatId}/messages?limit=100`,
|
||
{
|
||
headers: bearer(token),
|
||
},
|
||
);
|
||
}
|
||
|
||
async function apiContainers(): Promise<string[]> {
|
||
const stdout = await compose('ps', '-q', apiService);
|
||
return stdout
|
||
.split('\n')
|
||
.map((line) => line.trim())
|
||
.filter((line) => line.length > 0)
|
||
.sort();
|
||
}
|
||
|
||
async function containerIp(container: string): Promise<string> {
|
||
return docker(
|
||
'inspect',
|
||
'-f',
|
||
'{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}',
|
||
container,
|
||
);
|
||
}
|
||
|
||
// The replicas publish no ports, so a single replica is probed from inside its
|
||
// own network namespace (loopback).
|
||
const probeScript = `
|
||
const params = JSON.parse(process.env.LAB3_PROBE);
|
||
const port = process.env.APP_PORT || 8080;
|
||
const response = await fetch('http://127.0.0.1:' + port + params.path, {
|
||
method: params.method || 'GET',
|
||
headers: params.headers || {},
|
||
body: params.body,
|
||
});
|
||
const text = await response.text();
|
||
console.log(JSON.stringify({
|
||
status: response.status,
|
||
text,
|
||
instanceId: response.headers.get('x-instance-id'),
|
||
requestId: response.headers.get('x-request-id'),
|
||
}));
|
||
`;
|
||
|
||
type ProbeInit = {
|
||
body?: string;
|
||
headers?: Record<string, string>;
|
||
method?: string;
|
||
};
|
||
|
||
async function probeReplica(
|
||
container: string,
|
||
path: string,
|
||
init: ProbeInit = {},
|
||
): Promise<ReplicaProbe> {
|
||
const stdout = await docker(
|
||
'exec',
|
||
'-e',
|
||
`LAB3_PROBE=${JSON.stringify({ path, ...init })}`,
|
||
container,
|
||
'node',
|
||
'--input-type=module',
|
||
'-e',
|
||
probeScript,
|
||
);
|
||
return JSON.parse(stdout) as ReplicaProbe;
|
||
}
|
||
|
||
async function waitForReplicaReadiness(
|
||
container: string,
|
||
expected: number,
|
||
timeoutMs: number,
|
||
): Promise<void> {
|
||
let last = 'no probe';
|
||
await waitFor(
|
||
async () => {
|
||
try {
|
||
const probe = await probeReplica(container, '/health/ready');
|
||
last = `${probe.status} ${probe.text}`;
|
||
return probe.status === expected;
|
||
} catch (error) {
|
||
last = error instanceof Error ? error.message : 'probe failed';
|
||
return false;
|
||
}
|
||
},
|
||
async () => last,
|
||
timeoutMs,
|
||
);
|
||
}
|
||
|
||
function parseCsvLine(line: string): string[] {
|
||
const values: string[] = [];
|
||
let current = '';
|
||
let quoted = false;
|
||
for (const character of line) {
|
||
if (character === '"') quoted = !quoted;
|
||
else if (character === ',' && !quoted) {
|
||
values.push(current);
|
||
current = '';
|
||
} else current += character;
|
||
}
|
||
values.push(current.replace(/\r$/u, ''));
|
||
return values;
|
||
}
|
||
|
||
// Resolved backend slots of the api backend: the unused slots of the
|
||
// server-template have no address and are ignored.
|
||
async function lbSlots(): Promise<LbSlot[]> {
|
||
const response = await fetch(lbStatsUrl);
|
||
const lines = (await response.text()).trim().split('\n');
|
||
const header = parseCsvLine(lines[0].replace(/^#\s*/u, '')).map((name) =>
|
||
name.trim(),
|
||
);
|
||
const column = (name: string): number => header.indexOf(name);
|
||
return lines
|
||
.slice(1)
|
||
.map(parseCsvLine)
|
||
.filter(
|
||
(columns) =>
|
||
columns[column('pxname')] === 'api' &&
|
||
/^api\d+$/u.test(columns[column('svname')] ?? '') &&
|
||
(columns[column('addr')] ?? '').length > 0,
|
||
)
|
||
.map((columns) => ({
|
||
addr: columns[column('addr')],
|
||
check: columns[column('check_status')],
|
||
name: columns[column('svname')],
|
||
status: columns[column('status')],
|
||
}));
|
||
}
|
||
|
||
async function waitForSlot(
|
||
predicate: (slots: LbSlot[]) => boolean,
|
||
describe: string,
|
||
timeoutMs = slotBudgetMs,
|
||
): Promise<LbSlot[]> {
|
||
let last = 'no stats';
|
||
await waitFor(
|
||
async () => {
|
||
try {
|
||
const slots = await lbSlots();
|
||
last = slots
|
||
.map(
|
||
(slot) =>
|
||
`${slot.name}@${slot.addr} ${slot.status}/${slot.check}`,
|
||
)
|
||
.join(', ');
|
||
return slots.length > 0 && predicate(slots);
|
||
} catch (error) {
|
||
last = error instanceof Error ? error.message : 'stats failed';
|
||
return false;
|
||
}
|
||
},
|
||
async () => `${describe} (last: ${last})`,
|
||
timeoutMs,
|
||
);
|
||
return lbSlots();
|
||
}
|
||
|
||
function upSlots(slots: LbSlot[]): LbSlot[] {
|
||
return slots.filter((slot) => slot.status === 'UP');
|
||
}
|
||
|
||
async function psql(query: string): Promise<string> {
|
||
return dockerExecService(
|
||
postgresService,
|
||
'psql',
|
||
'-U',
|
||
postgresUser,
|
||
'-d',
|
||
postgresDatabase,
|
||
'-tA',
|
||
'-c',
|
||
query,
|
||
);
|
||
}
|
||
|
||
// docker exec needs a container, while the suite is configured with Compose
|
||
// service names, so the container of the service is resolved first.
|
||
async function dockerExecService(
|
||
service: string,
|
||
...args: string[]
|
||
): Promise<string> {
|
||
const container = (await compose('ps', '-q', service))
|
||
.split('\n')[0]
|
||
?.trim();
|
||
if (!container) throw new Error(`no container for service ${service}`);
|
||
return docker('exec', container, ...args);
|
||
}
|
||
|
||
// Takes away the replica's own path to PostgreSQL without touching the shared
|
||
// database: a reject entry for the replica's address becomes the first matching
|
||
// line of pg_hba.conf, so its new connections fail while the server stays up
|
||
// for everyone else. The file is rewritten through the existing inode (`cat >
|
||
// file`) to preserve the owner and mode the server needs to reload it.
|
||
async function applyReplicaDatabaseReject(address: string): Promise<void> {
|
||
const script = [
|
||
`test -f ${pgHbaBackup} || cat ${pgHbaPath} > ${pgHbaBackup}`,
|
||
`printf 'host all all ${address}/32 reject\\n' > /tmp/lab3-pg-hba`,
|
||
`cat ${pgHbaPath} >> /tmp/lab3-pg-hba`,
|
||
`cat /tmp/lab3-pg-hba > ${pgHbaPath}`,
|
||
'rm -f /tmp/lab3-pg-hba',
|
||
].join('; ');
|
||
await dockerExecService(postgresService, 'sh', '-c', script);
|
||
await psql('select pg_reload_conf()');
|
||
await psql(
|
||
`select count(pg_terminate_backend(pid)) from pg_stat_activity where client_addr = '${address}'`,
|
||
);
|
||
}
|
||
|
||
// Idempotent restore: also runs from afterAll, so an assertion that failed
|
||
// mid-fault cannot leave the stand rejecting one replica.
|
||
async function restoreDatabaseAccess(): Promise<void> {
|
||
await dockerExecService(
|
||
postgresService,
|
||
'sh',
|
||
'-c',
|
||
`if [ -f ${pgHbaBackup} ]; then cat ${pgHbaBackup} > ${pgHbaPath}; rm -f ${pgHbaBackup}; fi`,
|
||
);
|
||
await psql('select pg_reload_conf()');
|
||
}
|
||
|
||
describe.skipIf(!enabled)('dependency outages', () => {
|
||
let user: User;
|
||
let token: string;
|
||
let chat: Chat;
|
||
let message: Message;
|
||
let replicas: string[];
|
||
|
||
beforeAll(async () => {
|
||
await compose('up', '-d');
|
||
await waitForReadiness(200);
|
||
const username = `outage_${runId}`;
|
||
const registered = await request<User>(
|
||
'/api/v1/users',
|
||
json({ username, display_name: 'Отказ зависимости', password }),
|
||
);
|
||
expect(registered.response.status).toBe(201);
|
||
user = registered.body;
|
||
const session = await request<SessionResponse>(
|
||
'/api/v1/auth/sessions',
|
||
{
|
||
method: 'POST',
|
||
headers: { authorization: basic(username, password) },
|
||
},
|
||
);
|
||
expect(session.response.status).toBe(201);
|
||
token = session.body.token;
|
||
const created = await request<Chat>(
|
||
'/api/v1/chats',
|
||
json(
|
||
{ title: `outage-${runId}`, member_ids: [user.id] },
|
||
bearer(token),
|
||
),
|
||
);
|
||
expect(created.response.status).toBe(201);
|
||
chat = created.body;
|
||
const sent = await request<Message>(
|
||
`/api/v1/chats/${chat.id}/messages`,
|
||
json({ text: 'Сообщение до отказа' }, messageHeaders(token)),
|
||
);
|
||
expect(sent.response.status).toBe(201);
|
||
message = sent.body;
|
||
replicas = await apiContainers();
|
||
expect(replicas.length).toBeGreaterThanOrEqual(2);
|
||
await waitForSlot(
|
||
(slots) => upSlots(slots).length === replicas.length,
|
||
'not every replica is UP',
|
||
30_000,
|
||
);
|
||
}, outagedRequestBudgetMs);
|
||
|
||
afterAll(async () => {
|
||
// Always restore the stand, even when an assertion failed mid-outage:
|
||
// first the services (pg_hba is only reachable in a running container),
|
||
// then the temporary per-replica rule, then rotation.
|
||
await compose('start', redisService);
|
||
await compose('start', postgresService);
|
||
await waitForReadiness(200);
|
||
await restoreDatabaseAccess();
|
||
const expected = (await apiContainers()).length;
|
||
await waitForSlot(
|
||
(slots) => upSlots(slots).length === expected,
|
||
'backends did not return to rotation',
|
||
30_000,
|
||
);
|
||
}, outagedRequestBudgetMs);
|
||
|
||
it(
|
||
'keeps every replica in rotation while the database pool is saturated',
|
||
async () => {
|
||
// A row lock on the chat parks every write in the shared pool; the
|
||
// readiness probe uses its own connection and must keep answering.
|
||
const lock = psql(
|
||
`begin; select id from chats where id = '${chat.id}' for update; select pg_sleep(4); commit;`,
|
||
);
|
||
await sleep(300);
|
||
const statuses = new Set<string>();
|
||
let sampling = true;
|
||
const sampler = (async () => {
|
||
while (sampling) {
|
||
for (const slot of await lbSlots()) statuses.add(slot.status);
|
||
await sleep(200);
|
||
}
|
||
})();
|
||
const writes = await Promise.all(
|
||
Array.from({ length: saturatingWrites }, () =>
|
||
request(
|
||
`/api/v1/chats/${chat.id}/messages`,
|
||
json({ text: 'saturation' }, messageHeaders(token)),
|
||
),
|
||
),
|
||
);
|
||
await lock;
|
||
await sleep(1000);
|
||
sampling = false;
|
||
await sampler;
|
||
expect([...statuses]).toEqual(['UP']);
|
||
expect(
|
||
writes
|
||
.map((write) => write.response.status)
|
||
.filter((status) => status !== 201),
|
||
).toEqual([]);
|
||
},
|
||
outagedRequestBudgetMs,
|
||
);
|
||
|
||
it(
|
||
'serves bounded 503 for the Redis outage and recovers the session',
|
||
async () => {
|
||
await compose('stop', redisService);
|
||
const replica = replicas[0];
|
||
|
||
expect((await probeReplica(replica, '/health/live')).status).toBe(
|
||
200,
|
||
);
|
||
await waitForReplicaReadiness(replica, 503, 30_000);
|
||
const readiness = await probeReplica(replica, '/health/ready');
|
||
expectBoundedError(
|
||
readiness.status,
|
||
readiness.text,
|
||
readiness.requestId,
|
||
);
|
||
|
||
const me = await probeReplica(replica, '/api/v1/users/me', {
|
||
headers: bearer(token),
|
||
});
|
||
expectBoundedError(me.status, me.text, me.requestId);
|
||
|
||
// A request without a session must stay unauthorized during an outage:
|
||
// the outage may not turn an anonymous caller into a trusted one.
|
||
const anonymous = await probeReplica(replica, '/api/v1/chats', {
|
||
headers: { 'x-user-id': user.id },
|
||
});
|
||
expect(anonymous.status).toBe(401);
|
||
expect(
|
||
(JSON.parse(anonymous.text) as ApiErrorBody).error.code,
|
||
).toBe('unauthorized');
|
||
|
||
expectUnavailableThroughLb(
|
||
await request<ApiErrorBody>('/api/v1/users/me', {
|
||
headers: bearer(token),
|
||
}),
|
||
);
|
||
await waitForSlot(
|
||
// 'UP 1/2' is still in rotation: wait for DOWN itself.
|
||
(slots) => slots.every((slot) => slot.status === 'DOWN'),
|
||
'not every slot left rotation',
|
||
);
|
||
expectLbOutage(
|
||
await request('/api/v1/users/me', { headers: bearer(token) }),
|
||
);
|
||
|
||
await compose('start', redisService);
|
||
await waitForReadiness(200);
|
||
await waitForSlot(
|
||
(slots) => upSlots(slots).length === replicas.length,
|
||
'backends did not return to rotation',
|
||
30_000,
|
||
);
|
||
|
||
// Sessions live in Redis with persistence, so the old token still works.
|
||
const recovered = await request<User>('/api/v1/users/me', {
|
||
headers: bearer(token),
|
||
});
|
||
expect(recovered.response.status).toBe(200);
|
||
expect(recovered.body.id).toBe(user.id);
|
||
|
||
const history = await historyOf(chat.id, token);
|
||
expect(history.response.status).toBe(200);
|
||
expect(history.body.items.map((item) => item.id)).toContain(
|
||
message.id,
|
||
);
|
||
},
|
||
outagedRequestBudgetMs,
|
||
);
|
||
|
||
it(
|
||
'serves bounded 503 for the PostgreSQL outage and keeps confirmed data',
|
||
async () => {
|
||
await compose('stop', postgresService);
|
||
const replica = replicas[0];
|
||
|
||
expect((await probeReplica(replica, '/health/live')).status).toBe(
|
||
200,
|
||
);
|
||
await waitForReplicaReadiness(replica, 503, 30_000);
|
||
const readiness = await probeReplica(replica, '/health/ready');
|
||
expectBoundedError(
|
||
readiness.status,
|
||
readiness.text,
|
||
readiness.requestId,
|
||
);
|
||
|
||
// Redis still answers, so these requests reach the database layer and
|
||
// must fail with the bounded 503, not an unhandled 500.
|
||
const me = await probeReplica(replica, '/api/v1/users/me', {
|
||
headers: bearer(token),
|
||
});
|
||
expectBoundedError(me.status, me.text, me.requestId);
|
||
|
||
const chats = await probeReplica(replica, '/api/v1/chats', {
|
||
headers: bearer(token),
|
||
});
|
||
expectBoundedError(chats.status, chats.text, chats.requestId);
|
||
|
||
const history = await probeReplica(
|
||
replica,
|
||
`/api/v1/chats/${chat.id}/messages?limit=100`,
|
||
{ headers: bearer(token) },
|
||
);
|
||
expectBoundedError(history.status, history.text, history.requestId);
|
||
|
||
const registration = await probeReplica(replica, '/api/v1/users', {
|
||
method: 'POST',
|
||
headers: { 'content-type': 'application/json' },
|
||
body: JSON.stringify({
|
||
username: `outage_${runId}_late`,
|
||
display_name: 'Отказ зависимости',
|
||
password,
|
||
}),
|
||
});
|
||
expectBoundedError(
|
||
registration.status,
|
||
registration.text,
|
||
registration.requestId,
|
||
);
|
||
|
||
expectUnavailableThroughLb(
|
||
await request<ApiErrorBody>('/api/v1/users/me', {
|
||
headers: bearer(token),
|
||
}),
|
||
);
|
||
await waitForSlot(
|
||
// 'UP 1/2' is still in rotation: wait for DOWN itself.
|
||
(slots) => slots.every((slot) => slot.status === 'DOWN'),
|
||
'not every slot left rotation',
|
||
);
|
||
expectLbOutage(
|
||
await request('/api/v1/users/me', { headers: bearer(token) }),
|
||
);
|
||
|
||
await compose('start', postgresService);
|
||
await waitForReadiness(200);
|
||
await waitForSlot(
|
||
(slots) => upSlots(slots).length === replicas.length,
|
||
'backends did not return to rotation',
|
||
30_000,
|
||
);
|
||
|
||
const recovered = await request<User>('/api/v1/users/me', {
|
||
headers: bearer(token),
|
||
});
|
||
expect(recovered.response.status).toBe(200);
|
||
expect(recovered.body.id).toBe(user.id);
|
||
|
||
const after = await historyOf(chat.id, token);
|
||
expect(after.response.status).toBe(200);
|
||
expect(after.body.items.map((item) => item.id)).toContain(
|
||
message.id,
|
||
);
|
||
|
||
const sent = await request<Message>(
|
||
`/api/v1/chats/${chat.id}/messages`,
|
||
json({ text: 'Сообщение после отказа' }, messageHeaders(token)),
|
||
);
|
||
expect(sent.response.status).toBe(201);
|
||
},
|
||
outagedRequestBudgetMs,
|
||
);
|
||
|
||
it(
|
||
'excludes only the replica that lost its own database path',
|
||
async () => {
|
||
const selected = replicas[0];
|
||
const healthy = replicas[1];
|
||
const selectedIp = await containerIp(selected);
|
||
const selectedAddress = `${selectedIp}:8080`;
|
||
const selectedInstance = (
|
||
await probeReplica(selected, '/health/live')
|
||
).instanceId;
|
||
expect(selectedInstance).not.toBeNull();
|
||
const healthyInstance = (
|
||
await probeReplica(healthy, '/health/live')
|
||
).instanceId;
|
||
|
||
try {
|
||
await applyReplicaDatabaseReject(selectedIp);
|
||
const faultAt = Date.now();
|
||
|
||
await waitForReplicaReadiness(selected, 503, 30_000);
|
||
expect(
|
||
(await probeReplica(healthy, '/health/ready')).status,
|
||
).toBe(200);
|
||
|
||
// Only the affected slot leaves rotation. While `fall 2` counts the
|
||
// failed checks the slot passes through "UP 1/2 (L7STS)", so the
|
||
// wait is for DOWN itself, not for "no longer plain UP".
|
||
const slots = await waitForSlot(
|
||
(candidates) =>
|
||
upSlots(candidates).length === replicas.length - 1 &&
|
||
candidates.some(
|
||
(slot) =>
|
||
slot.addr === selectedAddress &&
|
||
slot.status === 'DOWN',
|
||
),
|
||
`selected slot ${selectedAddress} did not go down`,
|
||
30_000,
|
||
);
|
||
// inter 500 ms × fall 2 + timeout check 500 ms + polling step 250 ms.
|
||
expect(Date.now() - faultAt).toBeLessThan(3_000);
|
||
expect(
|
||
slots.find((slot) => slot.addr === selectedAddress)?.status,
|
||
).toBe('DOWN');
|
||
|
||
const served = new Set<string>();
|
||
for (let index = 0; index < readyReads; index += 1) {
|
||
const read = await historyOf(chat.id, token);
|
||
expect(read.response.status).toBe(200);
|
||
served.add(
|
||
read.response.headers.get('x-instance-id') ?? 'unknown',
|
||
);
|
||
}
|
||
expect(served.has(selectedInstance as string)).toBe(false);
|
||
expect(served.has(healthyInstance as string)).toBe(true);
|
||
} finally {
|
||
await restoreDatabaseAccess();
|
||
}
|
||
|
||
await waitForReplicaReadiness(selected, 200, 30_000);
|
||
await waitForSlot(
|
||
(slots) =>
|
||
upSlots(slots).length === replicas.length &&
|
||
slots.some(
|
||
(slot) =>
|
||
slot.addr === selectedAddress &&
|
||
slot.status === 'UP',
|
||
),
|
||
'restored replica did not return to rotation',
|
||
30_000,
|
||
);
|
||
|
||
let servedAgain = false;
|
||
for (
|
||
let index = 0;
|
||
index < readyReads * 2 && !servedAgain;
|
||
index += 1
|
||
) {
|
||
const read = await historyOf(chat.id, token);
|
||
expect(read.response.status).toBe(200);
|
||
servedAgain =
|
||
read.response.headers.get('x-instance-id') ===
|
||
selectedInstance;
|
||
}
|
||
expect(servedAgain).toBe(true);
|
||
},
|
||
outagedRequestBudgetMs,
|
||
);
|
||
});
|