Carry over the lab 3 solution
Bring the application, own tests, Dockerfile, toolchain, compose stack,
HAProxy and k6 configuration, migrations, hooks, evidence and level 4
scripts and the CI job from the lab 3 solution commit 61c0c28, merge
the lab 3 Makefile targets into the lab 4 skeleton Makefile without
touching its targets, and append the lab 3 solution variables to the
skeleton .env.example. The package and the application follow the
contract version 1.4.0, and the image and the throwaway test containers
are tagged lab4 so they do not collide with a lab 3 stand on the same
host.
This is the starting point of the lab 4 cycle: lint, own tests and the
offline skeleton check pass before any lab 4 change.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
b0f463ac5c
commit
bf8e4e3cd1
Executable
+20
@@ -0,0 +1,20 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
out=${1:-evidence/detection}
|
||||
mkdir -p "$out"
|
||||
scripts/restore.sh
|
||||
replica() { docker compose ps -q "${API_SERVICE:-api}" | head -1; }
|
||||
detect() { python3 scripts/probe.py detect "$@"; }
|
||||
failed=0
|
||||
detect --max 1.5 --within-action --restore scripts/restore.sh --out "$out/sigterm.json" \
|
||||
-- docker stop -t "${STOP_TIMEOUT:-6}" "$(replica)" || failed=1
|
||||
detect --max 1.5 --restore scripts/restore.sh --out "$out/kill.json" \
|
||||
-- docker kill "$(replica)" || failed=1
|
||||
hung=$(replica)
|
||||
detect --max 3.5 --restore "docker unpause $hung && scripts/lb-wait.sh" --out "$out/pause.json" \
|
||||
-- docker pause "$hung" || failed=1
|
||||
db=$(docker compose ps -q postgres)
|
||||
detect --max 1.5 --until 0 --lb-status 503 --restore "docker start $db && scripts/lb-wait.sh" \
|
||||
--out "$out/postgres.json" -- docker stop "$db" || failed=1
|
||||
exit "$failed"
|
||||
Executable
+24
@@ -0,0 +1,24 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
cli() { docker compose exec -T valkey sh -c "REDISCLI_AUTH=\"\$(sed -n 's/^requirepass //p' /run/secrets/valkey.conf)\" valkey-cli $1"; }
|
||||
psql() { docker compose exec -T postgres psql -U messenger -d messenger -tA -c "$1"; }
|
||||
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"
|
||||
echo "\$ valkey-cli config get appendonly appendfsync maxmemory maxmemory-policy"
|
||||
cli "config get appendonly" ; cli "config get appendfsync"; cli "config get maxmemory"; cli "config get maxmemory-policy"
|
||||
echo "\$ valkey-cli info persistence | grep aof_enabled,aof_last_write_status"
|
||||
cli "info persistence" | grep -E '^aof_enabled|^aof_last_write_status'
|
||||
echo "\$ valkey-cli dbsize; info stats | grep keyspace"
|
||||
cli dbsize
|
||||
cli "info stats" | grep -E '^keyspace_(hits|misses)'
|
||||
key=$(cli "--scan --pattern session:* --count 1000" | head -1)
|
||||
hash=${key#session:}
|
||||
echo "\$ сравнение срока ключа кэша и срока сессии для одного ключа (значение и токен не печатаются)"
|
||||
echo "valkey pexpiretime, ms: $(cli "pexpiretime $key")"
|
||||
echo "postgres expires_at, ms: $(psql "SELECT (extract(epoch FROM expires_at) * 1000)::bigint FROM sessions WHERE token_hash = decode('$hash', 'hex')")"
|
||||
echo "\$ количество отметок отзыва и закэшированных сессий"
|
||||
revoked=0; cached=0
|
||||
for k in $(cli "--scan --pattern session:* --count 1000"); do
|
||||
if [ "$(cli "get $k")" = "-" ]; then revoked=$((revoked + 1)); else cached=$((cached + 1)); fi
|
||||
done
|
||||
echo "tombstones: $revoked, cached sessions: $cached"
|
||||
Executable
+48
@@ -0,0 +1,48 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
base=${1:-http://$(docker compose port lb 8080)}
|
||||
stats=http://$(docker compose port lb 8404)/stats\;csv
|
||||
header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"; }
|
||||
{
|
||||
header
|
||||
echo "\$ docker compose --env-file .env config --quiet && echo valid"
|
||||
docker compose --env-file .env config --quiet && echo valid
|
||||
echo "\$ docker compose ps --format '{{.Service}} {{.Name}} {{.Status}} {{.Ports}}'"
|
||||
docker compose ps --format '{{.Service}} {{.Name}} {{.Status}} {{.Ports}}'
|
||||
echo "\$ docker compose config | grep -c container_name"
|
||||
docker compose config | grep -c container_name || true
|
||||
echo "\$ docker port <каждая реплика api> # реплики не публикуют порт"
|
||||
for id in $(docker compose ps -q api); do echo "$(docker inspect -f '{{.Config.Hostname}}' "$id"): [$(docker port "$id")]"; done
|
||||
echo "\$ docker inspect: hostname = X-Instance-Id, сети, лимиты CPU (nano) и памяти"
|
||||
for id in $(docker compose ps -q); do
|
||||
docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}} {{.Config.Hostname}} {{.Name}} nets={{range $k, $v := .NetworkSettings.Networks}}{{$k}},{{end}} cpus={{.HostConfig.NanoCpus}} mem={{.HostConfig.Memory}} user={{.Config.User}} ro={{.HostConfig.ReadonlyRootfs}}' "$id"
|
||||
done | sort
|
||||
echo "\$ curl LB stats: backend api, server status check"
|
||||
curl -fsS "$stats" | python3 -c 'import csv,sys
|
||||
for r in csv.DictReader(sys.stdin.read().lstrip("# ").splitlines()):
|
||||
if r["pxname"] == "api" and r["status"] != "MAINT (resolution)":
|
||||
print(r["svname"], r["status"], r["check_status"], r["check_code"])'
|
||||
echo "\$ for i in 1..6: curl -D - $base/health/live | X-Instance-Id"
|
||||
for _ in 1 2 3 4 5 6; do curl -fsS -D - -o /dev/null "$base/health/live" | tr -d '\r' | grep -i '^x-instance-id'; done
|
||||
} > evidence/stack.txt
|
||||
{
|
||||
header
|
||||
echo "host: $(sysctl -n machdep.cpu.brand_string 2>/dev/null || uname -m), $(sysctl -n hw.ncpu 2>/dev/null) CPU, $(($(sysctl -n hw.memsize 2>/dev/null || echo 0) / 1073741824)) GiB RAM, $(sw_vers -productName 2>/dev/null) $(sw_vers -productVersion 2>/dev/null)"
|
||||
echo "docker: $(docker version -f '{{.Server.Version}}'), VM: $(docker info -f '{{.NCPU}}') CPU, $(docker info -f '{{.MemTotal}}') bytes"
|
||||
echo "compose: $(docker compose version --short)"
|
||||
echo "images:"
|
||||
grep -h -E 'image:|@sha256' compose.yaml Dockerfile | sed 's/^ */ /'
|
||||
echo "python in image: $(docker exec "$(docker compose ps -q api | head -1)" python -V)"
|
||||
echo "haproxy: $(docker compose exec -T lb haproxy -v | head -1)"
|
||||
echo "packages (uv.lock):"
|
||||
for name in fastapi uvicorn uvicorn-worker gunicorn asyncpg argon2-cffi pydantic pydantic-settings dishka structlog redis; do
|
||||
printf ' %s %s\n' "$name" "$(sed -n "/^name = \"$name\"$/{n;s/version = \"\(.*\)\"/\1/p;}" uv.lock)"
|
||||
done
|
||||
echo "process users (service uid command):"
|
||||
for id in $(docker compose ps -q); do
|
||||
service=$(docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}}' "$id")
|
||||
docker top "$id" -eo pid,uid,comm | awk -v s="$service" 'NR > 1 {print " " s, $2, $3}'
|
||||
done | sort -u
|
||||
} > evidence/environment.txt
|
||||
cat evidence/stack.txt evidence/environment.txt
|
||||
Executable
+44
@@ -0,0 +1,44 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
psql() { docker compose exec -T postgres psql -U messenger -d messenger -c "$1"; }
|
||||
header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"; }
|
||||
counts="SELECT (SELECT count(*) FROM users) users, (SELECT count(*) FROM sessions) sessions, (SELECT count(*) FROM chats) chats, (SELECT count(*) FROM chat_members) members, (SELECT count(*) FROM messages) messages, (SELECT count(*) FROM idempotency_keys) keys, (SELECT count(*) FROM schema_migrations) versions"
|
||||
kdf="SELECT substring(password_hash from '^[$]argon2id[$]v=[0-9]+[$]m=[0-9]+,t=[0-9]+,p=[0-9]+[$]') AS kdf_prefix, count(*) AS users, min(length(password_hash)) AS hash_length FROM users GROUP BY 1"
|
||||
tokens="SELECT octet_length(token_hash) AS stored_bytes, count(*) AS sessions FROM sessions GROUP BY 1"
|
||||
{
|
||||
header
|
||||
for table in users sessions chats chat_members messages idempotency_keys schema_migrations; do
|
||||
echo "\$ psql -c '\\d+ $table'"
|
||||
psql "\\d+ $table"
|
||||
done
|
||||
echo "\$ psql -c 'SELECT version, left(checksum, 12), applied_at FROM schema_migrations'"
|
||||
psql "SELECT version, left(checksum, 12) AS checksum, applied_at FROM schema_migrations"
|
||||
} > evidence/schema.txt
|
||||
{
|
||||
header
|
||||
echo "\$ psql -c \"$counts\""
|
||||
psql "$counts"
|
||||
echo "\$ make migrate"
|
||||
docker compose run --rm migrate 2>/dev/null
|
||||
echo "\$ psql -c \"$counts\""
|
||||
psql "$counts"
|
||||
} > evidence/migrations.txt
|
||||
{
|
||||
header
|
||||
echo "\$ psql -c \"$kdf\""
|
||||
psql "$kdf"
|
||||
echo "\$ psql -c \"$tokens\""
|
||||
psql "$tokens"
|
||||
} > evidence/kdf.txt
|
||||
{
|
||||
header
|
||||
echo "\$ git ls-files secrets .env backups volumes"
|
||||
git ls-files secrets .env backups volumes
|
||||
echo "(пусто: секретов, .env, дампов и volumes в Git нет)"
|
||||
echo "\$ git check-ignore -v secrets/database_url secrets/postgres_password .env backups/"
|
||||
git check-ignore -v secrets/database_url secrets/postgres_password .env backups/
|
||||
echo "\$ docker volume ls --filter name=mtusi2026lab4"
|
||||
docker volume ls --filter name=mtusi2026lab4
|
||||
} > evidence/secrets.txt
|
||||
echo "evidence/{schema,migrations,kdf,secrets}.txt обновлены"
|
||||
Executable
+19
@@ -0,0 +1,19 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
out=${1:-evidence/failover-each}
|
||||
rm -rf "$out"
|
||||
mkdir -p "$out"
|
||||
scripts/restore.sh
|
||||
hosts=$(docker inspect -f '{{.Config.Hostname}}' $(docker compose ps -q "${API_SERVICE:-api}"))
|
||||
failed=0
|
||||
set --
|
||||
for host in $hosts; do
|
||||
TARGET_INSTANCE_ID=$host python3 scripts/probe.py load --expect dip --out "$out/$host" \
|
||||
-- sh -c "scripts/stop-one.sh && sleep ${HOLD:-10} && scripts/restore.sh" || failed=1
|
||||
scripts/restore.sh
|
||||
set -- "$@" "$out/$host/result.json"
|
||||
done
|
||||
python3 -c 'import json,sys; print(json.dumps([json.load(open(p)) for p in sys.argv[1:]], ensure_ascii=False, indent=2))' "$@" > "$out/summary.tmp"
|
||||
mv "$out/summary.tmp" "$out/summary.json"
|
||||
exit "$failed"
|
||||
Executable
+28
@@ -0,0 +1,28 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
replicas() {
|
||||
docker inspect -f '{{.Config.Hostname}} {{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}' $(docker compose ps -aq "${API_SERVICE:-api}")
|
||||
}
|
||||
tmp=evidence/failover.tmp
|
||||
since=$(date -u +%FT%TZ)
|
||||
{
|
||||
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"
|
||||
echo "== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt"
|
||||
replicas
|
||||
echo "\$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh"
|
||||
make --no-print-directory test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL="${1:-http://localhost:8080}"
|
||||
echo "== after: api replicas"
|
||||
replicas
|
||||
} > "$tmp" 2>&1
|
||||
stopped=$(replicas | awk '$2 == "exited" {print $1}')
|
||||
{
|
||||
echo "== stopped replica: $stopped; shutdown: signal, drain window with readiness 503, close, exit"
|
||||
docker logs --since "$since" "$stopped" 2>&1 | grep -E 'Handling signal|Draining|"status":503|Shutting down|Waiting for conn|Finished server|service.stopped|SIGTERM' | tail -16
|
||||
echo "\$ scripts/restore.sh"
|
||||
scripts/restore.sh
|
||||
echo "== after restore"
|
||||
replicas
|
||||
} >> "$tmp" 2>&1
|
||||
mv "$tmp" "evidence/failover-$stopped.txt"
|
||||
cat "evidence/failover-$stopped.txt"
|
||||
Executable
+10
@@ -0,0 +1,10 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
out=${1:?usage: k6.sh <output dir> [k6 run args]}
|
||||
shift
|
||||
mkdir -p "$out"
|
||||
out=$(cd "$out" && pwd)
|
||||
cd "$(dirname "$0")/.."
|
||||
exec docker compose --profile bench run --rm --no-deps -T -u "$(id -u):$(id -g)" -v "$out:/out" \
|
||||
-e RATE -e WARMUP -e DURATION -e USERS -e READ_RATIO -e VUS -e MAX_VUS -e RUN_ID \
|
||||
k6 run --quiet "$@" "/scripts/${K6_SCRIPT:-messages.js}"
|
||||
Executable
+11
@@ -0,0 +1,11 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
service=${API_SERVICE:-api}
|
||||
want=${1:-$(docker compose ps -q "$service" | wc -l | tr -d ' ')}
|
||||
url=${LB_STATS_URL:-http://$(docker compose port lb 8404)/stats;csv}
|
||||
deadline=$(($(date +%s) + ${WAIT_TIMEOUT:-60}))
|
||||
until [ "$(curl -fsS "$url" | awk -F, -v b="$service" '$1 == b && $2 != "BACKEND" && $18 == "UP"' | wc -l)" -ge "$want" ]; do
|
||||
[ "$(date +%s)" -lt "$deadline" ] || { echo "LB does not see $want UP $service servers" >&2; exit 4; }
|
||||
sleep 0.5
|
||||
done
|
||||
@@ -0,0 +1,132 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
POLICY = {
|
||||
"lb": {"secrets": [], "volumes": [], "networks": ["backend", "edge"], "ports": True},
|
||||
"api": {
|
||||
"secrets": ["cache_url", "cursor_secret", "database_url"],
|
||||
"volumes": [],
|
||||
"networks": ["backend"],
|
||||
},
|
||||
"migrate": {"secrets": ["database_url"], "volumes": [], "networks": ["backend"]},
|
||||
"postgres": {"secrets": ["postgres_password"], "volumes": ["pgdata"], "networks": ["backend"]},
|
||||
"valkey": {"secrets": ["valkey.conf"], "volumes": ["valkeydata"], "networks": ["backend"]},
|
||||
"k6": {"secrets": [], "volumes": [], "networks": ["edge"]},
|
||||
}
|
||||
|
||||
|
||||
def run(*command):
|
||||
done = subprocess.run(command, cwd=str(ROOT), capture_output=True, text=True, check=False)
|
||||
return done.returncode, done.stdout.strip()
|
||||
|
||||
|
||||
def declared(name, service, networks, problems):
|
||||
limits = service.get("deploy", {}).get("resources", {}).get("limits", {})
|
||||
secrets = sorted(s["source"] for s in service.get("secrets", []))
|
||||
volumes = sorted(v["source"] for v in service.get("volumes", []) if v["type"] == "volume")
|
||||
nets = sorted(service.get("networks", {}))
|
||||
ports = [
|
||||
f"{p.get('host_ip', '0.0.0.0')}:{p.get('published')}" for p in service.get("ports", [])
|
||||
]
|
||||
want = POLICY.get(name)
|
||||
if want is None:
|
||||
problems.append(f"{name}: service is not covered by the policy")
|
||||
want = {"secrets": [], "volumes": [], "networks": []}
|
||||
if not limits.get("cpus") or not limits.get("memory"):
|
||||
problems.append(f"{name}: no CPU or memory limit")
|
||||
if "container_name" in service:
|
||||
problems.append(f"{name}: container_name prevents scaling")
|
||||
for kind, actual in (("secrets", secrets), ("volumes", volumes), ("networks", nets)):
|
||||
if actual != want[kind]:
|
||||
problems.append(f"{name}: {kind} {actual}, policy {want[kind]}")
|
||||
if ports and not want.get("ports"):
|
||||
problems.append(f"{name}: publishes {ports} in bypass of the balancer")
|
||||
if any(not port.startswith("127.0.0.1:") for port in ports):
|
||||
problems.append(f"{name}: publishes {ports} beyond loopback")
|
||||
if name == "api" and not service.get("read_only"):
|
||||
problems.append("api: root filesystem is writable")
|
||||
internal = [n for n in nets if networks.get(n, {}).get("internal")]
|
||||
return [
|
||||
name,
|
||||
f"{limits.get('cpus', '-')}/{int(limits.get('memory') or 0) >> 20}MiB",
|
||||
",".join(ports) or "-",
|
||||
",".join(secrets) or "-",
|
||||
",".join(volumes) or "-",
|
||||
",".join(f"{n}{'(internal)' if n in internal else ''}" for n in nets),
|
||||
str(service.get("read_only", False)).lower(),
|
||||
]
|
||||
|
||||
|
||||
def running(problems):
|
||||
_, ids = run("docker", "compose", "ps", "-q")
|
||||
template = (
|
||||
'{{index .Config.Labels "com.docker.compose.service"}} {{.Config.Hostname}} '
|
||||
"{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}} {{json .HostConfig.PortBindings}}"
|
||||
)
|
||||
_, found = run("docker", "inspect", "-f", template, *ids.split())
|
||||
rows = []
|
||||
for line in found.splitlines():
|
||||
service, host, cpus, memory, bindings = line.split(" ", 4)
|
||||
published = [
|
||||
f"{b['HostIp']}:{b['HostPort']}->{port}"
|
||||
for port, items in (json.loads(bindings) or {}).items()
|
||||
for b in items or []
|
||||
]
|
||||
if int(cpus) <= 0 or int(memory) <= 0:
|
||||
problems.append(f"container {service}/{host}: no CPU or memory limit applied")
|
||||
if published and service != "lb":
|
||||
problems.append(f"container {service}/{host}: publishes {published}")
|
||||
rows.append(
|
||||
[service, host, f"{int(cpus) / 1e9:g}", f"{int(memory) >> 20}MiB", ",".join(published)]
|
||||
)
|
||||
return sorted(rows)
|
||||
|
||||
|
||||
def table(header, rows):
|
||||
widths = [max(len(str(r[i])) for r in [header, *rows]) for i in range(len(header))]
|
||||
lines = [header, ["-" * w for w in widths], *rows]
|
||||
return [" ".join(str(c).ljust(w) for c, w in zip(r, widths)).rstrip() for r in lines]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="limits, ports, secrets, volumes and networks")
|
||||
parser.add_argument("--out", help="text report, written only after a complete check")
|
||||
args = parser.parse_args()
|
||||
problems = []
|
||||
_, raw = run("docker", "compose", "--profile", "bench", "config", "--format", "json")
|
||||
config = json.loads(raw)
|
||||
services = [
|
||||
declared(name, service, config.get("networks", {}), problems)
|
||||
for name, service in sorted(config["services"].items())
|
||||
]
|
||||
containers = running(problems)
|
||||
if not any(row[0] == "api" for row in containers):
|
||||
problems.append("no running api replicas to inspect")
|
||||
lines = [
|
||||
f"date: {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}",
|
||||
"$ docker compose --profile bench config --format json (declared)",
|
||||
*table(["service", "cpus/mem", "ports", "secrets", "volumes", "networks", "ro"], services),
|
||||
"",
|
||||
"$ docker inspect (running containers: hostname = X-Instance-Id, applied limits)",
|
||||
*table(["service", "hostname", "cpus", "memory", "published"], containers),
|
||||
"",
|
||||
*(f"FAIL: {p}" for p in problems),
|
||||
"PASS: limits, ports, secrets, volumes and networks match the policy"
|
||||
if not problems
|
||||
else f"FAIL: {len(problems)} violation(s)",
|
||||
]
|
||||
text = "\n".join(lines) + "\n"
|
||||
if args.out:
|
||||
Path(args.out).write_text(text)
|
||||
print(text, end="")
|
||||
return 1 if problems else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+14
@@ -0,0 +1,14 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
started() {
|
||||
for id in $(docker compose ps -aq); do
|
||||
docker inspect --format '{{index .Config.Labels "com.docker.compose.service"}} {{.Config.Hostname}} {{.State.StartedAt}} {{.State.Status}}' "$id"
|
||||
done | sort
|
||||
}
|
||||
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"
|
||||
echo "== before: service hostname StartedAt status"
|
||||
started
|
||||
make --no-print-directory test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL="${1:-http://localhost:8080}"
|
||||
echo "== after: service hostname StartedAt status"
|
||||
started
|
||||
Executable
+436
@@ -0,0 +1,436 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import base64
|
||||
import collections
|
||||
import functools
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
COUNTERS = ("wretr", "wredis", "econ", "eresp", "hrsp_5xx")
|
||||
|
||||
|
||||
@functools.cache
|
||||
def published(port, fallback):
|
||||
command = ["docker", "compose", "port", "lb", str(port)]
|
||||
found = subprocess.run(command, cwd=str(ROOT), capture_output=True, text=True, check=False)
|
||||
return found.stdout.strip() or fallback
|
||||
|
||||
|
||||
def base_url():
|
||||
return os.environ.get("BASE_URL") or "http://" + published(8080, "localhost:8080")
|
||||
|
||||
|
||||
def stats_url():
|
||||
return os.environ.get("LB_STATS_URL") or f"http://{published(8404, '127.0.0.1:8404')}/stats;csv"
|
||||
|
||||
|
||||
def call(method, path, body=None, headers=None, timeout=2.0):
|
||||
data = None if body is None else json.dumps(body, ensure_ascii=False).encode()
|
||||
head = {"Accept": "application/json"}
|
||||
if data is not None:
|
||||
head["Content-Type"] = "application/json"
|
||||
head.update(headers or {})
|
||||
request = urllib.request.Request(
|
||||
base_url().rstrip("/") + path, data=data, headers=head, method=method
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
payload = json.loads(response.read() or b"null")
|
||||
return response.status, response.headers.get("X-Instance-Id", ""), payload
|
||||
except urllib.error.HTTPError as error:
|
||||
try:
|
||||
payload = json.loads(error.read() or b"null")
|
||||
except (OSError, ValueError):
|
||||
payload = None
|
||||
return error.code, error.headers.get("X-Instance-Id", ""), payload
|
||||
except (OSError, ValueError):
|
||||
return 0, "", None
|
||||
|
||||
|
||||
def emit(report, out=None):
|
||||
text = json.dumps(report, ensure_ascii=False, indent=2)
|
||||
if out:
|
||||
Path(out).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(out).write_text(text + "\n")
|
||||
print(text)
|
||||
|
||||
|
||||
def created(result, what):
|
||||
if result[0] != 201:
|
||||
raise SystemExit(f"{what} failed: HTTP {result[0]}")
|
||||
return result[2]
|
||||
|
||||
|
||||
def actor(label):
|
||||
username = f"p_{label}_{uuid.uuid4().hex[:12]}"
|
||||
password = "Probe-" + uuid.uuid4().hex
|
||||
body = {"username": username, "display_name": "probe " + label, "password": password}
|
||||
user = created(call("POST", "/api/v1/users", body, timeout=10), "register")
|
||||
basic = base64.b64encode(f"{username}:{password}".encode()).decode()
|
||||
login = call(
|
||||
"POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}, timeout=10
|
||||
)
|
||||
return {
|
||||
"id": user["id"],
|
||||
"auth": {"Authorization": "Bearer " + created(login, "login")["token"]},
|
||||
}
|
||||
|
||||
|
||||
def chat_of(owner, other):
|
||||
body = {"title": "probe", "member_ids": [owner["id"], other["id"]]}
|
||||
return created(call("POST", "/api/v1/chats", body, owner["auth"], timeout=10), "chat")["id"]
|
||||
|
||||
|
||||
def send(owner, chat, text, key):
|
||||
headers = dict(owner["auth"], **{"Idempotency-Key": key})
|
||||
return call("POST", f"/api/v1/chats/{chat}/messages", {"text": text}, headers)
|
||||
|
||||
|
||||
def lb_rows(backend="api"):
|
||||
try:
|
||||
with urllib.request.urlopen(stats_url(), timeout=2) as response:
|
||||
lines = response.read().decode().splitlines()
|
||||
except OSError:
|
||||
return None
|
||||
header = lines[0].lstrip("# ").split(",")
|
||||
rows = [dict(zip(header, line.split(","))) for line in lines[1:] if line]
|
||||
return [r for r in rows if r["pxname"] == backend]
|
||||
|
||||
|
||||
def lb_up():
|
||||
rows = lb_rows()
|
||||
if rows is None:
|
||||
return None
|
||||
return sum(1 for r in rows if r["svname"] != "BACKEND" and r["status"].startswith("UP"))
|
||||
|
||||
|
||||
def lb_counters():
|
||||
row = next((r for r in lb_rows() or [] if r["svname"] == "BACKEND"), {})
|
||||
return {name: int(row.get(name) or 0) for name in COUNTERS}
|
||||
|
||||
|
||||
def run_k6(out, env, on_start=None):
|
||||
Path(out).mkdir(parents=True, exist_ok=True)
|
||||
started = []
|
||||
process = subprocess.Popen(
|
||||
[str(ROOT / "scripts/k6.sh"), str(out)],
|
||||
env=dict(os.environ, **env),
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
)
|
||||
with (Path(out) / "k6.txt").open("w") as log:
|
||||
for line in process.stdout:
|
||||
log.write(line)
|
||||
if not started and "LOAD-START" in line:
|
||||
started.append(time.time())
|
||||
if on_start:
|
||||
on_start()
|
||||
return process.wait(), started[0] if started else None
|
||||
|
||||
|
||||
def k6_metrics(path, duration):
|
||||
metrics = json.loads(Path(path).read_text())["metrics"]
|
||||
|
||||
def value(name, field):
|
||||
return metrics.get(name + "{scenario:measure}", {}).get("values", {}).get(field, 0.0)
|
||||
|
||||
count = value("http_reqs", "count")
|
||||
return {
|
||||
"requests": int(count),
|
||||
"rps": round(count / duration, 2),
|
||||
"ok_rps": round(count * (1 - value("http_req_failed", "rate")) / duration, 2),
|
||||
"failed_ratio": round(value("http_req_failed", "rate"), 5),
|
||||
"dropped_iterations": int(value("dropped_iterations", "count")),
|
||||
"p50_ms": round(value("http_req_duration", "med"), 2),
|
||||
"p95_ms": round(value("http_req_duration", "p(95)"), 2),
|
||||
"p99_ms": round(value("http_req_duration", "p(99)"), 2),
|
||||
}
|
||||
|
||||
|
||||
def race(alice, chat, parallel):
|
||||
key, text = uuid.uuid4().hex, "same key " + uuid.uuid4().hex
|
||||
barrier, results = threading.Barrier(parallel), []
|
||||
|
||||
def worker():
|
||||
barrier.wait()
|
||||
results.append(send(alice, chat, text, key))
|
||||
|
||||
threads = [threading.Thread(target=worker) for _ in range(parallel)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
bodies = [r[2] for r in results if r[0] == 201]
|
||||
identical = bool(bodies) and all(b == bodies[0] for b in bodies)
|
||||
return text, results, identical, send(alice, chat, text + " changed", key)[0]
|
||||
|
||||
|
||||
def check(args):
|
||||
alice, bob = actor("alice"), actor("bob")
|
||||
chat = chat_of(alice, bob)
|
||||
seen = collections.Counter(
|
||||
call("GET", "/api/v1/users/me", headers=alice["auth"])[1] for _ in range(args.samples)
|
||||
)
|
||||
rounds = [race(alice, chat, args.parallel) for _ in range(min(args.rounds, 100))]
|
||||
page = call("GET", f"/api/v1/chats/{chat}/messages?limit=100", headers=bob["auth"])[2]
|
||||
copies = collections.Counter(m.get("text") for m in (page or {"items": []})["items"])
|
||||
statuses = collections.Counter(r[0] for _, results, _, _ in rounds for r in results)
|
||||
served = collections.Counter(r[1] for _, results, _, _ in rounds for r in results)
|
||||
same = {
|
||||
"rounds": len(rounds),
|
||||
"parallel": args.parallel,
|
||||
"statuses": dict(statuses),
|
||||
"instances": dict(served),
|
||||
"identical_bodies_rounds": sum(1 for _, _, identical, _ in rounds if identical),
|
||||
"single_copy_rounds": sum(1 for text, _, _, _ in rounds if copies[text] == 1),
|
||||
"conflict_rounds": sum(1 for _, _, _, other in rounds if other == 409),
|
||||
}
|
||||
passed = (
|
||||
len(seen) >= 2
|
||||
and statuses == {201: len(rounds) * args.parallel}
|
||||
and len(served) >= 2
|
||||
and same["identical_bodies_rounds"] == len(rounds)
|
||||
and same["single_copy_rounds"] == len(rounds)
|
||||
and same["conflict_rounds"] == len(rounds)
|
||||
)
|
||||
emit({"instances_one_token": dict(seen), "same_key": same, "pass": passed}, args.out)
|
||||
return 0 if passed else 1
|
||||
|
||||
|
||||
def record(stop, alice, chat):
|
||||
probes, trace = [], []
|
||||
|
||||
def prober():
|
||||
key = uuid.uuid4().hex
|
||||
while not stop.is_set():
|
||||
started = time.time()
|
||||
status, instance, _ = send(alice, chat, "probe " + key, key)
|
||||
probes.append((started, status, round(1000 * (time.time() - started), 1), instance))
|
||||
if status == 201:
|
||||
key = uuid.uuid4().hex
|
||||
time.sleep(0.1)
|
||||
|
||||
def poller():
|
||||
while not stop.is_set():
|
||||
trace.append((time.time(), lb_up()))
|
||||
time.sleep(0.1)
|
||||
|
||||
threads = [threading.Thread(target=t, daemon=True) for t in (prober, poller)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
return probes, trace, threads
|
||||
|
||||
|
||||
def write_csv(path, header, rows):
|
||||
with path.open("w") as handle:
|
||||
handle.write(header + "\n")
|
||||
handle.writelines(",".join(str(v) for v in row) + "\n" for row in rows)
|
||||
|
||||
|
||||
def summarize(args, probes, trace, action, expected):
|
||||
t0, end = action["start"], action["end"]
|
||||
label = args.label or os.environ.get("TARGET_INSTANCE_ID", "action")
|
||||
failures = [t - t0 for t, s, _, _ in probes if s != 201 and t >= t0]
|
||||
down = [t for t, up in trace if t >= t0 and up is not None and up < expected]
|
||||
back = [t for t, up in trace if down and t > down[0] and up is not None and up >= expected]
|
||||
outage = [t for t in down if not back or t < back[0]] or [end]
|
||||
served = [(t, i) for t, s, _, i in probes if s == 201]
|
||||
before = collections.Counter(i for t, i in served if t < t0)
|
||||
during = collections.Counter(i for t, i in served if outage[0] <= t <= outage[-1])
|
||||
after = collections.Counter(i for t, i in served if t > end)
|
||||
return {
|
||||
"label": label,
|
||||
"action": args.command,
|
||||
"action_exit": action["exit"],
|
||||
"action_seconds": round(end - t0, 3),
|
||||
"action_finished_before_load_end": end < action["load_end"],
|
||||
"lb_up_before": expected,
|
||||
"lb_up_min": min((up for _, up in trace if up is not None), default=None),
|
||||
"detection_seconds": round(down[0] - t0, 3) if down else None,
|
||||
"rejoin_seconds": round(back[0] - t0, 3) if back else None,
|
||||
"probe": {
|
||||
"attempts": len(probes),
|
||||
"failed_attempts": sum(1 for _, s, _, _ in probes if s != 201),
|
||||
"first_failure_s": round(failures[0], 3) if failures else None,
|
||||
"recovery_s": round(failures[-1], 3) if failures else 0.0,
|
||||
"instances_before": dict(before),
|
||||
"instances_during_outage": dict(during),
|
||||
"instances_after": dict(after),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def verdict(args, result):
|
||||
probe, k6 = result["probe"], result["k6"]
|
||||
checks = {
|
||||
"action_ok": result["action_exit"] == 0 and result["action_finished_before_load_end"],
|
||||
"load_errors_ok": k6["exit"] == 0 and k6["failed_ratio"] <= args.max_failed_ratio,
|
||||
"write_recovery_ok": probe["recovery_s"] <= args.max_recovery,
|
||||
}
|
||||
if args.expect == "dip":
|
||||
checks["detected"] = result["detection_seconds"] is not None
|
||||
checks["target_excluded"] = probe["instances_during_outage"].get(result["label"], 0) == 0
|
||||
checks["rejoined"] = result["rejoin_seconds"] is not None and bool(
|
||||
probe["instances_after"].get(result["label"])
|
||||
)
|
||||
if args.expect == "replace":
|
||||
checks["capacity_kept"] = (result["lb_up_min"] or 0) >= result["lb_up_before"]
|
||||
old, new = set(probe["instances_before"]), set(probe["instances_after"])
|
||||
checks["all_replaced"] = bool(new) and not old & new
|
||||
return checks
|
||||
|
||||
|
||||
def load(args):
|
||||
out = Path(args.out)
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
expected = lb_up() or 0
|
||||
alice, bob = actor("alice"), actor("bob")
|
||||
stop, action = threading.Event(), {}
|
||||
probes, trace, threads = record(stop, alice, chat_of(alice, bob))
|
||||
|
||||
def act():
|
||||
time.sleep(args.warmup + args.lead)
|
||||
action["start"] = time.time()
|
||||
with (out / "action.txt").open("w") as log:
|
||||
action["exit"] = subprocess.call(
|
||||
args.command, cwd=str(ROOT), stdout=log, stderr=subprocess.STDOUT
|
||||
)
|
||||
action["end"] = time.time()
|
||||
|
||||
runner = threading.Thread(target=act)
|
||||
env = {"RATE": str(args.rate), "WARMUP": str(args.warmup), "DURATION": str(args.duration)}
|
||||
counters = lb_counters()
|
||||
code, load_start = run_k6(out / "k6", env, runner.start)
|
||||
if load_start is not None:
|
||||
runner.join()
|
||||
action["load_end"] = load_start + args.warmup + args.duration
|
||||
stop.set()
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
if "start" not in action:
|
||||
raise SystemExit(f"k6 failed before the action (exit {code}), see {out / 'k6/k6.txt'}")
|
||||
t0 = action["start"]
|
||||
write_csv(
|
||||
out / "probe.csv",
|
||||
"t_rel_s,status,latency_ms,instance",
|
||||
((round(t - t0, 3), *r) for t, *r in probes),
|
||||
)
|
||||
write_csv(out / "lb_up.csv", "t_rel_s,up", ((round(t - t0, 3), up) for t, up in trace))
|
||||
result = summarize(args, probes, trace, action, expected)
|
||||
result["lb_counters"] = {k: v - counters[k] for k, v in lb_counters().items()}
|
||||
result["k6"] = dict(
|
||||
k6_metrics(out / "k6/summary.json", args.duration), exit=code, rate=args.rate
|
||||
)
|
||||
result["checks"] = verdict(args, result)
|
||||
result["pass"] = all(result["checks"].values())
|
||||
(out / "result.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
return 0 if result["pass"] else 1
|
||||
|
||||
|
||||
def detect(args):
|
||||
before = lb_up() or 0
|
||||
until = before - 1 if args.until is None else args.until
|
||||
started = time.time()
|
||||
process = subprocess.Popen(args.command, cwd=str(ROOT))
|
||||
detected = acted = None
|
||||
while time.time() - started < args.timeout and (detected is None or acted is None):
|
||||
if acted is None and process.poll() is not None:
|
||||
acted = time.time() - started
|
||||
if detected is None and (lb_up() or 0) <= until:
|
||||
detected = time.time() - started
|
||||
time.sleep(0.05)
|
||||
code = process.wait()
|
||||
acted = time.time() - started if acted is None else acted
|
||||
status, instance, payload = call("GET", "/health/live")
|
||||
restored = None
|
||||
if args.restore:
|
||||
begun = time.time()
|
||||
subprocess.call(["sh", "-c", args.restore], cwd=str(ROOT))
|
||||
while (lb_up() or 0) < before and time.time() - begun < 3 * args.timeout:
|
||||
time.sleep(0.1)
|
||||
restored = round(time.time() - begun, 3) if (lb_up() or 0) >= before else None
|
||||
checks = {
|
||||
"action_ok": code == 0,
|
||||
"detected_in_time": detected is not None and detected <= args.max + args.slack,
|
||||
"lb_answer_ok": status == args.lb_status,
|
||||
}
|
||||
if args.within_action:
|
||||
checks["excluded_before_action_end"] = detected is not None and detected < acted
|
||||
if args.restore:
|
||||
checks["restored"] = restored is not None
|
||||
report = {
|
||||
"action": args.command,
|
||||
"action_exit": code,
|
||||
"action_seconds": round(acted, 3),
|
||||
"lb_up_before": before,
|
||||
"lb_up_counted_as_detected": until,
|
||||
"detection_seconds": None if detected is None else round(detected, 3),
|
||||
"declared_max_seconds": args.max,
|
||||
"measurement_slack_seconds": args.slack,
|
||||
"lb_answer_after_detection": {
|
||||
"path": "/health/live",
|
||||
"status": status,
|
||||
"x_instance_id_present": bool(instance),
|
||||
"error_code": ((payload or {}).get("error") or {}).get("code"),
|
||||
},
|
||||
"restore": args.restore,
|
||||
"restore_seconds": restored,
|
||||
"lb_up_after_restore": lb_up(),
|
||||
"checks": checks,
|
||||
"pass": all(checks.values()),
|
||||
}
|
||||
emit(report, args.out)
|
||||
return 0 if report["pass"] else 1
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="LB probes over the public HTTP contract")
|
||||
sub = parser.add_subparsers(dest="mode", required=True)
|
||||
one = sub.add_parser("check", help="instances behind one token and same-key races")
|
||||
one.add_argument("--samples", type=int, default=40)
|
||||
one.add_argument("--parallel", type=int, default=8)
|
||||
one.add_argument("--rounds", type=int, default=1)
|
||||
one.add_argument("--out", help="JSON file, written only after a complete report")
|
||||
under = sub.add_parser("load", help="run an action under k6 load and measure recovery")
|
||||
under.add_argument("--out", required=True)
|
||||
under.add_argument("--label")
|
||||
under.add_argument("--expect", choices=("dip", "replace"), default="dip")
|
||||
under.add_argument("--rate", type=int, default=int(os.environ.get("RATE", "50")))
|
||||
under.add_argument("--warmup", type=int, default=10)
|
||||
under.add_argument("--duration", type=int, default=int(os.environ.get("DURATION", "60")))
|
||||
under.add_argument("--lead", type=int, default=15)
|
||||
under.add_argument("--max-failed-ratio", type=float, default=0.01)
|
||||
under.add_argument("--max-recovery", type=float, default=10.0)
|
||||
under.add_argument("command", nargs=argparse.REMAINDER)
|
||||
watch = sub.add_parser("detect", help="time until the LB takes replicas out of rotation")
|
||||
watch.add_argument("--max", type=float, required=True, help="declared worst detection, s")
|
||||
watch.add_argument("--slack", type=float, default=0.25, help="docker CLI and polling, s")
|
||||
watch.add_argument("--until", type=int, help="UP servers counted as detected")
|
||||
watch.add_argument("--within-action", action="store_true")
|
||||
watch.add_argument("--lb-status", type=int, default=200)
|
||||
watch.add_argument("--restore", help="shell command that brings replicas back")
|
||||
watch.add_argument("--timeout", type=float, default=20)
|
||||
watch.add_argument("--out", help="JSON file, written only after a complete report")
|
||||
watch.add_argument("command", nargs=argparse.REMAINDER)
|
||||
args = parser.parse_args()
|
||||
if args.mode == "check":
|
||||
return check(args)
|
||||
args.command = [c for c in args.command if c != "--"]
|
||||
if not args.command:
|
||||
parser.error(f"{args.mode} needs an action command after --")
|
||||
return load(args) if args.mode == "load" else detect(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+12
@@ -0,0 +1,12 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
defined=$(docker compose config --services)
|
||||
stores=""
|
||||
for service in ${STATE_SERVICES:-postgres valkey}; do
|
||||
if printf '%s\n' "$defined" | grep -qx "$service"; then stores="$stores $service"; fi
|
||||
done
|
||||
[ -n "$stores" ] || { echo "no state services found" >&2; exit 3; }
|
||||
docker compose stop lb "${API_SERVICE:-api}"
|
||||
docker compose restart $stores
|
||||
docker compose up -d --no-recreate --wait --wait-timeout "${WAIT_TIMEOUT:-75}"
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose up -d --no-deps --no-recreate --wait --wait-timeout "${WAIT_TIMEOUT:-60}" "${API_SERVICE:-api}"
|
||||
scripts/lb-wait.sh
|
||||
Executable
+20
@@ -0,0 +1,20 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
service=${API_SERVICE:-api}
|
||||
show() { docker inspect -f '{{.Config.Hostname}} image={{slice .Image 7 19}}' "$1"; }
|
||||
echo "$(date -u +%T) migrate once before the first replacement"
|
||||
docker compose run --rm --no-deps -T migrate 2>&1
|
||||
old=$(docker compose ps -q "$service")
|
||||
count=$(printf '%s\n' "$old" | grep -c .)
|
||||
for id in $old; do
|
||||
running=$(docker compose ps -q "$service")
|
||||
docker compose up -d --no-deps --no-recreate --wait --wait-timeout "${WAIT_TIMEOUT:-90}" --scale "$service=$((count + 1))" "$service" 2>&1
|
||||
scripts/lb-wait.sh $((count + 1))
|
||||
new=$(docker compose ps -q "$service" | grep -v -F "$running")
|
||||
echo "$(date -u +%T) replace $(show "$id") -> $(show "$new")"
|
||||
docker stop -t "${STOP_TIMEOUT:-6}" "$id" >/dev/null
|
||||
docker rm "$id" >/dev/null
|
||||
done
|
||||
scripts/lb-wait.sh "$count"
|
||||
echo "$(date -u +%T) done: $count replicas replaced"
|
||||
Executable
+20
@@ -0,0 +1,20 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
umask 077
|
||||
mkdir -p secrets
|
||||
random() { od -An -N32 -tx1 /dev/urandom | tr -d ' \n'; }
|
||||
put() {
|
||||
if [ ! -s "secrets/$1" ]; then
|
||||
printf '%s' "$2" > "secrets/$1"
|
||||
chmod 644 "secrets/$1"
|
||||
echo "создан secrets/$1"
|
||||
fi
|
||||
}
|
||||
put postgres_password "$(random)"
|
||||
put database_url "postgresql://messenger:$(cat secrets/postgres_password)@postgres:5432/messenger"
|
||||
put cursor_secret "$(random)"
|
||||
put valkey_password "$(random)"
|
||||
put valkey.conf "requirepass $(cat secrets/valkey_password)"
|
||||
put cache_url "redis://:$(cat secrets/valkey_password)@valkey:6379/0"
|
||||
[ -f .env ] || { cp .env.example .env; echo "создан .env из .env.example"; }
|
||||
@@ -0,0 +1,59 @@
|
||||
\set ON_ERROR_STOP on
|
||||
\set big '''00000000-0000-7000-8000-00000000c001'''
|
||||
\set cursor_at '''2026-01-01T02:46:40Z'''
|
||||
\set cursor_id '''00000000-0000-7000-8001-000000002710'''
|
||||
|
||||
PREPARE history_first (uuid, int) AS
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = $1
|
||||
ORDER BY created_at, id
|
||||
LIMIT $2;
|
||||
|
||||
PREPARE history_after (uuid, timestamptz, uuid, int) AS
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = $1 AND (created_at, id) > ($2, $3)
|
||||
ORDER BY created_at, id
|
||||
LIMIT $4;
|
||||
|
||||
PREPARE history_offset (uuid, int, int) AS
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = $1
|
||||
ORDER BY created_at, id
|
||||
LIMIT $2 OFFSET $3;
|
||||
|
||||
SELECT count(*) AS chat_messages FROM messages WHERE chat_id = :big;
|
||||
SELECT count(*) AS all_messages FROM messages;
|
||||
SELECT pg_size_pretty(pg_relation_size('messages_history_idx')) AS history_index_size;
|
||||
|
||||
\echo '== Первая страница, limit=50 (запрос HISTORY_FIRST приложения, LIMIT limit+1)'
|
||||
EXPLAIN (ANALYZE, BUFFERS) EXECUTE history_first(:big, 51);
|
||||
|
||||
\echo '== Глубокая страница: keyset после 10 000-го сообщения (запрос HISTORY_AFTER приложения)'
|
||||
EXPLAIN (ANALYZE, BUFFERS) EXECUTE history_after(:big, :cursor_at, :cursor_id, 51);
|
||||
|
||||
\echo '== Та же глубокая страница через OFFSET 10000 (не используется приложением, для сравнения)'
|
||||
EXPLAIN (ANALYZE, BUFFERS) EXECUTE history_offset(:big, 51, 10000);
|
||||
|
||||
\echo '== Проверка: keyset и OFFSET возвращают одну и ту же страницу'
|
||||
SELECT (
|
||||
SELECT array_agg(id ORDER BY created_at, id) FROM (
|
||||
SELECT id, created_at FROM messages
|
||||
WHERE chat_id = :big AND (created_at, id) > (:cursor_at, :cursor_id)
|
||||
ORDER BY created_at, id LIMIT 51
|
||||
) AS keyset
|
||||
) = (
|
||||
SELECT array_agg(id ORDER BY created_at, id) FROM (
|
||||
SELECT id, created_at FROM messages
|
||||
WHERE chat_id = :big
|
||||
ORDER BY created_at, id LIMIT 51 OFFSET 10000
|
||||
) AS skipped
|
||||
) AS same_page;
|
||||
|
||||
\echo '== Контрольный план без индекса истории (DROP INDEX внутри транзакции, затем ROLLBACK)'
|
||||
BEGIN;
|
||||
DROP INDEX messages_history_idx;
|
||||
EXPLAIN (ANALYZE, BUFFERS) EXECUTE history_after(:big, :cursor_at, :cursor_id, 51);
|
||||
ROLLBACK;
|
||||
@@ -0,0 +1,40 @@
|
||||
\set ON_ERROR_STOP on
|
||||
\set author '''00000000-0000-7000-8000-00000000a001'''
|
||||
\set big '''00000000-0000-7000-8000-00000000c001'''
|
||||
\set epoch '''2026-01-01T00:00:00Z'''
|
||||
|
||||
INSERT INTO users (id, username, display_name, password_hash, created_at)
|
||||
VALUES (:author, 'seed_author', 'Синтетический автор', '$argon2id$v=19$m=19456,t=2,p=1$c2VlZA$c2VlZA', :epoch)
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
INSERT INTO chats (id, title, created_at)
|
||||
SELECT ('00000000-0000-7000-8000-00000000c' || lpad(to_hex(chat), 3, '0'))::uuid,
|
||||
'Синтетический чат ' || chat, :epoch
|
||||
FROM generate_series(1, 21) AS chat
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
INSERT INTO chat_members (chat_id, user_id, position, chat_created_at)
|
||||
SELECT id, :author, 1, created_at FROM chats WHERE title LIKE 'Синтетический чат %'
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
SELECT ('00000000-0000-7000-8001-' || lpad(to_hex(n), 12, '0'))::uuid, :big, :author,
|
||||
'синтетическое сообщение ' || n, :epoch::timestamptz + n * interval '1 second'
|
||||
FROM generate_series(1, 20000) AS n
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
SELECT ('00000000-0000-7000-8002-' || lpad(to_hex(chat * 1000 + n), 12, '0'))::uuid,
|
||||
('00000000-0000-7000-8000-00000000c' || lpad(to_hex(chat), 3, '0'))::uuid, :author,
|
||||
'фоновое сообщение ' || n, :epoch::timestamptz + n * interval '1 second'
|
||||
FROM generate_series(2, 21) AS chat, generate_series(1, 500) AS n
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
UPDATE chats
|
||||
SET last_message_at = (SELECT max(created_at) FROM messages WHERE messages.chat_id = chats.id)
|
||||
WHERE title LIKE 'Синтетический чат %';
|
||||
|
||||
ANALYZE;
|
||||
|
||||
SELECT count(*) FILTER (WHERE chat_id = :big) AS big_chat_messages, count(*) AS all_messages
|
||||
FROM messages;
|
||||
Executable
+9
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
cd "$(dirname "$0")/.."
|
||||
ids=$(docker compose ps -q "${API_SERVICE:-api}")
|
||||
[ -n "$ids" ] || { echo "no running ${API_SERVICE:-api} replicas" >&2; exit 3; }
|
||||
target=$(docker inspect -f '{{.Id}} {{.Config.Hostname}}' $ids | awk -v t="$TARGET_INSTANCE_ID" '$2 == t {print $1}')
|
||||
[ -n "$target" ] || { echo "no running ${API_SERVICE:-api} replica answers as $TARGET_INSTANCE_ID" >&2; exit 3; }
|
||||
docker stop -t "${STOP_TIMEOUT:-6}" "$target" >/dev/null
|
||||
Reference in New Issue
Block a user