Carry over the lab 4 solution
Bring the application, own tests, Dockerfile, toolchain, compose stack, HAProxy, k6 and RustFS configuration, migrations, hooks, evidence and scenario scripts and the CI job from the lab 4 solution commit d2744bc, merge the lab 4 Makefile targets into the lab 5 skeleton Makefile without touching its targets, and append the lab 4 solution variables to the skeleton .env.example. The package and the application follow the contract version 1.5.0, and the image and the throwaway test containers are tagged lab5 so they do not collide with a lab 4 stand on the same host. This is the starting point of the lab 5 cycle: lint, own tests and the offline skeleton check pass before any lab 5 change. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
1210dd899c
commit
540fcb35ba
Executable
+20
@@ -0,0 +1,20 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
out=${1:-evidence/detection}
|
||||
mkdir -p "$out"
|
||||
scripts/restore.sh
|
||||
replica() { docker compose ps -q "${API_SERVICE:-api}" | head -1; }
|
||||
detect() { python3 scripts/probe.py detect "$@"; }
|
||||
failed=0
|
||||
detect --max 1.5 --within-action --restore scripts/restore.sh --out "$out/sigterm.json" \
|
||||
-- docker stop -t "${STOP_TIMEOUT:-6}" "$(replica)" || failed=1
|
||||
detect --max 1.5 --restore scripts/restore.sh --out "$out/kill.json" \
|
||||
-- docker kill "$(replica)" || failed=1
|
||||
hung=$(replica)
|
||||
detect --max 3.5 --restore "docker unpause $hung && scripts/lb-wait.sh" --out "$out/pause.json" \
|
||||
-- docker pause "$hung" || failed=1
|
||||
db=$(docker compose ps -q postgres)
|
||||
detect --max 1.5 --until 0 --lb-status 503 --restore "docker start $db && scripts/lb-wait.sh" \
|
||||
--out "$out/postgres.json" -- docker stop "$db" || failed=1
|
||||
exit "$failed"
|
||||
Executable
+40
@@ -0,0 +1,40 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
psql() { docker compose exec -T postgres psql -U messenger -d messenger -X -c "$1"; }
|
||||
value() { docker compose exec -T postgres psql -U messenger -d messenger -X -tA -c "$1"; }
|
||||
cli() { docker compose run --rm --no-deps -T s3-cli "$@"; }
|
||||
bucket=${S3_BUCKET:-messenger-attachments}
|
||||
{
|
||||
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"
|
||||
echo "\$ docker compose ps --format '{{.Service}} {{.Status}} {{.Ports}}'"
|
||||
docker compose ps -a --format '{{.Service}} {{.Status}} {{.Ports}}'
|
||||
for table in attachments message_attachments; do
|
||||
echo "\$ psql -c '\\d $table'"
|
||||
psql "\\d $table"
|
||||
done
|
||||
echo "\$ psql: столбцы bytea во всей схеме (хеши токенов и отпечатки, не файлы)"
|
||||
psql "SELECT table_name, column_name FROM information_schema.columns WHERE table_schema = 'public' AND data_type = 'bytea' ORDER BY 1, 2"
|
||||
echo "\$ psql: вложения по статусу и максимальный размер строки attachments"
|
||||
psql "SELECT status, count(*), max(pg_column_size(a.*)) AS max_row_bytes FROM attachments a GROUP BY status"
|
||||
echo "\$ rc object list --recursive root/$bucket/attachments/ | count"
|
||||
cli object list --recursive "root/$bucket/attachments/" --json | grep -c '"key"' || true
|
||||
echo "\$ rc object list root/$bucket/ (первые 3 объекта, ключи назначены сервером)"
|
||||
cli object list --recursive "root/$bucket/attachments/" | head -3
|
||||
row=$(value "SELECT object_key || ' ' || sha256 || ' ' || size_bytes FROM attachments WHERE status = 'ready' ORDER BY created_at DESC LIMIT 1")
|
||||
key=${row%% *}
|
||||
echo "\$ сверка вне API: sha256 и размер последнего ready-вложения в PostgreSQL против байтов объекта в bucket"
|
||||
echo "postgres: ${row#* }"
|
||||
echo "s3: $(cli sh -c "rc object show root/$bucket/$key | sha256sum | cut -d' ' -f1") $(cli sh -c "rc object show root/$bucket/$key | wc -c" | tr -d ' ')"
|
||||
echo "\$ docker inspect api: ReadonlyRootfs, tmpfs, volumes/bind mounts"
|
||||
for id in $(docker compose ps -q api); do
|
||||
docker inspect -f '{{.Config.Hostname}} ro={{.HostConfig.ReadonlyRootfs}} tmpfs={{json .HostConfig.Tmpfs}} mounts={{range .Mounts}}{{.Type}}:{{.Destination}} {{end}}' "$id"
|
||||
done
|
||||
echo "\$ docker inspect s3: volume данных"
|
||||
docker inspect -f '{{range .Mounts}}{{.Type}}:{{.Name}}:{{.Destination}} {{end}}' "$(docker compose ps -q s3)"
|
||||
echo "\$ секреты сервисов: root-ключи S3 только у s3/s3-init/s3-cli"
|
||||
docker compose config --format json | python3 -c 'import json,sys
|
||||
for name, service in sorted(json.load(sys.stdin)["services"].items()):
|
||||
print(name, sorted(s["source"] for s in service.get("secrets", [])))'
|
||||
} > evidence/storage-layout.txt
|
||||
cat evidence/storage-layout.txt
|
||||
Executable
+24
@@ -0,0 +1,24 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
cli() { docker compose exec -T valkey sh -c "REDISCLI_AUTH=\"\$(sed -n 's/^requirepass //p' /run/secrets/valkey.conf)\" valkey-cli $1"; }
|
||||
psql() { docker compose exec -T postgres psql -U messenger -d messenger -tA -c "$1"; }
|
||||
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"
|
||||
echo "\$ valkey-cli config get appendonly appendfsync maxmemory maxmemory-policy"
|
||||
cli "config get appendonly" ; cli "config get appendfsync"; cli "config get maxmemory"; cli "config get maxmemory-policy"
|
||||
echo "\$ valkey-cli info persistence | grep aof_enabled,aof_last_write_status"
|
||||
cli "info persistence" | grep -E '^aof_enabled|^aof_last_write_status'
|
||||
echo "\$ valkey-cli dbsize; info stats | grep keyspace"
|
||||
cli dbsize
|
||||
cli "info stats" | grep -E '^keyspace_(hits|misses)'
|
||||
key=$(cli "--scan --pattern session:* --count 1000" | head -1)
|
||||
hash=${key#session:}
|
||||
echo "\$ сравнение срока ключа кэша и срока сессии для одного ключа (значение и токен не печатаются)"
|
||||
echo "valkey pexpiretime, ms: $(cli "pexpiretime $key")"
|
||||
echo "postgres expires_at, ms: $(psql "SELECT (extract(epoch FROM expires_at) * 1000)::bigint FROM sessions WHERE token_hash = decode('$hash', 'hex')")"
|
||||
echo "\$ количество отметок отзыва и закэшированных сессий"
|
||||
revoked=0; cached=0
|
||||
for k in $(cli "--scan --pattern session:* --count 1000"); do
|
||||
if [ "$(cli "get $k")" = "-" ]; then revoked=$((revoked + 1)); else cached=$((cached + 1)); fi
|
||||
done
|
||||
echo "tombstones: $revoked, cached sessions: $cached"
|
||||
Executable
+49
@@ -0,0 +1,49 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
base=${1:-http://$(docker compose port lb 8080)}
|
||||
stats=http://$(docker compose port lb 8404)/stats\;csv
|
||||
header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"; }
|
||||
{
|
||||
header
|
||||
echo "\$ docker compose --env-file .env config --quiet && echo valid"
|
||||
docker compose --env-file .env config --quiet && echo valid
|
||||
echo "\$ docker compose ps --format '{{.Service}} {{.Name}} {{.Status}} {{.Ports}}'"
|
||||
docker compose ps --format '{{.Service}} {{.Name}} {{.Status}} {{.Ports}}'
|
||||
echo "\$ docker compose config | grep -c container_name"
|
||||
docker compose config | grep -c container_name || true
|
||||
echo "\$ docker port <каждая реплика api> # реплики не публикуют порт"
|
||||
for id in $(docker compose ps -q api); do echo "$(docker inspect -f '{{.Config.Hostname}}' "$id"): [$(docker port "$id")]"; done
|
||||
echo "\$ docker inspect: hostname = X-Instance-Id, сети, лимиты CPU (nano) и памяти"
|
||||
for id in $(docker compose ps -q); do
|
||||
docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}} {{.Config.Hostname}} {{.Name}} nets={{range $k, $v := .NetworkSettings.Networks}}{{$k}},{{end}} cpus={{.HostConfig.NanoCpus}} mem={{.HostConfig.Memory}} user={{.Config.User}} ro={{.HostConfig.ReadonlyRootfs}}' "$id"
|
||||
done | sort
|
||||
echo "\$ curl LB stats: backend api, server status check"
|
||||
curl -fsS "$stats" | python3 -c 'import csv,sys
|
||||
for r in csv.DictReader(sys.stdin.read().lstrip("# ").splitlines()):
|
||||
if r["pxname"] == "api" and r["status"] != "MAINT (resolution)":
|
||||
print(r["svname"], r["status"], r["check_status"], r["check_code"])'
|
||||
echo "\$ for i in 1..6: curl -D - $base/health/live | X-Instance-Id"
|
||||
for _ in 1 2 3 4 5 6; do curl -fsS -D - -o /dev/null "$base/health/live" | tr -d '\r' | grep -i '^x-instance-id'; done
|
||||
} > evidence/stack.txt
|
||||
{
|
||||
header
|
||||
echo "host: $(sysctl -n machdep.cpu.brand_string 2>/dev/null || uname -m), $(sysctl -n hw.ncpu 2>/dev/null) CPU, $(($(sysctl -n hw.memsize 2>/dev/null || echo 0) / 1073741824)) GiB RAM, $(sw_vers -productName 2>/dev/null) $(sw_vers -productVersion 2>/dev/null)"
|
||||
echo "docker: $(docker version -f '{{.Server.Version}}'), VM: $(docker info -f '{{.NCPU}}') CPU, $(docker info -f '{{.MemTotal}}') bytes"
|
||||
echo "compose: $(docker compose version --short)"
|
||||
echo "images:"
|
||||
grep -h -E 'image:|@sha256' compose.yaml Dockerfile | sed 's/^ */ /'
|
||||
echo "python in image: $(docker exec "$(docker compose ps -q api | head -1)" python -V)"
|
||||
echo "haproxy: $(docker compose exec -T lb haproxy -v | head -1)"
|
||||
echo "rustfs: $(docker compose exec -T s3 rustfs --version 2>/dev/null | head -1)"
|
||||
echo "packages (uv.lock):"
|
||||
for name in fastapi uvicorn uvicorn-worker gunicorn asyncpg argon2-cffi pydantic pydantic-settings dishka structlog redis aiobotocore botocore aiohttp pillow; do
|
||||
printf ' %s %s\n' "$name" "$(sed -n "/^name = \"$name\"$/{n;s/version = \"\(.*\)\"/\1/p;}" uv.lock)"
|
||||
done
|
||||
echo "process users (service uid command):"
|
||||
for id in $(docker compose ps -q); do
|
||||
service=$(docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}}' "$id")
|
||||
docker top "$id" -eo pid,uid,comm | awk -v s="$service" 'NR > 1 {print " " s, $2, $3}'
|
||||
done | sort -u
|
||||
} > evidence/environment.txt
|
||||
cat evidence/stack.txt evidence/environment.txt
|
||||
Executable
+44
@@ -0,0 +1,44 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
psql() { docker compose exec -T postgres psql -U messenger -d messenger -c "$1"; }
|
||||
header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"; }
|
||||
counts="SELECT (SELECT count(*) FROM users) users, (SELECT count(*) FROM sessions) sessions, (SELECT count(*) FROM chats) chats, (SELECT count(*) FROM chat_members) members, (SELECT count(*) FROM messages) messages, (SELECT count(*) FROM idempotency_keys) keys, (SELECT count(*) FROM attachments) attachments, (SELECT count(*) FROM message_attachments) links, (SELECT count(*) FROM schema_migrations) versions"
|
||||
kdf="SELECT substring(password_hash from '^[$]argon2id[$]v=[0-9]+[$]m=[0-9]+,t=[0-9]+,p=[0-9]+[$]') AS kdf_prefix, count(*) AS users, min(length(password_hash)) AS hash_length FROM users GROUP BY 1"
|
||||
tokens="SELECT octet_length(token_hash) AS stored_bytes, count(*) AS sessions FROM sessions GROUP BY 1"
|
||||
{
|
||||
header
|
||||
for table in users sessions chats chat_members messages idempotency_keys attachments message_attachments schema_migrations; do
|
||||
echo "\$ psql -c '\\d+ $table'"
|
||||
psql "\\d+ $table"
|
||||
done
|
||||
echo "\$ psql -c 'SELECT version, left(checksum, 12), applied_at FROM schema_migrations'"
|
||||
psql "SELECT version, left(checksum, 12) AS checksum, applied_at FROM schema_migrations"
|
||||
} > evidence/schema.txt
|
||||
{
|
||||
header
|
||||
echo "\$ psql -c \"$counts\""
|
||||
psql "$counts"
|
||||
echo "\$ make migrate"
|
||||
docker compose run --rm migrate 2>/dev/null
|
||||
echo "\$ psql -c \"$counts\""
|
||||
psql "$counts"
|
||||
} > evidence/migrations.txt
|
||||
{
|
||||
header
|
||||
echo "\$ psql -c \"$kdf\""
|
||||
psql "$kdf"
|
||||
echo "\$ psql -c \"$tokens\""
|
||||
psql "$tokens"
|
||||
} > evidence/kdf.txt
|
||||
{
|
||||
header
|
||||
echo "\$ git ls-files secrets .env backups volumes"
|
||||
git ls-files secrets .env backups volumes
|
||||
echo "(пусто: секретов, .env, дампов и volumes в Git нет)"
|
||||
echo "\$ git check-ignore -v secrets/database_url secrets/postgres_password secrets/s3_root_password secrets/s3_secret_access_key .env backups/"
|
||||
git check-ignore -v secrets/database_url secrets/postgres_password secrets/s3_root_password secrets/s3_secret_access_key .env backups/
|
||||
echo "\$ docker volume ls --filter name=mtusi2026lab5"
|
||||
docker volume ls --filter name=mtusi2026lab5
|
||||
} > evidence/secrets.txt
|
||||
echo "evidence/{schema,migrations,kdf,secrets}.txt обновлены"
|
||||
Executable
+19
@@ -0,0 +1,19 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
out=${1:-evidence/failover-each}
|
||||
rm -rf "$out"
|
||||
mkdir -p "$out"
|
||||
scripts/restore.sh
|
||||
hosts=$(docker inspect -f '{{.Config.Hostname}}' $(docker compose ps -q "${API_SERVICE:-api}"))
|
||||
failed=0
|
||||
set --
|
||||
for host in $hosts; do
|
||||
TARGET_INSTANCE_ID=$host python3 scripts/probe.py load --expect dip --out "$out/$host" \
|
||||
-- sh -c "scripts/stop-one.sh && sleep ${HOLD:-10} && scripts/restore.sh" || failed=1
|
||||
scripts/restore.sh
|
||||
set -- "$@" "$out/$host/result.json"
|
||||
done
|
||||
python3 -c 'import json,sys; print(json.dumps([json.load(open(p)) for p in sys.argv[1:]], ensure_ascii=False, indent=2))' "$@" > "$out/summary.tmp"
|
||||
mv "$out/summary.tmp" "$out/summary.json"
|
||||
exit "$failed"
|
||||
Executable
+28
@@ -0,0 +1,28 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
replicas() {
|
||||
docker inspect -f '{{.Config.Hostname}} {{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}' $(docker compose ps -aq "${API_SERVICE:-api}")
|
||||
}
|
||||
tmp=evidence/failover.tmp
|
||||
since=$(date -u +%FT%TZ)
|
||||
{
|
||||
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"
|
||||
echo "== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt"
|
||||
replicas
|
||||
echo "\$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh"
|
||||
make --no-print-directory test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL="${1:-http://localhost:8080}"
|
||||
echo "== after: api replicas"
|
||||
replicas
|
||||
} > "$tmp" 2>&1
|
||||
stopped=$(replicas | awk '$2 == "exited" {print $1}')
|
||||
{
|
||||
echo "== stopped replica: $stopped; shutdown: signal, drain window with readiness 503, close, exit"
|
||||
docker logs --since "$since" "$stopped" 2>&1 | grep -E 'Handling signal|Draining|"status":503|Shutting down|Waiting for conn|Finished server|service.stopped|SIGTERM' | tail -16
|
||||
echo "\$ scripts/restore.sh"
|
||||
scripts/restore.sh
|
||||
echo "== after restore"
|
||||
replicas
|
||||
} >> "$tmp" 2>&1
|
||||
mv "$tmp" "evidence/failover-$stopped.txt"
|
||||
cat "evidence/failover-$stopped.txt"
|
||||
Executable
+10
@@ -0,0 +1,10 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
out=${1:?usage: k6.sh <output dir> [k6 run args]}
|
||||
shift
|
||||
mkdir -p "$out"
|
||||
out=$(cd "$out" && pwd)
|
||||
cd "$(dirname "$0")/.."
|
||||
exec docker compose --profile bench run --rm --no-deps -T -u "$(id -u):$(id -g)" -v "$out:/out" \
|
||||
-e RATE -e WARMUP -e DURATION -e USERS -e READ_RATIO -e VUS -e MAX_VUS -e RUN_ID \
|
||||
k6 run --quiet "$@" "/scripts/${K6_SCRIPT:-messages.js}"
|
||||
Executable
+230
@@ -0,0 +1,230 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import hashlib
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
client = __import__("client")
|
||||
Client, Failure, check = client.Client, client.Failure, client.check
|
||||
FIXTURE = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
|
||||
|
||||
def compose(*args, stdin=None):
|
||||
cmd = ["docker", "compose", *args]
|
||||
return subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, text=True, check=False)
|
||||
|
||||
|
||||
def upload(c, actor, chat, expected=201):
|
||||
path = f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png"
|
||||
schema = "Attachment" if expected == 201 else None
|
||||
return c.call(
|
||||
"POST",
|
||||
path,
|
||||
expected,
|
||||
actor=actor,
|
||||
raw=FIXTURE,
|
||||
headers={"Content-Type": "image/png"},
|
||||
schema=schema,
|
||||
)
|
||||
|
||||
|
||||
def content(c, actor, attachment, expected=200):
|
||||
return c.call("GET", f"/api/v1/attachments/{attachment['id']}/content", expected, actor=actor)
|
||||
|
||||
|
||||
def timed(fn):
|
||||
started = time.monotonic()
|
||||
try:
|
||||
response = fn()
|
||||
except Failure as exc:
|
||||
return {
|
||||
"status": "failure",
|
||||
"seconds": round(time.monotonic() - started, 3),
|
||||
"error": str(exc),
|
||||
}
|
||||
seconds = round(time.monotonic() - started, 3)
|
||||
return {
|
||||
"status": response["status"],
|
||||
"seconds": seconds,
|
||||
"instance": response["headers"].get("X-Instance-Id"),
|
||||
}
|
||||
|
||||
|
||||
def wait(fn, seconds):
|
||||
deadline = time.monotonic() + seconds
|
||||
while True:
|
||||
try:
|
||||
return fn()
|
||||
except Failure:
|
||||
if time.monotonic() > deadline:
|
||||
raise
|
||||
time.sleep(0.2)
|
||||
|
||||
|
||||
def failover_upload(c, args):
|
||||
alice, bob, _, chat = c.bootstrap()
|
||||
sent = upload(c, alice, chat)
|
||||
att, target = sent["json"], sent["headers"]["X-Instance-Id"]
|
||||
started = time.monotonic()
|
||||
hook = subprocess.run(
|
||||
[str(ROOT / "scripts/stop-one.sh")],
|
||||
cwd=ROOT,
|
||||
env=dict(os.environ, TARGET_INSTANCE_ID=target),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
check(hook.returncode == 0, f"stop-one.sh failed: {hook.stderr.strip()}")
|
||||
hook_seconds = time.monotonic() - started
|
||||
got = wait(lambda: content(c, bob, att), 15)
|
||||
recovered = time.monotonic() - started
|
||||
served = got["headers"].get("X-Instance-Id")
|
||||
check(served != target, "download served by the stopped replica")
|
||||
digest = hashlib.sha256(got["bytes"]).hexdigest()
|
||||
check(
|
||||
digest == att["sha256"] == hashlib.sha256(FIXTURE).hexdigest(),
|
||||
"sha256 mismatch after failover",
|
||||
)
|
||||
if args.restore:
|
||||
compose("up", "-d", "--wait", "--no-recreate", "--no-deps", args.service)
|
||||
return {
|
||||
"scenario": "failover-upload",
|
||||
"upload_instance": target,
|
||||
"download_instance": served,
|
||||
"sha256_equal": True,
|
||||
"hook_seconds": round(hook_seconds, 3),
|
||||
"download_ok_after_seconds": round(recovered, 3),
|
||||
"restored": args.restore,
|
||||
}
|
||||
|
||||
|
||||
def s3_outage(c, args):
|
||||
alice, bob, _, chat = c.bootstrap()
|
||||
att = upload(c, alice, chat)["json"]
|
||||
check(
|
||||
compose("stop", "-t", "10", args.s3_service).returncode == 0, "docker compose stop failed"
|
||||
)
|
||||
try:
|
||||
during = {
|
||||
"upload": timed(lambda: upload(c, alice, chat, [503])),
|
||||
"metadata": timed(
|
||||
lambda: c.call("GET", f"/api/v1/attachments/{att['id']}", [200, 503], actor=bob)
|
||||
),
|
||||
"download": timed(lambda: content(c, bob, att, 503)),
|
||||
"text_message": timed(lambda: c.message(alice, chat, "S3 недоступен, текст работает")),
|
||||
"health_ready": timed(lambda: c.call("GET", "/health/ready", [200, 503])),
|
||||
}
|
||||
finally:
|
||||
started = time.monotonic()
|
||||
compose("up", "-d", "--wait", "--no-deps", "--no-recreate", args.s3_service)
|
||||
recovered = round(time.monotonic() - started, 3)
|
||||
got = wait(lambda: content(c, bob, att), 30)
|
||||
check(got["bytes"] == FIXTURE, "attachment changed across S3 outage")
|
||||
expected = {"upload": 503, "download": 503, "text_message": 201}
|
||||
return {
|
||||
"scenario": "s3-outage",
|
||||
"during_outage": during,
|
||||
"expected": expected,
|
||||
"as_expected": all(during[k]["status"] == v for k, v in expected.items()),
|
||||
"s3_back_healthy_seconds": recovered,
|
||||
"download_after_recovery": "sha256 equal",
|
||||
}
|
||||
|
||||
|
||||
def abort_upload(c, args):
|
||||
alice, _, _, chat = c.bootstrap()
|
||||
name = f"abort-{uuid.uuid4().hex}.png"
|
||||
parts = urlsplit(c.base)
|
||||
factory = http.client.HTTPSConnection if parts.scheme == "https" else http.client.HTTPConnection
|
||||
conn = factory(parts.netloc, timeout=10)
|
||||
conn.putrequest("POST", f"/api/v1/chats/{chat['id']}/attachments?filename={name}")
|
||||
headers = {
|
||||
"Authorization": f"Bearer {alice['token']}",
|
||||
"Content-Type": "image/png",
|
||||
"Content-Length": str(args.declared),
|
||||
"Accept": "application/json",
|
||||
}
|
||||
for key, value in headers.items():
|
||||
conn.putheader(key, value)
|
||||
conn.endheaders()
|
||||
conn.send(FIXTURE[:8] + os.urandom(args.sent - 8))
|
||||
conn.close()
|
||||
time.sleep(args.settle)
|
||||
user, db = os.environ.get("PGUSER", "messenger"), os.environ.get("PGDATABASE", "messenger")
|
||||
sql = "SELECT status, count(*) FROM attachments WHERE filename = :'name' GROUP BY status;\n"
|
||||
out = compose(
|
||||
"exec",
|
||||
"-T",
|
||||
"postgres",
|
||||
"psql",
|
||||
"-qAt",
|
||||
"-F",
|
||||
" ",
|
||||
"-U",
|
||||
user,
|
||||
"-d",
|
||||
db,
|
||||
"-v",
|
||||
f"name={name}",
|
||||
stdin=sql,
|
||||
)
|
||||
rows = out.stdout.split()
|
||||
counts = dict(zip(rows[::2], (int(x) for x in rows[1::2])))
|
||||
alive = c.call("GET", "/api/v1/users/me", actor=alice, schema="User")["status"]
|
||||
return {
|
||||
"scenario": "abort-upload",
|
||||
"declared_bytes": args.declared,
|
||||
"sent_bytes": args.sent,
|
||||
"settle_seconds": args.settle,
|
||||
"rows_by_status": counts,
|
||||
"ready_rows": counts.get("ready", 0),
|
||||
"api_still_serves": alive == 200,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(
|
||||
description="Lab4 failure scenarios over the public HTTP contract plus docker compose"
|
||||
)
|
||||
p.add_argument("scenario", choices=["failover-upload", "s3-outage", "abort-upload"])
|
||||
p.add_argument("--base-url", default=None)
|
||||
p.add_argument("--ca-file", default=None)
|
||||
p.add_argument("--timeout", type=float, default=15)
|
||||
p.add_argument("--service", default="api")
|
||||
p.add_argument("--s3-service", default="s3")
|
||||
p.add_argument(
|
||||
"--restore", action="store_true", help="failover-upload: start the stopped replica again"
|
||||
)
|
||||
p.add_argument("--declared", type=int, default=5 * 1024 * 1024)
|
||||
p.add_argument("--sent", type=int, default=256 * 1024)
|
||||
p.add_argument("--settle", type=float, default=3)
|
||||
p.add_argument("--output", metavar="FILE")
|
||||
args = p.parse_args()
|
||||
c = Client(args.base_url, args.ca_file, timeout=args.timeout)
|
||||
run = {
|
||||
"failover-upload": failover_upload,
|
||||
"s3-outage": s3_outage,
|
||||
"abort-upload": abort_upload,
|
||||
}[args.scenario]
|
||||
try:
|
||||
report = {"ok": True, **run(c, args)}
|
||||
except Failure as exc:
|
||||
report = {"scenario": args.scenario, "ok": False, "error": str(exc)}
|
||||
text = json.dumps(report, ensure_ascii=False, indent=2)
|
||||
if args.output:
|
||||
Path(args.output).write_text(text + "\n")
|
||||
print(text)
|
||||
sys.exit(0 if report["ok"] else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+352
@@ -0,0 +1,352 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import base64
|
||||
import hashlib
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import struct
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
MAX_UPLOAD = 10 * 1024 * 1024
|
||||
COLUMNS = (
|
||||
"id",
|
||||
"chat_id",
|
||||
"owner_id",
|
||||
"filename",
|
||||
"content_type",
|
||||
"size_bytes",
|
||||
"sha256",
|
||||
"width",
|
||||
"height",
|
||||
"object_key",
|
||||
"status",
|
||||
)
|
||||
|
||||
|
||||
def chunk(tag, data):
|
||||
return (
|
||||
struct.pack(">I", len(data))
|
||||
+ tag
|
||||
+ data
|
||||
+ struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF)
|
||||
)
|
||||
|
||||
|
||||
def png(width, height):
|
||||
raw = b"".join(b"\x00" + os.urandom(width * 3) for _ in range(height))
|
||||
header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
|
||||
return (
|
||||
b"\x89PNG\r\n\x1a\n"
|
||||
+ chunk(b"IHDR", header)
|
||||
+ chunk(b"IDAT", zlib.compress(raw, 1))
|
||||
+ chunk(b"IEND", b"")
|
||||
)
|
||||
|
||||
|
||||
def sized_png(target):
|
||||
height = max(1, target // (1024 * 3 + 1))
|
||||
body = png(1024, height)
|
||||
while len(body) > min(target, MAX_UPLOAD) and height > 1:
|
||||
height -= max(1, (len(body) - target) // 3073 + 1)
|
||||
body = png(1024, height)
|
||||
return body
|
||||
|
||||
|
||||
def run(cmd, stdin=None):
|
||||
out = subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, check=False)
|
||||
if out.returncode:
|
||||
raise SystemExit(
|
||||
f"command failed: {' '.join(cmd[:6])}\n{out.stderr.decode(errors='replace')[-500:]}"
|
||||
)
|
||||
return out.stdout.decode()
|
||||
|
||||
|
||||
def psql(sql, stdin=None):
|
||||
user, db = os.environ.get("PGUSER", "messenger"), os.environ.get("PGDATABASE", "messenger")
|
||||
psql = [
|
||||
"docker",
|
||||
"compose",
|
||||
"exec",
|
||||
"-T",
|
||||
"postgres",
|
||||
"psql",
|
||||
"-v",
|
||||
"ON_ERROR_STOP=1",
|
||||
"-qAt",
|
||||
"-F",
|
||||
" ",
|
||||
]
|
||||
return run([*psql, "-U", user, "-d", db, "-c", sql], stdin)
|
||||
|
||||
|
||||
def direct(args):
|
||||
bucket = os.environ.get("S3_BUCKET", "messenger-attachments")
|
||||
prefix = os.environ.get("S3_PREFIX", "attachments/")
|
||||
if args.chat_id and args.owner_id:
|
||||
chat_id, owner_id = args.chat_id, args.owner_id
|
||||
else:
|
||||
found = psql(
|
||||
"SELECT chat_id, owner_id FROM attachments ORDER BY created_at LIMIT 1"
|
||||
).split()
|
||||
if len(found) != 2:
|
||||
raise SystemExit(
|
||||
"no attachment rows yet: upload one via API or pass --chat-id and --owner-id"
|
||||
)
|
||||
chat_id, owner_id = found
|
||||
rows = []
|
||||
started = time.monotonic()
|
||||
with tempfile.TemporaryDirectory(prefix="lab4-seed-") as tmp:
|
||||
Path(tmp).chmod(0o755)
|
||||
for i in range(args.count):
|
||||
aid = str(uuid.uuid4())
|
||||
body = png(8, 8)
|
||||
path = Path(tmp) / aid
|
||||
path.write_bytes(body)
|
||||
path.chmod(0o644)
|
||||
digest = hashlib.sha256(body).hexdigest()
|
||||
rows.append(
|
||||
f"{aid},{chat_id},{owner_id},seed-{i}.png,image/png,{len(body)},{digest},8,8,{prefix}{aid},ready"
|
||||
)
|
||||
generated = time.monotonic()
|
||||
cli = [
|
||||
"docker",
|
||||
"compose",
|
||||
"run",
|
||||
"--rm",
|
||||
"--no-deps",
|
||||
"-T",
|
||||
"-v",
|
||||
f"{tmp}:/seed:ro",
|
||||
"s3-cli",
|
||||
]
|
||||
run([*cli, "put", "--recursive", "--no-progress", "/seed/", f"root/{bucket}/{prefix}"])
|
||||
uploaded = time.monotonic()
|
||||
columns = ", ".join(COLUMNS)
|
||||
psql(
|
||||
f"COPY attachments ({columns}) FROM STDIN WITH (FORMAT csv)",
|
||||
"".join(f"{r}\n" for r in rows).encode(),
|
||||
)
|
||||
seconds = {
|
||||
"generate": round(generated - started, 3),
|
||||
"s3_put": round(uploaded - generated, 3),
|
||||
"pg_copy": round(time.monotonic() - uploaded, 3),
|
||||
}
|
||||
return {"mode": "direct", "count": args.count, "seconds": seconds}
|
||||
|
||||
|
||||
class Http:
|
||||
def __init__(self, base, timeout):
|
||||
parts = urlsplit(base)
|
||||
self.cls = (
|
||||
http.client.HTTPSConnection if parts.scheme == "https" else http.client.HTTPConnection
|
||||
)
|
||||
self.host, self.timeout, self.local = parts.netloc, timeout, threading.local()
|
||||
|
||||
def call(self, method, path, body=None, headers=None, token=None):
|
||||
conn = getattr(self.local, "conn", None) or self.cls(self.host, timeout=self.timeout)
|
||||
self.local.conn = conn
|
||||
h = {"Accept": "application/json"}
|
||||
if token:
|
||||
h["Authorization"] = "Bearer " + token
|
||||
if isinstance(body, dict):
|
||||
body, h["Content-Type"] = json.dumps(body).encode(), "application/json"
|
||||
h.update(headers or {})
|
||||
try:
|
||||
conn.request(method, path, body=body, headers=h)
|
||||
response = conn.getresponse()
|
||||
data = response.read()
|
||||
except (OSError, http.client.HTTPException):
|
||||
conn.close()
|
||||
self.local.conn = None
|
||||
return 0, {}, b""
|
||||
return response.status, {k.lower(): v for k, v in response.getheaders()}, data
|
||||
|
||||
|
||||
def login(api, label):
|
||||
name = f"seed_{label}_{uuid.uuid4().hex[:12]}"
|
||||
password = "Seed-" + uuid.uuid4().hex
|
||||
body = {"username": name, "display_name": label, "password": password}
|
||||
status, _, data = api.call("POST", "/api/v1/users", body)
|
||||
if status != 201:
|
||||
raise SystemExit(f"user create failed: {status}")
|
||||
user = json.loads(data)
|
||||
basic = base64.b64encode(f"{name}:{password}".encode()).decode()
|
||||
status, _, data = api.call(
|
||||
"POST", "/api/v1/auth/sessions", headers={"Authorization": f"Basic {basic}"}
|
||||
)
|
||||
if status != 201:
|
||||
raise SystemExit(f"login failed: {status}")
|
||||
return user["id"], json.loads(data)["token"]
|
||||
|
||||
|
||||
def mib(text):
|
||||
m = re.match(r"([\d.]+)\s*([KMG]i?B)", text)
|
||||
if not m:
|
||||
return 0.0
|
||||
scale = {"KiB": 1 / 1024, "KB": 1 / 1024, "GiB": 1024, "GB": 1024}
|
||||
return float(m.group(1)) * scale.get(m.group(2), 1)
|
||||
|
||||
|
||||
def sampler(ids, peaks, stop):
|
||||
while not stop.is_set():
|
||||
cmd = ["docker", "stats", "--no-stream", "--format", "{{.ID}} {{.MemUsage}}", *ids]
|
||||
out = subprocess.run(cmd, capture_output=True, text=True, check=False).stdout
|
||||
for line in out.splitlines():
|
||||
cid, usage = line.split(" ", 1)
|
||||
peaks[cid] = max(peaks.get(cid, 0.0), mib(usage))
|
||||
|
||||
|
||||
def cgroup(cid, name):
|
||||
cmd = ["docker", "exec", cid, "cat", f"/sys/fs/cgroup/{name}"]
|
||||
return subprocess.run(cmd, capture_output=True, text=True, check=False).stdout.strip()
|
||||
|
||||
|
||||
def percentile(values, q):
|
||||
return (
|
||||
round(sorted(values)[min(len(values) - 1, int(q * len(values)))] * 1000, 1)
|
||||
if values
|
||||
else None
|
||||
)
|
||||
|
||||
|
||||
def over_http(args):
|
||||
api = Http(args.base_url, args.timeout)
|
||||
alice, token = login(api, "alice")
|
||||
bob, _ = login(api, "bob")
|
||||
status, _, data = api.call(
|
||||
"POST", "/api/v1/chats", {"title": "seed", "member_ids": [alice, bob]}, token=token
|
||||
)
|
||||
if status != 201:
|
||||
raise SystemExit(f"chat create failed: {status}")
|
||||
chat = json.loads(data)["id"]
|
||||
fixed = sized_png(args.size) if args.size else None
|
||||
counter, lock, results = iter(range(args.count)), threading.Lock(), []
|
||||
ids = run(["docker", "compose", "ps", "-q", args.service]).split() if args.sample else []
|
||||
idle = {cid: round(int(cgroup(cid, "memory.current") or 0) / 1048576, 1) for cid in ids}
|
||||
peaks, stop = {}, threading.Event()
|
||||
watcher = threading.Thread(target=sampler, args=(ids, peaks, stop), daemon=True)
|
||||
if ids:
|
||||
watcher.start()
|
||||
|
||||
def worker():
|
||||
while True:
|
||||
with lock:
|
||||
i = next(counter, None)
|
||||
if i is None:
|
||||
return
|
||||
body = fixed or png(8, 8)
|
||||
began = time.monotonic()
|
||||
path = f"/api/v1/chats/{chat}/attachments?filename=seed-{i}.png"
|
||||
status, headers, data = api.call(
|
||||
"POST", path, body, {"Content-Type": "image/png"}, token
|
||||
)
|
||||
created = json.loads(data)["id"] if status == 201 else None
|
||||
results.append(
|
||||
(status, time.monotonic() - began, headers.get("x-instance-id", ""), created)
|
||||
)
|
||||
|
||||
started = time.monotonic()
|
||||
threads = [threading.Thread(target=worker) for _ in range(args.concurrency)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
elapsed = time.monotonic() - started
|
||||
stop.set()
|
||||
if ids:
|
||||
watcher.join()
|
||||
created = [r[3] for r in results if r[3]]
|
||||
for start in range(0, min(args.link, len(created)), 10):
|
||||
body = {"text": "", "attachment_ids": created[start : min(start + 10, args.link)]}
|
||||
api.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat}/messages",
|
||||
body,
|
||||
{"Idempotency-Key": uuid.uuid4().hex},
|
||||
token,
|
||||
)
|
||||
if args.ids_out:
|
||||
Path(args.ids_out).write_text("".join(f"{i}\n" for i in created))
|
||||
codes, instances = {}, {}
|
||||
for status, _, instance, _ in results:
|
||||
codes[str(status)] = codes.get(str(status), 0) + 1
|
||||
instances[instance] = instances.get(instance, 0) + 1
|
||||
ok = [r[1] for r in results if r[0] == 201]
|
||||
memory = {}
|
||||
for cid in ids:
|
||||
peak = cgroup(cid, "memory.peak")
|
||||
events = dict(line.split() for line in cgroup(cid, "memory.events").splitlines())
|
||||
memory[cid[:12]] = {
|
||||
"idle_mib": idle[cid],
|
||||
"docker_stats_peak_mib": round(peaks.get(cid[:12], 0.0), 1),
|
||||
"cgroup_memory_peak_mib": round(int(peak) / 1048576, 1) if peak.isdigit() else None,
|
||||
"oom_kill": int(events.get("oom_kill", 0)),
|
||||
}
|
||||
return {
|
||||
"mode": "http",
|
||||
"count": args.count,
|
||||
"concurrency": args.concurrency,
|
||||
"body_bytes": len(fixed) if fixed else "~200 (unique 8x8 PNG)",
|
||||
"status_codes": codes,
|
||||
"instances": len([i for i in instances if i]),
|
||||
"linked_in_messages": min(args.link, len(created)),
|
||||
"elapsed_seconds": round(elapsed, 3),
|
||||
"successful_rps": round(len(ok) / elapsed, 2) if elapsed else None,
|
||||
"latency_ms": {
|
||||
"p50": percentile(ok, 0.5),
|
||||
"p95": percentile(ok, 0.95),
|
||||
"p99": percentile(ok, 0.99),
|
||||
"max": percentile(ok, 1.0),
|
||||
},
|
||||
"memory": memory,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(
|
||||
description="Lab4 attachment volume generator: HTTP uploads or direct S3+PG seeding"
|
||||
)
|
||||
p.add_argument("--mode", choices=["http", "direct"], default="http")
|
||||
p.add_argument("--count", type=int, default=1000)
|
||||
p.add_argument("--concurrency", type=int, default=16)
|
||||
p.add_argument(
|
||||
"--size",
|
||||
type=int,
|
||||
default=0,
|
||||
help="http: bytes per upload (one PNG reused); 0 = unique 8x8 PNGs",
|
||||
)
|
||||
p.add_argument(
|
||||
"--link", type=int, default=0, help="http: reference the first N attachments from messages"
|
||||
)
|
||||
p.add_argument(
|
||||
"--sample",
|
||||
action="store_true",
|
||||
help="http: sample docker stats and memory.peak of API containers",
|
||||
)
|
||||
p.add_argument("--service", default="api")
|
||||
p.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080"))
|
||||
p.add_argument("--timeout", type=float, default=60)
|
||||
p.add_argument("--ids-out", metavar="FILE")
|
||||
p.add_argument("--chat-id")
|
||||
p.add_argument("--owner-id")
|
||||
p.add_argument("--output", metavar="FILE")
|
||||
args = p.parse_args()
|
||||
report = direct(args) if args.mode == "direct" else over_http(args)
|
||||
text = json.dumps(report, ensure_ascii=False, indent=2)
|
||||
if args.output:
|
||||
Path(args.output).write_text(text + "\n")
|
||||
print(text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+11
@@ -0,0 +1,11 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
service=${API_SERVICE:-api}
|
||||
want=${1:-$(docker compose ps -q "$service" | wc -l | tr -d ' ')}
|
||||
url=${LB_STATS_URL:-http://$(docker compose port lb 8404)/stats;csv}
|
||||
deadline=$(($(date +%s) + ${WAIT_TIMEOUT:-60}))
|
||||
until [ "$(curl -fsS "$url" | awk -F, -v b="$service" '$1 == b && $2 != "BACKEND" && $18 == "UP"' | wc -l)" -ge "$want" ]; do
|
||||
[ "$(date +%s)" -lt "$deadline" ] || { echo "LB does not see $want UP $service servers" >&2; exit 4; }
|
||||
sleep 0.5
|
||||
done
|
||||
@@ -0,0 +1,156 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
S3_CLIENT_SECRETS = ["s3_access_key_id", "s3_root_password", "s3_root_user", "s3_secret_access_key"]
|
||||
POLICY = {
|
||||
"lb": {"secrets": [], "volumes": [], "networks": ["backend", "edge"], "ports": True},
|
||||
"api": {
|
||||
"secrets": [
|
||||
"cache_url",
|
||||
"cursor_secret",
|
||||
"database_url",
|
||||
"s3_access_key_id",
|
||||
"s3_secret_access_key",
|
||||
],
|
||||
"volumes": [],
|
||||
"networks": ["backend"],
|
||||
},
|
||||
"migrate": {"secrets": ["database_url"], "volumes": [], "networks": ["backend"]},
|
||||
"postgres": {"secrets": ["postgres_password"], "volumes": ["pgdata"], "networks": ["backend"]},
|
||||
"valkey": {"secrets": ["valkey.conf"], "volumes": ["valkeydata"], "networks": ["backend"]},
|
||||
"k6": {"secrets": [], "volumes": [], "networks": ["edge"]},
|
||||
"s3": {
|
||||
"secrets": ["s3_root_password", "s3_root_user"],
|
||||
"volumes": ["s3data"],
|
||||
"networks": ["backend"],
|
||||
},
|
||||
"s3-init": {"secrets": S3_CLIENT_SECRETS, "volumes": [], "networks": ["backend"]},
|
||||
"s3-cli": {"secrets": S3_CLIENT_SECRETS, "volumes": [], "networks": ["backend"]},
|
||||
}
|
||||
|
||||
|
||||
def run(*command):
|
||||
done = subprocess.run(command, cwd=str(ROOT), capture_output=True, text=True, check=False)
|
||||
return done.returncode, done.stdout.strip()
|
||||
|
||||
|
||||
def declared(name, service, networks, problems):
|
||||
limits = service.get("deploy", {}).get("resources", {}).get("limits", {})
|
||||
secrets = sorted(s["source"] for s in service.get("secrets", []))
|
||||
volumes = sorted(v["source"] for v in service.get("volumes", []) if v["type"] == "volume")
|
||||
nets = sorted(service.get("networks", {}))
|
||||
ports = [
|
||||
f"{p.get('host_ip', '0.0.0.0')}:{p.get('published')}" for p in service.get("ports", [])
|
||||
]
|
||||
want = POLICY.get(name)
|
||||
if want is None:
|
||||
problems.append(f"{name}: service is not covered by the policy")
|
||||
want = {"secrets": [], "volumes": [], "networks": []}
|
||||
if not limits.get("cpus") or not limits.get("memory"):
|
||||
problems.append(f"{name}: no CPU or memory limit")
|
||||
if "container_name" in service:
|
||||
problems.append(f"{name}: container_name prevents scaling")
|
||||
for kind, actual in (("secrets", secrets), ("volumes", volumes), ("networks", nets)):
|
||||
if actual != want[kind]:
|
||||
problems.append(f"{name}: {kind} {actual}, policy {want[kind]}")
|
||||
if ports and not want.get("ports"):
|
||||
problems.append(f"{name}: publishes {ports} in bypass of the balancer")
|
||||
if any(not port.startswith("127.0.0.1:") for port in ports):
|
||||
problems.append(f"{name}: publishes {ports} beyond loopback")
|
||||
if name == "api" and not service.get("read_only"):
|
||||
problems.append("api: root filesystem is writable")
|
||||
internal = [n for n in nets if networks.get(n, {}).get("internal")]
|
||||
return [
|
||||
name,
|
||||
f"{limits.get('cpus', '-')}/{int(limits.get('memory') or 0) >> 20}MiB",
|
||||
",".join(ports) or "-",
|
||||
",".join(secrets) or "-",
|
||||
",".join(volumes) or "-",
|
||||
",".join(f"{n}{'(internal)' if n in internal else ''}" for n in nets),
|
||||
str(service.get("read_only", False)).lower(),
|
||||
]
|
||||
|
||||
|
||||
def running(problems):
|
||||
_, ids = run("docker", "compose", "ps", "-q")
|
||||
template = (
|
||||
'{{index .Config.Labels "com.docker.compose.service"}} {{.Config.Hostname}} '
|
||||
"{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}} {{json .HostConfig.PortBindings}}"
|
||||
)
|
||||
_, found = run("docker", "inspect", "-f", template, *ids.split())
|
||||
rows = []
|
||||
for line in found.splitlines():
|
||||
service, host, cpus, memory, bindings = line.split(" ", 4)
|
||||
published = [
|
||||
f"{b['HostIp']}:{b['HostPort']}->{port}"
|
||||
for port, items in (json.loads(bindings) or {}).items()
|
||||
for b in items or []
|
||||
]
|
||||
if int(cpus) <= 0 or int(memory) <= 0:
|
||||
problems.append(f"container {service}/{host}: no CPU or memory limit applied")
|
||||
if published and service != "lb":
|
||||
problems.append(f"container {service}/{host}: publishes {published}")
|
||||
rows.append(
|
||||
[service, host, f"{int(cpus) / 1e9:g}", f"{int(memory) >> 20}MiB", ",".join(published)]
|
||||
)
|
||||
return sorted(rows)
|
||||
|
||||
|
||||
def table(header, rows):
|
||||
widths = [max(len(str(r[i])) for r in [header, *rows]) for i in range(len(header))]
|
||||
lines = [header, ["-" * w for w in widths], *rows]
|
||||
return [" ".join(str(c).ljust(w) for c, w in zip(r, widths)).rstrip() for r in lines]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="limits, ports, secrets, volumes and networks")
|
||||
parser.add_argument("--out", help="text report, written only after a complete check")
|
||||
args = parser.parse_args()
|
||||
problems = []
|
||||
_, raw = run(
|
||||
"docker",
|
||||
"compose",
|
||||
"--profile",
|
||||
"bench",
|
||||
"--profile",
|
||||
"tools",
|
||||
"config",
|
||||
"--format",
|
||||
"json",
|
||||
)
|
||||
config = json.loads(raw)
|
||||
services = [
|
||||
declared(name, service, config.get("networks", {}), problems)
|
||||
for name, service in sorted(config["services"].items())
|
||||
]
|
||||
containers = running(problems)
|
||||
if not any(row[0] == "api" for row in containers):
|
||||
problems.append("no running api replicas to inspect")
|
||||
lines = [
|
||||
f"date: {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}",
|
||||
"$ docker compose --profile bench --profile tools config --format json (declared)",
|
||||
*table(["service", "cpus/mem", "ports", "secrets", "volumes", "networks", "ro"], services),
|
||||
"",
|
||||
"$ docker inspect (running containers: hostname = X-Instance-Id, applied limits)",
|
||||
*table(["service", "hostname", "cpus", "memory", "published"], containers),
|
||||
"",
|
||||
*(f"FAIL: {p}" for p in problems),
|
||||
"PASS: limits, ports, secrets, volumes and networks match the policy"
|
||||
if not problems
|
||||
else f"FAIL: {len(problems)} violation(s)",
|
||||
]
|
||||
text = "\n".join(lines) + "\n"
|
||||
if args.out:
|
||||
Path(args.out).write_text(text)
|
||||
print(text, end="")
|
||||
return 1 if problems else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+14
@@ -0,0 +1,14 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
started() {
|
||||
for id in $(docker compose ps -aq); do
|
||||
docker inspect --format '{{index .Config.Labels "com.docker.compose.service"}} {{.Config.Hostname}} {{.State.StartedAt}} {{.State.Status}}' "$id"
|
||||
done | sort
|
||||
}
|
||||
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"
|
||||
echo "== before: service hostname StartedAt status"
|
||||
started
|
||||
make --no-print-directory test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL="${1:-http://localhost:8080}"
|
||||
echo "== after: service hostname StartedAt status"
|
||||
started
|
||||
Executable
+436
@@ -0,0 +1,436 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import base64
|
||||
import collections
|
||||
import functools
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
COUNTERS = ("wretr", "wredis", "econ", "eresp", "hrsp_5xx")
|
||||
|
||||
|
||||
@functools.cache
|
||||
def published(port, fallback):
|
||||
command = ["docker", "compose", "port", "lb", str(port)]
|
||||
found = subprocess.run(command, cwd=str(ROOT), capture_output=True, text=True, check=False)
|
||||
return found.stdout.strip() or fallback
|
||||
|
||||
|
||||
def base_url():
|
||||
return os.environ.get("BASE_URL") or "http://" + published(8080, "localhost:8080")
|
||||
|
||||
|
||||
def stats_url():
|
||||
return os.environ.get("LB_STATS_URL") or f"http://{published(8404, '127.0.0.1:8404')}/stats;csv"
|
||||
|
||||
|
||||
def call(method, path, body=None, headers=None, timeout=2.0):
|
||||
data = None if body is None else json.dumps(body, ensure_ascii=False).encode()
|
||||
head = {"Accept": "application/json"}
|
||||
if data is not None:
|
||||
head["Content-Type"] = "application/json"
|
||||
head.update(headers or {})
|
||||
request = urllib.request.Request(
|
||||
base_url().rstrip("/") + path, data=data, headers=head, method=method
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
payload = json.loads(response.read() or b"null")
|
||||
return response.status, response.headers.get("X-Instance-Id", ""), payload
|
||||
except urllib.error.HTTPError as error:
|
||||
try:
|
||||
payload = json.loads(error.read() or b"null")
|
||||
except (OSError, ValueError):
|
||||
payload = None
|
||||
return error.code, error.headers.get("X-Instance-Id", ""), payload
|
||||
except (OSError, ValueError):
|
||||
return 0, "", None
|
||||
|
||||
|
||||
def emit(report, out=None):
|
||||
text = json.dumps(report, ensure_ascii=False, indent=2)
|
||||
if out:
|
||||
Path(out).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(out).write_text(text + "\n")
|
||||
print(text)
|
||||
|
||||
|
||||
def created(result, what):
|
||||
if result[0] != 201:
|
||||
raise SystemExit(f"{what} failed: HTTP {result[0]}")
|
||||
return result[2]
|
||||
|
||||
|
||||
def actor(label):
|
||||
username = f"p_{label}_{uuid.uuid4().hex[:12]}"
|
||||
password = "Probe-" + uuid.uuid4().hex
|
||||
body = {"username": username, "display_name": "probe " + label, "password": password}
|
||||
user = created(call("POST", "/api/v1/users", body, timeout=10), "register")
|
||||
basic = base64.b64encode(f"{username}:{password}".encode()).decode()
|
||||
login = call(
|
||||
"POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}, timeout=10
|
||||
)
|
||||
return {
|
||||
"id": user["id"],
|
||||
"auth": {"Authorization": "Bearer " + created(login, "login")["token"]},
|
||||
}
|
||||
|
||||
|
||||
def chat_of(owner, other):
|
||||
body = {"title": "probe", "member_ids": [owner["id"], other["id"]]}
|
||||
return created(call("POST", "/api/v1/chats", body, owner["auth"], timeout=10), "chat")["id"]
|
||||
|
||||
|
||||
def send(owner, chat, text, key):
|
||||
headers = dict(owner["auth"], **{"Idempotency-Key": key})
|
||||
return call("POST", f"/api/v1/chats/{chat}/messages", {"text": text}, headers)
|
||||
|
||||
|
||||
def lb_rows(backend="api"):
|
||||
try:
|
||||
with urllib.request.urlopen(stats_url(), timeout=2) as response:
|
||||
lines = response.read().decode().splitlines()
|
||||
except OSError:
|
||||
return None
|
||||
header = lines[0].lstrip("# ").split(",")
|
||||
rows = [dict(zip(header, line.split(","))) for line in lines[1:] if line]
|
||||
return [r for r in rows if r["pxname"] == backend]
|
||||
|
||||
|
||||
def lb_up():
|
||||
rows = lb_rows()
|
||||
if rows is None:
|
||||
return None
|
||||
return sum(1 for r in rows if r["svname"] != "BACKEND" and r["status"].startswith("UP"))
|
||||
|
||||
|
||||
def lb_counters():
|
||||
row = next((r for r in lb_rows() or [] if r["svname"] == "BACKEND"), {})
|
||||
return {name: int(row.get(name) or 0) for name in COUNTERS}
|
||||
|
||||
|
||||
def run_k6(out, env, on_start=None):
|
||||
Path(out).mkdir(parents=True, exist_ok=True)
|
||||
started = []
|
||||
process = subprocess.Popen(
|
||||
[str(ROOT / "scripts/k6.sh"), str(out)],
|
||||
env=dict(os.environ, **env),
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
)
|
||||
with (Path(out) / "k6.txt").open("w") as log:
|
||||
for line in process.stdout:
|
||||
log.write(line)
|
||||
if not started and "LOAD-START" in line:
|
||||
started.append(time.time())
|
||||
if on_start:
|
||||
on_start()
|
||||
return process.wait(), started[0] if started else None
|
||||
|
||||
|
||||
def k6_metrics(path, duration):
|
||||
metrics = json.loads(Path(path).read_text())["metrics"]
|
||||
|
||||
def value(name, field):
|
||||
return metrics.get(name + "{scenario:measure}", {}).get("values", {}).get(field, 0.0)
|
||||
|
||||
count = value("http_reqs", "count")
|
||||
return {
|
||||
"requests": int(count),
|
||||
"rps": round(count / duration, 2),
|
||||
"ok_rps": round(count * (1 - value("http_req_failed", "rate")) / duration, 2),
|
||||
"failed_ratio": round(value("http_req_failed", "rate"), 5),
|
||||
"dropped_iterations": int(value("dropped_iterations", "count")),
|
||||
"p50_ms": round(value("http_req_duration", "med"), 2),
|
||||
"p95_ms": round(value("http_req_duration", "p(95)"), 2),
|
||||
"p99_ms": round(value("http_req_duration", "p(99)"), 2),
|
||||
}
|
||||
|
||||
|
||||
def race(alice, chat, parallel):
|
||||
key, text = uuid.uuid4().hex, "same key " + uuid.uuid4().hex
|
||||
barrier, results = threading.Barrier(parallel), []
|
||||
|
||||
def worker():
|
||||
barrier.wait()
|
||||
results.append(send(alice, chat, text, key))
|
||||
|
||||
threads = [threading.Thread(target=worker) for _ in range(parallel)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
bodies = [r[2] for r in results if r[0] == 201]
|
||||
identical = bool(bodies) and all(b == bodies[0] for b in bodies)
|
||||
return text, results, identical, send(alice, chat, text + " changed", key)[0]
|
||||
|
||||
|
||||
def check(args):
|
||||
alice, bob = actor("alice"), actor("bob")
|
||||
chat = chat_of(alice, bob)
|
||||
seen = collections.Counter(
|
||||
call("GET", "/api/v1/users/me", headers=alice["auth"])[1] for _ in range(args.samples)
|
||||
)
|
||||
rounds = [race(alice, chat, args.parallel) for _ in range(min(args.rounds, 100))]
|
||||
page = call("GET", f"/api/v1/chats/{chat}/messages?limit=100", headers=bob["auth"])[2]
|
||||
copies = collections.Counter(m.get("text") for m in (page or {"items": []})["items"])
|
||||
statuses = collections.Counter(r[0] for _, results, _, _ in rounds for r in results)
|
||||
served = collections.Counter(r[1] for _, results, _, _ in rounds for r in results)
|
||||
same = {
|
||||
"rounds": len(rounds),
|
||||
"parallel": args.parallel,
|
||||
"statuses": dict(statuses),
|
||||
"instances": dict(served),
|
||||
"identical_bodies_rounds": sum(1 for _, _, identical, _ in rounds if identical),
|
||||
"single_copy_rounds": sum(1 for text, _, _, _ in rounds if copies[text] == 1),
|
||||
"conflict_rounds": sum(1 for _, _, _, other in rounds if other == 409),
|
||||
}
|
||||
passed = (
|
||||
len(seen) >= 2
|
||||
and statuses == {201: len(rounds) * args.parallel}
|
||||
and len(served) >= 2
|
||||
and same["identical_bodies_rounds"] == len(rounds)
|
||||
and same["single_copy_rounds"] == len(rounds)
|
||||
and same["conflict_rounds"] == len(rounds)
|
||||
)
|
||||
emit({"instances_one_token": dict(seen), "same_key": same, "pass": passed}, args.out)
|
||||
return 0 if passed else 1
|
||||
|
||||
|
||||
def record(stop, alice, chat):
|
||||
probes, trace = [], []
|
||||
|
||||
def prober():
|
||||
key = uuid.uuid4().hex
|
||||
while not stop.is_set():
|
||||
started = time.time()
|
||||
status, instance, _ = send(alice, chat, "probe " + key, key)
|
||||
probes.append((started, status, round(1000 * (time.time() - started), 1), instance))
|
||||
if status == 201:
|
||||
key = uuid.uuid4().hex
|
||||
time.sleep(0.1)
|
||||
|
||||
def poller():
|
||||
while not stop.is_set():
|
||||
trace.append((time.time(), lb_up()))
|
||||
time.sleep(0.1)
|
||||
|
||||
threads = [threading.Thread(target=t, daemon=True) for t in (prober, poller)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
return probes, trace, threads
|
||||
|
||||
|
||||
def write_csv(path, header, rows):
|
||||
with path.open("w") as handle:
|
||||
handle.write(header + "\n")
|
||||
handle.writelines(",".join(str(v) for v in row) + "\n" for row in rows)
|
||||
|
||||
|
||||
def summarize(args, probes, trace, action, expected):
|
||||
t0, end = action["start"], action["end"]
|
||||
label = args.label or os.environ.get("TARGET_INSTANCE_ID", "action")
|
||||
failures = [t - t0 for t, s, _, _ in probes if s != 201 and t >= t0]
|
||||
down = [t for t, up in trace if t >= t0 and up is not None and up < expected]
|
||||
back = [t for t, up in trace if down and t > down[0] and up is not None and up >= expected]
|
||||
outage = [t for t in down if not back or t < back[0]] or [end]
|
||||
served = [(t, i) for t, s, _, i in probes if s == 201]
|
||||
before = collections.Counter(i for t, i in served if t < t0)
|
||||
during = collections.Counter(i for t, i in served if outage[0] <= t <= outage[-1])
|
||||
after = collections.Counter(i for t, i in served if t > end)
|
||||
return {
|
||||
"label": label,
|
||||
"action": args.command,
|
||||
"action_exit": action["exit"],
|
||||
"action_seconds": round(end - t0, 3),
|
||||
"action_finished_before_load_end": end < action["load_end"],
|
||||
"lb_up_before": expected,
|
||||
"lb_up_min": min((up for _, up in trace if up is not None), default=None),
|
||||
"detection_seconds": round(down[0] - t0, 3) if down else None,
|
||||
"rejoin_seconds": round(back[0] - t0, 3) if back else None,
|
||||
"probe": {
|
||||
"attempts": len(probes),
|
||||
"failed_attempts": sum(1 for _, s, _, _ in probes if s != 201),
|
||||
"first_failure_s": round(failures[0], 3) if failures else None,
|
||||
"recovery_s": round(failures[-1], 3) if failures else 0.0,
|
||||
"instances_before": dict(before),
|
||||
"instances_during_outage": dict(during),
|
||||
"instances_after": dict(after),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def verdict(args, result):
|
||||
probe, k6 = result["probe"], result["k6"]
|
||||
checks = {
|
||||
"action_ok": result["action_exit"] == 0 and result["action_finished_before_load_end"],
|
||||
"load_errors_ok": k6["exit"] == 0 and k6["failed_ratio"] <= args.max_failed_ratio,
|
||||
"write_recovery_ok": probe["recovery_s"] <= args.max_recovery,
|
||||
}
|
||||
if args.expect == "dip":
|
||||
checks["detected"] = result["detection_seconds"] is not None
|
||||
checks["target_excluded"] = probe["instances_during_outage"].get(result["label"], 0) == 0
|
||||
checks["rejoined"] = result["rejoin_seconds"] is not None and bool(
|
||||
probe["instances_after"].get(result["label"])
|
||||
)
|
||||
if args.expect == "replace":
|
||||
checks["capacity_kept"] = (result["lb_up_min"] or 0) >= result["lb_up_before"]
|
||||
old, new = set(probe["instances_before"]), set(probe["instances_after"])
|
||||
checks["all_replaced"] = bool(new) and not old & new
|
||||
return checks
|
||||
|
||||
|
||||
def load(args):
|
||||
out = Path(args.out)
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
expected = lb_up() or 0
|
||||
alice, bob = actor("alice"), actor("bob")
|
||||
stop, action = threading.Event(), {}
|
||||
probes, trace, threads = record(stop, alice, chat_of(alice, bob))
|
||||
|
||||
def act():
|
||||
time.sleep(args.warmup + args.lead)
|
||||
action["start"] = time.time()
|
||||
with (out / "action.txt").open("w") as log:
|
||||
action["exit"] = subprocess.call(
|
||||
args.command, cwd=str(ROOT), stdout=log, stderr=subprocess.STDOUT
|
||||
)
|
||||
action["end"] = time.time()
|
||||
|
||||
runner = threading.Thread(target=act)
|
||||
env = {"RATE": str(args.rate), "WARMUP": str(args.warmup), "DURATION": str(args.duration)}
|
||||
counters = lb_counters()
|
||||
code, load_start = run_k6(out / "k6", env, runner.start)
|
||||
if load_start is not None:
|
||||
runner.join()
|
||||
action["load_end"] = load_start + args.warmup + args.duration
|
||||
stop.set()
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
if "start" not in action:
|
||||
raise SystemExit(f"k6 failed before the action (exit {code}), see {out / 'k6/k6.txt'}")
|
||||
t0 = action["start"]
|
||||
write_csv(
|
||||
out / "probe.csv",
|
||||
"t_rel_s,status,latency_ms,instance",
|
||||
((round(t - t0, 3), *r) for t, *r in probes),
|
||||
)
|
||||
write_csv(out / "lb_up.csv", "t_rel_s,up", ((round(t - t0, 3), up) for t, up in trace))
|
||||
result = summarize(args, probes, trace, action, expected)
|
||||
result["lb_counters"] = {k: v - counters[k] for k, v in lb_counters().items()}
|
||||
result["k6"] = dict(
|
||||
k6_metrics(out / "k6/summary.json", args.duration), exit=code, rate=args.rate
|
||||
)
|
||||
result["checks"] = verdict(args, result)
|
||||
result["pass"] = all(result["checks"].values())
|
||||
(out / "result.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
return 0 if result["pass"] else 1
|
||||
|
||||
|
||||
def detect(args):
|
||||
before = lb_up() or 0
|
||||
until = before - 1 if args.until is None else args.until
|
||||
started = time.time()
|
||||
process = subprocess.Popen(args.command, cwd=str(ROOT))
|
||||
detected = acted = None
|
||||
while time.time() - started < args.timeout and (detected is None or acted is None):
|
||||
if acted is None and process.poll() is not None:
|
||||
acted = time.time() - started
|
||||
if detected is None and (lb_up() or 0) <= until:
|
||||
detected = time.time() - started
|
||||
time.sleep(0.05)
|
||||
code = process.wait()
|
||||
acted = time.time() - started if acted is None else acted
|
||||
status, instance, payload = call("GET", "/health/live")
|
||||
restored = None
|
||||
if args.restore:
|
||||
begun = time.time()
|
||||
subprocess.call(["sh", "-c", args.restore], cwd=str(ROOT))
|
||||
while (lb_up() or 0) < before and time.time() - begun < 3 * args.timeout:
|
||||
time.sleep(0.1)
|
||||
restored = round(time.time() - begun, 3) if (lb_up() or 0) >= before else None
|
||||
checks = {
|
||||
"action_ok": code == 0,
|
||||
"detected_in_time": detected is not None and detected <= args.max + args.slack,
|
||||
"lb_answer_ok": status == args.lb_status,
|
||||
}
|
||||
if args.within_action:
|
||||
checks["excluded_before_action_end"] = detected is not None and detected < acted
|
||||
if args.restore:
|
||||
checks["restored"] = restored is not None
|
||||
report = {
|
||||
"action": args.command,
|
||||
"action_exit": code,
|
||||
"action_seconds": round(acted, 3),
|
||||
"lb_up_before": before,
|
||||
"lb_up_counted_as_detected": until,
|
||||
"detection_seconds": None if detected is None else round(detected, 3),
|
||||
"declared_max_seconds": args.max,
|
||||
"measurement_slack_seconds": args.slack,
|
||||
"lb_answer_after_detection": {
|
||||
"path": "/health/live",
|
||||
"status": status,
|
||||
"x_instance_id_present": bool(instance),
|
||||
"error_code": ((payload or {}).get("error") or {}).get("code"),
|
||||
},
|
||||
"restore": args.restore,
|
||||
"restore_seconds": restored,
|
||||
"lb_up_after_restore": lb_up(),
|
||||
"checks": checks,
|
||||
"pass": all(checks.values()),
|
||||
}
|
||||
emit(report, args.out)
|
||||
return 0 if report["pass"] else 1
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="LB probes over the public HTTP contract")
|
||||
sub = parser.add_subparsers(dest="mode", required=True)
|
||||
one = sub.add_parser("check", help="instances behind one token and same-key races")
|
||||
one.add_argument("--samples", type=int, default=40)
|
||||
one.add_argument("--parallel", type=int, default=8)
|
||||
one.add_argument("--rounds", type=int, default=1)
|
||||
one.add_argument("--out", help="JSON file, written only after a complete report")
|
||||
under = sub.add_parser("load", help="run an action under k6 load and measure recovery")
|
||||
under.add_argument("--out", required=True)
|
||||
under.add_argument("--label")
|
||||
under.add_argument("--expect", choices=("dip", "replace"), default="dip")
|
||||
under.add_argument("--rate", type=int, default=int(os.environ.get("RATE", "50")))
|
||||
under.add_argument("--warmup", type=int, default=10)
|
||||
under.add_argument("--duration", type=int, default=int(os.environ.get("DURATION", "60")))
|
||||
under.add_argument("--lead", type=int, default=15)
|
||||
under.add_argument("--max-failed-ratio", type=float, default=0.01)
|
||||
under.add_argument("--max-recovery", type=float, default=10.0)
|
||||
under.add_argument("command", nargs=argparse.REMAINDER)
|
||||
watch = sub.add_parser("detect", help="time until the LB takes replicas out of rotation")
|
||||
watch.add_argument("--max", type=float, required=True, help="declared worst detection, s")
|
||||
watch.add_argument("--slack", type=float, default=0.25, help="docker CLI and polling, s")
|
||||
watch.add_argument("--until", type=int, help="UP servers counted as detected")
|
||||
watch.add_argument("--within-action", action="store_true")
|
||||
watch.add_argument("--lb-status", type=int, default=200)
|
||||
watch.add_argument("--restore", help="shell command that brings replicas back")
|
||||
watch.add_argument("--timeout", type=float, default=20)
|
||||
watch.add_argument("--out", help="JSON file, written only after a complete report")
|
||||
watch.add_argument("command", nargs=argparse.REMAINDER)
|
||||
args = parser.parse_args()
|
||||
if args.mode == "check":
|
||||
return check(args)
|
||||
args.command = [c for c in args.command if c != "--"]
|
||||
if not args.command:
|
||||
parser.error(f"{args.mode} needs an action command after --")
|
||||
return load(args) if args.mode == "load" else detect(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+12
@@ -0,0 +1,12 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
defined=$(docker compose config --services)
|
||||
stores=""
|
||||
for service in ${STATE_SERVICES:-postgres valkey s3}; do
|
||||
if printf '%s\n' "$defined" | grep -qx "$service"; then stores="$stores $service"; fi
|
||||
done
|
||||
[ -n "$stores" ] || { echo "no state services found" >&2; exit 3; }
|
||||
docker compose stop lb "${API_SERVICE:-api}"
|
||||
docker compose restart $stores
|
||||
docker compose up -d --no-recreate --wait --wait-timeout "${WAIT_TIMEOUT:-75}"
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose up -d --no-deps --no-recreate --wait --wait-timeout "${WAIT_TIMEOUT:-60}" "${API_SERVICE:-api}"
|
||||
scripts/lb-wait.sh
|
||||
Executable
+20
@@ -0,0 +1,20 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
service=${API_SERVICE:-api}
|
||||
show() { docker inspect -f '{{.Config.Hostname}} image={{slice .Image 7 19}}' "$1"; }
|
||||
echo "$(date -u +%T) migrate once before the first replacement"
|
||||
docker compose run --rm --no-deps -T migrate 2>&1
|
||||
old=$(docker compose ps -q "$service")
|
||||
count=$(printf '%s\n' "$old" | grep -c .)
|
||||
for id in $old; do
|
||||
running=$(docker compose ps -q "$service")
|
||||
docker compose up -d --no-deps --no-recreate --wait --wait-timeout "${WAIT_TIMEOUT:-90}" --scale "$service=$((count + 1))" "$service" 2>&1
|
||||
scripts/lb-wait.sh $((count + 1))
|
||||
new=$(docker compose ps -q "$service" | grep -v -F "$running")
|
||||
echo "$(date -u +%T) replace $(show "$id") -> $(show "$new")"
|
||||
docker stop -t "${STOP_TIMEOUT:-6}" "$id" >/dev/null
|
||||
docker rm "$id" >/dev/null
|
||||
done
|
||||
scripts/lb-wait.sh "$count"
|
||||
echo "$(date -u +%T) done: $count replicas replaced"
|
||||
Executable
+66
@@ -0,0 +1,66 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
cli() { docker compose run --rm --no-deps -T s3-cli "$@"; }
|
||||
snapshot() {
|
||||
cli sh -s <<'EOF' | grep -v '_date'
|
||||
ak=$(cat /run/secrets/s3_access_key_id)
|
||||
for c in "bucket list root" "bucket anonymous get root/$S3_BUCKET" "admin policy info root messenger-api" "admin user info root $ak" "bucket lifecycle rule export root/$S3_BUCKET"; do
|
||||
rc $c --json
|
||||
done | sed "s/$ak/<api-key>/g"
|
||||
EOF
|
||||
}
|
||||
failed=0
|
||||
verdict() { if [ "$2" = "$3" ]; then echo "PASS $1: $3"; else echo "FAIL $1: ожидалось $2, получено $3"; failed=1; fi; }
|
||||
echo "== $(date -u +%FT%TZ) S3 privacy and bootstrap checks"
|
||||
verdict "s3: опубликованные host-порты" "нет" "$(docker inspect -f '{{json .NetworkSettings.Ports}}' "$(docker compose ps -q s3)" | grep -q HostPort && echo есть || echo нет)"
|
||||
verdict "s3: консоль :9001 внутри контейнера" "000" "$(docker compose exec -T s3 curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:9001/ || true)"
|
||||
before=$(snapshot)
|
||||
first=$(docker compose run --rm --no-deps -T s3-init >/dev/null 2>&1 && echo 0 || echo 1)
|
||||
second=$(docker compose run --rm --no-deps -T s3-init 2>/dev/null || echo FAILED)
|
||||
after=$(snapshot)
|
||||
verdict "s3-init: повторный запуск 1" "0" "$first"
|
||||
verdict "s3-init: повторный запуск 2" "0" "$(echo "$second" | grep -q '^s3-init:' && echo 0 || echo 1)"
|
||||
verdict "s3-init: состояние IAM/bucket не изменилось" "same" "$([ "$before" = "$after" ] && echo same || echo changed)"
|
||||
cli sh -s <<'EOF' || failed=1
|
||||
b=$S3_BUCKET
|
||||
key="attachments/s3-check-$(date +%s)-$$"
|
||||
other="s3-check-$(date +%s)-$$"
|
||||
head -c 4096 /dev/urandom > /tmp/probe
|
||||
result=0
|
||||
expect() {
|
||||
want=$1; name=$2; shift 2
|
||||
if "$@" >/tmp/out 2>&1; then got=allow; elif grep -qE '403|AccessDenied|set_bucket_policy: service error' /tmp/out; then got=deny; else got=error; fi
|
||||
if [ "$got" = "$want" ]; then echo "PASS $name: $got"; else echo "FAIL $name: ожидалось $want, получено $got"; result=1; fi
|
||||
}
|
||||
expect allow "api: PutObject" rc put /tmp/probe "api/$b/$key"
|
||||
expect allow "api: HeadObject" rc object stat "api/$b/$key"
|
||||
expect allow "api: GetObject" rc object show "api/$b/$key"
|
||||
expect allow "api: ListBucket" rc object list "api/$b/attachments/"
|
||||
expect deny "api: CreateBucket" rc bucket create "api/$other"
|
||||
expect deny "api: DeleteBucket" rc bucket remove "api/$b"
|
||||
expect deny "api: PutBucketPolicy (public)" rc bucket anonymous set public "api/$b"
|
||||
expect deny "api: PutBucketLifecycle" rc bucket lifecycle rule import "api/$b" /s3/lifecycle.json
|
||||
expect deny "api: admin ListUsers" rc admin user list api
|
||||
rc bucket create --ignore-existing "root/$other" >/dev/null
|
||||
expect deny "api: PutObject в чужой bucket" rc put /tmp/probe "api/$other/x"
|
||||
expect deny "api: ListBucket чужого bucket" rc object list "api/$other"
|
||||
rc bucket remove "root/$other" >/dev/null
|
||||
expect deny "anonymous: HeadObject" rc object stat "anon/$b/$key"
|
||||
expect deny "anonymous: ListBucket" rc object list "anon/$b"
|
||||
expect deny "anonymous: PutObject" rc put /tmp/probe "anon/$b/attachments/anonymous"
|
||||
status=$(wget -S -O /dev/null "$S3_ENDPOINT/$b/$key" 2>&1 | sed -n 's/^ *HTTP\/[0-9.]* \([0-9]*\).*/\1/p' | head -n 1)
|
||||
if [ "$status" = 403 ]; then echo "PASS anonymous: прямой HTTP GET объекта из отдельного контейнера: $status"; else echo "FAIL anonymous: прямой HTTP GET: $status"; result=1; fi
|
||||
permission=$(rc bucket anonymous get "root/$b" --json | sed -n 's/.*"permission": *"\([a-z]*\)".*/\1/p')
|
||||
if [ "$permission" = private ]; then echo "PASS bucket: anonymous policy: $permission"; else echo "FAIL bucket: anonymous policy: $permission"; result=1; fi
|
||||
expect allow "api: DeleteObject" rc object remove --force "api/$b/$key"
|
||||
host=$(echo "$S3_ENDPOINT" | sed 's#^[a-z]*://##; s#:.*##')
|
||||
if wget -T 2 -O /dev/null "http://$host:9001/" >/tmp/out 2>&1; then echo "FAIL s3: консоль :9001 отвечает из сети стенда"; result=1
|
||||
elif grep -q refused /tmp/out; then echo "PASS s3: консоль :9001 из отдельного контейнера сети стенда: connection refused"
|
||||
else echo "FAIL s3: консоль :9001 из сети стенда: $(tail -n 1 /tmp/out)"; result=1; fi
|
||||
exit $result
|
||||
EOF
|
||||
echo "== IAM/bucket snapshot (без секретов)"
|
||||
echo "$after"
|
||||
if [ "$failed" != 0 ]; then echo "RESULT: FAIL"; exit 1; fi
|
||||
echo "RESULT: PASS"
|
||||
Executable
+24
@@ -0,0 +1,24 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
umask 077
|
||||
mkdir -p secrets
|
||||
random() { od -An -N32 -tx1 /dev/urandom | tr -d ' \n'; }
|
||||
put() {
|
||||
if [ ! -s "secrets/$1" ]; then
|
||||
printf '%s' "$2" > "secrets/$1"
|
||||
chmod 644 "secrets/$1"
|
||||
echo "создан secrets/$1"
|
||||
fi
|
||||
}
|
||||
put postgres_password "$(random)"
|
||||
put database_url "postgresql://messenger:$(cat secrets/postgres_password)@postgres:5432/messenger"
|
||||
put cursor_secret "$(random)"
|
||||
put valkey_password "$(random)"
|
||||
put valkey.conf "requirepass $(cat secrets/valkey_password)"
|
||||
put cache_url "redis://:$(cat secrets/valkey_password)@valkey:6379/0"
|
||||
put s3_root_user "root$(random | cut -c1-16)"
|
||||
put s3_root_password "$(random)"
|
||||
put s3_access_key_id "api$(random | cut -c1-16)"
|
||||
put s3_secret_access_key "$(random)"
|
||||
[ -f .env ] || { cp .env.example .env; echo "создан .env из .env.example"; }
|
||||
@@ -0,0 +1,59 @@
|
||||
\set ON_ERROR_STOP on
|
||||
\set big '''00000000-0000-7000-8000-00000000c001'''
|
||||
\set cursor_at '''2026-01-01T02:46:40Z'''
|
||||
\set cursor_id '''00000000-0000-7000-8001-000000002710'''
|
||||
|
||||
PREPARE history_first (uuid, int) AS
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = $1
|
||||
ORDER BY created_at, id
|
||||
LIMIT $2;
|
||||
|
||||
PREPARE history_after (uuid, timestamptz, uuid, int) AS
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = $1 AND (created_at, id) > ($2, $3)
|
||||
ORDER BY created_at, id
|
||||
LIMIT $4;
|
||||
|
||||
PREPARE history_offset (uuid, int, int) AS
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = $1
|
||||
ORDER BY created_at, id
|
||||
LIMIT $2 OFFSET $3;
|
||||
|
||||
SELECT count(*) AS chat_messages FROM messages WHERE chat_id = :big;
|
||||
SELECT count(*) AS all_messages FROM messages;
|
||||
SELECT pg_size_pretty(pg_relation_size('messages_history_idx')) AS history_index_size;
|
||||
|
||||
\echo '== Первая страница, limit=50 (запрос HISTORY_FIRST приложения, LIMIT limit+1)'
|
||||
EXPLAIN (ANALYZE, BUFFERS) EXECUTE history_first(:big, 51);
|
||||
|
||||
\echo '== Глубокая страница: keyset после 10 000-го сообщения (запрос HISTORY_AFTER приложения)'
|
||||
EXPLAIN (ANALYZE, BUFFERS) EXECUTE history_after(:big, :cursor_at, :cursor_id, 51);
|
||||
|
||||
\echo '== Та же глубокая страница через OFFSET 10000 (не используется приложением, для сравнения)'
|
||||
EXPLAIN (ANALYZE, BUFFERS) EXECUTE history_offset(:big, 51, 10000);
|
||||
|
||||
\echo '== Проверка: keyset и OFFSET возвращают одну и ту же страницу'
|
||||
SELECT (
|
||||
SELECT array_agg(id ORDER BY created_at, id) FROM (
|
||||
SELECT id, created_at FROM messages
|
||||
WHERE chat_id = :big AND (created_at, id) > (:cursor_at, :cursor_id)
|
||||
ORDER BY created_at, id LIMIT 51
|
||||
) AS keyset
|
||||
) = (
|
||||
SELECT array_agg(id ORDER BY created_at, id) FROM (
|
||||
SELECT id, created_at FROM messages
|
||||
WHERE chat_id = :big
|
||||
ORDER BY created_at, id LIMIT 51 OFFSET 10000
|
||||
) AS skipped
|
||||
) AS same_page;
|
||||
|
||||
\echo '== Контрольный план без индекса истории (DROP INDEX внутри транзакции, затем ROLLBACK)'
|
||||
BEGIN;
|
||||
DROP INDEX messages_history_idx;
|
||||
EXPLAIN (ANALYZE, BUFFERS) EXECUTE history_after(:big, :cursor_at, :cursor_id, 51);
|
||||
ROLLBACK;
|
||||
@@ -0,0 +1,40 @@
|
||||
\set ON_ERROR_STOP on
|
||||
\set author '''00000000-0000-7000-8000-00000000a001'''
|
||||
\set big '''00000000-0000-7000-8000-00000000c001'''
|
||||
\set epoch '''2026-01-01T00:00:00Z'''
|
||||
|
||||
INSERT INTO users (id, username, display_name, password_hash, created_at)
|
||||
VALUES (:author, 'seed_author', 'Синтетический автор', '$argon2id$v=19$m=19456,t=2,p=1$c2VlZA$c2VlZA', :epoch)
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
INSERT INTO chats (id, title, created_at)
|
||||
SELECT ('00000000-0000-7000-8000-00000000c' || lpad(to_hex(chat), 3, '0'))::uuid,
|
||||
'Синтетический чат ' || chat, :epoch
|
||||
FROM generate_series(1, 21) AS chat
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
INSERT INTO chat_members (chat_id, user_id, position, chat_created_at)
|
||||
SELECT id, :author, 1, created_at FROM chats WHERE title LIKE 'Синтетический чат %'
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
SELECT ('00000000-0000-7000-8001-' || lpad(to_hex(n), 12, '0'))::uuid, :big, :author,
|
||||
'синтетическое сообщение ' || n, :epoch::timestamptz + n * interval '1 second'
|
||||
FROM generate_series(1, 20000) AS n
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
SELECT ('00000000-0000-7000-8002-' || lpad(to_hex(chat * 1000 + n), 12, '0'))::uuid,
|
||||
('00000000-0000-7000-8000-00000000c' || lpad(to_hex(chat), 3, '0'))::uuid, :author,
|
||||
'фоновое сообщение ' || n, :epoch::timestamptz + n * interval '1 second'
|
||||
FROM generate_series(2, 21) AS chat, generate_series(1, 500) AS n
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
UPDATE chats
|
||||
SET last_message_at = (SELECT max(created_at) FROM messages WHERE messages.chat_id = chats.id)
|
||||
WHERE title LIKE 'Синтетический чат %';
|
||||
|
||||
ANALYZE;
|
||||
|
||||
SELECT count(*) FILTER (WHERE chat_id = :big) AS big_chat_messages, count(*) AS all_messages
|
||||
FROM messages;
|
||||
Executable
+9
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
cd "$(dirname "$0")/.."
|
||||
ids=$(docker compose ps -q "${API_SERVICE:-api}")
|
||||
[ -n "$ids" ] || { echo "no running ${API_SERVICE:-api} replicas" >&2; exit 3; }
|
||||
target=$(docker inspect -f '{{.Id}} {{.Config.Hostname}}' $ids | awk -v t="$TARGET_INSTANCE_ID" '$2 == t {print $1}')
|
||||
[ -n "$target" ] || { echo "no running ${API_SERVICE:-api} replica answers as $TARGET_INSTANCE_ID" >&2; exit 3; }
|
||||
docker stop -t "${STOP_TIMEOUT:-6}" "$target" >/dev/null
|
||||
Executable
+117
@@ -0,0 +1,117 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
seed = __import__("lab4_seed")
|
||||
FORMAT = "{:>5} {:>5} {:>4} {:<18} {:>7} {:>7} {:>8} {:>8} {:>7} {:>8} {:>6}"
|
||||
COLUMNS = FORMAT.format(
|
||||
"slots",
|
||||
"size",
|
||||
"conc",
|
||||
"codes",
|
||||
"p50_ms",
|
||||
"p95_ms",
|
||||
"api_idle",
|
||||
"api_peak",
|
||||
"api_oom",
|
||||
"lb_peak",
|
||||
"lb_oom",
|
||||
)
|
||||
|
||||
|
||||
OVERRIDES = ("UPLOAD_CONCURRENCY", "MALLOC_MMAP_THRESHOLD_")
|
||||
|
||||
|
||||
def recreate(slots):
|
||||
env = dict(os.environ, UPLOAD_CONCURRENCY=str(slots))
|
||||
if not slots:
|
||||
env = {k: v for k, v in os.environ.items() if k not in OVERRIDES}
|
||||
cmd = ["docker", "compose", "up", "-d", "--no-deps", "--force-recreate", "--wait", "api", "lb"]
|
||||
subprocess.run(cmd, cwd=ROOT, env=env, check=True, capture_output=True)
|
||||
subprocess.run([str(ROOT / "scripts/lb-wait.sh")], cwd=ROOT, check=True)
|
||||
|
||||
|
||||
def balancer():
|
||||
cid = seed.run(["docker", "compose", "ps", "-q", "lb"]).strip()
|
||||
events = dict(row.split() for row in seed.cgroup(cid, "memory.events").splitlines())
|
||||
peak = seed.cgroup(cid, "memory.peak")
|
||||
return {
|
||||
"cgroup_memory_peak_mib": round(int(peak) / 1048576, 1),
|
||||
"oom_kill": int(events["oom_kill"]),
|
||||
}
|
||||
|
||||
|
||||
def line(report):
|
||||
memory = report["memory"].values()
|
||||
codes = ",".join(f"{k}x{v}" for k, v in sorted(report["status_codes"].items()))
|
||||
return FORMAT.format(
|
||||
report["upload_concurrency"],
|
||||
f"{report['body_bytes'] / 1048576:.1f}M",
|
||||
report["concurrency"],
|
||||
codes,
|
||||
report["latency_ms"]["p50"],
|
||||
report["latency_ms"]["p95"],
|
||||
max(m["idle_mib"] for m in memory),
|
||||
max(m["cgroup_memory_peak_mib"] or 0 for m in memory),
|
||||
sum(m["oom_kill"] for m in memory),
|
||||
report["lb"]["cgroup_memory_peak_mib"],
|
||||
report["lb"]["oom_kill"],
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description="Peak memory of API replicas under concurrent uploads")
|
||||
p.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080"))
|
||||
p.add_argument("--slots", type=int, nargs="+", default=[64, 3])
|
||||
p.add_argument("--sizes", type=int, nargs="+", default=[1 << 20, 10 << 20])
|
||||
p.add_argument("--levels", type=int, nargs="+", default=[10, 20, 40])
|
||||
p.add_argument("--rounds", type=int, default=2, help="uploads per concurrent client")
|
||||
p.add_argument("--out", default="evidence/upload-memory")
|
||||
args = p.parse_args()
|
||||
out = ROOT / args.out
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
commit = seed.run(["git", "rev-parse", "--short", "HEAD"]).strip()
|
||||
threshold = os.environ.get("MALLOC_MMAP_THRESHOLD_", "131072")
|
||||
lines = [
|
||||
f"# {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} commit {commit}; api x2: 1 CPU,"
|
||||
f" 512 MiB, 2 workers; MALLOC_MMAP_THRESHOLD_={threshold}; {args.rounds} uploads"
|
||||
" per client; peaks from cgroup memory.peak, MiB",
|
||||
COLUMNS,
|
||||
]
|
||||
print("\n".join(lines), flush=True)
|
||||
try:
|
||||
for slots in args.slots:
|
||||
for size in args.sizes:
|
||||
for level in args.levels:
|
||||
recreate(slots)
|
||||
run = argparse.Namespace(
|
||||
base_url=args.base_url,
|
||||
timeout=60,
|
||||
size=size,
|
||||
count=level * args.rounds,
|
||||
concurrency=level,
|
||||
sample=True,
|
||||
service="api",
|
||||
link=0,
|
||||
ids_out=None,
|
||||
)
|
||||
report = {"upload_concurrency": slots, **seed.over_http(run)}
|
||||
report["lb"] = balancer()
|
||||
name = f"slots{slots}-{size >> 20}mib-c{level}.json"
|
||||
(out / name).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
|
||||
lines.append(line(report))
|
||||
print(lines[-1], flush=True)
|
||||
finally:
|
||||
recreate(None)
|
||||
(out / "summary.txt").write_text("\n".join(lines) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user