Carry over the lab 4 solution

Bring the application, own tests, Dockerfile, toolchain, compose stack,
HAProxy, k6 and RustFS configuration, migrations, hooks, evidence and
scenario scripts and the CI job from the lab 4 solution commit d2744bc,
merge the lab 4 Makefile targets into the lab 5 skeleton Makefile
without touching its targets, and append the lab 4 solution variables
to the skeleton .env.example. The package and the application follow
the contract version 1.5.0, and the image and the throwaway test
containers are tagged lab5 so they do not collide with a lab 4 stand on
the same host.

This is the starting point of the lab 5 cycle: lint, own tests and the
offline skeleton check pass before any lab 5 change.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Artyom
2026-10-29 00:34:03 +03:00
co-authored by Claude Opus 5.5
parent 1210dd899c
commit 540fcb35ba
171 changed files with 12909 additions and 1 deletions
+20
View File
@@ -0,0 +1,20 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
out=${1:-evidence/detection}
mkdir -p "$out"
scripts/restore.sh
replica() { docker compose ps -q "${API_SERVICE:-api}" | head -1; }
detect() { python3 scripts/probe.py detect "$@"; }
failed=0
detect --max 1.5 --within-action --restore scripts/restore.sh --out "$out/sigterm.json" \
-- docker stop -t "${STOP_TIMEOUT:-6}" "$(replica)" || failed=1
detect --max 1.5 --restore scripts/restore.sh --out "$out/kill.json" \
-- docker kill "$(replica)" || failed=1
hung=$(replica)
detect --max 3.5 --restore "docker unpause $hung && scripts/lb-wait.sh" --out "$out/pause.json" \
-- docker pause "$hung" || failed=1
db=$(docker compose ps -q postgres)
detect --max 1.5 --until 0 --lb-status 503 --restore "docker start $db && scripts/lb-wait.sh" \
--out "$out/postgres.json" -- docker stop "$db" || failed=1
exit "$failed"
+40
View File
@@ -0,0 +1,40 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
psql() { docker compose exec -T postgres psql -U messenger -d messenger -X -c "$1"; }
value() { docker compose exec -T postgres psql -U messenger -d messenger -X -tA -c "$1"; }
cli() { docker compose run --rm --no-deps -T s3-cli "$@"; }
bucket=${S3_BUCKET:-messenger-attachments}
{
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"
echo "\$ docker compose ps --format '{{.Service}} {{.Status}} {{.Ports}}'"
docker compose ps -a --format '{{.Service}} {{.Status}} {{.Ports}}'
for table in attachments message_attachments; do
echo "\$ psql -c '\\d $table'"
psql "\\d $table"
done
echo "\$ psql: столбцы bytea во всей схеме (хеши токенов и отпечатки, не файлы)"
psql "SELECT table_name, column_name FROM information_schema.columns WHERE table_schema = 'public' AND data_type = 'bytea' ORDER BY 1, 2"
echo "\$ psql: вложения по статусу и максимальный размер строки attachments"
psql "SELECT status, count(*), max(pg_column_size(a.*)) AS max_row_bytes FROM attachments a GROUP BY status"
echo "\$ rc object list --recursive root/$bucket/attachments/ | count"
cli object list --recursive "root/$bucket/attachments/" --json | grep -c '"key"' || true
echo "\$ rc object list root/$bucket/ (первые 3 объекта, ключи назначены сервером)"
cli object list --recursive "root/$bucket/attachments/" | head -3
row=$(value "SELECT object_key || ' ' || sha256 || ' ' || size_bytes FROM attachments WHERE status = 'ready' ORDER BY created_at DESC LIMIT 1")
key=${row%% *}
echo "\$ сверка вне API: sha256 и размер последнего ready-вложения в PostgreSQL против байтов объекта в bucket"
echo "postgres: ${row#* }"
echo "s3: $(cli sh -c "rc object show root/$bucket/$key | sha256sum | cut -d' ' -f1") $(cli sh -c "rc object show root/$bucket/$key | wc -c" | tr -d ' ')"
echo "\$ docker inspect api: ReadonlyRootfs, tmpfs, volumes/bind mounts"
for id in $(docker compose ps -q api); do
docker inspect -f '{{.Config.Hostname}} ro={{.HostConfig.ReadonlyRootfs}} tmpfs={{json .HostConfig.Tmpfs}} mounts={{range .Mounts}}{{.Type}}:{{.Destination}} {{end}}' "$id"
done
echo "\$ docker inspect s3: volume данных"
docker inspect -f '{{range .Mounts}}{{.Type}}:{{.Name}}:{{.Destination}} {{end}}' "$(docker compose ps -q s3)"
echo "\$ секреты сервисов: root-ключи S3 только у s3/s3-init/s3-cli"
docker compose config --format json | python3 -c 'import json,sys
for name, service in sorted(json.load(sys.stdin)["services"].items()):
print(name, sorted(s["source"] for s in service.get("secrets", [])))'
} > evidence/storage-layout.txt
cat evidence/storage-layout.txt
+24
View File
@@ -0,0 +1,24 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
cli() { docker compose exec -T valkey sh -c "REDISCLI_AUTH=\"\$(sed -n 's/^requirepass //p' /run/secrets/valkey.conf)\" valkey-cli $1"; }
psql() { docker compose exec -T postgres psql -U messenger -d messenger -tA -c "$1"; }
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"
echo "\$ valkey-cli config get appendonly appendfsync maxmemory maxmemory-policy"
cli "config get appendonly" ; cli "config get appendfsync"; cli "config get maxmemory"; cli "config get maxmemory-policy"
echo "\$ valkey-cli info persistence | grep aof_enabled,aof_last_write_status"
cli "info persistence" | grep -E '^aof_enabled|^aof_last_write_status'
echo "\$ valkey-cli dbsize; info stats | grep keyspace"
cli dbsize
cli "info stats" | grep -E '^keyspace_(hits|misses)'
key=$(cli "--scan --pattern session:* --count 1000" | head -1)
hash=${key#session:}
echo "\$ сравнение срока ключа кэша и срока сессии для одного ключа (значение и токен не печатаются)"
echo "valkey pexpiretime, ms: $(cli "pexpiretime $key")"
echo "postgres expires_at, ms: $(psql "SELECT (extract(epoch FROM expires_at) * 1000)::bigint FROM sessions WHERE token_hash = decode('$hash', 'hex')")"
echo "\$ количество отметок отзыва и закэшированных сессий"
revoked=0; cached=0
for k in $(cli "--scan --pattern session:* --count 1000"); do
if [ "$(cli "get $k")" = "-" ]; then revoked=$((revoked + 1)); else cached=$((cached + 1)); fi
done
echo "tombstones: $revoked, cached sessions: $cached"
+49
View File
@@ -0,0 +1,49 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
base=${1:-http://$(docker compose port lb 8080)}
stats=http://$(docker compose port lb 8404)/stats\;csv
header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"; }
{
header
echo "\$ docker compose --env-file .env config --quiet && echo valid"
docker compose --env-file .env config --quiet && echo valid
echo "\$ docker compose ps --format '{{.Service}} {{.Name}} {{.Status}} {{.Ports}}'"
docker compose ps --format '{{.Service}} {{.Name}} {{.Status}} {{.Ports}}'
echo "\$ docker compose config | grep -c container_name"
docker compose config | grep -c container_name || true
echo "\$ docker port <каждая реплика api> # реплики не публикуют порт"
for id in $(docker compose ps -q api); do echo "$(docker inspect -f '{{.Config.Hostname}}' "$id"): [$(docker port "$id")]"; done
echo "\$ docker inspect: hostname = X-Instance-Id, сети, лимиты CPU (nano) и памяти"
for id in $(docker compose ps -q); do
docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}} {{.Config.Hostname}} {{.Name}} nets={{range $k, $v := .NetworkSettings.Networks}}{{$k}},{{end}} cpus={{.HostConfig.NanoCpus}} mem={{.HostConfig.Memory}} user={{.Config.User}} ro={{.HostConfig.ReadonlyRootfs}}' "$id"
done | sort
echo "\$ curl LB stats: backend api, server status check"
curl -fsS "$stats" | python3 -c 'import csv,sys
for r in csv.DictReader(sys.stdin.read().lstrip("# ").splitlines()):
if r["pxname"] == "api" and r["status"] != "MAINT (resolution)":
print(r["svname"], r["status"], r["check_status"], r["check_code"])'
echo "\$ for i in 1..6: curl -D - $base/health/live | X-Instance-Id"
for _ in 1 2 3 4 5 6; do curl -fsS -D - -o /dev/null "$base/health/live" | tr -d '\r' | grep -i '^x-instance-id'; done
} > evidence/stack.txt
{
header
echo "host: $(sysctl -n machdep.cpu.brand_string 2>/dev/null || uname -m), $(sysctl -n hw.ncpu 2>/dev/null) CPU, $(($(sysctl -n hw.memsize 2>/dev/null || echo 0) / 1073741824)) GiB RAM, $(sw_vers -productName 2>/dev/null) $(sw_vers -productVersion 2>/dev/null)"
echo "docker: $(docker version -f '{{.Server.Version}}'), VM: $(docker info -f '{{.NCPU}}') CPU, $(docker info -f '{{.MemTotal}}') bytes"
echo "compose: $(docker compose version --short)"
echo "images:"
grep -h -E 'image:|@sha256' compose.yaml Dockerfile | sed 's/^ */ /'
echo "python in image: $(docker exec "$(docker compose ps -q api | head -1)" python -V)"
echo "haproxy: $(docker compose exec -T lb haproxy -v | head -1)"
echo "rustfs: $(docker compose exec -T s3 rustfs --version 2>/dev/null | head -1)"
echo "packages (uv.lock):"
for name in fastapi uvicorn uvicorn-worker gunicorn asyncpg argon2-cffi pydantic pydantic-settings dishka structlog redis aiobotocore botocore aiohttp pillow; do
printf ' %s %s\n' "$name" "$(sed -n "/^name = \"$name\"$/{n;s/version = \"\(.*\)\"/\1/p;}" uv.lock)"
done
echo "process users (service uid command):"
for id in $(docker compose ps -q); do
service=$(docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}}' "$id")
docker top "$id" -eo pid,uid,comm | awk -v s="$service" 'NR > 1 {print " " s, $2, $3}'
done | sort -u
} > evidence/environment.txt
cat evidence/stack.txt evidence/environment.txt
+44
View File
@@ -0,0 +1,44 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
psql() { docker compose exec -T postgres psql -U messenger -d messenger -c "$1"; }
header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"; }
counts="SELECT (SELECT count(*) FROM users) users, (SELECT count(*) FROM sessions) sessions, (SELECT count(*) FROM chats) chats, (SELECT count(*) FROM chat_members) members, (SELECT count(*) FROM messages) messages, (SELECT count(*) FROM idempotency_keys) keys, (SELECT count(*) FROM attachments) attachments, (SELECT count(*) FROM message_attachments) links, (SELECT count(*) FROM schema_migrations) versions"
kdf="SELECT substring(password_hash from '^[$]argon2id[$]v=[0-9]+[$]m=[0-9]+,t=[0-9]+,p=[0-9]+[$]') AS kdf_prefix, count(*) AS users, min(length(password_hash)) AS hash_length FROM users GROUP BY 1"
tokens="SELECT octet_length(token_hash) AS stored_bytes, count(*) AS sessions FROM sessions GROUP BY 1"
{
header
for table in users sessions chats chat_members messages idempotency_keys attachments message_attachments schema_migrations; do
echo "\$ psql -c '\\d+ $table'"
psql "\\d+ $table"
done
echo "\$ psql -c 'SELECT version, left(checksum, 12), applied_at FROM schema_migrations'"
psql "SELECT version, left(checksum, 12) AS checksum, applied_at FROM schema_migrations"
} > evidence/schema.txt
{
header
echo "\$ psql -c \"$counts\""
psql "$counts"
echo "\$ make migrate"
docker compose run --rm migrate 2>/dev/null
echo "\$ psql -c \"$counts\""
psql "$counts"
} > evidence/migrations.txt
{
header
echo "\$ psql -c \"$kdf\""
psql "$kdf"
echo "\$ psql -c \"$tokens\""
psql "$tokens"
} > evidence/kdf.txt
{
header
echo "\$ git ls-files secrets .env backups volumes"
git ls-files secrets .env backups volumes
echo "(пусто: секретов, .env, дампов и volumes в Git нет)"
echo "\$ git check-ignore -v secrets/database_url secrets/postgres_password secrets/s3_root_password secrets/s3_secret_access_key .env backups/"
git check-ignore -v secrets/database_url secrets/postgres_password secrets/s3_root_password secrets/s3_secret_access_key .env backups/
echo "\$ docker volume ls --filter name=mtusi2026lab5"
docker volume ls --filter name=mtusi2026lab5
} > evidence/secrets.txt
echo "evidence/{schema,migrations,kdf,secrets}.txt обновлены"
+19
View File
@@ -0,0 +1,19 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
out=${1:-evidence/failover-each}
rm -rf "$out"
mkdir -p "$out"
scripts/restore.sh
hosts=$(docker inspect -f '{{.Config.Hostname}}' $(docker compose ps -q "${API_SERVICE:-api}"))
failed=0
set --
for host in $hosts; do
TARGET_INSTANCE_ID=$host python3 scripts/probe.py load --expect dip --out "$out/$host" \
-- sh -c "scripts/stop-one.sh && sleep ${HOLD:-10} && scripts/restore.sh" || failed=1
scripts/restore.sh
set -- "$@" "$out/$host/result.json"
done
python3 -c 'import json,sys; print(json.dumps([json.load(open(p)) for p in sys.argv[1:]], ensure_ascii=False, indent=2))' "$@" > "$out/summary.tmp"
mv "$out/summary.tmp" "$out/summary.json"
exit "$failed"
+28
View File
@@ -0,0 +1,28 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
replicas() {
docker inspect -f '{{.Config.Hostname}} {{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}' $(docker compose ps -aq "${API_SERVICE:-api}")
}
tmp=evidence/failover.tmp
since=$(date -u +%FT%TZ)
{
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"
echo "== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt"
replicas
echo "\$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh"
make --no-print-directory test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL="${1:-http://localhost:8080}"
echo "== after: api replicas"
replicas
} > "$tmp" 2>&1
stopped=$(replicas | awk '$2 == "exited" {print $1}')
{
echo "== stopped replica: $stopped; shutdown: signal, drain window with readiness 503, close, exit"
docker logs --since "$since" "$stopped" 2>&1 | grep -E 'Handling signal|Draining|"status":503|Shutting down|Waiting for conn|Finished server|service.stopped|SIGTERM' | tail -16
echo "\$ scripts/restore.sh"
scripts/restore.sh
echo "== after restore"
replicas
} >> "$tmp" 2>&1
mv "$tmp" "evidence/failover-$stopped.txt"
cat "evidence/failover-$stopped.txt"
Executable
+10
View File
@@ -0,0 +1,10 @@
#!/bin/sh
set -eu
out=${1:?usage: k6.sh <output dir> [k6 run args]}
shift
mkdir -p "$out"
out=$(cd "$out" && pwd)
cd "$(dirname "$0")/.."
exec docker compose --profile bench run --rm --no-deps -T -u "$(id -u):$(id -g)" -v "$out:/out" \
-e RATE -e WARMUP -e DURATION -e USERS -e READ_RATIO -e VUS -e MAX_VUS -e RUN_ID \
k6 run --quiet "$@" "/scripts/${K6_SCRIPT:-messages.js}"
+230
View File
@@ -0,0 +1,230 @@
#!/usr/bin/env python3
import argparse
import hashlib
import http.client
import json
import os
import subprocess
import sys
import time
import uuid
from pathlib import Path
from urllib.parse import urlsplit
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
client = __import__("client")
Client, Failure, check = client.Client, client.Failure, client.check
FIXTURE = (ROOT / "tests/fixtures/sample.png").read_bytes()
def compose(*args, stdin=None):
cmd = ["docker", "compose", *args]
return subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, text=True, check=False)
def upload(c, actor, chat, expected=201):
path = f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png"
schema = "Attachment" if expected == 201 else None
return c.call(
"POST",
path,
expected,
actor=actor,
raw=FIXTURE,
headers={"Content-Type": "image/png"},
schema=schema,
)
def content(c, actor, attachment, expected=200):
return c.call("GET", f"/api/v1/attachments/{attachment['id']}/content", expected, actor=actor)
def timed(fn):
started = time.monotonic()
try:
response = fn()
except Failure as exc:
return {
"status": "failure",
"seconds": round(time.monotonic() - started, 3),
"error": str(exc),
}
seconds = round(time.monotonic() - started, 3)
return {
"status": response["status"],
"seconds": seconds,
"instance": response["headers"].get("X-Instance-Id"),
}
def wait(fn, seconds):
deadline = time.monotonic() + seconds
while True:
try:
return fn()
except Failure:
if time.monotonic() > deadline:
raise
time.sleep(0.2)
def failover_upload(c, args):
alice, bob, _, chat = c.bootstrap()
sent = upload(c, alice, chat)
att, target = sent["json"], sent["headers"]["X-Instance-Id"]
started = time.monotonic()
hook = subprocess.run(
[str(ROOT / "scripts/stop-one.sh")],
cwd=ROOT,
env=dict(os.environ, TARGET_INSTANCE_ID=target),
capture_output=True,
text=True,
check=False,
)
check(hook.returncode == 0, f"stop-one.sh failed: {hook.stderr.strip()}")
hook_seconds = time.monotonic() - started
got = wait(lambda: content(c, bob, att), 15)
recovered = time.monotonic() - started
served = got["headers"].get("X-Instance-Id")
check(served != target, "download served by the stopped replica")
digest = hashlib.sha256(got["bytes"]).hexdigest()
check(
digest == att["sha256"] == hashlib.sha256(FIXTURE).hexdigest(),
"sha256 mismatch after failover",
)
if args.restore:
compose("up", "-d", "--wait", "--no-recreate", "--no-deps", args.service)
return {
"scenario": "failover-upload",
"upload_instance": target,
"download_instance": served,
"sha256_equal": True,
"hook_seconds": round(hook_seconds, 3),
"download_ok_after_seconds": round(recovered, 3),
"restored": args.restore,
}
def s3_outage(c, args):
alice, bob, _, chat = c.bootstrap()
att = upload(c, alice, chat)["json"]
check(
compose("stop", "-t", "10", args.s3_service).returncode == 0, "docker compose stop failed"
)
try:
during = {
"upload": timed(lambda: upload(c, alice, chat, [503])),
"metadata": timed(
lambda: c.call("GET", f"/api/v1/attachments/{att['id']}", [200, 503], actor=bob)
),
"download": timed(lambda: content(c, bob, att, 503)),
"text_message": timed(lambda: c.message(alice, chat, "S3 недоступен, текст работает")),
"health_ready": timed(lambda: c.call("GET", "/health/ready", [200, 503])),
}
finally:
started = time.monotonic()
compose("up", "-d", "--wait", "--no-deps", "--no-recreate", args.s3_service)
recovered = round(time.monotonic() - started, 3)
got = wait(lambda: content(c, bob, att), 30)
check(got["bytes"] == FIXTURE, "attachment changed across S3 outage")
expected = {"upload": 503, "download": 503, "text_message": 201}
return {
"scenario": "s3-outage",
"during_outage": during,
"expected": expected,
"as_expected": all(during[k]["status"] == v for k, v in expected.items()),
"s3_back_healthy_seconds": recovered,
"download_after_recovery": "sha256 equal",
}
def abort_upload(c, args):
alice, _, _, chat = c.bootstrap()
name = f"abort-{uuid.uuid4().hex}.png"
parts = urlsplit(c.base)
factory = http.client.HTTPSConnection if parts.scheme == "https" else http.client.HTTPConnection
conn = factory(parts.netloc, timeout=10)
conn.putrequest("POST", f"/api/v1/chats/{chat['id']}/attachments?filename={name}")
headers = {
"Authorization": f"Bearer {alice['token']}",
"Content-Type": "image/png",
"Content-Length": str(args.declared),
"Accept": "application/json",
}
for key, value in headers.items():
conn.putheader(key, value)
conn.endheaders()
conn.send(FIXTURE[:8] + os.urandom(args.sent - 8))
conn.close()
time.sleep(args.settle)
user, db = os.environ.get("PGUSER", "messenger"), os.environ.get("PGDATABASE", "messenger")
sql = "SELECT status, count(*) FROM attachments WHERE filename = :'name' GROUP BY status;\n"
out = compose(
"exec",
"-T",
"postgres",
"psql",
"-qAt",
"-F",
" ",
"-U",
user,
"-d",
db,
"-v",
f"name={name}",
stdin=sql,
)
rows = out.stdout.split()
counts = dict(zip(rows[::2], (int(x) for x in rows[1::2])))
alive = c.call("GET", "/api/v1/users/me", actor=alice, schema="User")["status"]
return {
"scenario": "abort-upload",
"declared_bytes": args.declared,
"sent_bytes": args.sent,
"settle_seconds": args.settle,
"rows_by_status": counts,
"ready_rows": counts.get("ready", 0),
"api_still_serves": alive == 200,
}
def main():
p = argparse.ArgumentParser(
description="Lab4 failure scenarios over the public HTTP contract plus docker compose"
)
p.add_argument("scenario", choices=["failover-upload", "s3-outage", "abort-upload"])
p.add_argument("--base-url", default=None)
p.add_argument("--ca-file", default=None)
p.add_argument("--timeout", type=float, default=15)
p.add_argument("--service", default="api")
p.add_argument("--s3-service", default="s3")
p.add_argument(
"--restore", action="store_true", help="failover-upload: start the stopped replica again"
)
p.add_argument("--declared", type=int, default=5 * 1024 * 1024)
p.add_argument("--sent", type=int, default=256 * 1024)
p.add_argument("--settle", type=float, default=3)
p.add_argument("--output", metavar="FILE")
args = p.parse_args()
c = Client(args.base_url, args.ca_file, timeout=args.timeout)
run = {
"failover-upload": failover_upload,
"s3-outage": s3_outage,
"abort-upload": abort_upload,
}[args.scenario]
try:
report = {"ok": True, **run(c, args)}
except Failure as exc:
report = {"scenario": args.scenario, "ok": False, "error": str(exc)}
text = json.dumps(report, ensure_ascii=False, indent=2)
if args.output:
Path(args.output).write_text(text + "\n")
print(text)
sys.exit(0 if report["ok"] else 1)
if __name__ == "__main__":
main()
+352
View File
@@ -0,0 +1,352 @@
#!/usr/bin/env python3
import argparse
import base64
import hashlib
import http.client
import json
import os
import re
import struct
import subprocess
import tempfile
import threading
import time
import uuid
import zlib
from pathlib import Path
from urllib.parse import urlsplit
ROOT = Path(__file__).resolve().parents[1]
MAX_UPLOAD = 10 * 1024 * 1024
COLUMNS = (
"id",
"chat_id",
"owner_id",
"filename",
"content_type",
"size_bytes",
"sha256",
"width",
"height",
"object_key",
"status",
)
def chunk(tag, data):
return (
struct.pack(">I", len(data))
+ tag
+ data
+ struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF)
)
def png(width, height):
raw = b"".join(b"\x00" + os.urandom(width * 3) for _ in range(height))
header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
return (
b"\x89PNG\r\n\x1a\n"
+ chunk(b"IHDR", header)
+ chunk(b"IDAT", zlib.compress(raw, 1))
+ chunk(b"IEND", b"")
)
def sized_png(target):
height = max(1, target // (1024 * 3 + 1))
body = png(1024, height)
while len(body) > min(target, MAX_UPLOAD) and height > 1:
height -= max(1, (len(body) - target) // 3073 + 1)
body = png(1024, height)
return body
def run(cmd, stdin=None):
out = subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, check=False)
if out.returncode:
raise SystemExit(
f"command failed: {' '.join(cmd[:6])}\n{out.stderr.decode(errors='replace')[-500:]}"
)
return out.stdout.decode()
def psql(sql, stdin=None):
user, db = os.environ.get("PGUSER", "messenger"), os.environ.get("PGDATABASE", "messenger")
psql = [
"docker",
"compose",
"exec",
"-T",
"postgres",
"psql",
"-v",
"ON_ERROR_STOP=1",
"-qAt",
"-F",
" ",
]
return run([*psql, "-U", user, "-d", db, "-c", sql], stdin)
def direct(args):
bucket = os.environ.get("S3_BUCKET", "messenger-attachments")
prefix = os.environ.get("S3_PREFIX", "attachments/")
if args.chat_id and args.owner_id:
chat_id, owner_id = args.chat_id, args.owner_id
else:
found = psql(
"SELECT chat_id, owner_id FROM attachments ORDER BY created_at LIMIT 1"
).split()
if len(found) != 2:
raise SystemExit(
"no attachment rows yet: upload one via API or pass --chat-id and --owner-id"
)
chat_id, owner_id = found
rows = []
started = time.monotonic()
with tempfile.TemporaryDirectory(prefix="lab4-seed-") as tmp:
Path(tmp).chmod(0o755)
for i in range(args.count):
aid = str(uuid.uuid4())
body = png(8, 8)
path = Path(tmp) / aid
path.write_bytes(body)
path.chmod(0o644)
digest = hashlib.sha256(body).hexdigest()
rows.append(
f"{aid},{chat_id},{owner_id},seed-{i}.png,image/png,{len(body)},{digest},8,8,{prefix}{aid},ready"
)
generated = time.monotonic()
cli = [
"docker",
"compose",
"run",
"--rm",
"--no-deps",
"-T",
"-v",
f"{tmp}:/seed:ro",
"s3-cli",
]
run([*cli, "put", "--recursive", "--no-progress", "/seed/", f"root/{bucket}/{prefix}"])
uploaded = time.monotonic()
columns = ", ".join(COLUMNS)
psql(
f"COPY attachments ({columns}) FROM STDIN WITH (FORMAT csv)",
"".join(f"{r}\n" for r in rows).encode(),
)
seconds = {
"generate": round(generated - started, 3),
"s3_put": round(uploaded - generated, 3),
"pg_copy": round(time.monotonic() - uploaded, 3),
}
return {"mode": "direct", "count": args.count, "seconds": seconds}
class Http:
def __init__(self, base, timeout):
parts = urlsplit(base)
self.cls = (
http.client.HTTPSConnection if parts.scheme == "https" else http.client.HTTPConnection
)
self.host, self.timeout, self.local = parts.netloc, timeout, threading.local()
def call(self, method, path, body=None, headers=None, token=None):
conn = getattr(self.local, "conn", None) or self.cls(self.host, timeout=self.timeout)
self.local.conn = conn
h = {"Accept": "application/json"}
if token:
h["Authorization"] = "Bearer " + token
if isinstance(body, dict):
body, h["Content-Type"] = json.dumps(body).encode(), "application/json"
h.update(headers or {})
try:
conn.request(method, path, body=body, headers=h)
response = conn.getresponse()
data = response.read()
except (OSError, http.client.HTTPException):
conn.close()
self.local.conn = None
return 0, {}, b""
return response.status, {k.lower(): v for k, v in response.getheaders()}, data
def login(api, label):
name = f"seed_{label}_{uuid.uuid4().hex[:12]}"
password = "Seed-" + uuid.uuid4().hex
body = {"username": name, "display_name": label, "password": password}
status, _, data = api.call("POST", "/api/v1/users", body)
if status != 201:
raise SystemExit(f"user create failed: {status}")
user = json.loads(data)
basic = base64.b64encode(f"{name}:{password}".encode()).decode()
status, _, data = api.call(
"POST", "/api/v1/auth/sessions", headers={"Authorization": f"Basic {basic}"}
)
if status != 201:
raise SystemExit(f"login failed: {status}")
return user["id"], json.loads(data)["token"]
def mib(text):
m = re.match(r"([\d.]+)\s*([KMG]i?B)", text)
if not m:
return 0.0
scale = {"KiB": 1 / 1024, "KB": 1 / 1024, "GiB": 1024, "GB": 1024}
return float(m.group(1)) * scale.get(m.group(2), 1)
def sampler(ids, peaks, stop):
while not stop.is_set():
cmd = ["docker", "stats", "--no-stream", "--format", "{{.ID}} {{.MemUsage}}", *ids]
out = subprocess.run(cmd, capture_output=True, text=True, check=False).stdout
for line in out.splitlines():
cid, usage = line.split(" ", 1)
peaks[cid] = max(peaks.get(cid, 0.0), mib(usage))
def cgroup(cid, name):
cmd = ["docker", "exec", cid, "cat", f"/sys/fs/cgroup/{name}"]
return subprocess.run(cmd, capture_output=True, text=True, check=False).stdout.strip()
def percentile(values, q):
return (
round(sorted(values)[min(len(values) - 1, int(q * len(values)))] * 1000, 1)
if values
else None
)
def over_http(args):
api = Http(args.base_url, args.timeout)
alice, token = login(api, "alice")
bob, _ = login(api, "bob")
status, _, data = api.call(
"POST", "/api/v1/chats", {"title": "seed", "member_ids": [alice, bob]}, token=token
)
if status != 201:
raise SystemExit(f"chat create failed: {status}")
chat = json.loads(data)["id"]
fixed = sized_png(args.size) if args.size else None
counter, lock, results = iter(range(args.count)), threading.Lock(), []
ids = run(["docker", "compose", "ps", "-q", args.service]).split() if args.sample else []
idle = {cid: round(int(cgroup(cid, "memory.current") or 0) / 1048576, 1) for cid in ids}
peaks, stop = {}, threading.Event()
watcher = threading.Thread(target=sampler, args=(ids, peaks, stop), daemon=True)
if ids:
watcher.start()
def worker():
while True:
with lock:
i = next(counter, None)
if i is None:
return
body = fixed or png(8, 8)
began = time.monotonic()
path = f"/api/v1/chats/{chat}/attachments?filename=seed-{i}.png"
status, headers, data = api.call(
"POST", path, body, {"Content-Type": "image/png"}, token
)
created = json.loads(data)["id"] if status == 201 else None
results.append(
(status, time.monotonic() - began, headers.get("x-instance-id", ""), created)
)
started = time.monotonic()
threads = [threading.Thread(target=worker) for _ in range(args.concurrency)]
for t in threads:
t.start()
for t in threads:
t.join()
elapsed = time.monotonic() - started
stop.set()
if ids:
watcher.join()
created = [r[3] for r in results if r[3]]
for start in range(0, min(args.link, len(created)), 10):
body = {"text": "", "attachment_ids": created[start : min(start + 10, args.link)]}
api.call(
"POST",
f"/api/v1/chats/{chat}/messages",
body,
{"Idempotency-Key": uuid.uuid4().hex},
token,
)
if args.ids_out:
Path(args.ids_out).write_text("".join(f"{i}\n" for i in created))
codes, instances = {}, {}
for status, _, instance, _ in results:
codes[str(status)] = codes.get(str(status), 0) + 1
instances[instance] = instances.get(instance, 0) + 1
ok = [r[1] for r in results if r[0] == 201]
memory = {}
for cid in ids:
peak = cgroup(cid, "memory.peak")
events = dict(line.split() for line in cgroup(cid, "memory.events").splitlines())
memory[cid[:12]] = {
"idle_mib": idle[cid],
"docker_stats_peak_mib": round(peaks.get(cid[:12], 0.0), 1),
"cgroup_memory_peak_mib": round(int(peak) / 1048576, 1) if peak.isdigit() else None,
"oom_kill": int(events.get("oom_kill", 0)),
}
return {
"mode": "http",
"count": args.count,
"concurrency": args.concurrency,
"body_bytes": len(fixed) if fixed else "~200 (unique 8x8 PNG)",
"status_codes": codes,
"instances": len([i for i in instances if i]),
"linked_in_messages": min(args.link, len(created)),
"elapsed_seconds": round(elapsed, 3),
"successful_rps": round(len(ok) / elapsed, 2) if elapsed else None,
"latency_ms": {
"p50": percentile(ok, 0.5),
"p95": percentile(ok, 0.95),
"p99": percentile(ok, 0.99),
"max": percentile(ok, 1.0),
},
"memory": memory,
}
def main():
p = argparse.ArgumentParser(
description="Lab4 attachment volume generator: HTTP uploads or direct S3+PG seeding"
)
p.add_argument("--mode", choices=["http", "direct"], default="http")
p.add_argument("--count", type=int, default=1000)
p.add_argument("--concurrency", type=int, default=16)
p.add_argument(
"--size",
type=int,
default=0,
help="http: bytes per upload (one PNG reused); 0 = unique 8x8 PNGs",
)
p.add_argument(
"--link", type=int, default=0, help="http: reference the first N attachments from messages"
)
p.add_argument(
"--sample",
action="store_true",
help="http: sample docker stats and memory.peak of API containers",
)
p.add_argument("--service", default="api")
p.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080"))
p.add_argument("--timeout", type=float, default=60)
p.add_argument("--ids-out", metavar="FILE")
p.add_argument("--chat-id")
p.add_argument("--owner-id")
p.add_argument("--output", metavar="FILE")
args = p.parse_args()
report = direct(args) if args.mode == "direct" else over_http(args)
text = json.dumps(report, ensure_ascii=False, indent=2)
if args.output:
Path(args.output).write_text(text + "\n")
print(text)
if __name__ == "__main__":
main()
+11
View File
@@ -0,0 +1,11 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
service=${API_SERVICE:-api}
want=${1:-$(docker compose ps -q "$service" | wc -l | tr -d ' ')}
url=${LB_STATS_URL:-http://$(docker compose port lb 8404)/stats;csv}
deadline=$(($(date +%s) + ${WAIT_TIMEOUT:-60}))
until [ "$(curl -fsS "$url" | awk -F, -v b="$service" '$1 == b && $2 != "BACKEND" && $18 == "UP"' | wc -l)" -ge "$want" ]; do
[ "$(date +%s)" -lt "$deadline" ] || { echo "LB does not see $want UP $service servers" >&2; exit 4; }
sleep 0.5
done
+156
View File
@@ -0,0 +1,156 @@
#!/usr/bin/env python3
import argparse
import json
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
S3_CLIENT_SECRETS = ["s3_access_key_id", "s3_root_password", "s3_root_user", "s3_secret_access_key"]
POLICY = {
"lb": {"secrets": [], "volumes": [], "networks": ["backend", "edge"], "ports": True},
"api": {
"secrets": [
"cache_url",
"cursor_secret",
"database_url",
"s3_access_key_id",
"s3_secret_access_key",
],
"volumes": [],
"networks": ["backend"],
},
"migrate": {"secrets": ["database_url"], "volumes": [], "networks": ["backend"]},
"postgres": {"secrets": ["postgres_password"], "volumes": ["pgdata"], "networks": ["backend"]},
"valkey": {"secrets": ["valkey.conf"], "volumes": ["valkeydata"], "networks": ["backend"]},
"k6": {"secrets": [], "volumes": [], "networks": ["edge"]},
"s3": {
"secrets": ["s3_root_password", "s3_root_user"],
"volumes": ["s3data"],
"networks": ["backend"],
},
"s3-init": {"secrets": S3_CLIENT_SECRETS, "volumes": [], "networks": ["backend"]},
"s3-cli": {"secrets": S3_CLIENT_SECRETS, "volumes": [], "networks": ["backend"]},
}
def run(*command):
done = subprocess.run(command, cwd=str(ROOT), capture_output=True, text=True, check=False)
return done.returncode, done.stdout.strip()
def declared(name, service, networks, problems):
limits = service.get("deploy", {}).get("resources", {}).get("limits", {})
secrets = sorted(s["source"] for s in service.get("secrets", []))
volumes = sorted(v["source"] for v in service.get("volumes", []) if v["type"] == "volume")
nets = sorted(service.get("networks", {}))
ports = [
f"{p.get('host_ip', '0.0.0.0')}:{p.get('published')}" for p in service.get("ports", [])
]
want = POLICY.get(name)
if want is None:
problems.append(f"{name}: service is not covered by the policy")
want = {"secrets": [], "volumes": [], "networks": []}
if not limits.get("cpus") or not limits.get("memory"):
problems.append(f"{name}: no CPU or memory limit")
if "container_name" in service:
problems.append(f"{name}: container_name prevents scaling")
for kind, actual in (("secrets", secrets), ("volumes", volumes), ("networks", nets)):
if actual != want[kind]:
problems.append(f"{name}: {kind} {actual}, policy {want[kind]}")
if ports and not want.get("ports"):
problems.append(f"{name}: publishes {ports} in bypass of the balancer")
if any(not port.startswith("127.0.0.1:") for port in ports):
problems.append(f"{name}: publishes {ports} beyond loopback")
if name == "api" and not service.get("read_only"):
problems.append("api: root filesystem is writable")
internal = [n for n in nets if networks.get(n, {}).get("internal")]
return [
name,
f"{limits.get('cpus', '-')}/{int(limits.get('memory') or 0) >> 20}MiB",
",".join(ports) or "-",
",".join(secrets) or "-",
",".join(volumes) or "-",
",".join(f"{n}{'(internal)' if n in internal else ''}" for n in nets),
str(service.get("read_only", False)).lower(),
]
def running(problems):
_, ids = run("docker", "compose", "ps", "-q")
template = (
'{{index .Config.Labels "com.docker.compose.service"}} {{.Config.Hostname}} '
"{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}} {{json .HostConfig.PortBindings}}"
)
_, found = run("docker", "inspect", "-f", template, *ids.split())
rows = []
for line in found.splitlines():
service, host, cpus, memory, bindings = line.split(" ", 4)
published = [
f"{b['HostIp']}:{b['HostPort']}->{port}"
for port, items in (json.loads(bindings) or {}).items()
for b in items or []
]
if int(cpus) <= 0 or int(memory) <= 0:
problems.append(f"container {service}/{host}: no CPU or memory limit applied")
if published and service != "lb":
problems.append(f"container {service}/{host}: publishes {published}")
rows.append(
[service, host, f"{int(cpus) / 1e9:g}", f"{int(memory) >> 20}MiB", ",".join(published)]
)
return sorted(rows)
def table(header, rows):
widths = [max(len(str(r[i])) for r in [header, *rows]) for i in range(len(header))]
lines = [header, ["-" * w for w in widths], *rows]
return [" ".join(str(c).ljust(w) for c, w in zip(r, widths)).rstrip() for r in lines]
def main():
parser = argparse.ArgumentParser(description="limits, ports, secrets, volumes and networks")
parser.add_argument("--out", help="text report, written only after a complete check")
args = parser.parse_args()
problems = []
_, raw = run(
"docker",
"compose",
"--profile",
"bench",
"--profile",
"tools",
"config",
"--format",
"json",
)
config = json.loads(raw)
services = [
declared(name, service, config.get("networks", {}), problems)
for name, service in sorted(config["services"].items())
]
containers = running(problems)
if not any(row[0] == "api" for row in containers):
problems.append("no running api replicas to inspect")
lines = [
f"date: {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}",
"$ docker compose --profile bench --profile tools config --format json (declared)",
*table(["service", "cpus/mem", "ports", "secrets", "volumes", "networks", "ro"], services),
"",
"$ docker inspect (running containers: hostname = X-Instance-Id, applied limits)",
*table(["service", "hostname", "cpus", "memory", "published"], containers),
"",
*(f"FAIL: {p}" for p in problems),
"PASS: limits, ports, secrets, volumes and networks match the policy"
if not problems
else f"FAIL: {len(problems)} violation(s)",
]
text = "\n".join(lines) + "\n"
if args.out:
Path(args.out).write_text(text)
print(text, end="")
return 1 if problems else 0
if __name__ == "__main__":
sys.exit(main())
+14
View File
@@ -0,0 +1,14 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
started() {
for id in $(docker compose ps -aq); do
docker inspect --format '{{index .Config.Labels "com.docker.compose.service"}} {{.Config.Hostname}} {{.State.StartedAt}} {{.State.Status}}' "$id"
done | sort
}
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"
echo "== before: service hostname StartedAt status"
started
make --no-print-directory test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL="${1:-http://localhost:8080}"
echo "== after: service hostname StartedAt status"
started
+436
View File
@@ -0,0 +1,436 @@
#!/usr/bin/env python3
import argparse
import base64
import collections
import functools
import json
import os
import subprocess
import sys
import threading
import time
import urllib.error
import urllib.request
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
COUNTERS = ("wretr", "wredis", "econ", "eresp", "hrsp_5xx")
@functools.cache
def published(port, fallback):
command = ["docker", "compose", "port", "lb", str(port)]
found = subprocess.run(command, cwd=str(ROOT), capture_output=True, text=True, check=False)
return found.stdout.strip() or fallback
def base_url():
return os.environ.get("BASE_URL") or "http://" + published(8080, "localhost:8080")
def stats_url():
return os.environ.get("LB_STATS_URL") or f"http://{published(8404, '127.0.0.1:8404')}/stats;csv"
def call(method, path, body=None, headers=None, timeout=2.0):
data = None if body is None else json.dumps(body, ensure_ascii=False).encode()
head = {"Accept": "application/json"}
if data is not None:
head["Content-Type"] = "application/json"
head.update(headers or {})
request = urllib.request.Request(
base_url().rstrip("/") + path, data=data, headers=head, method=method
)
try:
with urllib.request.urlopen(request, timeout=timeout) as response:
payload = json.loads(response.read() or b"null")
return response.status, response.headers.get("X-Instance-Id", ""), payload
except urllib.error.HTTPError as error:
try:
payload = json.loads(error.read() or b"null")
except (OSError, ValueError):
payload = None
return error.code, error.headers.get("X-Instance-Id", ""), payload
except (OSError, ValueError):
return 0, "", None
def emit(report, out=None):
text = json.dumps(report, ensure_ascii=False, indent=2)
if out:
Path(out).parent.mkdir(parents=True, exist_ok=True)
Path(out).write_text(text + "\n")
print(text)
def created(result, what):
if result[0] != 201:
raise SystemExit(f"{what} failed: HTTP {result[0]}")
return result[2]
def actor(label):
username = f"p_{label}_{uuid.uuid4().hex[:12]}"
password = "Probe-" + uuid.uuid4().hex
body = {"username": username, "display_name": "probe " + label, "password": password}
user = created(call("POST", "/api/v1/users", body, timeout=10), "register")
basic = base64.b64encode(f"{username}:{password}".encode()).decode()
login = call(
"POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}, timeout=10
)
return {
"id": user["id"],
"auth": {"Authorization": "Bearer " + created(login, "login")["token"]},
}
def chat_of(owner, other):
body = {"title": "probe", "member_ids": [owner["id"], other["id"]]}
return created(call("POST", "/api/v1/chats", body, owner["auth"], timeout=10), "chat")["id"]
def send(owner, chat, text, key):
headers = dict(owner["auth"], **{"Idempotency-Key": key})
return call("POST", f"/api/v1/chats/{chat}/messages", {"text": text}, headers)
def lb_rows(backend="api"):
try:
with urllib.request.urlopen(stats_url(), timeout=2) as response:
lines = response.read().decode().splitlines()
except OSError:
return None
header = lines[0].lstrip("# ").split(",")
rows = [dict(zip(header, line.split(","))) for line in lines[1:] if line]
return [r for r in rows if r["pxname"] == backend]
def lb_up():
rows = lb_rows()
if rows is None:
return None
return sum(1 for r in rows if r["svname"] != "BACKEND" and r["status"].startswith("UP"))
def lb_counters():
row = next((r for r in lb_rows() or [] if r["svname"] == "BACKEND"), {})
return {name: int(row.get(name) or 0) for name in COUNTERS}
def run_k6(out, env, on_start=None):
Path(out).mkdir(parents=True, exist_ok=True)
started = []
process = subprocess.Popen(
[str(ROOT / "scripts/k6.sh"), str(out)],
env=dict(os.environ, **env),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
)
with (Path(out) / "k6.txt").open("w") as log:
for line in process.stdout:
log.write(line)
if not started and "LOAD-START" in line:
started.append(time.time())
if on_start:
on_start()
return process.wait(), started[0] if started else None
def k6_metrics(path, duration):
metrics = json.loads(Path(path).read_text())["metrics"]
def value(name, field):
return metrics.get(name + "{scenario:measure}", {}).get("values", {}).get(field, 0.0)
count = value("http_reqs", "count")
return {
"requests": int(count),
"rps": round(count / duration, 2),
"ok_rps": round(count * (1 - value("http_req_failed", "rate")) / duration, 2),
"failed_ratio": round(value("http_req_failed", "rate"), 5),
"dropped_iterations": int(value("dropped_iterations", "count")),
"p50_ms": round(value("http_req_duration", "med"), 2),
"p95_ms": round(value("http_req_duration", "p(95)"), 2),
"p99_ms": round(value("http_req_duration", "p(99)"), 2),
}
def race(alice, chat, parallel):
key, text = uuid.uuid4().hex, "same key " + uuid.uuid4().hex
barrier, results = threading.Barrier(parallel), []
def worker():
barrier.wait()
results.append(send(alice, chat, text, key))
threads = [threading.Thread(target=worker) for _ in range(parallel)]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
bodies = [r[2] for r in results if r[0] == 201]
identical = bool(bodies) and all(b == bodies[0] for b in bodies)
return text, results, identical, send(alice, chat, text + " changed", key)[0]
def check(args):
alice, bob = actor("alice"), actor("bob")
chat = chat_of(alice, bob)
seen = collections.Counter(
call("GET", "/api/v1/users/me", headers=alice["auth"])[1] for _ in range(args.samples)
)
rounds = [race(alice, chat, args.parallel) for _ in range(min(args.rounds, 100))]
page = call("GET", f"/api/v1/chats/{chat}/messages?limit=100", headers=bob["auth"])[2]
copies = collections.Counter(m.get("text") for m in (page or {"items": []})["items"])
statuses = collections.Counter(r[0] for _, results, _, _ in rounds for r in results)
served = collections.Counter(r[1] for _, results, _, _ in rounds for r in results)
same = {
"rounds": len(rounds),
"parallel": args.parallel,
"statuses": dict(statuses),
"instances": dict(served),
"identical_bodies_rounds": sum(1 for _, _, identical, _ in rounds if identical),
"single_copy_rounds": sum(1 for text, _, _, _ in rounds if copies[text] == 1),
"conflict_rounds": sum(1 for _, _, _, other in rounds if other == 409),
}
passed = (
len(seen) >= 2
and statuses == {201: len(rounds) * args.parallel}
and len(served) >= 2
and same["identical_bodies_rounds"] == len(rounds)
and same["single_copy_rounds"] == len(rounds)
and same["conflict_rounds"] == len(rounds)
)
emit({"instances_one_token": dict(seen), "same_key": same, "pass": passed}, args.out)
return 0 if passed else 1
def record(stop, alice, chat):
probes, trace = [], []
def prober():
key = uuid.uuid4().hex
while not stop.is_set():
started = time.time()
status, instance, _ = send(alice, chat, "probe " + key, key)
probes.append((started, status, round(1000 * (time.time() - started), 1), instance))
if status == 201:
key = uuid.uuid4().hex
time.sleep(0.1)
def poller():
while not stop.is_set():
trace.append((time.time(), lb_up()))
time.sleep(0.1)
threads = [threading.Thread(target=t, daemon=True) for t in (prober, poller)]
for thread in threads:
thread.start()
return probes, trace, threads
def write_csv(path, header, rows):
with path.open("w") as handle:
handle.write(header + "\n")
handle.writelines(",".join(str(v) for v in row) + "\n" for row in rows)
def summarize(args, probes, trace, action, expected):
t0, end = action["start"], action["end"]
label = args.label or os.environ.get("TARGET_INSTANCE_ID", "action")
failures = [t - t0 for t, s, _, _ in probes if s != 201 and t >= t0]
down = [t for t, up in trace if t >= t0 and up is not None and up < expected]
back = [t for t, up in trace if down and t > down[0] and up is not None and up >= expected]
outage = [t for t in down if not back or t < back[0]] or [end]
served = [(t, i) for t, s, _, i in probes if s == 201]
before = collections.Counter(i for t, i in served if t < t0)
during = collections.Counter(i for t, i in served if outage[0] <= t <= outage[-1])
after = collections.Counter(i for t, i in served if t > end)
return {
"label": label,
"action": args.command,
"action_exit": action["exit"],
"action_seconds": round(end - t0, 3),
"action_finished_before_load_end": end < action["load_end"],
"lb_up_before": expected,
"lb_up_min": min((up for _, up in trace if up is not None), default=None),
"detection_seconds": round(down[0] - t0, 3) if down else None,
"rejoin_seconds": round(back[0] - t0, 3) if back else None,
"probe": {
"attempts": len(probes),
"failed_attempts": sum(1 for _, s, _, _ in probes if s != 201),
"first_failure_s": round(failures[0], 3) if failures else None,
"recovery_s": round(failures[-1], 3) if failures else 0.0,
"instances_before": dict(before),
"instances_during_outage": dict(during),
"instances_after": dict(after),
},
}
def verdict(args, result):
probe, k6 = result["probe"], result["k6"]
checks = {
"action_ok": result["action_exit"] == 0 and result["action_finished_before_load_end"],
"load_errors_ok": k6["exit"] == 0 and k6["failed_ratio"] <= args.max_failed_ratio,
"write_recovery_ok": probe["recovery_s"] <= args.max_recovery,
}
if args.expect == "dip":
checks["detected"] = result["detection_seconds"] is not None
checks["target_excluded"] = probe["instances_during_outage"].get(result["label"], 0) == 0
checks["rejoined"] = result["rejoin_seconds"] is not None and bool(
probe["instances_after"].get(result["label"])
)
if args.expect == "replace":
checks["capacity_kept"] = (result["lb_up_min"] or 0) >= result["lb_up_before"]
old, new = set(probe["instances_before"]), set(probe["instances_after"])
checks["all_replaced"] = bool(new) and not old & new
return checks
def load(args):
out = Path(args.out)
out.mkdir(parents=True, exist_ok=True)
expected = lb_up() or 0
alice, bob = actor("alice"), actor("bob")
stop, action = threading.Event(), {}
probes, trace, threads = record(stop, alice, chat_of(alice, bob))
def act():
time.sleep(args.warmup + args.lead)
action["start"] = time.time()
with (out / "action.txt").open("w") as log:
action["exit"] = subprocess.call(
args.command, cwd=str(ROOT), stdout=log, stderr=subprocess.STDOUT
)
action["end"] = time.time()
runner = threading.Thread(target=act)
env = {"RATE": str(args.rate), "WARMUP": str(args.warmup), "DURATION": str(args.duration)}
counters = lb_counters()
code, load_start = run_k6(out / "k6", env, runner.start)
if load_start is not None:
runner.join()
action["load_end"] = load_start + args.warmup + args.duration
stop.set()
for thread in threads:
thread.join()
if "start" not in action:
raise SystemExit(f"k6 failed before the action (exit {code}), see {out / 'k6/k6.txt'}")
t0 = action["start"]
write_csv(
out / "probe.csv",
"t_rel_s,status,latency_ms,instance",
((round(t - t0, 3), *r) for t, *r in probes),
)
write_csv(out / "lb_up.csv", "t_rel_s,up", ((round(t - t0, 3), up) for t, up in trace))
result = summarize(args, probes, trace, action, expected)
result["lb_counters"] = {k: v - counters[k] for k, v in lb_counters().items()}
result["k6"] = dict(
k6_metrics(out / "k6/summary.json", args.duration), exit=code, rate=args.rate
)
result["checks"] = verdict(args, result)
result["pass"] = all(result["checks"].values())
(out / "result.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
print(json.dumps(result, ensure_ascii=False))
return 0 if result["pass"] else 1
def detect(args):
before = lb_up() or 0
until = before - 1 if args.until is None else args.until
started = time.time()
process = subprocess.Popen(args.command, cwd=str(ROOT))
detected = acted = None
while time.time() - started < args.timeout and (detected is None or acted is None):
if acted is None and process.poll() is not None:
acted = time.time() - started
if detected is None and (lb_up() or 0) <= until:
detected = time.time() - started
time.sleep(0.05)
code = process.wait()
acted = time.time() - started if acted is None else acted
status, instance, payload = call("GET", "/health/live")
restored = None
if args.restore:
begun = time.time()
subprocess.call(["sh", "-c", args.restore], cwd=str(ROOT))
while (lb_up() or 0) < before and time.time() - begun < 3 * args.timeout:
time.sleep(0.1)
restored = round(time.time() - begun, 3) if (lb_up() or 0) >= before else None
checks = {
"action_ok": code == 0,
"detected_in_time": detected is not None and detected <= args.max + args.slack,
"lb_answer_ok": status == args.lb_status,
}
if args.within_action:
checks["excluded_before_action_end"] = detected is not None and detected < acted
if args.restore:
checks["restored"] = restored is not None
report = {
"action": args.command,
"action_exit": code,
"action_seconds": round(acted, 3),
"lb_up_before": before,
"lb_up_counted_as_detected": until,
"detection_seconds": None if detected is None else round(detected, 3),
"declared_max_seconds": args.max,
"measurement_slack_seconds": args.slack,
"lb_answer_after_detection": {
"path": "/health/live",
"status": status,
"x_instance_id_present": bool(instance),
"error_code": ((payload or {}).get("error") or {}).get("code"),
},
"restore": args.restore,
"restore_seconds": restored,
"lb_up_after_restore": lb_up(),
"checks": checks,
"pass": all(checks.values()),
}
emit(report, args.out)
return 0 if report["pass"] else 1
def main():
parser = argparse.ArgumentParser(description="LB probes over the public HTTP contract")
sub = parser.add_subparsers(dest="mode", required=True)
one = sub.add_parser("check", help="instances behind one token and same-key races")
one.add_argument("--samples", type=int, default=40)
one.add_argument("--parallel", type=int, default=8)
one.add_argument("--rounds", type=int, default=1)
one.add_argument("--out", help="JSON file, written only after a complete report")
under = sub.add_parser("load", help="run an action under k6 load and measure recovery")
under.add_argument("--out", required=True)
under.add_argument("--label")
under.add_argument("--expect", choices=("dip", "replace"), default="dip")
under.add_argument("--rate", type=int, default=int(os.environ.get("RATE", "50")))
under.add_argument("--warmup", type=int, default=10)
under.add_argument("--duration", type=int, default=int(os.environ.get("DURATION", "60")))
under.add_argument("--lead", type=int, default=15)
under.add_argument("--max-failed-ratio", type=float, default=0.01)
under.add_argument("--max-recovery", type=float, default=10.0)
under.add_argument("command", nargs=argparse.REMAINDER)
watch = sub.add_parser("detect", help="time until the LB takes replicas out of rotation")
watch.add_argument("--max", type=float, required=True, help="declared worst detection, s")
watch.add_argument("--slack", type=float, default=0.25, help="docker CLI and polling, s")
watch.add_argument("--until", type=int, help="UP servers counted as detected")
watch.add_argument("--within-action", action="store_true")
watch.add_argument("--lb-status", type=int, default=200)
watch.add_argument("--restore", help="shell command that brings replicas back")
watch.add_argument("--timeout", type=float, default=20)
watch.add_argument("--out", help="JSON file, written only after a complete report")
watch.add_argument("command", nargs=argparse.REMAINDER)
args = parser.parse_args()
if args.mode == "check":
return check(args)
args.command = [c for c in args.command if c != "--"]
if not args.command:
parser.error(f"{args.mode} needs an action command after --")
return load(args) if args.mode == "load" else detect(args)
if __name__ == "__main__":
sys.exit(main())
+12
View File
@@ -0,0 +1,12 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
defined=$(docker compose config --services)
stores=""
for service in ${STATE_SERVICES:-postgres valkey s3}; do
if printf '%s\n' "$defined" | grep -qx "$service"; then stores="$stores $service"; fi
done
[ -n "$stores" ] || { echo "no state services found" >&2; exit 3; }
docker compose stop lb "${API_SERVICE:-api}"
docker compose restart $stores
docker compose up -d --no-recreate --wait --wait-timeout "${WAIT_TIMEOUT:-75}"
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
docker compose up -d --no-deps --no-recreate --wait --wait-timeout "${WAIT_TIMEOUT:-60}" "${API_SERVICE:-api}"
scripts/lb-wait.sh
+20
View File
@@ -0,0 +1,20 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
service=${API_SERVICE:-api}
show() { docker inspect -f '{{.Config.Hostname}} image={{slice .Image 7 19}}' "$1"; }
echo "$(date -u +%T) migrate once before the first replacement"
docker compose run --rm --no-deps -T migrate 2>&1
old=$(docker compose ps -q "$service")
count=$(printf '%s\n' "$old" | grep -c .)
for id in $old; do
running=$(docker compose ps -q "$service")
docker compose up -d --no-deps --no-recreate --wait --wait-timeout "${WAIT_TIMEOUT:-90}" --scale "$service=$((count + 1))" "$service" 2>&1
scripts/lb-wait.sh $((count + 1))
new=$(docker compose ps -q "$service" | grep -v -F "$running")
echo "$(date -u +%T) replace $(show "$id") -> $(show "$new")"
docker stop -t "${STOP_TIMEOUT:-6}" "$id" >/dev/null
docker rm "$id" >/dev/null
done
scripts/lb-wait.sh "$count"
echo "$(date -u +%T) done: $count replicas replaced"
+66
View File
@@ -0,0 +1,66 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
cli() { docker compose run --rm --no-deps -T s3-cli "$@"; }
snapshot() {
cli sh -s <<'EOF' | grep -v '_date'
ak=$(cat /run/secrets/s3_access_key_id)
for c in "bucket list root" "bucket anonymous get root/$S3_BUCKET" "admin policy info root messenger-api" "admin user info root $ak" "bucket lifecycle rule export root/$S3_BUCKET"; do
rc $c --json
done | sed "s/$ak/<api-key>/g"
EOF
}
failed=0
verdict() { if [ "$2" = "$3" ]; then echo "PASS $1: $3"; else echo "FAIL $1: ожидалось $2, получено $3"; failed=1; fi; }
echo "== $(date -u +%FT%TZ) S3 privacy and bootstrap checks"
verdict "s3: опубликованные host-порты" "нет" "$(docker inspect -f '{{json .NetworkSettings.Ports}}' "$(docker compose ps -q s3)" | grep -q HostPort && echo есть || echo нет)"
verdict "s3: консоль :9001 внутри контейнера" "000" "$(docker compose exec -T s3 curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:9001/ || true)"
before=$(snapshot)
first=$(docker compose run --rm --no-deps -T s3-init >/dev/null 2>&1 && echo 0 || echo 1)
second=$(docker compose run --rm --no-deps -T s3-init 2>/dev/null || echo FAILED)
after=$(snapshot)
verdict "s3-init: повторный запуск 1" "0" "$first"
verdict "s3-init: повторный запуск 2" "0" "$(echo "$second" | grep -q '^s3-init:' && echo 0 || echo 1)"
verdict "s3-init: состояние IAM/bucket не изменилось" "same" "$([ "$before" = "$after" ] && echo same || echo changed)"
cli sh -s <<'EOF' || failed=1
b=$S3_BUCKET
key="attachments/s3-check-$(date +%s)-$$"
other="s3-check-$(date +%s)-$$"
head -c 4096 /dev/urandom > /tmp/probe
result=0
expect() {
want=$1; name=$2; shift 2
if "$@" >/tmp/out 2>&1; then got=allow; elif grep -qE '403|AccessDenied|set_bucket_policy: service error' /tmp/out; then got=deny; else got=error; fi
if [ "$got" = "$want" ]; then echo "PASS $name: $got"; else echo "FAIL $name: ожидалось $want, получено $got"; result=1; fi
}
expect allow "api: PutObject" rc put /tmp/probe "api/$b/$key"
expect allow "api: HeadObject" rc object stat "api/$b/$key"
expect allow "api: GetObject" rc object show "api/$b/$key"
expect allow "api: ListBucket" rc object list "api/$b/attachments/"
expect deny "api: CreateBucket" rc bucket create "api/$other"
expect deny "api: DeleteBucket" rc bucket remove "api/$b"
expect deny "api: PutBucketPolicy (public)" rc bucket anonymous set public "api/$b"
expect deny "api: PutBucketLifecycle" rc bucket lifecycle rule import "api/$b" /s3/lifecycle.json
expect deny "api: admin ListUsers" rc admin user list api
rc bucket create --ignore-existing "root/$other" >/dev/null
expect deny "api: PutObject в чужой bucket" rc put /tmp/probe "api/$other/x"
expect deny "api: ListBucket чужого bucket" rc object list "api/$other"
rc bucket remove "root/$other" >/dev/null
expect deny "anonymous: HeadObject" rc object stat "anon/$b/$key"
expect deny "anonymous: ListBucket" rc object list "anon/$b"
expect deny "anonymous: PutObject" rc put /tmp/probe "anon/$b/attachments/anonymous"
status=$(wget -S -O /dev/null "$S3_ENDPOINT/$b/$key" 2>&1 | sed -n 's/^ *HTTP\/[0-9.]* \([0-9]*\).*/\1/p' | head -n 1)
if [ "$status" = 403 ]; then echo "PASS anonymous: прямой HTTP GET объекта из отдельного контейнера: $status"; else echo "FAIL anonymous: прямой HTTP GET: $status"; result=1; fi
permission=$(rc bucket anonymous get "root/$b" --json | sed -n 's/.*"permission": *"\([a-z]*\)".*/\1/p')
if [ "$permission" = private ]; then echo "PASS bucket: anonymous policy: $permission"; else echo "FAIL bucket: anonymous policy: $permission"; result=1; fi
expect allow "api: DeleteObject" rc object remove --force "api/$b/$key"
host=$(echo "$S3_ENDPOINT" | sed 's#^[a-z]*://##; s#:.*##')
if wget -T 2 -O /dev/null "http://$host:9001/" >/tmp/out 2>&1; then echo "FAIL s3: консоль :9001 отвечает из сети стенда"; result=1
elif grep -q refused /tmp/out; then echo "PASS s3: консоль :9001 из отдельного контейнера сети стенда: connection refused"
else echo "FAIL s3: консоль :9001 из сети стенда: $(tail -n 1 /tmp/out)"; result=1; fi
exit $result
EOF
echo "== IAM/bucket snapshot (без секретов)"
echo "$after"
if [ "$failed" != 0 ]; then echo "RESULT: FAIL"; exit 1; fi
echo "RESULT: PASS"
+24
View File
@@ -0,0 +1,24 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
umask 077
mkdir -p secrets
random() { od -An -N32 -tx1 /dev/urandom | tr -d ' \n'; }
put() {
if [ ! -s "secrets/$1" ]; then
printf '%s' "$2" > "secrets/$1"
chmod 644 "secrets/$1"
echo "создан secrets/$1"
fi
}
put postgres_password "$(random)"
put database_url "postgresql://messenger:$(cat secrets/postgres_password)@postgres:5432/messenger"
put cursor_secret "$(random)"
put valkey_password "$(random)"
put valkey.conf "requirepass $(cat secrets/valkey_password)"
put cache_url "redis://:$(cat secrets/valkey_password)@valkey:6379/0"
put s3_root_user "root$(random | cut -c1-16)"
put s3_root_password "$(random)"
put s3_access_key_id "api$(random | cut -c1-16)"
put s3_secret_access_key "$(random)"
[ -f .env ] || { cp .env.example .env; echo "создан .env из .env.example"; }
+59
View File
@@ -0,0 +1,59 @@
\set ON_ERROR_STOP on
\set big '''00000000-0000-7000-8000-00000000c001'''
\set cursor_at '''2026-01-01T02:46:40Z'''
\set cursor_id '''00000000-0000-7000-8001-000000002710'''
PREPARE history_first (uuid, int) AS
SELECT id, chat_id, sender_id, text, created_at
FROM messages
WHERE chat_id = $1
ORDER BY created_at, id
LIMIT $2;
PREPARE history_after (uuid, timestamptz, uuid, int) AS
SELECT id, chat_id, sender_id, text, created_at
FROM messages
WHERE chat_id = $1 AND (created_at, id) > ($2, $3)
ORDER BY created_at, id
LIMIT $4;
PREPARE history_offset (uuid, int, int) AS
SELECT id, chat_id, sender_id, text, created_at
FROM messages
WHERE chat_id = $1
ORDER BY created_at, id
LIMIT $2 OFFSET $3;
SELECT count(*) AS chat_messages FROM messages WHERE chat_id = :big;
SELECT count(*) AS all_messages FROM messages;
SELECT pg_size_pretty(pg_relation_size('messages_history_idx')) AS history_index_size;
\echo '== Первая страница, limit=50 (запрос HISTORY_FIRST приложения, LIMIT limit+1)'
EXPLAIN (ANALYZE, BUFFERS) EXECUTE history_first(:big, 51);
\echo '== Глубокая страница: keyset после 10 000-го сообщения (запрос HISTORY_AFTER приложения)'
EXPLAIN (ANALYZE, BUFFERS) EXECUTE history_after(:big, :cursor_at, :cursor_id, 51);
\echo '== Та же глубокая страница через OFFSET 10000 (не используется приложением, для сравнения)'
EXPLAIN (ANALYZE, BUFFERS) EXECUTE history_offset(:big, 51, 10000);
\echo '== Проверка: keyset и OFFSET возвращают одну и ту же страницу'
SELECT (
SELECT array_agg(id ORDER BY created_at, id) FROM (
SELECT id, created_at FROM messages
WHERE chat_id = :big AND (created_at, id) > (:cursor_at, :cursor_id)
ORDER BY created_at, id LIMIT 51
) AS keyset
) = (
SELECT array_agg(id ORDER BY created_at, id) FROM (
SELECT id, created_at FROM messages
WHERE chat_id = :big
ORDER BY created_at, id LIMIT 51 OFFSET 10000
) AS skipped
) AS same_page;
\echo '== Контрольный план без индекса истории (DROP INDEX внутри транзакции, затем ROLLBACK)'
BEGIN;
DROP INDEX messages_history_idx;
EXPLAIN (ANALYZE, BUFFERS) EXECUTE history_after(:big, :cursor_at, :cursor_id, 51);
ROLLBACK;
+40
View File
@@ -0,0 +1,40 @@
\set ON_ERROR_STOP on
\set author '''00000000-0000-7000-8000-00000000a001'''
\set big '''00000000-0000-7000-8000-00000000c001'''
\set epoch '''2026-01-01T00:00:00Z'''
INSERT INTO users (id, username, display_name, password_hash, created_at)
VALUES (:author, 'seed_author', 'Синтетический автор', '$argon2id$v=19$m=19456,t=2,p=1$c2VlZA$c2VlZA', :epoch)
ON CONFLICT DO NOTHING;
INSERT INTO chats (id, title, created_at)
SELECT ('00000000-0000-7000-8000-00000000c' || lpad(to_hex(chat), 3, '0'))::uuid,
'Синтетический чат ' || chat, :epoch
FROM generate_series(1, 21) AS chat
ON CONFLICT DO NOTHING;
INSERT INTO chat_members (chat_id, user_id, position, chat_created_at)
SELECT id, :author, 1, created_at FROM chats WHERE title LIKE 'Синтетический чат %'
ON CONFLICT DO NOTHING;
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
SELECT ('00000000-0000-7000-8001-' || lpad(to_hex(n), 12, '0'))::uuid, :big, :author,
'синтетическое сообщение ' || n, :epoch::timestamptz + n * interval '1 second'
FROM generate_series(1, 20000) AS n
ON CONFLICT DO NOTHING;
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
SELECT ('00000000-0000-7000-8002-' || lpad(to_hex(chat * 1000 + n), 12, '0'))::uuid,
('00000000-0000-7000-8000-00000000c' || lpad(to_hex(chat), 3, '0'))::uuid, :author,
'фоновое сообщение ' || n, :epoch::timestamptz + n * interval '1 second'
FROM generate_series(2, 21) AS chat, generate_series(1, 500) AS n
ON CONFLICT DO NOTHING;
UPDATE chats
SET last_message_at = (SELECT max(created_at) FROM messages WHERE messages.chat_id = chats.id)
WHERE title LIKE 'Синтетический чат %';
ANALYZE;
SELECT count(*) FILTER (WHERE chat_id = :big) AS big_chat_messages, count(*) AS all_messages
FROM messages;
+9
View File
@@ -0,0 +1,9 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
cd "$(dirname "$0")/.."
ids=$(docker compose ps -q "${API_SERVICE:-api}")
[ -n "$ids" ] || { echo "no running ${API_SERVICE:-api} replicas" >&2; exit 3; }
target=$(docker inspect -f '{{.Id}} {{.Config.Hostname}}' $ids | awk -v t="$TARGET_INSTANCE_ID" '$2 == t {print $1}')
[ -n "$target" ] || { echo "no running ${API_SERVICE:-api} replica answers as $TARGET_INSTANCE_ID" >&2; exit 3; }
docker stop -t "${STOP_TIMEOUT:-6}" "$target" >/dev/null
+117
View File
@@ -0,0 +1,117 @@
#!/usr/bin/env python3
import argparse
import json
import os
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
seed = __import__("lab4_seed")
FORMAT = "{:>5} {:>5} {:>4} {:<18} {:>7} {:>7} {:>8} {:>8} {:>7} {:>8} {:>6}"
COLUMNS = FORMAT.format(
"slots",
"size",
"conc",
"codes",
"p50_ms",
"p95_ms",
"api_idle",
"api_peak",
"api_oom",
"lb_peak",
"lb_oom",
)
OVERRIDES = ("UPLOAD_CONCURRENCY", "MALLOC_MMAP_THRESHOLD_")
def recreate(slots):
env = dict(os.environ, UPLOAD_CONCURRENCY=str(slots))
if not slots:
env = {k: v for k, v in os.environ.items() if k not in OVERRIDES}
cmd = ["docker", "compose", "up", "-d", "--no-deps", "--force-recreate", "--wait", "api", "lb"]
subprocess.run(cmd, cwd=ROOT, env=env, check=True, capture_output=True)
subprocess.run([str(ROOT / "scripts/lb-wait.sh")], cwd=ROOT, check=True)
def balancer():
cid = seed.run(["docker", "compose", "ps", "-q", "lb"]).strip()
events = dict(row.split() for row in seed.cgroup(cid, "memory.events").splitlines())
peak = seed.cgroup(cid, "memory.peak")
return {
"cgroup_memory_peak_mib": round(int(peak) / 1048576, 1),
"oom_kill": int(events["oom_kill"]),
}
def line(report):
memory = report["memory"].values()
codes = ",".join(f"{k}x{v}" for k, v in sorted(report["status_codes"].items()))
return FORMAT.format(
report["upload_concurrency"],
f"{report['body_bytes'] / 1048576:.1f}M",
report["concurrency"],
codes,
report["latency_ms"]["p50"],
report["latency_ms"]["p95"],
max(m["idle_mib"] for m in memory),
max(m["cgroup_memory_peak_mib"] or 0 for m in memory),
sum(m["oom_kill"] for m in memory),
report["lb"]["cgroup_memory_peak_mib"],
report["lb"]["oom_kill"],
)
def main():
p = argparse.ArgumentParser(description="Peak memory of API replicas under concurrent uploads")
p.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080"))
p.add_argument("--slots", type=int, nargs="+", default=[64, 3])
p.add_argument("--sizes", type=int, nargs="+", default=[1 << 20, 10 << 20])
p.add_argument("--levels", type=int, nargs="+", default=[10, 20, 40])
p.add_argument("--rounds", type=int, default=2, help="uploads per concurrent client")
p.add_argument("--out", default="evidence/upload-memory")
args = p.parse_args()
out = ROOT / args.out
out.mkdir(parents=True, exist_ok=True)
commit = seed.run(["git", "rev-parse", "--short", "HEAD"]).strip()
threshold = os.environ.get("MALLOC_MMAP_THRESHOLD_", "131072")
lines = [
f"# {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} commit {commit}; api x2: 1 CPU,"
f" 512 MiB, 2 workers; MALLOC_MMAP_THRESHOLD_={threshold}; {args.rounds} uploads"
" per client; peaks from cgroup memory.peak, MiB",
COLUMNS,
]
print("\n".join(lines), flush=True)
try:
for slots in args.slots:
for size in args.sizes:
for level in args.levels:
recreate(slots)
run = argparse.Namespace(
base_url=args.base_url,
timeout=60,
size=size,
count=level * args.rounds,
concurrency=level,
sample=True,
service="api",
link=0,
ids_out=None,
)
report = {"upload_concurrency": slots, **seed.over_http(run)}
report["lb"] = balancer()
name = f"slots{slots}-{size >> 20}mib-c{level}.json"
(out / name).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
lines.append(line(report))
print(lines[-1], flush=True)
finally:
recreate(None)
(out / "summary.txt").write_text("\n".join(lines) + "\n")
if __name__ == "__main__":
main()