Forward the lab 4 foundation and migrate the lab 5 pipeline to SQLAlchemy

This commit is contained in:
Artyom
2026-11-06 09:12:25 +03:00
parent ae9f60fe1f
commit 1217da5f5e
134 changed files with 8243 additions and 1409 deletions
+73
View File
@@ -0,0 +1,73 @@
import argparse
import json
import os
import shlex
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SETTINGS = (
"COMPOSE_PROJECT_NAME",
"COMPOSE_FILE",
"LB_HTTP_PORT",
"LB_STATS_PORT",
"BASE_URL",
"EVIDENCE_DIR",
"RECONCILE_EVIDENCE_DIR",
"NO_PROXY",
"S3_CPUS",
"S3_MEMORY",
"UPLOAD_GRACE_SECONDS",
"API_IMAGE",
"API_REPLICAS",
"WEB_CONCURRENCY",
"MALLOC_MMAP_THRESHOLD_",
)
def main():
parser = argparse.ArgumentParser(description="Capture a command without overwriting evidence")
parser.add_argument("output", type=Path)
parser.add_argument("command", nargs=argparse.REMAINDER)
args = parser.parse_args()
command = args.command[1:] if args.command[:1] == ["--"] else args.command
if not command:
parser.error("command is required")
args.output.parent.mkdir(parents=True, exist_ok=True)
commit = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip()
started = time.monotonic()
with args.output.open("x") as output:
def write(line):
line = line.replace(str(ROOT), "<repo>")
output.write(line)
output.flush()
sys.stdout.write(line)
sys.stdout.flush()
write(f"date: {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} commit: {commit}\n")
write(
"environment: "
+ json.dumps({k: os.environ[k] for k in SETTINGS if k in os.environ})
+ "\n"
)
write("$ " + shlex.join(command) + "\n")
with subprocess.Popen(
command,
cwd=ROOT,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
errors="replace",
) as process:
for line in process.stdout:
write(line)
code = process.wait()
write(f"exit_code: {code} elapsed_seconds: {time.monotonic() - started:.3f}\n")
return code
if __name__ == "__main__":
sys.exit(main())
+53
View File
@@ -0,0 +1,53 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
repo=$PWD
project="hlpipe-lab5-clean-$(date -u +%Y%m%d%H%M%S)"
dir="$repo/.verification/$project"
out="$repo/${EVIDENCE_DIR:-evidence}/clean-clone.txt"
sha=$(git rev-parse HEAD)
[ ! -e "$out" ] || { echo "Choose a new EVIDENCE_DIR" >&2; exit 2; }
mkdir -p "$(dirname "$dir")" "$(dirname "$out")"
exec 3>&1
exec > "$out" 2>&1
cleanup() {
status=$?
trap - EXIT HUP INT TERM
if [ -d "$dir/.git" ]; then
cd "$dir"
docker compose --profile bench --profile tools --profile reconcile down || status=1
docker volume ls --filter "label=com.docker.compose.project=$project"
fi
echo "exit_code: $status; volumes retained"
sed "s#$repo#<repo>#g" "$out" >&3
exit "$status"
}
trap cleanup EXIT
trap 'exit 130' HUP INT TERM
step() { echo "\$ $*"; "$@"; }
export COMPOSE_PROJECT_NAME="$project" COMPOSE_FILE=compose.yaml
export LB_HTTP_PORT=18085 LB_STATS_PORT=25085 BROKER_MANAGEMENT_PORT=25086 BASE_URL=http://127.0.0.1:18085
export NO_PROXY=localhost,127.0.0.1,::1
unset API_IMAGE
echo "date: $(date -u +%FT%TZ) commit: $sha project: $project"
echo "HTTP=$LB_HTTP_PORT stats=$LB_STATS_PORT"
step git clone --quiet --no-hardlinks "$repo" "$dir"
cd "$dir"
step git checkout --quiet --detach "$sha"
step make secrets
step docker compose --env-file .env config --quiet
step make up
step docker compose ps -a --format '{{.Service}} {{.State}} {{.Status}} {{.Ports}}'
step make test
step make test-persistence ACTION_SCRIPT=scripts/restart.sh
step make test-failover ACTION_SCRIPT=scripts/stop-one.sh
step docker compose ps -a --format '{{.Service}} {{.Name}} {{.State}}' api
step make restore
step make scenario SCENARIO=failover-upload
step make restore
step make scenario SCENARIO=s3-outage
step sh scripts/s3-check.sh
step make reconcile
step make migrate
step make down
step docker volume ls --filter "label=com.docker.compose.project=$project"
+1 -1
View File
@@ -8,7 +8,7 @@ replica() { docker compose ps -q "${API_SERVICE:-api}" | head -1; }
detect() { python3 scripts/probe.py detect "$@"; }
failed=0
detect --max 1.5 --within-action --restore scripts/restore.sh --out "$out/sigterm.json" \
-- docker stop -t "${STOP_TIMEOUT:-6}" "$(replica)" || failed=1
-- docker stop -t "${STOP_TIMEOUT:-7}" "$(replica)" || failed=1
detect --max 1.5 --restore scripts/restore.sh --out "$out/kill.json" \
-- docker kill "$(replica)" || failed=1
hung=$(replica)
+15
View File
@@ -0,0 +1,15 @@
#!/bin/sh
set -eu
out=${1:?usage: docker-stats.sh <csv>}
vm=$(dirname "$out")/vm_stat.csv
cd "$(dirname "$0")/.."
project=$(docker compose config | sed -n 's/^name: //p')
echo 'ts,name,cpu_pct,mem_usage,mem_pct,net_io,block_io,pids' > "$out"
echo 'ts,busy,total,cpus' > "$vm"
while :; do
ts=$(date +%s)
docker compose exec -T valkey cat /proc/stat 2>/dev/null | awk -v ts="$ts" '/^cpu /{b=$2+$3+$4+$7+$8+$9; t=b+$5+$6} /^cpu[0-9]/{n++} END{if (t) print ts "," b "," t "," n}' >> "$vm" || true
ids=$(docker ps -q --filter "label=com.docker.compose.project=$project")
[ -z "$ids" ] || docker stats --no-stream --format "$ts,{{.Name}},{{.CPUPerc}},{{.MemUsage}},{{.MemPerc}},{{.NetIO}},{{.BlockIO}},{{.PIDs}}" $ids >> "$out" || true
sleep "${INTERVAL:-1}"
done
+5 -3
View File
@@ -1,12 +1,14 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
out=${EVIDENCE_DIR:-evidence}
mkdir -p "$out"
psql() { docker compose exec -T postgres psql -U messenger -d messenger -X -c "$1"; }
value() { docker compose exec -T postgres psql -U messenger -d messenger -X -tA -c "$1"; }
cli() { docker compose run --rm --no-deps -T s3-cli "$@"; }
bucket=${S3_BUCKET:-messenger-attachments}
{
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD)"
echo "\$ docker compose ps --format '{{.Service}} {{.Status}} {{.Ports}}'"
docker compose ps -a --format '{{.Service}} {{.Status}} {{.Ports}}'
for table in attachments message_attachments; do
@@ -36,5 +38,5 @@ bucket=${S3_BUCKET:-messenger-attachments}
docker compose config --format json | python3 -c 'import json,sys
for name, service in sorted(json.load(sys.stdin)["services"].items()):
print(name, sorted(s["source"] for s in service.get("secrets", [])))'
} > evidence/storage-layout.txt
cat evidence/storage-layout.txt
} > "$out"/storage-layout.txt
cat "$out"/storage-layout.txt
+19 -8
View File
@@ -11,14 +11,25 @@ cli "info persistence" | grep -E '^aof_enabled|^aof_last_write_status'
echo "\$ valkey-cli dbsize; info stats | grep keyspace"
cli dbsize
cli "info stats" | grep -E '^keyspace_(hits|misses)'
key=$(cli "--scan --pattern session:* --count 1000" | head -1)
hash=${key#session:}
echo "\$ сравнение срока ключа кэша и срока сессии для одного ключа (значение и токен не печатаются)"
echo "valkey pexpiretime, ms: $(cli "pexpiretime $key")"
echo "postgres expires_at, ms: $(psql "SELECT (extract(epoch FROM expires_at) * 1000)::bigint FROM sessions WHERE token_hash = decode('$hash', 'hex')")"
echo "\$ количество отметок отзыва и закэшированных сессий"
revoked=0; cached=0
echo "\$ valkey-cli --scan --pattern 'session:*' и вид значения (хеш сокращён, значения не печатаются)"
revoked=0; cached=0; key=""; stored=""
for k in $(cli "--scan --pattern session:* --count 1000"); do
if [ "$(cli "get $k")" = "-" ]; then revoked=$((revoked + 1)); else cached=$((cached + 1)); fi
if [ "$(cli "get $k")" = "-" ]; then
revoked=$((revoked + 1)); printf '%.16s… отметка отзыва "-"\n' "$k"; continue
fi
cached=$((cached + 1)); printf '%.16s… сессия\n' "$k"
if [ -z "$key" ]; then
stored=$(psql "SELECT (extract(epoch FROM expires_at) * 1000)::bigint FROM sessions WHERE token_hash = decode('${k#session:}', 'hex')")
if [ -n "$stored" ]; then key=$k; fi
fi
done
echo "tombstones: $revoked, cached sessions: $cached"
if [ "$revoked" -eq 0 ] || [ -z "$key" ]; then
echo "нужны закэшированная сессия из БД и отметка отзыва: сначала make test" >&2
exit 1
fi
expires=$(cli "pexpiretime $key")
echo "\$ срок ключа кэша против срока сессии для $(printf '%.16s' "$key")…"
echo "valkey pexpiretime, ms: $expires"
echo "postgres expires_at, ms: $stored"
if [ "$expires" -le "$stored" ]; then echo "ключ не переживает сессию: да"; else echo "ключ не переживает сессию: НЕТ"; exit 1; fi
+7 -5
View File
@@ -1,9 +1,11 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
out=${EVIDENCE_DIR:-evidence}
mkdir -p "$out"
base=${1:-http://$(docker compose port lb 8080)}
stats=http://$(docker compose port lb 8404)/stats\;csv
header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"; }
header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD)"; }
{
header
echo "\$ docker compose --env-file .env config --quiet && echo valid"
@@ -25,7 +27,7 @@ for r in csv.DictReader(sys.stdin.read().lstrip("# ").splitlines()):
print(r["svname"], r["status"], r["check_status"], r["check_code"])'
echo "\$ for i in 1..6: curl -D - $base/health/live | X-Instance-Id"
for _ in 1 2 3 4 5 6; do curl -fsS -D - -o /dev/null "$base/health/live" | tr -d '\r' | grep -i '^x-instance-id'; done
} > evidence/stack.txt
} > "$out"/stack.txt
{
header
echo "host: $(sysctl -n machdep.cpu.brand_string 2>/dev/null || uname -m), $(sysctl -n hw.ncpu 2>/dev/null) CPU, $(($(sysctl -n hw.memsize 2>/dev/null || echo 0) / 1073741824)) GiB RAM, $(sw_vers -productName 2>/dev/null) $(sw_vers -productVersion 2>/dev/null)"
@@ -38,7 +40,7 @@ for r in csv.DictReader(sys.stdin.read().lstrip("# ").splitlines()):
echo "rustfs: $(docker compose exec -T s3 rustfs --version 2>/dev/null | head -1)"
echo "rabbitmq: $(docker compose exec -T broker rabbitmqctl -q version 2>/dev/null | head -1)"
echo "packages (uv.lock):"
for name in fastapi uvicorn uvicorn-worker gunicorn asyncpg argon2-cffi pydantic pydantic-settings dishka structlog redis aiobotocore botocore aiohttp pillow \
for name in fastapi uvicorn uvicorn-worker gunicorn sqlalchemy alembic asyncpg argon2-cffi pydantic pydantic-settings dishka structlog redis aiobotocore botocore aiohttp pillow \
faststream aio-pika aiormq dishka-faststream; do
printf ' %s %s\n' "$name" "$(sed -n "/^name = \"$name\"$/{n;s/version = \"\(.*\)\"/\1/p;}" uv.lock)"
done
@@ -47,5 +49,5 @@ for r in csv.DictReader(sys.stdin.read().lstrip("# ").splitlines()):
service=$(docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}}' "$id")
docker top "$id" -eo pid,uid,comm | awk -v s="$service" 'NR > 1 {print " " s, $2, $3}'
done | sort -u
} > evidence/environment.txt
cat evidence/stack.txt evidence/environment.txt
} > "$out"/environment.txt
cat "$out"/stack.txt "$out"/environment.txt
+16 -13
View File
@@ -1,20 +1,22 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
out=${EVIDENCE_DIR:-evidence}
mkdir -p "$out"
psql() { docker compose exec -T postgres psql -U messenger -d messenger -c "$1"; }
header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"; }
counts="SELECT (SELECT count(*) FROM users) users, (SELECT count(*) FROM sessions) sessions, (SELECT count(*) FROM chats) chats, (SELECT count(*) FROM chat_members) members, (SELECT count(*) FROM messages) messages, (SELECT count(*) FROM idempotency_keys) keys, (SELECT count(*) FROM attachments) attachments, (SELECT count(*) FROM message_attachments) links, (SELECT count(*) FROM schema_migrations) versions"
kdf="SELECT substring(password_hash from '^[$]argon2id[$]v=[0-9]+[$]m=[0-9]+,t=[0-9]+,p=[0-9]+[$]') AS kdf_prefix, count(*) AS users, min(length(password_hash)) AS hash_length FROM users GROUP BY 1"
header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD)"; }
counts="SELECT (SELECT count(*) FROM users) users, (SELECT count(*) FROM sessions) sessions, (SELECT count(*) FROM chats) chats, (SELECT count(*) FROM chat_members) members, (SELECT count(*) FROM messages) messages, (SELECT count(*) FROM idempotency_keys) keys, (SELECT count(*) FROM attachments) attachments, (SELECT count(*) FROM message_attachments) links, (SELECT count(*) FROM alembic_version) versions"
kdf="SELECT substring(password_hash from '^[$]argon2id[$]v=[0-9]+[$]m=[0-9]+,t=[0-9]+,p=[0-9]+[$]') AS kdf_prefix, username = 'seed_author' AS seed_stub, count(*) AS users, min(length(password_hash)) AS min_length, max(length(password_hash)) AS max_length FROM users GROUP BY 1, 2 ORDER BY 2"
tokens="SELECT octet_length(token_hash) AS stored_bytes, count(*) AS sessions FROM sessions GROUP BY 1"
{
header
for table in users sessions chats chat_members messages idempotency_keys attachments message_attachments schema_migrations; do
for table in users sessions chats chat_members messages idempotency_keys attachments message_attachments alembic_version; do
echo "\$ psql -c '\\d+ $table'"
psql "\\d+ $table"
done
echo "\$ psql -c 'SELECT version, left(checksum, 12), applied_at FROM schema_migrations'"
psql "SELECT version, left(checksum, 12) AS checksum, applied_at FROM schema_migrations"
} > evidence/schema.txt
echo "\$ psql -c 'SELECT version_num FROM alembic_version'"
psql "SELECT version_num FROM alembic_version"
} > "$out"/schema.txt
{
header
echo "\$ psql -c \"$counts\""
@@ -23,14 +25,14 @@ tokens="SELECT octet_length(token_hash) AS stored_bytes, count(*) AS sessions FR
docker compose run --rm migrate 2>/dev/null
echo "\$ psql -c \"$counts\""
psql "$counts"
} > evidence/migrations.txt
} > "$out"/migrations.txt
{
header
echo "\$ psql -c \"$kdf\""
psql "$kdf"
echo "\$ psql -c \"$tokens\""
psql "$tokens"
} > evidence/kdf.txt
} > "$out"/kdf.txt
{
header
echo "\$ git ls-files secrets .env backups volumes"
@@ -38,7 +40,8 @@ tokens="SELECT octet_length(token_hash) AS stored_bytes, count(*) AS sessions FR
echo "(пусто: секретов, .env, дампов и volumes в Git нет)"
echo "\$ git check-ignore -v secrets/database_url secrets/postgres_password secrets/s3_root_password secrets/s3_secret_access_key .env backups/"
git check-ignore -v secrets/database_url secrets/postgres_password secrets/s3_root_password secrets/s3_secret_access_key .env backups/
echo "\$ docker volume ls --filter name=mtusi2026lab5"
docker volume ls --filter name=mtusi2026lab5
} > evidence/secrets.txt
echo "evidence/{schema,migrations,kdf,secrets}.txt обновлены"
project=$(docker compose config --format json | python3 -c 'import json,sys; print(json.load(sys.stdin)["name"])')
echo "\$ docker volume ls --filter label=com.docker.compose.project=$project"
docker volume ls --filter "label=com.docker.compose.project=$project"
} > "$out"/secrets.txt
echo ""$out"/{schema,migrations,kdf,secrets}.txt обновлены"
+1 -1
View File
@@ -10,7 +10,7 @@ failed=0
set --
for host in $hosts; do
TARGET_INSTANCE_ID=$host python3 scripts/probe.py load --expect dip --out "$out/$host" \
-- sh -c "scripts/stop-one.sh && sleep ${HOLD:-10} && scripts/restore.sh" || failed=1
-- sh -c "scripts/stop-one.sh && docker compose ps -a --format '{{.ID}} {{.Service}} {{.Status}}' ${API_SERVICE:-api} && { scripts/stand-usage.sh & sleep ${HOLD:-10}; wait; } && scripts/restore.sh" || failed=1
scripts/restore.sh
set -- "$@" "$out/$host/result.json"
done
+11 -6
View File
@@ -1,28 +1,33 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
out=${EVIDENCE_DIR:-evidence}
mkdir -p "$out"
replicas() {
docker inspect -f '{{.Config.Hostname}} {{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}' $(docker compose ps -aq "${API_SERVICE:-api}")
}
tmp=evidence/failover.tmp
url=${1:-http://localhost:8080}
shift=0
while [ "$shift" -lt "${2:-0}" ]; do curl -fsS -o /dev/null "$url/health/live"; shift=$((shift + 1)); done
tmp="$out"/failover.tmp
since=$(date -u +%FT%TZ)
{
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD)"
echo "== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt"
replicas
echo "\$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh"
make --no-print-directory test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL="${1:-http://localhost:8080}"
make --no-print-directory test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL="$url"
echo "== after: api replicas"
replicas
} > "$tmp" 2>&1
stopped=$(replicas | awk '$2 == "exited" {print $1}')
{
echo "== stopped replica: $stopped; shutdown: signal, drain window with readiness 503, close, exit"
docker logs --since "$since" "$stopped" 2>&1 | grep -E 'Handling signal|Draining|"status":503|Shutting down|Waiting for conn|Finished server|service.stopped|SIGTERM' | tail -16
docker logs --since "$since" "$stopped" 2>&1 | grep -E 'Handling signal|service.draining|"status":503|Shutting down|Waiting for conn|Finished server|service.stopped|SIGTERM' | tail -16
echo "\$ scripts/restore.sh"
scripts/restore.sh
echo "== after restore"
replicas
} >> "$tmp" 2>&1
mv "$tmp" "evidence/failover-$stopped.txt"
cat "evidence/failover-$stopped.txt"
mv "$tmp" "$out/failover-$stopped.txt"
cat "$out/failover-$stopped.txt"
+34 -2
View File
@@ -23,6 +23,19 @@ def compose(*args, stdin=None):
return subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, text=True, check=False)
def containers(service):
ids = compose("ps", "-aq", service).stdout.split()
fields = ("hostname", "status", "exit_code", "started_at", "finished_at")
template = (
"{{.Config.Hostname}} {{.State.Status}} {{.State.ExitCode}}"
" {{.State.StartedAt}} {{.State.FinishedAt}}"
)
out = subprocess.run(
["docker", "inspect", "-f", template, *ids], capture_output=True, text=True, check=False
)
return [dict(zip(fields, line.split())) for line in out.stdout.splitlines()]
def upload(c, actor, chat, expected=202):
path = f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png"
schema = "Attachment" if expected == 202 else None
@@ -90,6 +103,10 @@ def failover_upload(c, args):
recovered = time.monotonic() - started
served = got["headers"].get("X-Instance-Id")
check(served != target, "download served by the stopped replica")
during = containers(args.service)
states = {item["hostname"]: item["status"] for item in during}
check(states.get(target) == "exited", f"target {target} is not exited: {states}")
check(states.get(served) == "running", f"neighbour {served} is not running: {states}")
digest = hashlib.sha256(got["bytes"]).hexdigest()
check(
digest == att["sha256"] == hashlib.sha256(FIXTURE).hexdigest(),
@@ -104,7 +121,9 @@ def failover_upload(c, args):
"sha256_equal": True,
"hook_seconds": round(hook_seconds, 3),
"download_ok_after_seconds": round(recovered, 3),
"replicas_at_download": during,
"restored": args.restore,
"replicas_after_restore": containers(args.service) if args.restore else None,
}
@@ -124,6 +143,7 @@ def s3_outage(c, args):
"text_message": timed(lambda: c.message(alice, chat, "S3 недоступен, текст работает")),
"health_ready": timed(lambda: c.call("GET", "/health/ready", [200, 503])),
}
stopped = containers(args.s3_service)
finally:
started = time.monotonic()
compose("up", "-d", "--wait", "--no-deps", "--no-recreate", args.s3_service)
@@ -131,8 +151,11 @@ def s3_outage(c, args):
got = wait(lambda: content(c, bob, att), 30)
check(got["bytes"] == FIXTURE, "attachment changed across S3 outage")
expected = {"upload": 503, "download": 503, "text_message": 201}
check(all(during[k]["status"] == v for k, v in expected.items()), f"S3 outage: {during}")
check(all(during[k]["seconds"] < 6 for k in ("upload", "download")), "S3 timeout exceeded")
return {
"scenario": "s3-outage",
"s3_container_during_outage": stopped,
"during_outage": during,
"expected": expected,
"as_expected": all(during[k]["status"] == v for k, v in expected.items()),
@@ -180,6 +203,8 @@ def abort_upload(c, args):
)
rows = out.stdout.split()
counts = dict(zip(rows[::2], (int(x) for x in rows[1::2])))
check(out.returncode == 0, "cannot inspect aborted upload")
check(counts.get("ready", 0) == 0, "aborted upload became ready")
alive = c.call("GET", "/api/v1/users/me", actor=alice, schema="User")["status"]
return {
"scenario": "abort-upload",
@@ -216,10 +241,17 @@ def main():
"s3-outage": s3_outage,
"abort-upload": abort_upload,
}[args.scenario]
commit = subprocess.run(
["git", "rev-parse", "HEAD"], cwd=ROOT, capture_output=True, check=False
)
stamp = {
"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"commit": commit.stdout.decode().strip(),
}
try:
report = {"ok": True, **run(c, args)}
report = {"ok": True, **stamp, **run(c, args)}
except Failure as exc:
report = {"scenario": args.scenario, "ok": False, "error": str(exc)}
report = {"scenario": args.scenario, "ok": False, **stamp, "error": str(exc)}
text = json.dumps(report, ensure_ascii=False, indent=2)
if args.output:
Path(args.output).write_text(text + "\n")
+11
View File
@@ -0,0 +1,11 @@
#!/bin/sh
set -eu
out=${1:?usage: lb-stats.sh <csv>}
cd "$(dirname "$0")/.."
url=${LB_STATS_URL:-http://$(docker compose port lb 8404)/stats;csv}
curl -fsS "$url" | sed -n '1s/^# /ts,/p' > "$out"
while :; do
ts=$(date +%s)
curl -fsS "$url" | grep -E '^(public|api),' | sed "s/^/$ts,/" >> "$out" || true
sleep "${INTERVAL:-1}"
done
+29 -5
View File
@@ -9,12 +9,20 @@ from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
S3_CLIENT_SECRETS = [
"s3_access_key_id",
"s3_reconciler_access_key_id",
"s3_reconciler_secret_access_key",
"s3_worker_access_key_id",
"s3_worker_secret_access_key",
"s3_root_password",
"s3_root_user",
"s3_secret_access_key",
"s3_worker_access_key_id",
"s3_worker_secret_access_key",
]
RECONCILER_SECRETS = [
"database_url",
"s3_reconciler_access_key_id",
"s3_reconciler_secret_access_key",
]
OWN_USER = ("postgres", "valkey", "s3-init", "s3-cli")
POLICY = {
"lb": {"secrets": [], "volumes": [], "networks": ["backend", "edge"], "ports": True},
"api": {
@@ -29,6 +37,11 @@ POLICY = {
"networks": ["backend"],
},
"migrate": {"secrets": ["database_url"], "volumes": [], "networks": ["backend"]},
"reconciler": {
"secrets": RECONCILER_SECRETS,
"volumes": [],
"networks": ["backend"],
},
"postgres": {"secrets": ["postgres_password"], "volumes": ["pgdata"], "networks": ["backend"]},
"valkey": {"secrets": ["valkey.conf"], "volumes": ["valkeydata"], "networks": ["backend"]},
"k6": {"secrets": [], "volumes": [], "networks": ["edge"]},
@@ -94,6 +107,9 @@ def declared(name, service, networks, problems):
problems.append(f"{name}: publishes {ports} beyond loopback")
if name == "api" and not service.get("read_only"):
problems.append("api: root filesystem is writable")
user = str(service.get("user", ""))
if name in OWN_USER and user.split(":", maxsplit=1)[0] in ("", "0", "root"):
problems.append(f"{name}: runs as root")
internal = [n for n in nets if networks.get(n, {}).get("internal")]
return [
name,
@@ -103,6 +119,7 @@ def declared(name, service, networks, problems):
",".join(volumes) or "-",
",".join(f"{n}{'(internal)' if n in internal else ''}" for n in nets),
str(service.get("read_only", False)).lower(),
user or "image",
]
@@ -149,6 +166,8 @@ def main():
"bench",
"--profile",
"tools",
"--profile",
"reconcile",
"config",
"--format",
"json",
@@ -162,9 +181,14 @@ def main():
if not any(row[0] == "api" for row in containers):
problems.append("no running api replicas to inspect")
lines = [
f"date: {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}",
"$ docker compose --profile bench --profile tools config --format json (declared)",
*table(["service", "cpus/mem", "ports", "secrets", "volumes", "networks", "ro"], services),
f"date: {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} "
f"commit: {run('git', 'rev-parse', '--short', 'HEAD')[1]}",
"$ docker compose --profile bench --profile tools --profile reconcile config --format json"
" (declared)",
*table(
["service", "cpus/mem", "ports", "secrets", "volumes", "networks", "ro", "user"],
services,
),
"",
"$ docker inspect (running containers: hostname = X-Instance-Id, applied limits)",
*table(["service", "hostname", "cpus", "memory", "published"], containers),
+62
View File
@@ -0,0 +1,62 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
runs="${RUNS:-3}"
pause="${PAUSE:-15}"
project=$(docker inspect -f '{{index .Config.Labels "com.docker.compose.project"}}' "$(docker compose ps -q lb)")
network="${project}_edge"
image=$(sed -n 's/^FROM \(python:[^ ]*\) AS runtime$/\1/p' Dockerfile)
name="$project-load-client"
work=$(mktemp -d)
trap 'rm -rf "$work"' EXIT
out=${EVIDENCE_DIR:-evidence}
mkdir -p "$out/load"
[ -z "$(ls -A "$out/load")" ] || { echo "Choose an empty EVIDENCE_DIR/load" >&2; exit 2; }
client() {
docker run --rm --name "$name" --network "$network" --cpus 2 --memory 512m \
--sysctl net.ipv4.tcp_tw_reuse=1 -v "$PWD/tests:/w/tests:ro" -v "$PWD/contracts:/w/contracts:ro" \
-v "$PWD/lab.json:/w/lab.json:ro" -v "$PWD/$out/load:/out" "$image" python /w/tests/load.py "$@"
}
handled() {
docker compose logs --no-log-prefix --since "$1" api | python3 -c 'import collections, json, sys
seen = collections.Counter()
for line in sys.stdin:
if line.startswith("{"):
event = json.loads(line)
if event.get("event") == "request.handled":
seen[(event["status"], event["method"], event["route"])] += 1
for (status, method, route), count in sorted(seen.items()):
print(status, method, route, count)'
}
{
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD) network: $network"
echo "# make load-evidence (scripts/load-evidence.sh): закрытый контур tests/load.py, 64 потока, 8 с, все записи"
echo "# прогона в один новый чат, $runs повтора с паузой $pause с; клиент — одноразовый контейнер в сети <проект>_edge"
echo "# стенда (2 CPU, 512 MiB, базовый образ API по digest, tests/, contracts/ и lab.json только для чтения,"
echo "# tcp_tw_reuse=1), адрес http://lb:8080. На 4-й секунде — docker stats; после прогона — request.handled реплик."
echo "\$ docker ps --format '{{.Names}}' | grep -v '^$project-' # чужие контейнеры на VM во время серии"
docker ps --format '{{.Names}}' | grep -v "^$project-" || true
echo "\$ client() { docker run --rm --name $name --network $network --cpus 2 --memory 512m --sysctl net.ipv4.tcp_tw_reuse=1 -v \"\$PWD/tests:/w/tests:ro\" -v \"\$PWD/contracts:/w/contracts:ro\" -v \"\$PWD/lab.json:/w/lab.json:ro\" -v \"\$PWD/$out/load:/out\" $image python /w/tests/load.py \"\$@\"; }"
run=1
while [ "$run" -le "$runs" ]; do
since=$(date -u +%FT%TZ)
echo "\$ client --base-url http://lb:8080 --duration 8 --concurrency 64 --output /out/run$run.json"
client --base-url http://lb:8080 --duration 8 --concurrency 64 --output "/out/run$run.json" > "$work/out" 2>&1 &
pid=$!
sleep 4
stats=$(docker stats --no-stream --format '{{.Name}} {{.CPUPerc}} {{.MemUsage}}' $(docker compose ps -q) "$name" | sed "s/^$project-//" | sort)
if wait "$pid"; then code=0; else code=$?; fi
echo "exit $code"
cat "$work/out"
echo "\$ docker stats --no-stream # 4-я секунда прогона: контейнер CPU% память"
echo "$stats"
echo "\$ docker compose logs --since $since api | request.handled: статус метод маршрут число"
handled "$since"
[ "$code" -eq 0 ] || { exit "$code"; }
[ "$run" -lt "$runs" ] && sleep "$pause"
run=$((run + 1))
done
echo "\$ docker compose exec -T postgres psql -c 'SELECT count(*) FROM messages'"
docker compose exec -T postgres psql -U messenger -d messenger -qAt -c 'SELECT count(*) FROM messages'
} > "$out/load.txt"
cat "$out/load.txt"
+42
View File
@@ -0,0 +1,42 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
limit="${WAIT_SECONDS:-1500}"
out=${EVIDENCE_DIR:-evidence}
mkdir -p "$out"
sql() { docker compose exec -T postgres psql -U messenger -d messenger -X "$@"; }
pending() { sql -tAc "SELECT count(*) FROM attachments WHERE status = 'pending'"; }
statuses="SELECT status, count(*), min(created_at), max(created_at) FROM attachments GROUP BY status ORDER BY status"
since=$(date -u +%FT%TZ)
{
echo "date: $since commit: $(git rev-parse HEAD)"
echo "# make overload-leftovers (scripts/overload-leftovers.sh) после серии памяти: невидимые намерения от 503 по"
echo "# таймауту видны сверке свежими, уборка API удаляет их после UPLOAD_GRACE_SECONDS; стенд тот же"
echo "\$ make reconcile # dry-run"
docker compose run --rm --no-deps -T reconciler 2>/dev/null
echo "\$ psql -c \"$statuses\""
sql -c "$statuses"
echo "\$ until [ \"\$(psql -tAc \"SELECT count(*) FROM attachments WHERE status = 'pending'\")\" = 0 ]; do sleep 10; done"
started=$(date +%s)
until [ "$(pending)" = 0 ] || [ $(( $(date +%s) - started )) -gt "$limit" ]; do sleep 10; done
echo "pending = $(pending) в $(date -u +%FT%TZ)"
[ "$(pending)" = 0 ] || { echo "FAIL: pending cleanup timed out"; exit 1; }
echo "\$ docker compose logs --since $since api | audit.attachment_discarded и attachments.swept"
docker compose logs --no-log-prefix --since "$since" api | python3 -c 'import collections, json, sys
reasons, swept, times = collections.Counter(), 0, []
for line in sys.stdin:
if not line.startswith("{"):
continue
event = json.loads(line)
if event.get("event") == "audit.attachment_discarded":
reasons[event["reason"]] += 1
times.append(event["ts"][:19] + "Z")
elif event.get("event") == "attachments.swept":
swept += event["count"]
print(dict(reasons), "swept", swept, "first", min(times, default="-"), "last", max(times, default="-"))'
echo "\$ psql -c \"$statuses\""
sql -c "$statuses"
echo "\$ make reconcile # dry-run после уборки"
docker compose run --rm --no-deps -T reconciler 2>/dev/null
} > "$out/overload-leftovers.txt"
cat "$out/overload-leftovers.txt"
+1 -1
View File
@@ -6,7 +6,7 @@ started() {
docker inspect --format '{{index .Config.Labels "com.docker.compose.service"}} {{.Config.Hostname}} {{.State.StartedAt}} {{.State.Status}}' "$id"
done | sort
}
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD)"
echo "== before: service hostname StartedAt status"
started
make --no-print-directory test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL="${1:-http://localhost:8080}"
+30
View File
@@ -0,0 +1,30 @@
#!/bin/sh
set -eu
mode=${1:?usage: pg-stats.sh activity <csv> | statements <csv> | size | reset}
cd "$(dirname "$0")/.."
psql() {
docker compose exec -T postgres sh -c 'exec psql -X -q -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "${POSTGRES_DB:-$POSTGRES_USER}" "$@"' psql "$@"
}
case $mode in
reset)
psql -c 'set client_min_messages = warning' -c 'create extension if not exists pg_stat_statements' -c 'select pg_stat_statements_reset()' >/dev/null
;;
size)
psql -At -F, -c "select pg_database_size(current_database()), coalesce((select n_live_tup from pg_stat_user_tables where relname = 'messages'), 0)"
;;
statements)
psql -c "copy (select calls, round(total_exec_time::numeric, 3) as total_ms, round(mean_exec_time::numeric, 3) as mean_ms, rows, shared_blks_hit, shared_blks_read, left(regexp_replace(query, '\s+', ' ', 'g'), 200) as query from pg_stat_statements where dbid = (select oid from pg_database where datname = current_database()) and query not like '%pg_stat%' order by total_exec_time desc limit ${TOP:-20}) to stdout with csv header" > "${2:?csv}"
;;
activity)
echo 'ts,state,wait_event_type,wait_event,backends,max_connections' > "${2:?csv}"
while :; do
ts=$(date +%s)
psql -At -F, -c "select coalesce(state, ''), coalesce(wait_event_type, ''), coalesce(wait_event, ''), count(*), current_setting('max_connections') from pg_stat_activity where backend_type = 'client backend' and pid <> pg_backend_pid() group by 1, 2, 3" | sed "s/^/$ts,/" >> "$2" || true
sleep "${INTERVAL:-1}"
done
;;
*)
echo "unknown mode $mode" >&2
exit 2
;;
esac
+163 -7
View File
@@ -10,12 +10,16 @@ import sys
import threading
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
from datetime import datetime
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
COUNTERS = ("wretr", "wredis", "econ", "eresp", "hrsp_5xx")
PSQL = ["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger"]
PSQL += ["-X", "-q", "-At"]
@functools.cache
@@ -207,16 +211,112 @@ def check(args):
return 0 if passed else 1
def output(*command):
done = subprocess.run(command, cwd=str(ROOT), capture_output=True, text=True, check=False)
return done.stdout + done.stderr
def replicas():
ids = output("docker", "compose", "ps", "-q", os.environ.get("API_SERVICE", "api")).split()
form = "{{.Id}} {{.Config.Hostname}} {{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}"
return [line.split() for line in output("docker", "inspect", "-f", form, *ids).splitlines()]
def answered(container, request_id):
for line in output("docker", "logs", container).splitlines():
if request_id in line and "request.handled" in line:
return json.loads(line).get("status")
return None
def lost(alice, bob, chat, kill, args):
key, text = uuid.uuid4().hex, "lost " + uuid.uuid4().hex
request_id = "lost-" + key[:12]
lock = subprocess.Popen(
PSQL, cwd=str(ROOT), stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True
)
lock.stdin.write(
f"BEGIN;\nSELECT 'locked' FROM chats WHERE id = '{uuid.UUID(chat)}' FOR UPDATE;\n"
f"SELECT pg_sleep({args.hold});\nCOMMIT;\n"
)
lock.stdin.flush()
lock.stdout.readline()
sent = time.time()
headers = dict(alice["auth"], **{"Idempotency-Key": key, "X-Request-Id": request_id})
path = f"/api/v1/chats/{chat}/messages"
first = call("POST", path, {"text": text}, headers, args.timeout)[0]
sql = "SELECT host(client_addr) FROM pg_stat_activity WHERE wait_event_type = 'Lock'"
waiting = set(output(*PSQL, "-c", sql).split())
holder = next((r for r in replicas() if waiting & set(r[2:])), [None, None])
if kill and holder[0]:
subprocess.call(["docker", "kill", holder[0]], stdout=subprocess.DEVNULL)
lock.stdin.close()
lock.wait()
retried = time.time()
attempts, (status, instance, body) = 1, send(alice, chat, text, key)
while status != 201 and attempts < 20:
time.sleep(0.2)
attempts, (status, instance, body) = attempts + 1, send(alice, chat, text, key)
page = call("GET", path + "?limit=100", headers=bob["auth"])[2] or {"items": []}
copies = [m for m in page["items"] if m["text"] == text]
stamp = copies[0]["created_at"].replace("Z", "+00:00") if copies else None
report = {
"variant": "kill" if kill else "timeout",
"lost_attempt": {
"status": first,
"client_timeout_s": args.timeout,
"chat_lock_held_s": args.hold,
"replica": holder[1],
"replica_killed": bool(kill and holder[0]),
"replica_answered": answered(holder[0], request_id) if holder[0] else None,
},
"retry": {
"after_s": round(retried - sent, 3),
"status": status,
"attempts": attempts,
"replica": instance,
},
"copies_in_history": len(copies),
"same_message": bool(copies) and copies[0]["id"] == (body or {}).get("id"),
"written_by": None
if stamp is None
else ("lost attempt" if datetime.fromisoformat(stamp).timestamp() < retried else "retry"),
}
if kill and holder[0]:
subprocess.call(
[str(ROOT / "scripts/restore.sh")], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
)
report["pass"] = (
first == 0
and holder[0] is not None
and status == 201
and report["copies_in_history"] == 1
and report["same_message"]
and (kill or report["lost_attempt"]["replica_answered"] == 201)
)
return report
def retry(args):
alice, bob = actor("alice"), actor("bob")
chat = chat_of(alice, bob)
variants = [lost(alice, bob, chat, kill, args) for kill in (False, True)]
passed = all(v["pass"] for v in variants)
emit({"variants": variants, "pass": passed}, args.out)
return 0 if passed else 1
def record(stop, alice, chat):
probes, trace = [], []
probes, trace, confirmed = [], [], []
def prober():
key = uuid.uuid4().hex
while not stop.is_set():
started = time.time()
status, instance, _ = send(alice, chat, "probe " + key, key)
status, instance, body = send(alice, chat, "probe " + key, key)
probes.append((started, status, round(1000 * (time.time() - started), 1), instance))
if status == 201:
confirmed.append(body)
key = uuid.uuid4().hex
time.sleep(0.1)
@@ -228,7 +328,38 @@ def record(stop, alice, chat):
threads = [threading.Thread(target=t, daemon=True) for t in (prober, poller)]
for thread in threads:
thread.start()
return probes, trace, threads
return probes, trace, threads, confirmed
def history_check(reader, chat, confirmed):
items, cursors = [], set()
cursor = None
for _ in range(len(confirmed) // 100 + 2):
path = f"/api/v1/chats/{chat}/messages?limit=100"
if cursor:
path += "&cursor=" + urllib.parse.quote(cursor, safe="")
status, _, page = call("GET", path, headers=reader["auth"])
if status != 200 or not isinstance(page, dict):
break
items.extend(page["items"])
cursor = page["next_cursor"]
if cursor is None:
same = sorted(items, key=lambda m: m["id"]) == sorted(confirmed, key=lambda m: m["id"])
return {
"confirmed": len(confirmed),
"stored": len(items),
"exact_match": bool(confirmed) and same,
"unique": len({m["id"] for m in items}) == len(items),
}
if cursor in cursors:
break
cursors.add(cursor)
return {
"confirmed": len(confirmed),
"stored": len(items),
"exact_match": False,
"unique": False,
}
def write_csv(path, header, rows):
@@ -241,6 +372,10 @@ def summarize(args, probes, trace, action, expected):
t0, end = action["start"], action["end"]
label = args.label or os.environ.get("TARGET_INSTANCE_ID", "action")
failures = [t - t0 for t, s, _, _ in probes if s != 201 and t >= t0]
successes = [(t, t + ms / 1000 - t0) for t, s, ms, _ in probes if s == 201 and t >= t0]
recovery = (
next((end for t, end in successes if t - t0 > failures[-1]), None) if failures else 0.0
)
down = [t for t, up in trace if t >= t0 and up is not None and up < expected]
back = [t for t, up in trace if down and t > down[0] and up is not None and up >= expected]
outage = [t for t in down if not back or t < back[0]] or [end]
@@ -256,13 +391,15 @@ def summarize(args, probes, trace, action, expected):
"action_finished_before_load_end": end < action["load_end"],
"lb_up_before": expected,
"lb_up_min": min((up for _, up in trace if up is not None), default=None),
"lb_samples_missing": sum(up is None for _, up in trace),
"detection_seconds": round(down[0] - t0, 3) if down else None,
"rejoin_seconds": round(back[0] - t0, 3) if back else None,
"probe": {
"attempts": len(probes),
"failed_attempts": sum(1 for _, s, _, _ in probes if s != 201),
"first_failure_s": round(failures[0], 3) if failures else None,
"recovery_s": round(failures[-1], 3) if failures else 0.0,
"first_success_s": round(successes[0][1], 3) if successes else None,
"recovery_s": round(recovery, 3) if recovery is not None else None,
"instances_before": dict(before),
"instances_during_outage": dict(during),
"instances_after": dict(after),
@@ -275,7 +412,13 @@ def verdict(args, result):
checks = {
"action_ok": result["action_exit"] == 0 and result["action_finished_before_load_end"],
"load_errors_ok": k6["exit"] == 0 and k6["failed_ratio"] <= args.max_failed_ratio,
"write_recovery_ok": probe["recovery_s"] <= args.max_recovery,
"load_delivered": k6["requests"] > 0 and k6["dropped_iterations"] == 0,
"lb_observed": result["lb_up_before"] >= 2 and result["lb_samples_missing"] == 0,
"write_recovery_ok": all(
probe[k] is not None and probe[k] <= args.max_recovery
for k in ("first_success_s", "recovery_s")
),
"history_preserved": result["history"]["exact_match"] and result["history"]["unique"],
}
if args.expect == "dip":
checks["detected"] = result["detection_seconds"] is not None
@@ -296,7 +439,8 @@ def load(args):
expected = lb_up() or 0
alice, bob = actor("alice"), actor("bob")
stop, action = threading.Event(), {}
probes, trace, threads = record(stop, alice, chat_of(alice, bob))
chat = chat_of(alice, bob)
probes, trace, threads, confirmed = record(stop, alice, chat)
def act():
time.sleep(args.warmup + args.lead)
@@ -327,6 +471,7 @@ def load(args):
)
write_csv(out / "lb_up.csv", "t_rel_s,up", ((round(t - t0, 3), up) for t, up in trace))
result = summarize(args, probes, trace, action, expected)
result["history"] = history_check(bob, chat, confirmed)
result["lb_counters"] = {k: v - counters[k] for k, v in lb_counters().items()}
result["k6"] = dict(
k6_metrics(out / "k6/summary.json", args.duration), exit=code, rate=args.rate
@@ -344,10 +489,13 @@ def detect(args):
started = time.time()
process = subprocess.Popen(args.command, cwd=str(ROOT))
detected = acted = None
missing = 0
while time.time() - started < args.timeout and (detected is None or acted is None):
if acted is None and process.poll() is not None:
acted = time.time() - started
if detected is None and (lb_up() or 0) <= until:
up = lb_up()
missing += up is None
if detected is None and up is not None and up <= until:
detected = time.time() - started
time.sleep(0.05)
code = process.wait()
@@ -362,6 +510,7 @@ def detect(args):
restored = round(time.time() - begun, 3) if (lb_up() or 0) >= before else None
checks = {
"action_ok": code == 0,
"lb_observed": before >= 2 and missing == 0,
"detected_in_time": detected is not None and detected <= args.max + args.slack,
"lb_answer_ok": status == args.lb_status,
}
@@ -374,6 +523,7 @@ def detect(args):
"action_exit": code,
"action_seconds": round(acted, 3),
"lb_up_before": before,
"lb_samples_missing": missing,
"lb_up_counted_as_detected": until,
"detection_seconds": None if detected is None else round(detected, 3),
"declared_max_seconds": args.max,
@@ -413,6 +563,10 @@ def main():
under.add_argument("--max-failed-ratio", type=float, default=0.01)
under.add_argument("--max-recovery", type=float, default=10.0)
under.add_argument("command", nargs=argparse.REMAINDER)
again = sub.add_parser("retry", help="retry a write whose answer was lost, same key")
again.add_argument("--hold", type=float, default=1.5, help="chat row lock, s (< lock_timeout)")
again.add_argument("--timeout", type=float, default=0.5, help="first attempt timeout, s")
again.add_argument("--out", help="JSON file, written only after a complete report")
watch = sub.add_parser("detect", help="time until the LB takes replicas out of rotation")
watch.add_argument("--max", type=float, required=True, help="declared worst detection, s")
watch.add_argument("--slack", type=float, default=0.25, help="docker CLI and polling, s")
@@ -426,6 +580,8 @@ def main():
args = parser.parse_args()
if args.mode == "check":
return check(args)
if args.mode == "retry":
return retry(args)
args.command = [c for c in args.command if c != "--"]
if not args.command:
parser.error(f"{args.mode} needs an action command after --")
+37
View File
@@ -0,0 +1,37 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
kind="${1:?usage: reconcile-inject.sh orphan|missing|corrupt|bitflip|stale-pending|pending-object|multipart [count]}"
count="${2:-1}"
age="${INJECT_AGE_SECONDS:-7200}"
prefix="${S3_PREFIX:-attachments/}"
sql() { docker compose exec -T postgres psql -v ON_ERROR_STOP=1 -qAt -F ' ' -U "${PGUSER:-messenger}" -d "${PGDATABASE:-messenger}" -c "$1"; }
s3() { docker compose run --rm --no-deps -T -e ROWS="$1" s3-cli sh -c "$2"; }
ready() { sql "SELECT id, object_key, size_bytes FROM attachments WHERE status = 'ready' ${INJECT_IDS:+AND id = ANY('{$INJECT_IDS}'::uuid[])} ORDER BY random() LIMIT $count"; }
pending() {
sql "INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, object_key, status, created_at, updated_at)
SELECT g.id, s.chat_id, s.owner_id, 'injected.png', 'image/png', 2048, repeat('0', 64), 1, 1, '$prefix' || g.id, 'pending',
clock_timestamp() - make_interval(secs => $age), clock_timestamp() - make_interval(secs => $age)
FROM (SELECT chat_id, owner_id FROM attachments ORDER BY created_at LIMIT 1) s,
(SELECT gen_random_uuid() AS id FROM generate_series(1, $count)) g
RETURNING id, object_key"
}
fresh() { sql "SELECT g, '$prefix' || g FROM (SELECT gen_random_uuid() AS g FROM generate_series(1, $count)) s"; }
put_random='printf "%s\n" "$ROWS" | while read -r id key size; do [ -n "$key" ] || continue; head -c "${size:-2048}" /dev/urandom | rc pipe "root/$S3_BUCKET/$key" >/dev/null; echo "$id $key"; done'
case "$kind" in
orphan) rows=$(fresh); s3 "$rows" "$put_random" | sed "s/^/orphan /" ;;
missing) rows=$(ready); s3 "$rows" 'printf "%s\n" "$ROWS" | while read -r id key size; do [ -n "$key" ] || continue; rc object remove --force "root/$S3_BUCKET/$key" >/dev/null; echo "$id $key"; done' | sed "s/^/missing /" ;;
corrupt) rows=$(ready | awk '{print $1, $2, int($3 / 2) + 1}'); s3 "$rows" "$put_random" | sed "s/^/corrupt /" ;;
bitflip) rows=$(ready); s3 "$rows" "$put_random" | sed "s/^/bitflip /" ;;
stale-pending) pending | sed "s/^/stale-pending /" ;;
pending-object) rows=$(pending); s3 "$rows" "$put_random" | sed "s/^/pending-object /" ;;
multipart)
fresh | while read -r _ key; do
docker compose exec -T -e KEY="$key" -e S3_BUCKET="${S3_BUCKET:-messenger-attachments}" s3 sh -c 'u="http://127.0.0.1:9000/$S3_BUCKET/$KEY"; a="$(cat /run/secrets/s3_root_user):$(cat /run/secrets/s3_root_password)"; s="aws:amz:${RUSTFS_REGION:-us-east-1}:s3"
upload=$(curl -fsS --aws-sigv4 "$s" --user "$a" -X POST "$u?uploads" | sed -n "s:.*<UploadId>\(.*\)</UploadId>.*:\1:p")
head -c 4096 /dev/urandom | curl -fsS --aws-sigv4 "$s" --user "$a" -X PUT --data-binary @- "$u?partNumber=1&uploadId=$upload" -o /dev/null' </dev/null
echo "multipart - $key"
done
;;
*) echo "unknown kind: $kind" >&2; exit 2 ;;
esac
+199
View File
@@ -0,0 +1,199 @@
#!/usr/bin/env python3
import argparse
import datetime as dt
import json
import os
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SQL = (
"SELECT a.id, a.object_key, a.status, a.size_bytes, a.sha256,"
" floor(extract(epoch FROM clock_timestamp() - a.created_at))::bigint,"
" EXISTS (SELECT 1 FROM message_attachments m WHERE m.attachment_id = a.id),"
" (to_jsonb(a) ->> 'quarantined_at') IS NOT NULL"
" FROM attachments a ORDER BY a.object_key"
)
HASH = (
'while read -r k; do printf "%s %s\\n" "$k" '
'"$(rc object show "root/$S3_BUCKET/$k" | sha256sum | cut -c1-64)"; done'
)
CLASSES = (
"orphan_stale",
"orphan_fresh",
"missing_referenced",
"missing_unreferenced",
"size_mismatch",
"sha_mismatch",
"pending_stale_with_object",
"pending_stale_without_object",
"pending_fresh",
"quarantined",
"incomplete_stale",
"incomplete_fresh",
"lost_ready",
)
TOLERATED = {"orphan_fresh", "pending_fresh", "incomplete_fresh", "quarantined"}
BROKEN = (
"missing_referenced",
"missing_unreferenced",
"size_mismatch",
"sha_mismatch",
"quarantined",
)
def timed(cmd, stdin=None):
started = time.monotonic()
out = subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, text=True, check=False)
if out.returncode:
raise SystemExit(f"command failed: {' '.join(cmd[:6])}\n{out.stderr[-500:]}")
return out.stdout, round(time.monotonic() - started, 3)
def s3(*args, stdin=None):
return timed(["docker", "compose", "run", "--rm", "--no-deps", "-T", "s3-cli", *args], stdin)
def age(stamp, now):
return now - dt.datetime.fromisoformat(stamp[:19] + "+00:00").timestamp()
def load(bucket, prefix, deep):
listing, list_seconds = s3("object", "list", "--recursive", "--json", f"root/{bucket}/{prefix}")
uploads, uploads_seconds = s3(
"object", "list", "--incomplete", "--recursive", "--json", f"root/{bucket}"
)
user, db = os.environ.get("PGUSER", "messenger"), os.environ.get("PGDATABASE", "messenger")
psql = ["docker", "compose", "exec", "-T", "postgres", "psql", "-v", "ON_ERROR_STOP=1", "-qAt"]
rows_raw, pg_seconds = timed([*psql, "-F", "\t", "-U", user, "-d", db, "-c", SQL])
objects = {}
for item in json.loads(listing).get("items", []):
key = item["key"] if item["key"].startswith(prefix) else prefix + item["key"]
objects[key] = item
rows = {}
for line in rows_raw.splitlines():
rid, key, status, size, sha, row_age, referenced, quarantined = line.split("\t")
rows[key] = {
"id": rid,
"status": status,
"size": int(size),
"sha256": sha,
"age": int(row_age),
"referenced": referenced == "t",
"quarantined": quarantined == "t",
}
hashes, deep_seconds = {}, 0.0
if deep:
present = "".join(
f"{k}\n" for k, r in rows.items() if r["status"] == "ready" and k in objects
)
out, deep_seconds = s3("sh", "-c", HASH, stdin=present)
hashes = dict(line.split(" ", 1) for line in out.splitlines() if " " in line)
seconds = {
"s3_list": list_seconds,
"s3_incomplete": uploads_seconds,
"pg_query": pg_seconds,
"deep_hash": deep_seconds,
}
return objects, rows, hashes, json.loads(uploads).get("data", {}).get("items", []), seconds
def row_class(key, row, objects, hashes, grace):
if row["quarantined"]:
return "quarantined"
if row["status"] == "pending":
stale = "pending_stale_with_object" if key in objects else "pending_stale_without_object"
return "pending_fresh" if row["age"] <= grace else stale
if key not in objects:
return "missing_referenced" if row["referenced"] else "missing_unreferenced"
if objects[key]["size_bytes"] != row["size"]:
return "size_mismatch"
return "sha_mismatch" if hashes and hashes.get(key) != row["sha256"] else None
def classify(objects, rows, hashes, uploads, grace):
now = time.time()
classes = {name: [] for name in CLASSES}
for key, item in objects.items():
if key not in rows:
classes[
"orphan_stale" if age(item["last_modified"], now) > grace else "orphan_fresh"
].append(key)
for key, row in rows.items():
name = row_class(key, row, objects, hashes, grace)
if name:
classes[name].append(key if name.startswith("orphan") else row["id"])
for item in uploads:
classes[
"incomplete_stale" if age(item["initiated"], now) > grace else "incomplete_fresh"
].append(item["key"])
return classes
def main():
p = argparse.ArgumentParser(
description="Independent PG <-> S3 invariant check for lab4 attachments"
)
p.add_argument(
"--grace", type=int, default=int(os.environ.get("RECONCILE_GRACE_SECONDS", "3600"))
)
p.add_argument(
"--deep", action="store_true", help="stream every present object and compare sha256"
)
p.add_argument(
"--save-ready", metavar="FILE", help="store ids and sha256 of healthy ready attachments"
)
p.add_argument(
"--expect-ready",
metavar="FILE",
help="fail if any attachment from FILE is no longer healthy",
)
p.add_argument("--limit", type=int, default=20, help="max ids per class in the report")
p.add_argument("--output", metavar="FILE")
args = p.parse_args()
started = time.monotonic()
prefix = os.environ.get("S3_PREFIX", "attachments/")
objects, rows, hashes, uploads, seconds = load(
os.environ.get("S3_BUCKET", "messenger-attachments"), prefix, args.deep
)
classes = classify(objects, rows, hashes, uploads, args.grace)
bad = {i for c in BROKEN for i in classes[c]}
healthy = {
r["id"]: r["sha256"]
for k, r in rows.items()
if r["status"] == "ready" and k in objects and r["id"] not in bad
}
if args.expect_ready:
expected = json.loads(Path(args.expect_ready).read_text())
classes["lost_ready"] = sorted(i for i, sha in expected.items() if healthy.get(i) != sha)
if args.save_ready:
Path(args.save_ready).write_text(json.dumps(healthy, sort_keys=True))
violations = [c for c in CLASSES if classes[c] and c not in TOLERATED]
seconds["total"] = round(time.monotonic() - started, 3)
report = {
"checked_at": dt.datetime.now(dt.timezone.utc)
.isoformat(timespec="seconds")
.replace("+00:00", "Z"),
"grace_seconds": args.grace,
"deep": args.deep,
"objects": len(objects),
"rows": len(rows),
"ready_healthy": len(healthy),
"counts": {c: len(v) for c, v in classes.items()},
"samples": {c: sorted(v)[: args.limit] for c, v in classes.items() if v},
"seconds": seconds,
"violations": violations,
"ok": not violations,
}
text = json.dumps(report, ensure_ascii=False, indent=2)
if args.output:
Path(args.output).write_text(text + "\n")
print(text)
sys.exit(0 if report["ok"] else 1)
if __name__ == "__main__":
main()
+823
View File
@@ -0,0 +1,823 @@
#!/usr/bin/env python3
import argparse
import http.client
import json
import os
import re
import statistics
import struct
import subprocess
import sys
import threading
import time
import uuid
import zlib
from pathlib import Path
from urllib.parse import urlsplit
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
client = __import__("client")
FIXTURE = (ROOT / "tests/fixtures/sample.png").read_bytes()
GRACE = 60
FLAGS = ("--grace", str(GRACE), "--skew", "0")
INTENT_AGE = 300
LOCK_KEY = 20260004
FIXABLE = ("orphan_stale", "missing", "size_mismatch", "sha_mismatch", "pending_stale")
USER_HZ = 100
BUSY = (0, 1, 2, 5, 6, 7)
def run(cmd, env=None, stdin=None):
started = time.monotonic()
done = subprocess.run(
cmd,
cwd=ROOT,
input=stdin,
capture_output=True,
check=False,
env=dict(os.environ, **(env or {})),
)
out, err = done.stdout.decode(errors="replace"), done.stderr.decode(errors="replace")
return done.returncode, out, err, round(time.monotonic() - started, 3)
def must(cmd, env=None, stdin=None):
code, out, err, _ = run(cmd, env, stdin)
if code:
raise SystemExit(f"failed: {' '.join(cmd[:7])}\n{err[-800:]}")
return out
def compose(*args, env=None, stdin=None):
return must(["docker", "compose", *args], env, stdin)
def psql(sql):
return compose(
"exec",
"-T",
"postgres",
"psql",
"-v",
"ON_ERROR_STOP=1",
"-qAt",
"-U",
"messenger",
"-d",
"messenger",
"-c",
sql,
).strip()
def reconcile(*flags, env=None):
cmd = ["docker", "compose", "run", "--rm", "--no-deps", "-T", "reconciler", *flags]
code, out, err, wall = run(cmd, env)
if code not in (0, 1) or not out.strip():
raise SystemExit(f"reconciler failed ({code}):\n{err[-800:]}")
report = json.loads(out)
audit = [json.loads(line) for line in err.splitlines() if '"audit.' in line]
report.update(exit_code=code, wall_seconds=wall, audit=audit)
return report
def oracle(*flags):
code, out, err, _ = run([sys.executable, "scripts/reconcile_check.py", *flags])
if not out.strip():
raise SystemExit(f"reconcile_check failed ({code}):\n{err[-800:]}")
return json.loads(out)
def inject(kind, count, ids=(), age=None):
env = {"INJECT_IDS": ",".join(ids)} if ids else {}
if age is not None:
env["INJECT_AGE_SECONDS"] = str(age)
out = must(["sh", "scripts/reconcile-inject.sh", kind, str(count)], env)
return [line.split()[1] for line in out.splitlines() if line.strip()]
def head():
return must(["git", "rev-parse", "HEAD"]).strip()
def evidence_root():
return ROOT / os.environ.get("RECONCILE_EVIDENCE_DIR", "evidence/reconcile")
def save(name, data, *, stamp=True):
path = evidence_root() / name
path.parent.mkdir(parents=True, exist_ok=True)
stamped = {"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "commit": head(), **data}
path.write_text(json.dumps(stamped if stamp else data, ensure_ascii=False, indent=2) + "\n")
return path
class Checks:
def __init__(self):
self.items = []
def __call__(self, name, ok, **detail):
self.items.append(dict(name=name, ok=bool(ok), **detail))
sys.stderr.write(("PASS " if ok else "FAIL ") + name + "\n")
@property
def ok(self):
return all(item["ok"] for item in self.items)
def chunk(tag, data):
return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", zlib.crc32(tag + data))
def tiny_png():
raw = b"".join(b"\x00" + os.urandom(24) for _ in range(8))
header = struct.pack(">IIBBBBB", 8, 8, 8, 2, 0, 0, 0)
return (
b"\x89PNG\r\n\x1a\n"
+ chunk(b"IHDR", header)
+ chunk(b"IDAT", zlib.compress(raw))
+ chunk(b"IEND", b"")
)
def pick(found, names):
return {name: found.get(name, 0) for name in names}
def faults(c, check, pre):
alice, bob, _, chat = c.bootstrap()
ids = [c.attachment(alice, chat)[0]["id"] for _ in range(18)]
linked = ids[:6]
for part in (linked[:3], linked[3:]):
c.message(alice, chat, "", attachments=part)
targets = {
"missing": [linked[0], linked[1], ids[6]],
"corrupt": [linked[2], ids[7], ids[8]],
"bitflip": [linked[3], ids[9], ids[10]],
}
damaged = {i for group in targets.values() for i in group}
ready = evidence_root() / "ready-before.json"
oracle("--grace", str(GRACE), "--deep", "--save-ready", str(ready))
untouched = {k: v for k, v in json.loads(ready.read_text()).items() if k not in damaged}
save("ready-untouched.json", untouched, stamp=False)
started = time.monotonic()
injected = {
"orphan_stale": inject("orphan", 3),
"missing": inject("missing", 3, targets["missing"]),
"size_mismatch": inject("corrupt", 3, targets["corrupt"]),
"sha_mismatch": inject("bitflip", 3, targets["bitflip"]),
"pending_stale": inject("stale-pending", 3, age=INTENT_AGE)
+ inject("pending-object", 3, age=INTENT_AGE),
}
time.sleep(max(0.0, GRACE + 5 - (time.monotonic() - started)))
injected["orphan_fresh"] = inject("orphan", 2)
injected["pending_fresh"] = inject("stale-pending", 2, age=0)
expected = {name: len(keys) for name, keys in injected.items()}
expected.update(quarantined=pre["counts"]["quarantined"])
before = oracle(
"--grace", str(GRACE), "--deep", "--output", str(evidence_root() / "check-before.json")
)
dry = reconcile("--dry-run", "--deep", *FLAGS)
save("dry-run.json", dry)
after_dry = oracle("--grace", str(GRACE), "--deep")
named = {k: sorted(s.rsplit("/", 1)[-1] for s in dry["samples"].get(k, [])) for k in injected}
check(
"dry-run находит каждый внедрённый класс, и найдено ровно внедрённое",
pick(dry["found"], expected) == expected
and named == {name: sorted(ids) for name, ids in injected.items()},
expected=expected,
found=dry["found"],
)
check(
"dry-run ничего не меняет",
not any(dry["fixed"].values())
and after_dry["counts"] == before["counts"]
and (after_dry["objects"], after_dry["rows"]) == (before["objects"], before["rows"]),
objects=[before["objects"], after_dry["objects"]],
rows=[before["rows"], after_dry["rows"]],
)
first = reconcile("--apply", "--deep", *FLAGS)
save("apply-1.json", first)
discarded = sorted(e["attachment_id"] for e in first["audit"] if e["reason"] == "reconcile")
check(
"apply исправляет ровно найденное dry-run, удаление намерений — в журнале аудита",
pick(first["fixed"], FIXABLE) == pick(dry["found"], FIXABLE)
and not first["errors"]
and discarded == sorted(injected["pending_stale"]),
dry_run=pick(dry["found"], FIXABLE),
fixed=pick(first["fixed"], FIXABLE),
audit_discarded=len(discarded),
)
second = reconcile("--apply", "--deep", *FLAGS)
save("apply-2.json", second)
check(
"повторный apply — 0 действий",
not any(second["fixed"].values()),
fixed=second["fixed"],
found=second["found"],
)
after = oracle(
"--grace",
str(GRACE),
"--deep",
"--expect-ready",
str(evidence_root() / "ready-untouched.json"),
"--output",
str(evidence_root() / "check-after.json"),
)
tolerated = ("orphan_fresh", "pending_fresh")
check(
"независимая проверка после apply: нарушений нет, готовые не потеряны",
after["ok"]
and not after["counts"]["lost_ready"]
and pick(after["counts"], tolerated) == pick(before["counts"], tolerated)
and after["counts"]["quarantined"] == pre["counts"]["quarantined"] + 9,
counts=after["counts"],
)
marks = dict(
line.split("|") for line in psql("SELECT id, quarantine_reason FROM attachments").split()
)
links = psql("SELECT attachment_id FROM message_attachments").split()
kept = {i: marks.get(i) for i in sorted(damaged)}
check(
"строки с расхождениями и ссылки сообщений сохранены",
all(kept.values()) and sum(i in linked for i in links) == len(linked),
quarantine=kept,
message_links=sum(i in linked for i in links),
)
meta = c.call("GET", f"/api/v1/attachments/{linked[0]}", 200, actor=bob, schema="Attachment")
blocked = c.call("GET", f"/api/v1/attachments/{linked[2]}/content", 404, actor=bob)
intact = c.call("GET", f"/api/v1/attachments/{linked[4]}/content", 200, actor=bob)
check(
"карантин: метаданные 200, /content 404; нетронутое скачивается",
meta["status"] == 200 and blocked["status"] == 404 and intact["bytes"] == FIXTURE,
)
bucket = os.environ.get("S3_BUCKET", "messenger-attachments")
for attachment, data in ((linked[0], FIXTURE), (linked[1], os.urandom(len(FIXTURE)))):
compose(
"run",
"--rm",
"--no-deps",
"-T",
"s3-cli",
"pipe",
f"root/{bucket}/attachments/{attachment}",
stdin=data,
)
healed = reconcile("--apply", *FLAGS)
save("healed.json", healed)
restored = c.call("GET", f"/api/v1/attachments/{linked[0]}/content", 200, actor=bob)
foreign = c.call("GET", f"/api/v1/attachments/{linked[1]}/content", 404, actor=bob)
check(
"вернули те же байты — карантин снят; другие байты той же длины — остался",
healed["fixed"]["healed"] == 1
and healed["samples"].get("healed") == [linked[0]]
and linked[1] in healed["samples"].get("quarantined", [])
and restored["bytes"] == FIXTURE
and foreign["status"] == 404,
fixed=healed["fixed"],
s3_get=healed["cost"]["requests"]["s3_get"],
)
return {"injected": injected, "targets": targets, "linked": linked}
def single(check):
holder = subprocess.Popen(
[
"docker",
"compose",
"exec",
"-T",
"postgres",
"psql",
"-U",
"messenger",
"-d",
"messenger",
"-c",
f"SELECT pg_advisory_lock({LOCK_KEY}), pg_sleep(15)",
],
cwd=ROOT,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
try:
time.sleep(3)
busy = reconcile("--apply", *FLAGS)
finally:
holder.wait()
save("lock-busy.json", busy)
check(
"второй одновременный проход пропущен",
busy["skipped"] and busy["exit_code"] == 0 and not any(busy["fixed"].values()),
)
def healthy(service, seconds=90):
container = compose("ps", "-q", service).strip()
probe = ["docker", "inspect", "-f", "{{.State.Health.Status}}", container]
deadline = time.monotonic() + seconds
while must(probe).strip() != "healthy":
if time.monotonic() > deadline:
raise SystemExit(f"{service} is not healthy after {seconds} s")
time.sleep(1)
def interrupt(check):
name = f"hlpipe-lab5-reconcile-sigterm-{os.getpid()}"
compose("pause", "s3")
try:
cmd = ["docker", "compose", "run", "--rm", "--no-deps", "-T", "--name", name, "reconciler"]
runner = subprocess.Popen(
[*cmd, "--apply", *FLAGS],
cwd=ROOT,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
time.sleep(4)
state = must(["docker", "inspect", "-f", "{{.State.Status}} {{.HostConfig.Init}}", name])
started = time.monotonic()
must(["docker", "stop", name])
stopped = round(time.monotonic() - started, 2)
code = runner.wait(timeout=60)
finally:
compose("unpause", "s3")
healthy("s3")
after = reconcile("--dry-run", *FLAGS)
status, init = state.split()
report = {
"s3": "paused",
"state_before_stop": status,
"init": init,
"stop_seconds": stopped,
"exit_code": code,
"next_pass_skipped": after["skipped"],
}
save("sigterm.json", report)
check(
"SIGTERM: идущий проход останавливается за секунды, блокировка освобождена",
status == "running" and code == 143 and stopped < 5 and not after["skipped"],
**report,
)
def outage(check):
before = oracle("--grace", str(GRACE))
compose("stop", "s3")
try:
cmd = ["docker", "compose", "run", "--rm", "--no-deps", "-T", "reconciler", "--apply"]
code, out, err, wall = run([*cmd, *FLAGS])
finally:
compose("up", "-d", "--wait", "--no-deps", "s3")
after = oracle("--grace", str(GRACE))
log = [line for line in err.splitlines() if "reconcile." in line]
same = all(after[k] == before[k] for k in ("objects", "rows", "counts"))
save("s3-down.json", {"exit_code": code, "wall_seconds": wall, "stdout": out, "log": log})
check(
"S3 остановлен: код 1 за ограниченное время, состояние не изменилось",
code == 1 and wall < 30 and not out.strip() and same,
exit_code=code,
wall_seconds=wall,
)
def uploader(base, token, chat, *, deadline, results, lock):
host = urlsplit(base).netloc
conn = http.client.HTTPConnection(host, timeout=30)
headers = {"Authorization": f"Bearer {token}", "Accept": "application/json"}
sent = 0
while time.monotonic() < deadline:
path = f"/api/v1/chats/{chat}/attachments?filename=load.png"
try:
conn.request("POST", path, tiny_png(), {**headers, "Content-Type": "image/png"})
response = conn.getresponse()
status, data = response.status, response.read()
if status == 201 and sent % 10 == 0:
body = json.dumps({"text": "", "attachment_ids": [json.loads(data)["id"]]})
extra = {"Content-Type": "application/json", "Idempotency-Key": uuid.uuid4().hex}
conn.request("POST", f"/api/v1/chats/{chat}/messages", body, {**headers, **extra})
linked = conn.getresponse()
linked.read()
status = 201 if linked.status == 201 else -linked.status
except (OSError, http.client.HTTPException):
conn.close()
conn = http.client.HTTPConnection(host, timeout=30)
status, data = 0, b""
with lock:
results.append((status, json.loads(data) if status == 201 else None))
sent += 1
time.sleep(0.05)
def under_load(c, check, base, seconds, threads):
alice, _, _, chat = c.bootstrap()
ready = evidence_root() / "ready-load.json"
left = oracle("--grace", str(GRACE), "--save-ready", str(ready))["counts"]
leftover = {
"orphan_stale": left["orphan_fresh"] + left["orphan_stale"],
"pending_stale": left["pending_fresh"]
+ left["pending_stale_with_object"]
+ left["pending_stale_without_object"],
}
orphans = inject("orphan", 3)
intents = inject("stale-pending", 3, age=INTENT_AGE)
results, lock = [], threading.Lock()
deadline = time.monotonic() + seconds
workers = [
threading.Thread(
target=uploader,
args=(base, alice["token"], chat["id"]),
kwargs={"deadline": deadline, "results": results, "lock": lock},
)
for _ in range(threads)
]
for worker in workers:
worker.start()
runs = []
while time.monotonic() < deadline:
runs.append(reconcile("--apply", *FLAGS))
for worker in workers:
worker.join()
runs.append(reconcile("--apply", *FLAGS))
created = {att["id"]: att["sha256"] for status, att in results if att}
expected = dict(json.loads(ready.read_text()), **created)
save("ready-load.json", expected, stamp=False)
after = oracle(
"--grace",
str(GRACE),
"--deep",
"--expect-ready",
str(ready),
"--output",
str(evidence_root() / "check-under-load.json"),
)
statuses = {}
for status, _ in results:
statuses[str(status)] = statuses.get(str(status), 0) + 1
fixed = {name: sum(r["fixed"][name] for r in runs) for name in FIXABLE}
rechecked = {}
for r in runs:
for name, count in r["rechecked"].items():
rechecked[name] = rechecked.get(name, 0) + count
seen = {name: sum(r["found"][name] for r in runs) for name in ("orphan_fresh", "pending_fresh")}
summary = {
"seconds": seconds,
"upload_threads": threads,
"uploads": statuses,
"created": len(created),
"linked_every": 10,
"reconciler_runs": len(runs),
"run_seconds": [r["cost"]["seconds"] for r in runs],
"fixed_total": fixed,
"fresh_seen_total": seen,
"rechecked_total": rechecked,
"errors": sum(r["errors"] for r in runs),
"skipped": sum(1 for r in runs if r["skipped"]),
"injected_before_load": {"orphan": len(orphans), "pending_stale": len(intents)},
"left_from_faults": leftover,
"check_after": {k: after[k] for k in ("objects", "rows", "ready_healthy", "counts", "ok")},
}
save("under-load.json", summary)
check(
"под нагрузкой: исправлены только внедрённые, ошибок нет",
fixed
== {
**dict.fromkeys(FIXABLE, 0),
"orphan_stale": leftover["orphan_stale"] + len(orphans),
"pending_stale": leftover["pending_stale"] + len(intents),
}
and not summary["errors"]
and not summary["skipped"],
fixed=fixed,
runs=len(runs),
)
check(
"под нагрузкой: все 201 готовы и совпадают по SHA-256, потерь 0",
after["ok"] and not after["counts"]["lost_ready"] and set(statuses) == {"201"},
uploads=statuses,
lost_ready=after["counts"]["lost_ready"],
)
def demo(args):
evidence_root().mkdir(parents=True, exist_ok=False)
c = client.Client(args.base_url, timeout=30)
check = Checks()
pre = oracle("--grace", str(GRACE))
dirty = {k: v for k, v in pre["counts"].items() if v and k != "quarantined"}
if dirty:
raise SystemExit(f"стенд не в исходном состоянии {dirty}: make reconcile ARGS=--apply")
started = time.monotonic()
result = {"started_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "commit": head()}
result["faults"] = faults(c, check, pre)
single(check)
interrupt(check)
outage(check)
under_load(c, check, args.base_url, args.load_seconds, args.load_threads)
result.update(seconds=round(time.monotonic() - started, 1), checks=check.items, ok=check.ok)
save("demo.json", result)
print(json.dumps({"ok": check.ok, "checks": len(check.items)}, ensure_ascii=False))
sys.exit(0 if check.ok else 1)
def pins(args):
cpus = int(must(["docker", "info", "--format", "{{.NCPU}}"]).strip())
sized = {"S3_CPUS": "2", "S3_MEMORY": args.s3_memory}
if args.no_pin or cpus < 8:
return sized
return dict(
sized,
S3_CPUSET=f"{cpus - 5}-{cpus - 4}",
PG_CPUSET=f"{cpus - 3}-{cpus - 2}",
RECONCILE_CPUSET=str(cpus - 1),
)
def cpu_set(env):
cpus = set()
for name in ("S3_CPUSET", "PG_CPUSET", "RECONCILE_CPUSET"):
for part in filter(None, env.get(name, "").split(",")):
low, _, high = part.partition("-")
cpus.update(range(int(low), int(high or low) + 1))
return cpus
def limits():
ids = compose("ps", "-q", "s3", "postgres").split()
fmt = (
"{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}}"
" cpuset={{.HostConfig.CpusetCpus}}"
)
return must(["docker", "inspect", "-f", fmt, *ids]).strip().splitlines()
def host(title):
stats = must(
[
"docker",
"stats",
"--no-stream",
"--all",
"--format",
"table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}\t{{.PIDs}}",
]
)
running = must(["docker", "ps", "--format", "{{.Names}} {{.Image}} {{.Status}}"])
stamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
load = " ".join(f"{v:.2f}" for v in os.getloadavg())
return (
f"== {title} date: {stamp} commit: {head()} host loadavg: {load}\n"
f"$ docker stats --no-stream --all\n{stats}\n$ docker ps\n{running}\n"
)
def usage(text):
return int(re.search(r"^usage_usec (\d+)$", text, re.M).group(1)) / 1e6
def sample():
vm = compose(
"exec", "-T", "postgres", "cat", "/proc/uptime", "/proc/stat", "/sys/fs/cgroup/cpu.stat"
)
s3 = compose("exec", "-T", "s3", "cat", "/sys/fs/cgroup/cpu.stat")
busy = {}
for line in vm.splitlines()[1:]:
name, *values = line.split() or [""]
if name.startswith("cpu") and name[3:].isdigit():
busy[int(name[3:])] = sum(int(values[i]) for i in BUSY) / USER_HZ
return float(vm.split()[0]), busy, usage(vm) + usage(s3)
def screen(first, last, own, pinned, args, *, started=True):
window = max(last[0] - first[0], 0.01)
busy = {cpu: spent - first[1].get(cpu, 0.0) for cpu, spent in last[1].items()}
ours = last[2] - first[2] + own
other = sum(busy.values()) - ours - args.start_other * started
on_pinned = sum(busy.get(cpu, 0.0) for cpu in pinned) - ours - args.start_pinned * started
other, on_pinned = max(0.0, other / window), max(0.0, on_pinned / window) if pinned else 0.0
return {
"window_seconds": round(window, 2),
"ours_cores": round(ours / window, 2),
"other_cores": round(other, 2),
"other_on_pinned_cores": round(on_pinned, 2),
"host_loadavg_1m": round(os.getloadavg()[0], 2),
"disturbed": other > args.max_other or on_pinned > args.max_pinned,
}
def wait_quiet(args, pinned):
deadline, calm = time.monotonic() + args.quiet_timeout, 0
while calm < 3 and time.monotonic() < deadline:
first = sample()
time.sleep(5)
quiet = screen(first, sample(), 0.0, pinned, args, started=False)
calm = 0 if quiet["disturbed"] else calm + 1
sys.stderr.write(f"waiting for quiet: {json.dumps(quiet)}\n")
return calm == 3
def measure(flags, env, args, pinned):
rejected = []
for attempt in range(args.retries + 1):
first = sample()
report = reconcile(*flags, "--samples", "3", env=env)
report["screen"] = screen(first, sample(), report["cost"]["cpu_seconds"], pinned, args)
if not report["screen"]["disturbed"]:
return report, rejected
rejected.append(report)
if attempt < args.retries and not wait_quiet(args, pinned):
break
return None, rejected
def seed(count, env):
if count <= 0:
return {"added": 0}
env = dict(env, API_CPUS="4", API_MEMORY="512m")
tools = f"{ROOT / 'student-tests' / 'tools'}:/tools:ro"
out = compose(
"run",
"--rm",
"--no-deps",
"-T",
"-v",
tools,
"--entrypoint",
"python",
"api",
"/tools/seed_storage.py",
"--count",
str(count),
"--concurrency",
"4",
env=env,
)
records = [json.loads(line) for line in out.strip().splitlines()]
return {**records[-1], "progress": records[:-1]}
def median(runs, path):
values = []
for r in runs:
value = r
for key in path:
value = value[key]
values.append(value)
return statistics.median(values)
def summarize(results, rejected):
rows = []
for (n, mode), runs in sorted(results.items()):
seconds = median(runs, ("cost", "seconds"))
rows.append(
{
"objects": n,
"mode": mode,
"runs": len(runs),
"seconds_median": seconds,
"seconds_all": [r["cost"]["seconds"] for r in runs],
"wall_seconds_median": median(runs, ("wall_seconds",)),
"us_per_object": round(seconds / n * 1e6, 1),
"cpu_seconds_median": median(runs, ("cost", "cpu_seconds")),
"s3_list_wait_median": median(runs, ("cost", "waited_seconds", "s3_list")),
"sql_wait_median": median(runs, ("cost", "waited_seconds", "sql")),
"requests": runs[0]["cost"]["requests"],
"max_rss_mib_median": median(runs, ("cost", "max_rss_mib")),
"other_cores_max": max(r["screen"]["other_cores"] for r in runs),
"other_on_pinned_cores_max": max(
r["screen"]["other_on_pinned_cores"] for r in runs
),
"disturbed_kept": sum(r["screen"]["disturbed"] for r in runs),
"rejected": rejected.get((n, mode), 0),
"scanned": runs[0]["scanned"],
"found_nonzero": {k: v for k, v in runs[0]["found"].items() if v},
}
)
return rows
def table(rows):
head = (
"objects mode runs t_med,s wall,s us/obj cpu,s list,s sql,s lists sqls"
" gets rss,MiB other pinned rejected"
)
lines = [head]
for r in rows:
q = r["requests"]
lines.append(
f"{r['objects']:>7} {r['mode']:<9} {r['runs']:>4} {r['seconds_median']:>7.3f} "
f"{r['wall_seconds_median']:>6.2f} {r['us_per_object']:>6.1f} "
f"{r['cpu_seconds_median']:>5.2f} {r['s3_list_wait_median']:>6.3f} "
f"{r['sql_wait_median']:>5.3f} {q['s3_list']:>5} {q['sql']:>4} {q['s3_get']:>5} "
f"{r['max_rss_mib_median']:>7.1f} {r['other_cores_max']:>5.2f} "
f"{r['other_on_pinned_cores_max']:>6.2f} {r['rejected']:>8}"
)
return "\n".join(lines) + "\n"
def cost(args):
out = evidence_root() / "cost"
out.mkdir(parents=True, exist_ok=False)
(out / "disturbed").mkdir()
rows = int(psql("SELECT count(*) FROM attachments"))
if rows > args.sizes[0]:
raise SystemExit(f"на стенде уже {rows} вложений: используйте новый отдельный project")
snapshot = out / "host-docker-stats.txt"
snapshot.write_text(host("before the series"))
env = pins(args)
pinned = cpu_set(env)
compose("up", "-d", "--wait", "--no-deps", "s3", "postgres", env=env)
stamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
plan = {"date": stamp, "commit": head(), "pins": env, "limits": limits(), "seeds": []}
plan["screen"] = {
"max_other_cores": args.max_other,
"max_other_on_pinned_cores": args.max_pinned,
"start_allowance_core_seconds": {"vm": args.start_other, "pinned": args.start_pinned},
"retries": args.retries,
}
results, rejected = {}, {}
try:
for n in args.sizes:
plan["seeds"].append(dict(seed(n - rows, env), target=n))
rows = n
time.sleep(args.settle)
save("cost/plan.json", plan)
if not wait_quiet(args, pinned):
raise SystemExit("no quiet measurement window before timeout")
modes = [("shallow", ["--dry-run"])]
if n <= args.deep_max:
modes.append(("deep", ["--dry-run", "--deep"]))
if n == args.page_compare:
modes.append(("page100", ["--dry-run", "--page-size", "100"]))
for mode, flags in modes:
for attempt in range(1, args.repeats + 1):
report, dropped = measure(flags, env, args, pinned)
name = f"n{n}-{mode}-{attempt}"
for number, bad in enumerate(dropped, 1):
save(f"cost/disturbed/{name}-try{number}.json", bad)
rejected[(n, mode)] = rejected.get((n, mode), 0) + len(dropped)
if report is None:
raise SystemExit(f"{name}: no clean run after retries; rejected preserved")
if (
report["exit_code"]
or report["errors"]
or report["skipped"]
or report["scanned"] != {"objects": n, "rows": n}
or any(report["found"].values())
):
save(f"cost/invalid/{name}.json", report)
raise SystemExit(f"{name}: inconsistent dataset or failed pass")
save(f"cost/{name}.json", report)
results.setdefault((n, mode), []).append(report)
save("cost/summary.json", {**plan, "results": summarize(results, rejected)})
sys.stderr.write(f"{name}: {report['cost']['seconds']} s {report['screen']}\n")
finally:
save("cost/summary.json", {**plan, "results": summarize(results, rejected)})
compose("up", "-d", "--wait", "--no-deps", "s3", "postgres")
with snapshot.open("a") as handle:
handle.write("\n" + host("after the series"))
summary = summarize(results, rejected)
save("cost/summary.json", {**plan, "results": summary})
(out / "summary.txt").write_text(
f"date: {stamp} commit: {plan['commit']} pins: {env}\n" + table(summary)
)
sys.stdout.write(table(summary))
def main():
p = argparse.ArgumentParser(description="Lab4 track 2: reconciler demo and cost of a pass")
sub = p.add_subparsers(dest="command", required=True)
d = sub.add_parser("demo", help="fault injection, dry-run/apply/repeat, safety, under load")
d.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080"))
d.add_argument("--load-seconds", type=int, default=75)
d.add_argument("--load-threads", type=int, default=3)
k = sub.add_parser("cost", help="cost of a pass for N objects, repeated")
k.add_argument("--sizes", type=int, nargs="+", default=[1000, 10000, 30000, 100000])
k.add_argument("--repeats", type=int, default=3)
k.add_argument("--deep-max", type=int, default=100000)
k.add_argument("--page-compare", type=int, default=10000)
k.add_argument("--settle", type=float, default=10)
k.add_argument("--s3-memory", default="3g")
k.add_argument("--no-pin", action="store_true")
k.add_argument("--max-other", type=float, default=2.0, help="foreign CPU in the VM, cores")
k.add_argument("--max-pinned", type=float, default=0.25, help="foreign CPU on pinned cores")
k.add_argument("--start-other", type=float, default=1.5, help="container start, core-s")
k.add_argument("--start-pinned", type=float, default=0.5, help="the same on pinned cores")
k.add_argument("--retries", type=int, default=3, help="reruns of a disturbed run")
k.add_argument("--quiet-timeout", type=float, default=900.0)
for command in (d, k):
command.add_argument(
"--out", default=os.environ.get("RECONCILE_EVIDENCE_DIR", "evidence/reconcile")
)
args = p.parse_args()
os.environ["RECONCILE_EVIDENCE_DIR"] = args.out
if args.command == "demo":
demo(args)
else:
cost(args)
if __name__ == "__main__":
main()
+4 -2
View File
@@ -2,7 +2,8 @@
set -eu
cd "$(dirname "$0")/.."
service=${API_SERVICE:-api}
show() { docker inspect -f '{{.Config.Hostname}} image={{slice .Image 7 19}}' "$1"; }
show() { docker inspect -f '{{.Config.Hostname}} image={{.Config.Image}}@{{slice .Image 7 19}}' "$1"; }
scripts/stand-usage.sh &
echo "$(date -u +%T) migrate once before the first replacement"
docker compose run --rm --no-deps -T migrate 2>&1
old=$(docker compose ps -q "$service")
@@ -13,8 +14,9 @@ for id in $old; do
scripts/lb-wait.sh $((count + 1))
new=$(docker compose ps -q "$service" | grep -v -F "$running")
echo "$(date -u +%T) replace $(show "$id") -> $(show "$new")"
docker stop -t "${STOP_TIMEOUT:-6}" "$id" >/dev/null
docker stop -t "${STOP_TIMEOUT:-7}" "$id" >/dev/null
docker rm "$id" >/dev/null
done
scripts/lb-wait.sh "$count"
wait
echo "$(date -u +%T) done: $count replicas replaced"
+30 -7
View File
@@ -2,17 +2,30 @@
set -eu
cd "$(dirname "$0")/.."
cli() { docker compose run --rm --no-deps -T s3-cli "$@"; }
undated() {
python3 -c 'import json, sys
def clean(value):
if isinstance(value, dict):
return {k: clean(v) for k, v in value.items() if not k.endswith("_date")}
return [clean(v) for v in value] if isinstance(value, list) else value
text, decoder, at = sys.stdin.read(), json.JSONDecoder(), 0
while text[at:].strip():
at += len(text[at:]) - len(text[at:].lstrip())
value, at = decoder.raw_decode(text, at)
print(json.dumps(clean(value), indent=2, ensure_ascii=False))'
}
snapshot() {
cli sh -s <<'EOF' | grep -v '_date'
cli sh -s <<'EOF' | undated
ak=$(cat /run/secrets/s3_access_key_id)
for c in "bucket list root" "bucket anonymous get root/$S3_BUCKET" "admin policy info root messenger-api" "admin user info root $ak" "bucket lifecycle rule export root/$S3_BUCKET"; do
rk=$(cat /run/secrets/s3_reconciler_access_key_id)
for c in "bucket list root" "bucket anonymous get root/$S3_BUCKET" "admin policy info root messenger-api" "admin policy info root messenger-reconciler" "admin user info root $ak" "admin user info root $rk" "bucket lifecycle rule export root/$S3_BUCKET"; do
rc $c --json
done | sed "s/$ak/<api-key>/g"
done | sed "s/$ak/<api-key>/g; s/$rk/<reconciler-key>/g"
EOF
}
failed=0
verdict() { if [ "$2" = "$3" ]; then echo "PASS $1: $3"; else echo "FAIL $1: ожидалось $2, получено $3"; failed=1; fi; }
echo "== $(date -u +%FT%TZ) S3 privacy and bootstrap checks"
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD) == S3 privacy and bootstrap checks"
verdict "s3: опубликованные host-порты" "нет" "$(docker inspect -f '{{json .NetworkSettings.Ports}}' "$(docker compose ps -q s3)" | grep -q HostPort && echo есть || echo нет)"
verdict "s3: консоль :9001 внутри контейнера" "000" "$(docker compose exec -T s3 curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:9001/ || true)"
before=$(snapshot)
@@ -25,22 +38,32 @@ verdict "s3-init: состояние IAM/bucket не изменилось" "same
cli sh -s <<'EOF' || failed=1
b=$S3_BUCKET
key="attachments/s3-check-$(date +%s)-$$"
spare="$key-reconciler"
other="s3-check-$(date +%s)-$$"
head -c 4096 /dev/urandom > /tmp/probe
result=0
expect() {
want=$1; name=$2; shift 2
if "$@" >/tmp/out 2>&1; then got=allow; elif grep -qE '403|AccessDenied|set_bucket_policy: service error' /tmp/out; then got=deny; else got=error; fi
if "$@" >/tmp/out 2>&1; then got=allow; elif grep -qE '403|AccessDenied|set_bucket_policy: service error|check bucket: Network error: service error' /tmp/out; then got=deny; else got=error; fi
if [ "$got" = "$want" ]; then echo "PASS $name: $got"; else echo "FAIL $name: ожидалось $want, получено $got"; result=1; fi
}
expect allow "api: PutObject" rc put /tmp/probe "api/$b/$key"
expect allow "api: HeadObject" rc object stat "api/$b/$key"
expect allow "api: GetObject" rc object show "api/$b/$key"
expect allow "api: ListBucket" rc object list "api/$b/attachments/"
expect deny "api: ListBucket" rc object list "api/$b/attachments/"
rc put /tmp/probe "api/$b/$spare" >/dev/null
expect allow "reconciler: ListBucket" rc object list "reconciler/$b/attachments/"
expect allow "reconciler: HeadObject" rc object stat "reconciler/$b/$key"
expect allow "reconciler: GetObject" rc object show "reconciler/$b/$key"
expect deny "reconciler: PutObject" rc put /tmp/probe "reconciler/$b/$key-put"
expect allow "reconciler: DeleteObject" rc object remove --force "reconciler/$b/$spare"
expect deny "reconciler: CreateBucket" rc bucket create "reconciler/$other"
expect deny "reconciler: PutBucketPolicy (public)" rc bucket anonymous set public "reconciler/$b"
expect deny "reconciler: admin ListUsers" rc admin user list reconciler
expect deny "api: CreateBucket" rc bucket create "api/$other"
expect deny "api: DeleteBucket" rc bucket remove "api/$b"
expect deny "api: PutBucketPolicy (public)" rc bucket anonymous set public "api/$b"
expect deny "api: PutBucketLifecycle" rc bucket lifecycle rule import "api/$b" /s3/lifecycle.json
expect deny "api: PutBucketLifecycle (HeadBucket уже 403)" rc bucket lifecycle rule import "api/$b" /s3/lifecycle.json
expect deny "api: admin ListUsers" rc admin user list api
rc bucket create --ignore-existing "root/$other" >/dev/null
expect deny "api: PutObject в чужой bucket" rc put /tmp/probe "api/$other/x"
+524
View File
@@ -0,0 +1,524 @@
#!/usr/bin/env python3
import argparse
import collections
import contextlib
import csv
import json
import math
import os
import re
import signal
import statistics
import subprocess
import sys
import time
from pathlib import Path
from probe import ROOT, k6_metrics, lb_up, run_k6
SCRIPTS = ROOT / "scripts"
UNITS = {"B": 1, "KIB": 2**10, "MIB": 2**20, "GIB": 2**30, "KB": 10**3, "MB": 10**6, "GB": 10**9}
PINNED = {
"PG_CPUSET": "0-1",
"LB_CPUSET": "2",
"VALKEY_CPUSET": "3",
"K6_CPUSET": "4-6",
"K6_CPUS": "3",
"K6_MEMORY": "1g",
}
API_FIRST_CPU = 7
FIELDS = [
"raw_dir",
"variant",
"config",
"replicas",
"cpus",
"memory_mb",
"workers",
"pool",
"maxconn",
"api_cpuset",
"rate",
"repeat",
"started",
"db_mb",
"messages",
"k6_exit",
"saturated",
"requests",
"rps",
"ok_rps",
"failed_ratio",
"dropped_iterations",
"p50_ms",
"p95_ms",
"p99_ms",
"p95_post_ms",
"p95_get_ms",
"cpu_api",
"cpu_postgres",
"cpu_lb",
"cpu_valkey",
"cpu_k6",
"mem_api_mb",
"mem_postgres_mb",
"mem_k6_mb",
"pg_backends_max",
"pg_active_avg",
"pg_active_max",
"pg_idle_avg",
"pg_idle_tx_avg",
"pg_lock_avg",
"pg_lock_max",
"pg_lwlock_avg",
"pg_io_avg",
"lb_scur_max",
"lb_qcur_max",
"lb_qtime_max_ms",
"lb_rtime_avg_ms",
"cpu_vm",
"cpu_other",
"gap_s",
"wall_s",
"disturbed",
]
def compose(*args, env=None):
command = ["docker", "compose", *args]
return subprocess.run(
command, cwd=str(ROOT), env=env, check=True, capture_output=True, text=True
).stdout
def size(text):
match = re.match(r"([\d.]+)\s*([A-Za-z]+)", text.strip())
return float(match.group(1)) * UNITS.get(match.group(2).upper(), 1) if match else 0.0
def rows(path, start, end):
with path.open() as handle:
return [r for r in csv.DictReader(handle) if r["ts"] and start <= float(r["ts"]) <= end]
def mean(values):
return round(statistics.mean(values), 2) if values else 0.0
def docker_usage(path, project, start, end):
ticks = collections.defaultdict(collections.Counter)
memory = collections.Counter()
for r in rows(path, start, end):
name = re.sub(r"(-run-[0-9a-f]+|-\d+)$", "", r["name"][len(project) + 1 :])
ticks[r["ts"]][name] += float(r["cpu_pct"].rstrip("%") or 0)
memory[name] = max(memory[name], size(r["mem_usage"].split("/")[0]) / 2**20)
cpu = {
name: mean([t[name] for t in ticks.values() if name in t])
for name in ("api", "postgres", "lb", "valkey", "k6")
}
usage = {f"cpu_{name}": value for name, value in cpu.items()}
usage.update({f"mem_{name}_mb": round(memory[name], 1) for name in ("api", "postgres", "k6")})
return usage
def vm_usage(path, start, end):
samples = [(int(r["busy"]), int(r["total"]), int(r["cpus"])) for r in rows(path, start, end)]
if len(samples) < 2 or samples[-1][1] == samples[0][1]:
return 0.0
first, last = samples[0], samples[-1]
return round(100 * last[2] * (last[0] - first[0]) / (last[1] - first[1]), 1)
def gap(path, start, end):
ticks = sorted({float(r["ts"]) for r in rows(path, start - 5, end + 5)})
edges = [start, *[t for t in ticks if start <= t <= end], end]
return max(b - a for a, b in zip(edges, edges[1:]))
def vm_counters():
lines = compose("exec", "-T", "valkey", "cat", "/proc/stat").splitlines()
values = [int(v) for v in lines[0].split()[1:]]
cpus = sum(1 for line in lines if re.match(r"cpu\d", line))
return sum(values) - values[3] - values[4], sum(values), cpus
def wait_quiet(args):
deadline, calm = time.monotonic() + args.quiet_timeout, 0
while calm < 3 and time.monotonic() < deadline:
first = vm_counters()
time.sleep(5)
last = vm_counters()
busy = 100 * last[2] * (last[0] - first[0]) / max(1, last[1] - first[1])
calm = calm + 1 if busy < args.max_other else 0
print(json.dumps({"waiting_for_quiet_vm_cpu": round(busy, 1)}), flush=True)
def pg_usage(path, start, end):
ticks = collections.defaultdict(collections.Counter)
for r in rows(path, start, end):
count, state, wait = int(r["backends"]), r["state"], r["wait_event_type"]
tick = ticks[r["ts"]]
tick["backends"] += count
tick["active"] += count * (state == "active")
tick["idle"] += count * (state == "idle")
tick["idle_tx"] += count * state.startswith("idle in transaction")
tick["lock"] += count * (wait == "Lock")
tick["lwlock"] += count * (wait == "LWLock")
tick["io"] += count * (wait == "IO")
series = list(ticks.values())
return {
"pg_backends_max": max((t["backends"] for t in series), default=0),
"pg_active_avg": mean([t["active"] for t in series]),
"pg_active_max": max((t["active"] for t in series), default=0),
"pg_idle_avg": mean([t["idle"] for t in series]),
"pg_idle_tx_avg": mean([t["idle_tx"] for t in series]),
"pg_lock_avg": mean([t["lock"] for t in series]),
"pg_lock_max": max((t["lock"] for t in series), default=0),
"pg_lwlock_avg": mean([t["lwlock"] for t in series]),
"pg_io_avg": mean([t["io"] for t in series]),
}
def lb_usage(path, start, end):
backend = [
r for r in rows(path, start, end) if r["pxname"] == "api" and r["svname"] == "BACKEND"
]
def numbers(field):
return [int(r[field] or 0) for r in backend]
return {
"lb_scur_max": max(numbers("scur"), default=0),
"lb_qcur_max": max(numbers("qcur"), default=0),
"lb_qtime_max_ms": max(numbers("qtime"), default=0),
"lb_rtime_avg_ms": mean(numbers("rtime")),
}
def trend(path, name):
metrics = json.loads(Path(path).read_text())["metrics"]
found = metrics.get(f"http_req_duration{{scenario:measure,name:{name}}}", {})
return round(found.get("values", {}).get("p(95)", 0.0), 2)
def sampler(*command):
return subprocess.Popen([str(SCRIPTS / command[0]), *command[1:]], start_new_session=True)
def stop(process):
try:
os.killpg(process.pid, signal.SIGTERM)
except ProcessLookupError:
return
process.wait()
def wait_lb(expected, timeout=120):
deadline, stable = time.monotonic() + timeout, 0
while stable < 4:
if time.monotonic() > deadline:
raise SystemExit(f"LB does not settle on {expected} UP servers (now {lb_up()})")
stable = stable + 1 if lb_up() == expected else 0
time.sleep(0.5)
def layout(config, args, variant):
name, _, extra = variant.partition(":")
replicas, cpus = (float(v) for v in config.split("x"))
replicas = int(replicas)
workers = args.workers or max(1, int(cpus))
width = max(1, math.ceil(replicas * cpus))
cpuset = str(API_FIRST_CPU) + (f"-{API_FIRST_CPU + width - 1}" if width > 1 else "")
memory = args.api_memory or args.api_memory_total // replicas
settings = {
"API_REPLICAS": str(replicas),
"API_CPUS": f"{cpus:g}",
"API_MEMORY": f"{memory}m",
"WEB_CONCURRENCY": str(workers),
"DB_POOL_MAX_SIZE": str(args.pool),
"API_SERVER_MAXCONN": str(args.conn_per_worker * workers),
"API_CPUSET": cpuset if args.pin else "",
}
settings.update(dict(kv.split("=", 1) for kv in [*args.env, *filter(None, extra.split(","))]))
base = {
"variant": name,
"config": config,
"replicas": replicas,
"cpus": cpus,
"memory_mb": memory,
"workers": int(settings["WEB_CONCURRENCY"]),
"pool": int(settings["DB_POOL_MAX_SIZE"]),
"maxconn": int(settings["API_SERVER_MAXCONN"]),
"api_cpuset": settings["API_CPUSET"],
}
return base, settings
def apply(base, settings, env, out):
env = dict(env, **settings)
for attempt in range(3):
try:
compose("up", "-d", "--wait", "--wait-timeout", "180", env=env)
break
except subprocess.CalledProcessError as error:
log = Path(out) / "compose-up-failed.txt"
with log.open("a") as handle:
stamp = time.strftime("%FT%TZ", time.gmtime())
commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
handle.write(f"date: {stamp} commit: {commit}\n")
handle.write(f"== {stamp} {base} attempt {attempt + 1}\n")
handle.write(error.stdout + error.stderr + "\n")
print(json.dumps({"compose_up_failed": attempt + 1, "log": str(log)}))
if attempt == 2:
raise
time.sleep(30)
wait_lb(base["replicas"])
return env
def deployment(env):
ids = compose("ps", "-q", env=env).split()
template = "{{json .Name}} {{json .Image}} {{json .HostConfig}}"
result = subprocess.run(
["docker", "inspect", "--format", template, *ids],
check=True,
capture_output=True,
text=True,
)
containers = []
for line in result.stdout.splitlines():
name, image, config = line.split(" ", 2)
limits = json.loads(config)
containers.append(
dict(
name=json.loads(name),
image=json.loads(image),
**{k: limits[k] for k in ("NanoCpus", "Memory", "CpusetCpus", "PortBindings")},
)
)
return containers
def provenance(folder, args, env):
stamp = time.strftime("%FT%TZ", time.gmtime())
metadata = {
"date": stamp,
"commit": args.commit,
"project": args.project,
"command": sys.argv,
"api_image": env.get("API_IMAGE", "compose default"),
"warmup_seconds": args.warmup,
"duration_seconds": args.duration,
"containers": args.deployment,
}
(folder / "metadata.json").write_text(json.dumps(metadata, ensure_ascii=False) + "\n")
for path in folder.rglob("*"):
if path.suffix in {".csv", ".txt"}:
path.write_text(f"# date: {stamp} commit: {args.commit}\n" + path.read_text())
elif path.name == "summary.json":
data = dict(date=stamp, commit=args.commit, **json.loads(path.read_text()))
path.write_text(json.dumps(data) + "\n")
def attempt_folder(out, base, rate, repeat):
name = f"{base['variant']}-{base['config']}_rate{rate}_rep{repeat}"
attempt = 1
while (Path(out) / f"{name}_attempt{attempt}").exists():
attempt += 1
folder = Path(out) / f"{name}_attempt{attempt}"
folder.mkdir(parents=True)
return folder
def measure(folder, args, rate, env):
subprocess.run([str(SCRIPTS / "pg-stats.sh"), "reset"], check=True)
database = subprocess.run(
[str(SCRIPTS / "pg-stats.sh"), "size"], check=True, capture_output=True, text=True
).stdout.split(",")
samplers = [
sampler("docker-stats.sh", str(folder / "docker_stats.csv")),
sampler("lb-stats.sh", str(folder / "lb_stats.csv")),
sampler("pg-stats.sh", "activity", str(folder / "pg_activity.csv")),
]
load = dict(env, RATE=str(rate), WARMUP=str(args.warmup), DURATION=str(args.duration))
began = time.time()
try:
code, started = run_k6(folder / "k6", load)
finally:
wall = time.time() - began
for process in samplers:
stop(process)
subprocess.run(
[str(SCRIPTS / "pg-stats.sh"), "statements", str(folder / "pg_statements.csv")], check=True
)
if started is None:
raise SystemExit(f"k6 failed in setup, see {folder / 'k6/k6.txt'}")
return code, started + args.warmup, database, wall
def step(base, rate, repeat, args, env):
folder = attempt_folder(args.out, base, rate, repeat)
code, start, database, wall = measure(folder, args, rate, env)
end = start + args.duration
summary = folder / "k6/summary.json"
metrics = k6_metrics(summary, args.duration)
saturated = (
metrics["failed_ratio"] > args.max_error_ratio
or metrics["dropped_iterations"] > args.max_error_ratio * rate * args.duration
or metrics["p95_ms"] > args.slo_p95_ms
)
row = dict(
base,
raw_dir=folder.name,
rate=rate,
repeat=repeat,
k6_exit=code,
saturated=saturated,
**metrics,
)
row["started"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(start))
row["db_mb"] = round(int(database[0]) / 2**20, 1)
row["messages"] = int(database[1])
row["p95_post_ms"] = trend(summary, "post_message")
row["p95_get_ms"] = trend(summary, "get_history")
row.update(docker_usage(folder / "docker_stats.csv", args.project, start, end))
row.update(pg_usage(folder / "pg_activity.csv", start, end))
row.update(lb_usage(folder / "lb_stats.csv", start, end))
row["cpu_vm"] = vm_usage(folder / "vm_stat.csv", start, end)
own = sum(row[f"cpu_{name}"] for name in ("api", "postgres", "lb", "valkey", "k6"))
row["cpu_other"] = round(row["cpu_vm"] - own, 1)
row["gap_s"] = round(gap(folder / "docker_stats.csv", start, end), 1)
row["wall_s"] = round(wall, 1)
row["disturbed"] = (
row["cpu_other"] > args.max_other
or row["gap_s"] > args.max_gap
or wall > args.warmup + args.duration + args.max_overhead
or row["cpu_k6"] >= 95 * float(env.get("K6_CPUS", "2"))
or any(row[f"cpu_{name}"] <= 0 for name in ("vm", "api", "postgres", "k6"))
or not rows(folder / "pg_activity.csv", start, end)
or not rows(folder / "lb_stats.csv", start, end)
)
provenance(folder, args, env)
if code or not metrics["requests"]:
raise SystemExit(f"invalid k6 run: {folder}")
print(json.dumps({k: row[k] for k in FIELDS}, ensure_ascii=False), flush=True)
return row
def rates(args, base):
if args.rates:
return [int(r) for r in args.rates.split(",")]
factor, top = (float(v) for v in args.search.split(":"))
value, values = args.start_per_cpu * base["replicas"] * base["cpus"], []
while value <= top:
values.append(int(round(value, -1)))
value *= factor
return values
def arguments():
parser = argparse.ArgumentParser(description="Track 1: replicas against throughput")
parser.add_argument("--out", required=True)
parser.add_argument("--configs", default="1x4,2x2,4x1", help="replicas x cpus per replica")
parser.add_argument(
"--variant",
dest="variants",
action="append",
help="NAME[:KEY=VALUE,...]; several variants alternate per repeat",
)
parser.add_argument("--api-memory-total", type=int, default=2048, help="MiB for all replicas")
parser.add_argument("--api-memory", type=int, default=0, help="MiB per replica")
parser.add_argument("--workers", type=int, default=0, help="gunicorn workers, default = cpus")
parser.add_argument("--pool", type=int, default=8, help="DB pool per worker")
parser.add_argument("--conn-per-worker", type=int, default=32, help="LB maxconn per worker")
parser.add_argument("--env", action="append", default=[], help="extra compose KEY=VALUE")
parser.add_argument("--no-pin", dest="pin", action="store_false", help="do not pin CPUs")
parser.add_argument("--rates", help="comma separated offered rates, rps")
parser.add_argument("--search", default="1.5:8000", help="factor:max, from start-per-cpu")
parser.add_argument("--start-per-cpu", type=float, default=100.0)
parser.add_argument("--repeats", type=int, default=3)
parser.add_argument("--first-repeat", type=int, default=1, help="number of the first repeat")
parser.add_argument("--warmup", type=int, default=10)
parser.add_argument("--duration", type=int, default=30)
parser.add_argument("--pause", type=float, default=5.0)
parser.add_argument("--slo-p95-ms", type=float, default=500.0)
parser.add_argument("--max-error-ratio", type=float, default=0.01)
parser.add_argument("--keep", action="store_true", help="leave the last layout running")
parser.add_argument("--max-other", type=float, default=100.0, help="foreign VM CPU, %% core")
parser.add_argument("--max-gap", type=float, default=10.0, help="sampling gap, s")
parser.add_argument("--max-overhead", type=float, default=90.0, help="k6 run beyond load, s")
parser.add_argument("--retries", type=int, default=3, help="reruns of a disturbed step")
parser.add_argument("--quiet-timeout", type=float, default=1200.0)
args = parser.parse_args()
args.variants = args.variants or ["base"]
if len(args.variants) > 1 and not args.rates:
parser.error("several variants need explicit --rates")
return args
def block(args, output, config, variant, repeats):
base, settings = layout(config, args, variant)
current, streak = apply(base, settings, args.base_env, args.out), 0
args.deployment = deployment(current)
for rate in rates(args, base):
results = []
for repeat in repeats:
for _ in range(args.retries + 1):
row = step(base, rate, repeat, args, current)
time.sleep(args.pause)
if not row["disturbed"]:
break
args.rejected.writerow(row)
wait_quiet(args)
else:
raise SystemExit(f"disturbed step exhausted retries: {row['raw_dir']}")
results.append(row)
output.writerows(results)
streak = streak + 1 if not args.rates and all(r["saturated"] for r in results) else 0
if streak == 2:
break
def series(args, output):
every = range(args.first_repeat, args.first_repeat + args.repeats)
configs = args.configs.split(",")
if not args.rates:
for config in configs:
block(args, output, config, args.variants[0], every)
return
for repeat in every:
for config in configs:
for variant in args.variants:
block(args, output, config, variant, [repeat])
@contextlib.contextmanager
def table(path):
fresh = not path.exists()
with path.open("a", newline="", buffering=1) as handle:
writer = csv.DictWriter(handle, fieldnames=FIELDS, extrasaction="ignore")
if fresh:
commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
handle.write(f"# date: {time.strftime('%FT%TZ', time.gmtime())} commit: {commit}\n")
writer.writeheader()
yield writer
def main():
args = arguments()
out = Path(args.out)
out.mkdir(parents=True, exist_ok=True)
args.project = json.loads(compose("config", "--format", "json"))["name"]
args.commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
args.base_env = dict(os.environ, **(PINNED if args.pin else {}))
try:
with table(out / "summary.csv") as writer, table(out / "disturbed.csv") as args.rejected:
series(args, writer)
finally:
if not args.keep:
compose("up", "-d", "--wait", "--wait-timeout", "180")
return 0
if __name__ == "__main__":
sys.exit(main())
+2
View File
@@ -21,4 +21,6 @@ put s3_root_user "root$(random | cut -c1-16)"
put s3_root_password "$(random)"
put s3_access_key_id "api$(random | cut -c1-16)"
put s3_secret_access_key "$(random)"
put s3_reconciler_access_key_id "rec$(random | cut -c1-16)"
put s3_reconciler_secret_access_key "$(random)"
[ -f .env ] || { cp .env.example .env; echo "создан .env из .env.example"; }
+6
View File
@@ -0,0 +1,6 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
size="select pg_database_size(current_database()), coalesce((select n_live_tup from pg_stat_user_tables where relname = 'messages'), 0)"
echo "$(date -u +%T) database bytes,messages: $(docker compose exec -T postgres psql -X -At -F, -U messenger -d messenger -c "$size")"
docker stats --no-stream --format '{{.Name}} cpu={{.CPUPerc}} mem={{.MemUsage}}' $(docker compose ps -q)
+1 -1
View File
@@ -6,4 +6,4 @@ ids=$(docker compose ps -q "${API_SERVICE:-api}")
[ -n "$ids" ] || { echo "no running ${API_SERVICE:-api} replicas" >&2; exit 3; }
target=$(docker inspect -f '{{.Id}} {{.Config.Hostname}}' $ids | awk -v t="$TARGET_INSTANCE_ID" '$2 == t {print $1}')
[ -n "$target" ] || { echo "no running ${API_SERVICE:-api} replica answers as $TARGET_INSTANCE_ID" >&2; exit 3; }
docker stop -t "${STOP_TIMEOUT:-6}" "$target" >/dev/null
docker stop -t "${STOP_TIMEOUT:-7}" "$target" >/dev/null
+9 -2
View File
@@ -77,7 +77,9 @@ def main():
args = p.parse_args()
out = ROOT / args.out
out.mkdir(parents=True, exist_ok=True)
commit = seed.run(["git", "rev-parse", "--short", "HEAD"]).strip()
if any(out.iterdir()):
p.error("output directory must be empty; choose a new --out")
commit = seed.run(["git", "rev-parse", "HEAD"]).strip()
threshold = os.environ.get("MALLOC_MMAP_THRESHOLD_", "131072")
lines = [
f"# {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} commit {commit}; api x2: 1 CPU,"
@@ -102,7 +104,12 @@ def main():
link=0,
ids_out=None,
)
report = {"upload_concurrency": slots, **seed.over_http(run)}
report = {
"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"commit": commit,
"upload_concurrency": slots,
**seed.over_http(run),
}
report["lb"] = balancer()
name = f"slots{slots}-{size >> 20}mib-c{level}.json"
(out / name).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")