Carry over the lab 3 fixes made after 61c0c28

Bring the system changes of lab 3 from 61c0c28..72a1e57 onto the lab 4
code, resolving each conflict by meaning and keeping everything lab 4
added:

- a message is posted with one SQL statement under the chat row lock,
  and the attachment links go into the same statement; a foreign key
  violation on the attachment link still answers 404 and leaves nothing
  behind. Text and the attachment list are checked before the write, an
  outsider gets 404 for both, as before the carry over;
- the request pipeline counts the body bytes actually received against
  the limit of the route, so a chunked body over 1 MiB gets 413 and the
  upload route keeps its MAX_UPLOAD_BYTES limit;
- a request cancelled at shutdown gets 503 unavailable in the contract
  format, the worker waits a second less than the rest of the shutdown
  budget, SHUTDOWN_TIMEOUT_SECONDS becomes 6 s and the stop hooks 7 s;
- readiness sits behind a Readiness port with the draining flag in
  shared infrastructure, and the drain window is logged through Logger;
- logout revokes through SessionRevocations and withdraws the mark when
  PostgreSQL rejects the delete; an empty cursor is 400;
- EventBus.project is gone, the sweep test subscribes and waits on a
  queue; domain tests check attachment_list instead of Chat.post;
- make up waits for every replica in the balancer, BASE_URL follows the
  lb port, the image is tagged by the compose project, retry-check and
  faults-evidence are added, detection runs three times, failover-each
  and rolling-update record the stand usage, CI checks numeric uids.
  down also takes the tools and reconcile profiles of lab 4.

Track 1 tooling of lab 3 (scaling.py, its samplers, tools/ and the CPU
pinning of api, lb, valkey and k6) stays out; stand-usage.sh queries the
database size itself instead of the track 1 sampler.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27
This commit is contained in:
Artyom
2026-10-30 06:25:23 +03:00
co-authored by Claude Opus 5.5
parent 7ac1a4a11b
commit e927afdb09
56 changed files with 932 additions and 397 deletions
+1 -1
View File
@@ -8,7 +8,7 @@ replica() { docker compose ps -q "${API_SERVICE:-api}" | head -1; }
detect() { python3 scripts/probe.py detect "$@"; }
failed=0
detect --max 1.5 --within-action --restore scripts/restore.sh --out "$out/sigterm.json" \
-- docker stop -t "${STOP_TIMEOUT:-6}" "$(replica)" || failed=1
-- docker stop -t "${STOP_TIMEOUT:-7}" "$(replica)" || failed=1
detect --max 1.5 --restore scripts/restore.sh --out "$out/kill.json" \
-- docker kill "$(replica)" || failed=1
hung=$(replica)
+19 -8
View File
@@ -11,14 +11,25 @@ cli "info persistence" | grep -E '^aof_enabled|^aof_last_write_status'
echo "\$ valkey-cli dbsize; info stats | grep keyspace"
cli dbsize
cli "info stats" | grep -E '^keyspace_(hits|misses)'
key=$(cli "--scan --pattern session:* --count 1000" | head -1)
hash=${key#session:}
echo "\$ сравнение срока ключа кэша и срока сессии для одного ключа (значение и токен не печатаются)"
echo "valkey pexpiretime, ms: $(cli "pexpiretime $key")"
echo "postgres expires_at, ms: $(psql "SELECT (extract(epoch FROM expires_at) * 1000)::bigint FROM sessions WHERE token_hash = decode('$hash', 'hex')")"
echo "\$ количество отметок отзыва и закэшированных сессий"
revoked=0; cached=0
echo "\$ valkey-cli --scan --pattern 'session:*' и вид значения (хеш сокращён, значения не печатаются)"
revoked=0; cached=0; key=""; stored=""
for k in $(cli "--scan --pattern session:* --count 1000"); do
if [ "$(cli "get $k")" = "-" ]; then revoked=$((revoked + 1)); else cached=$((cached + 1)); fi
if [ "$(cli "get $k")" = "-" ]; then
revoked=$((revoked + 1)); printf '%.16s… отметка отзыва "-"\n' "$k"; continue
fi
cached=$((cached + 1)); printf '%.16s… сессия\n' "$k"
if [ -z "$key" ]; then
stored=$(psql "SELECT (extract(epoch FROM expires_at) * 1000)::bigint FROM sessions WHERE token_hash = decode('${k#session:}', 'hex')")
if [ -n "$stored" ]; then key=$k; fi
fi
done
echo "tombstones: $revoked, cached sessions: $cached"
if [ "$revoked" -eq 0 ] || [ -z "$key" ]; then
echo "нужны закэшированная сессия из БД и отметка отзыва: сначала make test" >&2
exit 1
fi
expires=$(cli "pexpiretime $key")
echo "\$ срок ключа кэша против срока сессии для $(printf '%.16s' "$key")…"
echo "valkey pexpiretime, ms: $expires"
echo "postgres expires_at, ms: $stored"
if [ "$expires" -le "$stored" ]; then echo "ключ не переживает сессию: да"; else echo "ключ не переживает сессию: НЕТ"; exit 1; fi
+4 -3
View File
@@ -4,7 +4,7 @@ cd "$(dirname "$0")/.."
psql() { docker compose exec -T postgres psql -U messenger -d messenger -c "$1"; }
header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"; }
counts="SELECT (SELECT count(*) FROM users) users, (SELECT count(*) FROM sessions) sessions, (SELECT count(*) FROM chats) chats, (SELECT count(*) FROM chat_members) members, (SELECT count(*) FROM messages) messages, (SELECT count(*) FROM idempotency_keys) keys, (SELECT count(*) FROM attachments) attachments, (SELECT count(*) FROM message_attachments) links, (SELECT count(*) FROM schema_migrations) versions"
kdf="SELECT substring(password_hash from '^[$]argon2id[$]v=[0-9]+[$]m=[0-9]+,t=[0-9]+,p=[0-9]+[$]') AS kdf_prefix, count(*) AS users, min(length(password_hash)) AS hash_length FROM users GROUP BY 1"
kdf="SELECT substring(password_hash from '^[$]argon2id[$]v=[0-9]+[$]m=[0-9]+,t=[0-9]+,p=[0-9]+[$]') AS kdf_prefix, username = 'seed_author' AS seed_stub, count(*) AS users, min(length(password_hash)) AS min_length, max(length(password_hash)) AS max_length FROM users GROUP BY 1, 2 ORDER BY 2"
tokens="SELECT octet_length(token_hash) AS stored_bytes, count(*) AS sessions FROM sessions GROUP BY 1"
{
header
@@ -38,7 +38,8 @@ tokens="SELECT octet_length(token_hash) AS stored_bytes, count(*) AS sessions FR
echo "(пусто: секретов, .env, дампов и volumes в Git нет)"
echo "\$ git check-ignore -v secrets/database_url secrets/postgres_password secrets/s3_root_password secrets/s3_secret_access_key .env backups/"
git check-ignore -v secrets/database_url secrets/postgres_password secrets/s3_root_password secrets/s3_secret_access_key .env backups/
echo "\$ docker volume ls --filter name=mtusi2026lab4"
docker volume ls --filter name=mtusi2026lab4
project=$(docker compose config --format json | python3 -c 'import json,sys; print(json.load(sys.stdin)["name"])')
echo "\$ docker volume ls --filter label=com.docker.compose.project=$project"
docker volume ls --filter "label=com.docker.compose.project=$project"
} > evidence/secrets.txt
echo "evidence/{schema,migrations,kdf,secrets}.txt обновлены"
+1 -1
View File
@@ -10,7 +10,7 @@ failed=0
set --
for host in $hosts; do
TARGET_INSTANCE_ID=$host python3 scripts/probe.py load --expect dip --out "$out/$host" \
-- sh -c "scripts/stop-one.sh && sleep ${HOLD:-10} && scripts/restore.sh" || failed=1
-- sh -c "scripts/stop-one.sh && docker compose ps -a --format '{{.ID}} {{.Service}} {{.Status}}' ${API_SERVICE:-api} && { scripts/stand-usage.sh & sleep ${HOLD:-10}; wait; } && scripts/restore.sh" || failed=1
scripts/restore.sh
set -- "$@" "$out/$host/result.json"
done
+5 -2
View File
@@ -4,6 +4,9 @@ cd "$(dirname "$0")/.."
replicas() {
docker inspect -f '{{.Config.Hostname}} {{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}' $(docker compose ps -aq "${API_SERVICE:-api}")
}
url=${1:-http://localhost:8080}
shift=0
while [ "$shift" -lt "${2:-0}" ]; do curl -fsS -o /dev/null "$url/health/live"; shift=$((shift + 1)); done
tmp=evidence/failover.tmp
since=$(date -u +%FT%TZ)
{
@@ -11,14 +14,14 @@ since=$(date -u +%FT%TZ)
echo "== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt"
replicas
echo "\$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh"
make --no-print-directory test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL="${1:-http://localhost:8080}"
make --no-print-directory test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL="$url"
echo "== after: api replicas"
replicas
} > "$tmp" 2>&1
stopped=$(replicas | awk '$2 == "exited" {print $1}')
{
echo "== stopped replica: $stopped; shutdown: signal, drain window with readiness 503, close, exit"
docker logs --since "$since" "$stopped" 2>&1 | grep -E 'Handling signal|Draining|"status":503|Shutting down|Waiting for conn|Finished server|service.stopped|SIGTERM' | tail -16
docker logs --since "$since" "$stopped" 2>&1 | grep -E 'Handling signal|service.draining|"status":503|Shutting down|Waiting for conn|Finished server|service.stopped|SIGTERM' | tail -16
echo "\$ scripts/restore.sh"
scripts/restore.sh
echo "== after restore"
+2 -1
View File
@@ -140,7 +140,8 @@ def main():
if not any(row[0] == "api" for row in containers):
problems.append("no running api replicas to inspect")
lines = [
f"date: {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}",
f"date: {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} "
f"commit: {run('git', 'rev-parse', '--short', 'HEAD')[1]}",
"$ docker compose --profile bench --profile tools --profile reconcile config --format json"
" (declared)",
*table(["service", "cpus/mem", "ports", "secrets", "volumes", "networks", "ro"], services),
+104
View File
@@ -12,10 +12,13 @@ import time
import urllib.error
import urllib.request
import uuid
from datetime import datetime
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
COUNTERS = ("wretr", "wredis", "econ", "eresp", "hrsp_5xx")
PSQL = ["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger"]
PSQL += ["-X", "-q", "-At"]
@functools.cache
@@ -207,6 +210,101 @@ def check(args):
return 0 if passed else 1
def output(*command):
done = subprocess.run(command, cwd=str(ROOT), capture_output=True, text=True, check=False)
return done.stdout + done.stderr
def replicas():
ids = output("docker", "compose", "ps", "-q", os.environ.get("API_SERVICE", "api")).split()
form = "{{.Id}} {{.Config.Hostname}} {{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}"
return [line.split() for line in output("docker", "inspect", "-f", form, *ids).splitlines()]
def answered(container, request_id):
for line in output("docker", "logs", container).splitlines():
if request_id in line and "request.handled" in line:
return json.loads(line).get("status")
return None
def lost(alice, bob, chat, kill, args):
key, text = uuid.uuid4().hex, "lost " + uuid.uuid4().hex
request_id = "lost-" + key[:12]
lock = subprocess.Popen(
PSQL, cwd=str(ROOT), stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True
)
lock.stdin.write(
f"BEGIN;\nSELECT 'locked' FROM chats WHERE id = '{uuid.UUID(chat)}' FOR UPDATE;\n"
f"SELECT pg_sleep({args.hold});\nCOMMIT;\n"
)
lock.stdin.flush()
lock.stdout.readline()
sent = time.time()
headers = dict(alice["auth"], **{"Idempotency-Key": key, "X-Request-Id": request_id})
path = f"/api/v1/chats/{chat}/messages"
first = call("POST", path, {"text": text}, headers, args.timeout)[0]
sql = "SELECT host(client_addr) FROM pg_stat_activity WHERE wait_event_type = 'Lock'"
waiting = set(output(*PSQL, "-c", sql).split())
holder = next((r for r in replicas() if waiting & set(r[2:])), [None, None])
if kill and holder[0]:
subprocess.call(["docker", "kill", holder[0]], stdout=subprocess.DEVNULL)
lock.stdin.close()
lock.wait()
retried = time.time()
attempts, (status, instance, body) = 1, send(alice, chat, text, key)
while status != 201 and attempts < 20:
time.sleep(0.2)
attempts, (status, instance, body) = attempts + 1, send(alice, chat, text, key)
page = call("GET", path + "?limit=100", headers=bob["auth"])[2] or {"items": []}
copies = [m for m in page["items"] if m["text"] == text]
stamp = copies[0]["created_at"].replace("Z", "+00:00") if copies else None
report = {
"variant": "kill" if kill else "timeout",
"lost_attempt": {
"status": first,
"client_timeout_s": args.timeout,
"chat_lock_held_s": args.hold,
"replica": holder[1],
"replica_killed": bool(kill and holder[0]),
"replica_answered": answered(holder[0], request_id) if holder[0] else None,
},
"retry": {
"after_s": round(retried - sent, 3),
"status": status,
"attempts": attempts,
"replica": instance,
},
"copies_in_history": len(copies),
"same_message": bool(copies) and copies[0]["id"] == (body or {}).get("id"),
"written_by": None
if stamp is None
else ("lost attempt" if datetime.fromisoformat(stamp).timestamp() < retried else "retry"),
}
if kill and holder[0]:
subprocess.call(
[str(ROOT / "scripts/restore.sh")], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
)
report["pass"] = (
first == 0
and holder[0] is not None
and status == 201
and report["copies_in_history"] == 1
and report["same_message"]
and (kill or report["lost_attempt"]["replica_answered"] == 201)
)
return report
def retry(args):
alice, bob = actor("alice"), actor("bob")
chat = chat_of(alice, bob)
variants = [lost(alice, bob, chat, kill, args) for kill in (False, True)]
passed = all(v["pass"] for v in variants)
emit({"variants": variants, "pass": passed}, args.out)
return 0 if passed else 1
def record(stop, alice, chat):
probes, trace = [], []
@@ -413,6 +511,10 @@ def main():
under.add_argument("--max-failed-ratio", type=float, default=0.01)
under.add_argument("--max-recovery", type=float, default=10.0)
under.add_argument("command", nargs=argparse.REMAINDER)
again = sub.add_parser("retry", help="retry a write whose answer was lost, same key")
again.add_argument("--hold", type=float, default=1.5, help="chat row lock, s (< lock_timeout)")
again.add_argument("--timeout", type=float, default=0.5, help="first attempt timeout, s")
again.add_argument("--out", help="JSON file, written only after a complete report")
watch = sub.add_parser("detect", help="time until the LB takes replicas out of rotation")
watch.add_argument("--max", type=float, required=True, help="declared worst detection, s")
watch.add_argument("--slack", type=float, default=0.25, help="docker CLI and polling, s")
@@ -426,6 +528,8 @@ def main():
args = parser.parse_args()
if args.mode == "check":
return check(args)
if args.mode == "retry":
return retry(args)
args.command = [c for c in args.command if c != "--"]
if not args.command:
parser.error(f"{args.mode} needs an action command after --")
+4 -2
View File
@@ -2,7 +2,8 @@
set -eu
cd "$(dirname "$0")/.."
service=${API_SERVICE:-api}
show() { docker inspect -f '{{.Config.Hostname}} image={{slice .Image 7 19}}' "$1"; }
show() { docker inspect -f '{{.Config.Hostname}} image={{.Config.Image}}@{{slice .Image 7 19}}' "$1"; }
scripts/stand-usage.sh &
echo "$(date -u +%T) migrate once before the first replacement"
docker compose run --rm --no-deps -T migrate 2>&1
old=$(docker compose ps -q "$service")
@@ -13,8 +14,9 @@ for id in $old; do
scripts/lb-wait.sh $((count + 1))
new=$(docker compose ps -q "$service" | grep -v -F "$running")
echo "$(date -u +%T) replace $(show "$id") -> $(show "$new")"
docker stop -t "${STOP_TIMEOUT:-6}" "$id" >/dev/null
docker stop -t "${STOP_TIMEOUT:-7}" "$id" >/dev/null
docker rm "$id" >/dev/null
done
scripts/lb-wait.sh "$count"
wait
echo "$(date -u +%T) done: $count replicas replaced"
+6
View File
@@ -0,0 +1,6 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
size="select pg_database_size(current_database()), coalesce((select n_live_tup from pg_stat_user_tables where relname = 'messages'), 0)"
echo "$(date -u +%T) database bytes,messages: $(docker compose exec -T postgres psql -X -At -F, -U messenger -d messenger -c "$size")"
docker stats --no-stream --format '{{.Name}} cpu={{.CPUPerc}} mem={{.MemUsage}}' $(docker compose ps -q)
+1 -1
View File
@@ -6,4 +6,4 @@ ids=$(docker compose ps -q "${API_SERVICE:-api}")
[ -n "$ids" ] || { echo "no running ${API_SERVICE:-api} replicas" >&2; exit 3; }
target=$(docker inspect -f '{{.Id}} {{.Config.Hostname}}' $ids | awk -v t="$TARGET_INSTANCE_ID" '$2 == t {print $1}')
[ -n "$target" ] || { echo "no running ${API_SERVICE:-api} replica answers as $TARGET_INSTANCE_ID" >&2; exit 3; }
docker stop -t "${STOP_TIMEOUT:-6}" "$target" >/dev/null
docker stop -t "${STOP_TIMEOUT:-7}" "$target" >/dev/null