Carry over the lab 3 fixes made after 61c0c28
Bring the system changes of lab 3 from 61c0c28..72a1e57 onto the lab 4 code, resolving each conflict by meaning and keeping everything lab 4 added: - a message is posted with one SQL statement under the chat row lock, and the attachment links go into the same statement; a foreign key violation on the attachment link still answers 404 and leaves nothing behind. Text and the attachment list are checked before the write, an outsider gets 404 for both, as before the carry over; - the request pipeline counts the body bytes actually received against the limit of the route, so a chunked body over 1 MiB gets 413 and the upload route keeps its MAX_UPLOAD_BYTES limit; - a request cancelled at shutdown gets 503 unavailable in the contract format, the worker waits a second less than the rest of the shutdown budget, SHUTDOWN_TIMEOUT_SECONDS becomes 6 s and the stop hooks 7 s; - readiness sits behind a Readiness port with the draining flag in shared infrastructure, and the drain window is logged through Logger; - logout revokes through SessionRevocations and withdraws the mark when PostgreSQL rejects the delete; an empty cursor is 400; - EventBus.project is gone, the sweep test subscribes and waits on a queue; domain tests check attachment_list instead of Chat.post; - make up waits for every replica in the balancer, BASE_URL follows the lb port, the image is tagged by the compose project, retry-check and faults-evidence are added, detection runs three times, failover-each and rolling-update record the stand usage, CI checks numeric uids. down also takes the tools and reconcile profiles of lab 4. Track 1 tooling of lab 3 (scaling.py, its samplers, tools/ and the CPU pinning of api, lb, valkey and k6) stays out; stand-usage.sh queries the database size itself instead of the track 1 sampler. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
7ac1a4a11b
commit
e927afdb09
@@ -8,7 +8,7 @@ replica() { docker compose ps -q "${API_SERVICE:-api}" | head -1; }
|
||||
detect() { python3 scripts/probe.py detect "$@"; }
|
||||
failed=0
|
||||
detect --max 1.5 --within-action --restore scripts/restore.sh --out "$out/sigterm.json" \
|
||||
-- docker stop -t "${STOP_TIMEOUT:-6}" "$(replica)" || failed=1
|
||||
-- docker stop -t "${STOP_TIMEOUT:-7}" "$(replica)" || failed=1
|
||||
detect --max 1.5 --restore scripts/restore.sh --out "$out/kill.json" \
|
||||
-- docker kill "$(replica)" || failed=1
|
||||
hung=$(replica)
|
||||
|
||||
@@ -11,14 +11,25 @@ cli "info persistence" | grep -E '^aof_enabled|^aof_last_write_status'
|
||||
echo "\$ valkey-cli dbsize; info stats | grep keyspace"
|
||||
cli dbsize
|
||||
cli "info stats" | grep -E '^keyspace_(hits|misses)'
|
||||
key=$(cli "--scan --pattern session:* --count 1000" | head -1)
|
||||
hash=${key#session:}
|
||||
echo "\$ сравнение срока ключа кэша и срока сессии для одного ключа (значение и токен не печатаются)"
|
||||
echo "valkey pexpiretime, ms: $(cli "pexpiretime $key")"
|
||||
echo "postgres expires_at, ms: $(psql "SELECT (extract(epoch FROM expires_at) * 1000)::bigint FROM sessions WHERE token_hash = decode('$hash', 'hex')")"
|
||||
echo "\$ количество отметок отзыва и закэшированных сессий"
|
||||
revoked=0; cached=0
|
||||
echo "\$ valkey-cli --scan --pattern 'session:*' и вид значения (хеш сокращён, значения не печатаются)"
|
||||
revoked=0; cached=0; key=""; stored=""
|
||||
for k in $(cli "--scan --pattern session:* --count 1000"); do
|
||||
if [ "$(cli "get $k")" = "-" ]; then revoked=$((revoked + 1)); else cached=$((cached + 1)); fi
|
||||
if [ "$(cli "get $k")" = "-" ]; then
|
||||
revoked=$((revoked + 1)); printf '%.16s… отметка отзыва "-"\n' "$k"; continue
|
||||
fi
|
||||
cached=$((cached + 1)); printf '%.16s… сессия\n' "$k"
|
||||
if [ -z "$key" ]; then
|
||||
stored=$(psql "SELECT (extract(epoch FROM expires_at) * 1000)::bigint FROM sessions WHERE token_hash = decode('${k#session:}', 'hex')")
|
||||
if [ -n "$stored" ]; then key=$k; fi
|
||||
fi
|
||||
done
|
||||
echo "tombstones: $revoked, cached sessions: $cached"
|
||||
if [ "$revoked" -eq 0 ] || [ -z "$key" ]; then
|
||||
echo "нужны закэшированная сессия из БД и отметка отзыва: сначала make test" >&2
|
||||
exit 1
|
||||
fi
|
||||
expires=$(cli "pexpiretime $key")
|
||||
echo "\$ срок ключа кэша против срока сессии для $(printf '%.16s' "$key")…"
|
||||
echo "valkey pexpiretime, ms: $expires"
|
||||
echo "postgres expires_at, ms: $stored"
|
||||
if [ "$expires" -le "$stored" ]; then echo "ключ не переживает сессию: да"; else echo "ключ не переживает сессию: НЕТ"; exit 1; fi
|
||||
|
||||
@@ -4,7 +4,7 @@ cd "$(dirname "$0")/.."
|
||||
psql() { docker compose exec -T postgres psql -U messenger -d messenger -c "$1"; }
|
||||
header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"; }
|
||||
counts="SELECT (SELECT count(*) FROM users) users, (SELECT count(*) FROM sessions) sessions, (SELECT count(*) FROM chats) chats, (SELECT count(*) FROM chat_members) members, (SELECT count(*) FROM messages) messages, (SELECT count(*) FROM idempotency_keys) keys, (SELECT count(*) FROM attachments) attachments, (SELECT count(*) FROM message_attachments) links, (SELECT count(*) FROM schema_migrations) versions"
|
||||
kdf="SELECT substring(password_hash from '^[$]argon2id[$]v=[0-9]+[$]m=[0-9]+,t=[0-9]+,p=[0-9]+[$]') AS kdf_prefix, count(*) AS users, min(length(password_hash)) AS hash_length FROM users GROUP BY 1"
|
||||
kdf="SELECT substring(password_hash from '^[$]argon2id[$]v=[0-9]+[$]m=[0-9]+,t=[0-9]+,p=[0-9]+[$]') AS kdf_prefix, username = 'seed_author' AS seed_stub, count(*) AS users, min(length(password_hash)) AS min_length, max(length(password_hash)) AS max_length FROM users GROUP BY 1, 2 ORDER BY 2"
|
||||
tokens="SELECT octet_length(token_hash) AS stored_bytes, count(*) AS sessions FROM sessions GROUP BY 1"
|
||||
{
|
||||
header
|
||||
@@ -38,7 +38,8 @@ tokens="SELECT octet_length(token_hash) AS stored_bytes, count(*) AS sessions FR
|
||||
echo "(пусто: секретов, .env, дампов и volumes в Git нет)"
|
||||
echo "\$ git check-ignore -v secrets/database_url secrets/postgres_password secrets/s3_root_password secrets/s3_secret_access_key .env backups/"
|
||||
git check-ignore -v secrets/database_url secrets/postgres_password secrets/s3_root_password secrets/s3_secret_access_key .env backups/
|
||||
echo "\$ docker volume ls --filter name=mtusi2026lab4"
|
||||
docker volume ls --filter name=mtusi2026lab4
|
||||
project=$(docker compose config --format json | python3 -c 'import json,sys; print(json.load(sys.stdin)["name"])')
|
||||
echo "\$ docker volume ls --filter label=com.docker.compose.project=$project"
|
||||
docker volume ls --filter "label=com.docker.compose.project=$project"
|
||||
} > evidence/secrets.txt
|
||||
echo "evidence/{schema,migrations,kdf,secrets}.txt обновлены"
|
||||
|
||||
@@ -10,7 +10,7 @@ failed=0
|
||||
set --
|
||||
for host in $hosts; do
|
||||
TARGET_INSTANCE_ID=$host python3 scripts/probe.py load --expect dip --out "$out/$host" \
|
||||
-- sh -c "scripts/stop-one.sh && sleep ${HOLD:-10} && scripts/restore.sh" || failed=1
|
||||
-- sh -c "scripts/stop-one.sh && docker compose ps -a --format '{{.ID}} {{.Service}} {{.Status}}' ${API_SERVICE:-api} && { scripts/stand-usage.sh & sleep ${HOLD:-10}; wait; } && scripts/restore.sh" || failed=1
|
||||
scripts/restore.sh
|
||||
set -- "$@" "$out/$host/result.json"
|
||||
done
|
||||
|
||||
@@ -4,6 +4,9 @@ cd "$(dirname "$0")/.."
|
||||
replicas() {
|
||||
docker inspect -f '{{.Config.Hostname}} {{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}' $(docker compose ps -aq "${API_SERVICE:-api}")
|
||||
}
|
||||
url=${1:-http://localhost:8080}
|
||||
shift=0
|
||||
while [ "$shift" -lt "${2:-0}" ]; do curl -fsS -o /dev/null "$url/health/live"; shift=$((shift + 1)); done
|
||||
tmp=evidence/failover.tmp
|
||||
since=$(date -u +%FT%TZ)
|
||||
{
|
||||
@@ -11,14 +14,14 @@ since=$(date -u +%FT%TZ)
|
||||
echo "== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt"
|
||||
replicas
|
||||
echo "\$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh"
|
||||
make --no-print-directory test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL="${1:-http://localhost:8080}"
|
||||
make --no-print-directory test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL="$url"
|
||||
echo "== after: api replicas"
|
||||
replicas
|
||||
} > "$tmp" 2>&1
|
||||
stopped=$(replicas | awk '$2 == "exited" {print $1}')
|
||||
{
|
||||
echo "== stopped replica: $stopped; shutdown: signal, drain window with readiness 503, close, exit"
|
||||
docker logs --since "$since" "$stopped" 2>&1 | grep -E 'Handling signal|Draining|"status":503|Shutting down|Waiting for conn|Finished server|service.stopped|SIGTERM' | tail -16
|
||||
docker logs --since "$since" "$stopped" 2>&1 | grep -E 'Handling signal|service.draining|"status":503|Shutting down|Waiting for conn|Finished server|service.stopped|SIGTERM' | tail -16
|
||||
echo "\$ scripts/restore.sh"
|
||||
scripts/restore.sh
|
||||
echo "== after restore"
|
||||
|
||||
+2
-1
@@ -140,7 +140,8 @@ def main():
|
||||
if not any(row[0] == "api" for row in containers):
|
||||
problems.append("no running api replicas to inspect")
|
||||
lines = [
|
||||
f"date: {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}",
|
||||
f"date: {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} "
|
||||
f"commit: {run('git', 'rev-parse', '--short', 'HEAD')[1]}",
|
||||
"$ docker compose --profile bench --profile tools --profile reconcile config --format json"
|
||||
" (declared)",
|
||||
*table(["service", "cpus/mem", "ports", "secrets", "volumes", "networks", "ro"], services),
|
||||
|
||||
@@ -12,10 +12,13 @@ import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
COUNTERS = ("wretr", "wredis", "econ", "eresp", "hrsp_5xx")
|
||||
PSQL = ["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger"]
|
||||
PSQL += ["-X", "-q", "-At"]
|
||||
|
||||
|
||||
@functools.cache
|
||||
@@ -207,6 +210,101 @@ def check(args):
|
||||
return 0 if passed else 1
|
||||
|
||||
|
||||
def output(*command):
|
||||
done = subprocess.run(command, cwd=str(ROOT), capture_output=True, text=True, check=False)
|
||||
return done.stdout + done.stderr
|
||||
|
||||
|
||||
def replicas():
|
||||
ids = output("docker", "compose", "ps", "-q", os.environ.get("API_SERVICE", "api")).split()
|
||||
form = "{{.Id}} {{.Config.Hostname}} {{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}"
|
||||
return [line.split() for line in output("docker", "inspect", "-f", form, *ids).splitlines()]
|
||||
|
||||
|
||||
def answered(container, request_id):
|
||||
for line in output("docker", "logs", container).splitlines():
|
||||
if request_id in line and "request.handled" in line:
|
||||
return json.loads(line).get("status")
|
||||
return None
|
||||
|
||||
|
||||
def lost(alice, bob, chat, kill, args):
|
||||
key, text = uuid.uuid4().hex, "lost " + uuid.uuid4().hex
|
||||
request_id = "lost-" + key[:12]
|
||||
lock = subprocess.Popen(
|
||||
PSQL, cwd=str(ROOT), stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True
|
||||
)
|
||||
lock.stdin.write(
|
||||
f"BEGIN;\nSELECT 'locked' FROM chats WHERE id = '{uuid.UUID(chat)}' FOR UPDATE;\n"
|
||||
f"SELECT pg_sleep({args.hold});\nCOMMIT;\n"
|
||||
)
|
||||
lock.stdin.flush()
|
||||
lock.stdout.readline()
|
||||
sent = time.time()
|
||||
headers = dict(alice["auth"], **{"Idempotency-Key": key, "X-Request-Id": request_id})
|
||||
path = f"/api/v1/chats/{chat}/messages"
|
||||
first = call("POST", path, {"text": text}, headers, args.timeout)[0]
|
||||
sql = "SELECT host(client_addr) FROM pg_stat_activity WHERE wait_event_type = 'Lock'"
|
||||
waiting = set(output(*PSQL, "-c", sql).split())
|
||||
holder = next((r for r in replicas() if waiting & set(r[2:])), [None, None])
|
||||
if kill and holder[0]:
|
||||
subprocess.call(["docker", "kill", holder[0]], stdout=subprocess.DEVNULL)
|
||||
lock.stdin.close()
|
||||
lock.wait()
|
||||
retried = time.time()
|
||||
attempts, (status, instance, body) = 1, send(alice, chat, text, key)
|
||||
while status != 201 and attempts < 20:
|
||||
time.sleep(0.2)
|
||||
attempts, (status, instance, body) = attempts + 1, send(alice, chat, text, key)
|
||||
page = call("GET", path + "?limit=100", headers=bob["auth"])[2] or {"items": []}
|
||||
copies = [m for m in page["items"] if m["text"] == text]
|
||||
stamp = copies[0]["created_at"].replace("Z", "+00:00") if copies else None
|
||||
report = {
|
||||
"variant": "kill" if kill else "timeout",
|
||||
"lost_attempt": {
|
||||
"status": first,
|
||||
"client_timeout_s": args.timeout,
|
||||
"chat_lock_held_s": args.hold,
|
||||
"replica": holder[1],
|
||||
"replica_killed": bool(kill and holder[0]),
|
||||
"replica_answered": answered(holder[0], request_id) if holder[0] else None,
|
||||
},
|
||||
"retry": {
|
||||
"after_s": round(retried - sent, 3),
|
||||
"status": status,
|
||||
"attempts": attempts,
|
||||
"replica": instance,
|
||||
},
|
||||
"copies_in_history": len(copies),
|
||||
"same_message": bool(copies) and copies[0]["id"] == (body or {}).get("id"),
|
||||
"written_by": None
|
||||
if stamp is None
|
||||
else ("lost attempt" if datetime.fromisoformat(stamp).timestamp() < retried else "retry"),
|
||||
}
|
||||
if kill and holder[0]:
|
||||
subprocess.call(
|
||||
[str(ROOT / "scripts/restore.sh")], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
|
||||
)
|
||||
report["pass"] = (
|
||||
first == 0
|
||||
and holder[0] is not None
|
||||
and status == 201
|
||||
and report["copies_in_history"] == 1
|
||||
and report["same_message"]
|
||||
and (kill or report["lost_attempt"]["replica_answered"] == 201)
|
||||
)
|
||||
return report
|
||||
|
||||
|
||||
def retry(args):
|
||||
alice, bob = actor("alice"), actor("bob")
|
||||
chat = chat_of(alice, bob)
|
||||
variants = [lost(alice, bob, chat, kill, args) for kill in (False, True)]
|
||||
passed = all(v["pass"] for v in variants)
|
||||
emit({"variants": variants, "pass": passed}, args.out)
|
||||
return 0 if passed else 1
|
||||
|
||||
|
||||
def record(stop, alice, chat):
|
||||
probes, trace = [], []
|
||||
|
||||
@@ -413,6 +511,10 @@ def main():
|
||||
under.add_argument("--max-failed-ratio", type=float, default=0.01)
|
||||
under.add_argument("--max-recovery", type=float, default=10.0)
|
||||
under.add_argument("command", nargs=argparse.REMAINDER)
|
||||
again = sub.add_parser("retry", help="retry a write whose answer was lost, same key")
|
||||
again.add_argument("--hold", type=float, default=1.5, help="chat row lock, s (< lock_timeout)")
|
||||
again.add_argument("--timeout", type=float, default=0.5, help="first attempt timeout, s")
|
||||
again.add_argument("--out", help="JSON file, written only after a complete report")
|
||||
watch = sub.add_parser("detect", help="time until the LB takes replicas out of rotation")
|
||||
watch.add_argument("--max", type=float, required=True, help="declared worst detection, s")
|
||||
watch.add_argument("--slack", type=float, default=0.25, help="docker CLI and polling, s")
|
||||
@@ -426,6 +528,8 @@ def main():
|
||||
args = parser.parse_args()
|
||||
if args.mode == "check":
|
||||
return check(args)
|
||||
if args.mode == "retry":
|
||||
return retry(args)
|
||||
args.command = [c for c in args.command if c != "--"]
|
||||
if not args.command:
|
||||
parser.error(f"{args.mode} needs an action command after --")
|
||||
|
||||
@@ -2,7 +2,8 @@
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
service=${API_SERVICE:-api}
|
||||
show() { docker inspect -f '{{.Config.Hostname}} image={{slice .Image 7 19}}' "$1"; }
|
||||
show() { docker inspect -f '{{.Config.Hostname}} image={{.Config.Image}}@{{slice .Image 7 19}}' "$1"; }
|
||||
scripts/stand-usage.sh &
|
||||
echo "$(date -u +%T) migrate once before the first replacement"
|
||||
docker compose run --rm --no-deps -T migrate 2>&1
|
||||
old=$(docker compose ps -q "$service")
|
||||
@@ -13,8 +14,9 @@ for id in $old; do
|
||||
scripts/lb-wait.sh $((count + 1))
|
||||
new=$(docker compose ps -q "$service" | grep -v -F "$running")
|
||||
echo "$(date -u +%T) replace $(show "$id") -> $(show "$new")"
|
||||
docker stop -t "${STOP_TIMEOUT:-6}" "$id" >/dev/null
|
||||
docker stop -t "${STOP_TIMEOUT:-7}" "$id" >/dev/null
|
||||
docker rm "$id" >/dev/null
|
||||
done
|
||||
scripts/lb-wait.sh "$count"
|
||||
wait
|
||||
echo "$(date -u +%T) done: $count replicas replaced"
|
||||
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
size="select pg_database_size(current_database()), coalesce((select n_live_tup from pg_stat_user_tables where relname = 'messages'), 0)"
|
||||
echo "$(date -u +%T) database bytes,messages: $(docker compose exec -T postgres psql -X -At -F, -U messenger -d messenger -c "$size")"
|
||||
docker stats --no-stream --format '{{.Name}} cpu={{.CPUPerc}} mem={{.MemUsage}}' $(docker compose ps -q)
|
||||
+1
-1
@@ -6,4 +6,4 @@ ids=$(docker compose ps -q "${API_SERVICE:-api}")
|
||||
[ -n "$ids" ] || { echo "no running ${API_SERVICE:-api} replicas" >&2; exit 3; }
|
||||
target=$(docker inspect -f '{{.Id}} {{.Config.Hostname}}' $ids | awk -v t="$TARGET_INSTANCE_ID" '$2 == t {print $1}')
|
||||
[ -n "$target" ] || { echo "no running ${API_SERVICE:-api} replica answers as $TARGET_INSTANCE_ID" >&2; exit 3; }
|
||||
docker stop -t "${STOP_TIMEOUT:-6}" "$target" >/dev/null
|
||||
docker stop -t "${STOP_TIMEOUT:-7}" "$target" >/dev/null
|
||||
|
||||
Reference in New Issue
Block a user