forked from mtusi2026/mtusi2026lab4
Carry over the lab 3 fixes made after 61c0c28
Bring the system changes of lab 3 from 61c0c28..72a1e57 onto the lab 4 code, resolving each conflict by meaning and keeping everything lab 4 added: - a message is posted with one SQL statement under the chat row lock, and the attachment links go into the same statement; a foreign key violation on the attachment link still answers 404 and leaves nothing behind. Text and the attachment list are checked before the write, an outsider gets 404 for both, as before the carry over; - the request pipeline counts the body bytes actually received against the limit of the route, so a chunked body over 1 MiB gets 413 and the upload route keeps its MAX_UPLOAD_BYTES limit; - a request cancelled at shutdown gets 503 unavailable in the contract format, the worker waits a second less than the rest of the shutdown budget, SHUTDOWN_TIMEOUT_SECONDS becomes 6 s and the stop hooks 7 s; - readiness sits behind a Readiness port with the draining flag in shared infrastructure, and the drain window is logged through Logger; - logout revokes through SessionRevocations and withdraws the mark when PostgreSQL rejects the delete; an empty cursor is 400; - EventBus.project is gone, the sweep test subscribes and waits on a queue; domain tests check attachment_list instead of Chat.post; - make up waits for every replica in the balancer, BASE_URL follows the lb port, the image is tagged by the compose project, retry-check and faults-evidence are added, detection runs three times, failover-each and rolling-update record the stand usage, CI checks numeric uids. down also takes the tools and reconcile profiles of lab 4. Track 1 tooling of lab 3 (scaling.py, its samplers, tools/ and the CPU pinning of api, lb, valkey and k6) stays out; stand-usage.sh queries the database size itself instead of the track 1 sampler. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
7ac1a4a11b
commit
e927afdb09
@@ -12,10 +12,13 @@ import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
COUNTERS = ("wretr", "wredis", "econ", "eresp", "hrsp_5xx")
|
||||
PSQL = ["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger"]
|
||||
PSQL += ["-X", "-q", "-At"]
|
||||
|
||||
|
||||
@functools.cache
|
||||
@@ -207,6 +210,101 @@ def check(args):
|
||||
return 0 if passed else 1
|
||||
|
||||
|
||||
def output(*command):
|
||||
done = subprocess.run(command, cwd=str(ROOT), capture_output=True, text=True, check=False)
|
||||
return done.stdout + done.stderr
|
||||
|
||||
|
||||
def replicas():
|
||||
ids = output("docker", "compose", "ps", "-q", os.environ.get("API_SERVICE", "api")).split()
|
||||
form = "{{.Id}} {{.Config.Hostname}} {{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}"
|
||||
return [line.split() for line in output("docker", "inspect", "-f", form, *ids).splitlines()]
|
||||
|
||||
|
||||
def answered(container, request_id):
|
||||
for line in output("docker", "logs", container).splitlines():
|
||||
if request_id in line and "request.handled" in line:
|
||||
return json.loads(line).get("status")
|
||||
return None
|
||||
|
||||
|
||||
def lost(alice, bob, chat, kill, args):
|
||||
key, text = uuid.uuid4().hex, "lost " + uuid.uuid4().hex
|
||||
request_id = "lost-" + key[:12]
|
||||
lock = subprocess.Popen(
|
||||
PSQL, cwd=str(ROOT), stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True
|
||||
)
|
||||
lock.stdin.write(
|
||||
f"BEGIN;\nSELECT 'locked' FROM chats WHERE id = '{uuid.UUID(chat)}' FOR UPDATE;\n"
|
||||
f"SELECT pg_sleep({args.hold});\nCOMMIT;\n"
|
||||
)
|
||||
lock.stdin.flush()
|
||||
lock.stdout.readline()
|
||||
sent = time.time()
|
||||
headers = dict(alice["auth"], **{"Idempotency-Key": key, "X-Request-Id": request_id})
|
||||
path = f"/api/v1/chats/{chat}/messages"
|
||||
first = call("POST", path, {"text": text}, headers, args.timeout)[0]
|
||||
sql = "SELECT host(client_addr) FROM pg_stat_activity WHERE wait_event_type = 'Lock'"
|
||||
waiting = set(output(*PSQL, "-c", sql).split())
|
||||
holder = next((r for r in replicas() if waiting & set(r[2:])), [None, None])
|
||||
if kill and holder[0]:
|
||||
subprocess.call(["docker", "kill", holder[0]], stdout=subprocess.DEVNULL)
|
||||
lock.stdin.close()
|
||||
lock.wait()
|
||||
retried = time.time()
|
||||
attempts, (status, instance, body) = 1, send(alice, chat, text, key)
|
||||
while status != 201 and attempts < 20:
|
||||
time.sleep(0.2)
|
||||
attempts, (status, instance, body) = attempts + 1, send(alice, chat, text, key)
|
||||
page = call("GET", path + "?limit=100", headers=bob["auth"])[2] or {"items": []}
|
||||
copies = [m for m in page["items"] if m["text"] == text]
|
||||
stamp = copies[0]["created_at"].replace("Z", "+00:00") if copies else None
|
||||
report = {
|
||||
"variant": "kill" if kill else "timeout",
|
||||
"lost_attempt": {
|
||||
"status": first,
|
||||
"client_timeout_s": args.timeout,
|
||||
"chat_lock_held_s": args.hold,
|
||||
"replica": holder[1],
|
||||
"replica_killed": bool(kill and holder[0]),
|
||||
"replica_answered": answered(holder[0], request_id) if holder[0] else None,
|
||||
},
|
||||
"retry": {
|
||||
"after_s": round(retried - sent, 3),
|
||||
"status": status,
|
||||
"attempts": attempts,
|
||||
"replica": instance,
|
||||
},
|
||||
"copies_in_history": len(copies),
|
||||
"same_message": bool(copies) and copies[0]["id"] == (body or {}).get("id"),
|
||||
"written_by": None
|
||||
if stamp is None
|
||||
else ("lost attempt" if datetime.fromisoformat(stamp).timestamp() < retried else "retry"),
|
||||
}
|
||||
if kill and holder[0]:
|
||||
subprocess.call(
|
||||
[str(ROOT / "scripts/restore.sh")], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
|
||||
)
|
||||
report["pass"] = (
|
||||
first == 0
|
||||
and holder[0] is not None
|
||||
and status == 201
|
||||
and report["copies_in_history"] == 1
|
||||
and report["same_message"]
|
||||
and (kill or report["lost_attempt"]["replica_answered"] == 201)
|
||||
)
|
||||
return report
|
||||
|
||||
|
||||
def retry(args):
|
||||
alice, bob = actor("alice"), actor("bob")
|
||||
chat = chat_of(alice, bob)
|
||||
variants = [lost(alice, bob, chat, kill, args) for kill in (False, True)]
|
||||
passed = all(v["pass"] for v in variants)
|
||||
emit({"variants": variants, "pass": passed}, args.out)
|
||||
return 0 if passed else 1
|
||||
|
||||
|
||||
def record(stop, alice, chat):
|
||||
probes, trace = [], []
|
||||
|
||||
@@ -413,6 +511,10 @@ def main():
|
||||
under.add_argument("--max-failed-ratio", type=float, default=0.01)
|
||||
under.add_argument("--max-recovery", type=float, default=10.0)
|
||||
under.add_argument("command", nargs=argparse.REMAINDER)
|
||||
again = sub.add_parser("retry", help="retry a write whose answer was lost, same key")
|
||||
again.add_argument("--hold", type=float, default=1.5, help="chat row lock, s (< lock_timeout)")
|
||||
again.add_argument("--timeout", type=float, default=0.5, help="first attempt timeout, s")
|
||||
again.add_argument("--out", help="JSON file, written only after a complete report")
|
||||
watch = sub.add_parser("detect", help="time until the LB takes replicas out of rotation")
|
||||
watch.add_argument("--max", type=float, required=True, help="declared worst detection, s")
|
||||
watch.add_argument("--slack", type=float, default=0.25, help="docker CLI and polling, s")
|
||||
@@ -426,6 +528,8 @@ def main():
|
||||
args = parser.parse_args()
|
||||
if args.mode == "check":
|
||||
return check(args)
|
||||
if args.mode == "retry":
|
||||
return retry(args)
|
||||
args.command = [c for c in args.command if c != "--"]
|
||||
if not args.command:
|
||||
parser.error(f"{args.mode} needs an action command after --")
|
||||
|
||||
Reference in New Issue
Block a user