lab 5 + kafka message broker

This commit is contained in:
2026-11-07 19:45:45 +03:00
parent 276dbdad8a
commit d26ef6e660
54 changed files with 7203 additions and 4817 deletions
+37
View File
@@ -0,0 +1,37 @@
#!/usr/bin/env python3
"""Apply SQL migrations once, before any API replica is started."""
from pathlib import Path
import os
import time
import psycopg
ROOT = Path(__file__).resolve().parents[1]
CONNINFO = os.getenv("DATABASE_URL", "")
def main():
last_error = None
for _ in range(30):
try:
with psycopg.connect(CONNINFO, connect_timeout=3) as conn:
conn.execute("CREATE TABLE IF NOT EXISTS schema_migrations (version TEXT PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT now())")
conn.commit()
for path in sorted((ROOT / "migrations").glob("*.sql")):
version = path.name
if conn.execute("SELECT 1 FROM schema_migrations WHERE version=%s", (version,)).fetchone():
continue
sql = path.read_text(encoding="utf-8")
with conn.transaction():
for statement in sql.split(";"):
if statement.strip():
conn.execute(statement)
conn.execute("INSERT INTO schema_migrations(version) VALUES (%s)", (version,))
print(f"APPLIED {version}", flush=True)
print("PASS: migrations complete", flush=True)
return
except Exception as exc:
last_error = exc
time.sleep(1)
raise SystemExit(f"Migration failed after retries: {type(last_error).__name__}")
if __name__ == "__main__":
main()
+53
View File
@@ -0,0 +1,53 @@
#!/usr/bin/env python3
"""Operator tool: failed-job registry and SAFE explicit replay (lab5, level 4).
docker compose exec worker python scripts/replay_failed.py --list
docker compose exec worker python scripts/replay_failed.py --job-id <uuid> --reason "ticket-123"
Replay never executes a duplicate of a successful job: it requires status=failed, that the staging source still exists,
and that the effect (all variants of the pipeline version) is NOT already present. Everything happens in ONE transaction
(state change + audit event + outbox row), so the new attempt is delivered by the normal dispatcher path.
"""
import argparse
import os
import sys
import psycopg
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from src import jobs # noqa: E402
def main() -> int:
p = argparse.ArgumentParser()
p.add_argument("--list", action="store_true")
p.add_argument("--job-id")
p.add_argument("--reason", default="operator replay")
a = p.parse_args()
with psycopg.connect(connect_timeout=5) as conn:
if a.list or not a.job_id:
for r in conn.execute("SELECT job_id, attachment_id, error_code, attempt, updated_at FROM attachment_jobs WHERE status='failed' ORDER BY updated_at"):
print(*r, sep="\t")
return 0
row = conn.execute("SELECT j.status, j.attachment_id, j.pipeline_version, a.staging_key FROM attachment_jobs j "
"JOIN attachments a ON a.id=j.attachment_id WHERE j.job_id=%s FOR UPDATE OF j", (a.job_id,)).fetchone()
if row is None:
print("REFUSED: unknown job"); return 1
status, att_id, pv, staging_key = row
if status != "failed":
print(f"REFUSED: job is {status}, only failed jobs can be replayed"); return 1
done = conn.execute("SELECT count(*) FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s", (att_id, pv)).fetchone()[0]
if done >= len(jobs.VARIANT_ORDER):
print("REFUSED: the effect already exists (all variants present)"); return 1
if not staging_key:
print("REFUSED: staging source expired (retention), upload the file again"); return 1
conn.execute("UPDATE attachment_jobs SET status='queued', attempt=0, error_code=NULL, lease_until=NULL, next_attempt_at=now(), updated_at=now() WHERE job_id=%s", (a.job_id,))
conn.execute("UPDATE attachments SET status='queued', error_code=NULL, updated_at=now() WHERE id=%s", (att_id,))
jobs.add_event(conn, a.job_id, "replay", a.reason[:200])
jobs.enqueue(conn, a.job_id, att_id, pv, 1)
print(f"REPLAYED job={a.job_id} attachment={att_id}")
return 0
if __name__ == "__main__":
sys.exit(main())
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -eu
# Restart ALL state stores (PostgreSQL, Redis, S3, Kafka), then API, dispatcher and worker; named volumes are preserved.
docker compose restart postgres redis s3 kafka
docker compose restart api dispatcher worker
+24
View File
@@ -0,0 +1,24 @@
#!/bin/sh
set -eu
# Sequential replacement-equivalent exercise for Compose (which has no native rolling-update primitive).
# Run a load generator in another terminal while this script executes.
replicas="$(docker compose ps -q --all api)"
[ -n "$replicas" ] || { echo "No API containers found; start the stack first" >&2; exit 1; }
for id in $replicas; do
# Remove one replica; the LB excludes it after health-check failure.
docker stop --time 10 "$id" >/dev/null || true
docker rm -f "$id" >/dev/null
docker compose up -d --no-deps --scale api=2 api
# Wait for at least two healthy API containers before moving to the next one.
tries=0
while [ "$tries" -lt 30 ]; do
healthy=0
for candidate in $(docker compose ps -q api); do
status=$(docker inspect --format '{{.State.Health.Status}}' "$candidate" 2>/dev/null || echo starting)
[ "$status" = healthy ] && healthy=$((healthy+1))
done
[ "$healthy" -ge 2 ] && break
tries=$((tries+1)); sleep 1
done
[ "$healthy" -ge 2 ] || { echo "Fewer than two healthy replicas after replacement" >&2; exit 1; }
done
+74
View File
@@ -0,0 +1,74 @@
#!/usr/bin/env python3
"""Orphan sweeper (lab5): staging/ and attachments/ cleanup, retention of failed staging (see run_once).
Safety: only objects older than --grace-seconds are candidates (default 3600, far above the few seconds between
PUT and COMMIT), and the DB is re-checked immediately before each delete. Default mode is --dry-run.
"""
import argparse, os, sys, time
from datetime import datetime, timezone
import boto3, psycopg
from botocore.config import Config
def run_once(s3, conn, bucket, grace, apply, retention):
"""Lab5 cleanup rules (documented in REPORT.md):
1) staging/<k> without any attachments.staging_key row older than grace = orphan of a crash between PUT and COMMIT -> delete
(staging of ready attachments is deleted by the worker and its staging_key set to NULL, so a leftover is also an orphan);
2) attachments/<id>/... whose attachment row no longer exists -> delete;
3) staging of FAILED attachments older than retention -> delete object and NULL staging_key (replay is then impossible).
Current work (queued/processing, or failed within retention) is never touched; the DB is re-checked right before each delete."""
now = datetime.now(timezone.utc)
seen = old = orphans = deleted = 0
for prefix in ("staging/", "attachments/"):
for page in s3.get_paginator("list_objects_v2").paginate(Bucket=bucket, Prefix=prefix):
for obj in page.get("Contents", []):
seen += 1
key = obj["Key"]
if key == "attachments/.ready" or (now - obj["LastModified"]).total_seconds() < grace:
continue # fresh: may still be between PUT and COMMIT
old += 1
if prefix == "staging/":
referenced = lambda: conn.execute("SELECT 1 FROM attachments WHERE staging_key=%s", (key,)).fetchone()
else:
att = key.split("/")[1] if key.count("/") >= 2 else ""
referenced = lambda: conn.execute("SELECT 1 FROM attachments WHERE id::text=%s", (att,)).fetchone()
if referenced():
continue
orphans += 1
print(("DELETE " if apply else "WOULD-DELETE ") + key, flush=True)
if apply and not referenced():
s3.delete_object(Bucket=bucket, Key=key); deleted += 1
expired = conn.execute("SELECT id, staging_key FROM attachments WHERE status='failed' AND staging_key IS NOT NULL "
"AND updated_at < now() - make_interval(secs=>%s)", (retention,)).fetchall()
for att_id, skey in expired:
print(("EXPIRE-STAGING " if apply else "WOULD-EXPIRE-STAGING ") + skey, flush=True)
if apply:
s3.delete_object(Bucket=bucket, Key=skey)
conn.execute("UPDATE attachments SET staging_key=NULL WHERE id=%s", (att_id,))
deleted += 1
print(f"sweep: seen={seen} older_than_grace={old} orphans={orphans} failed_expired={len(expired)} deleted={deleted} apply={apply}", flush=True)
return orphans, deleted
def main():
p = argparse.ArgumentParser()
p.add_argument("--apply", action="store_true", help="really delete (default: dry-run)")
p.add_argument("--grace-seconds", type=int, default=int(os.getenv("SWEEP_GRACE_SECONDS", "3600")))
p.add_argument("--loop", action="store_true"); p.add_argument("--interval", type=int, default=int(os.getenv("SWEEP_INTERVAL_SECONDS", "600")))
p.add_argument("--retention-seconds", type=int, default=int(os.getenv("STAGING_RETENTION_SECONDS", "86400")))
a = p.parse_args()
if a.grace_seconds < 0: sys.exit("grace must be >= 0")
s3 = boto3.client("s3", endpoint_url=os.environ["S3_ENDPOINT"], region_name=os.getenv("S3_REGION", "us-east-1"),
aws_access_key_id=os.environ["S3_ACCESS_KEY_ID"], aws_secret_access_key=os.environ["S3_SECRET_ACCESS_KEY"],
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}, connect_timeout=3, read_timeout=15, retries={"max_attempts": 2}))
bucket = os.environ["S3_BUCKET"]
while True:
try:
with psycopg.connect(connect_timeout=5, autocommit=True) as conn:
run_once(s3, conn, bucket, a.grace_seconds, a.apply, a.retention_seconds)
except Exception as exc: # keep looping through outages
print("sweep failed:", type(exc).__name__, flush=True)
if not a.loop: raise
if not a.loop: return
time.sleep(a.interval)
if __name__ == "__main__":
main()
+17
View File
@@ -0,0 +1,17 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID must be supplied by tests/resilience.py}"
found=""
for id in $(docker compose ps -q api); do
host=$(docker inspect --format '{{.Config.Hostname}}' "$id")
if [ "$host" = "$TARGET_INSTANCE_ID" ]; then
found="$id"
break
fi
done
if [ -z "$found" ]; then
echo "No running API container matches TARGET_INSTANCE_ID" >&2
exit 1
fi
# Stop exactly one API replica and intentionally leave it stopped for the failover check.
docker stop --time 10 "$found" >/dev/null