lab 6 observability
This commit is contained in:
@@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Apply SQL migrations once, before any API replica is started."""
|
||||
from pathlib import Path
|
||||
import os
|
||||
import time
|
||||
import psycopg
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
CONNINFO = os.getenv("DATABASE_URL", "")
|
||||
|
||||
def main():
|
||||
last_error = None
|
||||
for _ in range(30):
|
||||
try:
|
||||
with psycopg.connect(CONNINFO, connect_timeout=3) as conn:
|
||||
conn.execute("CREATE TABLE IF NOT EXISTS schema_migrations (version TEXT PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT now())")
|
||||
conn.commit()
|
||||
for path in sorted((ROOT / "migrations").glob("*.sql")):
|
||||
version = path.name
|
||||
if conn.execute("SELECT 1 FROM schema_migrations WHERE version=%s", (version,)).fetchone():
|
||||
continue
|
||||
sql = path.read_text(encoding="utf-8")
|
||||
with conn.transaction():
|
||||
for statement in sql.split(";"):
|
||||
if statement.strip():
|
||||
conn.execute(statement)
|
||||
conn.execute("INSERT INTO schema_migrations(version) VALUES (%s)", (version,))
|
||||
print(f"APPLIED {version}", flush=True)
|
||||
print("PASS: migrations complete", flush=True)
|
||||
return
|
||||
except Exception as exc:
|
||||
last_error = exc
|
||||
time.sleep(1)
|
||||
raise SystemExit(f"Migration failed after retries: {type(last_error).__name__}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Operator tool: failed-job registry and SAFE explicit replay (lab5, level 4).
|
||||
|
||||
docker compose exec worker python scripts/replay_failed.py --list
|
||||
docker compose exec worker python scripts/replay_failed.py --job-id <uuid> --reason "ticket-123"
|
||||
|
||||
Replay never executes a duplicate of a successful job: it requires status=failed, that the staging source still exists,
|
||||
and that the effect (all variants of the pipeline version) is NOT already present. Everything happens in ONE transaction
|
||||
(state change + audit event + outbox row), so the new attempt is delivered by the normal dispatcher path.
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
|
||||
import psycopg
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from src import jobs # noqa: E402
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--list", action="store_true")
|
||||
p.add_argument("--job-id")
|
||||
p.add_argument("--reason", default="operator replay")
|
||||
a = p.parse_args()
|
||||
with psycopg.connect(connect_timeout=5) as conn:
|
||||
if a.list or not a.job_id:
|
||||
for r in conn.execute("SELECT job_id, attachment_id, error_code, attempt, updated_at FROM attachment_jobs WHERE status='failed' ORDER BY updated_at"):
|
||||
print(*r, sep="\t")
|
||||
return 0
|
||||
row = conn.execute("SELECT j.status, j.attachment_id, j.pipeline_version, a.staging_key FROM attachment_jobs j "
|
||||
"JOIN attachments a ON a.id=j.attachment_id WHERE j.job_id=%s FOR UPDATE OF j", (a.job_id,)).fetchone()
|
||||
if row is None:
|
||||
print("REFUSED: unknown job"); return 1
|
||||
status, att_id, pv, staging_key = row
|
||||
if status != "failed":
|
||||
print(f"REFUSED: job is {status}, only failed jobs can be replayed"); return 1
|
||||
done = conn.execute("SELECT count(*) FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s", (att_id, pv)).fetchone()[0]
|
||||
if done >= len(jobs.VARIANT_ORDER):
|
||||
print("REFUSED: the effect already exists (all variants present)"); return 1
|
||||
if not staging_key:
|
||||
print("REFUSED: staging source expired (retention), upload the file again"); return 1
|
||||
conn.execute("UPDATE attachment_jobs SET status='queued', attempt=0, error_code=NULL, lease_until=NULL, next_attempt_at=now(), updated_at=now() WHERE job_id=%s", (a.job_id,))
|
||||
conn.execute("UPDATE attachments SET status='queued', error_code=NULL, updated_at=now() WHERE id=%s", (att_id,))
|
||||
jobs.add_event(conn, a.job_id, "replay", a.reason[:200])
|
||||
jobs.enqueue(conn, a.job_id, att_id, pv, 1)
|
||||
print(f"REPLAYED job={a.job_id} attachment={att_id}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Restart ALL state stores (PostgreSQL, Redis, S3, Kafka + Prometheus, Loki, Tempo), then API, dispatcher, worker, Collector, Grafana; named volumes are preserved.
|
||||
docker compose restart postgres redis s3 kafka prometheus loki tempo
|
||||
docker compose restart api dispatcher worker otel-collector grafana
|
||||
@@ -0,0 +1,24 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Sequential replacement-equivalent exercise for Compose (which has no native rolling-update primitive).
|
||||
# Run a load generator in another terminal while this script executes.
|
||||
replicas="$(docker compose ps -q --all api)"
|
||||
[ -n "$replicas" ] || { echo "No API containers found; start the stack first" >&2; exit 1; }
|
||||
for id in $replicas; do
|
||||
# Remove one replica; the LB excludes it after health-check failure.
|
||||
docker stop --time 10 "$id" >/dev/null || true
|
||||
docker rm -f "$id" >/dev/null
|
||||
docker compose up -d --no-deps --scale api=2 api
|
||||
# Wait for at least two healthy API containers before moving to the next one.
|
||||
tries=0
|
||||
while [ "$tries" -lt 30 ]; do
|
||||
healthy=0
|
||||
for candidate in $(docker compose ps -q api); do
|
||||
status=$(docker inspect --format '{{.State.Health.Status}}' "$candidate" 2>/dev/null || echo starting)
|
||||
[ "$status" = healthy ] && healthy=$((healthy+1))
|
||||
done
|
||||
[ "$healthy" -ge 2 ] && break
|
||||
tries=$((tries+1)); sleep 1
|
||||
done
|
||||
[ "$healthy" -ge 2 ] || { echo "Fewer than two healthy replicas after replacement" >&2; exit 1; }
|
||||
done
|
||||
@@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Orphan sweeper (lab5): staging/ and attachments/ cleanup, retention of failed staging (see run_once).
|
||||
|
||||
Safety: only objects older than --grace-seconds are candidates (default 3600, far above the few seconds between
|
||||
PUT and COMMIT), and the DB is re-checked immediately before each delete. Default mode is --dry-run.
|
||||
"""
|
||||
import argparse, os, sys, time
|
||||
from datetime import datetime, timezone
|
||||
import boto3, psycopg
|
||||
from botocore.config import Config
|
||||
|
||||
def run_once(s3, conn, bucket, grace, apply, retention):
|
||||
"""Lab5 cleanup rules (documented in REPORT.md):
|
||||
1) staging/<k> without any attachments.staging_key row older than grace = orphan of a crash between PUT and COMMIT -> delete
|
||||
(staging of ready attachments is deleted by the worker and its staging_key set to NULL, so a leftover is also an orphan);
|
||||
2) attachments/<id>/... whose attachment row no longer exists -> delete;
|
||||
3) staging of FAILED attachments older than retention -> delete object and NULL staging_key (replay is then impossible).
|
||||
Current work (queued/processing, or failed within retention) is never touched; the DB is re-checked right before each delete."""
|
||||
now = datetime.now(timezone.utc)
|
||||
seen = old = orphans = deleted = 0
|
||||
for prefix in ("staging/", "attachments/"):
|
||||
for page in s3.get_paginator("list_objects_v2").paginate(Bucket=bucket, Prefix=prefix):
|
||||
for obj in page.get("Contents", []):
|
||||
seen += 1
|
||||
key = obj["Key"]
|
||||
if key == "attachments/.ready" or (now - obj["LastModified"]).total_seconds() < grace:
|
||||
continue # fresh: may still be between PUT and COMMIT
|
||||
old += 1
|
||||
if prefix == "staging/":
|
||||
referenced = lambda: conn.execute("SELECT 1 FROM attachments WHERE staging_key=%s", (key,)).fetchone()
|
||||
else:
|
||||
att = key.split("/")[1] if key.count("/") >= 2 else ""
|
||||
referenced = lambda: conn.execute("SELECT 1 FROM attachments WHERE id::text=%s", (att,)).fetchone()
|
||||
if referenced():
|
||||
continue
|
||||
orphans += 1
|
||||
print(("DELETE " if apply else "WOULD-DELETE ") + key, flush=True)
|
||||
if apply and not referenced():
|
||||
s3.delete_object(Bucket=bucket, Key=key); deleted += 1
|
||||
expired = conn.execute("SELECT id, staging_key FROM attachments WHERE status='failed' AND staging_key IS NOT NULL "
|
||||
"AND updated_at < now() - make_interval(secs=>%s)", (retention,)).fetchall()
|
||||
for att_id, skey in expired:
|
||||
print(("EXPIRE-STAGING " if apply else "WOULD-EXPIRE-STAGING ") + skey, flush=True)
|
||||
if apply:
|
||||
s3.delete_object(Bucket=bucket, Key=skey)
|
||||
conn.execute("UPDATE attachments SET staging_key=NULL WHERE id=%s", (att_id,))
|
||||
deleted += 1
|
||||
print(f"sweep: seen={seen} older_than_grace={old} orphans={orphans} failed_expired={len(expired)} deleted={deleted} apply={apply}", flush=True)
|
||||
return orphans, deleted
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--apply", action="store_true", help="really delete (default: dry-run)")
|
||||
p.add_argument("--grace-seconds", type=int, default=int(os.getenv("SWEEP_GRACE_SECONDS", "3600")))
|
||||
p.add_argument("--loop", action="store_true"); p.add_argument("--interval", type=int, default=int(os.getenv("SWEEP_INTERVAL_SECONDS", "600")))
|
||||
p.add_argument("--retention-seconds", type=int, default=int(os.getenv("STAGING_RETENTION_SECONDS", "86400")))
|
||||
a = p.parse_args()
|
||||
if a.grace_seconds < 0: sys.exit("grace must be >= 0")
|
||||
s3 = boto3.client("s3", endpoint_url=os.environ["S3_ENDPOINT"], region_name=os.getenv("S3_REGION", "us-east-1"),
|
||||
aws_access_key_id=os.environ["S3_ACCESS_KEY_ID"], aws_secret_access_key=os.environ["S3_SECRET_ACCESS_KEY"],
|
||||
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}, connect_timeout=3, read_timeout=15, retries={"max_attempts": 2}))
|
||||
bucket = os.environ["S3_BUCKET"]
|
||||
while True:
|
||||
try:
|
||||
with psycopg.connect(connect_timeout=5, autocommit=True) as conn:
|
||||
run_once(s3, conn, bucket, a.grace_seconds, a.apply, a.retention_seconds)
|
||||
except Exception as exc: # keep looping through outages
|
||||
print("sweep failed:", type(exc).__name__, flush=True)
|
||||
if not a.loop: raise
|
||||
if not a.loop: return
|
||||
time.sleep(a.interval)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,17 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID must be supplied by tests/resilience.py}"
|
||||
found=""
|
||||
for id in $(docker compose ps -q api); do
|
||||
host=$(docker inspect --format '{{.Config.Hostname}}' "$id")
|
||||
if [ "$host" = "$TARGET_INSTANCE_ID" ]; then
|
||||
found="$id"
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [ -z "$found" ]; then
|
||||
echo "No running API container matches TARGET_INSTANCE_ID" >&2
|
||||
exit 1
|
||||
fi
|
||||
# Stop exactly one API replica and intentionally leave it stopped for the failover check.
|
||||
docker stop --time 10 "$found" >/dev/null
|
||||
Reference in New Issue
Block a user