forked from mtusi2026/mtusi2026lab6
lab6: observability — OTel Collector, Prometheus, Loki, Tempo, Grafana
This commit is contained in:
@@ -0,0 +1,85 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import boto3
|
||||
import psycopg
|
||||
from botocore.client import Config
|
||||
from botocore.exceptions import ClientError
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s"
|
||||
)
|
||||
log = logging.getLogger("cleanup")
|
||||
|
||||
|
||||
def _env(name: str, default: str | None = None) -> str:
|
||||
value = os.environ.get(name, default)
|
||||
if value is None:
|
||||
raise RuntimeError(f"Environment variable {name} is required")
|
||||
return value
|
||||
|
||||
|
||||
def main() -> None:
|
||||
grace_hours = int(os.environ.get("CLEANUP_GRACE_HOURS", "1"))
|
||||
grace = timedelta(hours=grace_hours)
|
||||
bucket = _env("S3_BUCKET", "messenger-attachments")
|
||||
|
||||
s3 = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=_env("S3_ENDPOINT"),
|
||||
region_name=_env("S3_REGION", "us-east-1"),
|
||||
aws_access_key_id=_env("S3_ACCESS_KEY_ID"),
|
||||
aws_secret_access_key=_env("S3_SECRET_ACCESS_KEY"),
|
||||
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}),
|
||||
)
|
||||
|
||||
s3_objects: dict[str, datetime] = {}
|
||||
paginator = s3.get_paginator("list_objects_v2")
|
||||
for page in paginator.paginate(Bucket=bucket):
|
||||
for obj in page.get("Contents", []):
|
||||
s3_objects[obj["Key"]] = obj["LastModified"]
|
||||
|
||||
log.info("S3 objects: %d", len(s3_objects))
|
||||
|
||||
db_keys: set[str] = set()
|
||||
with psycopg.connect(_env("DATABASE_URL"), row_factory=dict_row) as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT s3_key FROM attachments")
|
||||
for row in cur.fetchall():
|
||||
db_keys.add(row["s3_key"])
|
||||
|
||||
log.info("DB attachments: %d", len(db_keys))
|
||||
|
||||
orphans = set(s3_objects) - db_keys
|
||||
log.info("Orphans: %d", len(orphans))
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
deleted = 0
|
||||
skipped = 0
|
||||
for key in orphans:
|
||||
age = now - s3_objects[key]
|
||||
if age < grace:
|
||||
log.info("skip %s (age %s < grace %s)", key, age, grace)
|
||||
skipped += 1
|
||||
continue
|
||||
try:
|
||||
s3.delete_object(Bucket=bucket, Key=key)
|
||||
log.info("deleted orphan: %s (age %s)", key, age)
|
||||
deleted += 1
|
||||
except ClientError as exc:
|
||||
log.error("failed to delete %s: %s", key, exc)
|
||||
|
||||
log.info("done: deleted=%d, skipped=%d", deleted, skipped)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except Exception as exc:
|
||||
log.exception("cleanup failed")
|
||||
sys.exit(1)
|
||||
Executable
+29
@@ -0,0 +1,29 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
docker compose --env-file .env restart postgres cache s3 broker
|
||||
|
||||
i=0
|
||||
while [ $i -lt 120 ]; do
|
||||
if docker compose --env-file .env exec -T broker rabbitmq-diagnostics -q ping > /dev/null 2>&1; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
i=$((i + 1))
|
||||
done
|
||||
|
||||
docker compose --env-file .env restart api worker
|
||||
|
||||
i=0
|
||||
while [ $i -lt 90 ]; do
|
||||
if curl -sf http://localhost:8080/health/ready > /dev/null 2>&1; then
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
i=$((i + 1))
|
||||
done
|
||||
|
||||
echo "stack did not become healthy in 90 seconds" >&2
|
||||
exit 1
|
||||
Executable
+25
@@ -0,0 +1,25 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID environment variable is required}"
|
||||
|
||||
echo "Looking for API container with instance_id=$TARGET_INSTANCE_ID"
|
||||
|
||||
found=""
|
||||
for container in $(docker ps --format '{{.Names}}' | grep -E '^mtusi.*-api-[0-9]+$'); do
|
||||
if docker logs "$container" 2>&1 | grep -q "instance_id=$TARGET_INSTANCE_ID"; then
|
||||
found="$container"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -z "$found" ]; then
|
||||
echo "ERROR: no container found with instance_id=$TARGET_INSTANCE_ID" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Stopping container: $found"
|
||||
docker stop "$found"
|
||||
|
||||
echo "Container $found stopped. Other replicas continue serving."
|
||||
exit 0
|
||||
Reference in New Issue
Block a user