lab6: observability — OTel Collector, Prometheus, Loki, Tempo, Grafana

This commit is contained in:
2026-11-13 13:20:46 +03:00
parent 470d9d6f1e
commit 5945e035e0
63 changed files with 6436 additions and 19 deletions
+85
View File
@@ -0,0 +1,85 @@
from __future__ import annotations
import logging
import os
import sys
from datetime import datetime, timedelta, timezone
import boto3
import psycopg
from botocore.client import Config
from botocore.exceptions import ClientError
from psycopg.rows import dict_row
logging.basicConfig(
level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s"
)
log = logging.getLogger("cleanup")
def _env(name: str, default: str | None = None) -> str:
value = os.environ.get(name, default)
if value is None:
raise RuntimeError(f"Environment variable {name} is required")
return value
def main() -> None:
grace_hours = int(os.environ.get("CLEANUP_GRACE_HOURS", "1"))
grace = timedelta(hours=grace_hours)
bucket = _env("S3_BUCKET", "messenger-attachments")
s3 = boto3.client(
"s3",
endpoint_url=_env("S3_ENDPOINT"),
region_name=_env("S3_REGION", "us-east-1"),
aws_access_key_id=_env("S3_ACCESS_KEY_ID"),
aws_secret_access_key=_env("S3_SECRET_ACCESS_KEY"),
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}),
)
s3_objects: dict[str, datetime] = {}
paginator = s3.get_paginator("list_objects_v2")
for page in paginator.paginate(Bucket=bucket):
for obj in page.get("Contents", []):
s3_objects[obj["Key"]] = obj["LastModified"]
log.info("S3 objects: %d", len(s3_objects))
db_keys: set[str] = set()
with psycopg.connect(_env("DATABASE_URL"), row_factory=dict_row) as conn:
with conn.cursor() as cur:
cur.execute("SELECT s3_key FROM attachments")
for row in cur.fetchall():
db_keys.add(row["s3_key"])
log.info("DB attachments: %d", len(db_keys))
orphans = set(s3_objects) - db_keys
log.info("Orphans: %d", len(orphans))
now = datetime.now(timezone.utc)
deleted = 0
skipped = 0
for key in orphans:
age = now - s3_objects[key]
if age < grace:
log.info("skip %s (age %s < grace %s)", key, age, grace)
skipped += 1
continue
try:
s3.delete_object(Bucket=bucket, Key=key)
log.info("deleted orphan: %s (age %s)", key, age)
deleted += 1
except ClientError as exc:
log.error("failed to delete %s: %s", key, exc)
log.info("done: deleted=%d, skipped=%d", deleted, skipped)
if __name__ == "__main__":
try:
main()
except Exception as exc:
log.exception("cleanup failed")
sys.exit(1)
+29
View File
@@ -0,0 +1,29 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
docker compose --env-file .env restart postgres cache s3 broker
i=0
while [ $i -lt 120 ]; do
if docker compose --env-file .env exec -T broker rabbitmq-diagnostics -q ping > /dev/null 2>&1; then
break
fi
sleep 1
i=$((i + 1))
done
docker compose --env-file .env restart api worker
i=0
while [ $i -lt 90 ]; do
if curl -sf http://localhost:8080/health/ready > /dev/null 2>&1; then
exit 0
fi
sleep 1
i=$((i + 1))
done
echo "stack did not become healthy in 90 seconds" >&2
exit 1
+25
View File
@@ -0,0 +1,25 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID environment variable is required}"
echo "Looking for API container with instance_id=$TARGET_INSTANCE_ID"
found=""
for container in $(docker ps --format '{{.Names}}' | grep -E '^mtusi.*-api-[0-9]+$'); do
if docker logs "$container" 2>&1 | grep -q "instance_id=$TARGET_INSTANCE_ID"; then
found="$container"
break
fi
done
if [ -z "$found" ]; then
echo "ERROR: no container found with instance_id=$TARGET_INSTANCE_ID" >&2
exit 1
fi
echo "Stopping container: $found"
docker stop "$found"
echo "Container $found stopped. Other replicas continue serving."
exit 0