forked from mtusi2026/mtusi2026lab5
Лаба 5: Redis Streams очередь + асинхронная обработка изображений (оценка 4)
Переиспользован стек lab4 (LB/2 реплики/Postgres/SeaweedFS). upload теперь 202 queued; отдельный worker-процесс генерирует thumbnail/crop/optimized. Transactional outbox закрывает разрыв COMMIT/publish; durable retry (job_retry_schedule) переживает падение worker во время backoff; lease recovery через XAUTOCLAIM; DLQ-реестр (job_failures) с явным operator replay (scripts/replay_failed.py). make test/test-persistence/test-failover/ validate зелёные; в процессе экспериментов найдены и исправлены два реальных бага (неполный try/except вокруг S3-записи вариантов, ошибка типов asyncpg в построении interval) — см. REPORT.md и evidence/. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,72 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Find S3 objects with no matching `attachments` row (e.g. PUT succeeded but
|
||||
the PostgreSQL INSERT that followed it never committed -- a crash between the
|
||||
two writes). Dry-run by default; --delete removes orphans older than the
|
||||
grace period, never anything referenced in the database.
|
||||
|
||||
Usage:
|
||||
python3 scripts/reconcile_attachments.py --grace-minutes 60
|
||||
python3 scripts/reconcile_attachments.py --grace-minutes 60 --delete
|
||||
"""
|
||||
import argparse
|
||||
import asyncio
|
||||
import datetime as dt
|
||||
import os
|
||||
import sys
|
||||
|
||||
import asyncpg
|
||||
import boto3
|
||||
|
||||
|
||||
def read_secret(file_env, inline_env):
|
||||
path = os.environ.get(file_env)
|
||||
if path and os.path.isfile(path):
|
||||
return open(path, encoding="utf-8").read().strip()
|
||||
return os.environ[inline_env]
|
||||
|
||||
|
||||
async def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--grace-minutes", type=int, default=60,
|
||||
help="Only objects older than this are candidates (protects in-flight uploads).")
|
||||
p.add_argument("--delete", action="store_true", help="Actually delete orphans (default: dry-run).")
|
||||
args = p.parse_args()
|
||||
|
||||
database_url = read_secret("DATABASE_URL_FILE", "DATABASE_URL")
|
||||
conn = await asyncpg.connect(database_url)
|
||||
known_keys = {r["s3_key"] for r in await conn.fetch("SELECT s3_key FROM attachments")}
|
||||
await conn.close()
|
||||
|
||||
s3 = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=os.environ["S3_ENDPOINT"],
|
||||
region_name=os.environ.get("S3_REGION", "us-east-1"),
|
||||
aws_access_key_id=read_secret("S3_ACCESS_KEY_ID_FILE", "S3_ACCESS_KEY_ID"),
|
||||
aws_secret_access_key=read_secret("S3_SECRET_ACCESS_KEY_FILE", "S3_SECRET_ACCESS_KEY"),
|
||||
)
|
||||
bucket = os.environ.get("S3_BUCKET", "messenger-attachments")
|
||||
cutoff = dt.datetime.now(dt.timezone.utc) - dt.timedelta(minutes=args.grace_minutes)
|
||||
|
||||
orphans = []
|
||||
paginator = s3.get_paginator("list_objects_v2")
|
||||
for page in paginator.paginate(Bucket=bucket, Prefix="attachments/"):
|
||||
for obj in page.get("Contents", []):
|
||||
if obj["Key"] in known_keys:
|
||||
continue
|
||||
if obj["LastModified"] > cutoff:
|
||||
continue # might still be an in-flight upload; not an orphan yet
|
||||
orphans.append(obj["Key"])
|
||||
|
||||
print(f"{len(orphans)} orphaned object(s) older than {args.grace_minutes} minutes with no DB row.")
|
||||
for key in orphans:
|
||||
print(" ", key)
|
||||
if args.delete:
|
||||
s3.delete_object(Bucket=bucket, Key=key)
|
||||
if orphans and args.delete:
|
||||
print(f"Deleted {len(orphans)} orphan(s).")
|
||||
elif orphans:
|
||||
print("Dry-run: nothing deleted. Re-run with --delete to remove them.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(asyncio.run(main()) or 0)
|
||||
Executable
+66
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Explicit operator replay for a failed attachment: resets status to
|
||||
'queued' and republishes a fresh job (new job_id, attempt=1). Never replays
|
||||
an attachment that is already 'ready' -- the same protection the worker
|
||||
itself applies against stale duplicates.
|
||||
|
||||
Usage: python3 scripts/replay_failed.py <attachment-id>
|
||||
"""
|
||||
import asyncio
|
||||
import datetime as dt
|
||||
import os
|
||||
import sys
|
||||
import uuid
|
||||
|
||||
import asyncpg
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from app import broker # noqa: E402
|
||||
|
||||
|
||||
def read_secret(file_env, inline_env):
|
||||
path = os.environ.get(file_env)
|
||||
if path and os.path.isfile(path):
|
||||
return open(path, encoding="utf-8").read().strip()
|
||||
return os.environ[inline_env]
|
||||
|
||||
|
||||
async def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: replay_failed.py <attachment-id>", file=sys.stderr)
|
||||
return 2
|
||||
attachment_id = uuid.UUID(sys.argv[1])
|
||||
|
||||
conn = await asyncpg.connect(read_secret("DATABASE_URL_FILE", "DATABASE_URL"))
|
||||
record = await conn.fetchrow("SELECT status FROM attachments WHERE id=$1", attachment_id)
|
||||
if record is None:
|
||||
print("no such attachment", file=sys.stderr)
|
||||
return 1
|
||||
if record["status"] == "ready":
|
||||
print("attachment is already ready; refusing to replay a finished job")
|
||||
return 0
|
||||
if record["status"] != "failed":
|
||||
print(f"attachment status is '{record['status']}', not 'failed'; nothing to replay")
|
||||
return 0
|
||||
|
||||
job_id = uuid.uuid4()
|
||||
await conn.execute("UPDATE attachments SET status='queued', error_code=NULL, updated_at=now() WHERE id=$1", attachment_id)
|
||||
await conn.close()
|
||||
|
||||
broker.init(read_secret("CACHE_URL_FILE", "CACHE_URL"))
|
||||
job = {
|
||||
"schema_version": 1,
|
||||
"job_id": str(job_id),
|
||||
"type": "attachment.process.v1",
|
||||
"attachment_id": str(attachment_id),
|
||||
"pipeline_version": 1,
|
||||
"attempt": 1,
|
||||
"created_at": dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z"),
|
||||
}
|
||||
await broker.publish(job)
|
||||
print(f"replayed attachment {attachment_id} as job {job_id}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(asyncio.run(main()))
|
||||
Executable
+26
@@ -0,0 +1,26 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
wait_healthy() {
|
||||
for svc in "$@"; do
|
||||
for _ in $(seq 1 60); do
|
||||
status=$(docker compose ps --format '{{.Health}}' "$svc" 2>/dev/null || true)
|
||||
[ "$status" = "healthy" ] && break
|
||||
sleep 1
|
||||
done
|
||||
done
|
||||
}
|
||||
|
||||
# Sequenced: API/worker eagerly connect to Postgres/Redis/S3 at startup and
|
||||
# exit if unreachable; Nginx resolves upstream hostnames only at its own
|
||||
# startup. Each stage must be actually healthy before the next one starts.
|
||||
docker compose restart postgres redis seaweedfs
|
||||
wait_healthy postgres redis seaweedfs
|
||||
|
||||
docker compose restart api1 api2
|
||||
wait_healthy api1 api2
|
||||
|
||||
docker compose restart worker
|
||||
docker compose restart lb
|
||||
exit 0
|
||||
Executable
+12
@@ -0,0 +1,12 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Stops exactly the API replica identified by TARGET_INSTANCE_ID (api1 or api2),
|
||||
# leaving it stopped. The other replica keeps serving traffic through the LB.
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID must be set by the test harness}"
|
||||
case "$TARGET_INSTANCE_ID" in
|
||||
api1|api2) ;;
|
||||
*) echo "unknown instance id: $TARGET_INSTANCE_ID" >&2; exit 1 ;;
|
||||
esac
|
||||
docker compose stop "$TARGET_INSTANCE_ID"
|
||||
exit 0
|
||||
Reference in New Issue
Block a user