Carry forward the lab4 API and storage baseline, stage verified originals in MinIO, and atomically commit attachment, job, and outbox records before 202. Publish identifier-only jobs with broker confirms and reclaimable outbox leases. Clean abandoned and completed staging objects after database checks.
105 lines
3.6 KiB
Python
105 lines
3.6 KiB
Python
"""Remove abandoned staging objects after a grace period or a committed result."""
|
|
|
|
import logging
|
|
import os
|
|
import time
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
from uuid import UUID
|
|
|
|
from minio import Minio
|
|
|
|
from .db import pool
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
BUCKET = "messenger-attachments"
|
|
|
|
|
|
def storage():
|
|
def secret(name):
|
|
return Path(os.environ[f"{name}_FILE"]).read_text(encoding="utf-8").strip()
|
|
|
|
return Minio(
|
|
os.environ.get("S3_ENDPOINT", "s3:9000"),
|
|
access_key=secret("S3_MAINTENANCE_KEY_ID"),
|
|
secret_key=secret("S3_MAINTENANCE_SECRET_KEY"),
|
|
secure=False,
|
|
region="us-east-1",
|
|
)
|
|
|
|
|
|
def clean_once(client, grace_seconds=3600):
|
|
"""Return removed count; lock DB state through each S3 delete."""
|
|
if grace_seconds < 1:
|
|
raise ValueError("invalid staging grace period")
|
|
cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds)
|
|
removed = 0
|
|
for obj in client.list_objects(BUCKET, prefix="staging/", recursive=True):
|
|
key = obj.object_name
|
|
suffix = key.removeprefix("staging/")
|
|
try:
|
|
if key != f"staging/{UUID(suffix)}":
|
|
continue
|
|
except ValueError:
|
|
continue
|
|
with pool().connection() as conn, conn.transaction():
|
|
attachment = conn.execute(
|
|
"SELECT a.status,EXISTS("
|
|
"SELECT 1 FROM attachment_variants v WHERE v.attachment_id=a.id "
|
|
"AND v.pipeline_version=a.pipeline_version AND v.name='original' "
|
|
"AND v.object_key=a.object_key) AS original_ready "
|
|
"FROM attachments a WHERE a.staging_key=%s FOR UPDATE OF a",
|
|
(key,),
|
|
).fetchone()
|
|
if attachment:
|
|
if attachment["status"] != "ready" or not attachment["original_ready"]:
|
|
continue
|
|
else:
|
|
if obj.last_modified is None or obj.last_modified > cutoff:
|
|
continue
|
|
marker = conn.execute(
|
|
"SELECT lease_expires_at>clock_timestamp() AS active "
|
|
"FROM attachment_uploads WHERE staging_key=%s FOR UPDATE",
|
|
(key,),
|
|
).fetchone()
|
|
if marker and marker["active"]:
|
|
continue
|
|
client.remove_object(BUCKET, key)
|
|
if not attachment:
|
|
conn.execute(
|
|
"DELETE FROM attachment_uploads WHERE staging_key=%s "
|
|
"AND lease_expires_at<=clock_timestamp()",
|
|
(key,),
|
|
)
|
|
removed += 1
|
|
with pool().connection() as conn, conn.transaction():
|
|
conn.execute(
|
|
"DELETE FROM attachment_uploads u WHERE u.lease_expires_at<"
|
|
"clock_timestamp()-make_interval(secs => %s) "
|
|
"AND NOT EXISTS (SELECT 1 FROM attachments a WHERE a.staging_key=u.staging_key)",
|
|
(grace_seconds,),
|
|
)
|
|
return removed
|
|
|
|
|
|
def main():
|
|
logging.basicConfig(level=logging.INFO)
|
|
client = storage()
|
|
grace = int(os.environ.get("STAGING_GRACE_SECONDS", "3600"))
|
|
interval = int(os.environ.get("STAGING_CLEANUP_INTERVAL_SECONDS", "60"))
|
|
if interval < 1:
|
|
raise ValueError("invalid cleanup interval")
|
|
while True:
|
|
try:
|
|
removed = clean_once(client, grace)
|
|
if removed:
|
|
logger.info("Removed %s staging objects", removed)
|
|
except Exception as exc:
|
|
logger.warning("Staging cleanup retrying after %s", type(exc).__name__)
|
|
time.sleep(interval)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|