forked from mtusi2026/mtusi2026lab5
593 lines
23 KiB
Python
593 lines
23 KiB
Python
"""Idempotent attachment pipeline.
|
|
|
|
The effect key is (attachment_id, pipeline_version, variant). Object keys include that
|
|
identity, database rows use it as a primary key, and a completed job is a no-op.
|
|
Delivery is at least once; this module does not claim exactly-once delivery.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import logging
|
|
import os
|
|
import threading
|
|
import time
|
|
import uuid
|
|
|
|
from psycopg.types.json import Jsonb
|
|
|
|
from app import runtime
|
|
from app.db import connection
|
|
from app.images import ImageRejected, decode_image, pipeline_params, render_variants
|
|
from app.outbox import add_outbox, audit, publish_dlq
|
|
|
|
log = logging.getLogger("messenger.pipeline")
|
|
|
|
VARIANT_NAMES = ("thumbnail", "crop", "optimized")
|
|
|
|
|
|
class TransientJobError(Exception):
|
|
def __init__(self, message: str):
|
|
super().__init__(message)
|
|
self.message = message
|
|
|
|
|
|
class PermanentJobError(Exception):
|
|
def __init__(self, code: str, message: str):
|
|
super().__init__(message)
|
|
self.code = code
|
|
self.message = message
|
|
|
|
|
|
def _safe(text: str) -> str:
|
|
cleaned = " ".join(text.split())
|
|
return cleaned[:180] or "processing failed"
|
|
|
|
|
|
def _parse_message(body: dict) -> dict:
|
|
required = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at"}
|
|
if not isinstance(body, dict) or set(body) - required - {"traceparent", "tracestate"}:
|
|
raise PermanentJobError("processing_failed", "invalid job message")
|
|
if any(key not in body for key in required):
|
|
raise PermanentJobError("processing_failed", "invalid job message")
|
|
if body["schema_version"] != 1 or body["type"] != "attachment.process.v1":
|
|
raise PermanentJobError("processing_failed", "invalid job message")
|
|
try:
|
|
return {
|
|
"job_id": uuid.UUID(str(body["job_id"])),
|
|
"attachment_id": uuid.UUID(str(body["attachment_id"])),
|
|
"pipeline_version": int(body["pipeline_version"]),
|
|
"attempt": int(body["attempt"]),
|
|
"created_at": str(body["created_at"]),
|
|
}
|
|
except (ValueError, TypeError) as exc:
|
|
raise PermanentJobError("processing_failed", "invalid job message") from exc
|
|
|
|
|
|
def _read_fault(conn) -> dict:
|
|
row = conn.execute("SELECT value FROM runtime_control WHERE key = 'worker_fault' FOR UPDATE").fetchone()
|
|
value = row["value"] if row else {}
|
|
return dict(value) if isinstance(value, dict) else {}
|
|
|
|
|
|
def _write_fault(conn, value: dict) -> None:
|
|
conn.execute("UPDATE runtime_control SET value = %s WHERE key = 'worker_fault'", (Jsonb(value),))
|
|
|
|
|
|
def _consume_counted_fault() -> str | None:
|
|
"""Disarm one-shot and counted faults. Crash faults stay armed until their injection point."""
|
|
with connection() as conn:
|
|
value = _read_fault(conn)
|
|
if not value.get("armed"):
|
|
return None
|
|
mode = value.get("mode")
|
|
if mode == "transient":
|
|
remaining = int(value.get("remaining") or 1) - 1
|
|
if remaining <= 0:
|
|
_write_fault(conn, {"mode": None, "armed": False})
|
|
else:
|
|
_write_fault(conn, {"mode": "transient", "armed": True, "remaining": remaining})
|
|
return "transient"
|
|
if mode in {"permanent", "duplicate"}:
|
|
_write_fault(conn, {"mode": None, "armed": False})
|
|
return mode
|
|
if mode in {"before_effect", "after_s3", "after_commit"}:
|
|
return mode
|
|
return None
|
|
|
|
|
|
def _crash(mode: str, job_id, attachment_id) -> None:
|
|
with connection() as conn:
|
|
value = _read_fault(conn)
|
|
if value.get("armed") and value.get("mode") == mode:
|
|
_write_fault(conn, {"mode": None, "armed": False})
|
|
audit(conn, job_id, attachment_id, "fault_" + mode, {"mode": mode})
|
|
if mode in {"before_effect", "after_s3"}:
|
|
conn.execute(
|
|
"""
|
|
UPDATE jobs
|
|
SET status = 'queued', lease_until = NULL, lease_owner = NULL, updated_at = clock_timestamp()
|
|
WHERE job_id = %s AND status = 'processing'
|
|
""",
|
|
(job_id,),
|
|
)
|
|
conn.execute(
|
|
"""
|
|
UPDATE attachments AS a
|
|
SET status = 'queued', updated_at = clock_timestamp()
|
|
FROM jobs AS j
|
|
WHERE j.job_id = %s AND a.id = j.attachment_id
|
|
AND a.pipeline_version = j.pipeline_version AND a.status = 'processing'
|
|
""",
|
|
(job_id,),
|
|
)
|
|
log.warning("fault injection exit: %s", mode)
|
|
os._exit(75)
|
|
|
|
|
|
def _load(conn, message: dict):
|
|
job = conn.execute("SELECT * FROM jobs WHERE job_id = %s FOR UPDATE", (message["job_id"],)).fetchone()
|
|
attachment = None
|
|
if job is not None:
|
|
attachment = conn.execute("SELECT * FROM attachments WHERE id = %s FOR UPDATE", (job["attachment_id"],)).fetchone()
|
|
return job, attachment
|
|
|
|
|
|
def _finish_noop(conn, job, attachment, action: str, redelivered: bool) -> None:
|
|
audit(conn, job["job_id"] if job else None, attachment["id"] if attachment else None, action, {"redelivered": redelivered})
|
|
|
|
|
|
def _supersede(conn, job, attachment, redelivered: bool) -> None:
|
|
conn.execute(
|
|
"""
|
|
UPDATE jobs
|
|
SET status = 'superseded', lease_until = NULL, lease_owner = NULL, updated_at = clock_timestamp()
|
|
WHERE job_id = %s AND status IN ('queued', 'processing')
|
|
""",
|
|
(job["job_id"],),
|
|
)
|
|
_finish_noop(conn, job, attachment, "superseded", redelivered)
|
|
|
|
|
|
def _acquire(conn, job, worker_id: str, timeout_seconds: int):
|
|
return conn.execute(
|
|
"""
|
|
UPDATE jobs
|
|
SET status = 'processing',
|
|
lease_owner = %s,
|
|
lease_until = clock_timestamp() + make_interval(secs => %s),
|
|
updated_at = clock_timestamp()
|
|
WHERE job_id = %s
|
|
AND status IN ('queued', 'processing')
|
|
AND (lease_until IS NULL OR lease_until < clock_timestamp() OR lease_owner = %s)
|
|
RETURNING job_id
|
|
""",
|
|
(worker_id, timeout_seconds, job["job_id"], worker_id),
|
|
).fetchone()
|
|
|
|
|
|
def _schedule_retry(conn, job, attachment, message: str) -> None:
|
|
settings, _store, _publisher = runtime.require()
|
|
attempt = int(job["attempt"])
|
|
if attempt >= settings.job_max_attempts:
|
|
_fail(conn, job, attachment, "retry_exhausted", "transient", "retry budget exhausted")
|
|
return
|
|
import random
|
|
|
|
delay = min(settings.job_backoff_cap_seconds, settings.job_backoff_base_seconds * (2 ** (attempt - 1)))
|
|
delay = delay * (0.5 + random.random())
|
|
next_attempt = attempt + 1
|
|
conn.execute(
|
|
"""
|
|
UPDATE jobs
|
|
SET status = 'queued', attempt = %s, next_attempt_at = clock_timestamp() + make_interval(secs => %s),
|
|
lease_until = NULL, lease_owner = NULL, error_class = 'transient', error_code = 'processing_failed',
|
|
last_error = %s, updated_at = clock_timestamp()
|
|
WHERE job_id = %s
|
|
""",
|
|
(next_attempt, delay, _safe(message), job["job_id"]),
|
|
)
|
|
conn.execute(
|
|
"""
|
|
UPDATE attachments
|
|
SET status = 'queued', error_code = NULL, updated_at = clock_timestamp()
|
|
WHERE id = %s AND pipeline_version = %s AND status <> 'ready'
|
|
""",
|
|
(attachment["id"], job["pipeline_version"]),
|
|
)
|
|
add_outbox(conn, job["job_id"], attachment["id"], job["pipeline_version"], next_attempt, job["created_at"])
|
|
audit(conn, job["job_id"], attachment["id"], "retry", {"attempt": next_attempt, "delay_seconds": round(delay, 3)})
|
|
|
|
|
|
def _fail(conn, job, attachment, code: str, error_class: str, message: str) -> None:
|
|
conn.execute(
|
|
"""
|
|
UPDATE jobs
|
|
SET status = 'failed', error_code = %s, error_class = %s, last_error = %s,
|
|
lease_until = NULL, lease_owner = NULL, next_attempt_at = NULL, updated_at = clock_timestamp()
|
|
WHERE job_id = %s
|
|
""",
|
|
(code, error_class, _safe(message), job["job_id"]),
|
|
)
|
|
if attachment["ready_pipeline_version"] is not None:
|
|
# A failed newer attempt must not discard an already published result.
|
|
conn.execute(
|
|
"""
|
|
UPDATE attachments
|
|
SET status = 'ready', error_code = NULL, pipeline_version = ready_pipeline_version,
|
|
updated_at = clock_timestamp()
|
|
WHERE id = %s AND ready_pipeline_version IS NOT NULL
|
|
""",
|
|
(attachment["id"],),
|
|
)
|
|
else:
|
|
conn.execute(
|
|
"""
|
|
UPDATE attachments
|
|
SET status = 'failed', error_code = %s, updated_at = clock_timestamp()
|
|
WHERE id = %s AND pipeline_version = %s AND status <> 'ready'
|
|
""",
|
|
(code, attachment["id"], job["pipeline_version"]),
|
|
)
|
|
conn.execute(
|
|
"""
|
|
INSERT INTO failed_jobs (job_id, attachment_id, pipeline_version, attempt, error_code, error_class, safe_message)
|
|
VALUES (%s, %s, %s, %s, %s, %s, %s)
|
|
""",
|
|
(job["job_id"], attachment["id"], job["pipeline_version"], job["attempt"], code, error_class, _safe(message)),
|
|
)
|
|
audit(conn, job["job_id"], attachment["id"], "failed", {"error_code": code, "error_class": error_class})
|
|
publish_dlq(
|
|
{
|
|
"schema_version": 1,
|
|
"job_id": str(job["job_id"]),
|
|
"type": "attachment.process.v1",
|
|
"attachment_id": str(attachment["id"]),
|
|
"pipeline_version": int(job["pipeline_version"]),
|
|
"attempt": int(job["attempt"]),
|
|
"created_at": message_created(job),
|
|
}
|
|
)
|
|
|
|
|
|
def message_created(job) -> str:
|
|
from app.timeutil import iso as format_iso
|
|
|
|
value = job["created_at"]
|
|
return value if isinstance(value, str) else format_iso(value)
|
|
|
|
|
|
def _put_original(store, attachment, data: bytes) -> None:
|
|
existing = store.get(attachment["original_key"])
|
|
digest = hashlib.sha256(data).hexdigest()
|
|
if digest != attachment["sha256"]:
|
|
raise PermanentJobError("processing_failed", "checksum mismatch")
|
|
if existing is None:
|
|
store.put(attachment["original_key"], data, attachment["content_type"])
|
|
return
|
|
if hashlib.sha256(existing).hexdigest() != attachment["sha256"]:
|
|
raise TransientJobError("stored original does not match the accepted checksum")
|
|
|
|
|
|
def _store_variants(store, attachment, version: int, rendered: dict, max_pixels: int) -> list[dict]:
|
|
stored = []
|
|
for name in VARIANT_NAMES:
|
|
item = rendered[name]
|
|
key = f"variants/{attachment['id']}/v{version}/{name}"
|
|
existing = store.get(key)
|
|
if existing is None:
|
|
store.put(key, item["bytes"], item["content_type"])
|
|
blob = item["bytes"]
|
|
width, height = item["width"], item["height"]
|
|
else:
|
|
blob = existing
|
|
if hashlib.sha256(blob).hexdigest() == hashlib.sha256(item["bytes"]).hexdigest():
|
|
width, height = item["width"], item["height"]
|
|
else:
|
|
probed = decode_image(blob, max_pixels)
|
|
width, height = probed.size
|
|
stored.append(
|
|
{
|
|
"name": name,
|
|
"content_type": item["content_type"] if existing is None else _content_type_for(name, blob, item["content_type"]),
|
|
"size_bytes": len(blob),
|
|
"width": width,
|
|
"height": height,
|
|
"sha256": hashlib.sha256(blob).hexdigest(),
|
|
"object_key": key,
|
|
}
|
|
)
|
|
return stored
|
|
|
|
|
|
def _content_type_for(name: str, blob: bytes, encoded_type: str) -> str:
|
|
if blob.startswith(b"\xff\xd8"):
|
|
return "image/jpeg"
|
|
if len(blob) >= 12 and blob[:4] == b"RIFF" and blob[8:12] == b"WEBP":
|
|
return "image/webp"
|
|
return encoded_type
|
|
|
|
|
|
def _source_bytes(store, attachment) -> bytes:
|
|
data = store.get(attachment["staging_key"])
|
|
if data is None:
|
|
data = store.get(attachment["original_key"])
|
|
if data is None:
|
|
raise TransientJobError("source object is not available yet")
|
|
return data
|
|
|
|
|
|
def _commit_success(job, attachment, variants: list[dict], worker_id: str, crash_after_commit: bool) -> bool:
|
|
"""Returns True when this version became the published result."""
|
|
settings, store, _publisher = runtime.require()
|
|
with connection() as conn:
|
|
current = conn.execute("SELECT pipeline_version, ready_pipeline_version FROM attachments WHERE id = %s FOR UPDATE", (attachment["id"],)).fetchone()
|
|
for item in variants:
|
|
conn.execute(
|
|
"""
|
|
INSERT INTO attachment_variants (
|
|
attachment_id, pipeline_version, name, content_type, size_bytes, width, height, sha256, object_key
|
|
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
|
|
ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING
|
|
""",
|
|
(
|
|
attachment["id"],
|
|
job["pipeline_version"],
|
|
item["name"],
|
|
item["content_type"],
|
|
item["size_bytes"],
|
|
item["width"],
|
|
item["height"],
|
|
item["sha256"],
|
|
item["object_key"],
|
|
),
|
|
)
|
|
updated = conn.execute(
|
|
"""
|
|
UPDATE attachments
|
|
SET status = 'ready', ready_pipeline_version = %s, error_code = NULL, updated_at = clock_timestamp()
|
|
WHERE id = %s
|
|
AND pipeline_version = %s
|
|
AND (ready_pipeline_version IS NULL OR ready_pipeline_version <= %s)
|
|
RETURNING id
|
|
""",
|
|
(job["pipeline_version"], attachment["id"], job["pipeline_version"], job["pipeline_version"]),
|
|
).fetchone()
|
|
conn.execute(
|
|
"""
|
|
UPDATE jobs
|
|
SET status = 'ready', error_code = NULL, error_class = NULL, last_error = NULL,
|
|
lease_until = NULL, lease_owner = NULL, next_attempt_at = NULL, updated_at = clock_timestamp()
|
|
WHERE job_id = %s
|
|
""",
|
|
(job["job_id"],),
|
|
)
|
|
if crash_after_commit:
|
|
_write_fault(conn, {"mode": None, "armed": False})
|
|
audit(conn, job["job_id"], attachment["id"], "fault_after_commit", {"worker": worker_id})
|
|
audit(
|
|
conn,
|
|
job["job_id"],
|
|
attachment["id"],
|
|
"committed",
|
|
{"pipeline_version": job["pipeline_version"], "published": bool(updated), "variants": len(variants)},
|
|
)
|
|
published = updated is not None and current is not None
|
|
if crash_after_commit:
|
|
os._exit(75)
|
|
if published:
|
|
try:
|
|
store.delete(attachment["staging_key"])
|
|
with connection() as conn:
|
|
conn.execute(
|
|
"UPDATE attachments SET staging_deleted_at = clock_timestamp() WHERE id = %s AND status = 'ready'",
|
|
(attachment["id"],),
|
|
)
|
|
except Exception:
|
|
log.warning("staging cleanup deferred for %s", attachment["id"])
|
|
return published
|
|
|
|
|
|
def _execute(message: dict, attachment: dict, worker_id: str, fault: str | None) -> None:
|
|
settings, store, _publisher = runtime.require()
|
|
if fault == "before_effect":
|
|
_crash("before_effect", message["job_id"], attachment["id"])
|
|
deadline = time.monotonic() + settings.job_timeout_seconds
|
|
data = _source_bytes(store, attachment)
|
|
if len(data) > settings.max_upload_bytes:
|
|
raise PermanentJobError("invalid_image", "object exceeds the byte limit")
|
|
if time.monotonic() > deadline:
|
|
raise TransientJobError("processing deadline exceeded")
|
|
try:
|
|
image = _decode_limited(data, settings.max_image_pixels, max(1, deadline - time.monotonic()))
|
|
params = pipeline_params(int(attachment_version(message)))
|
|
rendered = render_variants(image, params)
|
|
except ImageRejected as exc:
|
|
raise PermanentJobError("invalid_image", exc.message) from exc
|
|
if time.monotonic() > deadline:
|
|
raise TransientJobError("processing deadline exceeded")
|
|
_put_original(store, attachment, data)
|
|
variants = _store_variants(store, attachment, message["pipeline_version"], rendered, settings.max_image_pixels)
|
|
if fault == "after_s3":
|
|
_crash("after_s3", message["job_id"], attachment["id"])
|
|
_commit_success(message_as_job(message, attachment), attachment, variants, worker_id, fault == "after_commit")
|
|
|
|
|
|
def attachment_version(message: dict) -> int:
|
|
return int(message["pipeline_version"])
|
|
|
|
|
|
def message_as_job(message: dict, attachment: dict) -> dict:
|
|
return {
|
|
"job_id": message["job_id"],
|
|
"pipeline_version": message["pipeline_version"],
|
|
"attempt": message["attempt"],
|
|
"created_at": message["created_at"],
|
|
"attachment_id": attachment["id"],
|
|
}
|
|
|
|
|
|
def _decode_limited(data: bytes, max_pixels: int, timeout: float):
|
|
box: dict = {}
|
|
|
|
def run() -> None:
|
|
try:
|
|
box["image"] = decode_image(data, max_pixels)
|
|
except Exception as exc: # noqa: BLE001 - transferred to the caller
|
|
box["error"] = exc
|
|
|
|
thread = threading.Thread(target=run, daemon=True)
|
|
thread.start()
|
|
thread.join(timeout)
|
|
if thread.is_alive():
|
|
raise TransientJobError("decode timed out")
|
|
if "error" in box:
|
|
raise box["error"]
|
|
return box["image"]
|
|
|
|
|
|
def _handle_known_error(message: dict, exc: Exception) -> None:
|
|
with connection() as conn:
|
|
job, attachment = _load(conn, message)
|
|
if job is None or attachment is None or job["status"] in {"ready", "failed", "superseded"}:
|
|
return
|
|
if isinstance(exc, PermanentJobError):
|
|
_fail(conn, job, attachment, exc.code, "permanent", exc.message)
|
|
elif isinstance(exc, TransientJobError):
|
|
_schedule_retry(conn, job, attachment, exc.message)
|
|
else:
|
|
_schedule_retry(conn, job, attachment, "temporary processing failure")
|
|
|
|
|
|
def process_delivery(raw: bytes, redelivered: bool, worker_id: str) -> str:
|
|
"""Returns 'ack' or 'nack'. Ack happens only after the durable decision is committed."""
|
|
import json
|
|
|
|
try:
|
|
body = json.loads(raw.decode("utf-8"))
|
|
message = _parse_message(body)
|
|
except (UnicodeError, json.JSONDecodeError, PermanentJobError):
|
|
log.warning("discarding a job message that does not match the contract")
|
|
return "ack"
|
|
try:
|
|
fault = _consume_counted_fault()
|
|
except Exception:
|
|
log.warning("fault control unreadable", exc_info=True)
|
|
return "nack"
|
|
if fault == "transient":
|
|
_handle_known_error(message, TransientJobError("injected transient failure"))
|
|
return "ack"
|
|
if fault == "permanent":
|
|
_handle_known_error(message, PermanentJobError("invalid_image", "injected permanent failure"))
|
|
return "ack"
|
|
runs = 2 if fault == "duplicate" else 1
|
|
for _ in range(runs):
|
|
outcome = _process_once(message, redelivered, worker_id, fault if runs == 1 else None)
|
|
if outcome == "nack":
|
|
return "nack"
|
|
fault = None
|
|
return "ack"
|
|
|
|
|
|
def _process_once(message: dict, redelivered: bool, worker_id: str, fault: str | None) -> str:
|
|
settings, _store, _publisher = runtime.require()
|
|
with connection() as conn:
|
|
job, attachment = _load(conn, message)
|
|
if job is None or attachment is None:
|
|
audit(conn, message["job_id"], message["attachment_id"], "missing", {"redelivered": redelivered})
|
|
return "ack"
|
|
if job["status"] in {"ready", "superseded", "failed"}:
|
|
_finish_noop(conn, job, attachment, "noop_" + job["status"], redelivered)
|
|
return "ack"
|
|
if message["attempt"] != int(job["attempt"]):
|
|
_finish_noop(conn, job, attachment, "noop_attempt", redelivered)
|
|
return "ack"
|
|
newer_ready = (
|
|
attachment["ready_pipeline_version"] is not None
|
|
and int(attachment["ready_pipeline_version"]) > int(job["pipeline_version"])
|
|
)
|
|
newer_desired = int(attachment["pipeline_version"]) > int(job["pipeline_version"])
|
|
if newer_ready or newer_desired:
|
|
_supersede(conn, job, attachment, redelivered)
|
|
return "ack"
|
|
if _acquire(conn, job, worker_id, settings.job_timeout_seconds) is None:
|
|
return "nack"
|
|
conn.execute(
|
|
"""
|
|
UPDATE attachments
|
|
SET status = 'processing', updated_at = clock_timestamp()
|
|
WHERE id = %s AND pipeline_version = %s AND status IN ('queued', 'processing')
|
|
""",
|
|
(attachment["id"], job["pipeline_version"]),
|
|
)
|
|
audit(conn, job["job_id"], attachment["id"], "leased", {"worker": worker_id, "redelivered": redelivered})
|
|
try:
|
|
_execute(message, attachment, worker_id, fault)
|
|
except (PermanentJobError, TransientJobError, ImageRejected) as exc:
|
|
if isinstance(exc, ImageRejected):
|
|
exc = PermanentJobError("invalid_image", exc.message)
|
|
_handle_known_error(message, exc)
|
|
except Exception:
|
|
log.warning("unexpected pipeline error", exc_info=True)
|
|
_handle_known_error(message, TransientJobError("temporary processing failure"))
|
|
return "ack"
|
|
|
|
|
|
def recover_expired_leases() -> int:
|
|
recovered = 0
|
|
with connection() as conn:
|
|
rows = conn.execute(
|
|
"""
|
|
UPDATE jobs
|
|
SET status = 'queued', lease_until = NULL, lease_owner = NULL, updated_at = clock_timestamp()
|
|
WHERE status = 'processing' AND lease_until IS NOT NULL AND lease_until < clock_timestamp()
|
|
RETURNING job_id, attachment_id, pipeline_version, attempt, created_at
|
|
"""
|
|
).fetchall()
|
|
for job in rows:
|
|
conn.execute(
|
|
"""
|
|
UPDATE attachments
|
|
SET status = 'queued', updated_at = clock_timestamp()
|
|
WHERE id = %s AND pipeline_version = %s AND status = 'processing'
|
|
""",
|
|
(job["attachment_id"], job["pipeline_version"]),
|
|
)
|
|
add_outbox(conn, job["job_id"], job["attachment_id"], job["pipeline_version"], job["attempt"], job["created_at"])
|
|
audit(conn, job["job_id"], job["attachment_id"], "lease_expired", {})
|
|
recovered += 1
|
|
if recovered:
|
|
log.info("recovered %s expired lease(s)", recovered)
|
|
return recovered
|
|
|
|
|
|
def sweep_staging() -> None:
|
|
settings, store, _publisher = runtime.require()
|
|
with connection() as conn:
|
|
ready = conn.execute(
|
|
"""
|
|
SELECT id, staging_key FROM attachments
|
|
WHERE staging_deleted_at IS NULL AND status = 'ready'
|
|
LIMIT 25
|
|
"""
|
|
).fetchall()
|
|
failed = conn.execute(
|
|
"""
|
|
SELECT id, staging_key FROM attachments
|
|
WHERE staging_deleted_at IS NULL AND status = 'failed'
|
|
AND updated_at < clock_timestamp() - make_interval(secs => %s)
|
|
LIMIT 25
|
|
""",
|
|
(settings.staging_retention_seconds,),
|
|
).fetchall()
|
|
for row in list(ready) + list(failed):
|
|
try:
|
|
store.delete(row["staging_key"])
|
|
except Exception:
|
|
log.warning("staging delete failed")
|
|
continue
|
|
with connection() as conn:
|
|
conn.execute(
|
|
"UPDATE attachments SET staging_deleted_at = clock_timestamp() WHERE id = %s",
|
|
(row["id"],),
|
|
)
|