forked from mtusi2026/mtusi2026lab6
628 lines
25 KiB
Python
628 lines
25 KiB
Python
import os
|
|
import io
|
|
import json
|
|
import time
|
|
import signal
|
|
import random
|
|
import hashlib
|
|
import logging
|
|
from datetime import datetime, timezone
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
import psycopg2
|
|
import psycopg2.extras
|
|
import redis
|
|
import boto3
|
|
import botocore.exceptions
|
|
from botocore.client import Config
|
|
from PIL import Image, ImageOps, UnidentifiedImageError
|
|
|
|
from opentelemetry import metrics, trace
|
|
from opentelemetry._logs import set_logger_provider
|
|
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
|
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
|
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
|
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
|
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
|
from opentelemetry.sdk.metrics import MeterProvider
|
|
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
|
from opentelemetry.sdk.resources import Resource
|
|
from opentelemetry.sdk.trace import TracerProvider
|
|
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
|
from opentelemetry.propagate import extract
|
|
|
|
|
|
class JsonLogFormatter(logging.Formatter):
|
|
def format(self, record: logging.LogRecord) -> str:
|
|
span = trace.get_current_span()
|
|
ctx = span.get_span_context() if span else None
|
|
trace_id = format(ctx.trace_id, "032x") if ctx and ctx.is_valid else None
|
|
span_id = format(ctx.span_id, "016x") if ctx and ctx.is_valid else None
|
|
|
|
payload = {
|
|
"ts": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
|
|
"level": record.levelname,
|
|
"logger": record.name,
|
|
"service": os.getenv("OTEL_SERVICE_NAME", "messenger-worker"),
|
|
"msg": record.getMessage(),
|
|
}
|
|
if trace_id and span_id:
|
|
payload["trace_id"] = trace_id
|
|
payload["span_id"] = span_id
|
|
|
|
for attr in ("method", "path", "status_code", "job_id", "attachment_id", "job_status", "attempt", "error_code"):
|
|
if hasattr(record, attr):
|
|
payload[attr] = getattr(record, attr)
|
|
|
|
if record.exc_info:
|
|
payload["exception"] = record.exc_info[0].__name__ if record.exc_info[0] else "Exception"
|
|
|
|
return json.dumps(payload, ensure_ascii=False)
|
|
|
|
|
|
def setup_worker_telemetry():
|
|
service_name = os.getenv("OTEL_SERVICE_NAME", "messenger-worker")
|
|
endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318").rstrip("/")
|
|
env_attrs = os.getenv("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment.name=lab")
|
|
|
|
attrs = {"service.name": service_name}
|
|
for part in env_attrs.split(","):
|
|
if "=" in part:
|
|
k, v = part.split("=", 1)
|
|
attrs[k.strip()] = v.strip()
|
|
|
|
resource = Resource.create(attrs)
|
|
|
|
tracer_provider = TracerProvider(resource=resource)
|
|
tracer_provider.add_span_processor(
|
|
BatchSpanProcessor(OTLPSpanExporter(endpoint=f"{endpoint}/v1/traces"))
|
|
)
|
|
trace.set_tracer_provider(tracer_provider)
|
|
|
|
metric_reader = PeriodicExportingMetricReader(
|
|
OTLPMetricExporter(endpoint=f"{endpoint}/v1/metrics"),
|
|
export_interval_millis=5000,
|
|
)
|
|
meter_provider = MeterProvider(resource=resource, metric_readers=[metric_reader])
|
|
metrics.set_meter_provider(meter_provider)
|
|
|
|
logger_provider = LoggerProvider(resource=resource)
|
|
logger_provider.add_log_record_processor(
|
|
BatchLogRecordProcessor(OTLPLogExporter(endpoint=f"{endpoint}/v1/logs"))
|
|
)
|
|
set_logger_provider(logger_provider)
|
|
|
|
root_logger = logging.getLogger()
|
|
root_logger.setLevel(logging.INFO)
|
|
root_logger.handlers.clear()
|
|
|
|
stream_handler = logging.StreamHandler()
|
|
stream_handler.setFormatter(JsonLogFormatter())
|
|
root_logger.addHandler(stream_handler)
|
|
|
|
otel_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
|
|
otel_handler.setFormatter(JsonLogFormatter())
|
|
root_logger.addHandler(otel_handler)
|
|
|
|
return trace.get_tracer(service_name), metrics.get_meter(service_name)
|
|
|
|
|
|
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger")
|
|
REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0")
|
|
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333")
|
|
S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
|
|
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY_ID", "appuser")
|
|
S3_SECRET_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "appsecretpassword")
|
|
|
|
STREAM_NAME = os.getenv("BROKER_STREAM", "attachment_jobs")
|
|
DLQ_STREAM_NAME = f"{STREAM_NAME}_dlq"
|
|
GROUP_NAME = os.getenv("BROKER_GROUP", "attachment_workers")
|
|
CONSUMER_NAME = os.getenv("HOSTNAME", "worker-1")
|
|
JOB_MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
|
|
MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000"))
|
|
JOB_TIMEOUT_SECONDS = int(os.getenv("JOB_TIMEOUT_SECONDS", "30"))
|
|
WORKER_CONCURRENCY = int(os.getenv("WORKER_CONCURRENCY", "4"))
|
|
|
|
CLAIM_MIN_IDLE_MS = (JOB_TIMEOUT_SECONDS + 20) * 1000
|
|
Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS
|
|
|
|
redis_client = redis.Redis.from_url(REDIS_URL, decode_responses=True)
|
|
s3_client = boto3.client(
|
|
"s3", endpoint_url=S3_ENDPOINT,
|
|
aws_access_key_id=S3_ACCESS_KEY, aws_secret_access_key=S3_SECRET_KEY,
|
|
config=Config(signature_version="s3v4"), region_name="us-east-1"
|
|
)
|
|
|
|
|
|
def observe_queue_metrics(options):
|
|
try:
|
|
q_len = redis_client.xlen(STREAM_NAME)
|
|
entries = redis_client.xrange(STREAM_NAME, count=1)
|
|
oldest_age = 0.0
|
|
if entries:
|
|
msg_id, fields = entries[0]
|
|
try:
|
|
payload = json.loads(fields.get("payload", "{}"))
|
|
created_str = payload.get("created_at")
|
|
if created_str:
|
|
created_dt = datetime.fromisoformat(created_str.replace("Z", "+00:00"))
|
|
oldest_age = max(0.0, (datetime.now(timezone.utc) - created_dt).total_seconds())
|
|
except Exception:
|
|
ms = int(msg_id.split("-")[0])
|
|
oldest_age = max(0.0, time.time() - (ms / 1000.0))
|
|
yield metrics.Observation(value=q_len, attributes={"stream": STREAM_NAME})
|
|
except Exception:
|
|
yield metrics.Observation(value=0, attributes={"stream": STREAM_NAME})
|
|
|
|
|
|
def observe_oldest_job_age(options):
|
|
try:
|
|
entries = redis_client.xrange(STREAM_NAME, count=1)
|
|
if not entries:
|
|
yield metrics.Observation(value=0.0, attributes={"stream": STREAM_NAME})
|
|
return
|
|
msg_id, fields = entries[0]
|
|
try:
|
|
payload = json.loads(fields.get("payload", "{}"))
|
|
created_str = payload.get("created_at")
|
|
if created_str:
|
|
created_dt = datetime.fromisoformat(created_str.replace("Z", "+00:00"))
|
|
age = max(0.0, (datetime.now(timezone.utc) - created_dt).total_seconds())
|
|
yield metrics.Observation(value=age, attributes={"stream": STREAM_NAME})
|
|
return
|
|
except Exception:
|
|
pass
|
|
ms = int(msg_id.split("-")[0])
|
|
yield metrics.Observation(value=max(0.0, time.time() - (ms / 1000.0)), attributes={"stream": STREAM_NAME})
|
|
except Exception:
|
|
yield metrics.Observation(value=0.0, attributes={"stream": STREAM_NAME})
|
|
|
|
|
|
tracer, meter = setup_worker_telemetry()
|
|
logger = logging.getLogger("attachment-worker")
|
|
|
|
worker_jobs_counter = meter.create_counter(
|
|
name="worker_jobs_total",
|
|
description="Completed and failed attachment jobs",
|
|
unit="1",
|
|
)
|
|
worker_retries_counter = meter.create_counter(
|
|
name="worker_job_retries_total",
|
|
description="Transient retry attempts scheduled for attachment jobs",
|
|
unit="1",
|
|
)
|
|
worker_duration_histogram = meter.create_histogram(
|
|
name="worker_job_duration_seconds",
|
|
description="Attachment job processing duration in seconds",
|
|
unit="s",
|
|
)
|
|
worker_jobs_in_progress = meter.create_up_down_counter(
|
|
name="worker_jobs_in_progress",
|
|
description="Jobs currently being processed",
|
|
unit="1",
|
|
)
|
|
meter.create_observable_gauge(
|
|
name="worker_queue_depth",
|
|
callbacks=[observe_queue_metrics],
|
|
description="Number of messages currently in attachment jobs stream",
|
|
unit="1",
|
|
)
|
|
meter.create_observable_gauge(
|
|
name="worker_oldest_job_age_seconds",
|
|
callbacks=[observe_oldest_job_age],
|
|
description="Age of the oldest message in attachment jobs stream in seconds",
|
|
unit="s",
|
|
)
|
|
|
|
running = True
|
|
|
|
|
|
def handle_sigterm(signum, frame):
|
|
global running
|
|
logger.info("Received shutdown signal, finishing in-flight jobs...")
|
|
running = False
|
|
|
|
|
|
signal.signal(signal.SIGTERM, handle_sigterm)
|
|
signal.signal(signal.SIGINT, handle_sigterm)
|
|
|
|
|
|
def get_db_connection():
|
|
return psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor)
|
|
|
|
|
|
def ensure_consumer_groups():
|
|
while running:
|
|
try:
|
|
redis_client.xgroup_create(STREAM_NAME, GROUP_NAME, id="0", mkstream=True)
|
|
redis_client.xgroup_create(DLQ_STREAM_NAME, GROUP_NAME, id="0", mkstream=True)
|
|
logger.info("Consumer groups ensured on '%s' and '%s'", STREAM_NAME, DLQ_STREAM_NAME)
|
|
return
|
|
except redis.exceptions.ResponseError as e:
|
|
if "BUSYGROUP" in str(e):
|
|
return
|
|
raise
|
|
except Exception as e:
|
|
logger.warning("Waiting for Redis: %s", type(e).__name__)
|
|
time.sleep(1)
|
|
|
|
|
|
def prepare_base_image(raw_bytes):
|
|
img = Image.open(io.BytesIO(raw_bytes))
|
|
img = ImageOps.exif_transpose(img)
|
|
if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info):
|
|
rgba = img.convert("RGBA")
|
|
bg = Image.new("RGB", rgba.size, (255, 255, 255))
|
|
bg.paste(rgba, mask=rgba.split()[3])
|
|
img = bg
|
|
elif img.mode != "RGB":
|
|
img = img.convert("RGB")
|
|
return img
|
|
|
|
|
|
def generate_grade4_variants(raw_bytes):
|
|
base_img = prepare_base_image(raw_bytes)
|
|
orig_w, orig_h = base_img.size
|
|
meta = []
|
|
blobs = {}
|
|
|
|
thumb = base_img.copy()
|
|
if orig_w <= 256 and orig_h <= 256:
|
|
t_w, t_h = orig_w, orig_h
|
|
else:
|
|
s = min(256.0 / orig_w, 256.0 / orig_h)
|
|
t_w, t_h = max(1, int(orig_w * s)), max(1, int(orig_h * s))
|
|
if (t_w, t_h) != (orig_w, orig_h):
|
|
thumb = thumb.resize((t_w, t_h), Image.Resampling.LANCZOS)
|
|
buf = io.BytesIO()
|
|
thumb.save(buf, format="JPEG", quality=85, optimize=True)
|
|
tb = buf.getvalue()
|
|
blobs["thumbnail"] = tb
|
|
meta.append({
|
|
"name": "thumbnail", "content_type": "image/jpeg",
|
|
"size_bytes": len(tb), "width": t_w, "height": t_h,
|
|
"sha256": hashlib.sha256(tb).hexdigest()
|
|
})
|
|
|
|
opt = base_img.copy()
|
|
if orig_w <= 1280 and orig_h <= 1280:
|
|
o_w, o_h = orig_w, orig_h
|
|
else:
|
|
s = min(1280.0 / orig_w, 1280.0 / orig_h)
|
|
o_w, o_h = max(1, int(orig_w * s)), max(1, int(orig_h * s))
|
|
if (o_w, o_h) != (orig_w, orig_h):
|
|
opt = opt.resize((o_w, o_h), Image.Resampling.LANCZOS)
|
|
buf = io.BytesIO()
|
|
opt.save(buf, format="WEBP", quality=85)
|
|
ob = buf.getvalue()
|
|
blobs["optimized"] = ob
|
|
meta.append({
|
|
"name": "optimized", "content_type": "image/webp",
|
|
"size_bytes": len(ob), "width": o_w, "height": o_h,
|
|
"sha256": hashlib.sha256(ob).hexdigest()
|
|
})
|
|
|
|
c = min(orig_w, orig_h)
|
|
left, top = (orig_w - c) // 2, (orig_h - c) // 2
|
|
crop = base_img.crop((left, top, left + c, top + c))
|
|
if c != 128:
|
|
crop = crop.resize((128, 128), Image.Resampling.LANCZOS)
|
|
buf = io.BytesIO()
|
|
crop.save(buf, format="JPEG", quality=85, optimize=True)
|
|
cb = buf.getvalue()
|
|
blobs["crop"] = cb
|
|
meta.append({
|
|
"name": "crop", "content_type": "image/jpeg",
|
|
"size_bytes": len(cb), "width": 128, "height": 128,
|
|
"sha256": hashlib.sha256(cb).hexdigest()
|
|
})
|
|
|
|
return meta, blobs
|
|
|
|
|
|
def mark_failed(attachment_id, error_code):
|
|
now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
|
conn = get_db_connection()
|
|
try:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"UPDATE attachments SET status='failed', error_code=%s, updated_at=%s "
|
|
"WHERE id=%s AND status != 'ready'",
|
|
(error_code, now, attachment_id)
|
|
)
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
TRANSIENT_ERRORS = (
|
|
TimeoutError,
|
|
botocore.exceptions.EndpointConnectionError,
|
|
botocore.exceptions.ReadTimeoutError,
|
|
psycopg2.OperationalError,
|
|
redis.exceptions.ConnectionError,
|
|
)
|
|
|
|
|
|
def _record_job_outcome(span, status: str, elapsed: float):
|
|
span.set_attribute("job.outcome", status)
|
|
worker_jobs_counter.add(1, {"status": status})
|
|
worker_duration_histogram.record(elapsed, {"status": status})
|
|
|
|
|
|
def process_job(message_id, fields):
|
|
job_start = time.monotonic()
|
|
try:
|
|
job = json.loads(fields.get("payload", "{}"))
|
|
except Exception:
|
|
logger.error("Malformed job %s, ack to drop", message_id)
|
|
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
|
return
|
|
|
|
ctx = extract(job)
|
|
|
|
with tracer.start_as_current_span("worker.process_attachment", context=ctx) as span:
|
|
worker_jobs_in_progress.add(1)
|
|
try:
|
|
attachment_id = job.get("attachment_id")
|
|
job_id = job.get("job_id")
|
|
pipeline_version = int(job.get("pipeline_version", 1))
|
|
attempt = int(job.get("attempt", 1))
|
|
|
|
span.set_attribute("job.pipeline_version", pipeline_version)
|
|
span.set_attribute("job.attempt", attempt)
|
|
|
|
if not attachment_id:
|
|
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
|
_record_job_outcome(span, "failed", time.monotonic() - job_start)
|
|
return
|
|
|
|
conn = get_db_connection()
|
|
try:
|
|
with conn.cursor() as cur:
|
|
cur.execute("SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,))
|
|
att = cur.fetchone()
|
|
if not att:
|
|
conn.commit()
|
|
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
|
_record_job_outcome(span, "failed", time.monotonic() - job_start)
|
|
return
|
|
if att["status"] == "ready" or att["pipeline_version"] > pipeline_version:
|
|
conn.commit()
|
|
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
|
_record_job_outcome(span, "success", time.monotonic() - job_start)
|
|
return
|
|
if att["status"] == "failed":
|
|
conn.commit()
|
|
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
|
_record_job_outcome(span, "failed", time.monotonic() - job_start)
|
|
return
|
|
|
|
if att["status"] == "processing":
|
|
updated = att["updated_at"]
|
|
if isinstance(updated, str):
|
|
updated = datetime.fromisoformat(updated.replace("Z", "+00:00"))
|
|
age_seconds = (datetime.now(timezone.utc) - updated).total_seconds()
|
|
if age_seconds < (JOB_TIMEOUT_SECONDS + 10):
|
|
conn.commit()
|
|
logger.info(
|
|
"Attachment already processing, ack duplicate",
|
|
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "processing", "attempt": attempt},
|
|
)
|
|
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
|
return
|
|
|
|
now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
|
cur.execute(
|
|
"UPDATE attachments SET status='processing', attempt=%s, updated_at=%s WHERE id=%s",
|
|
(attempt, now, attachment_id)
|
|
)
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
chat_id = str(att["chat_id"])
|
|
staging_key = f"staging/{chat_id}/{attachment_id}/original"
|
|
final_original_key = f"attachments/{chat_id}/{attachment_id}/original"
|
|
|
|
start_time = time.monotonic()
|
|
|
|
def check_timeout():
|
|
if time.monotonic() - start_time > JOB_TIMEOUT_SECONDS:
|
|
raise TimeoutError("Job exceeded time limit")
|
|
|
|
try:
|
|
try:
|
|
obj = s3_client.get_object(Bucket=S3_BUCKET, Key=staging_key)
|
|
except Exception:
|
|
obj = s3_client.get_object(Bucket=S3_BUCKET, Key=final_original_key)
|
|
raw_bytes = obj["Body"].read()
|
|
check_timeout()
|
|
|
|
if hashlib.sha256(raw_bytes).hexdigest() != att["sha256"] or len(raw_bytes) != att["size_bytes"]:
|
|
raise ValueError("Staging checksum mismatch")
|
|
|
|
variants_meta, variants_bytes = generate_grade4_variants(raw_bytes)
|
|
check_timeout()
|
|
|
|
s3_client.put_object(
|
|
Bucket=S3_BUCKET, Key=final_original_key,
|
|
Body=raw_bytes, ContentType=att["content_type"]
|
|
)
|
|
for var_name, var_bytes in variants_bytes.items():
|
|
ct = "image/webp" if var_name == "optimized" else "image/jpeg"
|
|
s3_client.put_object(
|
|
Bucket=S3_BUCKET,
|
|
Key=f"attachments/{chat_id}/{attachment_id}/{var_name}",
|
|
Body=var_bytes, ContentType=ct
|
|
)
|
|
check_timeout()
|
|
|
|
ready_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
|
conn = get_db_connection()
|
|
try:
|
|
with conn.cursor() as cur:
|
|
cur.execute("""
|
|
UPDATE attachments SET status='ready', variants=%s::jsonb,
|
|
error_code=NULL, updated_at=%s
|
|
WHERE id=%s AND pipeline_version <= %s
|
|
""", (json.dumps(variants_meta), ready_at, attachment_id, pipeline_version))
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
|
_record_job_outcome(span, "success", time.monotonic() - job_start)
|
|
logger.info(
|
|
"Job completed: attachment ready",
|
|
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "ready", "attempt": attempt},
|
|
)
|
|
|
|
try:
|
|
s3_client.delete_object(Bucket=S3_BUCKET, Key=staging_key)
|
|
except Exception:
|
|
pass
|
|
|
|
except (UnidentifiedImageError, Image.DecompressionBombError):
|
|
logger.error(
|
|
"Permanent image error",
|
|
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "invalid_image"},
|
|
)
|
|
mark_failed(attachment_id, "invalid_image")
|
|
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
|
_record_job_outcome(span, "failed", time.monotonic() - job_start)
|
|
|
|
except ValueError as e:
|
|
logger.error(
|
|
"Checksum error",
|
|
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "invalid_checksum"},
|
|
)
|
|
mark_failed(attachment_id, "invalid_checksum")
|
|
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
|
_record_job_outcome(span, "failed", time.monotonic() - job_start)
|
|
|
|
except psycopg2.errors.DataError:
|
|
logger.error(
|
|
"DB data error",
|
|
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "data_error"},
|
|
)
|
|
mark_failed(attachment_id, "data_error")
|
|
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
|
_record_job_outcome(span, "failed", time.monotonic() - job_start)
|
|
|
|
except TRANSIENT_ERRORS as e:
|
|
logger.warning(
|
|
"Transient error during job processing",
|
|
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "processing", "attempt": attempt},
|
|
)
|
|
if attempt >= JOB_MAX_ATTEMPTS:
|
|
mark_failed(attachment_id, "retry_exhausted")
|
|
try:
|
|
dlq_job = dict(job)
|
|
dlq_job["attempt"] = 1
|
|
redis_client.xadd(
|
|
DLQ_STREAM_NAME,
|
|
{"payload": json.dumps(dlq_job), "error": type(e).__name__}
|
|
)
|
|
except Exception:
|
|
pass
|
|
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
|
_record_job_outcome(span, "failed", time.monotonic() - job_start)
|
|
else:
|
|
worker_retries_counter.add(1, {"reason": type(e).__name__})
|
|
delay = min(2 ** attempt, 30) + random.uniform(0, 1)
|
|
time.sleep(delay)
|
|
next_job = dict(job)
|
|
next_job["attempt"] = attempt + 1
|
|
redis_client.xadd(STREAM_NAME, {"payload": json.dumps(next_job)})
|
|
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
|
|
|
except Exception:
|
|
logger.error(
|
|
"Unknown error during job processing",
|
|
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "processing_failed"},
|
|
)
|
|
mark_failed(attachment_id, "processing_failed")
|
|
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
|
_record_job_outcome(span, "failed", time.monotonic() - job_start)
|
|
finally:
|
|
worker_jobs_in_progress.add(-1)
|
|
|
|
|
|
def run_staging_cleanup():
|
|
try:
|
|
cutoff = datetime.now(timezone.utc).timestamp() - 3600
|
|
paginator = s3_client.get_paginator("list_objects_v2")
|
|
for page in paginator.paginate(Bucket=S3_BUCKET, Prefix="staging/"):
|
|
for obj in page.get("Contents", []):
|
|
if obj["LastModified"].timestamp() < cutoff:
|
|
try:
|
|
s3_client.delete_object(Bucket=S3_BUCKET, Key=obj["Key"])
|
|
except Exception:
|
|
pass
|
|
except Exception as e:
|
|
logger.debug("Staging cleanup: %s", type(e).__name__)
|
|
|
|
|
|
def safe_process(msg_id, fields):
|
|
try:
|
|
process_job(msg_id, fields)
|
|
except Exception as e:
|
|
logger.error("Unhandled in process_job %s: %s", msg_id, type(e).__name__)
|
|
|
|
|
|
def main():
|
|
ensure_consumer_groups()
|
|
logger.info("Worker '%s' started, concurrency=%d", CONSUMER_NAME, WORKER_CONCURRENCY)
|
|
|
|
executor = ThreadPoolExecutor(max_workers=WORKER_CONCURRENCY)
|
|
last_claim = 0
|
|
last_cleanup = 0
|
|
|
|
try:
|
|
while running:
|
|
try:
|
|
now = time.time()
|
|
|
|
if now - last_claim > 15:
|
|
last_claim = now
|
|
try:
|
|
claimed = redis_client.xautoclaim(
|
|
STREAM_NAME, GROUP_NAME, CONSUMER_NAME,
|
|
min_idle_time=CLAIM_MIN_IDLE_MS,
|
|
start_id="0-0", count=WORKER_CONCURRENCY
|
|
)
|
|
if claimed and len(claimed) > 1 and claimed[1]:
|
|
for msg_id, fields in claimed[1]:
|
|
if not running:
|
|
break
|
|
executor.submit(safe_process, msg_id, fields)
|
|
except Exception:
|
|
pass
|
|
|
|
if now - last_cleanup > 600:
|
|
last_cleanup = now
|
|
executor.submit(run_staging_cleanup)
|
|
|
|
resp = redis_client.xreadgroup(
|
|
GROUP_NAME, CONSUMER_NAME, {STREAM_NAME: ">"},
|
|
count=WORKER_CONCURRENCY, block=2000
|
|
)
|
|
if not resp:
|
|
continue
|
|
for _, messages in resp:
|
|
for msg_id, fields in messages:
|
|
if not running:
|
|
break
|
|
executor.submit(safe_process, msg_id, fields)
|
|
except Exception as e:
|
|
logger.warning("Worker loop: %s", type(e).__name__)
|
|
time.sleep(1)
|
|
finally:
|
|
logger.info("Shutting down, waiting for in-flight jobs...")
|
|
executor.shutdown(wait=True)
|
|
logger.info("Worker stopped cleanly")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |