Files
mtusi2026lab6/worker.py
T
2026-11-19 18:05:43 +03:00

628 lines
25 KiB
Python

import os
import io
import json
import time
import signal
import random
import hashlib
import logging
from datetime import datetime, timezone
from concurrent.futures import ThreadPoolExecutor
import psycopg2
import psycopg2.extras
import redis
import boto3
import botocore.exceptions
from botocore.client import Config
from PIL import Image, ImageOps, UnidentifiedImageError
from opentelemetry import metrics, trace
from opentelemetry._logs import set_logger_provider
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.propagate import extract
class JsonLogFormatter(logging.Formatter):
def format(self, record: logging.LogRecord) -> str:
span = trace.get_current_span()
ctx = span.get_span_context() if span else None
trace_id = format(ctx.trace_id, "032x") if ctx and ctx.is_valid else None
span_id = format(ctx.span_id, "016x") if ctx and ctx.is_valid else None
payload = {
"ts": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
"level": record.levelname,
"logger": record.name,
"service": os.getenv("OTEL_SERVICE_NAME", "messenger-worker"),
"msg": record.getMessage(),
}
if trace_id and span_id:
payload["trace_id"] = trace_id
payload["span_id"] = span_id
for attr in ("method", "path", "status_code", "job_id", "attachment_id", "job_status", "attempt", "error_code"):
if hasattr(record, attr):
payload[attr] = getattr(record, attr)
if record.exc_info:
payload["exception"] = record.exc_info[0].__name__ if record.exc_info[0] else "Exception"
return json.dumps(payload, ensure_ascii=False)
def setup_worker_telemetry():
service_name = os.getenv("OTEL_SERVICE_NAME", "messenger-worker")
endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318").rstrip("/")
env_attrs = os.getenv("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment.name=lab")
attrs = {"service.name": service_name}
for part in env_attrs.split(","):
if "=" in part:
k, v = part.split("=", 1)
attrs[k.strip()] = v.strip()
resource = Resource.create(attrs)
tracer_provider = TracerProvider(resource=resource)
tracer_provider.add_span_processor(
BatchSpanProcessor(OTLPSpanExporter(endpoint=f"{endpoint}/v1/traces"))
)
trace.set_tracer_provider(tracer_provider)
metric_reader = PeriodicExportingMetricReader(
OTLPMetricExporter(endpoint=f"{endpoint}/v1/metrics"),
export_interval_millis=5000,
)
meter_provider = MeterProvider(resource=resource, metric_readers=[metric_reader])
metrics.set_meter_provider(meter_provider)
logger_provider = LoggerProvider(resource=resource)
logger_provider.add_log_record_processor(
BatchLogRecordProcessor(OTLPLogExporter(endpoint=f"{endpoint}/v1/logs"))
)
set_logger_provider(logger_provider)
root_logger = logging.getLogger()
root_logger.setLevel(logging.INFO)
root_logger.handlers.clear()
stream_handler = logging.StreamHandler()
stream_handler.setFormatter(JsonLogFormatter())
root_logger.addHandler(stream_handler)
otel_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
otel_handler.setFormatter(JsonLogFormatter())
root_logger.addHandler(otel_handler)
return trace.get_tracer(service_name), metrics.get_meter(service_name)
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger")
REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0")
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333")
S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY_ID", "appuser")
S3_SECRET_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "appsecretpassword")
STREAM_NAME = os.getenv("BROKER_STREAM", "attachment_jobs")
DLQ_STREAM_NAME = f"{STREAM_NAME}_dlq"
GROUP_NAME = os.getenv("BROKER_GROUP", "attachment_workers")
CONSUMER_NAME = os.getenv("HOSTNAME", "worker-1")
JOB_MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000"))
JOB_TIMEOUT_SECONDS = int(os.getenv("JOB_TIMEOUT_SECONDS", "30"))
WORKER_CONCURRENCY = int(os.getenv("WORKER_CONCURRENCY", "4"))
CLAIM_MIN_IDLE_MS = (JOB_TIMEOUT_SECONDS + 20) * 1000
Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS
redis_client = redis.Redis.from_url(REDIS_URL, decode_responses=True)
s3_client = boto3.client(
"s3", endpoint_url=S3_ENDPOINT,
aws_access_key_id=S3_ACCESS_KEY, aws_secret_access_key=S3_SECRET_KEY,
config=Config(signature_version="s3v4"), region_name="us-east-1"
)
def observe_queue_metrics(options):
try:
q_len = redis_client.xlen(STREAM_NAME)
entries = redis_client.xrange(STREAM_NAME, count=1)
oldest_age = 0.0
if entries:
msg_id, fields = entries[0]
try:
payload = json.loads(fields.get("payload", "{}"))
created_str = payload.get("created_at")
if created_str:
created_dt = datetime.fromisoformat(created_str.replace("Z", "+00:00"))
oldest_age = max(0.0, (datetime.now(timezone.utc) - created_dt).total_seconds())
except Exception:
ms = int(msg_id.split("-")[0])
oldest_age = max(0.0, time.time() - (ms / 1000.0))
yield metrics.Observation(value=q_len, attributes={"stream": STREAM_NAME})
except Exception:
yield metrics.Observation(value=0, attributes={"stream": STREAM_NAME})
def observe_oldest_job_age(options):
try:
entries = redis_client.xrange(STREAM_NAME, count=1)
if not entries:
yield metrics.Observation(value=0.0, attributes={"stream": STREAM_NAME})
return
msg_id, fields = entries[0]
try:
payload = json.loads(fields.get("payload", "{}"))
created_str = payload.get("created_at")
if created_str:
created_dt = datetime.fromisoformat(created_str.replace("Z", "+00:00"))
age = max(0.0, (datetime.now(timezone.utc) - created_dt).total_seconds())
yield metrics.Observation(value=age, attributes={"stream": STREAM_NAME})
return
except Exception:
pass
ms = int(msg_id.split("-")[0])
yield metrics.Observation(value=max(0.0, time.time() - (ms / 1000.0)), attributes={"stream": STREAM_NAME})
except Exception:
yield metrics.Observation(value=0.0, attributes={"stream": STREAM_NAME})
tracer, meter = setup_worker_telemetry()
logger = logging.getLogger("attachment-worker")
worker_jobs_counter = meter.create_counter(
name="worker_jobs_total",
description="Completed and failed attachment jobs",
unit="1",
)
worker_retries_counter = meter.create_counter(
name="worker_job_retries_total",
description="Transient retry attempts scheduled for attachment jobs",
unit="1",
)
worker_duration_histogram = meter.create_histogram(
name="worker_job_duration_seconds",
description="Attachment job processing duration in seconds",
unit="s",
)
worker_jobs_in_progress = meter.create_up_down_counter(
name="worker_jobs_in_progress",
description="Jobs currently being processed",
unit="1",
)
meter.create_observable_gauge(
name="worker_queue_depth",
callbacks=[observe_queue_metrics],
description="Number of messages currently in attachment jobs stream",
unit="1",
)
meter.create_observable_gauge(
name="worker_oldest_job_age_seconds",
callbacks=[observe_oldest_job_age],
description="Age of the oldest message in attachment jobs stream in seconds",
unit="s",
)
running = True
def handle_sigterm(signum, frame):
global running
logger.info("Received shutdown signal, finishing in-flight jobs...")
running = False
signal.signal(signal.SIGTERM, handle_sigterm)
signal.signal(signal.SIGINT, handle_sigterm)
def get_db_connection():
return psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor)
def ensure_consumer_groups():
while running:
try:
redis_client.xgroup_create(STREAM_NAME, GROUP_NAME, id="0", mkstream=True)
redis_client.xgroup_create(DLQ_STREAM_NAME, GROUP_NAME, id="0", mkstream=True)
logger.info("Consumer groups ensured on '%s' and '%s'", STREAM_NAME, DLQ_STREAM_NAME)
return
except redis.exceptions.ResponseError as e:
if "BUSYGROUP" in str(e):
return
raise
except Exception as e:
logger.warning("Waiting for Redis: %s", type(e).__name__)
time.sleep(1)
def prepare_base_image(raw_bytes):
img = Image.open(io.BytesIO(raw_bytes))
img = ImageOps.exif_transpose(img)
if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info):
rgba = img.convert("RGBA")
bg = Image.new("RGB", rgba.size, (255, 255, 255))
bg.paste(rgba, mask=rgba.split()[3])
img = bg
elif img.mode != "RGB":
img = img.convert("RGB")
return img
def generate_grade4_variants(raw_bytes):
base_img = prepare_base_image(raw_bytes)
orig_w, orig_h = base_img.size
meta = []
blobs = {}
thumb = base_img.copy()
if orig_w <= 256 and orig_h <= 256:
t_w, t_h = orig_w, orig_h
else:
s = min(256.0 / orig_w, 256.0 / orig_h)
t_w, t_h = max(1, int(orig_w * s)), max(1, int(orig_h * s))
if (t_w, t_h) != (orig_w, orig_h):
thumb = thumb.resize((t_w, t_h), Image.Resampling.LANCZOS)
buf = io.BytesIO()
thumb.save(buf, format="JPEG", quality=85, optimize=True)
tb = buf.getvalue()
blobs["thumbnail"] = tb
meta.append({
"name": "thumbnail", "content_type": "image/jpeg",
"size_bytes": len(tb), "width": t_w, "height": t_h,
"sha256": hashlib.sha256(tb).hexdigest()
})
opt = base_img.copy()
if orig_w <= 1280 and orig_h <= 1280:
o_w, o_h = orig_w, orig_h
else:
s = min(1280.0 / orig_w, 1280.0 / orig_h)
o_w, o_h = max(1, int(orig_w * s)), max(1, int(orig_h * s))
if (o_w, o_h) != (orig_w, orig_h):
opt = opt.resize((o_w, o_h), Image.Resampling.LANCZOS)
buf = io.BytesIO()
opt.save(buf, format="WEBP", quality=85)
ob = buf.getvalue()
blobs["optimized"] = ob
meta.append({
"name": "optimized", "content_type": "image/webp",
"size_bytes": len(ob), "width": o_w, "height": o_h,
"sha256": hashlib.sha256(ob).hexdigest()
})
c = min(orig_w, orig_h)
left, top = (orig_w - c) // 2, (orig_h - c) // 2
crop = base_img.crop((left, top, left + c, top + c))
if c != 128:
crop = crop.resize((128, 128), Image.Resampling.LANCZOS)
buf = io.BytesIO()
crop.save(buf, format="JPEG", quality=85, optimize=True)
cb = buf.getvalue()
blobs["crop"] = cb
meta.append({
"name": "crop", "content_type": "image/jpeg",
"size_bytes": len(cb), "width": 128, "height": 128,
"sha256": hashlib.sha256(cb).hexdigest()
})
return meta, blobs
def mark_failed(attachment_id, error_code):
now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
conn = get_db_connection()
try:
with conn.cursor() as cur:
cur.execute(
"UPDATE attachments SET status='failed', error_code=%s, updated_at=%s "
"WHERE id=%s AND status != 'ready'",
(error_code, now, attachment_id)
)
conn.commit()
finally:
conn.close()
TRANSIENT_ERRORS = (
TimeoutError,
botocore.exceptions.EndpointConnectionError,
botocore.exceptions.ReadTimeoutError,
psycopg2.OperationalError,
redis.exceptions.ConnectionError,
)
def _record_job_outcome(span, status: str, elapsed: float):
span.set_attribute("job.outcome", status)
worker_jobs_counter.add(1, {"status": status})
worker_duration_histogram.record(elapsed, {"status": status})
def process_job(message_id, fields):
job_start = time.monotonic()
try:
job = json.loads(fields.get("payload", "{}"))
except Exception:
logger.error("Malformed job %s, ack to drop", message_id)
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
return
ctx = extract(job)
with tracer.start_as_current_span("worker.process_attachment", context=ctx) as span:
worker_jobs_in_progress.add(1)
try:
attachment_id = job.get("attachment_id")
job_id = job.get("job_id")
pipeline_version = int(job.get("pipeline_version", 1))
attempt = int(job.get("attempt", 1))
span.set_attribute("job.pipeline_version", pipeline_version)
span.set_attribute("job.attempt", attempt)
if not attachment_id:
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
_record_job_outcome(span, "failed", time.monotonic() - job_start)
return
conn = get_db_connection()
try:
with conn.cursor() as cur:
cur.execute("SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,))
att = cur.fetchone()
if not att:
conn.commit()
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
_record_job_outcome(span, "failed", time.monotonic() - job_start)
return
if att["status"] == "ready" or att["pipeline_version"] > pipeline_version:
conn.commit()
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
_record_job_outcome(span, "success", time.monotonic() - job_start)
return
if att["status"] == "failed":
conn.commit()
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
_record_job_outcome(span, "failed", time.monotonic() - job_start)
return
if att["status"] == "processing":
updated = att["updated_at"]
if isinstance(updated, str):
updated = datetime.fromisoformat(updated.replace("Z", "+00:00"))
age_seconds = (datetime.now(timezone.utc) - updated).total_seconds()
if age_seconds < (JOB_TIMEOUT_SECONDS + 10):
conn.commit()
logger.info(
"Attachment already processing, ack duplicate",
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "processing", "attempt": attempt},
)
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
return
now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
cur.execute(
"UPDATE attachments SET status='processing', attempt=%s, updated_at=%s WHERE id=%s",
(attempt, now, attachment_id)
)
conn.commit()
finally:
conn.close()
chat_id = str(att["chat_id"])
staging_key = f"staging/{chat_id}/{attachment_id}/original"
final_original_key = f"attachments/{chat_id}/{attachment_id}/original"
start_time = time.monotonic()
def check_timeout():
if time.monotonic() - start_time > JOB_TIMEOUT_SECONDS:
raise TimeoutError("Job exceeded time limit")
try:
try:
obj = s3_client.get_object(Bucket=S3_BUCKET, Key=staging_key)
except Exception:
obj = s3_client.get_object(Bucket=S3_BUCKET, Key=final_original_key)
raw_bytes = obj["Body"].read()
check_timeout()
if hashlib.sha256(raw_bytes).hexdigest() != att["sha256"] or len(raw_bytes) != att["size_bytes"]:
raise ValueError("Staging checksum mismatch")
variants_meta, variants_bytes = generate_grade4_variants(raw_bytes)
check_timeout()
s3_client.put_object(
Bucket=S3_BUCKET, Key=final_original_key,
Body=raw_bytes, ContentType=att["content_type"]
)
for var_name, var_bytes in variants_bytes.items():
ct = "image/webp" if var_name == "optimized" else "image/jpeg"
s3_client.put_object(
Bucket=S3_BUCKET,
Key=f"attachments/{chat_id}/{attachment_id}/{var_name}",
Body=var_bytes, ContentType=ct
)
check_timeout()
ready_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
conn = get_db_connection()
try:
with conn.cursor() as cur:
cur.execute("""
UPDATE attachments SET status='ready', variants=%s::jsonb,
error_code=NULL, updated_at=%s
WHERE id=%s AND pipeline_version <= %s
""", (json.dumps(variants_meta), ready_at, attachment_id, pipeline_version))
conn.commit()
finally:
conn.close()
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
_record_job_outcome(span, "success", time.monotonic() - job_start)
logger.info(
"Job completed: attachment ready",
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "ready", "attempt": attempt},
)
try:
s3_client.delete_object(Bucket=S3_BUCKET, Key=staging_key)
except Exception:
pass
except (UnidentifiedImageError, Image.DecompressionBombError):
logger.error(
"Permanent image error",
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "invalid_image"},
)
mark_failed(attachment_id, "invalid_image")
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
_record_job_outcome(span, "failed", time.monotonic() - job_start)
except ValueError as e:
logger.error(
"Checksum error",
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "invalid_checksum"},
)
mark_failed(attachment_id, "invalid_checksum")
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
_record_job_outcome(span, "failed", time.monotonic() - job_start)
except psycopg2.errors.DataError:
logger.error(
"DB data error",
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "data_error"},
)
mark_failed(attachment_id, "data_error")
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
_record_job_outcome(span, "failed", time.monotonic() - job_start)
except TRANSIENT_ERRORS as e:
logger.warning(
"Transient error during job processing",
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "processing", "attempt": attempt},
)
if attempt >= JOB_MAX_ATTEMPTS:
mark_failed(attachment_id, "retry_exhausted")
try:
dlq_job = dict(job)
dlq_job["attempt"] = 1
redis_client.xadd(
DLQ_STREAM_NAME,
{"payload": json.dumps(dlq_job), "error": type(e).__name__}
)
except Exception:
pass
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
_record_job_outcome(span, "failed", time.monotonic() - job_start)
else:
worker_retries_counter.add(1, {"reason": type(e).__name__})
delay = min(2 ** attempt, 30) + random.uniform(0, 1)
time.sleep(delay)
next_job = dict(job)
next_job["attempt"] = attempt + 1
redis_client.xadd(STREAM_NAME, {"payload": json.dumps(next_job)})
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
except Exception:
logger.error(
"Unknown error during job processing",
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "processing_failed"},
)
mark_failed(attachment_id, "processing_failed")
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
_record_job_outcome(span, "failed", time.monotonic() - job_start)
finally:
worker_jobs_in_progress.add(-1)
def run_staging_cleanup():
try:
cutoff = datetime.now(timezone.utc).timestamp() - 3600
paginator = s3_client.get_paginator("list_objects_v2")
for page in paginator.paginate(Bucket=S3_BUCKET, Prefix="staging/"):
for obj in page.get("Contents", []):
if obj["LastModified"].timestamp() < cutoff:
try:
s3_client.delete_object(Bucket=S3_BUCKET, Key=obj["Key"])
except Exception:
pass
except Exception as e:
logger.debug("Staging cleanup: %s", type(e).__name__)
def safe_process(msg_id, fields):
try:
process_job(msg_id, fields)
except Exception as e:
logger.error("Unhandled in process_job %s: %s", msg_id, type(e).__name__)
def main():
ensure_consumer_groups()
logger.info("Worker '%s' started, concurrency=%d", CONSUMER_NAME, WORKER_CONCURRENCY)
executor = ThreadPoolExecutor(max_workers=WORKER_CONCURRENCY)
last_claim = 0
last_cleanup = 0
try:
while running:
try:
now = time.time()
if now - last_claim > 15:
last_claim = now
try:
claimed = redis_client.xautoclaim(
STREAM_NAME, GROUP_NAME, CONSUMER_NAME,
min_idle_time=CLAIM_MIN_IDLE_MS,
start_id="0-0", count=WORKER_CONCURRENCY
)
if claimed and len(claimed) > 1 and claimed[1]:
for msg_id, fields in claimed[1]:
if not running:
break
executor.submit(safe_process, msg_id, fields)
except Exception:
pass
if now - last_cleanup > 600:
last_cleanup = now
executor.submit(run_staging_cleanup)
resp = redis_client.xreadgroup(
GROUP_NAME, CONSUMER_NAME, {STREAM_NAME: ">"},
count=WORKER_CONCURRENCY, block=2000
)
if not resp:
continue
for _, messages in resp:
for msg_id, fields in messages:
if not running:
break
executor.submit(safe_process, msg_id, fields)
except Exception as e:
logger.warning("Worker loop: %s", type(e).__name__)
time.sleep(1)
finally:
logger.info("Shutting down, waiting for in-flight jobs...")
executor.shutdown(wait=True)
logger.info("Worker stopped cleanly")
if __name__ == "__main__":
main()