forked from mtusi2026/mtusi2026lab6
Переиспользован стек lab5. API/worker шлют OTLP (traces/metrics/logs) в otel-collector -> Prometheus/Loki/Tempo; Grafana с provisioning датасорсов, дашборда и alert rules в Git. W3C trace context переносится через поле traceparent в задании очереди (API producer span -> worker consumer span в одном trace, подтверждено прямым запросом к Tempo). Три контролируемых инцидента прогнаны и задокументированы (worker down -> queue lag + alert, DB down отличим от медленного handler по error-span, telemetry backend down не роняет API/Collector). В процессе найдены и исправлены 4 реальных бага (middleware после старта приложения, detached span context, замерзающая метрика очереди при падении её единственного источника, неатомарный XACK+XDEL) — см. REPORT.md и evidence/. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
249 lines
10 KiB
Python
249 lines
10 KiB
Python
"""Standalone worker process: consumes attachment.process.v1 jobs, generates
|
|
variants, writes results idempotently. Runs as its own container/command,
|
|
separate from the API (contracts/attachment-states.md requires this split).
|
|
"""
|
|
import asyncio
|
|
import datetime as dt
|
|
import hashlib
|
|
import logging
|
|
import os
|
|
import random
|
|
import socket
|
|
import time
|
|
import uuid
|
|
|
|
from opentelemetry import trace
|
|
from opentelemetry.trace import SpanKind
|
|
|
|
from . import broker, db, pipeline, storage, telemetry
|
|
from . import metrics as app_metrics
|
|
|
|
logger = logging.getLogger("worker")
|
|
|
|
CONSUMER_NAME = os.environ.get("INSTANCE_ID") or f"{socket.gethostname()}-{os.getpid()}"
|
|
MAX_ATTEMPTS = int(os.environ.get("JOB_MAX_ATTEMPTS", "5"))
|
|
JOB_TIMEOUT_SECONDS = int(os.environ.get("JOB_TIMEOUT_SECONDS", "60"))
|
|
RETRY_SCAN_INTERVAL_SECONDS = 1
|
|
WORKER_CAPACITY = 1 # this process handles one job at a time (see REPORT.md limitations)
|
|
|
|
_active_jobs = 0
|
|
_tracer = None
|
|
|
|
|
|
def read_secret(file_env: str, inline_env: str) -> str:
|
|
path = os.environ.get(file_env)
|
|
if path and os.path.isfile(path):
|
|
return open(path, encoding="utf-8").read().strip()
|
|
value = os.environ.get(inline_env)
|
|
if not value:
|
|
raise RuntimeError(f"{file_env} or {inline_env} must be set")
|
|
return value
|
|
|
|
|
|
async def process_job(job: dict) -> None:
|
|
global _active_jobs
|
|
ctx = telemetry.extract_context(job) if job.get("traceparent") else None
|
|
span_cm = _tracer.start_as_current_span("process_job", context=ctx, kind=SpanKind.CONSUMER) if _tracer else None
|
|
if span_cm:
|
|
span_cm.__enter__()
|
|
started = time.monotonic()
|
|
_active_jobs += 1
|
|
try:
|
|
await _process_job_inner(job, started)
|
|
finally:
|
|
_active_jobs -= 1
|
|
if span_cm:
|
|
span_cm.__exit__(None, None, None)
|
|
|
|
|
|
async def _process_job_inner(job: dict, started: float) -> None:
|
|
attachment_id = uuid.UUID(job["attachment_id"])
|
|
job_id = uuid.UUID(job["job_id"])
|
|
attempt = job["attempt"]
|
|
|
|
async with db.pool().acquire() as conn:
|
|
record = await conn.fetchrow("SELECT * FROM attachments WHERE id=$1", attachment_id)
|
|
if record is None:
|
|
logger.warning("job %s references missing attachment %s; dropping", job_id, attachment_id)
|
|
return
|
|
if record["status"] in ("ready", "failed"):
|
|
# Redelivery of an already-finished job, or a stale duplicate racing
|
|
# a newer attempt: never roll a finished result backwards.
|
|
logger.info("attachment %s already %s; no-op", attachment_id, record["status"])
|
|
app_metrics.record_job("attachment.process.v1", "duplicate_noop", time.monotonic() - started)
|
|
return
|
|
|
|
async with db.pool().acquire() as conn:
|
|
await conn.execute(
|
|
"UPDATE attachments SET status='processing', updated_at=now() WHERE id=$1 AND status != 'ready'",
|
|
attachment_id,
|
|
)
|
|
|
|
content_types = {"thumbnail": "image/jpeg", "crop": "image/jpeg", "optimized": "image/webp"}
|
|
try:
|
|
original = await storage.get_object(record["s3_key"])
|
|
generated = {
|
|
"thumbnail": pipeline.thumbnail(original),
|
|
"crop": pipeline.crop(original),
|
|
"optimized": pipeline.optimized(original),
|
|
}
|
|
uploaded = []
|
|
for name, (data, width, height) in generated.items():
|
|
key = f"attachments/{attachment_id}/{name}"
|
|
await storage.put_object(key, data, content_types[name]) # network call: can fail transiently too
|
|
uploaded.append((name, key, data, width, height))
|
|
except Exception: # noqa: BLE001 - covers S3 reachability (get+put) and image decode, not just decode
|
|
logger.exception("processing failed for attachment %s", attachment_id)
|
|
app_metrics.record_job("attachment.process.v1", "failed_attempt", time.monotonic() - started)
|
|
await _handle_failure(job, "processing_failed")
|
|
return
|
|
|
|
async with db.pool().acquire() as conn:
|
|
async with conn.transaction():
|
|
for name, key, data, width, height in uploaded:
|
|
digest = hashlib.sha256(data).hexdigest()
|
|
await conn.execute(
|
|
"""
|
|
INSERT INTO attachment_variants (attachment_id, name, content_type, size_bytes, width, height, sha256, s3_key)
|
|
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
|
ON CONFLICT (attachment_id, name) DO UPDATE SET
|
|
content_type=EXCLUDED.content_type, size_bytes=EXCLUDED.size_bytes,
|
|
width=EXCLUDED.width, height=EXCLUDED.height, sha256=EXCLUDED.sha256, s3_key=EXCLUDED.s3_key
|
|
""",
|
|
attachment_id, name, content_types[name], len(data), width, height, digest, key,
|
|
)
|
|
await conn.execute(
|
|
"UPDATE attachments SET status='ready', error_code=NULL, updated_at=now() WHERE id=$1 AND status != 'ready'",
|
|
attachment_id,
|
|
)
|
|
logger.info("attachment %s ready (attempt %d)", attachment_id, attempt)
|
|
app_metrics.record_job("attachment.process.v1", "success", time.monotonic() - started)
|
|
accepted_at = record["created_at"]
|
|
app_metrics.record_time_to_ready((dt.datetime.now(dt.timezone.utc) - accepted_at).total_seconds())
|
|
|
|
|
|
async def _handle_failure(job: dict, error_code: str) -> None:
|
|
attachment_id = uuid.UUID(job["attachment_id"])
|
|
job_id = uuid.UUID(job["job_id"])
|
|
attempt = job["attempt"]
|
|
async with db.pool().acquire() as conn:
|
|
if attempt >= MAX_ATTEMPTS:
|
|
async with conn.transaction():
|
|
await conn.execute(
|
|
"UPDATE attachments SET status='failed', error_code=$2, updated_at=now() WHERE id=$1",
|
|
attachment_id, error_code,
|
|
)
|
|
await conn.execute(
|
|
"INSERT INTO job_failures (attachment_id, job_id, attempt, error_code) VALUES ($1,$2,$3,$4) "
|
|
"ON CONFLICT (attachment_id, job_id) DO NOTHING",
|
|
attachment_id, job_id, attempt, error_code,
|
|
)
|
|
logger.warning("attachment %s permanently failed after %d attempts: %s", attachment_id, attempt, error_code)
|
|
return
|
|
# Transient: schedule a durable delayed retry instead of sleeping in-process.
|
|
delay = min(30, 2 ** (attempt - 1)) + random.uniform(0, 1)
|
|
await conn.execute(
|
|
"""
|
|
INSERT INTO job_retry_schedule (attachment_id, job_id, attempt, next_attempt_at)
|
|
VALUES ($1, $2, $3, now() + ($4 * interval '1 second'))
|
|
ON CONFLICT (attachment_id, job_id) DO UPDATE SET
|
|
attempt=EXCLUDED.attempt, next_attempt_at=EXCLUDED.next_attempt_at
|
|
""",
|
|
attachment_id, job_id, attempt + 1, delay,
|
|
)
|
|
await conn.execute(
|
|
"UPDATE attachments SET status='queued', updated_at=now() WHERE id=$1 AND status != 'ready'",
|
|
attachment_id,
|
|
)
|
|
logger.info("attachment %s scheduled for retry #%d in %.1fs", attachment_id, attempt + 1, delay)
|
|
|
|
|
|
async def retry_scheduler_loop() -> None:
|
|
while True:
|
|
try:
|
|
async with db.pool().acquire() as conn:
|
|
due = await conn.fetch(
|
|
"SELECT attachment_id, job_id, attempt FROM job_retry_schedule WHERE next_attempt_at <= now() LIMIT 50"
|
|
)
|
|
for row in due:
|
|
job = {
|
|
"schema_version": 1,
|
|
"job_id": str(row["job_id"]),
|
|
"type": "attachment.process.v1",
|
|
"attachment_id": str(row["attachment_id"]),
|
|
"pipeline_version": 1,
|
|
"attempt": row["attempt"],
|
|
"created_at": "1970-01-01T00:00:00Z", # informational only; not used for ordering
|
|
}
|
|
await broker.publish(job)
|
|
await conn.execute(
|
|
"DELETE FROM job_retry_schedule WHERE attachment_id=$1 AND job_id=$2",
|
|
row["attachment_id"], row["job_id"],
|
|
)
|
|
except Exception: # noqa: BLE001 - scheduler must survive a transient hiccup
|
|
logger.exception("retry scheduler tick failed")
|
|
await asyncio.sleep(RETRY_SCAN_INTERVAL_SECONDS)
|
|
|
|
|
|
async def reclaim_loop() -> None:
|
|
"""Hand this worker any job another (likely dead) worker left unacked
|
|
past JOB_TIMEOUT_SECONDS -- lease recovery for a crashed worker."""
|
|
while True:
|
|
try:
|
|
for message_id, job in await broker.reclaim_stale(CONSUMER_NAME, JOB_TIMEOUT_SECONDS * 1000):
|
|
logger.info("reclaimed stale job %s (message %s)", job["job_id"], message_id)
|
|
await process_job(job)
|
|
await broker.ack(message_id)
|
|
except Exception: # noqa: BLE001
|
|
logger.exception("reclaim tick failed")
|
|
await asyncio.sleep(JOB_TIMEOUT_SECONDS / 2)
|
|
|
|
|
|
async def consume_loop() -> None:
|
|
while True:
|
|
try:
|
|
for message_id, job in await broker.read(CONSUMER_NAME, count=1, block_ms=2000):
|
|
await process_job(job)
|
|
await broker.ack(message_id)
|
|
except Exception: # noqa: BLE001 - keep the loop alive across transient errors
|
|
logger.exception("consume tick failed")
|
|
await asyncio.sleep(1)
|
|
|
|
|
|
async def main() -> None:
|
|
global _tracer
|
|
telemetry.init(os.environ.get("OTEL_SERVICE_NAME", "messenger-worker"))
|
|
app_metrics.init()
|
|
_tracer = trace.get_tracer("messenger-worker")
|
|
|
|
database_url = read_secret("DATABASE_URL_FILE", "DATABASE_URL")
|
|
cache_url = read_secret("CACHE_URL_FILE", "CACHE_URL")
|
|
await db.init_pool(database_url)
|
|
broker.init(cache_url)
|
|
await broker.ensure_group()
|
|
storage.init(
|
|
endpoint=os.environ["S3_ENDPOINT"],
|
|
region=os.environ.get("S3_REGION", "us-east-1"),
|
|
access_key=read_secret("S3_ACCESS_KEY_ID_FILE", "S3_ACCESS_KEY_ID"),
|
|
secret_key=read_secret("S3_SECRET_ACCESS_KEY_FILE", "S3_SECRET_ACCESS_KEY"),
|
|
bucket=os.environ.get("S3_BUCKET", "messenger-attachments"),
|
|
)
|
|
|
|
loop = asyncio.get_event_loop()
|
|
|
|
def _queue_sync():
|
|
future = asyncio.run_coroutine_threadsafe(broker.depth_and_oldest_age_seconds(), loop)
|
|
return future.result(timeout=5)
|
|
|
|
app_metrics.observable_gauges(
|
|
queue_depth_getter=_queue_sync,
|
|
worker_slots_getter=lambda: (_active_jobs, WORKER_CAPACITY),
|
|
)
|
|
|
|
logger.info("worker %s starting", CONSUMER_NAME)
|
|
await asyncio.gather(consume_loop(), reclaim_loop(), retry_scheduler_loop())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|