Files
mtusi2026lab6/app/worker.py
T
Artu-RitoandClaude Sonnet 5 6c50a63743 Лаба 6: OpenTelemetry + Grafana наблюдаемость (оценка 4)
Переиспользован стек lab5. API/worker шлют OTLP (traces/metrics/logs) в
otel-collector -> Prometheus/Loki/Tempo; Grafana с provisioning датасорсов,
дашборда и alert rules в Git. W3C trace context переносится через поле
traceparent в задании очереди (API producer span -> worker consumer span
в одном trace, подтверждено прямым запросом к Tempo). Три контролируемых
инцидента прогнаны и задокументированы (worker down -> queue lag + alert,
DB down отличим от медленного handler по error-span, telemetry backend down
не роняет API/Collector). В процессе найдены и исправлены 4 реальных бага
(middleware после старта приложения, detached span context, замерзающая
метрика очереди при падении её единственного источника, неатомарный
XACK+XDEL) — см. REPORT.md и evidence/.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-10-30 01:59:33 +03:00

249 lines
10 KiB
Python

"""Standalone worker process: consumes attachment.process.v1 jobs, generates
variants, writes results idempotently. Runs as its own container/command,
separate from the API (contracts/attachment-states.md requires this split).
"""
import asyncio
import datetime as dt
import hashlib
import logging
import os
import random
import socket
import time
import uuid
from opentelemetry import trace
from opentelemetry.trace import SpanKind
from . import broker, db, pipeline, storage, telemetry
from . import metrics as app_metrics
logger = logging.getLogger("worker")
CONSUMER_NAME = os.environ.get("INSTANCE_ID") or f"{socket.gethostname()}-{os.getpid()}"
MAX_ATTEMPTS = int(os.environ.get("JOB_MAX_ATTEMPTS", "5"))
JOB_TIMEOUT_SECONDS = int(os.environ.get("JOB_TIMEOUT_SECONDS", "60"))
RETRY_SCAN_INTERVAL_SECONDS = 1
WORKER_CAPACITY = 1 # this process handles one job at a time (see REPORT.md limitations)
_active_jobs = 0
_tracer = None
def read_secret(file_env: str, inline_env: str) -> str:
path = os.environ.get(file_env)
if path and os.path.isfile(path):
return open(path, encoding="utf-8").read().strip()
value = os.environ.get(inline_env)
if not value:
raise RuntimeError(f"{file_env} or {inline_env} must be set")
return value
async def process_job(job: dict) -> None:
global _active_jobs
ctx = telemetry.extract_context(job) if job.get("traceparent") else None
span_cm = _tracer.start_as_current_span("process_job", context=ctx, kind=SpanKind.CONSUMER) if _tracer else None
if span_cm:
span_cm.__enter__()
started = time.monotonic()
_active_jobs += 1
try:
await _process_job_inner(job, started)
finally:
_active_jobs -= 1
if span_cm:
span_cm.__exit__(None, None, None)
async def _process_job_inner(job: dict, started: float) -> None:
attachment_id = uuid.UUID(job["attachment_id"])
job_id = uuid.UUID(job["job_id"])
attempt = job["attempt"]
async with db.pool().acquire() as conn:
record = await conn.fetchrow("SELECT * FROM attachments WHERE id=$1", attachment_id)
if record is None:
logger.warning("job %s references missing attachment %s; dropping", job_id, attachment_id)
return
if record["status"] in ("ready", "failed"):
# Redelivery of an already-finished job, or a stale duplicate racing
# a newer attempt: never roll a finished result backwards.
logger.info("attachment %s already %s; no-op", attachment_id, record["status"])
app_metrics.record_job("attachment.process.v1", "duplicate_noop", time.monotonic() - started)
return
async with db.pool().acquire() as conn:
await conn.execute(
"UPDATE attachments SET status='processing', updated_at=now() WHERE id=$1 AND status != 'ready'",
attachment_id,
)
content_types = {"thumbnail": "image/jpeg", "crop": "image/jpeg", "optimized": "image/webp"}
try:
original = await storage.get_object(record["s3_key"])
generated = {
"thumbnail": pipeline.thumbnail(original),
"crop": pipeline.crop(original),
"optimized": pipeline.optimized(original),
}
uploaded = []
for name, (data, width, height) in generated.items():
key = f"attachments/{attachment_id}/{name}"
await storage.put_object(key, data, content_types[name]) # network call: can fail transiently too
uploaded.append((name, key, data, width, height))
except Exception: # noqa: BLE001 - covers S3 reachability (get+put) and image decode, not just decode
logger.exception("processing failed for attachment %s", attachment_id)
app_metrics.record_job("attachment.process.v1", "failed_attempt", time.monotonic() - started)
await _handle_failure(job, "processing_failed")
return
async with db.pool().acquire() as conn:
async with conn.transaction():
for name, key, data, width, height in uploaded:
digest = hashlib.sha256(data).hexdigest()
await conn.execute(
"""
INSERT INTO attachment_variants (attachment_id, name, content_type, size_bytes, width, height, sha256, s3_key)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
ON CONFLICT (attachment_id, name) DO UPDATE SET
content_type=EXCLUDED.content_type, size_bytes=EXCLUDED.size_bytes,
width=EXCLUDED.width, height=EXCLUDED.height, sha256=EXCLUDED.sha256, s3_key=EXCLUDED.s3_key
""",
attachment_id, name, content_types[name], len(data), width, height, digest, key,
)
await conn.execute(
"UPDATE attachments SET status='ready', error_code=NULL, updated_at=now() WHERE id=$1 AND status != 'ready'",
attachment_id,
)
logger.info("attachment %s ready (attempt %d)", attachment_id, attempt)
app_metrics.record_job("attachment.process.v1", "success", time.monotonic() - started)
accepted_at = record["created_at"]
app_metrics.record_time_to_ready((dt.datetime.now(dt.timezone.utc) - accepted_at).total_seconds())
async def _handle_failure(job: dict, error_code: str) -> None:
attachment_id = uuid.UUID(job["attachment_id"])
job_id = uuid.UUID(job["job_id"])
attempt = job["attempt"]
async with db.pool().acquire() as conn:
if attempt >= MAX_ATTEMPTS:
async with conn.transaction():
await conn.execute(
"UPDATE attachments SET status='failed', error_code=$2, updated_at=now() WHERE id=$1",
attachment_id, error_code,
)
await conn.execute(
"INSERT INTO job_failures (attachment_id, job_id, attempt, error_code) VALUES ($1,$2,$3,$4) "
"ON CONFLICT (attachment_id, job_id) DO NOTHING",
attachment_id, job_id, attempt, error_code,
)
logger.warning("attachment %s permanently failed after %d attempts: %s", attachment_id, attempt, error_code)
return
# Transient: schedule a durable delayed retry instead of sleeping in-process.
delay = min(30, 2 ** (attempt - 1)) + random.uniform(0, 1)
await conn.execute(
"""
INSERT INTO job_retry_schedule (attachment_id, job_id, attempt, next_attempt_at)
VALUES ($1, $2, $3, now() + ($4 * interval '1 second'))
ON CONFLICT (attachment_id, job_id) DO UPDATE SET
attempt=EXCLUDED.attempt, next_attempt_at=EXCLUDED.next_attempt_at
""",
attachment_id, job_id, attempt + 1, delay,
)
await conn.execute(
"UPDATE attachments SET status='queued', updated_at=now() WHERE id=$1 AND status != 'ready'",
attachment_id,
)
logger.info("attachment %s scheduled for retry #%d in %.1fs", attachment_id, attempt + 1, delay)
async def retry_scheduler_loop() -> None:
while True:
try:
async with db.pool().acquire() as conn:
due = await conn.fetch(
"SELECT attachment_id, job_id, attempt FROM job_retry_schedule WHERE next_attempt_at <= now() LIMIT 50"
)
for row in due:
job = {
"schema_version": 1,
"job_id": str(row["job_id"]),
"type": "attachment.process.v1",
"attachment_id": str(row["attachment_id"]),
"pipeline_version": 1,
"attempt": row["attempt"],
"created_at": "1970-01-01T00:00:00Z", # informational only; not used for ordering
}
await broker.publish(job)
await conn.execute(
"DELETE FROM job_retry_schedule WHERE attachment_id=$1 AND job_id=$2",
row["attachment_id"], row["job_id"],
)
except Exception: # noqa: BLE001 - scheduler must survive a transient hiccup
logger.exception("retry scheduler tick failed")
await asyncio.sleep(RETRY_SCAN_INTERVAL_SECONDS)
async def reclaim_loop() -> None:
"""Hand this worker any job another (likely dead) worker left unacked
past JOB_TIMEOUT_SECONDS -- lease recovery for a crashed worker."""
while True:
try:
for message_id, job in await broker.reclaim_stale(CONSUMER_NAME, JOB_TIMEOUT_SECONDS * 1000):
logger.info("reclaimed stale job %s (message %s)", job["job_id"], message_id)
await process_job(job)
await broker.ack(message_id)
except Exception: # noqa: BLE001
logger.exception("reclaim tick failed")
await asyncio.sleep(JOB_TIMEOUT_SECONDS / 2)
async def consume_loop() -> None:
while True:
try:
for message_id, job in await broker.read(CONSUMER_NAME, count=1, block_ms=2000):
await process_job(job)
await broker.ack(message_id)
except Exception: # noqa: BLE001 - keep the loop alive across transient errors
logger.exception("consume tick failed")
await asyncio.sleep(1)
async def main() -> None:
global _tracer
telemetry.init(os.environ.get("OTEL_SERVICE_NAME", "messenger-worker"))
app_metrics.init()
_tracer = trace.get_tracer("messenger-worker")
database_url = read_secret("DATABASE_URL_FILE", "DATABASE_URL")
cache_url = read_secret("CACHE_URL_FILE", "CACHE_URL")
await db.init_pool(database_url)
broker.init(cache_url)
await broker.ensure_group()
storage.init(
endpoint=os.environ["S3_ENDPOINT"],
region=os.environ.get("S3_REGION", "us-east-1"),
access_key=read_secret("S3_ACCESS_KEY_ID_FILE", "S3_ACCESS_KEY_ID"),
secret_key=read_secret("S3_SECRET_ACCESS_KEY_FILE", "S3_SECRET_ACCESS_KEY"),
bucket=os.environ.get("S3_BUCKET", "messenger-attachments"),
)
loop = asyncio.get_event_loop()
def _queue_sync():
future = asyncio.run_coroutine_threadsafe(broker.depth_and_oldest_age_seconds(), loop)
return future.result(timeout=5)
app_metrics.observable_gauges(
queue_depth_getter=_queue_sync,
worker_slots_getter=lambda: (_active_jobs, WORKER_CAPACITY),
)
logger.info("worker %s starting", CONSUMER_NAME)
await asyncio.gather(consume_loop(), reclaim_loop(), retry_scheduler_loop())
if __name__ == "__main__":
asyncio.run(main())