Files
mtusi2026lab7/src/worker.py
T
Wladislava1 54dcf70ea6
Skeleton contracts / contracts (pull_request) Canceled after 0s
Lab7: HTTPS, mTLS, OpenBao, трек 2 PKI на оценку 5
2026-11-17 00:25:48 +03:00

262 lines
7.7 KiB
Python

"""Отдельный процесс. API не уменьшает картинку, даже если worker остановлен."""
import logging
import os
import socket
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
import pika
import psycopg
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool
from blobs import bucket_ready, read_secret
from broker import PROCESS_QUEUE, RETRY_QUEUE, _parameters, broker_channel, ensure_topology, publish_pending, queue_depth
from jobs import cleanup_staging, lease_seconds, process_delivery, release_expired
from telemetry import observe_process, set_queue_stats, set_worker_stats, setup as setup_telemetry, structured_log
LOGGER = logging.getLogger("worker")
SCHEMA_VERSION = "004_jobs"
BUSY = 0
BUSY_LOCK = threading.Lock()
def database_url():
return read_secret(os.environ["DATABASE_URL_FILE"])
def concurrency():
value = int(os.environ.get("WORKER_CONCURRENCY", "2"))
if not 1 <= value <= 8:
raise RuntimeError("WORKER_CONCURRENCY")
return value
def wait_schema(conninfo):
last = "error"
for _ in range(90):
try:
with psycopg.connect(conninfo, connect_timeout=3, row_factory=dict_row) as conn:
row = conn.execute(
"SELECT 1 FROM schema_migrations WHERE version = %s",
(SCHEMA_VERSION,),
).fetchone()
if row is not None:
return
last = "migration missing"
except psycopg.Error as exc:
last = type(exc).__name__
time.sleep(1)
LOGGER.error("schema not ready: %s", last)
raise SystemExit(1)
def wait_bucket():
for _ in range(60):
if bucket_ready():
return
time.sleep(1)
LOGGER.error("object storage not ready")
raise SystemExit(1)
def wait_broker():
last = "error"
for _ in range(60):
try:
with broker_channel():
return
except Exception as exc:
last = type(exc).__name__
time.sleep(1)
LOGGER.error("broker not ready: %s", last)
raise SystemExit(1)
def make_pool(conninfo, slots):
return ConnectionPool(
conninfo,
min_size=1,
max_size=slots * 2 + 2,
timeout=3,
kwargs={
"connect_timeout": 3,
"row_factory": dict_row,
"application_name": "mtusi-lab6-worker",
"options": "-c statement_timeout=5000 -c lock_timeout=3000 -c idle_in_transaction_session_timeout=30000",
},
open=True,
)
def oldest_job_age_seconds(pool):
try:
with pool.connection() as conn:
row = conn.execute(
"""
SELECT EXTRACT(EPOCH FROM (now() - MIN(created_at))) AS age
FROM attachment_jobs
WHERE status IN ('queued', 'processing')
"""
).fetchone()
if row is None or row["age"] is None:
return 0.0
return float(row["age"])
except Exception:
return 0.0
def refresh_runtime_stats(pool, slots):
global BUSY
try:
depth = queue_depth(PROCESS_QUEUE)
except Exception:
depth = 0
try:
retry_depth = queue_depth(RETRY_QUEUE)
except Exception:
retry_depth = 0
age = oldest_job_age_seconds(pool)
set_queue_stats(PROCESS_QUEUE, depth, age)
set_queue_stats(RETRY_QUEUE, retry_depth, 0.0)
with BUSY_LOCK:
busy = BUSY
set_worker_stats(busy, slots)
observe_process()
try:
stats = pool.get_stats()
from telemetry import set_pool_stats
used = int(stats.get("pool_size", 0)) - int(stats.get("pool_available", 0))
waiting = int(stats.get("requests_waiting", 0))
set_pool_stats(max(0, used), waiting)
except Exception:
pass
def consume(pool, owner, slots):
executor = ThreadPoolExecutor(max_workers=slots, thread_name_prefix="job")
while True:
connection = None
try:
connection = pika.BlockingConnection(_parameters())
channel = connection.channel()
ensure_topology(channel)
channel.basic_qos(prefetch_count=slots)
def on_message(ch, method, _props, body):
executor.submit(run_one, pool, owner, ch, method.delivery_tag, body)
channel.basic_consume(PROCESS_QUEUE, on_message, auto_ack=False)
channel.start_consuming()
except Exception as exc:
LOGGER.warning("consumer %s", type(exc).__name__)
time.sleep(1)
finally:
if connection is not None:
try:
connection.close()
except Exception:
pass
def run_one(pool, owner, channel, tag, body):
global BUSY
with BUSY_LOCK:
BUSY += 1
try:
process_delivery(pool, body, owner)
action = "ack"
except Exception as exc:
LOGGER.warning("delivery %s", type(exc).__name__)
action = "nack"
finally:
with BUSY_LOCK:
BUSY = max(0, BUSY - 1)
def finish():
try:
if action == "ack":
channel.basic_ack(tag)
else:
channel.basic_nack(tag, requeue=True)
except Exception:
pass
try:
channel.connection.add_callback_threadsafe(finish)
except Exception:
pass
def wait_bao():
"""Fail-closed: worker не стартует без scoped secret из OpenBao."""
if not os.environ.get("BAO_ADDR"):
return
from bao import BaoError, login_and_read
path = os.environ.get("BAO_SECRET_PATH", "secret/data/messenger/worker")
last = None
for _ in range(60):
try:
login_and_read(path)
return
except BaoError as exc:
last = exc
time.sleep(2)
LOGGER.error("openbao unavailable: %s", last)
raise SystemExit(1)
def main():
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
setup_telemetry(os.environ.get("OTEL_SERVICE_NAME", "messenger-worker"))
wait_bao()
conninfo = database_url()
wait_schema(conninfo)
wait_bucket()
wait_broker()
slots = concurrency()
pool = make_pool(conninfo, slots)
owner = socket.gethostname()
stop = threading.Event()
set_worker_stats(0, slots)
def drain():
while not stop.wait(1):
try:
with pool.connection() as conn:
publish_pending(conn)
except Exception as exc:
LOGGER.warning("drain %s", type(exc).__name__)
def sweep():
ticks = 0
while not stop.wait(5):
ticks += 1
try:
with pool.connection() as conn:
release_expired(conn)
if ticks % 12 == 0:
cleanup_staging(conn)
refresh_runtime_stats(pool, slots)
except Exception as exc:
LOGGER.warning("sweep %s", type(exc).__name__)
threading.Thread(target=drain, name="drain", daemon=True).start()
threading.Thread(target=sweep, name="sweep", daemon=True).start()
try:
with pool.connection() as conn:
publish_pending(conn)
except Exception as exc:
LOGGER.warning("startup drain %s", type(exc).__name__)
structured_log(LOGGER, "worker_ready", owner=owner, concurrency=slots, lease=lease_seconds())
LOGGER.info("worker ready owner=%s concurrency=%s lease=%s", owner, slots, lease_seconds())
consume(pool, owner, slots)
if __name__ == "__main__":
main()