262 lines
7.7 KiB
Python
262 lines
7.7 KiB
Python
"""Отдельный процесс. API не уменьшает картинку, даже если worker остановлен."""
|
|
import logging
|
|
import os
|
|
import socket
|
|
import threading
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import datetime, timezone
|
|
|
|
import pika
|
|
import psycopg
|
|
from psycopg.rows import dict_row
|
|
from psycopg_pool import ConnectionPool
|
|
|
|
from blobs import bucket_ready, read_secret
|
|
from broker import PROCESS_QUEUE, RETRY_QUEUE, _parameters, broker_channel, ensure_topology, publish_pending, queue_depth
|
|
from jobs import cleanup_staging, lease_seconds, process_delivery, release_expired
|
|
from telemetry import observe_process, set_queue_stats, set_worker_stats, setup as setup_telemetry, structured_log
|
|
|
|
LOGGER = logging.getLogger("worker")
|
|
SCHEMA_VERSION = "004_jobs"
|
|
BUSY = 0
|
|
BUSY_LOCK = threading.Lock()
|
|
|
|
|
|
def database_url():
|
|
return read_secret(os.environ["DATABASE_URL_FILE"])
|
|
|
|
|
|
def concurrency():
|
|
value = int(os.environ.get("WORKER_CONCURRENCY", "2"))
|
|
if not 1 <= value <= 8:
|
|
raise RuntimeError("WORKER_CONCURRENCY")
|
|
return value
|
|
|
|
|
|
def wait_schema(conninfo):
|
|
last = "error"
|
|
for _ in range(90):
|
|
try:
|
|
with psycopg.connect(conninfo, connect_timeout=3, row_factory=dict_row) as conn:
|
|
row = conn.execute(
|
|
"SELECT 1 FROM schema_migrations WHERE version = %s",
|
|
(SCHEMA_VERSION,),
|
|
).fetchone()
|
|
if row is not None:
|
|
return
|
|
last = "migration missing"
|
|
except psycopg.Error as exc:
|
|
last = type(exc).__name__
|
|
time.sleep(1)
|
|
LOGGER.error("schema not ready: %s", last)
|
|
raise SystemExit(1)
|
|
|
|
|
|
def wait_bucket():
|
|
for _ in range(60):
|
|
if bucket_ready():
|
|
return
|
|
time.sleep(1)
|
|
LOGGER.error("object storage not ready")
|
|
raise SystemExit(1)
|
|
|
|
|
|
def wait_broker():
|
|
last = "error"
|
|
for _ in range(60):
|
|
try:
|
|
with broker_channel():
|
|
return
|
|
except Exception as exc:
|
|
last = type(exc).__name__
|
|
time.sleep(1)
|
|
LOGGER.error("broker not ready: %s", last)
|
|
raise SystemExit(1)
|
|
|
|
|
|
def make_pool(conninfo, slots):
|
|
return ConnectionPool(
|
|
conninfo,
|
|
min_size=1,
|
|
max_size=slots * 2 + 2,
|
|
timeout=3,
|
|
kwargs={
|
|
"connect_timeout": 3,
|
|
"row_factory": dict_row,
|
|
"application_name": "mtusi-lab6-worker",
|
|
"options": "-c statement_timeout=5000 -c lock_timeout=3000 -c idle_in_transaction_session_timeout=30000",
|
|
},
|
|
open=True,
|
|
)
|
|
|
|
|
|
def oldest_job_age_seconds(pool):
|
|
try:
|
|
with pool.connection() as conn:
|
|
row = conn.execute(
|
|
"""
|
|
SELECT EXTRACT(EPOCH FROM (now() - MIN(created_at))) AS age
|
|
FROM attachment_jobs
|
|
WHERE status IN ('queued', 'processing')
|
|
"""
|
|
).fetchone()
|
|
if row is None or row["age"] is None:
|
|
return 0.0
|
|
return float(row["age"])
|
|
except Exception:
|
|
return 0.0
|
|
|
|
|
|
def refresh_runtime_stats(pool, slots):
|
|
global BUSY
|
|
try:
|
|
depth = queue_depth(PROCESS_QUEUE)
|
|
except Exception:
|
|
depth = 0
|
|
try:
|
|
retry_depth = queue_depth(RETRY_QUEUE)
|
|
except Exception:
|
|
retry_depth = 0
|
|
age = oldest_job_age_seconds(pool)
|
|
set_queue_stats(PROCESS_QUEUE, depth, age)
|
|
set_queue_stats(RETRY_QUEUE, retry_depth, 0.0)
|
|
with BUSY_LOCK:
|
|
busy = BUSY
|
|
set_worker_stats(busy, slots)
|
|
observe_process()
|
|
try:
|
|
stats = pool.get_stats()
|
|
from telemetry import set_pool_stats
|
|
|
|
used = int(stats.get("pool_size", 0)) - int(stats.get("pool_available", 0))
|
|
waiting = int(stats.get("requests_waiting", 0))
|
|
set_pool_stats(max(0, used), waiting)
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def consume(pool, owner, slots):
|
|
executor = ThreadPoolExecutor(max_workers=slots, thread_name_prefix="job")
|
|
while True:
|
|
connection = None
|
|
try:
|
|
connection = pika.BlockingConnection(_parameters())
|
|
channel = connection.channel()
|
|
ensure_topology(channel)
|
|
channel.basic_qos(prefetch_count=slots)
|
|
|
|
def on_message(ch, method, _props, body):
|
|
executor.submit(run_one, pool, owner, ch, method.delivery_tag, body)
|
|
|
|
channel.basic_consume(PROCESS_QUEUE, on_message, auto_ack=False)
|
|
channel.start_consuming()
|
|
except Exception as exc:
|
|
LOGGER.warning("consumer %s", type(exc).__name__)
|
|
time.sleep(1)
|
|
finally:
|
|
if connection is not None:
|
|
try:
|
|
connection.close()
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def run_one(pool, owner, channel, tag, body):
|
|
global BUSY
|
|
with BUSY_LOCK:
|
|
BUSY += 1
|
|
try:
|
|
process_delivery(pool, body, owner)
|
|
action = "ack"
|
|
except Exception as exc:
|
|
LOGGER.warning("delivery %s", type(exc).__name__)
|
|
action = "nack"
|
|
finally:
|
|
with BUSY_LOCK:
|
|
BUSY = max(0, BUSY - 1)
|
|
|
|
def finish():
|
|
try:
|
|
if action == "ack":
|
|
channel.basic_ack(tag)
|
|
else:
|
|
channel.basic_nack(tag, requeue=True)
|
|
except Exception:
|
|
pass
|
|
|
|
try:
|
|
channel.connection.add_callback_threadsafe(finish)
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def wait_bao():
|
|
"""Fail-closed: worker не стартует без scoped secret из OpenBao."""
|
|
if not os.environ.get("BAO_ADDR"):
|
|
return
|
|
from bao import BaoError, login_and_read
|
|
|
|
path = os.environ.get("BAO_SECRET_PATH", "secret/data/messenger/worker")
|
|
last = None
|
|
for _ in range(60):
|
|
try:
|
|
login_and_read(path)
|
|
return
|
|
except BaoError as exc:
|
|
last = exc
|
|
time.sleep(2)
|
|
LOGGER.error("openbao unavailable: %s", last)
|
|
raise SystemExit(1)
|
|
|
|
|
|
def main():
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
|
|
setup_telemetry(os.environ.get("OTEL_SERVICE_NAME", "messenger-worker"))
|
|
wait_bao()
|
|
conninfo = database_url()
|
|
wait_schema(conninfo)
|
|
wait_bucket()
|
|
wait_broker()
|
|
slots = concurrency()
|
|
pool = make_pool(conninfo, slots)
|
|
owner = socket.gethostname()
|
|
stop = threading.Event()
|
|
set_worker_stats(0, slots)
|
|
|
|
def drain():
|
|
while not stop.wait(1):
|
|
try:
|
|
with pool.connection() as conn:
|
|
publish_pending(conn)
|
|
except Exception as exc:
|
|
LOGGER.warning("drain %s", type(exc).__name__)
|
|
|
|
def sweep():
|
|
ticks = 0
|
|
while not stop.wait(5):
|
|
ticks += 1
|
|
try:
|
|
with pool.connection() as conn:
|
|
release_expired(conn)
|
|
if ticks % 12 == 0:
|
|
cleanup_staging(conn)
|
|
refresh_runtime_stats(pool, slots)
|
|
except Exception as exc:
|
|
LOGGER.warning("sweep %s", type(exc).__name__)
|
|
|
|
threading.Thread(target=drain, name="drain", daemon=True).start()
|
|
threading.Thread(target=sweep, name="sweep", daemon=True).start()
|
|
try:
|
|
with pool.connection() as conn:
|
|
publish_pending(conn)
|
|
except Exception as exc:
|
|
LOGGER.warning("startup drain %s", type(exc).__name__)
|
|
structured_log(LOGGER, "worker_ready", owner=owner, concurrency=slots, lease=lease_seconds())
|
|
LOGGER.info("worker ready owner=%s concurrency=%s lease=%s", owner, slots, lease_seconds())
|
|
consume(pool, owner, slots)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|