Lab7: HTTPS, mTLS, OpenBao, трек 2 PKI на оценку 5
Skeleton contracts / contracts (pull_request) Canceled after 0s
Skeleton contracts / contracts (pull_request) Canceled after 0s
This commit is contained in:
+261
@@ -0,0 +1,261 @@
|
||||
"""Отдельный процесс. API не уменьшает картинку, даже если worker остановлен."""
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pika
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool
|
||||
|
||||
from blobs import bucket_ready, read_secret
|
||||
from broker import PROCESS_QUEUE, RETRY_QUEUE, _parameters, broker_channel, ensure_topology, publish_pending, queue_depth
|
||||
from jobs import cleanup_staging, lease_seconds, process_delivery, release_expired
|
||||
from telemetry import observe_process, set_queue_stats, set_worker_stats, setup as setup_telemetry, structured_log
|
||||
|
||||
LOGGER = logging.getLogger("worker")
|
||||
SCHEMA_VERSION = "004_jobs"
|
||||
BUSY = 0
|
||||
BUSY_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def database_url():
|
||||
return read_secret(os.environ["DATABASE_URL_FILE"])
|
||||
|
||||
|
||||
def concurrency():
|
||||
value = int(os.environ.get("WORKER_CONCURRENCY", "2"))
|
||||
if not 1 <= value <= 8:
|
||||
raise RuntimeError("WORKER_CONCURRENCY")
|
||||
return value
|
||||
|
||||
|
||||
def wait_schema(conninfo):
|
||||
last = "error"
|
||||
for _ in range(90):
|
||||
try:
|
||||
with psycopg.connect(conninfo, connect_timeout=3, row_factory=dict_row) as conn:
|
||||
row = conn.execute(
|
||||
"SELECT 1 FROM schema_migrations WHERE version = %s",
|
||||
(SCHEMA_VERSION,),
|
||||
).fetchone()
|
||||
if row is not None:
|
||||
return
|
||||
last = "migration missing"
|
||||
except psycopg.Error as exc:
|
||||
last = type(exc).__name__
|
||||
time.sleep(1)
|
||||
LOGGER.error("schema not ready: %s", last)
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def wait_bucket():
|
||||
for _ in range(60):
|
||||
if bucket_ready():
|
||||
return
|
||||
time.sleep(1)
|
||||
LOGGER.error("object storage not ready")
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def wait_broker():
|
||||
last = "error"
|
||||
for _ in range(60):
|
||||
try:
|
||||
with broker_channel():
|
||||
return
|
||||
except Exception as exc:
|
||||
last = type(exc).__name__
|
||||
time.sleep(1)
|
||||
LOGGER.error("broker not ready: %s", last)
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def make_pool(conninfo, slots):
|
||||
return ConnectionPool(
|
||||
conninfo,
|
||||
min_size=1,
|
||||
max_size=slots * 2 + 2,
|
||||
timeout=3,
|
||||
kwargs={
|
||||
"connect_timeout": 3,
|
||||
"row_factory": dict_row,
|
||||
"application_name": "mtusi-lab6-worker",
|
||||
"options": "-c statement_timeout=5000 -c lock_timeout=3000 -c idle_in_transaction_session_timeout=30000",
|
||||
},
|
||||
open=True,
|
||||
)
|
||||
|
||||
|
||||
def oldest_job_age_seconds(pool):
|
||||
try:
|
||||
with pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT EXTRACT(EPOCH FROM (now() - MIN(created_at))) AS age
|
||||
FROM attachment_jobs
|
||||
WHERE status IN ('queued', 'processing')
|
||||
"""
|
||||
).fetchone()
|
||||
if row is None or row["age"] is None:
|
||||
return 0.0
|
||||
return float(row["age"])
|
||||
except Exception:
|
||||
return 0.0
|
||||
|
||||
|
||||
def refresh_runtime_stats(pool, slots):
|
||||
global BUSY
|
||||
try:
|
||||
depth = queue_depth(PROCESS_QUEUE)
|
||||
except Exception:
|
||||
depth = 0
|
||||
try:
|
||||
retry_depth = queue_depth(RETRY_QUEUE)
|
||||
except Exception:
|
||||
retry_depth = 0
|
||||
age = oldest_job_age_seconds(pool)
|
||||
set_queue_stats(PROCESS_QUEUE, depth, age)
|
||||
set_queue_stats(RETRY_QUEUE, retry_depth, 0.0)
|
||||
with BUSY_LOCK:
|
||||
busy = BUSY
|
||||
set_worker_stats(busy, slots)
|
||||
observe_process()
|
||||
try:
|
||||
stats = pool.get_stats()
|
||||
from telemetry import set_pool_stats
|
||||
|
||||
used = int(stats.get("pool_size", 0)) - int(stats.get("pool_available", 0))
|
||||
waiting = int(stats.get("requests_waiting", 0))
|
||||
set_pool_stats(max(0, used), waiting)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def consume(pool, owner, slots):
|
||||
executor = ThreadPoolExecutor(max_workers=slots, thread_name_prefix="job")
|
||||
while True:
|
||||
connection = None
|
||||
try:
|
||||
connection = pika.BlockingConnection(_parameters())
|
||||
channel = connection.channel()
|
||||
ensure_topology(channel)
|
||||
channel.basic_qos(prefetch_count=slots)
|
||||
|
||||
def on_message(ch, method, _props, body):
|
||||
executor.submit(run_one, pool, owner, ch, method.delivery_tag, body)
|
||||
|
||||
channel.basic_consume(PROCESS_QUEUE, on_message, auto_ack=False)
|
||||
channel.start_consuming()
|
||||
except Exception as exc:
|
||||
LOGGER.warning("consumer %s", type(exc).__name__)
|
||||
time.sleep(1)
|
||||
finally:
|
||||
if connection is not None:
|
||||
try:
|
||||
connection.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def run_one(pool, owner, channel, tag, body):
|
||||
global BUSY
|
||||
with BUSY_LOCK:
|
||||
BUSY += 1
|
||||
try:
|
||||
process_delivery(pool, body, owner)
|
||||
action = "ack"
|
||||
except Exception as exc:
|
||||
LOGGER.warning("delivery %s", type(exc).__name__)
|
||||
action = "nack"
|
||||
finally:
|
||||
with BUSY_LOCK:
|
||||
BUSY = max(0, BUSY - 1)
|
||||
|
||||
def finish():
|
||||
try:
|
||||
if action == "ack":
|
||||
channel.basic_ack(tag)
|
||||
else:
|
||||
channel.basic_nack(tag, requeue=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
channel.connection.add_callback_threadsafe(finish)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def wait_bao():
|
||||
"""Fail-closed: worker не стартует без scoped secret из OpenBao."""
|
||||
if not os.environ.get("BAO_ADDR"):
|
||||
return
|
||||
from bao import BaoError, login_and_read
|
||||
|
||||
path = os.environ.get("BAO_SECRET_PATH", "secret/data/messenger/worker")
|
||||
last = None
|
||||
for _ in range(60):
|
||||
try:
|
||||
login_and_read(path)
|
||||
return
|
||||
except BaoError as exc:
|
||||
last = exc
|
||||
time.sleep(2)
|
||||
LOGGER.error("openbao unavailable: %s", last)
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
|
||||
setup_telemetry(os.environ.get("OTEL_SERVICE_NAME", "messenger-worker"))
|
||||
wait_bao()
|
||||
conninfo = database_url()
|
||||
wait_schema(conninfo)
|
||||
wait_bucket()
|
||||
wait_broker()
|
||||
slots = concurrency()
|
||||
pool = make_pool(conninfo, slots)
|
||||
owner = socket.gethostname()
|
||||
stop = threading.Event()
|
||||
set_worker_stats(0, slots)
|
||||
|
||||
def drain():
|
||||
while not stop.wait(1):
|
||||
try:
|
||||
with pool.connection() as conn:
|
||||
publish_pending(conn)
|
||||
except Exception as exc:
|
||||
LOGGER.warning("drain %s", type(exc).__name__)
|
||||
|
||||
def sweep():
|
||||
ticks = 0
|
||||
while not stop.wait(5):
|
||||
ticks += 1
|
||||
try:
|
||||
with pool.connection() as conn:
|
||||
release_expired(conn)
|
||||
if ticks % 12 == 0:
|
||||
cleanup_staging(conn)
|
||||
refresh_runtime_stats(pool, slots)
|
||||
except Exception as exc:
|
||||
LOGGER.warning("sweep %s", type(exc).__name__)
|
||||
|
||||
threading.Thread(target=drain, name="drain", daemon=True).start()
|
||||
threading.Thread(target=sweep, name="sweep", daemon=True).start()
|
||||
try:
|
||||
with pool.connection() as conn:
|
||||
publish_pending(conn)
|
||||
except Exception as exc:
|
||||
LOGGER.warning("startup drain %s", type(exc).__name__)
|
||||
structured_log(LOGGER, "worker_ready", owner=owner, concurrency=slots, lease=lease_seconds())
|
||||
LOGGER.info("worker ready owner=%s concurrency=%s lease=%s", owner, slots, lease_seconds())
|
||||
consume(pool, owner, slots)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user