forked from mtusi2026/mtusi2026lab5
112 lines
5.8 KiB
Python
112 lines
5.8 KiB
Python
import os
|
|
import socket
|
|
from pathlib import Path
|
|
|
|
|
|
def env_str(name: str, default: str) -> str:
|
|
return os.environ.get(name, default).strip()
|
|
|
|
|
|
def env_int(name: str, default: int) -> int:
|
|
value = int(env_str(name, str(default)))
|
|
if value < 1:
|
|
raise ValueError(f"{name} must be >= 1")
|
|
return value
|
|
|
|
|
|
def read_secret(file_variable: str, plain_variable: str) -> str:
|
|
path = env_str(file_variable, "")
|
|
if path:
|
|
return Path(path).read_text(encoding="utf-8").strip()
|
|
return env_str(plain_variable, "")
|
|
|
|
|
|
APP_HOST = env_str("APP_HOST", "0.0.0.0")
|
|
APP_PORT = env_int("APP_PORT", 8080)
|
|
APP_ENV = env_str("APP_ENV", "lab")
|
|
APP_VERSION = env_str("APP_VERSION", "dev")
|
|
|
|
INSTANCE_ID = env_str("INSTANCE_ID", "") or socket.gethostname()
|
|
|
|
SESSION_TTL_SECONDS = env_int("SESSION_TTL_SECONDS", 3600)
|
|
SESSION_STORE = env_str("SESSION_STORE", "redis")
|
|
|
|
DATABASE_URL = read_secret("DATABASE_URL_FILE", "DATABASE_URL")
|
|
DB_POOL_MIN = env_int("DB_POOL_MIN", 1)
|
|
DB_POOL_MAX = env_int("DB_POOL_MAX", 10)
|
|
DB_POOL_WAIT_SECONDS = env_int("DB_POOL_WAIT_SECONDS", 3)
|
|
DB_CONNECT_TIMEOUT_SECONDS = env_int("DB_CONNECT_TIMEOUT_SECONDS", 3)
|
|
DB_STATEMENT_TIMEOUT_MS = env_int("DB_STATEMENT_TIMEOUT_MS", 5000)
|
|
|
|
REDIS_URL = read_secret("CACHE_URL_FILE", "CACHE_URL")
|
|
REDIS_POOL_MAX = env_int("REDIS_POOL_MAX", 20)
|
|
REDIS_TIMEOUT_MS = env_int("REDIS_TIMEOUT_MS", 500)
|
|
|
|
MIGRATIONS_DIR = Path(env_str("MIGRATIONS_DIR", str(Path(__file__).resolve().parent / "migrations")))
|
|
SCHEMA_WAIT_SECONDS = env_int("SCHEMA_WAIT_SECONDS", 60)
|
|
|
|
# S3 (SeaweedFS): адрес, bucket и отдельный ключ API с правами только на этот bucket.
|
|
S3_ENDPOINT = env_str("S3_ENDPOINT", "http://s3:8333")
|
|
S3_REGION = env_str("S3_REGION", "us-east-1")
|
|
S3_BUCKET = env_str("S3_BUCKET", "messenger-attachments")
|
|
S3_ACCESS_KEY_ID = read_secret("S3_ACCESS_KEY_ID_FILE", "S3_ACCESS_KEY_ID")
|
|
S3_SECRET_ACCESS_KEY = read_secret("S3_SECRET_ACCESS_KEY_FILE", "S3_SECRET_ACCESS_KEY")
|
|
S3_CONNECT_TIMEOUT_SECONDS = env_int("S3_CONNECT_TIMEOUT_SECONDS", 1)
|
|
S3_READ_TIMEOUT_SECONDS = env_int("S3_READ_TIMEOUT_SECONDS", 5)
|
|
S3_MAX_ATTEMPTS = env_int("S3_MAX_ATTEMPTS", 2)
|
|
S3_POOL_MAX = env_int("S3_POOL_MAX", 20)
|
|
|
|
# Вложения: лимиты из контракта и защита памяти/времени.
|
|
MAX_UPLOAD_BYTES = env_int("MAX_UPLOAD_BYTES", 10 * 1024 * 1024)
|
|
MAX_IMAGE_PIXELS = env_int("MAX_IMAGE_PIXELS", 40_000_000)
|
|
UPLOAD_TIMEOUT_SECONDS = env_int("UPLOAD_TIMEOUT_SECONDS", 60)
|
|
UPLOAD_IDLE_TIMEOUT_SECONDS = env_int("UPLOAD_IDLE_TIMEOUT_SECONDS", 10)
|
|
DECODE_CONCURRENCY = env_int("DECODE_CONCURRENCY", 2)
|
|
UPLOAD_TMP_DIR = env_str("UPLOAD_TMP_DIR", "/tmp/uploads")
|
|
|
|
# Очистка незавершённых загрузок (janitor.py).
|
|
ATTACHMENT_GRACE_SECONDS = env_int("ATTACHMENT_GRACE_SECONDS", 3600)
|
|
JANITOR_INTERVAL_SECONDS = env_int("JANITOR_INTERVAL_SECONDS", 300)
|
|
|
|
SHUTDOWN_DRAIN_SECONDS = env_int("SHUTDOWN_DRAIN_SECONDS", 3)
|
|
SHUTDOWN_TIMEOUT_SECONDS = env_int("SHUTDOWN_TIMEOUT_SECONDS", 10)
|
|
|
|
# lab5: брокер (RabbitMQ) и фоновая обработка вложений.
|
|
# У API адреса брокера нет вообще: API пишет задание в outbox (PostgreSQL), публикует dispatcher.
|
|
BROKER_URL = read_secret("BROKER_URL_FILE", "BROKER_URL")
|
|
BROKER_HEARTBEAT_SECONDS = env_int("BROKER_HEARTBEAT_SECONDS", 30)
|
|
PIPELINE_VERSION = env_int("PIPELINE_VERSION", 1)
|
|
WORKER_CONCURRENCY = env_int("WORKER_CONCURRENCY", 2) # потоков-потребителей в одном worker
|
|
WORKER_PREFETCH = env_int("WORKER_PREFETCH", 1) # неподтверждённых сообщений на поток
|
|
JOB_MAX_ATTEMPTS = env_int("JOB_MAX_ATTEMPTS", 5)
|
|
JOB_TIMEOUT_SECONDS = env_int("JOB_TIMEOUT_SECONDS", 60) # жёсткий предел одной обработки изображения
|
|
JOB_LEASE_SECONDS = env_int("JOB_LEASE_SECONDS", 15) # аренда processing, продлевается каждые lease/3
|
|
JOB_BACKOFF_BASE_SECONDS = env_int("JOB_BACKOFF_BASE_SECONDS", 2)
|
|
JOB_BACKOFF_MAX_SECONDS = env_int("JOB_BACKOFF_MAX_SECONDS", 60)
|
|
JOB_RECONCILE_SECONDS = env_int("JOB_RECONCILE_SECONDS", 120) # queued без сообщения дольше -> опубликовать заново
|
|
WORKER_MEMORY_LIMIT_MB = env_int("WORKER_MEMORY_LIMIT_MB", 640) # RLIMIT_AS процесса обработки
|
|
WORKER_TMP_DIR = env_str("WORKER_TMP_DIR", "/tmp/work")
|
|
DISPATCHER_POLL_SECONDS = env_int("DISPATCHER_POLL_SECONDS", 1)
|
|
DISPATCHER_BATCH = env_int("DISPATCHER_BATCH", 100)
|
|
STAGING_RETENTION_SECONDS = env_int("STAGING_RETENTION_SECONDS", 86400) # исходник failed-вложения для replay
|
|
OUTBOX_RETENTION_SECONDS = env_int("OUTBOX_RETENTION_SECONDS", 7 * 86400)
|
|
ALIVE_FILE = env_str("ALIVE_FILE", "/tmp/alive") # healthcheck фоновых процессов
|
|
|
|
# Учебное внедрение сбоев (faults.py). На стенде включено, в эксплуатации FAULT_INJECTION=0.
|
|
FAULT_INJECTION = env_str("FAULT_INJECTION", "0") == "1"
|
|
FAULTS_DIR = env_str("FAULTS_DIR", "/faults")
|
|
|
|
|
|
def check() -> None:
|
|
"""Проверка при старте: лучше сразу упасть с понятной ошибкой, чем на первом запросе."""
|
|
if not DATABASE_URL:
|
|
raise RuntimeError("PostgreSQL address is not set: DATABASE_URL_FILE")
|
|
if not REDIS_URL:
|
|
raise RuntimeError("Redis address is not set: CACHE_URL_FILE")
|
|
if not S3_ACCESS_KEY_ID or not S3_SECRET_ACCESS_KEY:
|
|
raise RuntimeError("S3 credentials are not set: S3_ACCESS_KEY_ID_FILE, S3_SECRET_ACCESS_KEY_FILE")
|
|
if ATTACHMENT_GRACE_SECONDS < 2 * UPLOAD_TIMEOUT_SECONDS:
|
|
raise RuntimeError("ATTACHMENT_GRACE_SECONDS must be at least twice UPLOAD_TIMEOUT_SECONDS")
|
|
if SESSION_STORE != "redis":
|
|
raise RuntimeError("SESSION_STORE: only 'redis' is supported in this solution")
|