forked from mtusi2026/mtusi2026lab5
RabbitMQ + transactional outbox, отдельный worker (thumbnail/crop/WebP), lease и heartbeat, retry с backoff+jitter, DLQ и replay, admission control; тесты отказов, трек 1 и отчёт.
99 lines
4.3 KiB
Python
99 lines
4.3 KiB
Python
import os
|
|
import socket
|
|
|
|
|
|
def _int(name, default):
|
|
value = os.environ.get(name)
|
|
return int(value) if value not in (None, "") else default
|
|
|
|
|
|
def _float(name, default):
|
|
value = os.environ.get(name)
|
|
return float(value) if value not in (None, "") else default
|
|
|
|
|
|
def _secret(name, default=""):
|
|
path = os.environ.get(name + "_FILE")
|
|
if path:
|
|
with open(path, encoding="utf-8") as handle:
|
|
return handle.read().strip()
|
|
return os.environ.get(name, default)
|
|
|
|
|
|
DATABASE_URL = _secret("DATABASE_URL")
|
|
CACHE_URL = _secret("CACHE_URL")
|
|
SESSION_STORE = os.environ.get("SESSION_STORE", "postgres")
|
|
SESSION_TTL_SECONDS = _int("SESSION_TTL_SECONDS", 3600)
|
|
SESSION_CACHE_TTL_SECONDS = _int("SESSION_CACHE_TTL_SECONDS", 30)
|
|
SESSION_SWEEP_SECONDS = _int("SESSION_SWEEP_SECONDS", 60)
|
|
DB_POOL_MIN = _int("DB_POOL_MIN", 1)
|
|
DB_POOL_MAX = _int("DB_POOL_MAX", 10)
|
|
DB_POOL_TIMEOUT = _float("DB_POOL_TIMEOUT", 3.0)
|
|
DB_CONNECT_TIMEOUT = _int("DB_CONNECT_TIMEOUT", 3)
|
|
DB_STATEMENT_TIMEOUT_MS = _int("DB_STATEMENT_TIMEOUT_MS", 5000)
|
|
DB_OP_TIMEOUT = _float("DB_OP_TIMEOUT", 6.0)
|
|
REQUEST_TIMEOUT = _float("REQUEST_TIMEOUT", 8.0)
|
|
DB_READY_TIMEOUT = _float("DB_READY_TIMEOUT", 1.5)
|
|
READY_CHECK_SECONDS = _float("READY_CHECK_SECONDS", 1.0)
|
|
READY_FAILURES = _int("READY_FAILURES", 2)
|
|
CACHE_TIMEOUT = _float("CACHE_TIMEOUT", 0.2)
|
|
MAX_INFLIGHT_REQUESTS = _int("MAX_INFLIGHT_REQUESTS", 64)
|
|
MAX_BODY_BYTES = _int("MAX_BODY_BYTES", 65536)
|
|
MAX_RSS_MB = _int("MAX_RSS_MB", 0)
|
|
PASSWORD_MIN = 12
|
|
PASSWORD_MAX = 128
|
|
SESSION_REVOKED_TTL_SECONDS = max(
|
|
_int("SESSION_REVOKED_TTL_SECONDS", 60),
|
|
SESSION_CACHE_TTL_SECONDS + int(REQUEST_TIMEOUT) + 5,
|
|
)
|
|
INSTANCE_ID = os.environ.get("INSTANCE_ID") or socket.gethostname()
|
|
IDEMPOTENCY_TTL_SECONDS = _int("IDEMPOTENCY_TTL_SECONDS", 86400)
|
|
|
|
S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "http://s3:9000")
|
|
S3_REGION = os.environ.get("S3_REGION", "us-east-1")
|
|
S3_BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments")
|
|
S3_ACCESS_KEY_ID = _secret("S3_ACCESS_KEY_ID")
|
|
S3_SECRET_ACCESS_KEY = _secret("S3_SECRET_ACCESS_KEY")
|
|
MAX_UPLOAD_BYTES = _int("MAX_UPLOAD_BYTES", 10485760)
|
|
MAX_IMAGE_PIXELS = _int("MAX_IMAGE_PIXELS", 40000000)
|
|
S3_CONNECT_TIMEOUT = _float("S3_CONNECT_TIMEOUT", 3.0)
|
|
S3_READ_TIMEOUT = _float("S3_READ_TIMEOUT", 6.0)
|
|
UPLOAD_SPOOL_BYTES = _int("UPLOAD_SPOOL_BYTES", 1048576)
|
|
RECONCILE_INTERVAL_SECONDS = _int("RECONCILE_INTERVAL_SECONDS", 300)
|
|
RECONCILE_GRACE_SECONDS = _int("RECONCILE_GRACE_SECONDS", 900)
|
|
UPLOAD_CONCURRENCY = _int("UPLOAD_CONCURRENCY", 2)
|
|
|
|
# --- lab5: broker, jobs, worker ---
|
|
BROKER_URL = _secret("BROKER_URL")
|
|
BROKER_EXCHANGE = os.environ.get("BROKER_EXCHANGE", "attachments")
|
|
BROKER_QUEUE = os.environ.get("BROKER_QUEUE", "attachments.process")
|
|
BROKER_DLX = os.environ.get("BROKER_DLX", "attachments.dlx")
|
|
BROKER_DLQ = os.environ.get("BROKER_DLQ", "attachments.dlq")
|
|
BROKER_DELIVERY_LIMIT = _int("BROKER_DELIVERY_LIMIT", 20)
|
|
PIPELINE_VERSION = _int("PIPELINE_VERSION", 1)
|
|
WORKER_CONCURRENCY = _int("WORKER_CONCURRENCY", 2)
|
|
WORKER_PREFETCH = _int("WORKER_PREFETCH", 0) or WORKER_CONCURRENCY
|
|
JOB_MAX_ATTEMPTS = _int("JOB_MAX_ATTEMPTS", 5)
|
|
JOB_TIMEOUT_SECONDS = _int("JOB_TIMEOUT_SECONDS", 60)
|
|
JOB_LEASE_SECONDS = _int("JOB_LEASE_SECONDS", 20)
|
|
JOB_BACKOFF_BASE_SECONDS = _float("JOB_BACKOFF_BASE_SECONDS", 2.0)
|
|
JOB_BACKOFF_MAX_SECONDS = _float("JOB_BACKOFF_MAX_SECONDS", 30.0)
|
|
JOB_SWEEP_SECONDS = _float("JOB_SWEEP_SECONDS", 5.0)
|
|
WORKER_CHILD_MEMORY_MB = _int("WORKER_CHILD_MEMORY_MB", 768)
|
|
WORKER_GRACE_SECONDS = _int("WORKER_GRACE_SECONDS", 25)
|
|
WORKER_HEARTBEAT_FILE = os.environ.get("WORKER_HEARTBEAT_FILE", "/tmp/worker-heartbeat")
|
|
OUTBOX_POLL_SECONDS = _float("OUTBOX_POLL_SECONDS", 1.0)
|
|
OUTBOX_BATCH = _int("OUTBOX_BATCH", 50)
|
|
OUTBOX_RETENTION_SECONDS = _int("OUTBOX_RETENTION_SECONDS", 86400)
|
|
STAGING_RETENTION_SECONDS = _int("STAGING_RETENTION_SECONDS", 86400)
|
|
# Admission control: uploads get 503 while this many jobs are queued/processing (0 disables).
|
|
MAX_PENDING_JOBS = _int("MAX_PENDING_JOBS", 200)
|
|
PENDING_CHECK_SECONDS = _float("PENDING_CHECK_SECONDS", 1.0)
|
|
THUMBNAIL_SIZE = 256
|
|
CROP_SIZE = 128
|
|
OPTIMIZED_SIZE = 1280
|
|
JPEG_QUALITY = _int("JPEG_QUALITY", 85)
|
|
WEBP_QUALITY = _int("WEBP_QUALITY", 80)
|
|
# Test stand only: filename markers like "fault-worker_crash_after_put" trigger crashes (see app/faults.py).
|
|
FAULT_INJECTION = os.environ.get("FAULT_INJECTION", "0") == "1"
|