forked from mtusi2026/mtusi2026lab5
96 lines
4.0 KiB
Python
96 lines
4.0 KiB
Python
from __future__ import annotations
|
|
|
|
import os
|
|
import secrets
|
|
from typing import Optional
|
|
|
|
|
|
def _secret(name: str) -> Optional[str]:
|
|
value = os.environ.get(name)
|
|
if value:
|
|
return value.strip()
|
|
path = os.environ.get(name + "_FILE")
|
|
if path and os.path.exists(path):
|
|
with open(path, encoding="utf-8") as fh:
|
|
return fh.read().strip() or None
|
|
return None
|
|
|
|
|
|
def _int(name: str, default: int) -> int:
|
|
return int(os.environ.get(name, default))
|
|
|
|
|
|
def _float(name: str, default: float) -> float:
|
|
return float(os.environ.get(name, default))
|
|
|
|
|
|
DATABASE_URL = _secret("DATABASE_URL")
|
|
CACHE_URL = _secret("CACHE_URL")
|
|
|
|
S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "http://s3:8333")
|
|
S3_REGION = os.environ.get("S3_REGION", "us-east-1")
|
|
S3_BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments")
|
|
S3_ACCESS_KEY_ID = _secret("S3_ACCESS_KEY_ID")
|
|
S3_SECRET_ACCESS_KEY = _secret("S3_SECRET_ACCESS_KEY")
|
|
S3_CONNECT_TIMEOUT = _float("S3_CONNECT_TIMEOUT", 2)
|
|
S3_READ_TIMEOUT = _float("S3_READ_TIMEOUT", 5)
|
|
S3_POOL_MAX = _int("S3_POOL_MAX", 20)
|
|
# Without this cap 20 parallel 10 MiB PUTs drove SeaweedFS (512M limit) into an OOM kill.
|
|
S3_PUT_CONCURRENCY = _int("S3_PUT_CONCURRENCY", 4)
|
|
|
|
SESSION_TTL_SECONDS = _int("SESSION_TTL_SECONDS", 3600)
|
|
SESSION_CACHE_MAX_SECONDS = _int("SESSION_CACHE_MAX_SECONDS", 300)
|
|
|
|
DB_POOL_MAX = _int("DB_POOL_MAX", 10)
|
|
DB_CONNECT_TIMEOUT = _float("DB_CONNECT_TIMEOUT", 3)
|
|
DB_ACQUIRE_TIMEOUT = _float("DB_ACQUIRE_TIMEOUT", 3)
|
|
DB_COMMAND_TIMEOUT = _float("DB_COMMAND_TIMEOUT", 5)
|
|
|
|
CACHE_TIMEOUT = _float("CACHE_TIMEOUT", 0.3)
|
|
CACHE_POOL_MAX = _int("CACHE_POOL_MAX", 20)
|
|
CACHE_RETRY_AFTER = _float("CACHE_RETRY_AFTER", 5)
|
|
|
|
PASSWORD_HASH_CONCURRENCY = _int("PASSWORD_HASH_CONCURRENCY", 4)
|
|
|
|
IDEMPOTENCY_TTL_SECONDS = _int("IDEMPOTENCY_TTL_SECONDS", 86400)
|
|
|
|
MAX_UPLOAD_BYTES = _int("MAX_UPLOAD_BYTES", 10 * 1024 * 1024)
|
|
MAX_IMAGE_PIXELS = _int("MAX_IMAGE_PIXELS", 40_000_000)
|
|
UPLOAD_SPOOL_BYTES = _int("UPLOAD_SPOOL_BYTES", 1024 * 1024)
|
|
UPLOAD_TMP_DIR = os.environ.get("UPLOAD_TMP_DIR") or None
|
|
UPLOAD_TIMEOUT_SECONDS = _float("UPLOAD_TIMEOUT_SECONDS", 60)
|
|
UPLOAD_CONCURRENCY = _int("UPLOAD_CONCURRENCY", 16)
|
|
DECODE_CONCURRENCY = _int("DECODE_CONCURRENCY", 1)
|
|
|
|
ORPHAN_GRACE_SECONDS = _int("ORPHAN_GRACE_SECONDS", 900)
|
|
SWEEP_INTERVAL_SECONDS = _int("SWEEP_INTERVAL_SECONDS", 60)
|
|
|
|
INSTANCE_ID = "api-" + secrets.token_hex(4)
|
|
INSTANCE_ID_FILE = os.environ.get("INSTANCE_ID_FILE", "/tmp/instance-id")
|
|
|
|
# --- lab5: broker, worker, retries ---------------------------------------------------------------
|
|
BROKER_URL = _secret("BROKER_URL")
|
|
BROKER_EXCHANGE = os.environ.get("BROKER_EXCHANGE", "attachments")
|
|
BROKER_QUEUE = os.environ.get("BROKER_QUEUE", "attachments.process")
|
|
BROKER_DLX = os.environ.get("BROKER_DLX", "attachments.dlx")
|
|
BROKER_DLQ = os.environ.get("BROKER_DLQ", "attachments.dlq")
|
|
BROKER_ROUTING_KEY = "attachment.process.v1"
|
|
# Poison-message guard of the quorum queue: a message redelivered more often goes to the DLQ.
|
|
BROKER_DELIVERY_LIMIT = _int("BROKER_DELIVERY_LIMIT", 10)
|
|
|
|
PIPELINE_VERSION = _int("PIPELINE_VERSION", 1)
|
|
WORKER_CONCURRENCY = _int("WORKER_CONCURRENCY", 2) # = AMQP prefetch = parallel child processes
|
|
JOB_MAX_ATTEMPTS = _int("JOB_MAX_ATTEMPTS", 5)
|
|
JOB_TIMEOUT_SECONDS = _float("JOB_TIMEOUT_SECONDS", 60) # hard limit of one image transformation
|
|
LEASE_SECONDS = _float("LEASE_SECONDS", 20) # processing lease, renewed by heartbeat
|
|
HEARTBEAT_SECONDS = _float("HEARTBEAT_SECONDS", 5)
|
|
REAPER_INTERVAL_SECONDS = _float("REAPER_INTERVAL_SECONDS", 2)
|
|
STUCK_QUEUED_SECONDS = _float("STUCK_QUEUED_SECONDS", 300) # republish a queued job whose message vanished
|
|
RETRY_BASE_SECONDS = _float("RETRY_BASE_SECONDS", 2)
|
|
RETRY_MAX_SECONDS = _float("RETRY_MAX_SECONDS", 60)
|
|
WORKER_MEMORY_LIMIT_MB = _int("WORKER_MEMORY_LIMIT_MB", 768) # RLIMIT_AS of one transformation process
|
|
STAGING_RETENTION_SECONDS = _int("STAGING_RETENTION_SECONDS", 86400)
|
|
RELAY_BATCH = _int("RELAY_BATCH", 100)
|
|
RELAY_POLL_SECONDS = _float("RELAY_POLL_SECONDS", 1)
|
|
FAULT_INJECTION = os.environ.get("FAULT_INJECTION", "0") == "1"
|