Files
mtusi2026lab5/src/messenger/config.py
T
Alexey Bodanyuk a633949919 lab5: схема 0005 (jobs, outbox, попытки, варианты, аудит) и модули pipeline
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Миграция только добавляет: статусы pending/queued/processing/ready/failed, staging-ключ,
published_version, jobs с lease и токеном попытки, job_attempts с заранее известными
ключами, attachment_variants (PK = ключ эффекта), outbox, job_events, view failed_jobs,
post_message_v3. CHECK attachments_active_has_source: у queued/processing всегда есть источник.
pipeline.py — версии 1/2/3 и геометрия по контракту; imaging.py — варианты в дочернем
процессе (RLIMIT_AS, таймаут, лимит пикселей); jobs.py — все переходы заданий одной
транзакцией с fencing по токену; broker.py — топология и строгая схема сообщения.
Зависимости: aio-pika 9.6.2, pillow 12.3.0 (lock с хешами).
2026-10-31 16:12:51 +03:00

344 lines
17 KiB
Python

"""Runtime configuration read from environment variables (12-factor style).
Non-secret settings have safe defaults. Connection strings with passwords are
read from files (`DATABASE_URL_FILE`, `CACHE_URL_FILE` → Docker secrets in
/run/secrets), so they never appear in `docker inspect`, compose files or logs.
Invalid values stop the process at startup with a clear message.
"""
from __future__ import annotations
import os
import re
import socket
from dataclasses import dataclass, field
from pathlib import Path
class ConfigError(ValueError):
pass
INSTANCE_ID_RE = re.compile(r"[A-Za-z0-9._-]{1,64}")
def _instance_id() -> str:
"""X-Instance-Id of this process: INSTANCE_ID or the container hostname.
Docker sets the hostname of a container to its short container ID, so the value
is unique per replica, stable for the life of the process and maps 1:1 to a real
container (`docker inspect`), which scripts/stop-one.sh relies on.
"""
value = os.environ.get("INSTANCE_ID", "").strip() or socket.gethostname()
if INSTANCE_ID_RE.fullmatch(value) is None:
raise ConfigError("INSTANCE_ID must match [A-Za-z0-9._-]{1,64}")
return value
def _int(name: str, default: int, low: int, high: int) -> int:
raw = os.environ.get(name)
if raw is None or raw.strip() == "":
return default
try:
value = int(raw.strip())
except ValueError:
raise ConfigError(f"{name} must be an integer, got {raw!r}") from None
if not low <= value <= high:
raise ConfigError(f"{name} must be in [{low}, {high}], got {value}")
return value
def _float(name: str, default: float, low: float, high: float) -> float:
raw = os.environ.get(name)
if raw is None or raw.strip() == "":
return default
try:
value = float(raw.strip())
except ValueError:
raise ConfigError(f"{name} must be a number, got {raw!r}") from None
if not low <= value <= high:
raise ConfigError(f"{name} must be in [{low}, {high}], got {value}")
return value
def _bool(name: str, default: bool) -> bool:
raw = os.environ.get(name, "").strip().lower()
if raw == "":
return default
if raw not in {"true", "false", "1", "0"}:
raise ConfigError(f"{name} must be true/false, got {raw!r}")
return raw in {"true", "1"}
def _choice(name: str, default: str, allowed: set[str]) -> str:
value = os.environ.get(name, "").strip().lower() or default
if value not in allowed:
raise ConfigError(f"{name} must be one of {sorted(allowed)}, got {value!r}")
return value
def _secret(name: str, required: bool) -> str:
"""Value of <NAME>_FILE (preferred, Docker secret) or <NAME> (local runs/tests)."""
path = os.environ.get(f"{name}_FILE", "").strip()
if path:
try:
value = Path(path).read_text(encoding="utf-8").strip()
except OSError as exc:
# The message names the file, never its content.
raise ConfigError(f"{name}_FILE={path} cannot be read: {exc.strerror}") from None
else:
value = os.environ.get(name, "").strip()
if required and not value:
raise ConfigError(f"{name}_FILE (or {name}) must be set")
return value
@dataclass(frozen=True)
class Settings:
host: str
port: int
env: str
log_level: str
shutdown_timeout: float
max_body_bytes: int
docs_enabled: bool = True
database_url: str = ""
session_store: str = "postgres"
cache_url: str = ""
session_ttl_seconds: int = 3600
db_pool_min: int = 1
db_pool_max: int = 10
db_timeout_seconds: float = 3.0
cache_timeout_seconds: float = 0.5
hash_concurrency: int = 2
instance_id: str = "local"
drain_seconds: float = 0.0
idempotency_ttl_seconds: int = 86400
stats_log_seconds: float = 0.0
message_write_path: str = "function"
# Lab 4: private S3 bucket and attachment limits (see storage.py, attachments.py).
s3_endpoint: str = "http://s3:8333"
s3_region: str = "us-east-1"
s3_bucket: str = "messenger-attachments"
s3_access_key_id: str = ""
s3_secret_access_key: str = ""
s3_timeout_seconds: float = 3.0
s3_connect_timeout_seconds: float = 1.0
s3_max_connections: int = 20
max_upload_bytes: int = 10 * 1024 * 1024
max_image_pixels: int = 40_000_000
upload_concurrency: int = 4
upload_queue_timeout_seconds: float = 5.0
upload_read_timeout_seconds: float = 30.0
image_verify_timeout_seconds: float = 10.0
image_verify_concurrency: int = 1
jpeg_max_memory_mb: int = 256
pending_grace_seconds: int = 900
janitor_interval_seconds: float = 60.0
# Lab 5: staging bucket (API writes, worker reads) and the asynchronous pipeline (jobs_settings()).
s3_staging_bucket: str = "messenger-staging"
broker_url: str = ""
pipeline_version: int = 1
job_max_attempts: int = 5
job_timeout_seconds: float = 60.0
job_lease_seconds: float = 30.0
job_retry_base_seconds: float = 2.0
job_retry_max_seconds: float = 60.0
worker_concurrency: int = 2
worker_shutdown_seconds: float = 20.0
image_max_memory_mb: int = 768
staging_retention_seconds: int = 86400
orphan_grace_seconds: int = 600
cleanup_interval_seconds: float = 30.0
relay_poll_seconds: float = 0.5
relay_batch: int = 50
queued_republish_seconds: float = 300.0
fault_points: frozenset[str] = frozenset()
fault_values: dict[str, int] = field(default_factory=dict)
def fault_value(self, point: str, default: int = 0) -> int:
return self.fault_values.get(point, default)
@property
def db_statement_timeout_ms(self) -> int:
# The server stops a statement a bit before the client-side deadline fires.
return max(100, int(self.db_timeout_seconds * 1000) - 500)
@classmethod
def from_env(cls, *, s3_credentials: bool = True) -> "Settings":
"""s3_credentials=False: processes without S3 access (relay, jobctl)."""
log_level = os.environ.get("LOG_LEVEL", "info").strip().lower() or "info"
if log_level not in {"debug", "info", "warning", "error"}:
raise ConfigError(f"LOG_LEVEL must be debug/info/warning/error, got {log_level!r}")
session_store = os.environ.get("SESSION_STORE", "postgres").strip().lower() or "postgres"
if session_store == "redis":
session_store = "valkey" # same protocol; the course allows Redis OR Valkey
if session_store not in {"postgres", "valkey"}:
raise ConfigError(f"SESSION_STORE must be postgres or valkey, got {session_store!r}")
db_pool_min = _int("DB_POOL_MIN", 1, 0, 50)
db_pool_max = _int("DB_POOL_MAX", 10, 1, 100)
if db_pool_min > db_pool_max:
raise ConfigError("DB_POOL_MIN must not exceed DB_POOL_MAX")
return cls(
host=os.environ.get("APP_HOST", "0.0.0.0").strip() or "0.0.0.0",
port=_int("APP_PORT", 8080, 1, 65535),
env=os.environ.get("APP_ENV", "lab").strip() or "lab",
log_level=log_level,
# DRAIN_SECONDS + SHUTDOWN_TIMEOUT_SECONDS must stay below compose stop_grace_period.
shutdown_timeout=_float("SHUTDOWN_TIMEOUT_SECONDS", 8.0, 0.0, 30.0),
max_body_bytes=_int("MAX_BODY_BYTES", 64 * 1024, 1024, 1024 * 1024),
docs_enabled=_bool("DOCS_ENABLED", True),
database_url=_secret("DATABASE_URL", required=True),
session_store=session_store,
cache_url=_secret("CACHE_URL", required=session_store == "valkey"),
# Contract name. Minimum 1 s so the expiry test can use 2 s.
session_ttl_seconds=_int("SESSION_TTL_SECONDS", 3600, 1, 30 * 24 * 3600),
db_pool_min=db_pool_min,
db_pool_max=db_pool_max,
db_timeout_seconds=_float("DB_TIMEOUT_SECONDS", 3.0, 0.5, 30.0),
cache_timeout_seconds=_float("CACHE_TIMEOUT_SECONDS", 0.5, 0.05, 10.0),
hash_concurrency=_int("PASSWORD_HASH_CONCURRENCY", 2, 1, 16),
instance_id=_instance_id(),
# After SIGTERM: readiness is 503 but requests are still served for this long,
# so the load balancer stops routing here before the listener closes.
drain_seconds=_float("DRAIN_SECONDS", 0.0, 0.0, 10.0),
# Contract: a replay within 24 h returns the original result.
idempotency_ttl_seconds=_int("IDEMPOTENCY_TTL_SECONDS", 86400, 86400, 30 * 24 * 3600),
# Experiments (bench/track1): log DB pool counters and process CPU every N s; 0 = off.
stats_log_seconds=_float("STATS_LOG_SECONDS", 0.0, 0.0, 60.0),
# Track 1: `function` = one SQL call per send (migration 0003), `multi` = the previous
# multi-statement transaction; both have identical semantics (A/B on one image).
message_write_path=_choice("MESSAGE_WRITE_PATH", "function", {"function", "multi"}),
**s3_settings(credentials_required=s3_credentials),
**jobs_settings(broker_required=False),
)
# Contract limits (contracts/README.md): configurable DOWN for experiments, never above them.
CONTRACT_MAX_UPLOAD_BYTES = 10 * 1024 * 1024
CONTRACT_MAX_IMAGE_PIXELS = 40_000_000
# Failure points for the experiments only (FAULT_POINTS of ONE service, compose variables
# API_/RELAY_/WORKER_FAULT_POINTS); empty in normal runs. `name` or `name=<integer>`.
FAULT_POINTS = {
# API, upload: between the staging PUT and the job COMMIT / after the job COMMIT, before 202.
"after_s3_put", "after_enqueue_commit",
# Relay: crash after the broker confirmed a publish but before outbox.published_at is
# committed; publish every message twice; publish a batch in reverse order.
"crash_after_publish", "duplicate_publish", "reverse_order",
# Worker: crash after the claim (before any side effect), after the S3 writes (before the
# result COMMIT), after the result COMMIT (before the ack); transient failure of the first N
# attempts; permanent failure; slow processing (N s, lease kept alive) of every job or only of
# pipeline version `slow_pipeline`; hang: stop the heartbeat and sleep N s inside processing.
"crash_after_claim", "crash_after_s3_write", "crash_after_commit", "transient_error", "permanent_error",
"slow", "slow_pipeline", "hang",
# Worker cleanup: pause N s after reading the candidates, before locking/deleting them.
"pause_cleanup",
}
def _faults() -> tuple[frozenset[str], dict[str, int]]:
names, values = set(), {}
for item in os.environ.get("FAULT_POINTS", "").split(","):
name, _, raw = item.strip().partition("=")
if not name:
continue
if name not in FAULT_POINTS:
raise ConfigError(f"FAULT_POINTS may contain only {sorted(FAULT_POINTS)}")
if raw:
if not raw.isdigit():
raise ConfigError(f"FAULT_POINTS value of {name} must be an integer")
values[name] = int(raw)
names.add(name)
return frozenset(names), values
def _bucket(name: str, default: str) -> str:
bucket = os.environ.get(name, default).strip() or default
if not re.fullmatch(r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]", bucket):
raise ConfigError(f"{name} is not a valid bucket name: {bucket!r}")
return bucket
def s3_settings(*, credentials_required: bool = True) -> dict:
"""S3 part of Settings; also used by the bootstrap CLI with its own keys."""
faults, fault_values = _faults()
endpoint = os.environ.get("S3_ENDPOINT", "http://s3:8333").strip() or "http://s3:8333"
if not re.fullmatch(r"https?://[A-Za-z0-9.\-]+(:[0-9]{1,5})?/?", endpoint):
raise ConfigError(f"S3_ENDPOINT must be http(s)://host[:port], got {endpoint!r}")
bucket = _bucket("S3_BUCKET", "messenger-attachments")
staging = _bucket("S3_STAGING_BUCKET", "messenger-staging")
if staging == bucket:
raise ConfigError("S3_STAGING_BUCKET must differ from S3_BUCKET")
return dict(
s3_endpoint=endpoint,
s3_region=os.environ.get("S3_REGION", "us-east-1").strip() or "us-east-1",
s3_bucket=bucket,
s3_staging_bucket=staging,
s3_access_key_id=_secret("S3_ACCESS_KEY_ID", required=credentials_required),
s3_secret_access_key=_secret("S3_SECRET_ACCESS_KEY", required=credentials_required),
s3_timeout_seconds=_float("S3_TIMEOUT_SECONDS", 3.0, 0.2, 30.0),
s3_connect_timeout_seconds=_float("S3_CONNECT_TIMEOUT_SECONDS", 1.0, 0.1, 10.0),
s3_max_connections=_int("S3_MAX_CONNECTIONS", 20, 1, 200),
max_upload_bytes=_int("MAX_UPLOAD_BYTES", CONTRACT_MAX_UPLOAD_BYTES, 1024, CONTRACT_MAX_UPLOAD_BYTES),
max_image_pixels=_int("MAX_IMAGE_PIXELS", CONTRACT_MAX_IMAGE_PIXELS, 1, CONTRACT_MAX_IMAGE_PIXELS),
# At most this many request bodies of one replica are buffered/checked at once (RAM bound:
# UPLOAD_CONCURRENCY x MAX_UPLOAD_BYTES); further uploads wait, then get 503.
upload_concurrency=_int("UPLOAD_CONCURRENCY", 4, 1, 64),
upload_queue_timeout_seconds=_float("UPLOAD_QUEUE_TIMEOUT_SECONDS", 5.0, 0.0, 60.0),
upload_read_timeout_seconds=_float("UPLOAD_READ_TIMEOUT_SECONDS", 30.0, 1.0, 300.0),
image_verify_timeout_seconds=_float("IMAGE_VERIFY_TIMEOUT_SECONDS", 10.0, 1.0, 120.0),
# Full checks running at once per replica: a strict JPEG decode of a 40 Mpx progressive
# file needs up to ~120-240 MB of coefficient buffers (capped by JPEG_MAX_MEMORY_MB).
image_verify_concurrency=_int("IMAGE_VERIFY_CONCURRENCY", 1, 1, 16),
jpeg_max_memory_mb=_int("JPEG_MAX_MEMORY_MB", 256, 16, 2048),
# An intent row still `pending` after this long belongs to a failed upload: the janitor
# deletes its object and the row. Far above the longest possible upload (~45 s).
pending_grace_seconds=_int("PENDING_GRACE_SECONDS", 900, 1, 7 * 24 * 3600),
janitor_interval_seconds=_float("JANITOR_INTERVAL_SECONDS", 60.0, 0.0, 3600.0),
fault_points=faults,
fault_values=fault_values,
)
def jobs_settings(*, broker_required: bool) -> dict:
"""Lab 5 pipeline: retries, lease, worker limits, cleanup, relay (docs/PIPELINE.md)."""
from .pipeline import PIPELINES # local import: pipeline.py imports nothing from here
timeout = _float("JOB_TIMEOUT_SECONDS", 60.0, 1.0, 600.0)
lease = _float("JOB_LEASE_SECONDS", 30.0, 2.0, 600.0)
base = _float("JOB_RETRY_BASE_SECONDS", 2.0, 0.0, 600.0)
cap = _float("JOB_RETRY_MAX_SECONDS", 60.0, 0.0, 3600.0)
if base > cap:
raise ConfigError("JOB_RETRY_BASE_SECONDS must not exceed JOB_RETRY_MAX_SECONDS")
orphan_grace = _int("ORPHAN_GRACE_SECONDS", 600, 1, 7 * 24 * 3600)
version = _int("PIPELINE_VERSION", 1, 1, 1000)
if version not in PIPELINES:
raise ConfigError(f"PIPELINE_VERSION must be one of {sorted(PIPELINES)}")
return dict(
broker_url=_secret("BROKER_URL", required=broker_required),
# Pipeline version of the jobs created by new uploads (track 2: versioned pipeline).
pipeline_version=version,
# Managed attempts (claims) per job before failed/retry_exhausted.
job_max_attempts=_int("JOB_MAX_ATTEMPTS", 5, 1, 100),
# Hard limit of ONE attempt: the image child process is killed, the lease is no longer renewed.
job_timeout_seconds=timeout,
# Lease of a claimed job, renewed by the heartbeat every lease/3 while the attempt runs.
job_lease_seconds=lease,
# Backoff of transient failures: full jitter, uniform(0, min(max, base * 2^(attempt-1))).
job_retry_base_seconds=base,
job_retry_max_seconds=cap,
# Jobs processed at once by ONE worker process = RabbitMQ prefetch (basic.qos).
worker_concurrency=_int("WORKER_CONCURRENCY", 2, 1, 32),
worker_shutdown_seconds=_float("WORKER_SHUTDOWN_SECONDS", 20.0, 0.0, 120.0),
# Address-space limit (RLIMIT_AS) of the child process that decodes/encodes ONE image.
image_max_memory_mb=_int("IMAGE_MAX_MEMORY_MB", 768, 128, 8192),
# Staging copy of a FAILED attachment is kept this long for a replay, then deleted.
staging_retention_seconds=_int("STAGING_RETENTION_SECONDS", 86400, 0, 30 * 24 * 3600),
# Objects of attempts that did not publish are deleted once the attempt ended this long ago.
orphan_grace_seconds=orphan_grace,
cleanup_interval_seconds=_float("CLEANUP_INTERVAL_SECONDS", 30.0, 0.0, 3600.0),
relay_poll_seconds=_float("RELAY_POLL_SECONDS", 0.5, 0.05, 60.0),
relay_batch=_int("RELAY_BATCH", 50, 1, 1000),
# A queued job whose message has not led to a claim for this long is published again
# (message lost in the broker, dead-lettered, purged): no accepted job waits forever.
queued_republish_seconds=_float("QUEUED_REPUBLISH_SECONDS", 300.0, 5.0, 86400.0),
)