forked from mtusi2026/mtusi2026lab5
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Миграция только добавляет: статусы pending/queued/processing/ready/failed, staging-ключ, published_version, jobs с lease и токеном попытки, job_attempts с заранее известными ключами, attachment_variants (PK = ключ эффекта), outbox, job_events, view failed_jobs, post_message_v3. CHECK attachments_active_has_source: у queued/processing всегда есть источник. pipeline.py — версии 1/2/3 и геометрия по контракту; imaging.py — варианты в дочернем процессе (RLIMIT_AS, таймаут, лимит пикселей); jobs.py — все переходы заданий одной транзакцией с fencing по токену; broker.py — топология и строгая схема сообщения. Зависимости: aio-pika 9.6.2, pillow 12.3.0 (lock с хешами).
344 lines
17 KiB
Python
344 lines
17 KiB
Python
"""Runtime configuration read from environment variables (12-factor style).
|
|
|
|
Non-secret settings have safe defaults. Connection strings with passwords are
|
|
read from files (`DATABASE_URL_FILE`, `CACHE_URL_FILE` → Docker secrets in
|
|
/run/secrets), so they never appear in `docker inspect`, compose files or logs.
|
|
Invalid values stop the process at startup with a clear message.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import re
|
|
import socket
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
|
|
|
|
class ConfigError(ValueError):
|
|
pass
|
|
|
|
|
|
INSTANCE_ID_RE = re.compile(r"[A-Za-z0-9._-]{1,64}")
|
|
|
|
|
|
def _instance_id() -> str:
|
|
"""X-Instance-Id of this process: INSTANCE_ID or the container hostname.
|
|
|
|
Docker sets the hostname of a container to its short container ID, so the value
|
|
is unique per replica, stable for the life of the process and maps 1:1 to a real
|
|
container (`docker inspect`), which scripts/stop-one.sh relies on.
|
|
"""
|
|
value = os.environ.get("INSTANCE_ID", "").strip() or socket.gethostname()
|
|
if INSTANCE_ID_RE.fullmatch(value) is None:
|
|
raise ConfigError("INSTANCE_ID must match [A-Za-z0-9._-]{1,64}")
|
|
return value
|
|
|
|
|
|
def _int(name: str, default: int, low: int, high: int) -> int:
|
|
raw = os.environ.get(name)
|
|
if raw is None or raw.strip() == "":
|
|
return default
|
|
try:
|
|
value = int(raw.strip())
|
|
except ValueError:
|
|
raise ConfigError(f"{name} must be an integer, got {raw!r}") from None
|
|
if not low <= value <= high:
|
|
raise ConfigError(f"{name} must be in [{low}, {high}], got {value}")
|
|
return value
|
|
|
|
|
|
def _float(name: str, default: float, low: float, high: float) -> float:
|
|
raw = os.environ.get(name)
|
|
if raw is None or raw.strip() == "":
|
|
return default
|
|
try:
|
|
value = float(raw.strip())
|
|
except ValueError:
|
|
raise ConfigError(f"{name} must be a number, got {raw!r}") from None
|
|
if not low <= value <= high:
|
|
raise ConfigError(f"{name} must be in [{low}, {high}], got {value}")
|
|
return value
|
|
|
|
|
|
def _bool(name: str, default: bool) -> bool:
|
|
raw = os.environ.get(name, "").strip().lower()
|
|
if raw == "":
|
|
return default
|
|
if raw not in {"true", "false", "1", "0"}:
|
|
raise ConfigError(f"{name} must be true/false, got {raw!r}")
|
|
return raw in {"true", "1"}
|
|
|
|
|
|
def _choice(name: str, default: str, allowed: set[str]) -> str:
|
|
value = os.environ.get(name, "").strip().lower() or default
|
|
if value not in allowed:
|
|
raise ConfigError(f"{name} must be one of {sorted(allowed)}, got {value!r}")
|
|
return value
|
|
|
|
|
|
def _secret(name: str, required: bool) -> str:
|
|
"""Value of <NAME>_FILE (preferred, Docker secret) or <NAME> (local runs/tests)."""
|
|
path = os.environ.get(f"{name}_FILE", "").strip()
|
|
if path:
|
|
try:
|
|
value = Path(path).read_text(encoding="utf-8").strip()
|
|
except OSError as exc:
|
|
# The message names the file, never its content.
|
|
raise ConfigError(f"{name}_FILE={path} cannot be read: {exc.strerror}") from None
|
|
else:
|
|
value = os.environ.get(name, "").strip()
|
|
if required and not value:
|
|
raise ConfigError(f"{name}_FILE (or {name}) must be set")
|
|
return value
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Settings:
|
|
host: str
|
|
port: int
|
|
env: str
|
|
log_level: str
|
|
shutdown_timeout: float
|
|
max_body_bytes: int
|
|
docs_enabled: bool = True
|
|
database_url: str = ""
|
|
session_store: str = "postgres"
|
|
cache_url: str = ""
|
|
session_ttl_seconds: int = 3600
|
|
db_pool_min: int = 1
|
|
db_pool_max: int = 10
|
|
db_timeout_seconds: float = 3.0
|
|
cache_timeout_seconds: float = 0.5
|
|
hash_concurrency: int = 2
|
|
instance_id: str = "local"
|
|
drain_seconds: float = 0.0
|
|
idempotency_ttl_seconds: int = 86400
|
|
stats_log_seconds: float = 0.0
|
|
message_write_path: str = "function"
|
|
# Lab 4: private S3 bucket and attachment limits (see storage.py, attachments.py).
|
|
s3_endpoint: str = "http://s3:8333"
|
|
s3_region: str = "us-east-1"
|
|
s3_bucket: str = "messenger-attachments"
|
|
s3_access_key_id: str = ""
|
|
s3_secret_access_key: str = ""
|
|
s3_timeout_seconds: float = 3.0
|
|
s3_connect_timeout_seconds: float = 1.0
|
|
s3_max_connections: int = 20
|
|
max_upload_bytes: int = 10 * 1024 * 1024
|
|
max_image_pixels: int = 40_000_000
|
|
upload_concurrency: int = 4
|
|
upload_queue_timeout_seconds: float = 5.0
|
|
upload_read_timeout_seconds: float = 30.0
|
|
image_verify_timeout_seconds: float = 10.0
|
|
image_verify_concurrency: int = 1
|
|
jpeg_max_memory_mb: int = 256
|
|
pending_grace_seconds: int = 900
|
|
janitor_interval_seconds: float = 60.0
|
|
# Lab 5: staging bucket (API writes, worker reads) and the asynchronous pipeline (jobs_settings()).
|
|
s3_staging_bucket: str = "messenger-staging"
|
|
broker_url: str = ""
|
|
pipeline_version: int = 1
|
|
job_max_attempts: int = 5
|
|
job_timeout_seconds: float = 60.0
|
|
job_lease_seconds: float = 30.0
|
|
job_retry_base_seconds: float = 2.0
|
|
job_retry_max_seconds: float = 60.0
|
|
worker_concurrency: int = 2
|
|
worker_shutdown_seconds: float = 20.0
|
|
image_max_memory_mb: int = 768
|
|
staging_retention_seconds: int = 86400
|
|
orphan_grace_seconds: int = 600
|
|
cleanup_interval_seconds: float = 30.0
|
|
relay_poll_seconds: float = 0.5
|
|
relay_batch: int = 50
|
|
queued_republish_seconds: float = 300.0
|
|
fault_points: frozenset[str] = frozenset()
|
|
fault_values: dict[str, int] = field(default_factory=dict)
|
|
|
|
def fault_value(self, point: str, default: int = 0) -> int:
|
|
return self.fault_values.get(point, default)
|
|
|
|
@property
|
|
def db_statement_timeout_ms(self) -> int:
|
|
# The server stops a statement a bit before the client-side deadline fires.
|
|
return max(100, int(self.db_timeout_seconds * 1000) - 500)
|
|
|
|
@classmethod
|
|
def from_env(cls, *, s3_credentials: bool = True) -> "Settings":
|
|
"""s3_credentials=False: processes without S3 access (relay, jobctl)."""
|
|
log_level = os.environ.get("LOG_LEVEL", "info").strip().lower() or "info"
|
|
if log_level not in {"debug", "info", "warning", "error"}:
|
|
raise ConfigError(f"LOG_LEVEL must be debug/info/warning/error, got {log_level!r}")
|
|
session_store = os.environ.get("SESSION_STORE", "postgres").strip().lower() or "postgres"
|
|
if session_store == "redis":
|
|
session_store = "valkey" # same protocol; the course allows Redis OR Valkey
|
|
if session_store not in {"postgres", "valkey"}:
|
|
raise ConfigError(f"SESSION_STORE must be postgres or valkey, got {session_store!r}")
|
|
db_pool_min = _int("DB_POOL_MIN", 1, 0, 50)
|
|
db_pool_max = _int("DB_POOL_MAX", 10, 1, 100)
|
|
if db_pool_min > db_pool_max:
|
|
raise ConfigError("DB_POOL_MIN must not exceed DB_POOL_MAX")
|
|
return cls(
|
|
host=os.environ.get("APP_HOST", "0.0.0.0").strip() or "0.0.0.0",
|
|
port=_int("APP_PORT", 8080, 1, 65535),
|
|
env=os.environ.get("APP_ENV", "lab").strip() or "lab",
|
|
log_level=log_level,
|
|
# DRAIN_SECONDS + SHUTDOWN_TIMEOUT_SECONDS must stay below compose stop_grace_period.
|
|
shutdown_timeout=_float("SHUTDOWN_TIMEOUT_SECONDS", 8.0, 0.0, 30.0),
|
|
max_body_bytes=_int("MAX_BODY_BYTES", 64 * 1024, 1024, 1024 * 1024),
|
|
docs_enabled=_bool("DOCS_ENABLED", True),
|
|
database_url=_secret("DATABASE_URL", required=True),
|
|
session_store=session_store,
|
|
cache_url=_secret("CACHE_URL", required=session_store == "valkey"),
|
|
# Contract name. Minimum 1 s so the expiry test can use 2 s.
|
|
session_ttl_seconds=_int("SESSION_TTL_SECONDS", 3600, 1, 30 * 24 * 3600),
|
|
db_pool_min=db_pool_min,
|
|
db_pool_max=db_pool_max,
|
|
db_timeout_seconds=_float("DB_TIMEOUT_SECONDS", 3.0, 0.5, 30.0),
|
|
cache_timeout_seconds=_float("CACHE_TIMEOUT_SECONDS", 0.5, 0.05, 10.0),
|
|
hash_concurrency=_int("PASSWORD_HASH_CONCURRENCY", 2, 1, 16),
|
|
instance_id=_instance_id(),
|
|
# After SIGTERM: readiness is 503 but requests are still served for this long,
|
|
# so the load balancer stops routing here before the listener closes.
|
|
drain_seconds=_float("DRAIN_SECONDS", 0.0, 0.0, 10.0),
|
|
# Contract: a replay within 24 h returns the original result.
|
|
idempotency_ttl_seconds=_int("IDEMPOTENCY_TTL_SECONDS", 86400, 86400, 30 * 24 * 3600),
|
|
# Experiments (bench/track1): log DB pool counters and process CPU every N s; 0 = off.
|
|
stats_log_seconds=_float("STATS_LOG_SECONDS", 0.0, 0.0, 60.0),
|
|
# Track 1: `function` = one SQL call per send (migration 0003), `multi` = the previous
|
|
# multi-statement transaction; both have identical semantics (A/B on one image).
|
|
message_write_path=_choice("MESSAGE_WRITE_PATH", "function", {"function", "multi"}),
|
|
**s3_settings(credentials_required=s3_credentials),
|
|
**jobs_settings(broker_required=False),
|
|
)
|
|
|
|
|
|
# Contract limits (contracts/README.md): configurable DOWN for experiments, never above them.
|
|
CONTRACT_MAX_UPLOAD_BYTES = 10 * 1024 * 1024
|
|
CONTRACT_MAX_IMAGE_PIXELS = 40_000_000
|
|
# Failure points for the experiments only (FAULT_POINTS of ONE service, compose variables
|
|
# API_/RELAY_/WORKER_FAULT_POINTS); empty in normal runs. `name` or `name=<integer>`.
|
|
FAULT_POINTS = {
|
|
# API, upload: between the staging PUT and the job COMMIT / after the job COMMIT, before 202.
|
|
"after_s3_put", "after_enqueue_commit",
|
|
# Relay: crash after the broker confirmed a publish but before outbox.published_at is
|
|
# committed; publish every message twice; publish a batch in reverse order.
|
|
"crash_after_publish", "duplicate_publish", "reverse_order",
|
|
# Worker: crash after the claim (before any side effect), after the S3 writes (before the
|
|
# result COMMIT), after the result COMMIT (before the ack); transient failure of the first N
|
|
# attempts; permanent failure; slow processing (N s, lease kept alive) of every job or only of
|
|
# pipeline version `slow_pipeline`; hang: stop the heartbeat and sleep N s inside processing.
|
|
"crash_after_claim", "crash_after_s3_write", "crash_after_commit", "transient_error", "permanent_error",
|
|
"slow", "slow_pipeline", "hang",
|
|
# Worker cleanup: pause N s after reading the candidates, before locking/deleting them.
|
|
"pause_cleanup",
|
|
}
|
|
|
|
|
|
def _faults() -> tuple[frozenset[str], dict[str, int]]:
|
|
names, values = set(), {}
|
|
for item in os.environ.get("FAULT_POINTS", "").split(","):
|
|
name, _, raw = item.strip().partition("=")
|
|
if not name:
|
|
continue
|
|
if name not in FAULT_POINTS:
|
|
raise ConfigError(f"FAULT_POINTS may contain only {sorted(FAULT_POINTS)}")
|
|
if raw:
|
|
if not raw.isdigit():
|
|
raise ConfigError(f"FAULT_POINTS value of {name} must be an integer")
|
|
values[name] = int(raw)
|
|
names.add(name)
|
|
return frozenset(names), values
|
|
|
|
|
|
def _bucket(name: str, default: str) -> str:
|
|
bucket = os.environ.get(name, default).strip() or default
|
|
if not re.fullmatch(r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]", bucket):
|
|
raise ConfigError(f"{name} is not a valid bucket name: {bucket!r}")
|
|
return bucket
|
|
|
|
|
|
def s3_settings(*, credentials_required: bool = True) -> dict:
|
|
"""S3 part of Settings; also used by the bootstrap CLI with its own keys."""
|
|
faults, fault_values = _faults()
|
|
endpoint = os.environ.get("S3_ENDPOINT", "http://s3:8333").strip() or "http://s3:8333"
|
|
if not re.fullmatch(r"https?://[A-Za-z0-9.\-]+(:[0-9]{1,5})?/?", endpoint):
|
|
raise ConfigError(f"S3_ENDPOINT must be http(s)://host[:port], got {endpoint!r}")
|
|
bucket = _bucket("S3_BUCKET", "messenger-attachments")
|
|
staging = _bucket("S3_STAGING_BUCKET", "messenger-staging")
|
|
if staging == bucket:
|
|
raise ConfigError("S3_STAGING_BUCKET must differ from S3_BUCKET")
|
|
return dict(
|
|
s3_endpoint=endpoint,
|
|
s3_region=os.environ.get("S3_REGION", "us-east-1").strip() or "us-east-1",
|
|
s3_bucket=bucket,
|
|
s3_staging_bucket=staging,
|
|
s3_access_key_id=_secret("S3_ACCESS_KEY_ID", required=credentials_required),
|
|
s3_secret_access_key=_secret("S3_SECRET_ACCESS_KEY", required=credentials_required),
|
|
s3_timeout_seconds=_float("S3_TIMEOUT_SECONDS", 3.0, 0.2, 30.0),
|
|
s3_connect_timeout_seconds=_float("S3_CONNECT_TIMEOUT_SECONDS", 1.0, 0.1, 10.0),
|
|
s3_max_connections=_int("S3_MAX_CONNECTIONS", 20, 1, 200),
|
|
max_upload_bytes=_int("MAX_UPLOAD_BYTES", CONTRACT_MAX_UPLOAD_BYTES, 1024, CONTRACT_MAX_UPLOAD_BYTES),
|
|
max_image_pixels=_int("MAX_IMAGE_PIXELS", CONTRACT_MAX_IMAGE_PIXELS, 1, CONTRACT_MAX_IMAGE_PIXELS),
|
|
# At most this many request bodies of one replica are buffered/checked at once (RAM bound:
|
|
# UPLOAD_CONCURRENCY x MAX_UPLOAD_BYTES); further uploads wait, then get 503.
|
|
upload_concurrency=_int("UPLOAD_CONCURRENCY", 4, 1, 64),
|
|
upload_queue_timeout_seconds=_float("UPLOAD_QUEUE_TIMEOUT_SECONDS", 5.0, 0.0, 60.0),
|
|
upload_read_timeout_seconds=_float("UPLOAD_READ_TIMEOUT_SECONDS", 30.0, 1.0, 300.0),
|
|
image_verify_timeout_seconds=_float("IMAGE_VERIFY_TIMEOUT_SECONDS", 10.0, 1.0, 120.0),
|
|
# Full checks running at once per replica: a strict JPEG decode of a 40 Mpx progressive
|
|
# file needs up to ~120-240 MB of coefficient buffers (capped by JPEG_MAX_MEMORY_MB).
|
|
image_verify_concurrency=_int("IMAGE_VERIFY_CONCURRENCY", 1, 1, 16),
|
|
jpeg_max_memory_mb=_int("JPEG_MAX_MEMORY_MB", 256, 16, 2048),
|
|
# An intent row still `pending` after this long belongs to a failed upload: the janitor
|
|
# deletes its object and the row. Far above the longest possible upload (~45 s).
|
|
pending_grace_seconds=_int("PENDING_GRACE_SECONDS", 900, 1, 7 * 24 * 3600),
|
|
janitor_interval_seconds=_float("JANITOR_INTERVAL_SECONDS", 60.0, 0.0, 3600.0),
|
|
fault_points=faults,
|
|
fault_values=fault_values,
|
|
)
|
|
|
|
|
|
def jobs_settings(*, broker_required: bool) -> dict:
|
|
"""Lab 5 pipeline: retries, lease, worker limits, cleanup, relay (docs/PIPELINE.md)."""
|
|
from .pipeline import PIPELINES # local import: pipeline.py imports nothing from here
|
|
timeout = _float("JOB_TIMEOUT_SECONDS", 60.0, 1.0, 600.0)
|
|
lease = _float("JOB_LEASE_SECONDS", 30.0, 2.0, 600.0)
|
|
base = _float("JOB_RETRY_BASE_SECONDS", 2.0, 0.0, 600.0)
|
|
cap = _float("JOB_RETRY_MAX_SECONDS", 60.0, 0.0, 3600.0)
|
|
if base > cap:
|
|
raise ConfigError("JOB_RETRY_BASE_SECONDS must not exceed JOB_RETRY_MAX_SECONDS")
|
|
orphan_grace = _int("ORPHAN_GRACE_SECONDS", 600, 1, 7 * 24 * 3600)
|
|
version = _int("PIPELINE_VERSION", 1, 1, 1000)
|
|
if version not in PIPELINES:
|
|
raise ConfigError(f"PIPELINE_VERSION must be one of {sorted(PIPELINES)}")
|
|
return dict(
|
|
broker_url=_secret("BROKER_URL", required=broker_required),
|
|
# Pipeline version of the jobs created by new uploads (track 2: versioned pipeline).
|
|
pipeline_version=version,
|
|
# Managed attempts (claims) per job before failed/retry_exhausted.
|
|
job_max_attempts=_int("JOB_MAX_ATTEMPTS", 5, 1, 100),
|
|
# Hard limit of ONE attempt: the image child process is killed, the lease is no longer renewed.
|
|
job_timeout_seconds=timeout,
|
|
# Lease of a claimed job, renewed by the heartbeat every lease/3 while the attempt runs.
|
|
job_lease_seconds=lease,
|
|
# Backoff of transient failures: full jitter, uniform(0, min(max, base * 2^(attempt-1))).
|
|
job_retry_base_seconds=base,
|
|
job_retry_max_seconds=cap,
|
|
# Jobs processed at once by ONE worker process = RabbitMQ prefetch (basic.qos).
|
|
worker_concurrency=_int("WORKER_CONCURRENCY", 2, 1, 32),
|
|
worker_shutdown_seconds=_float("WORKER_SHUTDOWN_SECONDS", 20.0, 0.0, 120.0),
|
|
# Address-space limit (RLIMIT_AS) of the child process that decodes/encodes ONE image.
|
|
image_max_memory_mb=_int("IMAGE_MAX_MEMORY_MB", 768, 128, 8192),
|
|
# Staging copy of a FAILED attachment is kept this long for a replay, then deleted.
|
|
staging_retention_seconds=_int("STAGING_RETENTION_SECONDS", 86400, 0, 30 * 24 * 3600),
|
|
# Objects of attempts that did not publish are deleted once the attempt ended this long ago.
|
|
orphan_grace_seconds=orphan_grace,
|
|
cleanup_interval_seconds=_float("CLEANUP_INTERVAL_SECONDS", 30.0, 0.0, 3600.0),
|
|
relay_poll_seconds=_float("RELAY_POLL_SECONDS", 0.5, 0.05, 60.0),
|
|
relay_batch=_int("RELAY_BATCH", 50, 1, 1000),
|
|
# A queued job whose message has not led to a claim for this long is published again
|
|
# (message lost in the broker, dead-lettered, purged): no accepted job waits forever.
|
|
queued_republish_seconds=_float("QUEUED_REPUBLISH_SECONDS", 300.0, 5.0, 86400.0),
|
|
)
|