feat: Carry runnable lab5 business stack into lab6 (U1)

Import committed lab5 baseline 03899d7 via git archive: API replicas,
PostgreSQL, private S3, RabbitMQ, publisher, worker, migrations, and hooks.
Preserve lab6 contracts and the separate telemetry stack. Static checks pass;
Docker runtime integration remains pending.
This commit is contained in:
Самохвалов Андрей Ремович
2026-10-02 14:43:04 +03:00
parent 22e1764d84
commit c60b3e652b
38 changed files with 2542 additions and 0 deletions
+12
View File
@@ -0,0 +1,12 @@
FROM python:3.12.12-slim-bookworm
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt && useradd --system --uid 10001 --home /app app
COPY --chown=app:app src/ src/
COPY --chown=app:app migrations/ migrations/
COPY --chown=app:app scripts/migrate.py scripts/migrate.py
USER app
EXPOSE 8080
CMD ["python", "-m", "src.server"]
+225
View File
@@ -0,0 +1,225 @@
name: mtusi2026lab5
services:
postgres:
image: postgres:17.6-bookworm
environment:
POSTGRES_DB: messenger
POSTGRES_USER: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets: [postgres_password]
volumes: [postgres_data:/var/lib/postgresql/data]
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
interval: 2s
timeout: 2s
retries: 20
migrate:
build: .
command: ["python", "scripts/migrate.py"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
secrets: [database_url]
depends_on:
postgres:
condition: service_healthy
restart: "no"
s3-volume-init:
image: alpine/minio:RELEASE.2025-10-15T17-29-55Z@sha256:cf23643a6cf9ce159c57643ceb88279e431262282428c9e0bf3a7ef1a97e84b4
user: "0:0"
entrypoint: ["/bin/sh", "-c"]
command: ["chown -R 100:100 /data"]
volumes: [s3_data:/data]
restart: "no"
s3:
image: alpine/minio:RELEASE.2025-10-15T17-29-55Z@sha256:cf23643a6cf9ce159c57643ceb88279e431262282428c9e0bf3a7ef1a97e84b4
command: ["server", "/data", "--console-address", ":9001"]
environment:
MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user
MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password
secrets: [minio_root_user, minio_root_password]
volumes: [s3_data:/data]
depends_on:
s3-volume-init:
condition: service_completed_successfully
healthcheck:
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9000/minio/health/ready"]
interval: 2s
timeout: 2s
retries: 30
cpus: 0.5
mem_limit: 384m
s3-bootstrap:
image: bitnamilegacy/minio-client:2025.7.21-debian-12-r3@sha256:73bd39f7899a0cef12b8dd5df13aa93a3ed1aaa44236542442e9ac76819ac158
user: "0:0"
entrypoint: ["/bin/sh", "/bootstrap.sh"]
command: []
volumes:
- ./infra/bootstrap-s3.sh:/bootstrap.sh:ro
- ./infra/minio-api-policy.json:/policies/minio-api-policy.json:ro
- ./infra/minio-maintenance-policy.json:/policies/minio-maintenance-policy.json:ro
secrets:
- minio_root_user
- minio_root_password
- s3_access_key_id
- s3_secret_access_key
- s3_maintenance_key_id
- s3_maintenance_secret_key
depends_on:
s3:
condition: service_healthy
restart: "no"
broker:
image: rabbitmq:4.1.8-management
hostname: broker
restart: unless-stopped
volumes:
- broker_data:/var/lib/rabbitmq
- ./secrets/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf:ro
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
interval: 5s
timeout: 5s
retries: 24
broker-init:
image: python:3.12.14-alpine3.24
restart: "no"
depends_on:
broker:
condition: service_healthy
environment:
BROKER_URL_FILE: /run/secrets/broker_url
volumes:
- ./scripts/bootstrap_broker.py:/app/bootstrap_broker.py:ro
secrets: [broker_url]
command: ["python", "/app/bootstrap_broker.py"]
api:
build: .
scale: 2
environment:
APP_HOST: 0.0.0.0
APP_PORT: 8080
DATABASE_URL_FILE: /run/secrets/database_url
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
S3_ENDPOINT: s3:9000
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
secrets: [database_url, s3_access_key_id, s3_secret_access_key]
depends_on:
migrate:
condition: service_completed_successfully
s3-bootstrap:
condition: service_completed_successfully
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).close()"]
interval: 2s
timeout: 3s
retries: 20
start_period: 5s
stop_grace_period: 10s
cpus: 0.5
mem_limit: 256m
publisher:
build: .
command: ["python", "-m", "src.publisher"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
BROKER_URL_FILE: /run/secrets/broker_url
BROKER_QUEUE: attachments.process
secrets: [database_url, broker_url]
depends_on:
migrate:
condition: service_completed_successfully
broker-init:
condition: service_completed_successfully
restart: unless-stopped
staging-cleanup:
build: .
command: ["python", "-m", "src.staging_cleanup"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
S3_ENDPOINT: s3:9000
S3_MAINTENANCE_KEY_ID_FILE: /run/secrets/s3_maintenance_key_id
S3_MAINTENANCE_SECRET_KEY_FILE: /run/secrets/s3_maintenance_secret_key
STAGING_GRACE_SECONDS: ${STAGING_GRACE_SECONDS:-3600}
STAGING_CLEANUP_INTERVAL_SECONDS: ${STAGING_CLEANUP_INTERVAL_SECONDS:-60}
secrets: [database_url, s3_maintenance_key_id, s3_maintenance_secret_key]
depends_on:
migrate:
condition: service_completed_successfully
s3-bootstrap:
condition: service_completed_successfully
restart: unless-stopped
worker:
build: .
command: ["python", "-m", "src.worker"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
BROKER_URL_FILE: /run/secrets/broker_url
BROKER_QUEUE: attachments.process
S3_ENDPOINT: s3:9000
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-1}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
secrets: [database_url, broker_url, s3_access_key_id, s3_secret_access_key]
depends_on:
migrate:
condition: service_completed_successfully
s3-bootstrap:
condition: service_completed_successfully
broker-init:
condition: service_completed_successfully
restart: unless-stopped
cpus: 0.5
mem_limit: 768m
lb:
image: haproxy:3.0.11-alpine
volumes:
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
ports:
- "127.0.0.1:${APP_PORT:-8080}:8080"
depends_on:
api:
condition: service_healthy
cpus: 0.5
mem_limit: 128m
secrets:
postgres_password:
file: ./secrets/postgres_password
database_url:
file: ./secrets/database_url
minio_root_user:
file: ./secrets/minio_root_user
minio_root_password:
file: ./secrets/minio_root_password
s3_access_key_id:
file: ./secrets/s3_access_key_id
s3_secret_access_key:
file: ./secrets/s3_secret_access_key
s3_maintenance_key_id:
file: ./secrets/s3_maintenance_key_id
s3_maintenance_secret_key:
file: ./secrets/s3_maintenance_secret_key
broker_url:
file: ./secrets/broker_url
volumes:
postgres_data:
s3_data:
broker_data:
+16
View File
@@ -0,0 +1,16 @@
#!/bin/sh
set -eu
read_secret() {
tr -d '\n' < "/run/secrets/$1"
}
mc alias set local http://s3:9000 "$(read_secret minio_root_user)" "$(read_secret minio_root_password)" >/dev/null
mc mb --ignore-existing local/messenger-attachments >/dev/null
mc anonymous set none local/messenger-attachments >/dev/null
mc admin policy create local messenger-api /policies/minio-api-policy.json >/dev/null
mc admin policy create local messenger-maintenance /policies/minio-maintenance-policy.json >/dev/null
mc admin user add local "$(read_secret s3_access_key_id)" "$(read_secret s3_secret_access_key)" >/dev/null
mc admin policy attach local messenger-api --user "$(read_secret s3_access_key_id)" >/dev/null
mc admin user add local "$(read_secret s3_maintenance_key_id)" "$(read_secret s3_maintenance_secret_key)" >/dev/null
mc admin policy attach local messenger-maintenance --user "$(read_secret s3_maintenance_key_id)" >/dev/null
+26
View File
@@ -0,0 +1,26 @@
global
log stdout format raw local0
defaults
mode http
log global
option httplog
timeout connect 2s
timeout client 15s
timeout server 15s
resolvers docker
nameserver docker 127.0.0.11:53
resolve_retries 2
timeout retry 1s
hold valid 1s
frontend public
bind *:8080
default_backend api
backend api
balance roundrobin
option httpchk GET /health/ready
http-check expect status 200
server-template api 1-4 api:8080 check inter 1s fall 2 rise 1 resolvers docker init-addr none
+15
View File
@@ -0,0 +1,15 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["s3:GetBucketLocation"],
"Resource": ["arn:aws:s3:::messenger-attachments"]
},
{
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
"Resource": ["arn:aws:s3:::messenger-attachments/*"]
}
]
}
+15
View File
@@ -0,0 +1,15 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["s3:GetBucketLocation", "s3:ListBucket"],
"Resource": ["arn:aws:s3:::messenger-attachments"]
},
{
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
"Resource": ["arn:aws:s3:::messenger-attachments/*"]
}
]
}
+36
View File
@@ -0,0 +1,36 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE INDEX chats_history_idx ON chats (created_at, id);
CREATE TABLE memberships (
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users(id),
position smallint NOT NULL CHECK (position BETWEEN 0 AND 99),
PRIMARY KEY (chat_id, user_id),
UNIQUE (chat_id, position)
);
CREATE INDEX memberships_user_idx ON memberships (user_id, chat_id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL,
sender_id uuid NOT NULL,
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
FOREIGN KEY (chat_id, sender_id) REFERENCES memberships(chat_id, user_id)
);
CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id);
+7
View File
@@ -0,0 +1,7 @@
CREATE TABLE sessions (
token_hash bytea PRIMARY KEY,
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
expires_at timestamptz NOT NULL
);
CREATE INDEX sessions_user_idx ON sessions (user_id);
+11
View File
@@ -0,0 +1,11 @@
CREATE TABLE message_replays (
user_id uuid NOT NULL REFERENCES users(id),
chat_id uuid NOT NULL REFERENCES chats(id),
request_key text NOT NULL,
request_hash bytea NOT NULL,
message_id uuid REFERENCES messages(id),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
PRIMARY KEY (user_id, chat_id, request_key)
);
CREATE INDEX message_replays_expiry_idx ON message_replays (created_at);
+32
View File
@@ -0,0 +1,32 @@
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
filename text NOT NULL,
object_key text NOT NULL UNIQUE,
content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
sha256 text NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'),
status text NOT NULL DEFAULT 'ready' CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
error_code text,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
UNIQUE (id, chat_id)
);
CREATE INDEX attachments_chat_idx ON attachments (chat_id, created_at);
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (char_length(text) BETWEEN 0 AND 4000);
ALTER TABLE messages ADD CONSTRAINT messages_id_chat_unique UNIQUE (id, chat_id);
CREATE TABLE message_attachments (
message_id uuid NOT NULL,
chat_id uuid NOT NULL,
attachment_id uuid NOT NULL,
position smallint NOT NULL CHECK (position BETWEEN 0 AND 9),
PRIMARY KEY (message_id, attachment_id),
UNIQUE (message_id, position),
FOREIGN KEY (message_id, chat_id) REFERENCES messages(id, chat_id) ON DELETE CASCADE,
FOREIGN KEY (attachment_id, chat_id) REFERENCES attachments(id, chat_id)
);
CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id);
+2
View File
@@ -0,0 +1,2 @@
ALTER TABLE attachments ADD COLUMN owner_id uuid NOT NULL REFERENCES users(id);
ALTER TABLE attachments ADD COLUMN updated_at timestamptz NOT NULL DEFAULT clock_timestamp();
+74
View File
@@ -0,0 +1,74 @@
-- Lab5: PostgreSQL is the source of truth; RabbitMQ carries only job IDs.
ALTER TABLE attachments
ADD COLUMN staging_key text UNIQUE,
ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version > 0);
CREATE TABLE attachment_jobs (
id uuid PRIMARY KEY,
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
pipeline_version integer NOT NULL CHECK (pipeline_version > 0),
attempt integer NOT NULL DEFAULT 1 CHECK (attempt > 0),
status text NOT NULL DEFAULT 'queued'
CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
next_attempt_at timestamptz NOT NULL DEFAULT clock_timestamp(),
lease_token uuid,
lease_expires_at timestamptz,
error_code text,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
UNIQUE (attachment_id, pipeline_version),
CHECK ((status = 'processing') =
(lease_token IS NOT NULL AND lease_expires_at IS NOT NULL))
);
CREATE INDEX attachment_jobs_due_idx
ON attachment_jobs (next_attempt_at, id) WHERE status = 'queued';
CREATE INDEX attachment_jobs_expired_idx
ON attachment_jobs (lease_expires_at, id) WHERE status = 'processing';
-- One logical publication per managed attempt. A crash after broker confirm
-- but before published_at may republish; consumers must tolerate that duplicate.
CREATE TABLE attachment_outbox (
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
job_id uuid NOT NULL REFERENCES attachment_jobs(id) ON DELETE CASCADE,
attempt integer NOT NULL CHECK (attempt > 0),
available_at timestamptz NOT NULL DEFAULT clock_timestamp(),
claim_token uuid,
claim_expires_at timestamptz,
published_at timestamptz,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
UNIQUE (job_id, attempt),
CHECK ((claim_token IS NULL) = (claim_expires_at IS NULL))
);
CREATE INDEX attachment_outbox_pending_idx
ON attachment_outbox (available_at, id) WHERE published_at IS NULL;
-- Internal 'original' is stored alongside public thumbnail/crop/optimized
-- so every physical effect has the same versioned identity and checksum.
CREATE TABLE attachment_variants (
attachment_id uuid NOT NULL,
pipeline_version integer NOT NULL,
name text NOT NULL CHECK (name IN ('original', 'thumbnail', 'crop', 'optimized')),
object_key text NOT NULL,
content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg', 'image/webp')),
size_bytes bigint NOT NULL CHECK (size_bytes > 0),
width integer NOT NULL CHECK (width > 0),
height integer NOT NULL CHECK (height > 0),
sha256 text NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
PRIMARY KEY (attachment_id, pipeline_version, name),
FOREIGN KEY (attachment_id, pipeline_version)
REFERENCES attachment_jobs (attachment_id, pipeline_version) ON DELETE CASCADE
);
-- An API upload reserves its staging key before S3 PUT. Cleanup only removes
-- objects without a committed job and without an unexpired upload marker.
CREATE TABLE attachment_uploads (
id uuid PRIMARY KEY,
staging_key text NOT NULL UNIQUE,
lease_expires_at timestamptz NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE INDEX attachment_uploads_expired_idx ON attachment_uploads (lease_expires_at);
+10
View File
@@ -0,0 +1,10 @@
fastapi==0.118.3
pydantic==2.12.4
uvicorn==0.38.0
psycopg[binary,pool]==3.2.12
psycopg-pool==3.3.3
argon2-cffi==25.1.0
minio==7.2.20
Pillow==11.3.0
urllib3==2.8.0
pika==1.4.4
+44
View File
@@ -0,0 +1,44 @@
#!/usr/bin/env python3
"""Idempotently declare the durable processing queue via RabbitMQ management API."""
import base64
import json
import os
import time
from urllib.error import HTTPError, URLError
from urllib.parse import quote, unquote, urlsplit
from urllib.request import Request, urlopen
broker_url = open(os.environ.get("BROKER_URL_FILE", "/run/secrets/broker_url"), encoding="utf-8").read().strip()
parsed = urlsplit(broker_url)
if parsed.scheme != "amqp" or not parsed.username or not parsed.password:
raise SystemExit("Invalid broker URL file")
credentials = f"{unquote(parsed.username)}:{unquote(parsed.password)}".encode()
headers = {
"Authorization": "Basic " + base64.b64encode(credentials).decode("ascii"),
"Content-Type": "application/json",
}
url = "http://broker:15672/api/queues/%2F/" + quote("attachments.process", safe="")
body = json.dumps(
{"durable": True, "auto_delete": False, "arguments": {"x-queue-type": "quorum"}}
).encode()
for attempt in range(30):
try:
with urlopen(Request(url, data=body, headers=headers, method="PUT"), timeout=3):
pass
with urlopen(Request(url, headers=headers), timeout=3) as response:
queue = json.load(response)
if queue.get("type") != "quorum" or not queue.get("durable"):
raise SystemExit("Processing queue is not durable quorum")
print("Durable quorum queue attachments.process is ready")
break
except HTTPError as exc:
if exc.code in (400, 401, 403, 406):
raise SystemExit(f"RabbitMQ queue declaration failed (HTTP {exc.code})") from None
except URLError:
pass
if attempt == 29:
raise SystemExit("RabbitMQ management API did not become ready")
time.sleep(1)
+30
View File
@@ -0,0 +1,30 @@
"""Apply numbered SQL migrations once, atomically and under a DB lock."""
from pathlib import Path
import sys
import psycopg
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from src.db import database_url
def main() -> None:
migration_dir = Path(__file__).resolve().parents[1] / "migrations"
with psycopg.connect(database_url(), connect_timeout=5, options="-c statement_timeout=30000") as conn:
with conn.transaction():
conn.execute("SELECT pg_advisory_xact_lock(2026, 2)")
conn.execute(
"CREATE TABLE IF NOT EXISTS schema_migrations ("
"version integer PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT clock_timestamp())"
)
for path in sorted(migration_dir.glob("[0-9][0-9][0-9]_*.sql")):
version = int(path.name.split("_", 1)[0])
if conn.execute("SELECT 1 FROM schema_migrations WHERE version=%s", (version,)).fetchone():
continue
conn.execute(path.read_text(encoding="utf-8"))
conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (version,))
if __name__ == "__main__":
main()
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
docker compose --env-file .env stop api lb publisher worker staging-cleanup broker postgres s3
docker compose --env-file .env up -d --wait --scale api=2
+42
View File
@@ -0,0 +1,42 @@
"""Create local Compose secrets without printing or overwriting existing values."""
import os
import secrets
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1] / "secrets"
def write(name: str, value: str) -> None:
fd = os.open(ROOT / name, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
with os.fdopen(fd, "w") as out:
out.write(value)
def main() -> None:
ROOT.mkdir(mode=0o700, exist_ok=True)
db_names = {"postgres_password", "database_url"}
s3_names = {
"minio_root_user", "minio_root_password", "s3_access_key_id", "s3_secret_access_key",
"s3_maintenance_key_id", "s3_maintenance_secret_key",
}
for names in (db_names, s3_names):
existing = {name for name in names if (ROOT / name).exists()}
if existing and existing != names:
raise SystemExit(f"Incomplete secret group: {', '.join(sorted(names - existing))}")
if not (ROOT / "postgres_password").exists():
password = secrets.token_urlsafe(32)
write("postgres_password", password)
write("database_url", f"postgresql://messenger:{password}@postgres:5432/messenger")
if not (ROOT / "minio_root_user").exists():
write("minio_root_user", "lab5admin")
write("minio_root_password", secrets.token_urlsafe(32))
write("s3_access_key_id", secrets.token_hex(10))
write("s3_secret_access_key", secrets.token_urlsafe(32))
write("s3_maintenance_key_id", secrets.token_hex(10))
write("s3_maintenance_secret_key", secrets.token_urlsafe(32))
if __name__ == "__main__":
main()
+13
View File
@@ -0,0 +1,13 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
for container in $(docker compose --env-file .env ps -q api); do
instance=$(docker exec "$container" python -c "import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=2).headers['X-Instance-Id'])")
if [ "$instance" = "$TARGET_INSTANCE_ID" ]; then
docker stop -t 5 "$container" >/dev/null
exit 0
fi
done
echo "target API instance not found" >&2
exit 1
+117
View File
@@ -0,0 +1,117 @@
"""Messenger HTTP routes."""
import re
from uuid import UUID
from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response
from .auth import basic_credentials, bearer_token
from .attachments import storage_ready, upload
from .attachment_read import download, get_view
from .models import CreateChat, CreateMessage, CreateUser
from .store import store
router = APIRouter()
@router.get("/health/live")
def live():
return {"status": "ok"}
@router.get("/health/ready")
def ready():
result = store.ready()
storage_ready()
return result
@router.post("/api/v1/users", status_code=201)
def register_user(body: CreateUser):
return store.register(body.username, body.display_name, body.password)
@router.post("/api/v1/auth/sessions", status_code=201)
async def create_session(request: Request, response: Response):
if await request.body():
raise HTTPException(400)
username, password = basic_credentials(request.headers.get("authorization"))
response.headers["Cache-Control"] = "no-store"
return store.create_session(username, password)
def current_user(request: Request) -> dict:
return store.authenticated(bearer_token(request.headers.get("authorization")))
def parse_limit(value: str) -> int:
try:
number = int(value) if value.isdecimal() else 0
except ValueError:
raise HTTPException(400) from None
if not 1 <= number <= 100:
raise HTTPException(400)
return number
@router.delete("/api/v1/auth/sessions/current", status_code=204)
def revoke_session(request: Request):
store.revoke_session(bearer_token(request.headers.get("authorization")))
return Response(status_code=204)
@router.get("/api/v1/users/me")
def user_me(actor: dict = Depends(current_user)):
return actor
@router.post("/api/v1/chats", status_code=201)
def create_chat(body: CreateChat, actor: dict = Depends(current_user)):
return store.create_chat(actor["id"], body.title, body.member_ids)
@router.get("/api/v1/chats")
def list_chats(
limit: str = Query("50"), cursor: str | None = Query(None, min_length=1), actor: dict = Depends(current_user)
):
return store.list_chats(actor["id"], parse_limit(limit), cursor)
@router.get("/api/v1/chats/{chat_id}")
def get_chat(chat_id: UUID, actor: dict = Depends(current_user)):
return store.chat(actor["id"], str(chat_id))
@router.post("/api/v1/chats/{chat_id}/attachments", status_code=202)
async def upload_attachment(chat_id: UUID, request: Request, filename: str = Query(...), actor: dict = Depends(current_user)):
return await upload(request, str(chat_id), actor["id"], filename)
@router.get("/api/v1/attachments/{attachment_id}")
def get_attachment(attachment_id: UUID, actor: dict = Depends(current_user)):
return get_view(actor["id"], attachment_id)
@router.get("/api/v1/attachments/{attachment_id}/content")
def get_attachment_content(
attachment_id: UUID, variant: str = Query("original"), actor: dict = Depends(current_user)
):
return download(actor["id"], attachment_id, variant)
@router.post("/api/v1/chats/{chat_id}/messages", status_code=201)
def create_message(chat_id: UUID, body: CreateMessage, request: Request, actor: dict = Depends(current_user)):
key = request.headers.get("Idempotency-Key")
if key is None or not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key):
raise HTTPException(400)
return store.create_message(actor["id"], str(chat_id), body.text, key, body.attachment_ids)
@router.get("/api/v1/chats/{chat_id}/messages")
def list_messages(
chat_id: UUID,
limit: str = Query("50"),
cursor: str | None = Query(None, min_length=1),
actor: dict = Depends(current_user),
):
return store.list_messages(actor["id"], str(chat_id), parse_limit(limit), cursor)
+72
View File
@@ -0,0 +1,72 @@
"""Authorize metadata and stream versioned variants from the private bucket."""
import logging
from uuid import UUID
from fastapi import HTTPException
from fastapi.responses import StreamingResponse
from .attachments import BUCKET, attachment_view, storage
from .store import connection
logger = logging.getLogger(__name__)
def metadata(actor_id: str, attachment_id: UUID) -> dict:
with connection() as conn:
row = conn.execute(
"SELECT a.* FROM attachments a "
"JOIN memberships m ON m.chat_id=a.chat_id "
"WHERE a.id=%s AND m.user_id=%s",
(attachment_id, UUID(actor_id)),
).fetchone()
if row is None:
raise HTTPException(404)
return row
def download(actor_id: str, attachment_id: UUID, variant: str) -> StreamingResponse:
row = metadata(actor_id, attachment_id)
if row["status"] != "ready":
raise HTTPException(409)
with connection() as conn:
selected = conn.execute(
"SELECT object_key,content_type,size_bytes FROM attachment_variants "
"WHERE attachment_id=%s AND pipeline_version=%s AND name=%s",
(attachment_id, row["pipeline_version"], variant),
).fetchone()
if selected is None:
raise HTTPException(404)
try:
obj = storage().get_object(BUCKET, selected["object_key"])
except Exception as exc:
logger.warning("S3 read failed for attachment %s (%s)", attachment_id, type(exc).__name__)
raise HTTPException(503) from None
def chunks():
try:
yield from obj.stream(64 * 1024)
finally:
obj.close()
obj.release_conn()
return StreamingResponse(
chunks(), media_type=selected["content_type"],
headers={"Content-Length": str(selected["size_bytes"])}
)
def get_view(actor_id: str, attachment_id: UUID) -> dict:
row = metadata(actor_id, attachment_id)
view = attachment_view(row)
if row["status"] == "ready":
with connection() as conn:
variants = conn.execute(
"SELECT name,content_type,size_bytes,width,height,sha256 "
"FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s "
"AND name <> 'original' ORDER BY name",
(attachment_id, row["pipeline_version"]),
).fetchall()
view["variants"] = [dict(variant) for variant in variants]
return view
+171
View File
@@ -0,0 +1,171 @@
"""Private S3 access shared by attachment routes and maintenance jobs."""
import asyncio
import hashlib
import logging
import os
import unicodedata
from functools import lru_cache
from pathlib import Path
from tempfile import SpooledTemporaryFile
from uuid import UUID, uuid4
from fastapi import HTTPException, Request
from minio import Minio
from minio.error import S3Error
from PIL import Image, UnidentifiedImageError
from urllib3 import PoolManager, Retry, Timeout
from .pipeline_jobs import queue_attachment, renew_upload, reserve_upload
from .store import connection, store, timestamp
BUCKET = "messenger-attachments"
MAX_BYTES = 10 * 1024 * 1024
MAX_PIXELS = 40_000_000
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
logger = logging.getLogger(__name__)
def secret(name: str) -> str:
return Path(os.environ[f"{name}_FILE"]).read_text(encoding="utf-8").strip()
@lru_cache(maxsize=1)
def storage() -> Minio:
return Minio(
os.environ.get("S3_ENDPOINT", "s3:9000"),
access_key=secret("S3_ACCESS_KEY_ID"),
secret_key=secret("S3_SECRET_ACCESS_KEY"),
secure=False,
region="us-east-1",
http_client=PoolManager(timeout=Timeout(connect=2, read=10), retries=Retry(total=0)),
)
def storage_ready() -> None:
try:
storage().stat_object(BUCKET, "health/ready-probe")
except S3Error as exc:
if exc.code != "NoSuchKey":
raise HTTPException(503) from None
except Exception:
raise HTTPException(503) from None
def attachment_view(row: dict) -> dict:
return {
"id": str(row["id"]), "chat_id": str(row["chat_id"]), "owner_id": str(row["owner_id"]),
"filename": row["filename"], "content_type": row["content_type"],
"size_bytes": row["size_bytes"], "sha256": row["sha256"], "status": row["status"],
"variants": [], "error_code": row["error_code"],
"created_at": timestamp(row["created_at"]), "updated_at": timestamp(row["updated_at"]),
}
def validate_filename(filename: str) -> None:
if not 1 <= len(filename) <= 255 or filename in {".", ".."}:
raise HTTPException(400)
if any(char in "/\\" or unicodedata.category(char) == "Cc" for char in filename):
raise HTTPException(400)
def validate_image(stream, declared_type: str) -> str:
try:
with Image.open(stream) as image:
actual_type = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(image.format)
if actual_type is None or actual_type != declared_type:
raise HTTPException(415)
if image.width * image.height > MAX_PIXELS:
raise HTTPException(413)
image.verify()
stream.seek(0)
with Image.open(stream) as image:
image.load()
return actual_type
except Image.DecompressionBombError:
raise HTTPException(413) from None
except (UnidentifiedImageError, OSError, ValueError, SyntaxError):
raise HTTPException(415) from None
def verify_staging(client: Minio, key: str, size: int, sha256: str, content_type: str) -> None:
"""Confirm complete durable bytes; an S3 ETag is not a SHA-256 checksum."""
info = client.stat_object(BUCKET, key)
if info.size != size or info.content_type.split(";", 1)[0] != content_type:
raise ValueError("staging metadata mismatch")
obj = client.get_object(BUCKET, key)
try:
digest = hashlib.sha256()
received = 0
for chunk in obj.stream(64 * 1024):
received += len(chunk)
if received > size:
raise ValueError("staging size mismatch")
digest.update(chunk)
if received != size or digest.hexdigest() != sha256:
raise ValueError("staging checksum mismatch")
finally:
obj.close()
obj.release_conn()
def persist_attachment(chat_id: str, actor_id: str, filename: str, content_type: str, size: int, sha256: str, stream) -> dict:
attachment_id = uuid4()
staging_key = f"staging/{attachment_id}"
object_key = f"originals/{attachment_id}"
with connection() as conn:
reserve_upload(conn, attachment_id, staging_key)
try:
stream.seek(0)
client = storage()
client.put_object(BUCKET, staging_key, stream, size, content_type=content_type)
with connection() as conn:
if not renew_upload(conn, attachment_id):
raise HTTPException(503)
verify_staging(client, staging_key, size, sha256, content_type)
with connection() as conn:
row, _job_id = queue_attachment(
conn, attachment_id=attachment_id, chat_id=UUID(chat_id), owner_id=UUID(actor_id),
filename=filename, staging_key=staging_key, object_key=object_key,
content_type=content_type, size_bytes=size, sha256=sha256,
)
except HTTPException:
raise
except Exception as exc:
# A lost COMMIT acknowledgement may leave a valid queued job. Keep the
# staging object; cleanup must check PostgreSQL before deleting it.
logger.warning("Attachment handoff failed for %s (%s)", attachment_id, type(exc).__name__)
raise HTTPException(503) from None
return attachment_view(row)
async def upload(request: Request, chat_id: str, actor_id: str, filename: str) -> dict:
await asyncio.to_thread(store.chat, actor_id, chat_id)
validate_filename(filename)
declared_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
if declared_type not in {"image/png", "image/jpeg"}:
raise HTTPException(415)
length = request.headers.get("content-length")
if length is not None:
try:
if int(length) > MAX_BYTES:
raise HTTPException(413)
except ValueError:
raise HTTPException(400) from None
with SpooledTemporaryFile(max_size=1024 * 1024) as stream:
digest = hashlib.sha256()
size = 0
async for chunk in request.stream():
size += len(chunk)
if size > MAX_BYTES:
raise HTTPException(413)
stream.write(chunk)
digest.update(chunk)
stream.seek(0)
validate_image(stream, declared_type)
return await asyncio.to_thread(
persist_attachment, chat_id, actor_id, filename, declared_type, size, digest.hexdigest(), stream
)
+43
View File
@@ -0,0 +1,43 @@
"""Parse HTTP credentials without exposing them in errors or logs."""
import base64
import binascii
import hashlib
import re
from fastapi import HTTPException
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
BEARER_CHALLENGE = 'Bearer realm="messenger"'
USERNAME = re.compile(r"[a-z0-9_]{3,32}\Z")
def unauthorized(challenge: str) -> HTTPException:
return HTTPException(401, headers={"WWW-Authenticate": challenge})
def basic_credentials(header: str | None) -> tuple[str, str]:
if not header or not header.startswith("Basic "):
raise unauthorized(BASIC_CHALLENGE)
try:
raw = base64.b64decode(header[6:], validate=True).decode("utf-8")
username, password = raw.split(":", 1)
except (binascii.Error, UnicodeError, ValueError):
raise unauthorized(BASIC_CHALLENGE) from None
if not USERNAME.fullmatch(username) or not 12 <= len(password) <= 128:
raise unauthorized(BASIC_CHALLENGE)
return username, password
def bearer_token(header: str | None) -> str:
if not header or not header.startswith("Bearer "):
raise unauthorized(BEARER_CHALLENGE)
token = header[7:]
if not 32 <= len(token) <= 256 or not token.isascii() or not re.fullmatch(r"[A-Za-z0-9_-]+", token):
raise unauthorized(BEARER_CHALLENGE)
return token
def token_hash(token: str) -> bytes:
return hashlib.sha256(token.encode("ascii")).digest()
+38
View File
@@ -0,0 +1,38 @@
"""Bounded PostgreSQL pool shared by API storage and health checks."""
import os
import atexit
from functools import lru_cache
from pathlib import Path
import psycopg
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool
from psycopg_pool import PoolTimeout
def database_url() -> str:
path = os.environ.get("DATABASE_URL_FILE")
if path:
return Path(path).read_text(encoding="utf-8").strip()
return os.environ.get("DATABASE_URL", "postgresql:///messenger")
@lru_cache(maxsize=1)
def pool() -> ConnectionPool:
result = ConnectionPool(
database_url(), min_size=0, max_size=8, timeout=5,
kwargs={"connect_timeout": 5, "options": "-c statement_timeout=5000", "row_factory": dict_row},
open=True,
)
atexit.register(result.close)
return result
def ready() -> bool:
try:
with pool().connection() as conn:
conn.execute("SELECT 1 FROM users LIMIT 0")
return True
except (psycopg.Error, PoolTimeout, OSError):
return False
+83
View File
@@ -0,0 +1,83 @@
"""Deterministic, bounded image variants for one accepted PNG or JPEG."""
import hashlib
from io import BytesIO
from PIL import Image, ImageOps, UnidentifiedImageError
MAX_BYTES = 10 * 1024 * 1024
MAX_PIXELS = 40_000_000
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
class InvalidImage(ValueError):
pass
def _fit(image: Image.Image, bound: int) -> Image.Image:
factor = min(1, bound / max(image.size))
size = (max(1, int(image.width * factor)), max(1, int(image.height * factor)))
return image.resize(size, Image.Resampling.LANCZOS) if size != image.size else image
def _jpeg(image: Image.Image) -> bytes:
output = BytesIO()
image.save(output, format="JPEG", quality=85, optimize=True)
return output.getvalue()
def _variant(data: bytes, content_type: str, image: Image.Image) -> dict:
return {
"data": data,
"content_type": content_type,
"size_bytes": len(data),
"width": image.width,
"height": image.height,
"sha256": hashlib.sha256(data).hexdigest(),
}
def render(data: bytes, content_type: str) -> dict[str, dict]:
"""Preserve original bytes; orient, flatten alpha, strip metadata on derivatives."""
if not 0 < len(data) <= MAX_BYTES or content_type not in {"image/png", "image/jpeg"}:
raise InvalidImage("invalid original")
try:
with Image.open(BytesIO(data)) as source:
if {"PNG": "image/png", "JPEG": "image/jpeg"}.get(source.format) != content_type:
raise InvalidImage("wrong image type")
if source.width * source.height > MAX_PIXELS:
raise InvalidImage("too many pixels")
source.load()
original_size = source.size
oriented = ImageOps.exif_transpose(source)
if oriented.mode in {"RGBA", "LA", "P"}:
rgba = oriented.convert("RGBA")
image = Image.new("RGB", rgba.size, "white")
image.paste(rgba, mask=rgba.getchannel("A"))
else:
image = oriented.convert("RGB")
thumbnail = _fit(image, 256)
side = min(image.size)
left = (image.width - side) // 2
top = (image.height - side) // 2
crop = image.resize(
(128, 128), Image.Resampling.LANCZOS,
box=(left, top, left + side, top + side),
)
optimized = _fit(image, 1280)
webp = BytesIO()
optimized.save(webp, format="WEBP", quality=82, method=4)
return {
"original": {
"data": data, "content_type": content_type,
"size_bytes": len(data), "width": original_size[0], "height": original_size[1],
"sha256": hashlib.sha256(data).hexdigest(),
},
"thumbnail": _variant(_jpeg(thumbnail), "image/jpeg", thumbnail),
"crop": _variant(_jpeg(crop), "image/jpeg", crop),
"optimized": _variant(webp.getvalue(), "image/webp", optimized),
}
except (UnidentifiedImageError, OSError, SyntaxError, Image.DecompressionBombError) as exc:
raise InvalidImage("image decoding failed") from exc
+67
View File
@@ -0,0 +1,67 @@
"""FastAPI entry point and contract-wide response handling."""
from uuid import uuid4
from fastapi import FastAPI, HTTPException, Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse
from .api import router
app = FastAPI()
app.include_router(router)
INSTANCE_ID = uuid4().hex
ERRORS = {
400: ("invalid_request", "Неверный запрос"),
401: ("unauthorized", "Требуется действительный пользователь"),
404: ("not_found", "Объект не найден"),
409: ("conflict", "Конфликт данных"),
413: ("payload_too_large", "Слишком большой запрос"),
415: ("unsupported_media_type", "Неподдерживаемый тип данных"),
429: ("rate_limited", "Слишком много запросов"),
503: ("unavailable", "Сервис недоступен"),
}
def error_response(request: Request, status: int, extra_headers: dict | None = None) -> JSONResponse:
code, message = ERRORS.get(status, ("internal_error", "Внутренняя ошибка"))
request_id = request.state.request_id
return JSONResponse(
status_code=status,
content={"error": {"code": code, "message": message, "request_id": request_id}},
headers={"X-Request-Id": request_id, "X-Instance-Id": INSTANCE_ID, **(extra_headers or {})},
)
@app.middleware("http")
async def request_contract(request: Request, call_next):
request.state.request_id = str(uuid4())
is_upload = request.url.path.startswith("/api/v1/chats/") and request.url.path.endswith("/attachments")
if request.method == "POST" and request.url.path.startswith("/api/v1/") and request.url.path != "/api/v1/auth/sessions" and not is_upload:
media_type, _, charset = request.headers.get("content-type", "").lower().partition(";")
if media_type.strip() != "application/json" or (charset and charset.strip() != "charset=utf-8"):
return error_response(request, 415)
try:
(await request.body()).decode("utf-8")
except UnicodeDecodeError:
return error_response(request, 400)
response = await call_next(request)
response.headers["X-Request-Id"] = request.state.request_id
response.headers["X-Instance-Id"] = INSTANCE_ID
return response
@app.exception_handler(RequestValidationError)
async def validation_error(request: Request, exc: RequestValidationError):
return error_response(request, 400)
@app.exception_handler(HTTPException)
async def http_error(request: Request, exc: HTTPException):
return error_response(request, exc.status_code, exc.headers)
@app.exception_handler(Exception)
async def unexpected_error(request: Request, exc: Exception):
return error_response(request, 500)
+68
View File
@@ -0,0 +1,68 @@
"""Request bodies from the lab 1 contract."""
from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
class StrictBody(BaseModel):
model_config = ConfigDict(extra="forbid", strict=True)
class CreateUser(StrictBody):
username: str = Field(pattern=r"^[a-z0-9_]{3,32}$")
display_name: str = Field(min_length=1, max_length=100)
password: str = Field(min_length=12, max_length=128)
@field_validator("display_name")
@classmethod
def nonblank_name(cls, value: str) -> str:
if not value.strip():
raise ValueError("blank display name")
return value
@field_validator("password")
@classmethod
def nonblank_password(cls, value: str) -> str:
if not value.strip():
raise ValueError("blank password")
return value
class CreateChat(StrictBody):
title: str = Field(min_length=1, max_length=100)
member_ids: list[str] = Field(min_length=1, max_length=100)
@field_validator("title")
@classmethod
def nonblank_title(cls, value: str) -> str:
if not value.strip():
raise ValueError("blank title")
return value
@field_validator("member_ids")
@classmethod
def valid_members(cls, value: list[str]) -> list[str]:
members = [str(UUID(member_id)) for member_id in value]
if len(set(members)) != len(members):
raise ValueError("duplicate member")
return members
class CreateMessage(StrictBody):
text: str = Field(max_length=4000)
attachment_ids: list[str] = Field(default_factory=list, max_length=10)
@field_validator("attachment_ids")
@classmethod
def valid_attachments(cls, value: list[str]) -> list[str]:
ids = [str(UUID(item)) for item in value]
if len(set(ids)) != len(ids):
raise ValueError("duplicate attachment")
return ids
@model_validator(mode="after")
def text_or_attachment(self):
if not self.text.strip() and not self.attachment_ids:
raise ValueError("message needs text or attachment")
return self
+278
View File
@@ -0,0 +1,278 @@
"""Atomic PostgreSQL state changes for the attachment processing pipeline.
Callers own the connection and must commit before replying 202 or acking AMQP.
S3 writes and RabbitMQ publication happen outside these transactions.
"""
from datetime import timezone
from uuid import uuid4
REQUIRED_VARIANTS = {"original", "thumbnail", "crop", "optimized"}
def job_payload(row):
"""Broker message per jobs.schema.json: identifiers only, never file bytes."""
return {
"schema_version": 1,
"job_id": str(row["job_id"]),
"type": "attachment.process.v1",
"attachment_id": str(row["attachment_id"]),
"pipeline_version": row["pipeline_version"],
"attempt": row["attempt"],
"created_at": row["created_at"].astimezone(timezone.utc).isoformat().replace("+00:00", "Z"),
}
def reserve_upload(conn, attachment_id, staging_key, lease_seconds=300):
"""Record an active staging PUT before it starts."""
if lease_seconds < 1:
raise ValueError("invalid upload lease")
with conn.transaction():
conn.execute(
"INSERT INTO attachment_uploads (id,staging_key,lease_expires_at) "
"VALUES (%s,%s,clock_timestamp()+make_interval(secs => %s))",
(attachment_id, staging_key, lease_seconds),
)
def renew_upload(conn, attachment_id, lease_seconds=300):
"""Keep a slow, active upload protected from orphan cleanup."""
if lease_seconds < 1:
raise ValueError("invalid upload lease")
with conn.transaction():
return conn.execute(
"UPDATE attachment_uploads SET lease_expires_at=clock_timestamp()+make_interval(secs => %s) "
"WHERE id=%s AND lease_expires_at>clock_timestamp() RETURNING id",
(lease_seconds, attachment_id),
).fetchone() is not None
def queue_attachment(conn, *, attachment_id, chat_id, owner_id, filename,
staging_key, object_key, content_type, size_bytes, sha256,
pipeline_version=1):
"""Commit attachment, job, and outbox after the full S3 staging PUT."""
job_id = uuid4()
with conn.transaction():
marker = conn.execute(
"SELECT id FROM attachment_uploads WHERE id=%s AND staging_key=%s "
"AND lease_expires_at>clock_timestamp() FOR UPDATE",
(attachment_id, staging_key),
).fetchone()
if marker is None:
raise ValueError("upload reservation is missing")
row = conn.execute(
"INSERT INTO attachments "
"(id,chat_id,owner_id,filename,object_key,staging_key,content_type,size_bytes,sha256,status,pipeline_version) "
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,'queued',%s) "
"RETURNING id,chat_id,owner_id,filename,content_type,size_bytes,sha256,status,error_code,created_at,updated_at",
(attachment_id, chat_id, owner_id, filename, object_key, staging_key,
content_type, size_bytes, sha256, pipeline_version),
).fetchone()
conn.execute(
"INSERT INTO attachment_jobs (id,attachment_id,pipeline_version) VALUES (%s,%s,%s)",
(job_id, attachment_id, pipeline_version),
)
conn.execute(
"INSERT INTO attachment_outbox (job_id,attempt) VALUES (%s,1)",
(job_id,),
)
conn.execute("DELETE FROM attachment_uploads WHERE id=%s", (attachment_id,))
return row, job_id
def claim_outbox(conn, limit=10, lease_seconds=30):
"""Lease pending publications; publisher confirms must call confirm_outbox."""
if not 1 <= limit <= 100 or lease_seconds < 1:
raise ValueError("invalid outbox lease")
token = uuid4()
with conn.transaction():
rows = conn.execute(
"WITH due AS ("
" SELECT id FROM attachment_outbox"
" WHERE published_at IS NULL AND available_at<=clock_timestamp()"
" AND (claim_expires_at IS NULL OR claim_expires_at<clock_timestamp())"
" ORDER BY available_at,id LIMIT %s FOR UPDATE SKIP LOCKED"
") UPDATE attachment_outbox o"
" SET claim_token=%s,claim_expires_at=clock_timestamp()+make_interval(secs => %s)"
" FROM due,attachment_jobs j"
" WHERE o.id=due.id AND j.id=o.job_id"
" RETURNING o.id,o.job_id,o.attempt,j.attachment_id,j.pipeline_version,j.created_at",
(limit, token, lease_seconds),
).fetchall()
return token, rows
def confirm_outbox(conn, outbox_id, token):
"""Mark published only after RabbitMQ publisher confirm."""
with conn.transaction():
return conn.execute(
"UPDATE attachment_outbox SET published_at=clock_timestamp(),"
"claim_token=NULL,claim_expires_at=NULL "
"WHERE id=%s AND claim_token=%s AND published_at IS NULL RETURNING id",
(outbox_id, token),
).fetchone() is not None
def claim_job(conn, job_id, delivery_attempt, lease_seconds=60):
"""A stale/redelivered message cannot steal an active or finished job."""
if lease_seconds < 1:
raise ValueError("invalid job lease")
token = uuid4()
with conn.transaction():
job = conn.execute(
"SELECT j.*,a.staging_key,a.object_key,a.sha256,a.content_type,a.size_bytes,"
"a.pipeline_version AS current_pipeline_version "
"FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id "
"WHERE j.id=%s FOR UPDATE OF j",
(job_id,),
).fetchone()
if (job is None or job["status"] != "queued" or
job["pipeline_version"] != job["current_pipeline_version"] or
job["attempt"] != delivery_attempt or
job["next_attempt_at"] > conn.execute("SELECT clock_timestamp() AS now").fetchone()["now"]):
return None
conn.execute(
"UPDATE attachment_jobs SET status='processing',lease_token=%s,"
"lease_expires_at=clock_timestamp()+make_interval(secs => %s),"
"updated_at=clock_timestamp() WHERE id=%s",
(token, lease_seconds, job_id),
)
conn.execute(
"UPDATE attachments SET status='processing',updated_at=clock_timestamp() "
"WHERE id=%s AND pipeline_version=%s AND status='queued'",
(job["attachment_id"], job["pipeline_version"]),
)
return {**job, "lease_token": token}
def _reschedule_locked(conn, job, *, max_attempts, delay_seconds, error_code, permanent):
if max_attempts < 1 or delay_seconds < 0:
raise ValueError("invalid retry configuration")
failed = permanent or job["attempt"] >= max_attempts
if failed:
code = error_code if permanent else "retry_exhausted"
conn.execute(
"UPDATE attachment_jobs SET status='failed',lease_token=NULL,lease_expires_at=NULL,"
"error_code=%s,updated_at=clock_timestamp() WHERE id=%s",
(code, job["id"]),
)
conn.execute(
"UPDATE attachments SET status='failed',error_code=%s,updated_at=clock_timestamp() "
"WHERE id=%s AND pipeline_version=%s",
(code, job["attachment_id"], job["pipeline_version"]),
)
else:
attempt = job["attempt"] + 1
due = conn.execute(
"UPDATE attachment_jobs SET status='queued',attempt=%s,lease_token=NULL,lease_expires_at=NULL,"
"next_attempt_at=clock_timestamp()+make_interval(secs => %s),"
"error_code=%s,updated_at=clock_timestamp() WHERE id=%s RETURNING next_attempt_at",
(attempt, delay_seconds, error_code, job["id"]),
).fetchone()["next_attempt_at"]
conn.execute(
"INSERT INTO attachment_outbox (job_id,attempt,available_at) VALUES (%s,%s,%s)",
(job["id"], attempt, due),
)
conn.execute(
"UPDATE attachments SET status='queued',error_code=NULL,updated_at=clock_timestamp() "
"WHERE id=%s AND pipeline_version=%s",
(job["attachment_id"], job["pipeline_version"]),
)
return "failed" if failed else "queued"
def finish_attempt(conn, job_id, token, *, max_attempts, delay_seconds=0,
error_code="processing_failed", permanent=False):
"""Persist retry or terminal failure before the worker acks delivery."""
if error_code not in {"invalid_image", "processing_failed", "storage_unavailable", "worker_timeout"}:
raise ValueError("unsafe error code")
with conn.transaction():
job = conn.execute(
"SELECT * FROM attachment_jobs WHERE id=%s FOR UPDATE", (job_id,)
).fetchone()
if job is None or job["status"] != "processing" or job["lease_token"] != token:
raise ValueError("job lease is no longer owned")
return _reschedule_locked(
conn, job, max_attempts=max_attempts, delay_seconds=delay_seconds,
error_code=error_code, permanent=permanent,
)
def recover_expired(conn, *, max_attempts, delay_seconds=0, limit=20):
"""Requeue dead workers; old delivery attempts become harmless duplicates."""
if not 1 <= limit <= 100:
raise ValueError("invalid recovery batch")
with conn.transaction():
jobs = conn.execute(
"SELECT * FROM attachment_jobs WHERE status='processing' "
"AND lease_expires_at<=clock_timestamp() "
"ORDER BY lease_expires_at,id LIMIT %s FOR UPDATE SKIP LOCKED",
(limit,),
).fetchall()
return [
(job["id"], _reschedule_locked(
conn, job, max_attempts=max_attempts, delay_seconds=delay_seconds,
error_code="worker_timeout", permanent=False,
))
for job in jobs
]
def complete_job(conn, job_id, token, variants):
"""Record every versioned effect, then expose ready atomically.
Returns True when this version is still current. A duplicate ready job is
a harmless no-op; an old version never changes the current attachment.
"""
if set(variants) != REQUIRED_VARIANTS:
raise ValueError("incomplete variant set")
with conn.transaction():
job = conn.execute(
"SELECT j.*,a.object_key,a.sha256,a.content_type,a.size_bytes,"
"a.pipeline_version AS current_pipeline_version "
"FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id "
"WHERE j.id=%s FOR UPDATE OF j",
(job_id,),
).fetchone()
if job is None:
raise ValueError("unknown job")
if job["status"] == "ready":
return job["pipeline_version"] == job["current_pipeline_version"]
if job["status"] != "processing" or job["lease_token"] != token:
raise ValueError("job lease is no longer owned")
original = variants["original"]
if (original["object_key"] != job["object_key"] or
original["sha256"] != job["sha256"] or
original["content_type"] != job["content_type"] or
original["size_bytes"] != job["size_bytes"]):
raise ValueError("original does not match accepted upload")
for name, value in variants.items():
fields = (value["object_key"], value["content_type"], value["size_bytes"],
value["width"], value["height"], value["sha256"])
conn.execute(
"INSERT INTO attachment_variants "
"(attachment_id,pipeline_version,name,object_key,content_type,size_bytes,width,height,sha256) "
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING",
(job["attachment_id"], job["pipeline_version"], name, *fields),
)
stored = conn.execute(
"SELECT object_key,content_type,size_bytes,width,height,sha256 "
"FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s AND name=%s",
(job["attachment_id"], job["pipeline_version"], name),
).fetchone()
if stored is None or tuple(stored[key] for key in
("object_key", "content_type", "size_bytes", "width", "height", "sha256")) != fields:
raise ValueError("versioned variant conflicts with a previous effect")
conn.execute(
"UPDATE attachment_jobs SET status='ready',lease_token=NULL,lease_expires_at=NULL,"
"error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",
(job_id,),
)
current = conn.execute(
"UPDATE attachments SET status='ready',error_code=NULL,"
"updated_at=clock_timestamp() "
"WHERE id=%s AND pipeline_version=%s RETURNING id",
(job["attachment_id"], job["pipeline_version"]),
).fetchone()
return current is not None
+70
View File
@@ -0,0 +1,70 @@
"""Publish PostgreSQL outbox jobs with RabbitMQ confirms."""
import json
import logging
import os
import time
from pathlib import Path
import pika
import psycopg
from psycopg_pool import PoolTimeout
from .db import pool
from .pipeline_jobs import claim_outbox, confirm_outbox, job_payload
logger = logging.getLogger(__name__)
QUEUE = os.environ.get("BROKER_QUEUE", "attachments.process")
def publish_batch(channel):
with pool().connection() as conn:
token, rows = claim_outbox(conn, limit=10, lease_seconds=30)
for row in rows:
body = json.dumps(job_payload(row), separators=(",", ":")).encode("utf-8")
channel.basic_publish(
exchange="", routing_key=QUEUE, body=body, mandatory=True,
properties=pika.BasicProperties(
content_type="application/json", delivery_mode=2,
message_id=str(row["job_id"]), type="attachment.process.v1",
),
)
# basic_publish returns only after a broker confirm in confirm mode.
with pool().connection() as conn:
if not confirm_outbox(conn, row["id"], token):
logger.warning("Outbox claim expired before confirm for row %s", row["id"])
return len(rows)
def main():
logging.basicConfig(level=logging.INFO)
logging.getLogger("pika").setLevel(logging.ERROR)
url = Path(os.environ["BROKER_URL_FILE"]).read_text(encoding="utf-8").strip()
params = pika.URLParameters(url)
params.heartbeat = 30
params.socket_timeout = 5
params.blocked_connection_timeout = 10
while True:
broker = None
try:
broker = pika.BlockingConnection(params)
channel = broker.channel()
channel.queue_declare(queue=QUEUE, passive=True)
channel.confirm_delivery()
while broker.is_open:
if not publish_batch(channel):
broker.process_data_events(time_limit=1)
except (pika.exceptions.AMQPError, psycopg.Error, PoolTimeout, OSError) as exc:
logger.warning("Publisher reconnecting after %s", type(exc).__name__)
time.sleep(2)
finally:
if broker is not None and broker.is_open:
try:
broker.close()
except (pika.exceptions.AMQPError, OSError):
pass
if __name__ == "__main__":
main()
+14
View File
@@ -0,0 +1,14 @@
"""Run the ASGI worker as the container's main process."""
import os
import uvicorn
if __name__ == "__main__":
uvicorn.run(
"src.main:app",
host=os.environ.get("APP_HOST", "0.0.0.0"),
port=int(os.environ.get("APP_PORT", "8080")),
workers=1,
)
+104
View File
@@ -0,0 +1,104 @@
"""Remove abandoned staging objects after a grace period or a committed result."""
import logging
import os
import time
from datetime import datetime, timedelta, timezone
from pathlib import Path
from uuid import UUID
from minio import Minio
from .db import pool
logger = logging.getLogger(__name__)
BUCKET = "messenger-attachments"
def storage():
def secret(name):
return Path(os.environ[f"{name}_FILE"]).read_text(encoding="utf-8").strip()
return Minio(
os.environ.get("S3_ENDPOINT", "s3:9000"),
access_key=secret("S3_MAINTENANCE_KEY_ID"),
secret_key=secret("S3_MAINTENANCE_SECRET_KEY"),
secure=False,
region="us-east-1",
)
def clean_once(client, grace_seconds=3600):
"""Return removed count; lock DB state through each S3 delete."""
if grace_seconds < 1:
raise ValueError("invalid staging grace period")
cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds)
removed = 0
for obj in client.list_objects(BUCKET, prefix="staging/", recursive=True):
key = obj.object_name
suffix = key.removeprefix("staging/")
try:
if key != f"staging/{UUID(suffix)}":
continue
except ValueError:
continue
with pool().connection() as conn, conn.transaction():
attachment = conn.execute(
"SELECT a.status,EXISTS("
"SELECT 1 FROM attachment_variants v WHERE v.attachment_id=a.id "
"AND v.pipeline_version=a.pipeline_version AND v.name='original' "
"AND v.object_key=a.object_key) AS original_ready "
"FROM attachments a WHERE a.staging_key=%s FOR UPDATE OF a",
(key,),
).fetchone()
if attachment:
if attachment["status"] != "ready" or not attachment["original_ready"]:
continue
else:
if obj.last_modified is None or obj.last_modified > cutoff:
continue
marker = conn.execute(
"SELECT lease_expires_at>clock_timestamp() AS active "
"FROM attachment_uploads WHERE staging_key=%s FOR UPDATE",
(key,),
).fetchone()
if marker and marker["active"]:
continue
client.remove_object(BUCKET, key)
if not attachment:
conn.execute(
"DELETE FROM attachment_uploads WHERE staging_key=%s "
"AND lease_expires_at<=clock_timestamp()",
(key,),
)
removed += 1
with pool().connection() as conn, conn.transaction():
conn.execute(
"DELETE FROM attachment_uploads u WHERE u.lease_expires_at<"
"clock_timestamp()-make_interval(secs => %s) "
"AND NOT EXISTS (SELECT 1 FROM attachments a WHERE a.staging_key=u.staging_key)",
(grace_seconds,),
)
return removed
def main():
logging.basicConfig(level=logging.INFO)
client = storage()
grace = int(os.environ.get("STAGING_GRACE_SECONDS", "3600"))
interval = int(os.environ.get("STAGING_CLEANUP_INTERVAL_SECONDS", "60"))
if interval < 1:
raise ValueError("invalid cleanup interval")
while True:
try:
removed = clean_once(client, grace)
if removed:
logger.info("Removed %s staging objects", removed)
except Exception as exc:
logger.warning("Staging cleanup retrying after %s", type(exc).__name__)
time.sleep(interval)
if __name__ == "__main__":
main()
+297
View File
@@ -0,0 +1,297 @@
"""PostgreSQL-backed users, chats and message history."""
import base64
import binascii
import hashlib
import json
import os
import secrets
from contextlib import contextmanager
from datetime import datetime, timezone
from uuid import UUID, uuid4
import psycopg
from argon2 import PasswordHasher
from argon2.exceptions import InvalidHashError, VerificationError
from fastapi import HTTPException
from psycopg.errors import UniqueViolation
from psycopg_pool import PoolTimeout
from . import db
from .auth import BASIC_CHALLENGE, BEARER_CHALLENGE, token_hash, unauthorized
password_hasher = PasswordHasher()
def timestamp(value: datetime) -> str:
return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
def encode_cursor(resource: str, item: dict) -> str:
value = [resource, item["created_at"], item["id"]]
return base64.urlsafe_b64encode(json.dumps(value, separators=(",", ":")).encode()).decode().rstrip("=")
def decode_cursor(cursor: str | None, resource: str) -> tuple[datetime, UUID] | None:
if cursor is None:
return None
try:
raw = base64.b64decode(cursor + "=" * (-len(cursor) % 4), altchars=b"-_", validate=True)
value = json.loads(raw)
if not isinstance(value, list) or len(value) != 3 or value[0] != resource:
raise ValueError
date_text, item_id = value[1:]
if not isinstance(date_text, str) or not date_text.endswith("Z") or not isinstance(item_id, str):
raise ValueError
date = datetime.fromisoformat(date_text[:-1] + "+00:00")
if date.utcoffset().total_seconds() != 0:
raise ValueError
return date, UUID(item_id)
except (binascii.Error, UnicodeError, ValueError, TypeError, AttributeError):
raise HTTPException(400) from None
@contextmanager
def connection():
try:
with db.pool().connection() as conn:
yield conn
except (psycopg.Error, PoolTimeout, OSError):
raise HTTPException(503) from None
def user_view(row: dict) -> dict:
return {
"id": str(row["id"]), "username": row["username"],
"display_name": row["display_name"], "created_at": timestamp(row["created_at"]),
}
def chat_view(row: dict) -> dict:
return {
"id": str(row["id"]), "title": row["title"],
"member_ids": [str(member_id) for member_id in row["member_ids"]],
"created_at": timestamp(row["created_at"]),
}
def message_view(row: dict) -> dict:
return {
"id": str(row["id"]), "chat_id": str(row["chat_id"]),
"sender_id": str(row["sender_id"]), "text": row["text"],
"created_at": timestamp(row["created_at"]),
"attachment_ids": [str(attachment_id) for attachment_id in row.get("attachment_ids", [])],
}
CHAT_COLUMNS = """c.id, c.title, c.created_at,
ARRAY(SELECT m.user_id FROM memberships m WHERE m.chat_id=c.id ORDER BY m.position) AS member_ids"""
MESSAGE_COLUMNS = """m.id, m.chat_id, m.sender_id, m.text, m.created_at,
ARRAY(SELECT a.attachment_id FROM message_attachments a
WHERE a.message_id=m.id ORDER BY a.position) AS attachment_ids"""
class Store:
def ready(self) -> dict:
if not db.ready():
raise HTTPException(503)
return {"status": "ok"}
def register(self, username: str, display_name: str, password: str) -> dict:
password_hash = password_hasher.hash(password)
with connection() as conn:
try:
row = conn.execute(
"INSERT INTO users (id, username, display_name, password_hash) VALUES (%s,%s,%s,%s) "
"RETURNING id,username,display_name,created_at",
(uuid4(), username, display_name, password_hash),
).fetchone()
except UniqueViolation:
raise HTTPException(409) from None
return user_view(row)
def user(self, user_id: str | None) -> dict:
if not user_id:
raise HTTPException(401)
try:
user_uuid = UUID(user_id)
except (ValueError, TypeError):
raise HTTPException(401) from None
with connection() as conn:
row = conn.execute(
"SELECT id,username,display_name,created_at FROM users WHERE id=%s", (user_uuid,)
).fetchone()
if row is None:
raise HTTPException(401)
return user_view(row)
def create_session(self, username: str, password: str) -> dict:
with connection() as conn:
row = conn.execute(
"SELECT id,username,display_name,created_at,password_hash FROM users WHERE username=%s",
(username,),
).fetchone()
try:
valid = row is not None and password_hasher.verify(row["password_hash"], password)
except (VerificationError, InvalidHashError):
valid = False
if not valid:
raise unauthorized(BASIC_CHALLENGE)
token = secrets.token_urlsafe(32)
ttl = int(os.environ.get("SESSION_TTL_SECONDS", "3600"))
expiry = conn.execute(
"INSERT INTO sessions (token_hash,user_id,expires_at) "
"VALUES (%s,%s,clock_timestamp()+make_interval(secs => %s)) RETURNING expires_at",
(token_hash(token), row["id"], ttl),
).fetchone()["expires_at"]
return {"token": token, "token_type": "Bearer", "expires_at": timestamp(expiry), "user": user_view(row)}
def authenticated(self, token: str) -> dict:
with connection() as conn:
row = conn.execute(
"SELECT u.id,u.username,u.display_name,u.created_at FROM sessions s "
"JOIN users u ON u.id=s.user_id "
"WHERE s.token_hash=%s AND s.expires_at>clock_timestamp()",
(token_hash(token),),
).fetchone()
if row is None:
raise unauthorized(BEARER_CHALLENGE)
return user_view(row)
def revoke_session(self, token: str) -> None:
with connection() as conn:
row = conn.execute(
"DELETE FROM sessions WHERE token_hash=%s AND expires_at>clock_timestamp() RETURNING token_hash",
(token_hash(token),),
).fetchone()
if row is None:
raise unauthorized(BEARER_CHALLENGE)
def create_chat(self, actor_id: str, title: str, member_ids: list[str]) -> dict:
if actor_id not in member_ids:
raise HTTPException(400)
members = [UUID(member_id) for member_id in member_ids]
with connection() as conn:
count = conn.execute("SELECT count(*) AS n FROM users WHERE id=ANY(%s)", (members,)).fetchone()["n"]
if count != len(members):
raise HTTPException(400)
row = conn.execute("INSERT INTO chats (id,title) VALUES (%s,%s) RETURNING id,title,created_at", (uuid4(), title)).fetchone()
for position, member_id in enumerate(members):
conn.execute(
"INSERT INTO memberships (chat_id,user_id,position) VALUES (%s,%s,%s)",
(row["id"], member_id, position),
)
return chat_view({**row, "member_ids": members})
def chat(self, actor_id: str, chat_id: str) -> dict:
with connection() as conn:
row = conn.execute(
f"SELECT {CHAT_COLUMNS} FROM chats c "
"JOIN memberships access ON access.chat_id=c.id "
"WHERE access.user_id=%s AND c.id=%s",
(UUID(actor_id), UUID(chat_id)),
).fetchone()
if row is None:
raise HTTPException(404)
return chat_view(row)
def list_chats(self, actor_id: str, limit: int, cursor: str | None) -> dict:
after = decode_cursor(cursor, f"chats:{actor_id}")
query = (f"SELECT {CHAT_COLUMNS} FROM chats c "
"JOIN memberships access ON access.chat_id=c.id WHERE access.user_id=%s")
params = [UUID(actor_id)]
if after:
query += " AND (c.created_at,c.id)>(%s,%s)"
params.extend(after)
query += " ORDER BY c.created_at,c.id LIMIT %s"
params.append(limit + 1)
with connection() as conn:
rows = conn.execute(query, params).fetchall()
items = [chat_view(row) for row in rows[:limit]]
return {"items": items, "next_cursor": encode_cursor(f"chats:{actor_id}", items[-1]) if len(rows) > limit else None}
def create_message(self, actor_id: str, chat_id: str, text: str, key: str, attachment_ids: list[str] | None = None) -> dict:
ids = [UUID(item) for item in attachment_ids or []]
if len(ids) > 10 or len(set(ids)) != len(ids) or (not text.strip() and not ids):
raise HTTPException(400)
chat_uuid, actor_uuid = UUID(chat_id), UUID(actor_id)
request_hash = hashlib.sha256(json.dumps(
{"text": text, "attachment_ids": [str(item) for item in ids]}, ensure_ascii=False, sort_keys=True,
).encode()).digest()
with connection() as conn:
member = conn.execute(
"SELECT 1 FROM memberships WHERE chat_id=%s AND user_id=%s", (chat_uuid, actor_uuid)
).fetchone()
if member is None:
raise HTTPException(404)
conn.execute(
"DELETE FROM message_replays WHERE user_id=%s AND chat_id=%s AND request_key=%s "
"AND created_at<clock_timestamp()-interval '24 hours'",
(actor_uuid, chat_uuid, key),
)
reserved = conn.execute(
"INSERT INTO message_replays (user_id,chat_id,request_key,request_hash) VALUES (%s,%s,%s,%s) "
"ON CONFLICT DO NOTHING RETURNING request_key",
(actor_uuid, chat_uuid, key, request_hash),
).fetchone()
if reserved is None:
replay = conn.execute(
f"SELECT r.request_hash,{MESSAGE_COLUMNS} "
"FROM message_replays r JOIN messages m ON m.id=r.message_id "
"WHERE r.user_id=%s AND r.chat_id=%s AND r.request_key=%s",
(actor_uuid, chat_uuid, key),
).fetchone()
if replay is None:
raise HTTPException(503)
if replay["request_hash"] != request_hash:
raise HTTPException(409)
return message_view(replay)
if ids:
attachments = conn.execute(
"SELECT id,status FROM attachments WHERE chat_id=%s AND id=ANY(%s)",
(chat_uuid, ids),
).fetchall()
if len(attachments) != len(ids):
raise HTTPException(404)
if any(item["status"] == "failed" for item in attachments):
raise HTTPException(409)
row = conn.execute(
"INSERT INTO messages (id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) "
"RETURNING id,chat_id,sender_id,text,created_at",
(uuid4(), chat_uuid, actor_uuid, text),
).fetchone()
for position, attachment_id in enumerate(ids):
conn.execute(
"INSERT INTO message_attachments (message_id,chat_id,attachment_id,position) "
"VALUES (%s,%s,%s,%s)",
(row["id"], chat_uuid, attachment_id, position),
)
conn.execute(
"UPDATE message_replays SET message_id=%s WHERE user_id=%s AND chat_id=%s AND request_key=%s",
(row["id"], actor_uuid, chat_uuid, key),
)
return message_view({**row, "attachment_ids": ids})
def list_messages(self, actor_id: str, chat_id: str, limit: int, cursor: str | None) -> dict:
after = decode_cursor(cursor, f"messages:{chat_id}")
chat_uuid, actor_uuid = UUID(chat_id), UUID(actor_id)
query = f"SELECT {MESSAGE_COLUMNS} FROM messages m WHERE m.chat_id=%s"
params = [chat_uuid]
if after:
query += " AND (m.created_at,m.id)>(%s,%s)"
params.extend(after)
query += " ORDER BY m.created_at,m.id LIMIT %s"
params.append(limit + 1)
with connection() as conn:
member = conn.execute(
"SELECT 1 FROM memberships WHERE chat_id=%s AND user_id=%s", (chat_uuid, actor_uuid)
).fetchone()
if member is None:
raise HTTPException(404)
rows = conn.execute(query, params).fetchall()
items = [message_view(row) for row in rows[:limit]]
return {"items": items, "next_cursor": encode_cursor(f"messages:{chat_id}", items[-1]) if len(rows) > limit else None}
store = Store()
+216
View File
@@ -0,0 +1,216 @@
"""At-least-once RabbitMQ consumer with versioned, idempotent S3 effects."""
import hashlib
import json
import logging
import os
import random
import re
import signal
import time
from contextlib import contextmanager
from datetime import datetime
from io import BytesIO
from pathlib import Path
from uuid import UUID
import pika
import psycopg
from minio.error import S3Error
from psycopg_pool import PoolTimeout
from .attachments import BUCKET, storage
from .db import pool
from .image_variants import InvalidImage, MAX_BYTES, render
from .pipeline_jobs import claim_job, complete_job, finish_attempt, recover_expired
logger = logging.getLogger(__name__)
QUEUE = os.environ.get("BROKER_QUEUE", "attachments.process")
MAX_ATTEMPTS = int(os.environ.get("JOB_MAX_ATTEMPTS", "5"))
LEASE_SECONDS = int(os.environ.get("JOB_TIMEOUT_SECONDS", "60"))
def parse_message(body: bytes) -> dict:
if len(body) > 4096:
raise ValueError("oversized job message")
item = json.loads(body)
required = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at"}
if (not isinstance(item, dict) or not required <= item.keys() or
item.keys() - required - {"traceparent", "tracestate"} or
type(item["schema_version"]) is not int or item["schema_version"] != 1 or
item["type"] != "attachment.process.v1" or
type(item["pipeline_version"]) is not int or item["pipeline_version"] < 1 or
type(item["attempt"]) is not int or item["attempt"] < 1 or
not isinstance(item["job_id"], str) or
not isinstance(item["attachment_id"], str) or
not isinstance(item["created_at"], str) or
("traceparent" in item and
(not isinstance(item["traceparent"], str) or
re.fullmatch(r"00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}", item["traceparent"]) is None)) or
("tracestate" in item and
(not isinstance(item["tracestate"], str) or len(item["tracestate"]) > 512))):
raise ValueError("invalid job message")
UUID(item["job_id"])
UUID(item["attachment_id"])
created_at = datetime.fromisoformat(item["created_at"].replace("Z", "+00:00"))
if created_at.utcoffset() is None:
raise ValueError("job timestamp lacks timezone")
return item
def _source_bytes(job) -> bytes:
if not 0 < job["size_bytes"] <= MAX_BYTES:
raise InvalidImage("invalid accepted size")
# Reprocessing uses the byte-identical final original, so ready staging
# can be reclaimed without making a future pipeline version impossible.
key = job["staging_key"] if job["pipeline_version"] == 1 else job["object_key"]
response = storage().get_object(BUCKET, key)
try:
chunks = []
size = 0
digest = hashlib.sha256()
for chunk in response.stream(64 * 1024):
size += len(chunk)
if size > job["size_bytes"]:
raise InvalidImage("source exceeds accepted size")
chunks.append(chunk)
digest.update(chunk)
if size != job["size_bytes"] or digest.hexdigest() != job["sha256"]:
raise InvalidImage("source checksum mismatch")
return b"".join(chunks)
finally:
response.close()
response.release_conn()
def _write_variants(job, variants: dict) -> dict:
extension = {"thumbnail": "jpg", "crop": "jpg", "optimized": "webp"}
stored = {}
for name, variant in variants.items():
key = (job["object_key"] if name == "original" else
f"variants/{job['attachment_id']}/v{job['pipeline_version']}/{name}.{extension[name]}")
data = variant["data"]
storage().put_object(
BUCKET, key, BytesIO(data), len(data), content_type=variant["content_type"]
)
stored[name] = {key_name: value for key_name, value in variant.items() if key_name != "data"}
stored[name]["object_key"] = key
return stored
def _delay(attempt: int) -> float:
base = min(30, 2 ** min(attempt - 1, 5))
return base + random.uniform(0, base / 2)
@contextmanager
def _deadline(seconds: int):
def expired(_signal, _frame):
raise TimeoutError("image job exceeded worker deadline")
previous = signal.signal(signal.SIGALRM, expired)
signal.setitimer(signal.ITIMER_REAL, seconds)
try:
yield
finally:
signal.setitimer(signal.ITIMER_REAL, 0)
signal.signal(signal.SIGALRM, previous)
def process_message(item: dict) -> str:
with pool().connection() as conn:
job = claim_job(conn, UUID(item["job_id"]), item["attempt"], lease_seconds=LEASE_SECONDS)
if job is None:
return "duplicate_or_stale"
token = job["lease_token"]
try:
with _deadline(LEASE_SECONDS):
data = _source_bytes(job)
variants = render(data, job["content_type"])
stored = _write_variants(job, variants)
with pool().connection() as conn:
current = complete_job(conn, job["id"], token, stored)
if current and job["staging_key"]:
try:
storage().remove_object(BUCKET, job["staging_key"])
except Exception as exc:
# Periodic cleanup recovers this post-COMMIT gap.
logger.warning("Staging cleanup deferred for %s (%s)", job["id"], type(exc).__name__)
return "ready"
except psycopg.Error:
raise
except Exception as exc:
if isinstance(exc, InvalidImage):
code, permanent = "invalid_image", True
elif isinstance(exc, TimeoutError):
code, permanent = "worker_timeout", False
elif isinstance(exc, S3Error) or isinstance(exc, OSError):
code, permanent = "storage_unavailable", False
elif isinstance(exc, ValueError):
code, permanent = "processing_failed", True
else:
code, permanent = "processing_failed", False
logger.warning("Job %s attempt %s: %s", job["id"], job["attempt"], code)
with pool().connection() as conn:
try:
return finish_attempt(
conn, job["id"], token, max_attempts=MAX_ATTEMPTS,
delay_seconds=_delay(job["attempt"]), error_code=code,
permanent=permanent,
)
except ValueError:
# A lease recovery already owns the next attempt.
return "stale_lease"
def main() -> None:
logging.basicConfig(level=logging.INFO)
if MAX_ATTEMPTS < 1 or LEASE_SECONDS < 1:
raise SystemExit("invalid worker retry settings")
url = Path(os.environ["BROKER_URL_FILE"]).read_text(encoding="utf-8").strip()
params = pika.URLParameters(url)
params.heartbeat = max(120, LEASE_SECONDS * 2)
params.socket_timeout = 5
params.blocked_connection_timeout = 10
while True:
broker = None
try:
broker = pika.BlockingConnection(params)
channel = broker.channel()
channel.queue_declare(queue=QUEUE, passive=True)
channel.basic_qos(prefetch_count=1)
last_recovery = 0.0
while broker.is_open:
if time.monotonic() - last_recovery >= 5:
with pool().connection() as conn:
recover_expired(conn, max_attempts=MAX_ATTEMPTS)
last_recovery = time.monotonic()
method, _, body = channel.basic_get(queue=QUEUE, auto_ack=False)
if method is None:
broker.process_data_events(time_limit=1)
continue
try:
item = parse_message(body)
except (ValueError, TypeError, KeyError, json.JSONDecodeError):
logger.warning("Rejecting invalid job message")
channel.basic_reject(method.delivery_tag, requeue=False)
continue
try:
result = process_message(item)
channel.basic_ack(method.delivery_tag)
logger.info("Job %s: %s", item["job_id"], result)
except (psycopg.Error, PoolTimeout, OSError) as exc:
logger.warning("Job %s database unavailable (%s)", item["job_id"], type(exc).__name__)
channel.basic_nack(method.delivery_tag, requeue=True)
time.sleep(2)
except (pika.exceptions.AMQPError, psycopg.Error, PoolTimeout, OSError) as exc:
logger.warning("Worker reconnecting after %s", type(exc).__name__)
time.sleep(2)
finally:
if broker is not None and broker.is_open:
broker.close()
if __name__ == "__main__":
main()
+74
View File
@@ -0,0 +1,74 @@
"""Small checks for the worker's deterministic image and payload boundary."""
import json
import unittest
from io import BytesIO
from PIL import Image
from src.image_variants import InvalidImage, render
from src.worker import parse_message
def encoded(image, format="PNG", **options):
stream = BytesIO()
image.save(stream, format=format, **options)
return stream.getvalue()
class ImagePipelineTest(unittest.TestCase):
def test_contract_sizes_original_and_content_types(self):
original = encoded(Image.new("RGB", (320, 200), "red"))
result = render(original, "image/png")
self.assertEqual(result["original"]["data"], original)
self.assertEqual((result["thumbnail"]["width"], result["thumbnail"]["height"]), (256, 160))
self.assertEqual((result["crop"]["width"], result["crop"]["height"]), (128, 128))
self.assertEqual((result["optimized"]["width"], result["optimized"]["height"]), (320, 200))
self.assertEqual([result[name]["content_type"] for name in ("thumbnail", "crop", "optimized")],
["image/jpeg", "image/jpeg", "image/webp"])
def test_alpha_is_white_and_small_image_is_not_upscaled(self):
original = encoded(Image.new("RGBA", (24, 12), (255, 0, 0, 0)))
result = render(original, "image/png")
self.assertEqual((result["thumbnail"]["width"], result["thumbnail"]["height"]), (24, 12))
with Image.open(BytesIO(result["thumbnail"]["data"])) as thumb:
self.assertTrue(all(channel >= 245 for channel in thumb.getpixel((0, 0))))
def test_exif_orientation_and_center_crop(self):
image = Image.new("RGB", (30, 10), "green")
image.paste("red", (0, 0, 10, 10))
image.paste("blue", (20, 0, 30, 10))
exif = Image.Exif()
exif[274] = 6
original = encoded(image, "JPEG", exif=exif)
result = render(original, "image/jpeg")
self.assertEqual((result["thumbnail"]["width"], result["thumbnail"]["height"]), (10, 30))
with Image.open(BytesIO(result["crop"]["data"])) as crop:
red, green, blue = crop.getpixel((64, 64))
self.assertGreater(green, red)
self.assertGreater(green, blue)
def test_rejects_wrong_type_and_corrupt_file(self):
original = encoded(Image.new("RGB", (2, 2), "black"))
with self.assertRaises(InvalidImage):
render(original, "image/jpeg")
with self.assertRaises(InvalidImage):
render(b"broken", "image/png")
def test_payload_contains_only_versioned_job_identifiers(self):
body = {
"schema_version": 1,
"job_id": "11111111-1111-4111-8111-111111111111",
"type": "attachment.process.v1",
"attachment_id": "22222222-2222-4222-8222-222222222222",
"pipeline_version": 1,
"attempt": 1,
"created_at": "2026-09-01T12:00:00Z",
}
self.assertEqual(parse_message(json.dumps(body).encode()), body)
with self.assertRaises(ValueError):
parse_message(json.dumps({**body, "staging_key": "private"}).encode())
if __name__ == "__main__":
unittest.main()
+35
View File
@@ -0,0 +1,35 @@
#!/usr/bin/env python3
"""Keep broker messages within the public job schema and free of file data."""
import json
import sys
import unittest
from datetime import datetime, timezone
from pathlib import Path
from uuid import uuid4
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
sys.path.insert(0, str(ROOT / "tests"))
from client import validate # noqa: E402
from src.pipeline_jobs import job_payload # noqa: E402
class OutboxPayloadTest(unittest.TestCase):
def test_job_message_has_only_contract_fields(self):
schema = json.loads((ROOT / "contracts/jobs.schema.json").read_text(encoding="utf-8"))
row = {
"job_id": uuid4(), "attachment_id": uuid4(), "pipeline_version": 2,
"attempt": 3, "created_at": datetime.now(timezone.utc),
"object_key": "staging/must-not-leak", "bytes": b"must-not-leak",
}
message = job_payload(row)
validate(message, schema)
self.assertEqual(set(message), set(schema["required"]))
self.assertNotIn("staging", json.dumps(message))
self.assertNotIn("bytes", json.dumps(message))
if __name__ == "__main__":
unittest.main()
+46
View File
@@ -0,0 +1,46 @@
"""Run inside the cleanup image against an isolated pair of staging keys."""
import time
import sys
from io import BytesIO
from pathlib import Path
from uuid import uuid4
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from src.db import pool
from src.pipeline_jobs import reserve_upload
from src.staging_cleanup import BUCKET, clean_once, storage
client = storage()
orphan_id, active_id = uuid4(), uuid4()
orphan_key, active_key = f"staging/{orphan_id}", f"staging/{active_id}"
def exists(key):
return any(obj.object_name == key for obj in client.list_objects(BUCKET, prefix=key))
try:
for key in (orphan_key, active_key):
client.put_object(BUCKET, key, BytesIO(b"test"), 4, content_type="image/png")
with pool().connection() as conn:
reserve_upload(conn, active_id, active_key, lease_seconds=60)
time.sleep(2)
clean_once(client, grace_seconds=1)
assert not exists(orphan_key), "old orphan was not removed"
assert exists(active_key), "active upload was removed"
with pool().connection() as conn:
conn.execute(
"UPDATE attachment_uploads SET lease_expires_at=clock_timestamp()-interval '1 second' WHERE id=%s",
(active_id,),
)
clean_once(client, grace_seconds=1)
assert not exists(active_key), "expired orphan was not removed"
print("PASS: cleanup removes old orphans and preserves active uploads")
finally:
for key in (orphan_key, active_key):
client.remove_object(BUCKET, key)
with pool().connection() as conn:
conn.execute("DELETE FROM attachment_uploads WHERE id=%s", (active_id,))
+61
View File
@@ -0,0 +1,61 @@
#!/usr/bin/env python3
"""Exercise 202 handoff while RabbitMQ and worker are unavailable."""
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, arguments, check, run # noqa: E402
def compose(*args):
subprocess.run(
["docker", "compose", "--env-file", ".env", *args],
cwd=ROOT, check=True, stdout=subprocess.DEVNULL,
)
def ready_after_restart(client):
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
try:
client.call("GET", "/health/ready", schema="Health")
return
except Failure:
time.sleep(1)
raise Failure("API did not recover after restart")
def main():
args = arguments("Lab5 durable upload handoff with stopped broker and worker")
client = Client(args.base_url, args.ca_file)
compose("stop", "worker", "publisher", "broker")
try:
alice, bob, _eve, chat = client.bootstrap()
attachment, original = client.attachment(alice, chat)
client.call("GET", f"/api/v1/attachments/{attachment['id']}/content", 409, actor=bob)
message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"]
check(message["attachment_ids"] == [attachment["id"]], "queued attachment cannot be linked")
compose("up", "-d", "broker")
compose("restart", "api")
ready_after_restart(client)
check(
client.call("GET", f"/api/v1/attachments/{attachment['id']}", actor=bob, schema="Attachment")["json"]["status"] == "queued",
"accepted job disappeared before publisher restart",
)
compose("up", "-d", "publisher", "worker")
client.wait_ready(bob, attachment)
result = client.call("GET", f"/api/v1/attachments/{attachment['id']}/content", actor=bob)
check(result["bytes"] == original, "accepted original changed after recovery")
finally:
compose("up", "-d", "broker", "publisher", "worker")
print("PASS: accepted upload survived broker/API outage and reached ready")
if __name__ == "__main__":
run(main)
+26
View File
@@ -0,0 +1,26 @@
#!/usr/bin/env python3
"""Public U2 check while publisher and worker are stopped."""
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, arguments, check, run # noqa: E402
def main():
args = arguments("Lab5 durable queued upload")
client = Client(args.base_url, args.ca_file)
alice, bob, _eve, chat = client.bootstrap()
attachment, _content = client.attachment(alice, chat)
check(attachment["status"] == "queued", "upload did not enter queued state")
client.call("GET", f"/api/v1/attachments/{attachment['id']}/content", 409, actor=bob)
message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"]
check(message["attachment_ids"] == [attachment["id"]], "queued attachment cannot be linked")
print("PASS: 202 queued upload is readable and linkable; content is pending")
if __name__ == "__main__":
run(main)
+47
View File
@@ -0,0 +1,47 @@
#!/usr/bin/env python3
"""Check every public variant against committed metadata and original bytes."""
import hashlib
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
from client import Client, arguments, check, run # noqa: E402
def main():
args = arguments("Lab5 ready variants")
client = Client(args.base_url, args.ca_file)
alice, bob, eve, chat = client.bootstrap()
attachment, original = client.attachment(alice, chat)
ready = client.wait_ready(bob, attachment)
variants = {item["name"]: item for item in ready["variants"]}
check(set(variants) == {"thumbnail", "crop", "optimized"}, "incomplete variant set")
expected = {
"thumbnail": ("image/jpeg", 256, 160),
"crop": ("image/jpeg", 128, 128),
"optimized": ("image/webp", 320, 200),
}
for name, (content_type, width, height) in expected.items():
variant = variants[name]
check((variant["content_type"], variant["width"], variant["height"]) ==
(content_type, width, height), f"wrong {name} metadata")
response = client.call(
"GET", f"/api/v1/attachments/{attachment['id']}/content?variant={name}", actor=bob
)
check(response["headers"].get_content_type() == content_type, f"wrong {name} MIME")
check(len(response["bytes"]) == variant["size_bytes"] and
hashlib.sha256(response["bytes"]).hexdigest() == variant["sha256"],
f"wrong {name} bytes")
client.call(
"GET", f"/api/v1/attachments/{attachment['id']}/content?variant={name}",
404, actor=eve,
)
downloaded = client.call("GET", f"/api/v1/attachments/{attachment['id']}/content", actor=bob)
check(downloaded["bytes"] == original, "original bytes changed")
print("PASS: all three variants match metadata; original preserved; outsider denied")
if __name__ == "__main__":
run(main)