feat(pipeline): add durable outbox and versioned image workers

This commit is contained in:
2026-11-15 22:16:04 +03:00
parent 276dbdad8a
commit 5be41d8646
47 changed files with 4728 additions and 4 deletions
+15
View File
@@ -18,3 +18,18 @@ __pycache__
node_modules
evidence
.DS_Store
**/__pycache__
**/*.py[cod]
.work
.pytest_cache
.ruff_cache
tests
student-tests
research
scripts
contracts
*.md
Makefile
compose.yaml
.work*
+12 -4
View File
@@ -13,13 +13,21 @@ API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
S3_BUCKET=messenger
S3_ACCESS_KEY_FILE=/run/secrets/s3_access_key
S3_SECRET_KEY_FILE=/run/secrets/s3_secret_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
PUBLIC_PORT=8085
API_CPUS=0.5
API_MEMORY=768m
DB_POOL_SIZE=10
RECONCILE_GRACE_SECONDS=300
BROKER_URL_FILE=/run/secrets/broker_url
WORKER_CONCURRENCY=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
JOB_TIMEOUT_SECONDS=20
JOB_LEASE_SECONDS=23
STAGING_RETENTION_SECONDS=86400
+5
View File
@@ -30,3 +30,8 @@ coverage/
.vscode/
# Keep small, sanitized evidence summaries in evidence/ under version control.
# Inspect git diff --cached: ignore rules are not a secret scanner.
.work/
.work-*.txt
.pytest_cache/
.ruff_cache/
+22
View File
@@ -0,0 +1,22 @@
FROM ghcr.io/astral-sh/uv:0.13.7@sha256:96f40d3904aad8d0bd010539001d36c8e2188b981e1334471a90c4a1d93bb490 AS uv
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS builder
COPY --from=uv /uv /usr/local/bin/uv
WORKDIR /app
ENV UV_PYTHON_DOWNLOADS=never UV_LINK_MODE=copy
COPY pyproject.toml uv.lock ./
RUN --mount=type=cache,target=/root/.cache/uv uv sync --locked --no-dev --no-install-project
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS runtime
ARG VCS_REF=unknown
LABEL org.opencontainers.image.revision=$VCS_REF
WORKDIR /app
ENV PATH="/app/.venv/bin:$PATH" PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 APP_HOST=0.0.0.0 APP_PORT=8080
RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app
COPY --from=builder /app/.venv /app/.venv
COPY src /app/src
COPY migrations /app/migrations
USER 10001:10001
EXPOSE 8080
HEALTHCHECK --interval=10s --timeout=2s --start-period=10s --start-interval=1s --retries=3 CMD ["python", "-c", "import os,urllib.request; urllib.request.urlopen('http://127.0.0.1:'+os.getenv('APP_PORT','8080')+'/health/ready',timeout=1).read()"]
STOPSIGNAL SIGTERM
CMD ["python", "-m", "src"]
+17
View File
@@ -28,3 +28,20 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
UV ?= uv
BASE_URL = http://127.0.0.1:8085
.PHONY: bootstrap up down lint test-student verify
bootstrap:
$(PYTHON) scripts/bootstrap.py
up: bootstrap
VCS_REF=$$(git rev-parse HEAD) docker compose up -d --build --wait
down:
docker compose down
lint:
$(UV) run --locked ruff format --check src student-tests scripts/bootstrap.py scripts/stand.py scripts/cluster.py $(wildcard scripts/verify_*.py)
$(UV) run --locked ruff check src student-tests scripts/bootstrap.py scripts/stand.py scripts/cluster.py $(wildcard scripts/verify_*.py)
test-student:
BASE_URL="$(BASE_URL)" $(UV) run --locked pytest student-tests
verify:
$(UV) run --locked python -m scripts.verify_pipeline --output evidence/pipeline-repeat.json
+13
View File
@@ -0,0 +1,13 @@
# Explicit test-only fault instrumentation, never enabled by default.
services:
api:
environment:
PIPELINE_FAULTS: "1"
relay:
environment:
PIPELINE_FAULTS: "1"
worker:
environment:
PIPELINE_FAULTS: "1"
JOB_TIMEOUT_SECONDS: "12"
JOB_LEASE_SECONDS: "15"
+219
View File
@@ -0,0 +1,219 @@
name: mtusi-lab5-maslov
services:
postgres:
image: postgres:18.3@sha256:7e32e9833a6fb1c92c32552794cb6ed569d51b445a54907d35fc112ef39684db
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
PGDATA: /var/lib/postgresql/data/pgdata
secrets: [postgres_password]
volumes:
- postgres_data:/var/lib/postgresql
healthcheck:
test: [CMD-SHELL, 'pg_isready -U messenger -d messenger']
interval: 2s
timeout: 2s
retries: 20
cpus: 1
mem_limit: 512m
memswap_limit: 512m
cache:
image: valkey/valkey:8.1.3@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571
command: [valkey-server, /run/secrets/valkey.conf]
secrets:
- cache_password
- source: valkey_config
target: valkey.conf
healthcheck:
test: [CMD-SHELL, 'REDISCLI_AUTH="$$(cat /run/secrets/cache_password)" valkey-cli ping']
interval: 2s
timeout: 2s
retries: 20
cpus: 0.25
mem_limit: 128m
memswap_limit: 128m
migrate:
image: ${API_IMAGE:-mtusi-messenger:lab5}
pull_policy: never
command: [python, -m, src.migrate]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
secrets: [database_url]
depends_on:
postgres: {condition: service_healthy}
cpus: 0.5
mem_limit: 256m
read_only: true
api:
image: ${API_IMAGE:-mtusi-messenger:lab5}
build:
context: .
args:
VCS_REF: ${VCS_REF:-unknown}
scale: ${API_REPLICAS:-2}
environment:
APP_HOST: 0.0.0.0
APP_PORT: 8080
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
DB_POOL_SIZE: ${DB_POOL_SIZE:-10}
secrets: [database_url, cache_url, s3_access_key, s3_secret_key]
depends_on:
migrate: {condition: service_completed_successfully}
cache: {condition: service_healthy}
s3-bootstrap: {condition: service_completed_successfully}
cpus: ${API_CPUS:-0.5}
mem_limit: ${API_MEMORY:-384m}
memswap_limit: ${API_MEMORY:-384m}
pids_limit: 128
read_only: true
tmpfs: ["/tmp:rw,noexec,nosuid,size=128m"]
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
stop_grace_period: 10s
s3:
image: ${S3_IMAGE:-mtusi-storage:minio-20251015}
build: {context: ., dockerfile: infra/s3.Dockerfile, target: server}
secrets: [s3_root_user, s3_root_password]
volumes: [s3_data:/data]
cpus: 1
mem_limit: 1024m
memswap_limit: 1024m
read_only: true
tmpfs: ["/tmp:rw,nosuid,size=32m"]
environment:
HOME: /tmp
security_opt: [no-new-privileges:true]
s3-bootstrap:
image: ${MC_IMAGE:-mtusi-storage:mc-20250813}
build: {context: ., dockerfile: infra/s3.Dockerfile, target: bootstrap}
secrets: [s3_root_user, s3_root_password, s3_access_key, s3_secret_key]
depends_on:
s3: {condition: service_healthy}
cpus: 0.25
mem_limit: 128m
read_only: true
tmpfs: ["/tmp:rw,nosuid,size=16m"]
reconciler:
image: ${API_IMAGE:-mtusi-messenger:lab5}
pull_policy: never
command: [python, -m, src.reconcile, --apply, --loop]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
secrets: [database_url, s3_access_key, s3_secret_key]
depends_on:
migrate: {condition: service_completed_successfully}
s3-bootstrap: {condition: service_completed_successfully}
cpus: 0.25
mem_limit: 256m
memswap_limit: 256m
read_only: true
healthcheck: {disable: true}
broker:
image: valkey/valkey:8.1.3@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571
command: [valkey-server, /run/secrets/broker.conf]
secrets:
- broker_password
- source: broker_config
target: broker.conf
volumes: [broker_data:/data]
healthcheck:
test: [CMD-SHELL, 'REDISCLI_AUTH="$$(cat /run/secrets/broker_password)" valkey-cli ping']
interval: 2s
timeout: 2s
retries: 20
cpus: 0.5
mem_limit: 256m
memswap_limit: 256m
relay:
image: ${API_IMAGE:-mtusi-messenger:lab5}
pull_policy: never
command: [python, -m, src.pipeline]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
BROKER_URL_FILE: /run/secrets/broker_url
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
secrets: [database_url, broker_url]
depends_on:
migrate: {condition: service_completed_successfully}
broker: {condition: service_healthy}
cpus: 0.25
mem_limit: 192m
memswap_limit: 192m
read_only: true
healthcheck: {disable: true}
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
worker:
image: ${API_IMAGE:-mtusi-messenger:lab5}
pull_policy: never
command: [python, -m, src.worker]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
BROKER_URL_FILE: /run/secrets/broker_url
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-20}
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-23}
secrets: [database_url, broker_url, s3_access_key, s3_secret_key]
depends_on:
migrate: {condition: service_completed_successfully}
broker: {condition: service_healthy}
s3-bootstrap: {condition: service_completed_successfully}
cpus: 2
mem_limit: 1536m
memswap_limit: 1536m
pids_limit: 128
read_only: true
tmpfs: ["/tmp:rw,noexec,nosuid,size=64m"]
healthcheck: {disable: true}
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
stop_grace_period: 25s
lb:
image: haproxy:3.2.1-alpine@sha256:ac79fe145f2bb6626ff26b584a2d0a34e791906c01015f2ae037aa3137b683d9
ports:
- "127.0.0.1:${PUBLIC_PORT:-8085}:8080"
volumes:
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
depends_on:
api: {condition: service_healthy}
cpus: 0.5
mem_limit: 128m
memswap_limit: 128m
read_only: true
tmpfs: ["/tmp:rw,nosuid,size=4m"]
security_opt: [no-new-privileges:true]
stop_grace_period: 10s
secrets:
broker_password: {file: ./secrets/broker_password}
broker_url: {file: ./secrets/broker_url}
broker_config: {file: ./secrets/broker.conf}
s3_root_user: {file: ./secrets/s3_root_user}
s3_root_password: {file: ./secrets/s3_root_password}
s3_access_key: {file: ./secrets/s3_access_key}
s3_secret_key: {file: ./secrets/s3_secret_key}
postgres_password:
file: ./secrets/postgres_password
database_url:
file: ./secrets/database_url
cache_password:
file: ./secrets/cache_password
cache_url:
file: ./secrets/cache_url
valkey_config:
file: ./secrets/valkey.conf
name: valkey.conf
volumes:
broker_data:
s3_data:
postgres_data:
+107
View File
@@ -0,0 +1,107 @@
// Fetch the immutable module ZIP in bounded HTTP ranges. The subsequent
// go mod download validates its h1 checksum through the Go checksum database.
package main
import (
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"sync"
"time"
)
func main() {
const version = "v0.0.0-20251015172955-9e49d5e7a648"
const total int64 = 25342606
const chunk int64 = 512 * 1024
const source = "https://proxy.golang.org/github.com/minio/minio/@v/" + version + ".zip"
directory := "/go/pkg/mod/cache/download/github.com/minio/minio/@v"
if err := os.MkdirAll(directory, 0755); err != nil {
panic(err)
}
target := filepath.Join(directory, version+".zip")
if info, err := os.Stat(target); err == nil && info.Size() == total {
fmt.Println("Source ZIP cached; Go will validate checksum")
return
}
client := &http.Client{Timeout: 90 * time.Second}
jobs := make(chan int64)
errors := make(chan error, 64)
var group sync.WaitGroup
for worker := 0; worker < 8; worker++ {
group.Add(1)
go func() {
defer group.Done()
for start := range jobs {
end := min(start+chunk, total) - 1
part := fmt.Sprintf("%s.part-%d", target, start)
if info, err := os.Stat(part); err == nil && info.Size() == end-start+1 {
continue
}
var last error
for attempt := 0; attempt < 4; attempt++ {
req, _ := http.NewRequest("GET", source, nil)
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end))
response, err := client.Do(req)
if err != nil {
last = fmt.Errorf("source range request failed")
continue
}
expected := fmt.Sprintf("bytes %d-%d/%d", start, end, total)
if response.StatusCode != 206 || response.Header.Get("Content-Range") != expected {
response.Body.Close()
last = fmt.Errorf("unexpected source range response: HTTP %d", response.StatusCode)
continue
}
data, err := io.ReadAll(io.LimitReader(response.Body, end-start+2))
response.Body.Close()
if err != nil || int64(len(data)) != end-start+1 {
last = fmt.Errorf("incomplete source range")
continue
}
if err := os.WriteFile(part, data, 0644); err != nil {
last = err
continue
}
last = nil
fmt.Printf("Source part %d verified\n", start/chunk+1)
break
}
if last != nil {
errors <- last
}
}
}()
}
for start := int64(0); start < total; start += chunk {
jobs <- start
}
close(jobs)
group.Wait()
close(errors)
for err := range errors {
panic(err)
}
out, err := os.Create(target + ".partial")
if err != nil {
panic(err)
}
for start := int64(0); start < total; start += chunk {
part, err := os.Open(fmt.Sprintf("%s.part-%d", target, start))
if err != nil {
panic(err)
}
if _, err := io.Copy(out, part); err != nil {
panic(err)
}
part.Close()
}
if err := out.Close(); err != nil {
panic(err)
}
if err := os.Rename(target+".partial", target); err != nil {
panic(err)
}
}
+36
View File
@@ -0,0 +1,36 @@
global
log stdout format raw local0
maxconn 512
stats socket /tmp/admin.sock mode 600 level admin
defaults
mode http
log global
option httplog
timeout connect 1s
timeout client 15s
timeout server 8s
timeout http-request 10s
timeout http-keep-alive 5s
retries 0
resolvers docker
nameserver dns 127.0.0.11:53
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 1s
hold obsolete 1s
hold nx 1s
frontend public
bind :8080
default_backend messenger
backend messenger
balance roundrobin
option httpchk
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
http-check expect status 200
default-server check inter 1s fastinter 250ms fall 2 rise 1 resolvers docker init-addr libc,none resolve-prefer ipv4
server-template api 1-8 api:8080
+14
View File
@@ -0,0 +1,14 @@
#!/bin/sh
set -eu
root_user=$(cat /run/secrets/s3_root_user)
root_password=$(cat /run/secrets/s3_root_password)
api_key=$(cat /run/secrets/s3_access_key)
api_secret=$(cat /run/secrets/s3_secret_key)
# Credentials are passed only inside this isolated bootstrap container.
mc alias set local http://s3:9000 "$root_user" "$root_password" >/dev/null 2>&1
mc mb --ignore-existing local/messenger >/dev/null 2>&1
mc anonymous set none local/messenger >/dev/null 2>&1
mc admin user add local "$api_key" "$api_secret" >/dev/null 2>&1
mc admin policy create local messenger-api /policy.json >/dev/null 2>&1
mc admin policy attach local messenger-api --user "$api_key" >/dev/null 2>&1
printf '%s\n' 'Private bucket and scoped application policy ready'
+6
View File
@@ -0,0 +1,6 @@
#!/bin/sh
set -eu
export MINIO_ROOT_USER="$(cat /run/secrets/s3_root_user)"
export MINIO_ROOT_PASSWORD="$(cat /run/secrets/s3_root_password)"
export MINIO_BROWSER_REDIRECT_URL=http://127.0.0.1:9001
exec minio server /data --address :9000 --console-address 127.0.0.1:9001
+15
View File
@@ -0,0 +1,15 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["s3:GetBucketLocation", "s3:ListBucket"],
"Resource": ["arn:aws:s3:::messenger"]
},
{
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
"Resource": ["arn:aws:s3:::messenger/objects/*"]
}
]
}
+49
View File
@@ -0,0 +1,49 @@
FROM golang:1.24.8-bookworm@sha256:4ed690d6649d63c312b99a6120025ec79ce3b542968a37da53d6236c7c61a848 AS toolchain
ENV CGO_ENABLED=0 GOMAXPROCS=2 GOFLAGS=-p=2 GOTOOLCHAIN=local
FROM toolchain AS minio-build
WORKDIR /source
COPY infra/fetch-source.go /tmp/fetch-source.go
RUN --mount=type=cache,target=/go/pkg/mod,sharing=locked \
--mount=type=cache,target=/root/.cache/go-build,sharing=locked \
go run /tmp/fetch-source.go && \
for attempt in 1 2 3 4; do \
go mod download -json github.com/minio/minio@v0.0.0-20251015172955-9e49d5e7a648 && break; \
test "$attempt" -lt 4 || exit 1; sleep 1; \
done && \
cp -a /go/pkg/mod/github.com/minio/minio@v0.0.0-20251015172955-9e49d5e7a648/. /source/
RUN --mount=type=cache,target=/go/pkg/mod,sharing=locked \
--mount=type=cache,target=/root/.cache/go-build,sharing=locked \
for attempt in 1 2 3; do \
timeout 900s go build -trimpath -ldflags='-s -w -X github.com/minio/minio/cmd.Version=2025-10-15T17:29:55Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-10-15T17-29-55Z -X github.com/minio/minio/cmd.CommitID=9e49d5e7a648f00e26f2246f4dc28e6b07f8c84a' -o /out/minio . && break; \
test "$attempt" -lt 3 || exit 1; sleep 1; \
done
FROM toolchain AS mc-build
WORKDIR /source
RUN git init . && git remote add origin https://github.com/minio/mc.git && \
git fetch --depth 1 origin 7394ce0dd2a80935aded936b09fa12cbb3cb8096 && git checkout --detach FETCH_HEAD
RUN --mount=type=cache,target=/go/pkg/mod,sharing=locked \
--mount=type=cache,target=/root/.cache/go-build,sharing=locked \
go build -trimpath -ldflags='-s -w' -o /out/mc .
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS server
LABEL org.opencontainers.image.revision=9e49d5e7a648f00e26f2246f4dc28e6b07f8c84a org.opencontainers.image.version=RELEASE.2025-10-15T17-29-55Z
COPY --from=minio-build /out/minio /usr/local/bin/minio
COPY --from=minio-build /source/LICENSE /licenses/minio.LICENSE
COPY --chmod=755 infra/s3-entrypoint.sh /usr/local/bin/s3-entrypoint
RUN mkdir /data && chown 10001:10001 /data
USER 10001:10001
ENTRYPOINT ["/usr/local/bin/s3-entrypoint"]
EXPOSE 9000
HEALTHCHECK --interval=2s --timeout=2s --start-period=10s --retries=30 CMD ["python","-c","import urllib.request; urllib.request.urlopen('http://127.0.0.1:9000/minio/health/ready',timeout=1).read()"]
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS bootstrap
LABEL org.opencontainers.image.revision=7394ce0dd2a80935aded936b09fa12cbb3cb8096 org.opencontainers.image.version=RELEASE.2025-08-13T08-35-41Z
COPY --from=mc-build /out/mc /usr/local/bin/mc
COPY --from=mc-build /source/LICENSE /licenses/mc.LICENSE
COPY --chmod=755 infra/s3-bootstrap.sh /usr/local/bin/s3-bootstrap
COPY infra/s3-policy.json /policy.json
USER 10001:10001
ENV MC_CONFIG_DIR=/tmp/mc
ENTRYPOINT ["/usr/local/bin/s3-bootstrap"]
+36
View File
@@ -0,0 +1,36 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
password_hash text NOT NULL,
created_at timestamptz NOT NULL
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
created_at timestamptz NOT NULL
);
CREATE TABLE members (
chat_id uuid NOT NULL REFERENCES chats(id),
user_id uuid NOT NULL REFERENCES users(id),
position integer NOT NULL CHECK (position BETWEEN 0 AND 99),
PRIMARY KEY (chat_id, user_id),
UNIQUE (chat_id, position)
);
CREATE INDEX members_user_chat_idx ON members(user_id,chat_id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
sender_id uuid NOT NULL,
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
created_at timestamptz NOT NULL,
FOREIGN KEY (chat_id, sender_id) REFERENCES members(chat_id,user_id)
);
CREATE INDEX messages_history_idx ON messages(chat_id,created_at,id);
CREATE TABLE sessions (
token_digest bytea PRIMARY KEY CHECK (octet_length(token_digest)=32),
user_id uuid NOT NULL REFERENCES users(id),
created_at timestamptz NOT NULL,
expires_at timestamptz NOT NULL CHECK (expires_at > created_at)
);
CREATE INDEX sessions_expiry_idx ON sessions(expires_at);
+10
View File
@@ -0,0 +1,10 @@
CREATE TABLE idempotency (
user_id uuid NOT NULL REFERENCES users(id),
chat_id uuid NOT NULL REFERENCES chats(id),
key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
body_hash bytea NOT NULL CHECK (octet_length(body_hash)=32),
message_id uuid NOT NULL REFERENCES messages(id),
expires_at timestamptz NOT NULL,
PRIMARY KEY(user_id, chat_id, key)
);
CREATE INDEX idempotency_expiry_idx ON idempotency(expires_at);
+32
View File
@@ -0,0 +1,32 @@
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
ALTER TABLE messages ADD CHECK (char_length(text) <= 4000);
ALTER TABLE messages ADD UNIQUE(chat_id,id);
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
owner_id uuid NOT NULL,
filename text NOT NULL CHECK(char_length(filename) BETWEEN 1 AND 255),
content_type text NOT NULL CHECK(content_type IN ('image/png','image/jpeg')),
size_bytes bigint NOT NULL CHECK(size_bytes BETWEEN 1 AND 10485760),
sha256 text NOT NULL CHECK(sha256 ~ '^[a-f0-9]{64}$'),
object_key text NOT NULL UNIQUE,
state text NOT NULL CHECK(state IN ('pending','ready','failed')),
error_code text,
uploader_instance text NOT NULL,
expires_at timestamptz NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
UNIQUE(chat_id,id),
FOREIGN KEY(chat_id,owner_id) REFERENCES members(chat_id,user_id)
);
CREATE INDEX attachments_state_created_idx ON attachments(state,created_at,id);
CREATE TABLE message_attachments (
message_id uuid NOT NULL,
chat_id uuid NOT NULL,
attachment_id uuid NOT NULL,
position integer NOT NULL CHECK(position BETWEEN 0 AND 9),
PRIMARY KEY(message_id,attachment_id),
UNIQUE(message_id,position),
FOREIGN KEY(chat_id,message_id) REFERENCES messages(chat_id,id),
FOREIGN KEY(chat_id,attachment_id) REFERENCES attachments(chat_id,id)
);
+54
View File
@@ -0,0 +1,54 @@
ALTER TABLE attachments DROP CONSTRAINT attachments_state_check;
ALTER TABLE attachments ADD CHECK(state IN ('queued','processing','ready','failed'));
ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK(pipeline_version>=1);
ALTER TABLE attachments ADD COLUMN published_version integer;
CREATE TABLE upload_reservations (
id uuid PRIMARY KEY, object_key text UNIQUE NOT NULL,
expires_at timestamptz NOT NULL DEFAULT clock_timestamp()+interval '120 seconds'
);
CREATE TABLE jobs (
id uuid PRIMARY KEY, attachment_id uuid NOT NULL REFERENCES attachments(id),
pipeline_version integer NOT NULL CHECK(pipeline_version>=1), source_key text NOT NULL,
state text NOT NULL CHECK(state IN ('queued','processing','ready','failed','obsolete')),
attempt integer NOT NULL DEFAULT 0, cycle integer NOT NULL DEFAULT 1,
fence bigint NOT NULL DEFAULT 0, lease_owner text, lease_until timestamptz,
next_at timestamptz NOT NULL DEFAULT clock_timestamp(), error_code text,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(), updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
UNIQUE(attachment_id,pipeline_version)
);
CREATE INDEX jobs_due_idx ON jobs(state,next_at);
CREATE TABLE outbox (
id bigserial PRIMARY KEY, job_id uuid NOT NULL REFERENCES jobs(id),
payload jsonb NOT NULL, due_at timestamptz NOT NULL DEFAULT clock_timestamp(),
published_at timestamptz
);
CREATE INDEX outbox_due_idx ON outbox(due_at,id) WHERE published_at IS NULL;
CREATE TABLE attachment_variants (
attachment_id uuid NOT NULL REFERENCES attachments(id), pipeline_version integer NOT NULL,
name text NOT NULL CHECK(name IN ('original','thumbnail','crop','optimized')),
object_key text UNIQUE NOT NULL, content_type text NOT NULL,
size_bytes bigint NOT NULL, sha256 text NOT NULL,
width integer, height integer,
PRIMARY KEY(attachment_id,pipeline_version,name)
);
CREATE TABLE effect_publications (
attachment_id uuid NOT NULL, pipeline_version integer NOT NULL,
job_id uuid NOT NULL REFERENCES jobs(id), fence bigint NOT NULL,
committed_at timestamptz NOT NULL DEFAULT clock_timestamp(),
PRIMARY KEY(attachment_id,pipeline_version)
);
CREATE TABLE pipeline_audit (
id bigserial PRIMARY KEY, job_id uuid NOT NULL, action text NOT NULL,
operator text NOT NULL, reason text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
-- Fault injection is disabled unless PIPELINE_FAULTS=1; no public endpoint.
CREATE TABLE pipeline_faults (
id bigserial PRIMARY KEY, target uuid NOT NULL, phase text NOT NULL,
action text NOT NULL CHECK(action IN ('wait','crash','transient','permanent','hang')),
remaining integer NOT NULL DEFAULT 1, hit_at timestamptz, released_at timestamptz
);
CREATE TABLE pipeline_events (
id bigserial PRIMARY KEY, job_id uuid, event text NOT NULL,
detail jsonb NOT NULL DEFAULT '{}', created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
+44
View File
@@ -0,0 +1,44 @@
[project]
name = "mtusi-messenger-lab5"
version = "1.0.0"
requires-python = ">=3.14,<3.15"
dependencies = [
"fastapi==0.143.3",
"pydantic==2.14.1",
"uvicorn==0.54.0",
"psycopg[binary,pool]==3.3.6",
"argon2-cffi>=25,<26",
"redis==8.1.0",
"minio==7.2.20",
"pillow==13.0.0",
]
[dependency-groups]
dev = ["pytest>=8,<10", "httpx>=0.28,<1", "ruff>=0.14,<1", "openapi-spec-validator==0.7.2", "psutil>=7,<8", "matplotlib>=3.10,<4"]
[tool.ruff]
line-length = 79
target-version = "py314"
include = ["src/**/*.py", "student-tests/**/*.py", "research/**/*.py", "scripts/verify_*.py"]
[tool.ruff.lint]
select = ["E", "F", "I", "B", "UP"]
[tool.ruff.per-file-target-version]
"scripts/*.py" = "py39"
# SQL и генерируемые Markdown-таблицы в инструментах не разрываем.
[tool.ruff.lint.per-file-ignores]
"research/*.py" = ["E501"]
"scripts/*.py" = ["E501"]
[tool.pytest.ini_options]
testpaths = ["student-tests"]
addopts = "-ra"
# Keep SQL statements together; the formatter controls executable statements.
"src/attachments.py" = ["E501"]
"src/pipeline.py" = ["E501"]
"src/worker.py" = ["E501"]
"src/operator.py" = ["E501"]
"src/reconcile.py" = ["E501"]
+63
View File
@@ -0,0 +1,63 @@
"""Создаёт только отсутствующие локальные файлы; секреты не печатает."""
import os
import secrets
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def create(path, value):
try:
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
except FileExistsError:
return
with os.fdopen(fd, "w") as stream:
stream.write(value + "\n")
def main():
directory = ROOT / "secrets"
directory.mkdir(mode=0o700, exist_ok=True)
pg = directory / "postgres_password"
cache = directory / "cache_password"
create(pg, secrets.token_urlsafe(32))
create(cache, secrets.token_urlsafe(32))
create(
directory / "database_url",
"postgresql://messenger:"
+ pg.read_text().strip()
+ "@postgres:5432/messenger",
)
create(
directory / "cache_url",
"redis://:" + cache.read_text().strip() + "@cache:6379/0",
)
create(
directory / "valkey.conf",
"bind 0.0.0.0\nprotected-mode yes\nport 6379\nmaxmemory 64mb\n"
'maxmemory-policy allkeys-lru\nsave ""\nappendonly no\nrequirepass '
+ cache.read_text().strip(),
)
create(directory / "s3_root_user", secrets.token_hex(10))
create(directory / "s3_root_password", secrets.token_urlsafe(32))
create(directory / "s3_access_key", secrets.token_hex(10))
create(directory / "s3_secret_key", secrets.token_urlsafe(32))
create(directory / "broker_password", secrets.token_urlsafe(32))
create(
directory / "broker_url",
"redis://:"
+ (directory / "broker_password").read_text().strip()
+ "@broker:6379/0",
)
create(
directory / "broker.conf",
"bind 0.0.0.0\nprotected-mode yes\nport 6379\nappendonly yes\nappendfsync always\nmaxmemory 192mb\nmaxmemory-policy noeviction\ndir /data\nrequirepass "
+ (directory / "broker_password").read_text().strip(),
)
create(ROOT / ".env", (ROOT / ".env.example").read_text().rstrip())
print("Local configuration ready; existing files preserved")
if __name__ == "__main__":
main()
+83
View File
@@ -0,0 +1,83 @@
"""Явные операторские действия только над API текущего Compose-проекта."""
import json
import os
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def command(*args):
return subprocess.check_output(args, cwd=ROOT, text=True).strip()
def containers():
ids = command("docker", "compose", "ps", "-q", "api").split()
return json.loads(command("docker", "inspect", *ids)) if ids else []
def alias(cid):
code = (
"import urllib.request; r=urllib.request.urlopen("
"'http://127.0.0.1:8080/health/live',timeout=2); "
"print(r.headers['X-Instance-Id'])"
)
return command("docker", "exec", cid, "python", "-c", code)
def wait_count(count):
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
data = containers()
if len(data) == count and all(
c["State"].get("Health", {}).get("Status") == "healthy"
for c in data
):
# DNS и проверки HAProxy должны успеть увидеть новый адрес.
time.sleep(3)
return
time.sleep(0.5)
raise TimeoutError("Replica readiness")
def main():
action = sys.argv[1]
data = containers()
if action == "stop":
target = os.environ["TARGET_INSTANCE_ID"]
matches = [c for c in data if alias(c["Id"]) == target]
if len(matches) != 1 or len(data) < 2:
raise RuntimeError("Target must identify one running replica")
command("docker", "stop", "--time", "10", matches[0]["Id"])
print("Selected API stopped")
elif action == "rolling":
count = len(data)
if count < 2:
raise RuntimeError("Need at least two replicas")
# Сначала добавляется готовая реплика, затем удаляется одна старая.
for old in data:
command(
"docker",
"compose",
"up",
"-d",
"--no-deps",
"--no-recreate",
"--scale",
f"api={count + 1}",
"api",
)
wait_count(count + 1)
command("docker", "stop", "--time", "10", old["Id"])
command("docker", "rm", old["Id"])
wait_count(count)
print("Sequential replacement completed")
else:
raise ValueError("Unknown action")
if __name__ == "__main__":
main()
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
docker compose restart api postgres cache s3 broker relay worker lb
docker compose up -d --no-build --no-recreate --wait
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
set -eu
exec python3 scripts/cluster.py rolling
+137
View File
@@ -0,0 +1,137 @@
import json
import os
import subprocess
import time
from pathlib import Path
from uuid import uuid4
import httpx
ROOT = Path(__file__).resolve().parents[1]
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8085")
def command(args, *, input=None):
result = subprocess.run(
args, cwd=ROOT, input=input, capture_output=True, text=True
)
if result.returncode:
raise RuntimeError(
"Command failed: "
+ args[0]
+ "; "
+ result.stderr.replace(str(ROOT), "<repository>")
)
return result.stdout.strip()
def compose(*args):
return command(["docker", "compose", *args])
def sql(query, *, database="messenger"):
return compose(
"exec",
"-T",
"postgres",
"psql",
"-U",
"messenger",
"-d",
database,
"-A",
"-t",
"-v",
"ON_ERROR_STOP=1",
"-c",
query,
)
def cache(command_name, *args):
script = (
'export REDISCLI_AUTH="$(cat /run/secrets/cache_password)"; '
'exec valkey-cli "$@"'
)
return compose(
"exec", "-T", "cache", "sh", "-c", script, "sh", command_name, *args
)
def ready(base=BASE, *, timeout=30):
end = time.monotonic() + timeout
with httpx.Client(base_url=base, trust_env=False, timeout=6) as client:
while time.monotonic() < end:
try:
if client.get("/health/ready").status_code == 200:
return
except httpx.HTTPError:
pass
time.sleep(0.2)
raise TimeoutError("API readiness")
def actor(client):
username = "verify_" + uuid4().hex[:20]
password = "Synthetic-" + uuid4().hex + ":🌍"
result = client.post(
"/api/v1/users",
json={
"username": username,
"display_name": "Synthetic",
"password": password,
},
)
assert result.status_code == 201
user = result.json()
session = client.post(
"/api/v1/auth/sessions", auth=httpx.BasicAuth(username, password)
)
assert session.status_code == 201
return user, password, session.json()
def auth(session):
return {"Authorization": "Bearer " + session["token"]}
def extra_api(*, ttl=3600, pool_size=1):
name = "mtusi-lab5-test-" + uuid4().hex[:10]
compose(
"run",
"-d",
"--no-deps",
"--name",
name,
"-e",
"SESSION_TTL_SECONDS=" + str(ttl),
"-e",
"DB_POOL_SIZE=" + str(pool_size),
"-p",
"127.0.0.1::8080",
"api",
)
try:
info = json.loads(command(["docker", "inspect", name]))[0]
base = (
"http://127.0.0.1:"
+ info["NetworkSettings"]["Ports"]["8080/tcp"][0]["HostPort"]
)
ready(base)
return name, base
except Exception:
command(["docker", "rm", "-f", name])
raise
def instances():
data = {}
for role in ["api", "postgres", "cache"]:
cid = compose("ps", "-q", role)
info = json.loads(command(["docker", "inspect", cid]))[0]
data[role] = {
"container": cid[:12],
"pid": info["State"]["Pid"],
"started_at": info["State"]["StartedAt"],
}
return data
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
set -eu
exec python3 scripts/cluster.py stop
View File
+44
View File
@@ -0,0 +1,44 @@
import json
import logging
import os
import signal
import sys
def early_stop(signum, _frame):
print(
json.dumps({"event": "stopped_before_startup", "signal": signum}),
flush=True,
)
sys.exit(0)
signal.signal(signal.SIGTERM, early_stop)
signal.signal(signal.SIGINT, early_stop)
import uvicorn # noqa: E402
from src.logging_config import JsonFormatter # noqa: E402
def main() -> None:
host = os.environ.get("APP_HOST", "127.0.0.1")
port = int(os.environ.get("APP_PORT", "8080"))
if not 1 <= port <= 65535:
raise ValueError("APP_PORT must be between 1 and 65535")
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(JsonFormatter())
logging.basicConfig(level=logging.INFO, handlers=[handler], force=True)
uvicorn.run(
"src.api:app",
host=host,
port=port,
workers=1,
access_log=False,
timeout_graceful_shutdown=5,
log_config=None,
)
if __name__ == "__main__":
main()
+424
View File
@@ -0,0 +1,424 @@
import hashlib
import json
import logging
import tempfile
import time
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from datetime import datetime
from functools import partial
from urllib.parse import quote
from uuid import UUID, uuid4
from anyio import Semaphore, fail_after, to_thread
from fastapi import FastAPI, Request
from fastapi.encoders import jsonable_encoder
from pydantic import ValidationError
from starlette.exceptions import HTTPException
from starlette.requests import ClientDisconnect
from starlette.responses import JSONResponse, Response, StreamingResponse
from src import images
from src.attachments import Attachments
from src.cache import UserCache
from src.cursor import parse_uuid
from src.database import Database
from src.domain import timestamp
from src.errors import DomainError, invalid
from src.object_store import ObjectStore
from src.password import Passwords
from src.repository import PostgresRepository
from src.schemas import CreateChat, CreateMessage, CreateUser, Input
from src.service import MessengerService
from src.settings import Settings
logger = logging.getLogger("messenger")
INSTANCE_ID = uuid4().hex
def response(data: object, *, status: int = 200) -> JSONResponse:
return JSONResponse(
jsonable_encoder(
data, custom_encoder={datetime: timestamp, UUID: str}
),
status_code=status,
)
async def body(request: Request, model: type[Input]) -> Input:
try:
data = await request.body()
return model.model_validate(
json.loads(
data.decode("utf-8"),
parse_constant=lambda _: (_ for _ in ()).throw(ValueError()),
)
)
except ValueError, ValidationError, UnicodeError, RecursionError:
raise invalid() from None
def pagination(request: Request) -> tuple[int, str | None]:
limits = request.query_params.getlist("limit")
cursors = request.query_params.getlist("cursor")
if len(limits) > 1 or len(cursors) > 1:
raise invalid()
raw = limits[0] if limits else "50"
if not raw.isascii() or not raw.isdecimal():
raise invalid()
raw = raw.lstrip("0") or "0"
if len(raw) > 3:
raise invalid()
limit = int(raw)
if not 1 <= limit <= 100:
raise invalid()
return limit, cursors[0] if cursors else None
def service(request: Request) -> MessengerService:
return request.app.state.service
async def run_io(function, **kwargs):
return await to_thread.run_sync(partial(function, **kwargs))
async def actor(request: Request):
return await run_io(
service(request).identify, value=request.headers.get("authorization")
)
@asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
settings = Settings.load()
database = Database(url=settings.database_url, size=settings.pool_size)
cache = UserCache(url=settings.cache_url)
store = ObjectStore()
try:
await run_io(database.open)
passwords = await run_io(Passwords)
app.state.database = database
app.state.store = store
app.state.upload_slots = Semaphore(2)
app.state.attachments = Attachments(
database=database, store=store, instance=INSTANCE_ID
)
app.state.service = MessengerService(
repository=PostgresRepository(database=database, cache=cache),
passwords=passwords,
session_ttl=settings.session_ttl,
)
app.state.ready = True
yield
finally:
app.state.ready = False
await run_io(store.close)
await run_io(cache.close)
await run_io(database.close)
app = FastAPI(
lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None
)
@app.middleware("http")
async def request_context(request: Request, call_next):
request_id = str(uuid4())
request.state.request_id = request_id
started = time.perf_counter()
try:
result = await call_next(request)
except Exception:
logger.error(
json.dumps({"event": "internal_error", "request_id": request_id})
)
result = response(
{
"error": {
"code": "internal_error",
"message": "Internal error",
"request_id": request_id,
}
},
status=500,
)
result.headers["X-Request-Id"] = request_id
result.headers["X-Instance-Id"] = INSTANCE_ID
route = request.scope.get("route")
logger.info(
json.dumps(
{
"event": "request",
"request_id": request_id,
"method": request.method,
"route": getattr(route, "path", "unmatched"),
"status": result.status_code,
"elapsed_ms": round((time.perf_counter() - started) * 1000, 3),
}
)
)
return result
@app.exception_handler(DomainError)
async def domain_error(request: Request, error: DomainError):
result = response(
{
"error": {
"code": error.code,
"message": error.message,
"request_id": request.state.request_id,
}
},
status=error.status,
)
result.headers.update(error.headers)
return result
@app.exception_handler(HTTPException)
async def http_error(request: Request, error: HTTPException):
code = "not_found" if error.status_code == 404 else "invalid_request"
return response(
{
"error": {
"code": code,
"message": "Resource not found"
if error.status_code == 404
else "Invalid request",
"request_id": request.state.request_id,
}
},
status=error.status_code,
)
@app.get("/health/live")
async def live():
return response({"status": "ok"})
@app.get("/health/ready")
async def ready():
if not getattr(app.state, "ready", False):
raise DomainError(503, "unavailable", "Not ready")
await run_io(app.state.database.ping)
await run_io(app.state.store.ping)
return response({"status": "ok"})
@app.post("/api/v1/users")
async def register(request: Request):
data = await body(request, CreateUser)
return response(
await run_io(service(request).register, **data.model_dump()),
status=201,
)
@app.get("/api/v1/users/me")
async def current(request: Request):
return response(await actor(request))
@app.post("/api/v1/chats")
async def create_chat(request: Request):
user = await actor(request)
data = await body(request, CreateChat)
return response(
await run_io(
service(request).create_chat, actor_id=user.id, **data.model_dump()
),
status=201,
)
@app.get("/api/v1/chats")
async def list_chats(request: Request):
user = await actor(request)
limit, cursor = pagination(request)
return response(
await run_io(
service(request).chats,
actor_id=user.id,
limit=limit,
cursor=cursor,
)
)
@app.get("/api/v1/chats/{chat_id}")
async def get_chat(request: Request, chat_id: str):
user = await actor(request)
return response(
await run_io(
service(request).get_chat,
actor_id=user.id,
chat_id=parse_uuid(chat_id),
)
)
@app.post("/api/v1/chats/{chat_id}/messages")
async def send(request: Request, chat_id: str):
user = await actor(request)
cid = parse_uuid(chat_id)
await run_io(service(request).get_chat, actor_id=user.id, chat_id=cid)
data = await body(request, CreateMessage)
return response(
await run_io(
service(request).send,
actor_id=user.id,
chat_id=cid,
idempotency_key=request.headers.get("idempotency-key"),
**data.model_dump(),
),
status=201,
)
@app.get("/api/v1/chats/{chat_id}/messages")
async def history(request: Request, chat_id: str):
user = await actor(request)
cid = parse_uuid(chat_id)
chat = await run_io(
service(request).get_chat, actor_id=user.id, chat_id=cid
)
limit, cursor = pagination(request)
return response(
await run_io(
service(request).history,
actor_id=user.id,
chat=chat,
limit=limit,
cursor=cursor,
)
)
@app.post("/api/v1/auth/sessions")
async def login(request: Request):
return response(
await run_io(
service(request).login,
authorization=request.headers.get("authorization"),
),
status=201,
)
@app.delete("/api/v1/auth/sessions/current")
async def logout(request: Request):
await run_io(
service(request).logout,
authorization=request.headers.get("authorization"),
)
return Response(status_code=204)
@app.post("/api/v1/chats/{chat_id}/attachments")
async def upload(request: Request, chat_id: str):
user = await actor(request)
cid = parse_uuid(chat_id)
await run_io(service(request).get_chat, actor_id=user.id, chat_id=cid)
names = request.query_params.getlist("filename")
if len(names) != 1:
raise invalid()
name = images.filename(names[0])
declared = (
request.headers.get("content-type", "")
.partition(";")[0]
.strip()
.lower()
)
if declared not in {"image/png", "image/jpeg"}:
raise DomainError(
415, "unsupported_media_type", "PNG or JPEG required"
)
length = request.headers.get("content-length")
if length and (not length.isdecimal() or len(length) > 12):
raise invalid()
if length and int(length) > images.MAX_BYTES:
raise DomainError(413, "payload_too_large", "Byte limit exceeded")
try:
with fail_after(20):
async with app.state.upload_slots:
with tempfile.SpooledTemporaryFile(
max_size=1024 * 1024, mode="w+b"
) as stream:
size = 0
digest = hashlib.sha256()
async for chunk in request.stream():
size += len(chunk)
if size > images.MAX_BYTES:
raise DomainError(
413, "payload_too_large", "Byte limit exceeded"
)
digest.update(chunk)
stream.write(chunk)
await run_io(
images.verify, stream=stream, declared=declared
)
result = await run_io(
app.state.attachments.upload,
chat_id=cid,
owner_id=user.id,
filename=name,
content_type=declared,
size=size,
sha256=digest.hexdigest(),
stream=stream,
)
return response(result, status=202)
except ClientDisconnect:
raise invalid() from None
except TimeoutError:
raise DomainError(
503, "unavailable", "Upload deadline exceeded"
) from None
@app.get("/api/v1/attachments/{attachment_id}")
async def attachment_metadata(request: Request, attachment_id: str):
user = await actor(request)
return response(
await run_io(
app.state.attachments.metadata,
identity=parse_uuid(attachment_id),
actor_id=user.id,
)
)
@app.get("/api/v1/attachments/{attachment_id}/content")
async def attachment_content(request: Request, attachment_id: str):
user = await actor(request)
variants = request.query_params.getlist("variant")
if len(variants) > 1:
raise invalid()
row, upstream = await run_io(
app.state.attachments.content,
identity=parse_uuid(attachment_id),
actor_id=user.id,
variant=variants[0] if variants else "original",
)
def chunks():
try:
while data := upstream.read(65536):
yield data
finally:
upstream.close()
upstream.release_conn()
return StreamingResponse(
chunks(),
media_type=row["content_type"],
headers={
"Content-Length": str(row["size_bytes"]),
"Content-Disposition": "inline; filename*=UTF-8''"
+ quote(row["filename"], safe=""),
},
)
+144
View File
@@ -0,0 +1,144 @@
from uuid import uuid4
from src.errors import DomainError, not_found
from src.pipeline import enqueue, fault
def dto(row, variants=()):
return {
k: row[k]
for k in [
"id",
"chat_id",
"owner_id",
"filename",
"content_type",
"size_bytes",
"sha256",
"created_at",
"updated_at",
]
} | {
"status": row["state"],
"variants": [
{
k: v[k]
for k in [
"name",
"content_type",
"size_bytes",
"sha256",
"width",
"height",
]
}
for v in variants
if v["name"] != "original"
],
"error_code": row["error_code"] if row["state"] == "failed" else None,
}
class Attachments:
def __init__(self, *, database, store, instance):
self.database, self.store, self.instance = database, store, instance
def upload(
self,
*,
chat_id,
owner_id,
filename,
content_type,
size,
sha256,
stream,
):
identity, job_id = uuid4(), uuid4()
key = f"objects/staging/{identity}/original"
with self.database.connection() as conn:
conn.execute(
"INSERT INTO upload_reservations(id,object_key) VALUES(%s,%s)",
(identity, key),
)
self.store.put(
key=key,
stream=stream,
size=size,
content_type=content_type,
sha256=sha256,
)
fault(self.database, chat_id, "api_after_staging")
with self.database.connection() as conn, conn.transaction():
reservation = conn.execute(
"SELECT * FROM upload_reservations WHERE id=%s AND expires_at>clock_timestamp() FOR UPDATE",
(identity,),
).fetchone()
if reservation is None:
raise DomainError(
503, "unavailable", "Upload reservation expired"
)
row = conn.execute(
"INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key,state,uploader_instance,expires_at) "
"VALUES(%s,%s,%s,%s,%s,%s,%s,%s,'queued',%s,clock_timestamp()) RETURNING *",
(
identity,
chat_id,
owner_id,
filename,
content_type,
size,
sha256,
key,
self.instance,
),
).fetchone()
conn.execute(
"INSERT INTO jobs(id,attachment_id,pipeline_version,source_key,state) VALUES(%s,%s,1,%s,'queued')",
(job_id, identity, key),
)
enqueue(conn, job_id)
conn.execute(
"DELETE FROM upload_reservations WHERE id=%s", (identity,)
)
fault(self.database, chat_id, "api_after_commit")
# Captured committed snapshot remains queued even if worker already finished.
return dto(row)
def row(self, *, identity, actor_id):
with self.database.connection() as conn:
row = conn.execute(
"SELECT a.* FROM attachments a JOIN members m ON m.chat_id=a.chat_id AND m.user_id=%s WHERE a.id=%s",
(actor_id, identity),
).fetchone()
if row is None:
raise not_found()
return row
def metadata(self, *, identity, actor_id):
row = self.row(identity=identity, actor_id=actor_id)
variants = []
if row["state"] == "ready":
with self.database.connection() as conn:
variants = conn.execute(
"SELECT * FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s ORDER BY name",
(identity, row["published_version"]),
).fetchall()
return dto(row, variants)
def content(self, *, identity, actor_id, variant):
row = self.row(identity=identity, actor_id=actor_id)
if row["state"] != "ready":
raise DomainError(409, "conflict", "Attachment is not ready")
with self.database.connection() as conn:
data = conn.execute(
"SELECT * FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s AND name=%s",
(identity, row["published_version"], variant),
).fetchone()
if data is None:
raise not_found()
if self.store.head(key=data["object_key"]) is None:
raise DomainError(503, "unavailable", "Variant unavailable")
return {**data, "filename": row["filename"]}, self.store.get(
key=data["object_key"]
)
+66
View File
@@ -0,0 +1,66 @@
import json
import threading
import time
from datetime import datetime
from uuid import UUID
import redis
from redis.backoff import NoBackoff
from redis.retry import Retry
from src.domain import User, timestamp
class UserCache:
def __init__(self, *, url: str):
self.client = redis.Redis.from_url(
url,
socket_connect_timeout=0.15,
socket_timeout=0.15,
max_connections=16,
retry=Retry(NoBackoff(), 0),
decode_responses=True,
)
self.disabled_until = 0.0
self.lock = threading.Lock()
def close(self):
self.client.close()
def _command(self, function, *args, **kwargs):
if time.monotonic() < self.disabled_until:
return None
try:
return function(*args, **kwargs)
except redis.RedisError, OSError:
with self.lock:
self.disabled_until = time.monotonic() + 5
return None
def get(self, *, user_id: UUID) -> User | None:
raw = self._command(self.client.get, "user:" + str(user_id))
if raw is None:
return None
try:
row = json.loads(raw)
if UUID(row["id"]) != user_id:
return None
return User(
user_id,
row["username"],
row["display_name"],
datetime.fromisoformat(row["created_at"]),
)
except ValueError, TypeError, KeyError:
return None
def put(self, *, user: User):
raw = json.dumps(
{
"id": str(user.id),
"username": user.username,
"display_name": user.display_name,
"created_at": timestamp(user.created_at),
}
)
self._command(self.client.set, "user:" + str(user.id), raw, ex=30)
+64
View File
@@ -0,0 +1,64 @@
import base64
import binascii
import json
import re
from datetime import datetime
from uuid import UUID
from src.domain import Key, timestamp
from src.errors import invalid
def parse_uuid(value: str) -> UUID:
if not isinstance(value, str) or not re.fullmatch(
r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}", value
):
raise invalid()
return UUID(value)
def encode_cursor(*, scope: str, key: Key) -> str:
payload = {
"v": 1,
"scope": scope,
"at": timestamp(key[0]),
"id": str(key[1]),
}
return (
base64.urlsafe_b64encode(
json.dumps(payload, separators=(",", ":")).encode()
)
.decode()
.rstrip("=")
)
def decode_cursor(*, value: str | None, scope: str) -> Key | None:
if value is None:
return None
try:
if (
not value
or len(value) > 2048
or not re.fullmatch(r"[A-Za-z0-9_-]+", value)
):
raise ValueError
data = json.loads(
base64.b64decode(
value + "=" * (-len(value) % 4), altchars=b"-_", validate=True
)
)
if (
not isinstance(data, dict)
or type(data.get("v")) is not int
or data["v"] != 1
or data.get("scope") != scope
or not isinstance(data.get("at"), str)
):
raise ValueError
at = datetime.fromisoformat(data["at"])
if at.tzinfo is None or timestamp(at) != data["at"]:
raise ValueError
return at, parse_uuid(data["id"])
except ValueError, TypeError, KeyError, binascii.Error, RecursionError:
raise invalid() from None
+69
View File
@@ -0,0 +1,69 @@
import time
from contextlib import contextmanager
import psycopg
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool, PoolTimeout
from src.errors import DomainError
class DeadlineConnection(psycopg.Connection):
"""Клиентский срок действует и при замороженном PostgreSQL."""
def wait(self, gen, interval=0.1, timeout=None):
deadline = getattr(self, "request_deadline", None)
remaining = (
max(0.001, deadline - time.monotonic()) if deadline else 3.0
)
bound = min(remaining, timeout) if timeout is not None else remaining
try:
return super().wait(gen, interval=interval, timeout=bound)
except psycopg.OperationalError:
self.close()
raise
class Database:
def __init__(self, *, url: str, size: int):
self.pool = ConnectionPool(
url,
connection_class=DeadlineConnection,
kwargs={
"autocommit": True,
"row_factory": dict_row,
"connect_timeout": 2,
"options": (
"-c statement_timeout=2500 -c lock_timeout=1000 "
"-c idle_in_transaction_session_timeout=6000"
),
},
min_size=1,
max_size=size,
timeout=1,
open=False,
)
def open(self):
self.pool.open(wait=True, timeout=10)
def close(self):
self.pool.close(timeout=2)
@contextmanager
def connection(self):
try:
with self.pool.connection(timeout=1) as conn:
conn.request_deadline = time.monotonic() + 4
try:
yield conn
finally:
conn.request_deadline = None
except psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout:
raise DomainError(
503, "unavailable", "Database unavailable"
) from None
def ping(self):
with self.connection() as conn:
conn.execute("SELECT 1").fetchone()
+45
View File
@@ -0,0 +1,45 @@
from dataclasses import dataclass
from datetime import UTC, datetime
from uuid import UUID
def utcnow() -> datetime:
return datetime.now(UTC)
def timestamp(value: datetime) -> str:
return value.isoformat(timespec="microseconds").replace("+00:00", "Z")
@dataclass(frozen=True, slots=True)
class User:
id: UUID
username: str
display_name: str
created_at: datetime
@dataclass(frozen=True, slots=True)
class Chat:
id: UUID
title: str
member_ids: tuple[UUID, ...]
created_at: datetime
@dataclass(frozen=True, slots=True)
class Message:
id: UUID
chat_id: UUID
sender_id: UUID
text: str
created_at: datetime
attachment_ids: tuple[UUID, ...] = ()
type Record = User | Chat | Message
type Key = tuple[datetime, UUID]
def order_key(record: Record) -> Key:
return record.created_at, record.id
+16
View File
@@ -0,0 +1,16 @@
class DomainError(Exception):
def __init__(
self, status: int, code: str, message: str, headers=None
) -> None:
self.status = status
self.code = code
self.message = message
self.headers = headers or {}
def invalid() -> DomainError:
return DomainError(400, "invalid_request", "Invalid request")
def not_found() -> DomainError:
return DomainError(404, "not_found", "Resource not found")
+77
View File
@@ -0,0 +1,77 @@
import struct
import threading
import unicodedata
import zlib
from PIL import Image, UnidentifiedImageError
from src.errors import DomainError, invalid
MAX_BYTES = 10 * 1024 * 1024
MAX_PIXELS = 40_000_000
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
DECODE_SLOT = threading.BoundedSemaphore(1)
def filename(value):
if (
not value
or not 1 <= len(value) <= 255
or not value.strip()
or any(
c in "/\\" or unicodedata.category(c) in {"Cc", "Cs"}
for c in value
)
):
raise invalid()
return value
def verify(stream, *, declared):
if declared not in {"image/png", "image/jpeg"}:
raise DomainError(
415, "unsupported_media_type", "PNG or JPEG required"
)
with DECODE_SLOT:
try:
stream.seek(0)
with Image.open(stream) as image:
actual = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(
image.format
)
if actual != declared:
raise DomainError(
415, "unsupported_media_type", "Image type mismatch"
)
if image.width * image.height > MAX_PIXELS:
raise DomainError(
413, "payload_too_large", "Pixel limit exceeded"
)
image.verify()
# JPEG verify alone does not decode entropy data.
stream.seek(0)
with Image.open(stream) as image:
image.load()
except Image.DecompressionBombError:
raise DomainError(
413, "payload_too_large", "Pixel limit exceeded"
) from None
except (
UnidentifiedImageError,
OSError,
ValueError,
SyntaxError,
struct.error,
zlib.error,
EOFError,
):
raise DomainError(
415, "unsupported_media_type", "Invalid image"
) from None
except MemoryError:
raise DomainError(
503, "unavailable", "Image memory budget exceeded"
) from None
finally:
stream.seek(0)
return declared
+17
View File
@@ -0,0 +1,17 @@
import json
import logging
from datetime import UTC, datetime
class JsonFormatter(logging.Formatter):
def format(self, record: logging.LogRecord) -> str:
message = record.getMessage()
try:
event = json.loads(message)
if not isinstance(event, dict):
event = {"event": message}
except ValueError:
event = {"event": message}
event["timestamp"] = datetime.now(UTC).isoformat()
event["level"] = record.levelname.lower()
return json.dumps(event, ensure_ascii=False)
+51
View File
@@ -0,0 +1,51 @@
import hashlib
import json
from pathlib import Path
import psycopg
from src.settings import secret
def main():
with psycopg.connect(
secret("DATABASE_URL"), autocommit=True, connect_timeout=3
) as conn:
conn.execute("SELECT pg_advisory_lock(2301002)")
try:
conn.execute(
"CREATE TABLE IF NOT EXISTS schema_migrations "
"(name text PRIMARY KEY, checksum text NOT NULL, "
"applied_at timestamptz NOT NULL DEFAULT now())"
)
for path in sorted(Path("migrations").glob("*.sql")):
content = path.read_bytes()
checksum = hashlib.sha256(content).hexdigest()
row = conn.execute(
"SELECT checksum FROM schema_migrations WHERE name=%s",
(path.name,),
).fetchone()
if row:
if row[0] != checksum:
raise RuntimeError(
"Applied migration checksum mismatch"
)
continue
with conn.transaction():
conn.execute(content.decode())
conn.execute(
"INSERT INTO schema_migrations(name,checksum) "
"VALUES(%s,%s)",
(path.name, checksum),
)
print(
json.dumps(
{"event": "migration_applied", "name": path.name}
)
)
finally:
conn.execute("SELECT pg_advisory_unlock(2301002)")
if __name__ == "__main__":
main()
+84
View File
@@ -0,0 +1,84 @@
from contextlib import contextmanager
import urllib3
from minio import Minio
from minio.error import S3Error
from src.errors import DomainError
from src.settings import secret
BUCKET = "messenger"
class ObjectStore:
def __init__(self):
self.http = urllib3.PoolManager(
maxsize=32,
block=True,
timeout=urllib3.Timeout(connect=1, read=3),
retries=urllib3.Retry(total=0),
)
self.client = Minio(
"s3:9000",
access_key=secret("S3_ACCESS_KEY"),
secret_key=secret("S3_SECRET_KEY"),
secure=False,
region="us-east-1",
http_client=self.http,
)
@contextmanager
def operation(self):
try:
yield
except S3Error, urllib3.exceptions.HTTPError, OSError:
raise DomainError(
503, "unavailable", "Object store unavailable"
) from None
def ping(self):
with self.operation():
if not self.client.bucket_exists(BUCKET):
raise DomainError(
503, "unavailable", "Object store unavailable"
)
def put(self, *, key, stream, size, content_type, sha256):
stream.seek(0)
with self.operation():
self.client.put_object(
BUCKET,
key,
stream,
length=size,
content_type=content_type,
metadata={"sha256": sha256},
part_size=10 * 1024 * 1024,
num_parallel_uploads=1,
)
def head(self, *, key):
with self.operation():
try:
return self.client.stat_object(BUCKET, key)
except S3Error as exc:
if exc.code in {"NoSuchKey", "NoSuchObject"}:
return None
raise
def get(self, *, key):
with self.operation():
return self.client.get_object(BUCKET, key)
def delete(self, *, key):
with self.operation():
self.client.remove_object(BUCKET, key)
def objects(self):
with self.operation():
yield from self.client.list_objects(
BUCKET, prefix="objects/", recursive=True
)
def close(self):
self.http.clear()
+123
View File
@@ -0,0 +1,123 @@
"""Explicit local operator actions. There is deliberately no HTTP replay endpoint."""
import argparse
import json
from uuid import UUID, uuid4
from src.object_store import ObjectStore
from src.pipeline import database, enqueue, event
def change(db, store, *, job_id, operator, reason, version=None):
if (
not operator.strip()
or not reason.strip()
or len(operator) > 100
or len(reason) > 500
):
raise ValueError("Operator and reason are required")
with db.connection() as conn, conn.transaction():
job = conn.execute(
"SELECT * FROM jobs WHERE id=%s FOR UPDATE", (job_id,)
).fetchone()
if not job:
raise ValueError("Unknown job")
attachment = conn.execute(
"SELECT * FROM attachments WHERE id=%s FOR UPDATE",
(job["attachment_id"],),
).fetchone()
if job["pipeline_version"] != attachment["pipeline_version"]:
raise ValueError("Superseded job cannot be replayed")
source = (
job["source_key"]
if attachment["published_version"] is None
else attachment["object_key"]
)
if store.head(key=source) is None:
raise ValueError("Source expired or missing; replay refused")
if version is None:
if job["state"] != "failed":
raise ValueError("Only failed jobs can be replayed")
conn.execute(
"UPDATE jobs SET state='queued',attempt=0,cycle=cycle+1,fence=fence+1,error_code=NULL,next_at=clock_timestamp(),lease_owner=NULL,lease_until=NULL,updated_at=clock_timestamp() WHERE id=%s",
(job_id,),
)
action = "replay"
target = job_id
else:
if (
version not in {1, 2}
or version <= attachment["pipeline_version"]
):
raise ValueError("Supported version must increase (1 or 2)")
conn.execute(
"UPDATE jobs SET state='obsolete',fence=fence+1,lease_owner=NULL,lease_until=NULL WHERE id=%s",
(job_id,),
)
target = uuid4()
conn.execute(
"INSERT INTO jobs(id,attachment_id,pipeline_version,source_key,state) VALUES(%s,%s,%s,%s,'queued')",
(target, attachment["id"], version, source),
)
conn.execute(
"UPDATE attachments SET pipeline_version=%s WHERE id=%s",
(version, attachment["id"]),
)
action = "new_version"
conn.execute(
"UPDATE attachments SET state='queued',error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",
(attachment["id"],),
)
conn.execute(
"INSERT INTO pipeline_audit(job_id,action,operator,reason) VALUES(%s,%s,%s,%s)",
(target, action, operator, reason),
)
event(conn, target, action, {"previous_job": str(job_id)})
enqueue(conn, target)
return str(target)
def main():
p = argparse.ArgumentParser()
p.add_argument("action", choices=["failed", "replay", "version"])
p.add_argument("--job-id", type=UUID)
p.add_argument("--operator")
p.add_argument("--reason")
p.add_argument("--version", type=int)
a = p.parse_args()
db, store = database(), ObjectStore()
try:
if a.action == "failed":
with db.connection() as conn:
rows = conn.execute(
"SELECT id,attachment_id,pipeline_version,attempt,cycle,error_code,updated_at FROM jobs WHERE state='failed' ORDER BY updated_at,id"
).fetchall()
print(json.dumps(rows, default=str))
else:
if not all([a.job_id, a.operator, a.reason]):
p.error("job-id/operator/reason required")
if a.action == "version" and a.version is None:
p.error("version required")
print(
json.dumps(
{
"job_id": change(
db,
store,
job_id=a.job_id,
operator=a.operator,
reason=a.reason,
version=a.version
if a.action == "version"
else None,
)
}
)
)
finally:
db.close()
store.close()
if __name__ == "__main__":
main()
+36
View File
@@ -0,0 +1,36 @@
import threading
from argon2 import PasswordHasher
from argon2.exceptions import InvalidHashError, VerificationError
from argon2.low_level import Type
from src.errors import DomainError
class Passwords:
def __init__(self):
self.hasher = PasswordHasher(
time_cost=3, memory_cost=65536, parallelism=4, type=Type.ID
)
self.slots = threading.BoundedSemaphore(2)
self.dummy = self.hasher.hash("Synthetic-dummy-password-not-for-login")
def _run(self, function, *args):
if not self.slots.acquire(timeout=5):
raise DomainError(503, "unavailable", "Authentication busy")
try:
return function(*args)
finally:
self.slots.release()
def hash(self, *, password: str) -> str:
return self._run(self.hasher.hash, password)
def verify(self, *, password: str, encoded: str | None) -> bool:
try:
return (
self._run(self.hasher.verify, encoded or self.dummy, password)
and encoded is not None
)
except VerificationError, InvalidHashError:
return False
+247
View File
@@ -0,0 +1,247 @@
"""Durable job state is authoritative; stream entries are wake-up notifications."""
import json
import os
import random
import time
from datetime import UTC, datetime
from uuid import UUID
import redis
from psycopg.types.json import Jsonb
from src.database import Database
from src.settings import integer, secret
STREAM = "attachments"
GROUP = "image-workers"
def database():
db = Database(url=secret("DATABASE_URL"), size=4)
db.open()
return db
def broker():
return redis.Redis.from_url(
secret("BROKER_URL"),
decode_responses=True,
socket_connect_timeout=1,
socket_timeout=3,
max_connections=8,
)
def envelope(row):
return dict(
schema_version=1,
job_id=str(row["id"]),
type="attachment.process.v1",
attachment_id=str(row["attachment_id"]),
pipeline_version=row["pipeline_version"],
attempt=row["attempt"] + 1,
created_at=datetime.now(UTC).isoformat().replace("+00:00", "Z"),
)
def validate_job(data):
required = {
"schema_version",
"job_id",
"type",
"attachment_id",
"pipeline_version",
"attempt",
"created_at",
}
if (
not isinstance(data, dict)
or not required <= data.keys()
or data.keys() - required - {"traceparent", "tracestate"}
):
raise ValueError("Invalid job envelope")
if (
type(data["schema_version"]) is not int
or data["schema_version"] != 1
or data["type"] != "attachment.process.v1"
):
raise ValueError("Invalid job type")
for k in ["job_id", "attachment_id"]:
UUID(data[k])
for k in ["pipeline_version", "attempt"]:
if type(data[k]) is not int or data[k] < 1:
raise ValueError("Invalid job number")
datetime.fromisoformat(data["created_at"].replace("Z", "+00:00"))
if "traceparent" in data:
import re
if not isinstance(data["traceparent"], str) or not re.fullmatch(
r"00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}", data["traceparent"]
):
raise ValueError("Invalid trace context")
if "tracestate" in data and (
not isinstance(data["tracestate"], str)
or len(data["tracestate"]) > 512
):
raise ValueError("Invalid trace context")
return data
def event(conn, job_id, name, detail=None):
conn.execute(
"INSERT INTO pipeline_events(job_id,event,detail) VALUES(%s,%s,%s)",
(job_id, name, Jsonb(detail or {})),
)
def enqueue(conn, job_id):
row = conn.execute("SELECT * FROM jobs WHERE id=%s", (job_id,)).fetchone()
conn.execute(
"INSERT INTO outbox(job_id,payload,due_at) VALUES(%s,%s,%s)",
(job_id, Jsonb(envelope(row)), row["next_at"]),
)
def fault(db, target, phase):
if os.getenv("PIPELINE_FAULTS") != "1":
return
with db.connection() as conn, conn.transaction():
row = conn.execute(
"SELECT * FROM pipeline_faults WHERE target=%s AND phase=%s AND remaining>0 ORDER BY id FOR UPDATE SKIP LOCKED LIMIT 1",
(target, phase),
).fetchone()
if row is None:
return
conn.execute(
"UPDATE pipeline_faults SET remaining=remaining-1,hit_at=clock_timestamp() WHERE id=%s",
(row["id"],),
)
action = row["action"]
if action == "crash":
os._exit(89)
if action == "permanent":
raise PermanentFailure("invalid_image")
if action == "transient":
raise RuntimeError("injected_transient")
if action == "hang":
time.sleep(120)
if action == "wait":
end = time.monotonic() + 60
while time.monotonic() < end:
with db.connection() as conn:
released = conn.execute(
"SELECT released_at FROM pipeline_faults WHERE id=%s",
(row["id"],),
).fetchone()
if released and released["released_at"]:
return
time.sleep(0.05)
raise RuntimeError("fault_wait_timeout")
class PermanentFailure(Exception):
pass
def retry(conn, row, error, *, permanent=False):
maximum = integer("JOB_MAX_ATTEMPTS", 5, 1, 20)
failed = permanent or row["attempt"] >= maximum
delay = random.uniform(0.25, min(30, 2 ** row["attempt"]))
state = "failed" if failed else "queued"
code = error if permanent else ("retry_exhausted" if failed else error)
conn.execute(
"UPDATE jobs SET state=%s,fence=fence+1,lease_owner=NULL,lease_until=NULL,error_code=%s,next_at=clock_timestamp()+%s*interval '1 second',updated_at=clock_timestamp() WHERE id=%s",
(state, code, delay, row["id"]),
)
conn.execute(
"UPDATE attachments SET state=%s,error_code=%s,updated_at=clock_timestamp() WHERE id=%s AND pipeline_version=%s",
(
state,
code if failed else None,
row["attachment_id"],
row["pipeline_version"],
),
)
event(
conn,
row["id"],
"failed" if failed else "retry",
{"attempt": row["attempt"], "code": code, "backoff_seconds": delay},
)
if not failed:
enqueue(conn, row["id"])
def watchdog(db):
with db.connection() as conn, conn.transaction():
rows = conn.execute(
"SELECT * FROM jobs WHERE state='processing' AND lease_until<clock_timestamp() ORDER BY lease_until FOR UPDATE SKIP LOCKED LIMIT 16"
).fetchall()
for row in rows:
retry(conn, row, "lease_expired")
# Recreate a wakeup if its stream entry was lost: never recreate logical jobs.
rows = conn.execute(
"SELECT j.* FROM jobs j WHERE j.state='queued' AND j.next_at<=clock_timestamp() AND NOT EXISTS(SELECT 1 FROM outbox o WHERE o.job_id=j.id AND (o.published_at IS NULL OR o.published_at>clock_timestamp()-interval '15 seconds')) FOR UPDATE OF j SKIP LOCKED LIMIT 16"
).fetchall()
for row in rows:
enqueue(conn, row["id"])
def relay_once(db, queue):
watchdog(db)
with db.connection() as conn, conn.transaction():
row = conn.execute(
"SELECT * FROM outbox WHERE published_at IS NULL AND due_at<=clock_timestamp() ORDER BY id FOR UPDATE SKIP LOCKED LIMIT 1"
).fetchone()
if not row:
return False
validate_job(row["payload"])
# WAITAOF must use the SAME connection as XADD (not two pooled calls).
with queue.client() as dedicated:
entry = dedicated.xadd(
STREAM,
{"job": json.dumps(row["payload"], separators=(",", ":"))},
)
confirmation = dedicated.execute_command("WAITAOF", 1, 0, 2000)
if int(confirmation[0]) < 1:
raise RuntimeError("Broker durability confirmation timed out")
fault(db, row["job_id"], "relay_after_publish")
conn.execute(
"UPDATE outbox SET published_at=clock_timestamp() WHERE id=%s",
(row["id"],),
)
event(
conn,
row["job_id"],
"published",
{"entry": entry, "outbox": row["id"]},
)
return True
def relay():
db, queue = database(), broker()
try:
while True:
try:
if not relay_once(db, queue):
time.sleep(0.2)
except Exception:
print(
json.dumps(
{
"event": "relay_retry",
"code": "dependency_unavailable",
}
),
flush=True,
)
time.sleep(1)
finally:
queue.close()
db.close()
if __name__ == "__main__":
relay()
+109
View File
@@ -0,0 +1,109 @@
"""Retention cleanup; default is dry-run. Active intent always protects bytes."""
import argparse
import json
import time
from src.object_store import ObjectStore
from src.pipeline import database
from src.settings import integer
def reconcile(db, store, *, apply=False, grace=None):
grace = (
grace
if grace is not None
else integer("STAGING_RETENTION_SECONDS", 86400, 300, 604800)
)
if grace < 300:
raise ValueError("Retention must be at least 300 seconds")
actions = []
for obj in store.objects():
# Recheck each candidate in a bounded transaction, including late PUT age.
with db.connection() as conn, conn.transaction():
now = conn.execute("SELECT clock_timestamp() now").fetchone()[
"now"
]
if (now - obj.last_modified).total_seconds() < grace:
continue
reservation = conn.execute(
"SELECT * FROM upload_reservations WHERE object_key=%s FOR UPDATE",
(obj.object_name,),
).fetchone()
if reservation and reservation["expires_at"] > now:
continue
jobs = conn.execute(
"SELECT * FROM jobs WHERE source_key=%s OR %s LIKE 'objects/' || attachment_id::text || '/v' || pipeline_version::text || '/%%' FOR UPDATE",
(obj.object_name, obj.object_name),
).fetchall()
if any(j["state"] in {"queued", "processing"} for j in jobs):
continue
current = conn.execute(
"SELECT v.* FROM attachment_variants v JOIN attachments a ON a.id=v.attachment_id AND a.published_version=v.pipeline_version WHERE v.object_key=%s",
(obj.object_name,),
).fetchone()
if current:
continue
if any(
j["state"] == "failed"
and (now - j["updated_at"]).total_seconds() < grace
for j in jobs
):
continue
head = store.head(key=obj.object_name)
if (
head is None
or (now - head.last_modified).total_seconds() < grace
):
continue
action = {
"action": "delete_expired_object",
"key": obj.object_name,
"applied": apply,
}
if apply:
# Taking the reservation lock fences an expired API before deletion.
if reservation:
conn.execute(
"DELETE FROM upload_reservations WHERE id=%s",
(reservation["id"],),
)
store.delete(key=obj.object_name)
actions.append(action)
return {"apply": apply, "retention_seconds": grace, "actions": actions}
def main():
p = argparse.ArgumentParser()
p.add_argument("--apply", action="store_true")
p.add_argument("--loop", action="store_true")
a = p.parse_args()
db, store = database(), ObjectStore()
try:
while True:
try:
print(
json.dumps(reconcile(db, store, apply=a.apply)), flush=True
)
except Exception:
if not a.loop:
raise
print(
json.dumps(
{
"event": "cleanup_retry",
"code": "dependency_unavailable",
}
),
flush=True,
)
if not a.loop:
break
time.sleep(60)
finally:
db.close()
store.close()
if __name__ == "__main__":
main()
+296
View File
@@ -0,0 +1,296 @@
import hashlib
import json
from uuid import UUID
from psycopg.errors import ForeignKeyViolation, UniqueViolation
from src.cache import UserCache
from src.database import Database
from src.domain import Chat, Key, Message, User
from src.errors import DomainError, invalid, not_found
MESSAGE_COLUMNS = (
"m.*, ARRAY(SELECT attachment_id FROM message_attachments ma "
"WHERE ma.message_id=m.id ORDER BY ma.position) attachment_ids"
)
USER_COLUMNS = "id, username, display_name, created_at"
def user_dto(row) -> User:
return User(
row["id"], row["username"], row["display_name"], row["created_at"]
)
def chat_dto(row) -> Chat:
return Chat(
row["id"], row["title"], tuple(row["member_ids"]), row["created_at"]
)
def message_dto(row) -> Message:
return Message(
row["id"],
row["chat_id"],
row["sender_id"],
row["text"],
row["created_at"],
tuple(row.get("attachment_ids", ())),
)
class PostgresRepository:
def __init__(self, *, database: Database, cache: UserCache):
self.database = database
self.cache = cache
def add_user(self, *, user: User, password_hash: str) -> User:
try:
with self.database.connection() as conn:
conn.execute(
"INSERT INTO users(id,username,display_name,"
"password_hash,created_at) VALUES(%s,%s,%s,%s,%s)",
(
user.id,
user.username,
user.display_name,
password_hash,
user.created_at,
),
)
except UniqueViolation:
raise DomainError(
409, "conflict", "Username already exists"
) from None
return user
def get_user(self, *, user_id: UUID) -> User | None:
cached = self.cache.get(user_id=user_id)
if cached is not None:
return cached
with self.database.connection() as conn:
row = conn.execute(
"SELECT " + USER_COLUMNS + " FROM users WHERE id=%s",
(user_id,),
).fetchone()
if row is None:
return None
user = user_dto(row)
self.cache.put(user=user)
return user
def credentials(self, *, username: str):
with self.database.connection() as conn:
row = conn.execute(
"SELECT "
+ USER_COLUMNS
+ ", password_hash FROM users WHERE username=%s",
(username,),
).fetchone()
if row is None:
return None, None
return user_dto(row), row["password_hash"]
def username_exists(self, *, username: str) -> bool:
with self.database.connection() as conn:
return (
conn.execute(
"SELECT 1 FROM users WHERE username=%s", (username,)
).fetchone()
is not None
)
def add_session(
self, *, digest: bytes, user_id: UUID, created_at, expires_at
):
with self.database.connection() as conn:
conn.execute(
"INSERT INTO sessions(token_digest,user_id,created_at,"
"expires_at) VALUES(%s,%s,%s,%s)",
(digest, user_id, created_at, expires_at),
)
def session_user(self, *, digest: bytes) -> UUID | None:
with self.database.connection() as conn:
row = conn.execute(
"SELECT user_id FROM sessions WHERE token_digest=%s "
"AND expires_at>clock_timestamp()",
(digest,),
).fetchone()
return row["user_id"] if row else None
def delete_session(self, *, digest: bytes):
with self.database.connection() as conn:
conn.execute(
"DELETE FROM sessions WHERE token_digest=%s", (digest,)
)
def add_chat(self, *, chat: Chat, actor_id: UUID) -> Chat:
members = chat.member_ids
if actor_id not in members or len(set(members)) != len(members):
raise invalid()
try:
with self.database.connection() as conn, conn.transaction():
conn.execute(
"INSERT INTO chats(id,title,created_at) VALUES(%s,%s,%s)",
(chat.id, chat.title, chat.created_at),
)
with conn.cursor() as cursor:
cursor.executemany(
"INSERT INTO members(chat_id,user_id,position) "
"VALUES(%s,%s,%s)",
[
(chat.id, user_id, position)
for position, user_id in enumerate(members)
],
)
except ForeignKeyViolation:
raise invalid() from None
return chat
def get_chat(self, *, chat_id: UUID, actor_id: UUID) -> Chat:
with self.database.connection() as conn:
row = conn.execute(
"SELECT c.id,c.title,c.created_at,"
"array_agg(m.user_id ORDER BY m.position) member_ids "
"FROM chats c JOIN members m ON m.chat_id=c.id "
"WHERE c.id=%s AND EXISTS(SELECT 1 FROM members own "
"WHERE own.chat_id=c.id AND own.user_id=%s) GROUP BY c.id",
(chat_id, actor_id),
).fetchone()
if row is None:
raise not_found()
return chat_dto(row)
def add_message(
self, *, message: Message, idempotency_key: str
) -> Message:
# Область включает пользователя, метод и нормализованный путь чата.
scope = f"{message.sender_id}:POST:{message.chat_id}:{idempotency_key}"
lock = int.from_bytes(
hashlib.sha256(scope.encode()).digest()[:8], "big", signed=True
)
fingerprint = hashlib.sha256(
json.dumps(
{
"text": message.text,
"attachment_ids": [str(x) for x in message.attachment_ids],
},
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
).encode()
).digest()
try:
with self.database.connection() as conn, conn.transaction():
conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock,))
prior = conn.execute(
"SELECT i.body_hash,i.message_id,"
"i.expires_at>clock_timestamp() active "
"FROM idempotency i WHERE user_id=%s AND chat_id=%s "
"AND key=%s FOR UPDATE",
(message.sender_id, message.chat_id, idempotency_key),
).fetchone()
if prior and prior["active"]:
if bytes(prior["body_hash"]) != fingerprint:
raise DomainError(409, "conflict", "Key already used")
row = conn.execute(
"SELECT "
+ MESSAGE_COLUMNS
+ " FROM messages m WHERE m.id=%s",
(prior["message_id"],),
).fetchone()
return message_dto(row)
if message.attachment_ids:
rows = conn.execute(
"SELECT id,state FROM attachments WHERE chat_id=%s "
"AND id=ANY(%s) FOR SHARE",
(message.chat_id, list(message.attachment_ids)),
).fetchall()
if len(rows) != len(message.attachment_ids):
raise not_found()
if any(row["state"] == "failed" for row in rows):
raise DomainError(
409, "conflict", "Attachment is not ready"
)
conn.execute(
"INSERT INTO messages(id,chat_id,sender_id,"
"text,created_at) "
"VALUES(%s,%s,%s,%s,%s)",
(
message.id,
message.chat_id,
message.sender_id,
message.text,
message.created_at,
),
)
if message.attachment_ids:
with conn.cursor() as cursor:
cursor.executemany(
"INSERT INTO message_attachments("
"message_id,chat_id,"
"attachment_id,position) "
"VALUES(%s,%s,%s,%s)",
[
(message.id, message.chat_id, aid, i)
for i, aid in enumerate(message.attachment_ids)
],
)
conn.execute(
"INSERT INTO idempotency(user_id,chat_id,key,body_hash,"
"message_id,expires_at) VALUES(%s,%s,%s,%s,%s,"
"clock_timestamp()+interval '24 hours') "
"ON CONFLICT(user_id,chat_id,key) DO UPDATE SET "
"body_hash=excluded.body_hash,message_id=excluded.message_id,"
"expires_at=excluded.expires_at",
(
message.sender_id,
message.chat_id,
idempotency_key,
fingerprint,
message.id,
),
)
except ForeignKeyViolation:
raise not_found() from None
return message
def list_chats(
self, *, actor_id: UUID, after: Key | None, limit: int
) -> list[Chat]:
condition = "AND (c.created_at,c.id)>(%s,%s)" if after else ""
parameters = (
[actor_id, *after, limit + 1] if after else [actor_id, limit + 1]
)
with self.database.connection() as conn:
rows = conn.execute(
"SELECT c.id,c.title,c.created_at,ARRAY(SELECT user_id "
"FROM members WHERE chat_id=c.id "
"ORDER BY position) member_ids "
"FROM chats c JOIN members own ON own.chat_id=c.id "
"WHERE own.user_id=%s "
+ condition
+ " ORDER BY c.created_at,c.id LIMIT %s",
parameters,
).fetchall()
return [chat_dto(row) for row in rows]
def list_messages(
self, *, chat_id: UUID, actor_id: UUID, after: Key | None, limit: int
) -> list[Message]:
condition = "AND (m.created_at,m.id)>(%s,%s)" if after else ""
parameters = (
[chat_id, *after, limit + 1] if after else [chat_id, limit + 1]
)
with self.database.connection() as conn:
rows = conn.execute(
"SELECT "
+ MESSAGE_COLUMNS
+ " FROM messages m WHERE m.chat_id=%s "
+ condition
+ " ORDER BY m.created_at,m.id LIMIT %s",
parameters,
).fetchall()
return [message_dto(row) for row in rows]
+65
View File
@@ -0,0 +1,65 @@
import re
from typing import Annotated
from uuid import UUID
from pydantic import (
BaseModel,
BeforeValidator,
ConfigDict,
Field,
StrictStr,
ValidationInfo,
field_validator,
model_validator,
)
from src.cursor import parse_uuid
UuidIn = Annotated[UUID, BeforeValidator(parse_uuid)]
class Input(BaseModel):
model_config = ConfigDict(extra="forbid")
@field_validator("*", mode="after")
@classmethod
def nonblank(cls, value: object, info: ValidationInfo) -> object:
if isinstance(value, str) and (
(not value.strip() and info.field_name != "text")
or any(0xD800 <= ord(c) <= 0xDFFF for c in value)
):
raise ValueError("Invalid text")
return value
class CreateUser(Input):
username: StrictStr
display_name: StrictStr = Field(min_length=1, max_length=100)
password: StrictStr = Field(min_length=12, max_length=128)
@field_validator("username")
@classmethod
def username_format(cls, value: str) -> str:
if not re.fullmatch(r"[a-z0-9_]{3,32}", value):
raise ValueError("Invalid username")
return value
class CreateChat(Input):
title: StrictStr = Field(min_length=1, max_length=100)
member_ids: list[UuidIn] = Field(min_length=1, max_length=100, strict=True)
class CreateMessage(Input):
text: StrictStr = Field(max_length=4000)
attachment_ids: list[UuidIn] = Field(
default_factory=list, max_length=10, strict=True
)
@model_validator(mode="after")
def content(self):
if len(set(self.attachment_ids)) != len(self.attachment_ids):
raise ValueError("Duplicate attachment")
if not self.text.strip() and not self.attachment_ids:
raise ValueError("Empty message")
return self
+192
View File
@@ -0,0 +1,192 @@
import base64
import hashlib
import re
import secrets
import threading
from datetime import timedelta
from uuid import UUID, uuid4
from src.cursor import decode_cursor, encode_cursor
from src.domain import Chat, Message, Record, User, order_key, utcnow
from src.errors import DomainError, invalid, not_found
from src.password import Passwords
from src.repository import PostgresRepository
def page(*, rows: list[Record], limit: int, scope: str) -> dict:
items = rows[:limit]
cursor = (
encode_cursor(scope=scope, key=order_key(items[-1]))
if len(rows) > limit
else None
)
return {"items": items, "next_cursor": cursor}
def unauthorized(*, basic=False):
challenge = (
'Basic realm="messenger", charset="UTF-8"'
if basic
else 'Bearer realm="messenger"'
)
return DomainError(
401,
"unauthorized",
"Authentication required",
{"WWW-Authenticate": challenge},
)
def bearer(value):
if not value:
raise unauthorized()
scheme, _, token = value.partition(" ")
if scheme.lower() != "bearer" or not re.fullmatch(
r"[A-Za-z0-9_-]{32,256}", token
):
raise unauthorized()
return token
def token_digest(value):
return hashlib.sha256(value.encode()).digest()
class MessengerService:
def __init__(
self,
*,
repository: PostgresRepository,
passwords: Passwords,
session_ttl: int,
) -> None:
self.repository = repository
self.passwords = passwords
self.session_ttl = session_ttl
self.registration_locks = [threading.Lock() for _ in range(64)]
def identify(self, *, value: str | None) -> User:
token = bearer(value)
user_id = self.repository.session_user(digest=token_digest(token))
user = self.repository.get_user(user_id=user_id) if user_id else None
if user is None:
raise unauthorized()
return user
def register(
self, *, username: str, display_name: str, password: str
) -> User:
# Bound lock memory and avoid running Argon2 for every local duplicate.
# Different replicas still race safely through PostgreSQL UNIQUE.
stripe = hashlib.sha256(username.encode()).digest()[0] % 64
lock = self.registration_locks[stripe]
if not lock.acquire(timeout=8):
raise DomainError(503, "unavailable", "Registration busy")
try:
if self.repository.username_exists(username=username):
raise DomainError(409, "conflict", "Username already exists")
encoded = self.passwords.hash(password=password)
return self.repository.add_user(
user=User(uuid4(), username, display_name, utcnow()),
password_hash=encoded,
)
finally:
lock.release()
def login(self, *, authorization: str | None):
try:
scheme, _, encoded = (authorization or "").partition(" ")
if scheme.lower() != "basic":
raise ValueError
plain = base64.b64decode(encoded, validate=True).decode("utf-8")
username, separator, password = plain.partition(":")
if (
not separator
or not re.fullmatch(r"[a-z0-9_]{3,32}", username)
or not 12 <= len(password) <= 128
or not password.strip()
):
raise ValueError
except ValueError, UnicodeError:
raise unauthorized(basic=True) from None
user, password_hash = self.repository.credentials(username=username)
if not self.passwords.verify(password=password, encoded=password_hash):
raise unauthorized(basic=True)
now = utcnow()
expires = now + timedelta(seconds=self.session_ttl)
token = secrets.token_urlsafe(32)
self.repository.add_session(
digest=token_digest(token),
user_id=user.id,
created_at=now,
expires_at=expires,
)
return {
"token": token,
"token_type": "Bearer",
"expires_at": expires,
"user": user,
}
def logout(self, *, authorization: str | None):
self.identify(value=authorization)
self.repository.delete_session(
digest=token_digest(bearer(authorization))
)
def create_chat(
self, *, actor_id: UUID, title: str, member_ids: list[UUID]
) -> Chat:
return self.repository.add_chat(
chat=Chat(uuid4(), title, tuple(member_ids), utcnow()),
actor_id=actor_id,
)
def get_chat(self, *, actor_id: UUID, chat_id: UUID) -> Chat:
return self.repository.get_chat(actor_id=actor_id, chat_id=chat_id)
def send(
self,
*,
actor_id: UUID,
chat_id: UUID,
text: str,
idempotency_key: str | None,
attachment_ids: list[UUID],
) -> Message:
if not idempotency_key or not re.fullmatch(
r"[A-Za-z0-9._:-]{1,128}", idempotency_key
):
raise invalid()
return self.repository.add_message(
message=Message(
uuid4(),
chat_id,
actor_id,
text,
utcnow(),
tuple(attachment_ids),
),
idempotency_key=idempotency_key,
)
def chats(self, *, actor_id: UUID, limit: int, cursor: str | None) -> dict:
scope = f"chats:{actor_id}"
after = decode_cursor(value=cursor, scope=scope)
rows = self.repository.list_chats(
actor_id=actor_id, after=after, limit=limit
)
return page(rows=rows, limit=limit, scope=scope)
def history(
self, *, actor_id: UUID, chat: Chat, limit: int, cursor: str | None
) -> dict:
if actor_id not in chat.member_ids:
raise not_found()
chat_id = chat.id
scope = f"messages:{chat_id}"
after = decode_cursor(value=cursor, scope=scope)
rows = self.repository.list_messages(
actor_id=actor_id, chat_id=chat_id, after=after, limit=limit
)
return page(rows=rows, limit=limit, scope=scope)
+37
View File
@@ -0,0 +1,37 @@
import os
from dataclasses import dataclass, field
from pathlib import Path
def secret(name: str) -> str:
file = os.environ.get(name + "_FILE")
if file:
return Path(file).read_text().strip()
value = os.environ.get(name)
if not value:
raise ValueError("Missing required secret setting: " + name)
return value
def integer(name: str, default: int, low: int, high: int) -> int:
value = int(os.environ.get(name, str(default)))
if not low <= value <= high:
raise ValueError("Invalid setting: " + name)
return value
@dataclass(frozen=True)
class Settings:
database_url: str = field(repr=False)
cache_url: str = field(repr=False)
session_ttl: int = 3600
pool_size: int = 10
@classmethod
def load(cls):
return cls(
secret("DATABASE_URL"),
secret("CACHE_URL"),
integer("SESSION_TTL_SECONDS", 3600, 1, 86400),
integer("DB_POOL_SIZE", 10, 1, 32),
)
+428
View File
@@ -0,0 +1,428 @@
"""Two bounded consumers; every decode runs in a killable isolated process."""
import hashlib
import io
import json
import multiprocessing
import resource
import threading
import time
from uuid import uuid4
import redis
from PIL import Image, ImageOps, UnidentifiedImageError
from src.images import MAX_BYTES, MAX_PIXELS
from src.object_store import ObjectStore
from src.pipeline import (
GROUP,
STREAM,
PermanentFailure,
broker,
database,
event,
fault,
retry,
validate_job,
)
from src.settings import integer
def sizes(width, height, bound):
longest = max(width, height)
return (
(width, height)
if longest <= bound
else (
max(1, width * bound // longest),
max(1, height * bound // longest),
)
)
def make_variants(raw, row):
if (
len(raw) != row["size_bytes"]
or hashlib.sha256(raw).hexdigest() != row["sha256"]
):
raise PermanentFailure("source_checksum_mismatch")
try:
with Image.open(io.BytesIO(raw)) as source:
if source.width * source.height > MAX_PIXELS:
raise PermanentFailure("pixel_limit")
if source.format not in {"PNG", "JPEG"}:
raise PermanentFailure("invalid_image")
source.load()
oriented = ImageOps.exif_transpose(source)
rgba = oriented.convert("RGBA")
rgb = Image.new("RGB", rgba.size, "white")
rgb.paste(rgba, mask=rgba.getchannel("A"))
rgb.info.clear()
yield "original", raw, row["content_type"], source.size
thumbnail = rgb.resize(
sizes(*rgb.size, 256), Image.Resampling.LANCZOS
)
crop = ImageOps.fit(
rgb, (128, 128), Image.Resampling.LANCZOS, centering=(0.5, 0.5)
)
optimized = rgb.resize(
sizes(*rgb.size, 1280), Image.Resampling.LANCZOS
)
for name, image, fmt, mime, options in [
(
"thumbnail",
thumbnail,
"JPEG",
"image/jpeg",
{"quality": 85, "subsampling": 0},
),
(
"crop",
crop,
"JPEG",
"image/jpeg",
{"quality": 85, "subsampling": 0},
),
(
"optimized",
optimized,
"WEBP",
"image/webp",
{
"quality": 82 if row["pipeline_version"] == 1 else 76,
"method": 4,
},
),
]:
output = io.BytesIO()
image.info.clear()
image.save(output, format=fmt, **options)
yield name, output.getvalue(), mime, image.size
except (
UnidentifiedImageError,
OSError,
ValueError,
Image.DecompressionBombError,
) as exc:
raise PermanentFailure("invalid_image") from exc
def process(job_id, fence, sender):
db = store = None
started = time.monotonic()
cpu = time.process_time()
try:
resource.setrlimit(resource.RLIMIT_AS, (768 * 1024**2, 768 * 1024**2))
db, store = database(), ObjectStore()
with db.connection() as conn:
row = conn.execute(
"SELECT j.*,a.size_bytes,a.sha256,a.content_type FROM jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=%s AND j.fence=%s",
(job_id, fence),
).fetchone()
if not row:
sender.send({"result": "stale"})
return
fault(db, job_id, "before_effect")
upstream = store.get(key=row["source_key"])
try:
raw = upstream.read(MAX_BYTES + 1)
finally:
upstream.close()
upstream.release_conn()
results = []
for name, data, mime, (width, height) in make_variants(raw, row):
key = f"objects/{row['attachment_id']}/v{row['pipeline_version']}/{name}"
digest = hashlib.sha256(data).hexdigest()
store.put(
key=key,
stream=io.BytesIO(data),
size=len(data),
content_type=mime,
sha256=digest,
)
results.append((name, key, mime, len(data), digest, width, height))
fault(db, job_id, "after_effect")
with db.connection() as conn, conn.transaction():
current = conn.execute(
"SELECT * FROM jobs WHERE id=%s FOR UPDATE", (job_id,)
).fetchone()
allowed = conn.execute(
"SELECT pipeline_version FROM attachments WHERE id=%s FOR UPDATE",
(row["attachment_id"],),
).fetchone()
valid_lease = conn.execute(
"SELECT %s::timestamptz>clock_timestamp() live",
(current["lease_until"],),
).fetchone()["live"]
if (
current["state"] != "processing"
or current["fence"] != fence
or not valid_lease
or allowed["pipeline_version"] != row["pipeline_version"]
):
event(
conn,
job_id,
"stale_completion_rejected",
{"fence": fence, "version": row["pipeline_version"]},
)
sender.send({"result": "stale"})
return
for result in results:
conn.execute(
"INSERT INTO attachment_variants(attachment_id,pipeline_version,name,object_key,content_type,size_bytes,sha256,width,height) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT(attachment_id,pipeline_version,name) DO NOTHING",
(row["attachment_id"], row["pipeline_version"], *result),
)
conn.execute(
"INSERT INTO effect_publications(attachment_id,pipeline_version,job_id,fence) VALUES(%s,%s,%s,%s)",
(row["attachment_id"], row["pipeline_version"], job_id, fence),
)
conn.execute(
"UPDATE attachments SET state='ready',object_key=%s,published_version=pipeline_version,error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",
(results[0][1], row["attachment_id"]),
)
conn.execute(
"UPDATE jobs SET state='ready',error_code=NULL,lease_owner=NULL,lease_until=NULL,updated_at=clock_timestamp() WHERE id=%s",
(job_id,),
)
event(
conn,
job_id,
"effect_committed",
{
"fence": fence,
"version": row["pipeline_version"],
"variants": 4,
},
)
fault(db, job_id, "after_commit")
# Cleanup is independently retryable; it cannot undo the committed effect.
if row["source_key"].startswith("objects/staging/"):
try:
store.delete(key=row["source_key"])
except Exception:
pass
sender.send(
{
"result": "ready",
"seconds": time.monotonic() - started,
"cpu_seconds": time.process_time() - cpu,
"rss_peak_mib": resource.getrusage(
resource.RUSAGE_SELF
).ru_maxrss
/ 1024,
}
)
except PermanentFailure as exc:
sender.send({"result": "permanent", "code": str(exc)})
except MemoryError:
sender.send({"result": "permanent", "code": "memory_limit"})
except Exception:
sender.send({"result": "transient", "code": "processing_unavailable"})
finally:
if store:
store.close()
if db:
db.close()
sender.close()
def claim(db, data, consumer):
with db.connection() as conn, conn.transaction():
row = conn.execute(
"SELECT * FROM jobs WHERE id=%s FOR UPDATE", (data["job_id"],)
).fetchone()
if (
not row
or str(row["attachment_id"]) != data["attachment_id"]
or row["pipeline_version"] != data["pipeline_version"]
):
return "obsolete", None
if row["state"] in {"ready", "failed", "obsolete"}:
return row["state"], row
if data["attempt"] < row["attempt"] or (
row["state"] == "queued" and data["attempt"] <= row["attempt"]
):
return "obsolete", row
if row["state"] == "processing":
return "busy", row
if data["attempt"] != row["attempt"] + 1:
return "obsolete", row
due = conn.execute(
"SELECT %s::timestamptz<=clock_timestamp() due", (row["next_at"],)
).fetchone()["due"]
if not due:
return "busy", row
lease = integer("JOB_LEASE_SECONDS", 23, 3, 120)
updated = conn.execute(
"UPDATE jobs SET state='processing',attempt=attempt+1,fence=fence+1,lease_owner=%s,lease_until=clock_timestamp()+%s*interval '1 second',updated_at=clock_timestamp() WHERE id=%s RETURNING *",
(consumer, lease, row["id"]),
).fetchone()
conn.execute(
"UPDATE attachments SET state='processing',updated_at=clock_timestamp() WHERE id=%s AND pipeline_version=%s",
(row["attachment_id"], row["pipeline_version"]),
)
event(
conn,
row["id"],
"claimed",
{
"attempt": updated["attempt"],
"fence": updated["fence"],
"version": updated["pipeline_version"],
},
)
return "claimed", updated
def ready_effect(db, store, row):
with db.connection() as conn:
variants = conn.execute(
"SELECT * FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s",
(row["attachment_id"], row["pipeline_version"]),
).fetchall()
if len(variants) != 4:
return False
for variant in variants:
head = store.head(key=variant["object_key"])
if (
head is None
or head.size != variant["size_bytes"]
or head.metadata.get("x-amz-meta-sha256") != variant["sha256"]
):
return False
return True
def consume(slot):
db, queue, store = database(), broker(), ObjectStore()
consumer = str(uuid4()) + f"-{slot}"
context = multiprocessing.get_context("spawn")
cursor = "0-0"
timeout = integer("JOB_TIMEOUT_SECONDS", 20, 2, 117)
if integer("JOB_LEASE_SECONDS", 23, 3, 120) <= timeout:
raise ValueError("Lease must exceed child timeout")
try:
while True:
try:
try:
queue.xgroup_create(STREAM, GROUP, id="0", mkstream=True)
except redis.ResponseError as exc:
if "BUSYGROUP" not in str(exc):
raise
reclaimed = queue.xautoclaim(
STREAM,
GROUP,
consumer,
min_idle_time=3000,
start_id=cursor,
count=1,
)
cursor, messages = reclaimed[0], reclaimed[1]
if not messages:
batch = queue.xreadgroup(
GROUP, consumer, {STREAM: ">"}, count=1, block=500
)
messages = batch[0][1] if batch else []
for entry, fields in messages:
try:
data = validate_job(
json.loads(fields.get("job", "null"))
)
except ValueError, TypeError, KeyError:
# Malformed notifications cannot become jobs; preserve diagnostic count.
with db.connection() as conn:
event(conn, None, "invalid_envelope")
queue.xack(STREAM, GROUP, entry)
queue.xdel(STREAM, entry)
continue
state, row = claim(db, data, consumer)
if state == "busy":
continue
if state == "ready" and not ready_effect(db, store, row):
with db.connection() as conn:
event(conn, row["id"], "ready_effect_missing")
continue
if state == "claimed":
receiver, sender = context.Pipe(duplex=False)
child = context.Process(
target=process,
args=(str(row["id"]), row["fence"], sender),
)
child.start()
sender.close()
child.join(timeout)
if child.is_alive():
child.kill()
child.join(2)
result = {
"result": "transient",
"code": "worker_timeout",
}
else:
try:
result = (
receiver.recv()
if receiver.poll()
else {
"result": "transient",
"code": "worker_crashed",
}
)
except EOFError:
result = {
"result": "transient",
"code": "worker_crashed",
}
receiver.close()
child.close()
with db.connection() as conn, conn.transaction():
current = conn.execute(
"SELECT * FROM jobs WHERE id=%s FOR UPDATE",
(row["id"],),
).fetchone()
event(conn, row["id"], "child_finished", result)
if (
current["state"] == "processing"
and current["fence"] == row["fence"]
):
retry(
conn,
current,
result.get("code", "processing_failed"),
permanent=result["result"] == "permanent",
)
# Commit result or a retry/terminal decision precedes ack.
queue.xack(STREAM, GROUP, entry)
queue.xdel(STREAM, entry)
except Exception:
print(
json.dumps(
{
"event": "worker_retry",
"code": "dependency_unavailable",
}
),
flush=True,
)
time.sleep(0.5)
finally:
queue.close()
store.close()
db.close()
def main():
count = integer("WORKER_CONCURRENCY", 2, 1, 4)
threads = [
threading.Thread(target=consume, args=(i,), daemon=False)
for i in range(count)
]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
if __name__ == "__main__":
main()
Generated
+1095
View File
File diff suppressed because it is too large Load Diff