Implement TLS identities, scoped OpenBao access and worker security study

This commit is contained in:
2026-11-16 11:57:32 +03:00
parent 20afc2c790
commit 32795a92ac
78 changed files with 8798 additions and 19 deletions
+10 -19
View File
@@ -1,20 +1,11 @@
.git
.github
.env
.env.*
!.env.example
*
!pyproject.toml
!uv.lock
!src/
!src/**
!migrations/
!migrations/**
operator-private
secrets
certs
volumes
data
backups
*.key
*.pem
*.p12
*.pfx
*.jks
.venv
__pycache__
node_modules
evidence
.DS_Store
.env
+6
View File
@@ -30,3 +30,9 @@ coverage/
.vscode/
# Keep small, sanitized evidence summaries in evidence/ under version control.
# Inspect git diff --cached: ignore rules are not a secret scanner.
.pytest_cache/
.ruff_cache/
operator-private/
secrets/
.env
+20
View File
@@ -0,0 +1,20 @@
FROM python:3.12-slim@sha256:646fb0bca3dd3ea1bcc6feb72c17ed16eed6e10cffc732fcc1478bd3e7f02d7b AS build
COPY --from=ghcr.io/astral-sh/uv:0.11.7 /uv /usr/local/bin/uv
WORKDIR /app
COPY pyproject.toml uv.lock ./
RUN uv sync --locked --no-dev --no-install-project --python /usr/local/bin/python
FROM python:3.12-slim@sha256:646fb0bca3dd3ea1bcc6feb72c17ed16eed6e10cffc732fcc1478bd3e7f02d7b
RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends libjpeg-turbo-progs && rm -rf /var/lib/apt/lists/*
RUN python -m pip uninstall --yes pip
ARG VCS_REF=uncommitted
LABEL org.opencontainers.image.revision=$VCS_REF
ENV PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=/app/src PATH=/app/.venv/bin:$PATH
WORKDIR /app
COPY --from=build /app/.venv /app/.venv
COPY src ./src
COPY migrations ./migrations
USER 10001:10001
EXPOSE 8080
HEALTHCHECK --interval=3s --timeout=2s --start-period=5s --retries=5 CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=1)"]
CMD ["python", "-m", "messenger"]
+25
View File
@@ -31,3 +31,28 @@ test-failover:
test-security:
$(PYTHON) tests/security.py --base-url "$(BASE_URL)" $(TLS_ARGS)
# Student implementation; teacher acceptance targets remain unchanged.
PUBLIC_PORT ?= 8447
BASE_URL = https://localhost:$(PUBLIC_PORT)
CA_FILE = $(CURDIR)/secrets/ca.pem
bootstrap:
python3 scripts/bootstrap.py
build:
VCS_REF=$$(git rev-parse HEAD) docker compose build
init: bootstrap build
docker compose up -d postgres openbao cache cache-tls broker broker-tls storage s3 storage-tls
python3 scripts/bao_operator.py init
up: init
VCS_REF=$$(git rev-parse HEAD) docker compose up -d --wait
unseal:
python3 scripts/bao_operator.py unseal
migrate:
docker compose run --rm migrate
down:
docker compose down
student-test:
BASE_URL=$(BASE_URL) SSL_CERT_FILE=$(CA_FILE) uv run --locked pytest -q
lint:
uv run --locked ruff check src student-tests scripts
.PHONY: bootstrap build init up unseal migrate down student-test lint
+755
View File
@@ -0,0 +1,755 @@
name: mtusi-vasilev-lab7
x-hardening:
cpus: 0.25
mem_limit: 128m
pids_limit: 96
cap_drop: &id001
- ALL
security_opt: &id002
- no-new-privileges:true
read_only: true
tmpfs: &id003
- /tmp:size=16m,noexec,nosuid
x-environment:
BAO_ADDR: https://openbao:8200
S3_ENDPOINT: https://storage:8443
S3_BUCKET: messenger
S3_CA_FILE: /run/tls/ca.pem
SSL_CERT_FILE: /run/tls/ca.pem
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-45}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
OTEL_EXPORTER_OTLP_ENDPOINT: https://collector:4318
OTEL_EXPORTER_OTLP_CERTIFICATE: /run/tls/ca.pem
OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/tls/identity/cert.pem
OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/tls/identity/key.pem
OTEL_ENABLED: '1'
OTEL_METRIC_INTERVAL_MS: '5000'
x-application:
cpus: 0.5
mem_limit: 512m
pids_limit: 96
cap_drop: *id001
security_opt: *id002
read_only: true
tmpfs: *id003
image: ${API_IMAGE:-mtusi-vasilev:lab7}
build: &id004
context: .
args:
VCS_REF: ${VCS_REF:-uncommitted}
stop_grace_period: 8s
services:
postgres:
image: postgres:17.10-alpine
entrypoint:
- /etc/pg-entrypoint.sh
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets:
- postgres_password
volumes:
- postgres_data:/var/lib/postgresql/data
- ./secrets/pki/postgres:/run/tls/identity:ro
- ./secrets/ca.pem:/run/tls/ca.pem:ro
- ./infra/pg-entrypoint.sh:/etc/pg-entrypoint.sh:ro
- ./infra/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro
- ./secrets/db_roles.sql:/docker-entrypoint-initdb.d/01_roles.sql:ro
cpus: 0.75
mem_limit: 768m
command:
- postgres
- -c
- fsync=on
- -c
- ssl=on
- -c
- ssl_cert_file=/run/tls/identity/cert.pem
- -c
- ssl_key_file=/tmp/postgres-key.pem
- -c
- ssl_ca_file=/run/tls/ca.pem
- -c
- hba_file=/etc/postgresql/pg_hba.conf
healthcheck:
test:
- CMD-SHELL
- pg_isready -U messenger -d messenger
interval: 2s
timeout: 2s
retries: 20
openbao:
cpus: 0.25
mem_limit: 192m
pids_limit: 96
cap_drop: *id001
security_opt: *id002
read_only: true
tmpfs: *id003
image: ghcr.io/openbao/openbao:2.5.2
user: 100:1000
command:
- server
- -config=/etc/openbao.hcl
ports:
- 127.0.0.1:18207:8200
volumes:
- bao_data:/bao/data
- ./infra/openbao.hcl:/etc/openbao.hcl:ro
- ./secrets/pki/openbao:/run/tls/identity:ro
- ./secrets/ca.pem:/run/tls/ca.pem:ro
depends_on:
state-init:
condition: service_completed_successfully
cache:
cpus: 0.25
mem_limit: 128m
pids_limit: 96
cap_drop: *id001
security_opt: *id002
read_only: true
tmpfs: *id003
image: valkey/valkey:8.1.3-alpine
command:
- /run/secrets/cache_config
secrets:
- cache_config
user: 1000:1000
entrypoint:
- valkey-server
cache-tls:
cpus: 0.5
mem_limit: 64m
pids_limit: 96
cap_drop: *id001
security_opt: *id002
read_only: true
tmpfs: *id003
image: ${API_IMAGE:-mtusi-vasilev:lab7}
build: *id004
stop_grace_period: 8s
network_mode: service:cache
command:
- python
- -m
- messenger.tls_gateway
- --config
- /run/tls/gateway.json
volumes:
- ./secrets/pki/cache:/run/tls/identity:ro
- ./secrets/ca.pem:/run/tls/ca.pem:ro
- ./secrets/revoked.json:/run/tls/revoked.json:ro
- ./secrets/cache_gateway.json:/run/tls/gateway.json:ro
healthcheck:
disable: true
broker:
cpus: 0.25
mem_limit: 192m
pids_limit: 96
cap_drop: *id001
security_opt: *id002
read_only: true
tmpfs: *id003
image: valkey/valkey:8.1.3-alpine
command:
- /run/secrets/broker_config
secrets:
- broker_config
volumes:
- broker_data:/data
depends_on:
state-init:
condition: service_completed_successfully
user: 1000:1000
entrypoint:
- valkey-server
broker-tls:
cpus: 0.5
mem_limit: 64m
pids_limit: 96
cap_drop: *id001
security_opt: *id002
read_only: true
tmpfs: *id003
image: ${API_IMAGE:-mtusi-vasilev:lab7}
build: *id004
stop_grace_period: 8s
network_mode: service:broker
command:
- python
- -m
- messenger.tls_gateway
- --config
- /run/tls/gateway.json
volumes:
- ./secrets/pki/broker:/run/tls/identity:ro
- ./secrets/ca.pem:/run/tls/ca.pem:ro
- ./secrets/revoked.json:/run/tls/revoked.json:ro
- ./secrets/broker_gateway.json:/run/tls/gateway.json:ro
healthcheck:
disable: true
storage:
image: chrislusf/seaweedfs:4.15@sha256:1593bd2a98964f5d4df83fa7db2b3d4ee7b20190813cc5bf8c3200e0e70d346e
command:
- server
- -dir=/data
- -ip=127.0.0.1
- -ip.bind=127.0.0.1
- -filer
- -volume.max=32
- -volume.preStopSeconds=0
- -master.volumeSizeLimitMB=64
volumes:
- s3_data:/data
- ./infra/filer.toml:/etc/seaweedfs/filer.toml:ro
cpus: 0.5
mem_limit: 512m
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
healthcheck:
test:
- CMD-SHELL
- wget -q -O /dev/null http://127.0.0.1:9333/dir/status
interval: 2s
timeout: 2s
retries: 25
depends_on:
state-init:
condition: service_completed_successfully
user: 1000:1000
entrypoint:
- /usr/bin/weed
s3:
image: chrislusf/seaweedfs:4.15@sha256:1593bd2a98964f5d4df83fa7db2b3d4ee7b20190813cc5bf8c3200e0e70d346e
network_mode: service:storage
command:
- s3
- -filer=127.0.0.1:8888
- -ip.bind=127.0.0.1
- -config=/run/secrets/s3_config
- -port=8333
- -port.iceberg=0
secrets:
- s3_config
depends_on:
storage:
condition: service_healthy
cpus: 0.25
mem_limit: 256m
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
user: 1000:1000
entrypoint:
- /usr/bin/weed
storage-tls:
cpus: 0.5
mem_limit: 96m
pids_limit: 96
cap_drop: *id001
security_opt: *id002
read_only: true
tmpfs: *id003
image: ${API_IMAGE:-mtusi-vasilev:lab7}
build: *id004
stop_grace_period: 8s
network_mode: service:storage
command:
- python
- -m
- messenger.tls_gateway
- --config
- /run/tls/gateway.json
volumes:
- ./secrets/pki/storage:/run/tls/identity:ro
- ./secrets/ca.pem:/run/tls/ca.pem:ro
- ./secrets/revoked.json:/run/tls/revoked.json:ro
- ./secrets/storage_gateway.json:/run/tls/gateway.json:ro
healthcheck:
disable: true
migrate:
cpus: 0.5
mem_limit: 512m
pids_limit: 96
cap_drop: *id001
security_opt: *id002
read_only: true
tmpfs: *id003
image: ${API_IMAGE:-mtusi-vasilev:lab7}
build: *id004
stop_grace_period: 8s
command:
- python
- -m
- messenger.migrate
environment:
DATABASE_URL_FILE: /run/secrets/admin_database_url
secrets:
- admin_database_url
volumes:
- ./secrets/ca.pem:/run/tls/ca.pem:ro
healthcheck:
disable: true
depends_on:
postgres:
condition: service_healthy
s3-bootstrap:
cpus: 0.5
mem_limit: 512m
pids_limit: 96
cap_drop: *id001
security_opt: *id002
read_only: true
tmpfs: *id003
image: ${API_IMAGE:-mtusi-vasilev:lab7}
build: *id004
stop_grace_period: 8s
command:
- python
- -m
- messenger.storage_bootstrap
environment:
S3_ENDPOINT: https://storage:8443
S3_ADMIN_ACCESS_KEY_FILE: /run/secrets/admin_s3_access_key
S3_ADMIN_SECRET_KEY_FILE: /run/secrets/admin_s3_secret_key
secrets:
- admin_s3_access_key
- admin_s3_secret_key
volumes:
- ./secrets/pki/operator:/run/tls/identity:ro
- ./secrets/ca.pem:/run/tls/ca.pem:ro
healthcheck:
disable: true
depends_on:
storage-tls:
condition: service_started
api:
cpus: 0.5
mem_limit: 512m
pids_limit: 96
cap_drop: *id001
security_opt: *id002
read_only: true
tmpfs: *id003
image: ${API_IMAGE:-mtusi-vasilev:lab7}
build: *id004
stop_grace_period: 8s
command:
- python
- -m
- messenger.secure_entry
environment:
BAO_ADDR: https://openbao:8200
S3_ENDPOINT: https://storage:8443
S3_BUCKET: messenger
S3_CA_FILE: /run/tls/ca.pem
SSL_CERT_FILE: /run/tls/ca.pem
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-45}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
OTEL_EXPORTER_OTLP_ENDPOINT: https://collector:4318
OTEL_EXPORTER_OTLP_CERTIFICATE: /run/tls/ca.pem
OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/tls/identity/cert.pem
OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/tls/identity/key.pem
BAO_ROLE: api
TLS_GATEWAY_CONFIG: /run/tls/gateway.json
OTEL_ENABLED: '1'
OTEL_METRIC_INTERVAL_MS: '5000'
volumes:
- ./secrets/pki/api:/run/tls/identity:ro
- ./secrets/ca.pem:/run/tls/ca.pem:ro
- ./secrets/revoked.json:/run/tls/revoked.json:ro
- ./secrets/api_gateway.json:/run/tls/gateway.json:ro
deploy:
replicas: ${API_REPLICAS:-2}
depends_on:
migrate:
condition: service_completed_successfully
s3-bootstrap:
condition: service_completed_successfully
cache-tls:
condition: service_started
dispatcher:
cpus: 0.5
mem_limit: 192m
pids_limit: 96
cap_drop: *id001
security_opt: *id002
read_only: true
tmpfs: *id003
image: ${API_IMAGE:-mtusi-vasilev:lab7}
build: *id004
stop_grace_period: 8s
command:
- python
- -m
- messenger.worker
- --dispatcher
environment:
BAO_ADDR: https://openbao:8200
S3_ENDPOINT: https://storage:8443
S3_BUCKET: messenger
S3_CA_FILE: /run/tls/ca.pem
SSL_CERT_FILE: /run/tls/ca.pem
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-45}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
OTEL_EXPORTER_OTLP_ENDPOINT: https://collector:4318
OTEL_EXPORTER_OTLP_CERTIFICATE: /run/tls/ca.pem
OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/tls/identity/cert.pem
OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/tls/identity/key.pem
BAO_ROLE: dispatcher
OTEL_ENABLED: '1'
OTEL_METRIC_INTERVAL_MS: '5000'
volumes:
- ./secrets/pki/dispatcher:/run/tls/identity:ro
- ./secrets/ca.pem:/run/tls/ca.pem:ro
healthcheck:
disable: true
restart: unless-stopped
depends_on:
migrate:
condition: service_completed_successfully
broker-tls:
condition: service_started
worker:
cpus: 0.75
mem_limit: 896m
pids_limit: 96
cap_drop: *id001
security_opt: *id002
read_only: true
tmpfs:
- /tmp:size=32m,noexec,nosuid
image: ${API_IMAGE:-mtusi-vasilev:lab7}
build: *id004
stop_grace_period: 30s
command:
- python
- -m
- messenger.worker
environment:
BAO_ADDR: https://openbao:8200
S3_ENDPOINT: https://storage:8443
S3_BUCKET: messenger
S3_CA_FILE: /run/tls/ca.pem
SSL_CERT_FILE: /run/tls/ca.pem
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-45}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
OTEL_EXPORTER_OTLP_ENDPOINT: https://collector:4318
OTEL_EXPORTER_OTLP_CERTIFICATE: /run/tls/ca.pem
OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/tls/identity/cert.pem
OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/tls/identity/key.pem
BAO_ROLE: worker
OTEL_ENABLED: '1'
OTEL_METRIC_INTERVAL_MS: '5000'
volumes:
- ./secrets/pki/worker:/run/tls/identity:ro
- ./secrets/ca.pem:/run/tls/ca.pem:ro
healthcheck:
disable: true
restart: unless-stopped
depends_on:
migrate:
condition: service_completed_successfully
broker-tls:
condition: service_started
s3-bootstrap:
condition: service_completed_successfully
lb:
cpus: 0.25
mem_limit: 128m
pids_limit: 96
cap_drop: *id001
security_opt: *id002
read_only: true
tmpfs: *id003
image: haproxy:3.2-alpine@sha256:96ec3c75a9264ba94e733eeacb44051ed6e7b79bb77a73c2c36f236b6423f1f0
ports:
- 127.0.0.1:${PUBLIC_PORT:-8447}:8443
volumes:
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
- ./secrets/pki/lb:/run/tls/identity:ro
- ./secrets/ca.pem:/run/tls/ca.pem:ro
depends_on:
api:
condition: service_healthy
healthcheck:
test:
- CMD-SHELL
- test -S /tmp/haproxy.sock
interval: 2s
timeout: 2s
retries: 20
collector:
cpus: 0.25
mem_limit: 256m
pids_limit: 96
cap_drop: *id001
security_opt: *id002
read_only: true
tmpfs: *id003
image: otel/opentelemetry-collector-contrib:0.123.0
command:
- --config=/etc/otelcol/config.yaml
volumes:
- ./infra/observability/collector.yaml:/etc/otelcol/config.yaml:ro
- ./secrets/pki/collector:/run/tls/identity:ro
- ./secrets/ca.pem:/run/tls/ca.pem:ro
prometheus:
cpus: 0.25
mem_limit: 256m
pids_limit: 96
cap_drop: *id001
security_opt: *id002
read_only: true
tmpfs: *id003
image: prom/prometheus:v3.2.1
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=24h
- --web.listen-address=127.0.0.1:9090
volumes:
- ./infra/observability/prometheus.yaml:/etc/prometheus/prometheus.yml:ro
- ./infra/observability/alerts.yaml:/etc/prometheus/alerts.yaml:ro
- prometheus_data:/prometheus
- ./secrets/pki/prometheus:/run/tls/identity:ro
- ./secrets/ca.pem:/run/tls/ca.pem:ro
ports:
- 127.0.0.1:19097:9443
depends_on:
state-init:
condition: service_completed_successfully
loki:
cpus: 0.25
mem_limit: 384m
pids_limit: 96
cap_drop: *id001
security_opt: *id002
read_only: true
tmpfs: *id003
image: grafana/loki:3.4.2
command:
- -config.file=/etc/loki/config.yaml
volumes:
- ./infra/observability/loki.yaml:/etc/loki/config.yaml:ro
- loki_data:/loki
depends_on:
state-init:
condition: service_completed_successfully
tempo:
cpus: 0.25
mem_limit: 768m
environment:
GOMEMLIMIT: 512MiB
pids_limit: 96
cap_drop: *id001
security_opt: *id002
read_only: true
tmpfs: *id003
image: grafana/tempo:2.7.2
command:
- -config.file=/etc/tempo/config.yaml
volumes:
- ./infra/observability/tempo.yaml:/etc/tempo/config.yaml:ro
- tempo_data:/var/tempo
depends_on:
state-init:
condition: service_completed_successfully
grafana:
cpus: 0.25
mem_limit: 256m
pids_limit: 96
cap_drop: *id001
security_opt: *id002
read_only: true
tmpfs: *id003
image: grafana/grafana:11.6.0
environment:
GF_AUTH_ANONYMOUS_ENABLED: 'false'
GF_AUTH_DISABLE_LOGIN_FORM: 'false'
GF_USERS_ALLOW_SIGN_UP: 'false'
GF_ANALYTICS_REPORTING_ENABLED: 'false'
GF_ANALYTICS_CHECK_FOR_UPDATES: 'false'
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_password
GF_SERVER_PROTOCOL: https
GF_SERVER_CERT_FILE: /run/tls/identity/cert.pem
GF_SERVER_CERT_KEY: /run/tls/identity/key.pem
volumes:
- ./secrets/grafana-provisioning/datasources:/etc/grafana/provisioning/datasources:ro
- ./infra/grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
- grafana_data:/var/lib/grafana
- ./secrets/pki/grafana:/run/tls/identity:ro
- ./secrets/ca.pem:/run/tls/ca.pem:ro
ports:
- 127.0.0.1:13007:3000
secrets:
- grafana_password
depends_on:
state-init:
condition: service_completed_successfully
collector-tls:
cpus: 0.5
mem_limit: 64m
pids_limit: 96
cap_drop: *id001
security_opt: *id002
read_only: true
tmpfs: *id003
image: ${API_IMAGE:-mtusi-vasilev:lab7}
build: *id004
stop_grace_period: 8s
network_mode: service:collector
command:
- python
- -m
- messenger.tls_gateway
- --config
- /run/tls/gateway.json
volumes:
- ./secrets/pki/collector:/run/tls/identity:ro
- ./secrets/ca.pem:/run/tls/ca.pem:ro
- ./secrets/revoked.json:/run/tls/revoked.json:ro
- ./secrets/collector_gateway.json:/run/tls/gateway.json:ro
healthcheck:
disable: true
prometheus-tls:
cpus: 0.5
mem_limit: 64m
pids_limit: 96
cap_drop: *id001
security_opt: *id002
read_only: true
tmpfs: *id003
image: ${API_IMAGE:-mtusi-vasilev:lab7}
build: *id004
stop_grace_period: 8s
network_mode: service:prometheus
command:
- python
- -m
- messenger.tls_gateway
- --config
- /run/tls/gateway.json
volumes:
- ./secrets/pki/prometheus:/run/tls/identity:ro
- ./secrets/ca.pem:/run/tls/ca.pem:ro
- ./secrets/revoked.json:/run/tls/revoked.json:ro
- ./secrets/prometheus_gateway.json:/run/tls/gateway.json:ro
healthcheck:
disable: true
loki-tls:
cpus: 0.5
mem_limit: 64m
pids_limit: 96
cap_drop: *id001
security_opt: *id002
read_only: true
tmpfs: *id003
image: ${API_IMAGE:-mtusi-vasilev:lab7}
build: *id004
stop_grace_period: 8s
network_mode: service:loki
command:
- python
- -m
- messenger.tls_gateway
- --config
- /run/tls/gateway.json
volumes:
- ./secrets/pki/loki:/run/tls/identity:ro
- ./secrets/ca.pem:/run/tls/ca.pem:ro
- ./secrets/revoked.json:/run/tls/revoked.json:ro
- ./secrets/loki_gateway.json:/run/tls/gateway.json:ro
healthcheck:
disable: true
tempo-tls:
cpus: 0.5
mem_limit: 64m
pids_limit: 96
cap_drop: *id001
security_opt: *id002
read_only: true
tmpfs: *id003
image: ${API_IMAGE:-mtusi-vasilev:lab7}
build: *id004
stop_grace_period: 8s
network_mode: service:tempo
command:
- python
- -m
- messenger.tls_gateway
- --config
- /run/tls/gateway.json
volumes:
- ./secrets/pki/tempo:/run/tls/identity:ro
- ./secrets/ca.pem:/run/tls/ca.pem:ro
- ./secrets/revoked.json:/run/tls/revoked.json:ro
- ./secrets/tempo_gateway.json:/run/tls/gateway.json:ro
healthcheck:
disable: true
state-init:
cpus: 0.5
mem_limit: 64m
pids_limit: 96
cap_drop: *id001
security_opt: *id002
read_only: true
tmpfs: *id003
image: ${API_IMAGE:-mtusi-vasilev:lab7}
build: *id004
stop_grace_period: 8s
user: 0:0
command:
- python
- -c
- import os; pairs=[('/state/bao',100),('/state/broker',1000),('/state/storage',1000),('/state/prometheus',65534),('/state/loki',10001),('/state/tempo',10001),('/state/grafana',472)];
[(os.chown(root,uid,uid),[os.chown(os.path.join(root,n),uid,uid) for n in dirs+files])
for path,uid in pairs for root,dirs,files in os.walk(path)]
cap_add:
- CHOWN
- DAC_OVERRIDE
healthcheck:
disable: true
volumes:
- bao_data:/state/bao
- broker_data:/state/broker
- s3_data:/state/storage
- prometheus_data:/state/prometheus
- loki_data:/state/loki
- tempo_data:/state/tempo
- grafana_data:/state/grafana
volumes:
postgres_data: null
s3_data: null
broker_data: null
bao_data: null
prometheus_data: {}
loki_data: {}
tempo_data: {}
grafana_data: {}
secrets:
postgres_password:
file: ./secrets/postgres_password
cache_config:
file: ./secrets/cache_config
broker_config:
file: ./secrets/broker_config
s3_config:
file: ./secrets/s3_config
admin_database_url:
file: ./secrets/admin_database_url
admin_s3_access_key:
file: ./secrets/admin_s3_access_key
admin_s3_secret_key:
file: ./secrets/admin_s3_secret_key
grafana_password:
file: ./secrets/grafana_password
+3
View File
@@ -0,0 +1,3 @@
[leveldb2]
enabled = true
dir = "/data/filerldb2"
+450
View File
@@ -0,0 +1,450 @@
{
"uid": "messenger-overview",
"title": "Messenger: API and image pipeline",
"schemaVersion": 39,
"version": 1,
"refresh": "5s",
"time": {
"from": "now-15m",
"to": "now"
},
"panels": [
{
"id": 1,
"title": "Business RPS",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "sum(rate(messenger_http_requests_total[1m]))"
}
],
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"fieldConfig": {
"defaults": {
"unit": "reqps"
},
"overrides": []
}
},
{
"id": 2,
"title": "Business 5xx ratio",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "sum(rate(messenger_http_requests_total{status_class=\"5xx\"}[1m])) / clamp_min(sum(rate(messenger_http_requests_total[1m])),0.001)"
}
],
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 0
},
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
}
},
{
"id": 3,
"title": "HTTP latency quantiles",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))",
"legendFormat": "p50"
},
{
"refId": "B",
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))",
"legendFormat": "p95"
},
{
"refId": "C",
"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))",
"legendFormat": "p99"
}
],
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 8
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
}
},
{
"id": 4,
"title": "Job outcomes",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "sum by (outcome) (rate(messenger_jobs_total[1m]))"
}
],
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 8
},
"fieldConfig": {
"defaults": {
"unit": "ops"
},
"overrides": []
}
},
{
"id": 5,
"title": "Job p95 processing",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket[1m])))"
}
],
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 16
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
}
},
{
"id": 6,
"title": "End-to-end p95 to ready",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_ready_duration_seconds_bucket[1m])))"
}
],
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 16
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
}
},
{
"id": 7,
"title": "Outstanding jobs",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "max(messenger_queue_depth)"
}
],
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 24
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
}
},
{
"id": 8,
"title": "Oldest outstanding age",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "max(messenger_queue_oldest_age_seconds)"
}
],
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 24
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
}
},
{
"id": 9,
"title": "DB pool used",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "messenger_db_pool_used"
}
],
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 32
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
}
},
{
"id": 10,
"title": "DB pool waiting",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "messenger_db_pool_waiting"
}
],
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 32
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
}
},
{
"id": 11,
"title": "Worker busy/capacity",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "messenger_worker_busy / messenger_worker_capacity"
}
],
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 40
},
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
}
},
{
"id": 12,
"title": "Process resident memory",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "messenger_process_memory_bytes"
}
],
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 40
},
"fieldConfig": {
"defaults": {
"unit": "bytes"
},
"overrides": []
}
},
{
"id": 13,
"title": "Process CPU (100%=1 core)",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "messenger_process_cpu_percent"
}
],
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 48
},
"fieldConfig": {
"defaults": {
"unit": "percent"
},
"overrides": []
}
},
{
"id": 14,
"title": "Sampling decisions",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "sum by (decision) (rate(messenger_http_trace_decisions_total[1m]))"
}
],
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 48
},
"fieldConfig": {
"defaults": {
"unit": "ops"
},
"overrides": []
}
},
{
"id": 15,
"title": "Collector failed span sends",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "rate(otelcol_exporter_send_failed_spans_total[1m])"
}
],
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 56
},
"fieldConfig": {
"defaults": {
"unit": "ops"
},
"overrides": []
}
},
{
"id": 99,
"title": "Structured application logs (trace_id and request_id)",
"type": "logs",
"datasource": {
"type": "loki",
"uid": "loki"
},
"targets": [
{
"refId": "A",
"expr": "{service_name=~\"messenger-api|messenger-worker\"}"
}
],
"gridPos": {
"h": 9,
"w": 24,
"x": 0,
"y": 64
}
}
]
}
@@ -0,0 +1,7 @@
apiVersion: 1
providers:
- name: messenger
folder: Messenger
type: file
options:
path: /var/lib/grafana/dashboards
@@ -0,0 +1,27 @@
apiVersion: 1
datasources:
- name: Prometheus
uid: prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
editable: false
- name: Loki
uid: loki
type: loki
access: proxy
url: http://loki:3100
editable: false
jsonData:
derivedFields:
- datasourceUid: tempo
matcherRegex: '"trace_id": "([a-f0-9]{32})"'
name: TraceID
url: '$${__value.raw}'
- name: Tempo
uid: tempo
type: tempo
access: proxy
url: http://tempo:3200
editable: false
+38
View File
@@ -0,0 +1,38 @@
global
log stdout format raw local0
maxconn 256
stats socket /tmp/haproxy.sock mode 600 level user
ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets
ssl-default-server-options ssl-min-ver TLSv1.2 no-tls-tickets
defaults
mode http
log global
timeout connect 1s
timeout client 20s
timeout server 15s
timeout check 2s
timeout queue 2s
retries 0
resolvers docker
nameserver dns 127.0.0.11:53
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 1s
hold obsolete 1s
hold nx 1s
frontend public
bind :8443 ssl crt /run/tls/identity/combined.pem
http-request set-var(txn.lb_request_id) uuid()
http-request return status 503 content-type application/json lf-string '{"error":{"code":"unavailable","message":"No ready API","request_id":"%[var(txn.lb_request_id)]"}}' hdr X-Request-Id '%[var(txn.lb_request_id)]' if { nbsrv(apis) eq 0 }
default_backend apis
backend apis
balance roundrobin
option httpchk GET /health/ready
http-check expect status 200
default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 2
server-template api 1-8 api:9443 ssl verify required ca-file /run/tls/ca.pem crt /run/tls/identity/combined.pem verifyhost api sni str(api) check check-sni api resolvers docker resolve-prefer ipv4 init-addr libc,none
+24
View File
@@ -0,0 +1,24 @@
groups:
- name: messenger-slo
rules:
- alert: MessengerHighErrorRate
expr: sum(rate(messenger_http_requests_total{status_class="5xx"}[1m])) / clamp_min(sum(rate(messenger_http_requests_total[1m])), 0.001) > 0.05
for: 10s
labels: {severity: warning}
annotations:
summary: Business HTTP 5xx ratio above 5% over 1 minute
runbook_url: file:RUNBOOK.md#api-errors
- alert: MessengerQueueLag
expr: max(messenger_queue_oldest_age_seconds) > 15 and max(messenger_queue_depth) > 0
for: 10s
labels: {severity: warning}
annotations:
summary: Outstanding attachment job age above 15 seconds
runbook_url: file:RUNBOOK.md#queue-lag
- alert: MessengerTelemetryUnavailable
expr: up{job="messenger"} == 0
for: 10s
labels: {severity: warning}
annotations:
summary: Collector metrics unavailable
runbook_url: file:RUNBOOK.md#telemetry-unavailable
+87
View File
@@ -0,0 +1,87 @@
extensions:
health_check:
endpoint: 127.0.0.1:13133
receivers:
otlp:
protocols:
http:
endpoint: 127.0.0.1:14318
processors:
memory_limiter:
check_interval: 1s
limit_mib: 160
spike_limit_mib: 32
batch:
timeout: 1s
send_batch_size: 128
send_batch_max_size: 256
exporters:
prometheus:
endpoint: 127.0.0.1:19464
metric_expiration: 15m
resource_to_telemetry_conversion:
enabled: true
otlphttp/tempo:
endpoint: https://tempo:44318
tls:
ca_file: /run/tls/ca.pem
cert_file: /run/tls/identity/cert.pem
key_file: /run/tls/identity/key.pem
timeout: 2s
sending_queue:
queue_size: 128
num_consumers: 1
retry_on_failure:
initial_interval: 1s
max_interval: 2s
max_elapsed_time: 5s
otlphttp/loki:
endpoint: https://loki:9443/otlp
timeout: 2s
sending_queue:
queue_size: 128
num_consumers: 1
retry_on_failure:
initial_interval: 1s
max_interval: 2s
max_elapsed_time: 5s
tls:
ca_file: /run/tls/ca.pem
cert_file: /run/tls/identity/cert.pem
key_file: /run/tls/identity/key.pem
service:
extensions:
- health_check
telemetry:
metrics:
readers:
- pull:
exporter:
prometheus:
host: 127.0.0.1
port: 18888
pipelines:
traces:
receivers:
- otlp
processors:
- memory_limiter
- batch
exporters:
- otlphttp/tempo
metrics:
receivers:
- otlp
processors:
- memory_limiter
- batch
exporters:
- prometheus
logs:
receivers:
- otlp
processors:
- memory_limiter
- batch
exporters:
- otlphttp/loki
+34
View File
@@ -0,0 +1,34 @@
auth_enabled: false
server:
http_listen_port: 3100
http_listen_address: 127.0.0.1
grpc_listen_address: 127.0.0.1
common:
instance_addr: 127.0.0.1
path_prefix: /loki
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
replication_factor: 1
ring:
kvstore:
store: inmemory
schema_config:
configs:
- from: 2024-01-01
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
limits_config:
allow_structured_metadata: true
retention_period: 24h
compactor:
working_directory: /loki/compactor
retention_enabled: true
delete_request_store: filesystem
analytics:
reporting_enabled: false
+26
View File
@@ -0,0 +1,26 @@
global:
scrape_interval: 5s
evaluation_interval: 5s
rule_files:
- /etc/prometheus/alerts.yaml
scrape_configs:
- job_name: messenger
static_configs:
- targets:
- collector:9464
scheme: https
tls_config:
ca_file: /run/tls/ca.pem
cert_file: /run/tls/identity/cert.pem
key_file: /run/tls/identity/key.pem
server_name: collector
- job_name: collector
static_configs:
- targets:
- collector:8888
scheme: https
tls_config:
ca_file: /run/tls/ca.pem
cert_file: /run/tls/identity/cert.pem
key_file: /run/tls/identity/key.pem
server_name: collector
+24
View File
@@ -0,0 +1,24 @@
server:
http_listen_port: 3200
http_listen_address: 127.0.0.1
grpc_listen_address: 127.0.0.1
distributor:
receivers:
otlp:
protocols:
http:
endpoint: 127.0.0.1:4318
ingester:
max_block_duration: 1m
compactor:
compaction:
block_retention: 24h
storage:
trace:
backend: local
wal:
path: /var/tempo/wal
local:
path: /var/tempo/blocks
usage_report:
reporting_enabled: false
+12
View File
@@ -0,0 +1,12 @@
ui = true
disable_mlock = true
storage "file" { path = "/bao/data" }
listener "tcp" {
address = "0.0.0.0:8200"
tls_cert_file = "/run/tls/identity/cert.pem"
tls_key_file = "/run/tls/identity/key.pem"
tls_client_ca_file = "/run/tls/ca.pem"
tls_require_and_verify_client_cert = true
tls_min_version = "tls12"
}
api_addr = "https://openbao:8200"
+6
View File
@@ -0,0 +1,6 @@
#!/bin/sh
set -eu
cp /run/tls/identity/key.pem /tmp/postgres-key.pem
chown postgres:postgres /tmp/postgres-key.pem
chmod 600 /tmp/postgres-key.pem
exec docker-entrypoint.sh "$@"
+3
View File
@@ -0,0 +1,3 @@
local all all trust
hostnossl all all 0.0.0.0/0 reject
hostssl messenger api,worker,dispatcher,messenger 0.0.0.0/0 scram-sha-256
+35
View File
@@ -0,0 +1,35 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
password_hash text NOT NULL,
created_at timestamptz NOT NULL
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
created_at timestamptz NOT NULL
);
CREATE TABLE members (
chat_id uuid NOT NULL REFERENCES chats(id),
user_id uuid NOT NULL REFERENCES users(id),
position integer NOT NULL CHECK (position >= 0),
PRIMARY KEY(chat_id,user_id),
UNIQUE(chat_id,position)
);
CREATE INDEX members_user_idx ON members(user_id,chat_id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL,
sender_id uuid NOT NULL,
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
created_at timestamptz NOT NULL,
FOREIGN KEY(chat_id,sender_id) REFERENCES members(chat_id,user_id)
);
CREATE INDEX messages_history_idx ON messages(chat_id,created_at,id);
CREATE TABLE sessions (
token_digest text PRIMARY KEY,
user_id uuid NOT NULL REFERENCES users(id),
expires_at timestamptz NOT NULL
);
CREATE INDEX sessions_expiry_idx ON sessions(expires_at);
+10
View File
@@ -0,0 +1,10 @@
CREATE TABLE message_requests (
actor_id uuid NOT NULL REFERENCES users(id),
chat_id uuid NOT NULL REFERENCES chats(id),
key text NOT NULL,
body_digest text NOT NULL,
response jsonb NOT NULL,
expires_at timestamptz NOT NULL,
PRIMARY KEY(actor_id,chat_id,key)
);
CREATE INDEX message_requests_expiry_idx ON message_requests(expires_at);
+13
View File
@@ -0,0 +1,13 @@
-- PostgreSQL text rejects U+0000, while the API accepts Unicode code points.
-- Preserve UTF-8 losslessly. API validation enforces code point limits; the DB
-- additionally bounds byte lengths, without stripping or replacing characters.
ALTER TABLE users DROP CONSTRAINT users_display_name_check;
ALTER TABLE users ALTER COLUMN display_name TYPE bytea USING convert_to(display_name,'UTF8');
ALTER TABLE users ADD CHECK (octet_length(display_name) BETWEEN 1 AND 400);
ALTER TABLE chats DROP CONSTRAINT chats_title_check;
ALTER TABLE chats ALTER COLUMN title TYPE bytea USING convert_to(title,'UTF8');
ALTER TABLE chats ADD CHECK (octet_length(title) BETWEEN 1 AND 400);
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
ALTER TABLE messages ALTER COLUMN text TYPE bytea USING convert_to(text,'UTF8');
ALTER TABLE messages ADD CHECK (octet_length(text) BETWEEN 1 AND 16000);
ALTER TABLE message_requests ALTER COLUMN response TYPE bytea USING convert_to(response::text,'UTF8');
+35
View File
@@ -0,0 +1,35 @@
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
owner_id uuid NOT NULL REFERENCES users(id),
filename text NOT NULL,
content_type text NOT NULL CHECK (content_type IN ('image/png','image/jpeg')),
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
sha256 text NOT NULL,
status text NOT NULL CONSTRAINT attachments_status_check CHECK (status IN ('queued','processing','ready','failed')),
error_code text,
created_at timestamptz NOT NULL,
updated_at timestamptz NOT NULL,
object_key text NOT NULL UNIQUE,
UNIQUE(id,chat_id)
);
CREATE TABLE message_attachments (
message_id uuid NOT NULL REFERENCES messages(id),
attachment_id uuid NOT NULL REFERENCES attachments(id),
position integer NOT NULL CHECK (position BETWEEN 0 AND 9),
PRIMARY KEY(message_id,attachment_id),
UNIQUE(message_id,position)
);
CREATE INDEX message_attachments_attachment_idx ON message_attachments(attachment_id);
CREATE TABLE upload_intents (
id uuid PRIMARY KEY,
object_key text NOT NULL UNIQUE,
created_at timestamptz NOT NULL
);
CREATE INDEX upload_intents_created_idx ON upload_intents(created_at,id);
CREATE TABLE attachment_reconcile_audit (
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
event text NOT NULL,
object_key text NOT NULL,
happened_at timestamptz NOT NULL DEFAULT now()
);
@@ -0,0 +1,3 @@
-- Empty text is valid only when the API supplies at least one attachment.
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (octet_length(text) <= 16000);
+49
View File
@@ -0,0 +1,49 @@
ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_status_check;
ALTER TABLE attachments ADD CONSTRAINT attachments_status_check
CHECK (status IN ('queued','processing','ready','failed'));
ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version > 0);
ALTER TABLE attachments ADD COLUMN variants jsonb NOT NULL DEFAULT '[]';
ALTER TABLE attachments ADD COLUMN stage_key text;
CREATE TABLE attachment_jobs (
id uuid PRIMARY KEY,
attachment_id uuid NOT NULL REFERENCES attachments(id),
pipeline_version integer NOT NULL CHECK (pipeline_version > 0),
source_key text,
state text NOT NULL CHECK (state IN ('queued','processing','ready','failed','superseded')),
attempt integer NOT NULL DEFAULT 0 CHECK (attempt >= 0),
attempt_limit integer NOT NULL CHECK (attempt_limit > 0),
lease_id uuid,
lease_until timestamptz,
available_at timestamptz NOT NULL DEFAULT now(),
error_code text,
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now(),
UNIQUE(attachment_id,pipeline_version)
);
CREATE INDEX jobs_recovery_idx ON attachment_jobs(state,lease_until);
CREATE TABLE job_outbox (
job_id uuid PRIMARY KEY REFERENCES attachment_jobs(id),
published_at timestamptz
);
CREATE TABLE job_audit (
id bigserial PRIMARY KEY,
job_id uuid NOT NULL REFERENCES attachment_jobs(id),
event text NOT NULL,
attempt integer NOT NULL,
created_at timestamptz NOT NULL DEFAULT now()
);
CREATE TABLE failed_jobs (
job_id uuid PRIMARY KEY REFERENCES attachment_jobs(id),
error_code text NOT NULL,
failed_at timestamptz NOT NULL DEFAULT now(),
replay_count integer NOT NULL DEFAULT 0
);
CREATE TABLE test_faults (
id bigserial PRIMARY KEY,
point text NOT NULL,
attachment_id uuid,
action text NOT NULL CHECK (action IN ('crash','pause','error')),
reached_at timestamptz,
released boolean NOT NULL DEFAULT false
);
+2
View File
@@ -0,0 +1,2 @@
ALTER TABLE attachment_jobs ADD COLUMN traceparent text;
ALTER TABLE attachment_jobs ADD COLUMN tracestate text;
+16
View File
@@ -0,0 +1,16 @@
CREATE TABLE login_windows (
bucket text PRIMARY KEY,
window_start timestamptz NOT NULL,
count integer NOT NULL CHECK (count > 0)
);
REVOKE CREATE ON SCHEMA public FROM PUBLIC;
GRANT USAGE ON SCHEMA public TO api,worker,dispatcher;
GRANT SELECT,INSERT,UPDATE,DELETE ON users,chats,members,messages,sessions,message_requests,attachments,message_attachments,upload_intents,attachment_reconcile_audit,attachment_jobs,job_outbox,job_audit,failed_jobs,login_windows TO api;
GRANT SELECT ON attachments,attachment_jobs,job_outbox,failed_jobs TO worker,dispatcher;
GRANT UPDATE(status,error_code,object_key,variants,updated_at,stage_key) ON attachments TO worker,dispatcher;
GRANT UPDATE(state,attempt,lease_id,lease_until,available_at,error_code,updated_at) ON attachment_jobs TO worker,dispatcher;
GRANT UPDATE(published_at) ON job_outbox TO worker,dispatcher;
GRANT INSERT ON job_audit TO worker,dispatcher;
GRANT INSERT,UPDATE ON failed_jobs TO worker,dispatcher;
GRANT USAGE,SELECT ON SEQUENCE job_audit_id_seq TO api,worker,dispatcher;
GRANT USAGE,SELECT ON SEQUENCE attachment_reconcile_audit_id_seq TO api;
+4
View File
@@ -0,0 +1,4 @@
ALTER TABLE attachment_jobs ADD COLUMN accepted_at timestamptz;
UPDATE attachment_jobs SET accepted_at=created_at;
ALTER TABLE attachment_jobs ALTER COLUMN accepted_at SET NOT NULL;
ALTER TABLE attachment_jobs ALTER COLUMN accepted_at SET DEFAULT clock_timestamp();
+1
View File
@@ -0,0 +1 @@
CREATE INDEX login_windows_expiry_idx ON login_windows(window_start);
+32
View File
@@ -0,0 +1,32 @@
[project]
name = "mtusi-messenger"
version = "1.0.0"
requires-python = ">=3.12,<3.13"
dependencies = [
"fastapi",
"uvicorn",
"psycopg[binary,pool]",
"redis",
"argon2-cffi",
"boto3>=1.43,<2",
"pillow>=13.0.0,<14",
"opentelemetry-sdk==1.45.2",
"opentelemetry-exporter-otlp-proto-http==1.45.2",
"psutil>=7,<8",
]
[dependency-groups]
dev = ["pytest", "httpx", "ruff", "openapi-spec-validator", "jsonschema"]
[tool.ruff]
target-version = "py312"
line-length = 100
include = ["src/**/*.py", "student-tests/**/*.py", "research/**/*.py"]
[tool.pytest.ini_options]
testpaths = ["student-tests"]
pythonpath = ["src", "tests"]
[tool.ruff.lint]
select = ["E4", "E7", "E9", "F", "I", "UP", "B"]
ignore = ["B008"] # FastAPI dependencies are declared in endpoint signatures.
+115
View File
@@ -0,0 +1,115 @@
#!/usr/bin/env python3
"""Shared-replica login window and real weak-hash rehash regression."""
import base64
import json
import subprocess
import sys
import time
from pathlib import Path
from argon2 import PasswordHasher
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, check # noqa: E402
def sql(query):
result = subprocess.run(
[
"docker",
"compose",
"exec",
"-T",
"postgres",
"psql",
"-U",
"messenger",
"-d",
"messenger",
"-At",
"-v",
"ON_ERROR_STOP=1",
],
cwd=ROOT,
input=query,
text=True,
capture_output=True,
check=True,
)
return result.stdout.strip()
def main():
c = Client("https://localhost:8447", str(ROOT / "secrets/ca.pem"))
actor = c.user("rehash")
password = actor["registration"]["password"]
weak = PasswordHasher(time_cost=1, memory_cost=8192, parallelism=1).hash(password)
sql("UPDATE users SET password_hash='" + weak + "' WHERE id='" + actor["id"] + "';")
authorization = (
"Basic "
+ base64.b64encode((actor["registration"]["username"] + ":" + password).encode()).decode()
)
c.call(
"POST",
"/api/v1/auth/sessions",
201,
headers={"Authorization": authorization},
schema="Session",
)
upgraded = sql("SELECT password_hash FROM users WHERE id='" + actor["id"] + "';")
check(
upgraded != weak and "$m=65536,t=2,p=1$" in upgraded,
"successful login did not upgrade hash",
)
time.sleep(10.2)
wrong = (
"Basic "
+ base64.b64encode((actor["registration"]["username"] + ":incorrect").encode()).decode()
)
rows = []
for _ in range(12):
response = c.call(
"POST", "/api/v1/auth/sessions", expected=[401, 429], headers={"Authorization": wrong}
)
rows.append(
{
"status": response["status"],
"instance": response["headers"]["X-Instance-Id"],
"retry_after": response["headers"].get("Retry-After"),
"milliseconds": round(response["elapsed"] * 1000, 3),
}
)
sql(
"UPDATE sessions SET expires_at=now()-interval '1 second' WHERE user_id='"
+ actor["id"]
+ "';"
)
expiry_rows = []
for _ in range(4):
expired = c.call("GET", "/api/v1/users/me", expected=401, actor=actor)
expiry_rows.append(
{"status": expired["status"], "instance": expired["headers"]["X-Instance-Id"]}
)
result = {
"weak_hash_upgraded": True,
"expired_session_requests": expiry_rows,
"expired_session_denied_both_replicas": len({r["instance"] for r in expiry_rows}) >= 2,
"configured_memory_kib": 65536,
"time_cost": 2,
"parallelism": 1,
"requests": rows,
"distinct_api_instances": len({r["instance"] for r in rows}),
"passed": len({r["instance"] for r in expiry_rows}) >= 2
and sum(r["status"] == 401 for r in rows) == 10
and all(r["status"] == 429 and r["retry_after"] == "10" for r in rows[10:])
and len({r["instance"] for r in rows}) >= 2,
}
(ROOT / "evidence/auth-budget.json").write_text(json.dumps(result, indent=2) + "\n")
check(result["passed"], "shared login window failed")
print(json.dumps({k: v for k, v in result.items() if k != "requests"}))
if __name__ == "__main__":
main()
+108
View File
@@ -0,0 +1,108 @@
#!/usr/bin/env python3
"""Accepted jobs survive worker identity revoke + working-secret rotation."""
import hashlib
import json
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, check # noqa: E402
def docker(*args):
return subprocess.check_output(["docker", "compose", *args], cwd=ROOT, text=True).strip()
def images():
ids = docker("ps", "-q", "api", "worker").split()
return sorted(
set(
subprocess.check_output(
["docker", "inspect", "--format", "{{.Image}}", *ids], text=True
).split()
)
)
def main():
c = Client("https://localhost:8447", str(ROOT / "secrets/ca.pem"))
alice, bob, _, chat = c.bootstrap()
before = images()
docker("stop", "worker")
accepted = [c.attachment(alice, chat) for _ in range(3)]
start = time.monotonic()
rotate = subprocess.run(
[sys.executable, "scripts/rotate_identity.py"], cwd=ROOT, text=True, capture_output=True
)
check(rotate.returncode == 0, "operator rotation failed; inspect private operator execution")
errors = 0
deadline = time.monotonic() + 90
pending = {a[0]["id"] for a in accepted}
while pending and time.monotonic() < deadline:
for att, _ in accepted:
if att["id"] not in pending:
continue
result = c.call(
"GET",
"/api/v1/attachments/" + att["id"],
expected=[200, 503],
actor=alice,
schema=None,
)
if result["status"] == 503:
errors += 1
else:
body = json.loads(result["bytes"])
check(body["status"] != "failed", "accepted task became failed")
if body["status"] == "ready":
pending.remove(att["id"])
time.sleep(0.2)
check(not pending, "accepted jobs did not recover")
rows = []
for att, original in accepted:
downloaded = c.call("GET", "/api/v1/attachments/" + att["id"] + "/content", actor=bob)[
"bytes"
]
thumbnail = c.call(
"GET", "/api/v1/attachments/" + att["id"] + "/content?variant=thumbnail", actor=bob
)["bytes"]
sha = hashlib.sha256(original).hexdigest()
rows.append(
{
"attachment_id": att["id"],
"original_sha256": sha,
"download_sha256": hashlib.sha256(downloaded).hexdigest(),
"thumbnail_sha256": hashlib.sha256(thumbnail).hexdigest(),
"thumbnail_nonempty": bool(thumbnail),
"accepted_original_sha_matches": sha == att["sha256"],
}
)
after = images()
result = {
"accepted_jobs": 3,
"ready_jobs": len(rows),
"recovery_seconds": round(time.monotonic() - start, 3),
"observed_api_503": errors,
"images_before": before,
"images_after": after,
"image_unchanged": before == after,
"rows": rows,
"passed": before == after
and all(
r["original_sha256"] == r["download_sha256"]
and r["thumbnail_nonempty"]
and r["accepted_original_sha_matches"]
for r in rows
),
}
(ROOT / "evidence/compromise-recovery.json").write_text(json.dumps(result, indent=2) + "\n")
check(result["passed"], "recovery invariant failed")
print(json.dumps({k: v for k, v in result.items() if k != "rows"}))
if __name__ == "__main__":
main()
+71
View File
@@ -0,0 +1,71 @@
#!/usr/bin/env python3
"""Fail closed after the service-auth lease expires, then bounded recovery."""
import json
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, check # noqa: E402
def operator(action):
subprocess.run(
[sys.executable, "scripts/bao_operator.py", action],
cwd=ROOT,
check=True,
capture_output=True,
)
def main():
c = Client("https://localhost:8447", str(ROOT / "secrets/ca.pem"))
actor = c.user("sealed")
c.call("GET", "/api/v1/users/me", actor=actor, schema="User")
start = time.monotonic()
operator("seal")
rows = []
try:
while time.monotonic() - start < 33:
live = c.call("GET", "/health/live", schema="Health")
response = c.call(
"GET", "/api/v1/users/me", expected=[200, 503], actor=actor
)
rows.append(
{
"seconds": round(time.monotonic() - start, 3),
"live": live["status"],
"business": response["status"],
}
)
time.sleep(1)
check(rows[-1]["business"] == 503, "service used credentials beyond lease")
finally:
unseal_start = time.monotonic()
operator("unseal")
for _ in range(20):
response = c.call("GET", "/health/ready", expected=[200, 503])
if response["status"] == 200:
break
time.sleep(0.25)
c.call("GET", "/api/v1/users/me", actor=actor, schema="User")
result = {
"lease_seconds": 30,
"local_cache_max_seconds": 29,
"rows": rows,
"first_business_503_seconds": next(
(r["seconds"] for r in rows if r["business"] == 503), None
),
"recovery_seconds": round(time.monotonic() - unseal_start, 3),
"passed": rows[-1]["business"] == 503 and response["status"] == 200,
}
(ROOT / "evidence/sealed-expiry.json").write_text(json.dumps(result, indent=2) + "\n")
check(result["passed"], "recovery failed")
print(json.dumps({k: v for k, v in result.items() if k != "rows"}))
if __name__ == "__main__":
main()
+115
View File
@@ -0,0 +1,115 @@
#!/usr/bin/env python3
"""Explicit operator init/unseal/configuration. Never prints credential values."""
import argparse
import json
import os
import ssl
import time
import urllib.error
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
PRIVATE = ROOT / "secrets"
OPERATOR = ROOT / "operator-private"
def call(path, data=None, *, method=None, token=True):
ctx = ssl.create_default_context(cafile=str(PRIVATE / "ca.pem"))
ctx.load_cert_chain(PRIVATE / "pki/operator/cert.pem", PRIVATE / "pki/operator/key.pem")
headers = {"Content-Type": "application/json"}
if token:
headers["X-Vault-Token"] = (OPERATOR / "root-token").read_text().strip()
req = urllib.request.Request(
os.getenv("BAO_OPERATOR_ADDR", "https://localhost:18207") + "/v1/" + path,
data=json.dumps(data).encode() if data is not None else None,
headers=headers,
method=method,
)
with urllib.request.urlopen(req, context=ctx, timeout=5) as response:
raw = response.read()
return json.loads(raw) if raw else {}
def unseal():
for share in json.loads((OPERATOR / "unseal-shares.json").read_text())[:2]:
result = call("sys/unseal", {"key": share}, token=False)
if result["sealed"]:
raise RuntimeError("OpenBao remained sealed")
def configure():
try:
call("sys/mounts/secret", {"type": "kv", "options": {"version": "2"}})
except urllib.error.HTTPError as exc:
if exc.code != 400:
raise
try:
call("sys/auth/cert", {"type": "cert"})
except urllib.error.HTTPError as exc:
if exc.code != 400:
raise
seed = json.loads((PRIVATE / "bao_seed.json").read_text())
for role, values in seed.items():
policy = (
'path "secret/data/services/'
+ role
+ '" { capabilities = ["read"] }\npath "auth/token/lookup-self" { capabilities = ["read"] }\npath "auth/token/renew-self" { capabilities = ["update"] }\n'
)
call("sys/policies/acl/" + role, {"policy": policy}, method="PUT")
call(
"auth/cert/certs/" + role,
{
"display_name": role,
"certificate": (PRIVATE / "pki" / role / "cert.pem").read_text(),
"allowed_common_names": role,
"token_policies": [role],
"token_no_default_policy": True,
"token_ttl": "30s",
"token_max_ttl": "60s",
},
)
call("secret/data/services/" + role, {"data": values})
try:
call("sys/audit/file", {"type": "file", "options": {"file_path": "/bao/data/audit.jsonl"}})
except urllib.error.HTTPError as exc:
if exc.code != 400:
raise
def main():
parser = argparse.ArgumentParser()
parser.add_argument("action", choices=["init", "unseal", "configure", "seal"])
args = parser.parse_args()
os.umask(0o077)
if args.action == "init":
for _attempt in range(30):
try:
status = call("sys/init", token=False)
break
except (OSError, urllib.error.URLError):
time.sleep(1)
else:
raise RuntimeError("OpenBao TLS endpoint unavailable")
if not status["initialized"]:
result = call(
"sys/init", {"secret_shares": 3, "secret_threshold": 2}, method="PUT", token=False
)
(OPERATOR / "root-token").write_text(result["root_token"])
(OPERATOR / "unseal-shares.json").write_text(json.dumps(result["keys_base64"]))
(OPERATOR / "root-token").chmod(0o600)
(OPERATOR / "unseal-shares.json").chmod(0o600)
unseal()
configure()
elif args.action == "unseal":
unseal()
elif args.action == "configure":
configure()
else:
call("sys/seal", {}, method="PUT")
print("OpenBao operator action complete; credential values were not printed.")
if __name__ == "__main__":
main()
+340
View File
@@ -0,0 +1,340 @@
#!/usr/bin/env python3
"""Idempotent local PKI/secrets bootstrap. Operator material never enters containers."""
import json
import os
import secrets
import subprocess
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
PRIVATE = ROOT / "secrets"
OPERATOR = ROOT / "operator-private"
ROLES = (
"api",
"worker",
"dispatcher",
"lb",
"postgres",
"storage",
"broker",
"cache",
"collector",
"prometheus",
"loki",
"tempo",
"grafana",
"openbao",
"operator",
)
def write(path, text, mode=0o644):
if not path.exists():
path.write_text(text)
path.chmod(mode)
def run(*args):
subprocess.run(
["openssl", *args], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
)
def certificate(name, replace=False):
dest = PRIVATE / "pki" / name
dest.mkdir(parents=True, exist_ok=True)
dest.chmod(0o755)
if (dest / "cert.pem").exists() and not replace:
valid = subprocess.run(
["openssl", "x509", "-in", str(dest / "cert.pem"), "-checkend", "3600", "-noout"],
capture_output=True,
)
if valid.returncode == 0:
return
key, csr = dest / "key.pem", OPERATOR / (name + ".csr")
run("genpkey", "-algorithm", "EC", "-pkeyopt", "ec_paramgen_curve:P-256", "-out", str(key))
run("req", "-new", "-key", str(key), "-subj", f"/CN={name}", "-out", str(csr))
ext = OPERATOR / (name + ".ext")
names = f"DNS:{name}"
if name in {
"lb",
"api",
"postgres",
"broker",
"cache",
"collector",
"prometheus",
"loki",
"tempo",
"openbao",
"grafana",
"storage",
}:
names += ",DNS:localhost,IP:127.0.0.1"
ext.write_text(
f"subjectAltName={names}\nbasicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth,clientAuth\n"
)
run(
"x509",
"-req",
"-in",
str(csr),
"-CA",
str(OPERATOR / "ca.pem"),
"-CAkey",
str(OPERATOR / "ca-key.pem"),
"-set_serial",
str(secrets.randbits(127) or 1),
"-days",
"2",
"-extfile",
str(ext),
"-out",
str(dest / "cert.pem"),
)
key.chmod(0o644)
(dest / "combined.pem").write_text((dest / "cert.pem").read_text() + key.read_text())
(dest / "combined.pem").chmod(0o644)
def main():
os.umask(0o077)
for folder in (PRIVATE, OPERATOR):
folder.mkdir(exist_ok=True)
folder.chmod(0o700)
if not (OPERATOR / "ca-key.pem").exists():
run(
"genpkey",
"-algorithm",
"EC",
"-pkeyopt",
"ec_paramgen_curve:P-256",
"-out",
str(OPERATOR / "ca-key.pem"),
)
run(
"req",
"-x509",
"-new",
"-key",
str(OPERATOR / "ca-key.pem"),
"-subj",
"/CN=MTUSI Lab7 local CA",
"-days",
"365",
"-addext",
"basicConstraints=critical,CA:TRUE",
"-addext",
"keyUsage=critical,keyCertSign,cRLSign",
"-out",
str(OPERATOR / "ca.pem"),
)
(PRIVATE / "ca.pem").write_text((OPERATOR / "ca.pem").read_text())
(PRIVATE / "ca.pem").chmod(0o644)
(PRIVATE / "pki").mkdir(exist_ok=True)
(PRIVATE / "pki").chmod(0o755)
for name in ROLES:
certificate(name)
write(PRIVATE / "revoked.json", "[]")
for name in (
"postgres_password",
"api_db_password",
"worker_db_password",
"dispatcher_db_password",
"cache_password",
"cache_worker_password",
"cache_dispatcher_password",
"broker_worker_password",
"broker_dispatcher_password",
"broker_admin_password",
"grafana_password",
):
write(PRIVATE / name, secrets.token_urlsafe(32))
for role in ("api", "worker", "admin", "study_wide"):
write(PRIVATE / (role + "_s3_access"), role + "-" + secrets.token_hex(12))
write(PRIVATE / (role + "_s3_secret"), secrets.token_urlsafe(32))
def value(name):
return (PRIVATE / name).read_text().strip()
identities = []
for role in ("api", "worker", "admin", "study_wide"):
identities.append(
{
"name": role,
"credentials": [
{
"accessKey": value(role + "_s3_access"),
"secretKey": value(role + "_s3_secret"),
}
],
"actions": ["Admin", "Read", "Write", "List"]
if role == "admin"
else (["Read:security-study", "Write:security-study", "List:security-study"] if role == "study_wide" else ["Read:messenger", "Write:messenger", "List:messenger"]),
}
)
(PRIVATE / "s3_config").write_text(json.dumps({"identities": identities}))
(PRIVATE / "s3_config").chmod(0o644)
write(
PRIVATE / "cache_config",
'bind 127.0.0.1\nport 6379\nprotected-mode yes\nsave ""\nmaxmemory 64mb\nmaxmemory-policy allkeys-lru\nuser default off\nuser api on >'
+ value("cache_password")
+ " ~user:v1:* +get +setex +ping +hello +client|setinfo\nuser worker on >"
+ value("cache_worker_password")
+ " +ping +hello +client|setinfo\nuser dispatcher on >"
+ value("cache_dispatcher_password")
+ " +ping +hello +client|setinfo\n",
)
write(
PRIVATE / "broker_config",
'bind 127.0.0.1\nport 6379\nprotected-mode yes\nappendonly yes\nappendfsync always\nsave ""\ndir /data\nmaxmemory 128mb\nmaxmemory-policy noeviction\nuser default off\nuser operator on >'
+ value("broker_admin_password")
+ " ~* &* +@all\nuser dispatcher on >"
+ value("broker_dispatcher_password")
+ " ~attachment-jobs:v1 +xadd +ping +hello +client|setinfo\nuser worker on >"
+ value("broker_worker_password")
+ " ~attachment-jobs:v1 +xgroup|create +xreadgroup +xautoclaim +xack +xdel +ping +hello +client|setinfo\n",
)
pg = "postgresql://{role}:{password}@postgres:5432/messenger?sslmode=verify-full&sslrootcert=/run/tls/ca.pem"
values = {}
for role in ("api", "worker", "dispatcher"):
values[role] = {
"DATABASE_URL": pg.format(role=role, password=value(role + "_db_password")),
"CACHE_URL": "rediss://"
+ role
+ ":"
+ value("cache_password" if role == "api" else "cache_" + role + "_password")
+ "@cache:6380/0?ssl_cert_reqs=required&ssl_ca_certs=/run/tls/ca.pem&ssl_certfile=/run/tls/identity/cert.pem&ssl_keyfile=/run/tls/identity/key.pem",
}
if role in {"worker", "dispatcher"}:
values[role]["BROKER_URL"] = (
"rediss://"
+ role
+ ":"
+ value("broker_" + role + "_password")
+ "@broker:6380/0?ssl_cert_reqs=required&ssl_ca_certs=/run/tls/ca.pem&ssl_certfile=/run/tls/identity/cert.pem&ssl_keyfile=/run/tls/identity/key.pem"
)
if role in {"api", "worker"}:
values[role].update(
S3_ACCESS_KEY=value(role + "_s3_access"), S3_SECRET_KEY=value(role + "_s3_secret")
)
write(PRIVATE / "bao_seed.json", json.dumps(values))
write(
PRIVATE / "admin_database_url",
pg.format(role="messenger", password=value("postgres_password")),
)
write(PRIVATE / "admin_s3_access_key", value("admin_s3_access"))
write(PRIVATE / "admin_s3_secret_key", value("admin_s3_secret"))
write(
PRIVATE / "db_roles.sql",
"\n".join(
f"CREATE ROLE {role} LOGIN PASSWORD '{value(role + '_db_password')}';"
for role in ("api", "worker", "dispatcher")
),
)
gateways = {
"api": {"port": 9443, "upstream": 8080, "allowed": ["lb"]},
"broker": {
"port": 6380,
"upstream": 6379,
"allowed": ["worker", "dispatcher", "operator"],
},
"cache": {
"port": 6380,
"upstream": 6379,
"allowed": ["api", "worker", "dispatcher", "operator"],
},
"storage": {
"port": 8443,
"upstream": 8333,
"allowed": ["api", "worker", "operator"],
"http": True,
"s3": True,
},
"collector": [
{
"port": 4318,
"upstream": 14318,
"allowed": ["api", "worker", "dispatcher", "operator"],
},
{"port": 9464, "upstream": 19464, "allowed": ["prometheus", "operator"]},
{"port": 8888, "upstream": 18888, "allowed": ["prometheus", "operator"]},
],
"prometheus": {
"http": True,
"backend": "prometheus",
"port": 9443,
"upstream": 9090,
"allowed": ["grafana", "operator"],
},
"loki": {
"http": True,
"backend": "loki",
"port": 9443,
"upstream": 3100,
"allowed": ["collector", "grafana", "operator"],
},
"tempo": [
{
"port": 9443,
"upstream": 3200,
"allowed": ["grafana", "operator"],
"http": True,
"backend": "tempo",
},
{"port": 44318, "upstream": 4318, "allowed": ["collector", "operator"]},
],
}
for role, configs in gateways.items():
for item in configs if isinstance(configs, list) else [configs]:
item.update(
cert="/run/tls/identity/cert.pem",
key="/run/tls/identity/key.pem",
ca="/run/tls/ca.pem",
)
(PRIVATE / (role + "_gateway.json")).write_text(json.dumps(configs))
(PRIVATE / (role + "_gateway.json")).chmod(0o644)
provisioning = PRIVATE / "grafana-provisioning" / "datasources"
provisioning.mkdir(parents=True, exist_ok=True)
provisioning.parent.chmod(0o755)
provisioning.chmod(0o755)
datasources = []
for name in ("prometheus", "loki", "tempo"):
item = {
"name": name.title(),
"uid": name,
"type": name,
"access": "proxy",
"url": f"https://{name}:9443",
"editable": False,
"jsonData": {"tlsAuth": True, "tlsAuthWithCACert": True, "serverName": name},
"secureJsonData": {
"tlsCACert": (PRIVATE / "ca.pem").read_text(),
"tlsClientCert": (PRIVATE / "pki/grafana/cert.pem").read_text(),
"tlsClientKey": (PRIVATE / "pki/grafana/key.pem").read_text(),
},
}
if name == "prometheus":
item["isDefault"] = True
if name == "loki":
item["jsonData"]["derivedFields"] = [
{
"datasourceUid": "tempo",
"matcherRegex": '"trace_id": "([a-f0-9]{32})"',
"name": "TraceID",
"url": "$${__value.raw}",
}
]
datasources.append(item)
(provisioning / "datasources.yaml").write_text(
json.dumps({"apiVersion": 1, "datasources": datasources})
)
(provisioning / "datasources.yaml").chmod(0o644)
if not (ROOT / ".env").exists():
(ROOT / ".env").write_text("PUBLIC_PORT=8447\nSESSION_TTL_SECONDS=3600\n")
print("Local PKI and scoped credential material ready; operator CA stays outside containers.")
if __name__ == "__main__":
main()
+9
View File
@@ -0,0 +1,9 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
# Namespace-sharing gateways are stopped before their backing services restart.
# Unseal is an explicit operator action; shares never enter application containers.
docker compose stop
docker compose up -d postgres openbao cache cache-tls broker broker-tls storage s3 storage-tls
python3 scripts/bao_operator.py unseal
docker compose up -d --wait
+91
View File
@@ -0,0 +1,91 @@
#!/usr/bin/env python3
"""Operator-driven worker revoke/cert renewal and DB credential rotation, no rebuild."""
import importlib.util
import json
import secrets
import ssl
import subprocess
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def main():
spec = importlib.util.spec_from_file_location("bootstrap", ROOT / "scripts/bootstrap.py")
bootstrap = importlib.util.module_from_spec(spec)
spec.loader.exec_module(bootstrap)
old = ROOT / "secrets/pki/worker"
cert = ssl._ssl._test_decode_cert(str(old / "cert.pem"))
snapshot = ROOT / "operator-private/revoked-worker"
snapshot.mkdir(exist_ok=True, mode=0o700)
for name in ("cert.pem", "key.pem"):
(snapshot / name).write_bytes((old / name).read_bytes())
(snapshot / name).chmod(0o600)
seed_before = json.loads((ROOT / "secrets/bao_seed.json").read_text())
(snapshot / "material.json").write_text(json.dumps(seed_before["worker"]))
(snapshot / "material.json").chmod(0o644)
# The containing operator-private directory stays 0700 on the host. This
# disposable old-material mount is used only by the explicit negative probe.
for name in ("cert.pem", "key.pem"):
(snapshot / name).chmod(0o644)
snapshot.chmod(0o755)
revoked = ROOT / "secrets/revoked.json"
serials = json.loads(revoked.read_text())
serials.append(cert["serialNumber"].lstrip("0").lower())
revoked.write_text(json.dumps(sorted(set(serials))))
subprocess.run(
["docker", "compose", "stop", "worker"], cwd=ROOT, check=True, capture_output=True
)
bootstrap.certificate("worker", replace=True)
password = secrets.token_urlsafe(32)
# psql receives SQL through stdin, never a process argument/output containing
# credentials. The operator/admin role is confined to this explicit command.
sql = f"ALTER ROLE worker PASSWORD '{password}'; SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE usename='worker' AND pid<>pg_backend_pid();"
subprocess.run(
[
"docker",
"compose",
"exec",
"-T",
"postgres",
"psql",
"-U",
"messenger",
"-d",
"messenger",
"-v",
"ON_ERROR_STOP=1",
],
cwd=ROOT,
input=sql,
text=True,
check=True,
capture_output=True,
)
path = ROOT / "secrets/worker_db_password"
path.write_text(password)
seed_path = ROOT / "secrets/bao_seed.json"
seed = json.loads(seed_path.read_text())
seed["worker"]["DATABASE_URL"] = (
"postgresql://worker:"
+ password
+ "@postgres:5432/messenger?sslmode=verify-full&sslrootcert=/run/tls/ca.pem"
)
seed_path.write_text(json.dumps(seed))
subprocess.run(
["python3", "scripts/bao_operator.py", "configure"],
cwd=ROOT,
check=True,
capture_output=True,
)
subprocess.run(
["docker", "compose", "start", "worker"], cwd=ROOT, check=True, capture_output=True
)
print(
"Worker identity revoked and replaced; database credential rotated; worker resumed without image rebuild."
)
if __name__ == "__main__":
main()
+217
View File
@@ -0,0 +1,217 @@
#!/usr/bin/env python3
"""Explicit operator probes; test identities never become application mounts."""
import argparse
import json
import os
import subprocess
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def run(args, *, output=None):
result = subprocess.run(args, cwd=ROOT, check=True, capture_output=True, text=True)
if output:
(ROOT / "evidence" / output).write_text(result.stdout)
else:
print(result.stdout, end="")
return result.stdout
def foreign_certificate():
path = ROOT / "secrets/pki/foreign"
path.mkdir(exist_ok=True, mode=0o755)
run(
[
"openssl",
"req",
"-x509",
"-newkey",
"ec",
"-pkeyopt",
"ec_paramgen_curve:P-256",
"-nodes",
"-keyout",
str(path / "key.pem"),
"-out",
str(path / "cert.pem"),
"-days",
"1",
"-subj",
"/CN=foreign",
"-addext",
"subjectAltName=DNS:lb,DNS:worker,DNS:api",
]
)
for item in path.iterdir():
item.chmod(0o644)
def audit_container(module, *, material=False):
command = [
"docker",
"compose",
"run",
"--rm",
"--no-deps",
"-e",
"BAO_ROLE=",
"-e",
"OTEL_ENABLED=0",
"-v",
str(ROOT / "secrets/pki") + ":/run/audit-identities:ro",
]
if material:
# Only selected synthetic/admin S3 credential files, never Bao root/shares.
path = ROOT / "operator-private/s3-study-material"
path.mkdir(exist_ok=True, mode=0o755)
for name in (
"admin_s3_access",
"admin_s3_secret",
"study_wide_s3_access",
"study_wide_s3_secret",
"worker_s3_access",
"worker_s3_secret",
):
(path / name).write_bytes((ROOT / "secrets" / name).read_bytes())
(path / name).chmod(0o644)
command += ["-v", str(path) + ":/run/audit-material:ro"]
command += ["api", "python", "-m", module]
return command
def main():
parser = argparse.ArgumentParser()
parser.add_argument("probe", choices=["transport", "policies", "baseline", "revoked", "kdf"])
args = parser.parse_args()
os.umask(0o077)
if args.probe == "transport":
foreign_certificate()
run(audit_container("messenger.transport_probe"), output="transport-matrix.json")
elif args.probe == "policies":
run(
[
"docker",
"compose",
"run",
"--rm",
"--no-deps",
"worker",
"python",
"-m",
"messenger.security_probe",
],
output="worker-policy-final.json",
)
elif args.probe == "baseline":
run(
[
"docker",
"compose",
"run",
"--rm",
"--no-deps",
"migrate",
"python",
"-m",
"messenger.baseline_db_probe",
],
output="baseline-db.json",
)
run(
[
"docker",
"compose",
"exec",
"-T",
"broker",
"valkey-server",
"--port",
"6378",
"--bind",
"127.0.0.1",
"--save",
"",
"--appendonly",
"no",
"--daemonize",
"yes",
"--pidfile",
"/tmp/security-study.pid",
"--logfile",
"/tmp/security-study.log",
]
)
broker = run(["docker", "compose", "ps", "-q", "broker"]).strip()
config = json.loads(
subprocess.check_output(
["docker", "compose", "config", "--format", "json"], cwd=ROOT, text=True
)
)
app_image = config["services"]["api"]["image"]
run(
[
"docker",
"run",
"--rm",
"--network",
"container:" + broker,
"--read-only",
"--cap-drop",
"ALL",
"--security-opt",
"no-new-privileges",
"--memory",
"128m",
"--cpus",
"0.5",
app_image,
"python",
"-m",
"messenger.baseline_queue_probe",
],
output="baseline-queue.json",
)
run(
audit_container("messenger.baseline_s3_probe", material=True), output="baseline-s3.json"
)
elif args.probe == "revoked":
run(
[
"docker",
"compose",
"run",
"--rm",
"--no-deps",
"-v",
str(ROOT / "operator-private/revoked-worker") + ":/run/old-worker:ro",
"worker",
"python",
"-m",
"messenger.revoked_probe",
],
output="revoked-identity.json",
)
else:
run(
[
"docker",
"compose",
"run",
"--rm",
"--no-deps",
"-e",
"OTEL_ENABLED=0",
"worker",
"python",
"-m",
"messenger.kdf_probe",
],
output="argon2-budget.json",
)
print(json.dumps({"probe": args.probe, "passed": True}))
if __name__ == "__main__":
main()
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
python3 scripts/stop_one.py
+22
View File
@@ -0,0 +1,22 @@
"""Map a process alias to its real container, then leave exactly it stopped."""
import json
import os
import subprocess
from pathlib import Path
wanted = os.environ["TARGET_INSTANCE_ID"]
ids = subprocess.check_output(["docker", "compose", "ps", "-q", "api"], text=True).split()
for cid in ids:
result = subprocess.run(
["docker", "exec", cid, "python", "-c",
"import urllib.request; r=urllib.request.urlopen('http://127.0.0.1:8080/health/live'); print(r.headers['X-Instance-Id'])"],
text=True, capture_output=True, check=False,
)
if result.returncode == 0 and result.stdout.strip() == wanted:
subprocess.run(["docker", "stop", "--time", "8", cid], check=True, stdout=subprocess.DEVNULL)
receipt = {"stopped_container": cid, "instance_id": wanted}
Path("evidence/last-stopped.json").write_text(json.dumps(receipt))
print(json.dumps(receipt))
break
else:
raise SystemExit("Target instance not found among this project's API containers")
+1
View File
@@ -0,0 +1 @@
"""Учебный REST-мессенджер."""
+12
View File
@@ -0,0 +1,12 @@
import os
import uvicorn
uvicorn.run(
"messenger.app:app",
host=os.getenv("APP_HOST", "0.0.0.0"),
port=int(os.getenv("APP_PORT", "8080")),
workers=1,
access_log=False,
timeout_graceful_shutdown=5,
)
+384
View File
@@ -0,0 +1,384 @@
import base64
import binascii
import json
import logging
import os
import re
import time
from contextlib import asynccontextmanager
from threading import BoundedSemaphore
from uuid import uuid4
from fastapi import Depends, FastAPI, Request, Response
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse
from opentelemetry import propagate
from opentelemetry.trace import SpanKind, Status, StatusCode
from starlette.concurrency import run_in_threadpool
from starlette.exceptions import HTTPException
from . import telemetry
from .attachments import AttachmentService
from .domain import Failure, boundary, identifier, page
from .models import CreateChat, CreateMessage, CreateUser
from .postgres import PostgresStore
from .secrets_provider import refresh
from .storage import ObjectStorage
logger = logging.getLogger("messenger")
logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO"), format="%(message)s")
INSTANCE_ID = telemetry.INSTANCE_ID
CODES = {
400: "invalid_request",
401: "unauthorized",
404: "not_found",
409: "conflict",
413: "payload_too_large",
415: "unsupported_media_type",
429: "rate_limited",
503: "unavailable",
500: "internal_error",
}
@asynccontextmanager
async def lifespan(app):
app.state.telemetry = telemetry.setup("messenger-api")
app.state.store = PostgresStore()
app.state.telemetry.start_background(app.state.store)
app.state.storage = ObjectStorage()
app.state.attachments = AttachmentService(app.state.store, app.state.storage)
app.state.upload_slots = BoundedSemaphore(1)
logger.info(json.dumps({"event": "started", "storage": "postgres"}))
yield
logger.info(json.dumps({"event": "stopped"}))
app.state.telemetry.close()
app.state.storage.close()
app.state.store.close()
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
def error(request, status, message, headers=None):
return JSONResponse(
{
"error": {
"code": CODES.get(status, "invalid_request"),
"message": message,
"request_id": request.state.request_id,
}
},
status_code=status,
headers=headers,
)
async def invalid_json_encoding(request: Request):
path = request.url.path
if request.method != "POST" or not (
path in {"/api/v1/users", "/api/v1/chats"}
or re.fullmatch(r"/api/v1/chats/[^/]+/messages", path)
):
return False
raw = await request.body()
try:
raw.decode("utf-8")
except UnicodeDecodeError:
return True
# Literal NUL is invalid JSON; it also prevents json.loads(bytes) from
# autodetecting BOM-less UTF-16/32. Escaped JSON \u0000 remains valid.
return b"\x00" in raw
@app.middleware("http")
async def request_log(request: Request, call_next):
request.state.request_id = str(uuid4())
start = time.perf_counter()
observed = not request.url.path.startswith("/health/")
if not observed:
try:
if request.url.path != "/health/live":
await run_in_threadpool(refresh)
response = await call_next(request)
except Failure as exc:
response = error(request, exc.status, exc.message, exc.headers)
response.headers["X-Request-Id"] = request.state.request_id
response.headers["X-Instance-Id"] = INSTANCE_ID
return response
method = (
request.method
if request.method in {"GET", "POST", "DELETE", "PUT", "PATCH", "HEAD", "OPTIONS"}
else "OTHER"
)
context = propagate.extract(dict(request.headers))
with telemetry.operation("HTTP " + method, context=context, kind=SpanKind.SERVER) as span:
try:
await run_in_threadpool(refresh)
if (
request.method == "POST"
and request.url.path == "/api/v1/auth/sessions"
and await request.body()
):
response = error(request, 400, "Вход не принимает тело запроса")
elif await invalid_json_encoding(request):
response = error(request, 400, "JSON должен быть закодирован в UTF-8")
else:
response = await call_next(request)
except Failure as exc:
response = error(request, exc.status, exc.message, exc.headers)
except Exception:
logger.error(
json.dumps({"event": "unexpected_error", "request_id": request.state.request_id})
)
response = error(request, 500, "Внутренняя ошибка")
response.headers["X-Request-Id"] = request.state.request_id
response.headers["X-Instance-Id"] = INSTANCE_ID
response.headers["X-Trace-Id"] = format(span.get_span_context().trace_id, "032x")
response.headers["X-Trace-Sampled"] = "true" if span.is_recording() else "false"
route = getattr(request.scope.get("route"), "path", "unknown")
elapsed = time.perf_counter() - start
attrs = {
"method": method,
"route": route,
"status_class": str(response.status_code // 100) + "xx",
}
span.update_name(method + " " + route)
span.set_attributes(
{
"http.request.method": method,
"http.route": route,
"http.response.status_code": response.status_code,
}
)
if response.status_code >= 500:
span.set_status(Status(StatusCode.ERROR, "server_error"))
t = telemetry.current()
if observed and t:
t.requests.add(1, attrs)
t.http_duration.record(elapsed, attrs)
t.sampling.add(1, {"decision": "sampled" if span.is_recording() else "not_sampled"})
logger.info(
json.dumps(
{
"event": "request",
"request_id": request.state.request_id,
**attrs,
"status": response.status_code,
"duration_ms": round(elapsed * 1000, 3),
}
)
)
return response
@app.exception_handler(Failure)
async def domain_error(request, exc):
return error(request, exc.status, exc.message, exc.headers)
@app.exception_handler(RequestValidationError)
async def validation_error(request, exc):
return error(request, 400, "Неверный JSON или поля запроса")
@app.exception_handler(HTTPException)
async def http_error(request, exc):
return error(request, exc.status_code, "Неверный запрос", exc.headers)
def bearer_failure():
return Failure(
401, "Требуется действующая сессия", {"WWW-Authenticate": 'Bearer realm="messenger"'}
)
def bearer(request):
parts = request.headers.get("Authorization", "").split()
if len(parts) != 2 or parts[0].lower() != "bearer":
raise bearer_failure()
return parts[1]
def actor(request: Request):
user = request.app.state.store.authenticate(bearer(request))
if user is None:
raise bearer_failure()
return user
@app.post("/api/v1/auth/sessions", status_code=201)
def login(request: Request):
headers = {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}
parts = request.headers.get("Authorization", "").split()
try:
if len(parts) != 2 or parts[0].lower() != "basic" or len(parts[1]) > 2048:
raise ValueError
decoded = base64.b64decode(parts[1], validate=True).decode("utf-8")
username, password = decoded.split(":", 1)
if not re.fullmatch(r"[a-z0-9_]{3,32}", username) or not 12 <= len(password) <= 128:
raise ValueError
except (ValueError, UnicodeError, binascii.Error):
raise Failure(401, "Неверные учётные данные", headers) from None
session = request.app.state.store.login(username, password)
if session is None:
raise Failure(401, "Неверные учётные данные", headers)
return session
@app.delete("/api/v1/auth/sessions/current", status_code=204)
def logout(request: Request, user=Depends(actor)):
if not request.app.state.store.logout(bearer(request)):
raise bearer_failure()
return Response(status_code=204)
def pagination(request, scope):
allowed = {"limit", "cursor"}
if any(k not in allowed for k in request.query_params):
raise Failure(400, "Неизвестный параметр запроса")
if any(len(request.query_params.getlist(k)) > 1 for k in allowed):
raise Failure(400, "Повторный параметр запроса")
value = request.query_params.get("limit", "50")
if len(value) > 3 or not re.fullmatch(r"[0-9]+", value) or not 1 <= int(value) <= 100:
raise Failure(400, "limit должен быть целым от 1 до 100")
return int(value), boundary(scope, request.query_params.get("cursor"))
@app.get("/health/live")
def live():
return {"status": "ok"}
@app.get("/health/ready")
def ready(request: Request):
if not request.app.state.store.ready() or not request.app.state.storage.ready():
raise Failure(503, "Хранилище недоступно")
return {"status": "ok"}
@app.post("/api/v1/users", status_code=201)
def register(data: CreateUser, request: Request):
return request.app.state.store.register(data.model_dump(mode="json"))
@app.get("/api/v1/users/me")
def me(user=Depends(actor)):
return user
@app.post("/api/v1/chats", status_code=201)
def create_chat(data: CreateChat, request: Request, user=Depends(actor)):
return request.app.state.store.create_chat(user["id"], data.model_dump(mode="json"))
@app.get("/api/v1/chats")
def chats(request: Request, user=Depends(actor)):
scope = "chats:" + user["id"]
limit, after = pagination(request, scope)
return page(request.app.state.store.list_chats(user["id"], after, limit), limit, scope)
@app.get("/api/v1/chats/{chat_id}")
def chat(chat_id: str, request: Request, user=Depends(actor)):
return request.app.state.store.chat(user["id"], identifier(chat_id))
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
def send(chat_id: str, data: CreateMessage, request: Request, user=Depends(actor)):
key = request.headers.get("Idempotency-Key", "")
if not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key):
raise Failure(400, "Требуется корректный Idempotency-Key")
return request.app.state.store.send(
user["id"], identifier(chat_id), data.text, key, [str(a) for a in data.attachment_ids]
)
@app.get("/api/v1/chats/{chat_id}/messages")
def history(chat_id: str, request: Request, user=Depends(actor)):
cid = identifier(chat_id)
scope = "messages:" + cid
limit, after = pagination(request, scope)
return page(request.app.state.store.history(user["id"], cid, after, limit), limit, scope)
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202)
async def upload_attachment(chat_id: str, request: Request, user=Depends(actor)):
import asyncio
import tempfile
from starlette.concurrency import run_in_threadpool
from .attachments import MAX_BYTES, filename_valid
if (
set(request.query_params) != {"filename"}
or len(request.query_params.getlist("filename")) != 1
):
raise Failure(400, "Требуется одно имя файла")
filename = filename_valid(request.query_params["filename"])
cid = identifier(chat_id)
await run_in_threadpool(request.app.state.store.chat, user["id"], cid)
content_type = request.headers.get("content-type", "").lower()
if content_type not in {"image/png", "image/jpeg"}:
raise Failure(415, "Ожидается PNG или JPEG")
if not request.app.state.upload_slots.acquire(blocking=False):
raise Failure(429, "Все места загрузки заняты", {"Retry-After": "1"})
try:
with tempfile.SpooledTemporaryFile(max_size=256 * 1024) as spool:
size = 0
try:
async with asyncio.timeout(15):
async for chunk in request.stream():
size += len(chunk)
if size > MAX_BYTES:
raise Failure(413, "Файл больше 10 MiB")
spool.write(chunk)
except TimeoutError:
raise Failure(400, "Истекло время загрузки") from None
spool.seek(0)
return await run_in_threadpool(
request.app.state.attachments.upload,
user["id"],
cid,
filename,
content_type,
spool,
size,
)
finally:
request.app.state.upload_slots.release()
@app.get("/api/v1/attachments/{attachment_id}")
def attachment_metadata(attachment_id: str, request: Request, user=Depends(actor)):
return request.app.state.attachments.metadata(user["id"], identifier(attachment_id))
@app.get("/api/v1/attachments/{attachment_id}/content")
def attachment_content(attachment_id: str, request: Request, user=Depends(actor)):
from fastapi.responses import StreamingResponse
if (
any(k != "variant" for k in request.query_params)
or len(request.query_params.getlist("variant")) > 1
):
raise Failure(400, "Неверный вариант")
variant = request.query_params.get("variant", "original")
if variant not in {"original", "thumbnail", "crop", "optimized"}:
raise Failure(400, "Неверный вариант")
obj = request.app.state.attachments.content(user["id"], identifier(attachment_id), variant)
def chunks():
try:
while data := obj["Body"].read(64 * 1024):
yield data
finally:
obj["Body"].close()
return StreamingResponse(
chunks(),
media_type=obj["ContentType"],
headers={"Content-Length": str(obj["ContentLength"])},
)
+176
View File
@@ -0,0 +1,176 @@
"""Durable asynchronous intake with full bounded validation before acceptance."""
import hashlib
import json
import os
import signal
import subprocess
import sys
import unicodedata
from pathlib import Path
from uuid import uuid4
from .domain import Failure, timestamp
from .jobs import Jobs, fault
from .postgres import public
MAX_BYTES = 10 * 1024 * 1024
MAX_PIXELS = 40_000_000
PUBLIC_FIELDS = (
"id",
"chat_id",
"owner_id",
"filename",
"content_type",
"size_bytes",
"sha256",
"status",
"error_code",
"created_at",
"updated_at",
)
def filename_valid(name):
if not isinstance(name, str) or not 1 <= len(name) <= 255 or not name.strip():
raise Failure(400, "Неверное имя файла")
if any(c in "/\\" or unicodedata.category(c) == "Cc" for c in name):
raise Failure(400, "Неверное имя файла")
return name
def validate_image(fileobj, declared_type):
if declared_type not in {"image/png", "image/jpeg"}:
raise Failure(415, "Ожидается PNG или JPEG")
fileobj.seek(0)
encoded = fileobj.read(MAX_BYTES + 1)
if len(encoded) > MAX_BYTES:
raise Failure(413, "Файл больше 10 MiB")
try:
with subprocess.Popen(
[sys.executable, "-m", "messenger.image_validation", declared_type],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
start_new_session=True,
env={
"PATH": os.environ.get("PATH", os.defpath),
"PYTHONPATH": str(Path(__file__).resolve().parents[1]),
"PYTHONDONTWRITEBYTECODE": "1",
"LANG": "C.UTF-8",
},
) as process:
try:
output, _ = process.communicate(input=encoded, timeout=5)
except subprocess.TimeoutExpired:
# The validator can itself run djpeg: kill its whole process group
# before releasing the upload slot, not only the Python parent.
try:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError:
pass # The child may have exited at the timeout boundary.
process.communicate()
raise Failure(415, "Изображение не прошло ограниченную проверку") from None
if process.returncode:
try:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError:
pass
raise Failure(415, "Изображение не прошло ограниченную проверку")
except OSError:
raise Failure(503, "Проверка изображения временно недоступна") from None
finally:
del encoded
try:
outcome = json.loads(output)
except (ValueError, UnicodeError):
raise Failure(415, "Повреждённое изображение") from None
if not isinstance(outcome, dict):
raise Failure(415, "Изображение не прошло ограниченную проверку")
if "error" in outcome:
status = outcome["error"]
if type(status) is not int or status not in {413, 415}:
status = 415
raise Failure(status, "Изображение не прошло ограниченную проверку")
values = outcome.get("ok")
if not isinstance(values, list) or len(values) != 3:
raise Failure(415, "Изображение не прошло ограниченную проверку")
fileobj.seek(0)
return tuple(values)
def metadata_public(row):
return {**public({k: row[k] for k in PUBLIC_FIELDS}), "variants": row["variants"]}
class AttachmentService:
def __init__(self, store, storage):
self.store = store
self.storage = storage
self.jobs = Jobs(store)
def upload(self, actor, cid, filename, content_type, fileobj, size):
filename_valid(filename)
if not 1 <= size <= MAX_BYTES:
raise Failure(413 if size > MAX_BYTES else 415, "Неверный размер изображения")
validate_image(fileobj, content_type)
digest = hashlib.file_digest(fileobj, "sha256").hexdigest()
aid = str(uuid4())
key = "staging/" + aid
now = timestamp()
with self.store.connection() as conn:
self.store._chat(conn, actor, cid)
conn.execute(
"INSERT INTO upload_intents(id,object_key,created_at) VALUES (%s,%s,%s)",
(aid, key, now),
)
# Keep the intent locked throughout PUT/finalization: reconciler skips live
# uploads even if a slow request crosses the grace boundary.
with self.store.connection() as conn, conn.transaction():
intent = conn.execute(
"SELECT id FROM upload_intents WHERE id=%s FOR UPDATE", (aid,)
).fetchone()
if not intent:
raise Failure(503, "Загрузка больше не активна")
self.storage.put(key, fileobj, size, content_type)
fault(self.store, "after_staging", aid)
row = conn.execute(
"INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,"
"status,error_code,created_at,updated_at,object_key,stage_key) "
"VALUES (%s,%s,%s,%s,%s,%s,%s,'queued',NULL,%s,%s,%s,%s) RETURNING *",
(aid, cid, actor, filename, content_type, size, digest, now, now, key, key),
).fetchone()
self.jobs.enqueue(conn, aid, key)
conn.execute("DELETE FROM upload_intents WHERE id=%s", (aid,))
# Never delete S3 on an ambiguous COMMIT: the transaction may have committed.
fault(self.store, "after_upload_commit", aid)
return metadata_public(row)
def metadata(self, actor, aid):
with self.store.connection() as conn:
row = conn.execute(
"SELECT a.* FROM attachments a JOIN members m ON m.chat_id=a.chat_id "
"WHERE a.id=%s AND m.user_id=%s",
(aid, actor),
).fetchone()
if not row:
raise Failure(404, "Объект не найден")
return metadata_public(row)
def content(self, actor, aid, variant="original"):
with self.store.connection() as conn:
row = conn.execute(
"SELECT a.* FROM attachments a JOIN members m ON m.chat_id=a.chat_id "
"WHERE a.id=%s AND m.user_id=%s",
(aid, actor),
).fetchone()
if not row:
raise Failure(404, "Объект не найден")
if row["status"] != "ready":
raise Failure(409, "Вложение не готово")
key = row["object_key"]
if variant != "original":
if variant not in {v["name"] for v in row["variants"]}:
raise Failure(404, "Вариант не найден")
key = key.rsplit("/", 1)[0] + "/" + variant
return self.storage.get(key)
+81
View File
@@ -0,0 +1,81 @@
"""Same SQL with actual PG roles, confined to a disposable synthetic schema."""
import json
import secrets
import psycopg
from psycopg import sql
from .config import secret
def main():
suffix = secrets.token_hex(5)
schema, wide, limited = (prefix + suffix for prefix in ("study_", "wide_", "limited_"))
admin = psycopg.connect(secret("DATABASE_URL"), autocommit=True)
rows = []
try:
# SET ROLE is an explicit operator act. Both NOLOGIN test roles have no
# grants on application tables. Wide owns ONLY the synthetic namespace.
for role in (wide, limited):
admin.execute(sql.SQL("CREATE ROLE {} NOLOGIN").format(sql.Identifier(role)))
admin.execute(
sql.SQL("CREATE SCHEMA {} AUTHORIZATION {}").format(
sql.Identifier(schema), sql.Identifier(wide)
)
)
admin.execute(sql.SQL("SET search_path TO {}").format(sql.Identifier(schema)))
admin.execute(sql.SQL("SET ROLE {}").format(sql.Identifier(wide)))
admin.execute("CREATE TABLE users(password_hash text); CREATE TABLE jobs(id int)")
admin.execute(
sql.SQL("GRANT USAGE ON SCHEMA {} TO {}; GRANT SELECT ON jobs TO {}").format(
sql.Identifier(schema), sql.Identifier(limited), sql.Identifier(limited)
)
)
admin.execute("RESET ROLE")
for label, role in (("wide", wide), ("limited", limited)):
admin.execute(sql.SQL("SET ROLE {}").format(sql.Identifier(role)))
assert admin.execute("SELECT current_user").fetchone()[0] == role
for name, query, restricted_expected in (
("read_password_hash", "SELECT password_hash FROM users LIMIT 0", False),
("create_table", "CREATE TABLE forbidden(id int)", False),
("read_jobs", "SELECT id FROM jobs LIMIT 0", True),
):
try:
admin.execute(query)
allowed = True
if name == "create_table":
admin.execute("DROP TABLE forbidden")
except psycopg.errors.InsufficientPrivilege:
allowed = False
rows.append(
{
"role": label,
"operation": name,
"allowed": allowed,
"expected": restricted_expected if label == "limited" else True,
}
)
admin.execute("RESET ROLE")
passed = all(r["allowed"] == r["expected"] for r in rows)
print(
json.dumps(
{
"scope": "disposable synthetic schema; operator SET ROLE to NOLOGIN roles",
"rows": rows,
"passed": passed,
}
)
)
if not passed:
raise SystemExit(1)
finally:
admin.execute("RESET ROLE")
admin.execute(sql.SQL("DROP SCHEMA IF EXISTS {} CASCADE").format(sql.Identifier(schema)))
for role in (limited, wide):
admin.execute(sql.SQL("DROP ROLE IF EXISTS {}").format(sql.Identifier(role)))
admin.close()
if __name__ == "__main__":
main()
+65
View File
@@ -0,0 +1,65 @@
"""Isolated loopback-only Valkey control: same operations before/after ACL narrowing."""
import json
import secrets
import redis
def main():
admin = redis.Redis(host="127.0.0.1", port=6378, decode_responses=True)
wide_pass, limited_pass = secrets.token_urlsafe(24), secrets.token_urlsafe(24)
admin.execute_command(
"ACL", "SETUSER", "wide", "on", ">" + wide_pass, "~security-study:*", "+@all"
)
admin.execute_command(
"ACL",
"SETUSER",
"limited",
"on",
">" + limited_pass,
"~security-study:jobs",
"+ping",
"+xgroup|create",
"+xreadgroup",
"+xautoclaim",
"+xack",
"+xdel",
)
rows = []
for role, password in (("wide", wide_pass), ("limited", limited_pass)):
client = redis.Redis(
host="127.0.0.1", port=6378, username=role, password=password, decode_responses=True
)
for name, operation in (
("queue_admin_acl_list", lambda client=client: client.acl_list()),
(
"queue_publish",
lambda client=client: client.xadd("security-study:jobs", {"body": "synthetic"}),
),
(
"queue_other_key_write",
lambda client=client: client.set("security-study:other", "synthetic"),
),
):
try:
operation()
allowed = True
except redis.ResponseError:
allowed = False
rows.append({"role": role, "operation": name, "allowed": allowed})
client.close()
print(
json.dumps(
{
"isolated_port": 6378,
"rows": rows,
"passed": all(r["allowed"] == (r["role"] == "wide") for r in rows),
}
)
)
admin.shutdown(nosave=True)
if __name__ == "__main__":
main()
+88
View File
@@ -0,0 +1,88 @@
"""Disposable bucket comparison; baseline credentials cannot access messenger."""
import json
from pathlib import Path
import boto3
from botocore.config import Config
from botocore.exceptions import ClientError
def client(access, secret, identity):
return boto3.client(
"s3",
endpoint_url="https://storage:8443",
aws_access_key_id=access,
aws_secret_access_key=secret,
region_name="us-east-1",
verify="/run/tls/ca.pem",
config=Config(
client_cert=(
f"/run/audit-identities/{identity}/cert.pem",
f"/run/audit-identities/{identity}/key.pem",
),
connect_timeout=2,
read_timeout=3,
retries={"total_max_attempts": 1},
s3={"addressing_style": "path"},
request_checksum_calculation="when_required",
response_checksum_validation="when_required",
),
)
def main():
root = Path("/run/audit-material")
def read(name):
return (root / name).read_text().strip()
admin = client(read("admin_s3_access"), read("admin_s3_secret"), "operator")
wide = client(read("study_wide_s3_access"), read("study_wide_s3_secret"), "operator")
limited = client(read("worker_s3_access"), read("worker_s3_secret"), "worker")
bucket = "security-study"
rows = []
try:
wide.get_object(Bucket="messenger", Key="private/not-a-real-user-object")
isolated = False
except ClientError as exc:
isolated = exc.response["ResponseMetadata"]["HTTPStatusCode"] == 403
admin.create_bucket(Bucket=bucket)
admin.put_object(Bucket=bucket, Key="private/synthetic", Body=b"synthetic-not-user-data")
try:
for role, c in (("wide", wide), ("limited", limited)):
for operation in ("read_private", "write_private"):
try:
if operation == "read_private":
response = c.get_object(Bucket=bucket, Key="private/synthetic")
response["Body"].read()
response["Body"].close()
else:
c.put_object(Bucket=bucket, Key="private/written", Body=b"synthetic")
allowed = True
except ClientError as exc:
if exc.response["ResponseMetadata"]["HTTPStatusCode"] != 403:
raise
allowed = False
rows.append({"role": role, "operation": operation, "allowed": allowed})
print(
json.dumps(
{
"scope": "disposable bucket security-study; broad role has no messenger rights",
"rows": rows,
"wide_role_denied_application_bucket": isolated,
"passed": isolated
and all(row["allowed"] == (row["role"] == "wide") for row in rows),
}
)
)
finally:
for key in ("private/synthetic", "private/written"):
admin.delete_object(Bucket=bucket, Key=key)
admin.delete_bucket(Bucket=bucket)
for c in (admin, wide, limited):
c.close()
if __name__ == "__main__":
main()
+14
View File
@@ -0,0 +1,14 @@
import os
from pathlib import Path
def secret(name: str) -> str:
if os.getenv("BAO_ROLE"):
from .secrets_provider import value
return value(name)
filename = os.environ.get(name + "_FILE")
value = Path(filename).read_text().strip() if filename else os.environ.get(name, "")
if not value:
raise RuntimeError("Отсутствует настройка " + name)
return value
+63
View File
@@ -0,0 +1,63 @@
import base64
import json
import re
from datetime import UTC, datetime
from uuid import UUID
class Failure(Exception):
def __init__(self, status: int, message: str, headers: dict | None = None):
self.status = status
self.message = message
self.headers = headers or {}
def timestamp() -> str:
return datetime.now(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z")
def identifier(value: str) -> str:
try:
return str(UUID(value))
except (ValueError, TypeError, AttributeError):
raise Failure(400, "Некорректный UUID") from None
def boundary(scope: str, cursor: str | None) -> tuple[str, str] | None:
if cursor is None:
return None
try:
if not cursor or len(cursor) > 1024 or not re.fullmatch(r"[A-Za-z0-9_-]+", cursor):
raise ValueError
data = json.loads(base64.urlsafe_b64decode(cursor + "=" * (-len(cursor) % 4)))
if (
not isinstance(data, list)
or len(data) != 4
or type(data[0]) is not int
or data[0] != 1
or data[1] != scope
):
raise ValueError
date, uid = data[2:]
if not isinstance(date, str) or not re.fullmatch(
r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]{1,6})?Z",
date,
):
raise ValueError
datetime.fromisoformat(date.replace("Z", "+00:00"))
if not isinstance(uid, str) or str(UUID(uid)) != uid:
raise ValueError
return date, uid
except (ValueError, TypeError, KeyError, UnicodeError):
raise Failure(400, "Некорректный cursor или область запроса") from None
def page(items: list[dict], limit: int, scope: str) -> dict:
more = len(items) > limit
selected = items[:limit]
cursor = None
if more:
last = selected[-1]
payload = json.dumps([1, scope, last["created_at"], last["id"]], separators=(",", ":"))
cursor = base64.urlsafe_b64encode(payload.encode()).decode().rstrip("=")
return {"items": selected, "next_cursor": cursor}
+82
View File
@@ -0,0 +1,82 @@
"""Image transformations, run in a bounded child process by the worker."""
import hashlib
import json
import sys
from pathlib import Path
from PIL import Image, ImageOps
Image.MAX_IMAGE_PIXELS = 40_000_000
def fitted_size(width, height, bound):
"""Fit without upscale; integer arithmetic deliberately rounds down."""
longest = max(width, height)
if longest <= bound:
return width, height
return max(1, width * bound // longest), max(1, height * bound // longest)
def render(source, destination, version=1):
if version not in (1, 2):
raise ValueError("unsupported_pipeline")
destination = Path(destination)
with Image.open(source) as image:
if image.width * image.height > 40_000_000:
raise ValueError("pixel_limit")
image.load()
ImageOps.exif_transpose(image, in_place=True)
if "A" in image.getbands() or (image.mode == "P" and "transparency" in image.info):
rgba = image if image.mode == "RGBA" else image.convert("RGBA")
rgb = Image.new("RGB", rgba.size, "white")
alpha = rgba.getchannel("A")
rgb.paste(rgba, mask=alpha)
alpha.close()
if rgba is not image:
rgba.close()
else:
rgb = image if image.mode == "RGB" else image.convert("RGB")
thumbnail = rgb.resize(fitted_size(*rgb.size, 256), Image.Resampling.LANCZOS)
crop = ImageOps.fit(rgb, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5))
optimized = rgb.resize(fitted_size(*rgb.size, 1280), Image.Resampling.LANCZOS)
variants = []
for name, variant, kind, mime, options in (
("thumbnail", thumbnail, "JPEG", "image/jpeg", {"quality": 85}),
("crop", crop, "JPEG", "image/jpeg", {"quality": 85}),
(
"optimized",
optimized,
"WEBP",
"image/webp",
{"quality": 80 if version == 1 else 75, "method": 4},
),
):
# No EXIF/ICC arguments: outputs do not propagate source metadata.
path = destination / name
variant.save(path, format=kind, **options)
data = path.read_bytes()
variants.append(
{
"name": name,
"content_type": mime,
"size_bytes": len(data),
"width": variant.width,
"height": variant.height,
"sha256": hashlib.sha256(data).hexdigest(),
}
)
variant.close()
if rgb is not image:
rgb.close()
return variants
if __name__ == "__main__":
import resource
resource.setrlimit(resource.RLIMIT_AS, (700 * 1024**2, 700 * 1024**2))
resource.setrlimit(resource.RLIMIT_CPU, (15, 15))
source, destination, version = sys.argv[1:]
result = render(source, destination, int(version))
(Path(destination) / "variants.json").write_text(json.dumps(result))
+146
View File
@@ -0,0 +1,146 @@
"""Short-lived bounded pixel validation; no DB, credentials or application imports."""
import json
import resource
import subprocess
import sys
from io import BytesIO
from PIL import Image, UnidentifiedImageError
from .domain import Failure
MAX_BYTES = 10 * 1024 * 1024
MAX_PIXELS = 40_000_000
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
def inspect_image(fileobj, declared_type):
if declared_type not in {"image/png", "image/jpeg"}:
raise Failure(415, "Ожидается PNG или JPEG")
try:
fileobj.seek(0)
with Image.open(fileobj) as header:
width, height = header.size
if width * height > MAX_PIXELS:
raise Failure(413, "Слишком много пикселей")
actual = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(header.format)
if actual != declared_type:
raise Failure(415, "Формат изображения не соответствует MIME")
frames = getattr(header, "n_frames", 1)
default_image = bool(getattr(header, "default_image", False))
if width * height * frames > MAX_PIXELS:
raise Failure(413, "Слишком много пикселей во всех кадрах")
header.verify()
# Drop parsed metadata before the separate strict decoder process starts.
del header
if actual == "image/png":
# verify() checks CRC, but does not compare acTL's frame count with
# every fcTL. Do not let a forged count hide a corrupt later frame.
fileobj.seek(8)
controls = frame_controls = 0
declared_frames = None
animation_data = False
while True:
chunk_header = fileobj.read(8)
if len(chunk_header) != 8:
raise ValueError("Truncated PNG chunk")
length = int.from_bytes(chunk_header[:4], "big")
kind = chunk_header[4:]
controls += kind == b"acTL"
frame_controls += kind == b"fcTL"
animation_data |= kind == b"fdAT"
if kind == b"acTL":
if length != 8:
raise ValueError("Invalid acTL length")
declared_frames = int.from_bytes(fileobj.read(4), "big")
fileobj.seek(length, 1) # remaining payload plus CRC
else:
fileobj.seek(length + 4, 1)
if kind == b"IEND":
break
if controls:
if (
controls != 1
or not declared_frames
or frame_controls != declared_frames
or declared_frames + int(default_image) != frames
):
raise ValueError("Invalid APNG frame count")
elif frame_controls or animation_data:
raise ValueError("Animation chunks without acTL")
if actual == "image/jpeg":
# Validate scan warnings before allocating Pillow's decoded canvas.
# 384 MiB covers coefficient arrays for <=40M pixels and <=4 channels;
# keeping both full decoded buffers alive could exceed the API cgroup.
fileobj.seek(0)
# A bounded compressed copy avoids sharing buffered file-descriptor
# offsets with the child, and supports BytesIO callers as well.
encoded = fileobj.read(MAX_BYTES + 1)
if len(encoded) > MAX_BYTES:
raise Failure(413, "Файл больше 10 MiB")
try:
decoded = subprocess.run(
["djpeg", "-strict", "-maxmemory", "384m", "-outfile", "/dev/null"],
input=encoded,
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
timeout=4,
check=False,
)
except subprocess.TimeoutExpired:
raise Failure(415, "Изображение не прошло ограниченную проверку") from None
finally:
del encoded
if decoded.returncode:
raise Failure(415, "Повреждённое изображение")
# djpeg has already decoded every JPEG pixel, streaming output rows.
# A second Pillow load would overlap full progressive coefficient
# arrays with a 160 MiB CMYK canvas and violate the API's 512 MiB cap.
fileobj.seek(0)
return actual, width, height
# Decode one frame at a time. Pillow may retain a previous APNG canvas
# for compositing, but no list/copies of every decoded frame is collected.
fileobj.seek(0)
with Image.open(fileobj) as image:
decoded_pixels = 0
for frame in range(frames):
if decoded_pixels + width * height > MAX_PIXELS:
raise Failure(413, "Слишком много пикселей во всех кадрах")
image.seek(frame)
decoded_pixels += image.size[0] * image.size[1]
if decoded_pixels > MAX_PIXELS:
raise Failure(413, "Слишком много пикселей во всех кадрах")
image.load()
# Image's context manager closes its input handle, not the decoded core.
# Releasing the object frees native canvases before hashing/S3 publication.
del image
fileobj.seek(0)
return actual, width, height
except Image.DecompressionBombError:
raise Failure(413, "Слишком много пикселей") from None
except (UnidentifiedImageError, OSError, ValueError, SyntaxError, EOFError):
raise Failure(415, "Повреждённое изображение") from None
def main():
resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
resource.setrlimit(resource.RLIMIT_CPU, (4, 5))
if sys.platform == "linux":
resource.setrlimit(resource.RLIMIT_AS, (512 * 1024 * 1024, 512 * 1024 * 1024))
try:
data = sys.stdin.buffer.read(MAX_BYTES + 1)
if len(data) > MAX_BYTES:
raise Failure(413, "Файл больше 10 MiB")
result = inspect_image(BytesIO(data), sys.argv[1])
print(json.dumps({"ok": result}))
except Failure as exc:
print(json.dumps({"error": exc.status}))
except MemoryError:
print(json.dumps({"error": 415}))
except Exception:
print(json.dumps({"error": 415}))
if __name__ == "__main__":
main()
+123
View File
@@ -0,0 +1,123 @@
"""Local operator commands; no public HTTP replay or arbitrary object paths."""
import argparse
import json
from .domain import Failure
from .jobs import Jobs, fault
from .postgres import PostgresStore
from .storage import ObjectStorage
from .worker import cleanup_ready
def cleanup(store, storage, failed_hours=168, intent_seconds=3600):
if failed_hours < 0 or intent_seconds < 0:
raise ValueError("Retention must be nonnegative")
counts = {"ready_staging": 0, "failed_sources": 0, "orphan_intents": 0}
with store.connection() as conn:
ready = conn.execute(
"SELECT id FROM attachments WHERE status='ready' AND stage_key IS NOT NULL"
).fetchall()
for row in ready:
cleanup_ready(store, storage, row["id"])
counts["ready_staging"] += 1
with store.connection() as conn:
candidates = conn.execute(
"SELECT id FROM attachments WHERE status='failed' AND updated_at<now()-(%s * interval '1 hour')",
(failed_hours,),
).fetchall()
for row in candidates:
with store.connection() as conn, conn.transaction():
a = conn.execute(
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (row["id"],)
).fetchone()
# Recheck under the same row lock used by replay and publication.
if not a or a["status"] != "failed":
continue
current = conn.execute(
"SELECT * FROM attachment_jobs WHERE attachment_id=%s AND pipeline_version=%s FOR UPDATE",
(a["id"], a["pipeline_version"]),
).fetchone()
if not current or current["state"] != "failed" or current["source_key"] is None:
continue
old = conn.execute(
"SELECT updated_at<now()-(%s * interval '1 hour') AS old FROM attachments WHERE id=%s",
(failed_hours, a["id"]),
).fetchone()["old"]
if not old:
continue
fault(store, "cleanup_before_delete", a["id"])
# If cleanup commits first, later replay refuses source_missing. If replay
# commits first, status is queued and cleanup skips it. DELETE stays locked.
source = current["source_key"]
storage.delete(source)
prefix = f"attachments/{a['id']}/v{a['pipeline_version']}"
for variant in ("original", "thumbnail", "crop", "optimized"):
storage.delete(prefix + "/" + variant)
conn.execute("UPDATE attachment_jobs SET source_key=NULL WHERE id=%s", (current["id"],))
conn.execute("UPDATE attachments SET stage_key=NULL WHERE id=%s", (a["id"],))
Jobs.audit(conn, current["id"], "source_retention_expired", current["attempt"])
counts["failed_sources"] += 1
with store.connection() as conn:
intents = conn.execute(
"SELECT id FROM upload_intents WHERE created_at<now()-(%s * interval '1 second')",
(intent_seconds,),
).fetchall()
for row in intents:
with store.connection() as conn, conn.transaction():
intent = conn.execute(
"SELECT * FROM upload_intents WHERE id=%s FOR UPDATE SKIP LOCKED", (row["id"],)
).fetchone()
if intent is None:
continue
referenced = conn.execute(
"SELECT 1 FROM attachments WHERE id=%s", (row["id"],)
).fetchone()
if referenced is None:
storage.delete(intent["object_key"])
counts["orphan_intents"] += 1
conn.execute("DELETE FROM upload_intents WHERE id=%s", (row["id"],))
return counts
def main():
parser = argparse.ArgumentParser()
sub = parser.add_subparsers(dest="command", required=True)
for name in ("replay", "reprocess"):
command = sub.add_parser(name)
command.add_argument("attachment_id")
if name == "reprocess":
command.add_argument("--version", type=int, required=True)
sub.add_parser("failed")
command = sub.add_parser("cleanup")
command.add_argument("--failed-hours", type=float, default=168)
command.add_argument("--intent-seconds", type=float, default=3600)
args = parser.parse_args()
store, storage = PostgresStore(), ObjectStorage()
try:
jobs = Jobs(store)
if args.command == "replay":
result = {"job_id": jobs.replay(args.attachment_id, storage)}
elif args.command == "reprocess":
result = {"job_id": jobs.reprocess(args.attachment_id, args.version, storage)}
elif args.command == "cleanup":
result = cleanup(store, storage, args.failed_hours, args.intent_seconds)
else:
with store.connection() as conn:
result = [
dict(row)
for row in conn.execute(
"SELECT j.id,j.attachment_id,j.attempt,f.error_code,f.replay_count FROM failed_jobs f JOIN attachment_jobs j ON j.id=f.job_id WHERE j.state='failed'"
).fetchall()
]
print(json.dumps(result, default=str))
except Failure as exc:
print(json.dumps({"status": exc.status, "error": exc.message}))
raise SystemExit(1) from None
finally:
storage.close()
store.close()
if __name__ == "__main__":
main()
+345
View File
@@ -0,0 +1,345 @@
"""PostgreSQL owns job state; broker deliveries are repeatable notifications."""
import json
import logging
import os
import random
import time
from datetime import UTC, datetime
from uuid import uuid4
import redis
from opentelemetry import propagate
from opentelemetry.trace import SpanKind
from . import telemetry
from .config import secret
from .domain import Failure
from .postgres import public
logger = logging.getLogger("messenger.jobs")
STREAM = "attachment-jobs:v1"
GROUP = "image-workers"
def broker_client():
return redis.Redis.from_url(
secret("BROKER_URL"),
decode_responses=False,
socket_connect_timeout=2,
socket_timeout=3,
)
def fault(store, point, attachment_id):
"""Only enabled in the explicit fault-test overlay; never accepts HTTP controls."""
if os.getenv("ENABLE_FAULT_INJECTION") != "1":
return
with store.connection() as conn:
row = conn.execute(
"UPDATE test_faults SET reached_at=now() WHERE id=("
"SELECT id FROM test_faults WHERE point=%s AND reached_at IS NULL "
"AND (attachment_id IS NULL OR attachment_id=%s) ORDER BY id LIMIT 1) RETURNING *",
(point, attachment_id),
).fetchone()
if row is None:
return
logger.warning(json.dumps({"event": "fault_reached", "point": point, "fault_id": row["id"]}))
if row["action"] == "crash":
os._exit(86)
if row["action"] == "error":
raise RuntimeError("injected_transient_failure")
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
with store.connection() as conn:
released = conn.execute(
"SELECT released FROM test_faults WHERE id=%s", (row["id"],)
).fetchone()
if released and released["released"]:
return
time.sleep(0.1)
raise RuntimeError("fault_barrier_timeout")
class Jobs:
def __init__(self, store):
self.store = store
self.attempt_limit = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
self.lease_seconds = int(os.getenv("JOB_LEASE_SECONDS", "45"))
if not 1 <= self.attempt_limit <= 20 or not 5 <= self.lease_seconds <= 300:
raise RuntimeError("Invalid job retry/lease configuration")
def enqueue(self, conn, attachment_id, source_key, version=1):
job_id = str(uuid4())
conn.execute(
"INSERT INTO attachment_jobs(id,attachment_id,pipeline_version,source_key,state,attempt_limit,traceparent,tracestate,accepted_at) "
"VALUES (%s,%s,%s,%s,'queued',%s,%s,%s,clock_timestamp())",
(
job_id,
attachment_id,
version,
source_key,
self.attempt_limit,
telemetry.carrier().get("traceparent"),
telemetry.carrier().get("tracestate"),
),
)
conn.execute("INSERT INTO job_outbox(job_id) VALUES (%s)", (job_id,))
self.audit(conn, job_id, "accepted", 0)
return job_id
@staticmethod
def audit(conn, job_id, event, attempt):
conn.execute(
"INSERT INTO job_audit(job_id,event,attempt) VALUES (%s,%s,%s)",
(job_id, event, attempt),
)
def publish_one(self, broker):
with self.store.connection() as conn, conn.transaction():
job = conn.execute(
"SELECT j.* FROM job_outbox o JOIN attachment_jobs j ON j.id=o.job_id "
"WHERE o.published_at IS NULL AND j.available_at<=now() "
"ORDER BY j.available_at LIMIT 1 FOR UPDATE OF o SKIP LOCKED"
).fetchone()
if job is None:
return False
if job["state"] == "queued":
body = {
"schema_version": 1,
"job_id": str(job["id"]),
"type": "attachment.process.v1",
"attachment_id": str(job["attachment_id"]),
"pipeline_version": job["pipeline_version"],
"attempt": job["attempt"] + 1,
"created_at": public({"v": job["created_at"]})["v"],
}
# XADD response with appendfsync=always is the broker acceptance boundary.
parent = {
key: job.get(key) for key in ("traceparent", "tracestate") if job.get(key)
}
with telemetry.operation(
"attachments publish", context=propagate.extract(parent), kind=SpanKind.PRODUCER
):
body.update(telemetry.carrier())
broker.xadd(STREAM, {"body": json.dumps(body)})
fault(self.store, "after_publish", job["attachment_id"])
conn.execute("UPDATE job_outbox SET published_at=now() WHERE job_id=%s", (job["id"],))
return True
def claim(self, job_id, attachment_id, version):
with self.store.connection() as conn, conn.transaction():
attachment = conn.execute(
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,)
).fetchone()
if attachment is None:
return None
job = conn.execute(
"SELECT * FROM attachment_jobs WHERE id=%s FOR UPDATE", (job_id,)
).fetchone()
if (
job is None
or str(job["attachment_id"]) != str(attachment_id)
or job["pipeline_version"] != version
):
return None
if attachment["pipeline_version"] != version:
conn.execute(
"UPDATE attachment_jobs SET state='superseded',updated_at=now() WHERE id=%s AND state NOT IN ('ready','superseded')",
(job_id,),
)
return None
if job["state"] != "queued":
return None
lease_id = str(uuid4())
job = conn.execute(
"UPDATE attachment_jobs SET state='processing',attempt=attempt+1,lease_id=%s,"
"lease_until=now()+(%s * interval '1 second'),updated_at=now() "
"WHERE id=%s AND available_at<=now() RETURNING *",
(lease_id, self.lease_seconds, job_id),
).fetchone()
if job is None:
return None
conn.execute(
"UPDATE attachments SET status='processing',updated_at=now() WHERE id=%s",
(attachment_id,),
)
self.audit(conn, job_id, "claimed", job["attempt"])
job["source_sha256"] = attachment["sha256"]
job["source_size"] = attachment["size_bytes"]
job["source_content_type"] = attachment["content_type"]
return job
def complete(self, job, object_key, variants):
with self.store.connection() as conn, conn.transaction():
attachment = conn.execute(
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (job["attachment_id"],)
).fetchone()
current = conn.execute(
"SELECT * FROM attachment_jobs WHERE id=%s FOR UPDATE", (job["id"],)
).fetchone()
if (
attachment["pipeline_version"] != job["pipeline_version"]
or current["state"] != "processing"
or current["lease_id"] != job["lease_id"]
):
self.audit(conn, job["id"], "stale_publish_rejected", job["attempt"])
return False
conn.execute(
"UPDATE attachments SET status='ready',error_code=NULL,object_key=%s,variants=%s::jsonb,updated_at=now() WHERE id=%s",
(object_key, json.dumps(variants), job["attachment_id"]),
)
conn.execute(
"UPDATE attachment_jobs SET state='ready',lease_until=NULL,error_code=NULL,updated_at=now() WHERE id=%s",
(job["id"],),
)
self.audit(conn, job["id"], "published", job["attempt"])
t = telemetry.current()
if t:
t.ready_duration.record(
(datetime.now(UTC) - job["accepted_at"]).total_seconds(),
{"job_type": "attachment.process.v1"},
)
return True
def fail(self, job, code, permanent=False):
with self.store.connection() as conn, conn.transaction():
attachment = conn.execute(
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (job["attachment_id"],)
).fetchone()
current = conn.execute(
"SELECT * FROM attachment_jobs WHERE id=%s FOR UPDATE", (job["id"],)
).fetchone()
if (
attachment["pipeline_version"] != job["pipeline_version"]
or current["lease_id"] != job["lease_id"]
or current["state"] != "processing"
):
return
if permanent or current["attempt"] >= current["attempt_limit"]:
final_code = code if permanent else "retry_exhausted"
self._failed(conn, current, final_code)
else:
delay = min(30, 2 ** (current["attempt"] - 1)) + random.uniform(0, 0.5)
conn.execute(
"UPDATE attachment_jobs SET state='queued',error_code=%s,lease_until=NULL,"
"available_at=now()+(%s * interval '1 second'),updated_at=now() WHERE id=%s",
(code, delay, job["id"]),
)
conn.execute(
"UPDATE job_outbox SET published_at=NULL WHERE job_id=%s", (job["id"],)
)
conn.execute(
"UPDATE attachments SET status='queued',updated_at=now() WHERE id=%s",
(job["attachment_id"],),
)
self.audit(conn, job["id"], "retry_scheduled", job["attempt"])
def _failed(self, conn, job, code):
conn.execute(
"UPDATE attachment_jobs SET state='failed',error_code=%s,lease_until=NULL,updated_at=now() WHERE id=%s",
(code, job["id"]),
)
conn.execute(
"UPDATE attachments SET status='failed',error_code=%s,updated_at=now() WHERE id=%s",
(code, job["attachment_id"]),
)
conn.execute(
"INSERT INTO failed_jobs(job_id,error_code) VALUES (%s,%s) "
"ON CONFLICT(job_id) DO UPDATE SET error_code=excluded.error_code,failed_at=now()",
(job["id"], code),
)
self.audit(conn, job["id"], "failed:" + code, job["attempt"])
def recover_one(self):
with self.store.connection() as conn, conn.transaction():
candidate = conn.execute(
"SELECT a.id FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id "
"WHERE j.state='processing' AND j.lease_until<now() LIMIT 1 FOR UPDATE OF a SKIP LOCKED"
).fetchone()
if candidate is None:
return False
job = conn.execute(
"SELECT j.* FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id "
"WHERE j.attachment_id=%s AND j.state='processing' AND j.lease_until<now() "
"AND j.pipeline_version=a.pipeline_version FOR UPDATE OF j",
(candidate["id"],),
).fetchone()
if job is None:
return False
if job["attempt"] >= job["attempt_limit"]:
self._failed(conn, job, "retry_exhausted")
else:
conn.execute(
"UPDATE attachment_jobs SET state='queued',lease_until=NULL,available_at=now(),updated_at=now() WHERE id=%s",
(job["id"],),
)
conn.execute(
"UPDATE attachments SET status='queued',updated_at=now() WHERE id=%s",
(job["attachment_id"],),
)
conn.execute(
"UPDATE job_outbox SET published_at=NULL WHERE job_id=%s", (job["id"],)
)
self.audit(conn, job["id"], "lease_recovered", job["attempt"])
return True
def replay(self, attachment_id, storage):
with self.store.connection() as conn, conn.transaction():
a = conn.execute(
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,)
).fetchone()
if not a or a["status"] != "failed":
raise Failure(409, "Повтор разрешён только для failed")
job = conn.execute(
"SELECT * FROM attachment_jobs WHERE attachment_id=%s AND pipeline_version=%s FOR UPDATE",
(attachment_id, a["pipeline_version"]),
).fetchone()
if not job["source_key"] or not storage.exists(job["source_key"]):
raise Failure(409, "Источник больше недоступен")
fault(self.store, "replay_before_commit", attachment_id)
conn.execute(
"UPDATE attachment_jobs SET state='queued',attempt_limit=attempt+%s,available_at=now(),error_code=NULL,updated_at=now() WHERE id=%s",
(self.attempt_limit, job["id"]),
)
conn.execute(
"UPDATE attachments SET status='queued',error_code=NULL,updated_at=now() WHERE id=%s",
(attachment_id,),
)
conn.execute("UPDATE job_outbox SET published_at=NULL WHERE job_id=%s", (job["id"],))
conn.execute(
"UPDATE failed_jobs SET replay_count=replay_count+1 WHERE job_id=%s", (job["id"],)
)
self.audit(conn, job["id"], "operator_replay", job["attempt"])
return str(job["id"])
def reprocess(self, attachment_id, version, storage):
if version not in (1, 2):
raise Failure(400, "Поддерживаются версии pipeline 1 и 2")
with self.store.connection() as conn, conn.transaction():
a = conn.execute(
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,)
).fetchone()
if not a:
raise Failure(404, "Вложение отсутствует")
if version <= a["pipeline_version"]:
raise Failure(409, "Нужна более новая версия pipeline")
source = a["object_key"] if a["status"] == "ready" else a["stage_key"]
if not source or not storage.exists(source):
raise Failure(409, "Источник больше недоступен")
if a["status"] == "ready" and a["stage_key"] and a["stage_key"].startswith("staging/"):
# A worker can be paused after publishing ready but before its
# staging cleanup. Do not orphan that key when choosing final v1
# as the new source. Delete stays under the same attachment lock;
# a storage failure leaves the ready version unchanged.
fault(self.store, "reprocess_before_staging_delete", attachment_id)
storage.delete(a["stage_key"])
conn.execute(
"UPDATE attachment_jobs SET state='superseded',updated_at=now() WHERE attachment_id=%s AND state IN ('queued','processing','failed')",
(attachment_id,),
)
conn.execute(
"UPDATE attachments SET pipeline_version=%s,status='queued',variants='[]',error_code=NULL,stage_key=%s,updated_at=now() WHERE id=%s",
(version, source, attachment_id),
)
return self.enqueue(conn, attachment_id, source, version)
+49
View File
@@ -0,0 +1,49 @@
"""Measure configured Argon2 costs without keeping passwords or hashes in evidence."""
import json
import resource
import statistics
import time
from concurrent.futures import ThreadPoolExecutor
from argon2 import PasswordHasher
def main():
kdf = PasswordHasher(time_cost=2, memory_cost=65536, parallelism=1)
samples = []
for _ in range(7):
start = time.perf_counter()
digest = kdf.hash("synthetic-measurement-password")
kdf.verify(digest, "synthetic-measurement-password")
samples.append((time.perf_counter() - start) * 1000)
def one(_):
start = time.perf_counter()
kdf.hash("synthetic-parallel-password")
return (time.perf_counter() - start) * 1000
with ThreadPoolExecutor(max_workers=2) as pool:
parallel = list(pool.map(one, range(4)))
print(
json.dumps(
{
"algorithm": "argon2id",
"memory_kib": 65536,
"time_cost": 2,
"parallelism": 1,
"runs_hash_plus_verify_ms": samples,
"median_hash_plus_verify_ms": statistics.median(samples),
"max_hash_plus_verify_ms": max(samples),
"two_concurrent_hash_ms": parallel,
"process_peak_rss_kib": resource.getrusage(resource.RUSAGE_SELF).ru_maxrss,
"kdf_slots_per_api": 2,
"api_replicas": 2,
"maximum_active_kdf_memory_mib": 256,
}
)
)
if __name__ == "__main__":
main()
+32
View File
@@ -0,0 +1,32 @@
import hashlib
from pathlib import Path
import psycopg
from .config import secret
def migrate():
with psycopg.connect(secret("DATABASE_URL")) as conn:
# Serialize independent migration processes without changing application data.
conn.execute("SELECT pg_advisory_xact_lock(2301002)")
conn.execute(
"CREATE TABLE IF NOT EXISTS schema_migrations "
"(name text PRIMARY KEY, checksum text NOT NULL)"
)
for path in sorted(Path("migrations").glob("*.sql")):
checksum = hashlib.sha256(path.read_bytes()).hexdigest()
old = conn.execute(
"SELECT checksum FROM schema_migrations WHERE name=%s", (path.name,)
).fetchone()
if old:
if old[0] != checksum:
raise RuntimeError("Изменена применённая миграция: " + path.name)
continue
conn.execute(path.read_text())
conn.execute("INSERT INTO schema_migrations VALUES (%s,%s)", (path.name, checksum))
print("Applied", path.name)
if __name__ == "__main__":
migrate()
+71
View File
@@ -0,0 +1,71 @@
import re
from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator, model_validator
class RequestModel(BaseModel):
model_config = ConfigDict(extra="forbid")
@field_validator("*", mode="after")
@classmethod
def nonblank(cls, value):
if isinstance(value, str):
value.encode("utf-8")
if isinstance(value, str) and not value.strip():
raise ValueError("Строка не может состоять из пробелов")
return value
class CreateUser(RequestModel):
username: StrictStr = Field(pattern=r"^[a-z0-9_]{3,32}$")
display_name: StrictStr = Field(min_length=1, max_length=100)
password: StrictStr = Field(min_length=12, max_length=128)
@field_validator("username")
@classmethod
def full_username(cls, value):
if not re.fullmatch(r"[a-z0-9_]{3,32}", value):
raise ValueError("Неверное имя пользователя")
return value
class CreateChat(RequestModel):
title: StrictStr = Field(min_length=1, max_length=100)
member_ids: list[UUID] = Field(min_length=1, max_length=100)
@field_validator("member_ids", mode="before")
@classmethod
def string_ids(cls, value):
if not isinstance(value, list) or any(not isinstance(x, str) for x in value):
raise ValueError("Ожидается массив UUID-строк")
return value
@field_validator("member_ids")
@classmethod
def unique_ids(cls, value):
if len(set(value)) != len(value):
raise ValueError("Участники не должны повторяться")
return value
class CreateMessage(BaseModel):
model_config = ConfigDict(extra="forbid")
text: StrictStr = Field(max_length=4000)
attachment_ids: list[UUID] = Field(default_factory=list, max_length=10)
@field_validator("attachment_ids", mode="before")
@classmethod
def string_ids(cls, value):
if not isinstance(value, list) or any(not isinstance(x, str) for x in value):
raise ValueError("Ожидается массив UUID-строк")
return value
@model_validator(mode="after")
def content(self):
self.text.encode("utf-8")
if len(set(self.attachment_ids)) != len(self.attachment_ids):
raise ValueError("Вложения не должны повторяться")
if not self.text.strip() and not self.attachment_ids:
raise ValueError("Пустое сообщение без вложений")
return self
+365
View File
@@ -0,0 +1,365 @@
import hashlib
import json
import logging
import os
import secrets
from contextlib import contextmanager
from datetime import UTC, datetime, timedelta
from threading import BoundedSemaphore
from uuid import UUID, uuid4
import psycopg
import redis
from argon2 import PasswordHasher
from argon2.exceptions import VerificationError
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool, PoolTimeout, TooManyRequests
from .config import secret
from .domain import Failure, timestamp
from .secrets_provider import refresh
from .telemetry import operation
logger = logging.getLogger("messenger")
KDF_SLOTS = BoundedSemaphore(2)
KDF = PasswordHasher(time_cost=2, memory_cost=65536, parallelism=1)
DUMMY_HASH = KDF.hash("invalid-user-dummy-password")
class DeadlineConnection(psycopg.Connection):
def wait(self, gen, interval=0.1, timeout=None):
try:
return super().wait(gen, interval=interval, timeout=timeout or 3.0)
except psycopg.errors._WaitTimeout:
# A paused server cannot complete cancellation. Discard an uncertain
# connection instead of returning it to the pool or waiting forever.
self.pgconn.finish()
raise
def public(row):
return {
k: (
str(v)
if isinstance(v, UUID)
else v.astimezone(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z")
if isinstance(v, datetime)
else bytes(v).decode("utf-8")
if isinstance(v, (bytes, memoryview))
else v
)
for k, v in row.items()
if k not in {"password_hash", "token_digest"}
}
class PostgresStore:
def __init__(self):
self.pool = ConnectionPool(
secret("DATABASE_URL"),
connection_class=DeadlineConnection,
min_size=1,
max_size=int(os.getenv("POOL_MAX_SIZE", "8")),
timeout=2,
max_waiting=32,
kwargs={
"autocommit": True,
"row_factory": dict_row,
"connect_timeout": 2,
"options": "-c statement_timeout=2500 -c lock_timeout=1500",
},
check=ConnectionPool.check_connection,
)
self.pool.wait(timeout=15)
self.cache = redis.Redis.from_url(
secret("CACHE_URL"),
socket_connect_timeout=0.2,
socket_timeout=0.2,
decode_responses=True,
)
self.ttl = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
if self.ttl < 1:
raise RuntimeError("SESSION_TTL_SECONDS должен быть положительным")
@contextmanager
def connection(self):
refresh()
try:
with operation("postgres.transaction", {"db.system.name": "postgresql"}):
with self.pool.connection() as conn:
yield conn
except (
psycopg.OperationalError,
psycopg.InterfaceError,
PoolTimeout,
TooManyRequests,
psycopg.errors.QueryCanceled,
psycopg.errors.LockNotAvailable,
):
raise Failure(503, "Хранилище временно недоступно") from None
def close(self):
self.cache.close()
self.pool.close()
def ready(self):
try:
with self.connection() as conn:
conn.execute(
"SELECT u.id,u.password_hash,c.id,m.chat_id,msg.id,s.token_digest,r.key,"
"a.id,a.stage_key,a.pipeline_version,a.variants,ma.attachment_id,ui.object_key,"
"j.id,j.source_key,j.attempt_limit,j.lease_until,j.traceparent,j.tracestate,j.accepted_at,o.job_id,o.published_at,"
"audit.job_id,f.error_code FROM users u,chats c,members m,messages msg,"
"sessions s,message_requests r,attachments a,message_attachments ma,"
"upload_intents ui,attachment_jobs j,job_outbox o,job_audit audit,failed_jobs f "
"WHERE false"
)
return True
except (Failure, psycopg.Error):
return False
def _cached_user(self, uid):
key = "user:v1:" + uid
try:
value = self.cache.get(key)
if value:
cached = json.loads(value)
if (
isinstance(cached, dict)
and cached.get("id") == uid
and set(cached) == {"id", "username", "display_name", "created_at"}
):
return cached
except (redis.RedisError, ValueError, TypeError):
logger.warning(json.dumps({"event": "cache_bypass"}))
user = self.user(uid)
if user:
try:
self.cache.setex(key, 60, json.dumps(user))
except redis.RedisError:
logger.warning(json.dumps({"event": "cache_bypass"}))
return user
def user(self, uid):
with self.connection() as conn:
row = conn.execute(
"SELECT id,username,display_name,created_at FROM users WHERE id=%s", (uid,)
).fetchone()
return public(row) if row else None
def register(self, data):
with KDF_SLOTS:
hashed = KDF.hash(data.pop("password"))
uid = str(uuid4())
try:
with self.connection() as conn:
row = conn.execute(
"INSERT INTO users VALUES (%s,%s,%s,%s,%s) "
"RETURNING id,username,display_name,created_at",
(
uid,
data["username"],
data["display_name"].encode("utf-8"),
hashed,
timestamp(),
),
).fetchone()
return public(row)
except psycopg.errors.UniqueViolation:
raise Failure(409, "Имя пользователя занято") from None
def login(self, username, password):
self.login_budget(username)
with self.connection() as conn:
row = conn.execute("SELECT * FROM users WHERE username=%s", (username,)).fetchone()
try:
if not KDF_SLOTS.acquire(blocking=False):
raise Failure(429, "Слишком много попыток входа", {"Retry-After": "1"})
try:
KDF.verify(row["password_hash"] if row else DUMMY_HASH, password)
finally:
KDF_SLOTS.release()
except VerificationError:
return None
if row is None:
return None
if KDF.check_needs_rehash(row["password_hash"]):
with KDF_SLOTS:
upgraded = KDF.hash(password)
with self.connection() as conn:
conn.execute(
"UPDATE users SET password_hash=%s WHERE id=%s AND password_hash=%s",
(upgraded, row["id"], row["password_hash"]),
)
token = secrets.token_urlsafe(32)
digest = hashlib.sha256(token.encode()).hexdigest()
expires = datetime.now(UTC) + timedelta(seconds=self.ttl)
with self.connection() as conn:
conn.execute("INSERT INTO sessions VALUES (%s,%s,%s)", (digest, row["id"], expires))
return {
"token": token,
"token_type": "Bearer",
"expires_at": public({"v": expires})["v"],
"user": public(row),
}
def login_budget(self, username):
key = hashlib.sha256(username.encode("utf-8")).hexdigest()
with self.connection() as conn, conn.transaction():
# Shared PostgreSQL window, serialized across both API replicas.
conn.execute("SELECT pg_advisory_xact_lock(2301007)")
conn.execute("DELETE FROM login_windows WHERE window_start<now()-interval '1 hour'")
for bucket, maximum in (("global", 30), ("user:" + key, 10)):
count = conn.execute(
"INSERT INTO login_windows(bucket,window_start,count) VALUES (%s,now(),1) "
"ON CONFLICT(bucket) DO UPDATE SET count=CASE WHEN login_windows.window_start<now()-interval '10 seconds' THEN 1 ELSE login_windows.count+1 END, "
"window_start=CASE WHEN login_windows.window_start<now()-interval '10 seconds' THEN now() ELSE login_windows.window_start END RETURNING count",
(bucket,),
).fetchone()["count"]
if count > maximum:
raise Failure(429, "Слишком много попыток входа", {"Retry-After": "10"})
def authenticate(self, token):
if not token or len(token) > 512:
return None
digest = hashlib.sha256(token.encode()).hexdigest()
# Session validity always comes from PostgreSQL, including on a cache hit.
with self.connection() as conn:
row = conn.execute(
"SELECT s.user_id FROM sessions s WHERE s.token_digest=%s AND s.expires_at>now()",
(digest,),
).fetchone()
return self._cached_user(str(row["user_id"])) if row else None
def logout(self, token):
with self.connection() as conn:
row = conn.execute(
"DELETE FROM sessions WHERE token_digest=%s AND expires_at>now() RETURNING user_id",
(hashlib.sha256(token.encode()).hexdigest(),),
).fetchone()
return row is not None
def create_chat(self, actor, data):
if actor not in data["member_ids"]:
raise Failure(400, "Вызывающий должен входить в чат")
cid = str(uuid4())
try:
with self.connection() as conn, conn.transaction():
row = conn.execute(
"INSERT INTO chats VALUES (%s,%s,%s) RETURNING *",
(cid, data["title"].encode("utf-8"), timestamp()),
).fetchone()
for index, uid in enumerate(data["member_ids"]):
conn.execute("INSERT INTO members VALUES (%s,%s,%s)", (cid, uid, index))
return {**public(row), "member_ids": data["member_ids"]}
except psycopg.errors.ForeignKeyViolation:
raise Failure(400, "Неизвестный участник") from None
@staticmethod
def _chat(conn, actor, cid):
row = conn.execute(
"SELECT c.* FROM chats c JOIN members m ON m.chat_id=c.id "
"WHERE c.id=%s AND m.user_id=%s",
(cid, actor),
).fetchone()
if row is None:
raise Failure(404, "Объект не найден")
members = conn.execute(
"SELECT user_id FROM members WHERE chat_id=%s ORDER BY position", (cid,)
).fetchall()
return {**public(row), "member_ids": [str(m["user_id"]) for m in members]}
def chat(self, actor, cid):
with self.connection() as conn:
return self._chat(conn, actor, cid)
def list_chats(self, actor, after, limit):
with self.connection() as conn:
sql = "SELECT c.* FROM chats c JOIN members m ON m.chat_id=c.id WHERE m.user_id=%s"
params = [actor]
if after:
sql += " AND (c.created_at,c.id)>(%s,%s)"
params.extend(after)
sql += " ORDER BY c.created_at,c.id LIMIT %s"
params.append(limit + 1)
rows = conn.execute(sql, params).fetchall()
return [self._chat(conn, actor, str(row["id"])) for row in rows]
def send(self, actor, cid, text, key, attachment_ids=None):
attachment_ids = attachment_ids or []
body_digest = hashlib.sha256(
json.dumps(
{"text": text, "attachment_ids": attachment_ids},
sort_keys=True,
ensure_ascii=False,
separators=(",", ":"),
).encode()
).hexdigest()
with self.connection() as conn, conn.transaction():
self._chat(conn, actor, cid)
# Serializes exactly this semantic key across all API processes. Hash
# collisions can only serialize unrelated requests, never mix results.
scope = json.dumps([actor, "POST", cid, key], separators=(",", ":"))
lock = int.from_bytes(hashlib.sha256(scope.encode()).digest()[:8], "big", signed=True)
conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock,))
old = conn.execute(
"SELECT body_digest,response FROM message_requests "
"WHERE actor_id=%s AND chat_id=%s AND key=%s AND expires_at>now()",
(actor, cid, key),
).fetchone()
if old:
if old["body_digest"] != body_digest:
raise Failure(409, "Ключ уже использован для другого сообщения")
return json.loads(bytes(old["response"]))
conn.execute(
"DELETE FROM message_requests WHERE actor_id=%s AND chat_id=%s AND key=%s",
(actor, cid, key),
)
for aid in attachment_ids:
attachment = conn.execute(
"SELECT id,status FROM attachments WHERE id=%s AND chat_id=%s FOR SHARE",
(aid, cid),
).fetchone()
if not attachment:
raise Failure(404, "Объект не найден")
if attachment["status"] == "failed":
raise Failure(409, "Вложение недоступно")
row = conn.execute(
"INSERT INTO messages VALUES (%s,%s,%s,%s,%s) RETURNING *",
(str(uuid4()), cid, actor, text.encode("utf-8"), timestamp()),
).fetchone()
for position, aid in enumerate(attachment_ids):
conn.execute(
"INSERT INTO message_attachments VALUES (%s,%s,%s)", (row["id"], aid, position)
)
response = {**public(row), "attachment_ids": attachment_ids}
conn.execute(
"INSERT INTO message_requests(actor_id,chat_id,key,body_digest,response,expires_at) "
"VALUES (%s,%s,%s,%s,%s,now()+interval '24 hours')",
(actor, cid, key, body_digest, json.dumps(response).encode("utf-8")),
)
return response
def history(self, actor, cid, after, limit):
with self.connection() as conn:
self._chat(conn, actor, cid)
sql = "SELECT * FROM messages WHERE chat_id=%s"
params = [cid]
if after:
sql += " AND (created_at,id)>(%s,%s)"
params.extend(after)
sql += " ORDER BY created_at,id LIMIT %s"
params.append(limit + 1)
result = []
for row in conn.execute(sql, params).fetchall():
attachments = conn.execute(
"SELECT attachment_id FROM message_attachments WHERE message_id=%s ORDER BY position",
(row["id"],),
).fetchall()
result.append(
{
**public(row),
"attachment_ids": [str(a["attachment_id"]) for a in attachments],
}
)
return result
+60
View File
@@ -0,0 +1,60 @@
"""Fresh connections with deliberately revoked worker material; no fallback."""
import json
import ssl
import urllib.error
import urllib.request
from pathlib import Path
import psycopg
import redis
def main():
old = json.loads(Path("/run/old-worker/material.json").read_text())
checks = {}
try:
conn = psycopg.connect(old["DATABASE_URL"], connect_timeout=2)
conn.close()
checks["old_database_password_denied"] = False
except psycopg.OperationalError:
checks["old_database_password_denied"] = True
url = old["BROKER_URL"].replace("/run/tls/identity/", "/run/old-worker/")
broker = redis.Redis.from_url(url, socket_connect_timeout=2, socket_timeout=2)
try:
broker.ping()
checks["old_broker_client_certificate_denied"] = False
except redis.RedisError:
checks["old_broker_client_certificate_denied"] = True
finally:
broker.close()
ctx = ssl.create_default_context(cafile="/run/tls/ca.pem")
ctx.load_cert_chain("/run/old-worker/cert.pem", "/run/old-worker/key.pem")
for name, url, body in (
("old_s3_certificate_denied", "https://storage:8443/messenger/staging/probe", None),
(
"old_bao_certificate_auth_denied",
"https://openbao:8200/v1/auth/cert/login",
b'{"name":"worker"}',
),
):
try:
with urllib.request.urlopen(
urllib.request.Request(
url, data=body, headers={"Content-Type": "application/json"}
),
context=ctx,
timeout=3,
):
checks[name] = False
except urllib.error.HTTPError as exc:
checks[name] = exc.code in {400, 403}
except (OSError, urllib.error.URLError):
checks[name] = True
print(json.dumps({"checks": checks, "passed": all(checks.values())}))
if not all(checks.values()):
raise SystemExit(1)
if __name__ == "__main__":
main()
+59
View File
@@ -0,0 +1,59 @@
"""Short-lived OpenBao certificate authentication; no root token or SecretID in app."""
import json
import os
import ssl
import threading
import time
import urllib.error
import urllib.request
from .domain import Failure
_lock = threading.Lock()
_values = {}
_valid_until = 0.0
def request(path, *, data=None, token=None):
context = ssl.create_default_context(cafile=os.getenv("BAO_CA_FILE", "/run/tls/ca.pem"))
context.load_cert_chain("/run/tls/identity/cert.pem", "/run/tls/identity/key.pem")
headers = {"Content-Type": "application/json"}
if token:
headers["X-Vault-Token"] = token
payload = json.dumps(data).encode() if data is not None else None
req = urllib.request.Request(
os.environ["BAO_ADDR"] + "/v1/" + path, data=payload, headers=headers
)
with urllib.request.urlopen(req, context=context, timeout=2) as response:
return json.load(response)
def refresh():
global _values, _valid_until
if not os.getenv("BAO_ROLE"):
return
with _lock:
if time.monotonic() < _valid_until:
return
try:
role = os.environ["BAO_ROLE"]
auth = request("auth/cert/login", data={"name": role})["auth"]
values = request("secret/data/services/" + role, token=auth["client_token"])["data"][
"data"
]
# Never keep credentials usable beyond the auth token's short lease.
_values = values
_valid_until = time.monotonic() + min(float(auth["lease_duration"]), 30) - 1
except (OSError, ValueError, KeyError, urllib.error.URLError):
_values = {}
_valid_until = 0
raise Failure(503, "Сервис секретов временно недоступен") from None
def value(name):
refresh()
try:
return _values[name]
except KeyError:
raise RuntimeError("Scoped secret unavailable: " + name) from None
+35
View File
@@ -0,0 +1,35 @@
"""Run API TLS boundary and loopback business server in one namespace."""
import os
import signal
import subprocess
import sys
import time
def main():
children = [subprocess.Popen([sys.executable, "-m", "messenger.tls_gateway"])]
env = dict(os.environ, APP_HOST="127.0.0.1")
children.append(subprocess.Popen([sys.executable, "-m", "messenger"], env=env))
def stop(*_):
for child in children:
child.terminate()
signal.signal(signal.SIGTERM, stop)
signal.signal(signal.SIGINT, stop)
try:
while all(child.poll() is None for child in children):
time.sleep(0.2)
finally:
stop()
for child in children:
try:
child.wait(timeout=6)
except subprocess.TimeoutExpired:
child.kill()
child.wait()
if __name__ == "__main__":
main()
+108
View File
@@ -0,0 +1,108 @@
"""Read-only/rollback permission probes plus disposable synthetic prefix objects."""
import io
import json
import os
import socket
import psycopg
import redis
from .domain import Failure
from .jobs import STREAM, broker_client
from .postgres import PostgresStore
from .secrets_provider import request
from .storage import ObjectStorage
def denied(action):
try:
action()
except (psycopg.errors.InsufficientPrivilege, redis.ResponseError, Failure):
return True
return False
def main():
checks = {}
store = PostgresStore()
broker = broker_client()
storage = ObjectStorage()
try:
with store.connection() as conn:
checks["db_identity_worker"] = (
conn.execute("SELECT current_user AS role").fetchone()["role"] == "worker"
)
checks["db_business_users_denied"] = denied(
lambda: conn.execute("SELECT password_hash FROM users LIMIT 0")
)
checks["db_business_messages_denied"] = denied(
lambda: conn.execute("SELECT text FROM messages LIMIT 0")
)
checks["db_create_denied"] = denied(
lambda: conn.execute("CREATE TABLE security_forbidden(id int)")
)
checks["db_owner_mutation_denied"] = denied(
lambda: conn.execute("UPDATE attachments SET owner_id=owner_id WHERE false")
)
checks["db_job_read_allowed"] = (
conn.execute("SELECT count(*) AS n FROM attachment_jobs").fetchone()["n"] >= 0
)
checks["broker_ping_allowed"] = broker.ping()
checks["broker_acl_admin_denied"] = denied(lambda: broker.acl_list())
checks["broker_publish_denied"] = denied(lambda: broker.xadd(STREAM, {"body": "forbidden"}))
checks["broker_wrong_stream_denied"] = denied(lambda: broker.xlen("other-stream"))
checks["broker_group_destroy_denied"] = denied(
lambda: broker.xgroup_destroy(STREAM, "nonexistent-security-probe")
)
key = "attachments/security-probe/v1/original"
storage.put(key, io.BytesIO(b"synthetic"), 9, "application/octet-stream")
checks["s3_results_prefix_write_allowed"] = storage.head(key)["ContentLength"] == 9
checks["s3_staging_write_denied"] = denied(
lambda: storage.put("staging/security-forbidden", io.BytesIO(b"x"), 1, "text/plain")
)
checks["s3_private_prefix_read_denied"] = denied(
lambda: storage.get("private/security-probe")
)
auth = request("auth/cert/login", data={"name": "worker"})["auth"]["client_token"]
checks["bao_own_path_allowed"] = bool(
request("secret/data/services/worker", token=auth)["data"]["data"]
)
import urllib.error
for name, path, data in (
("bao_other_path_denied", "secret/data/services/api", None),
("bao_policy_write_denied", "sys/policies/acl/forbidden", {"policy": ""}),
):
try:
request(path, data=data, token=auth)
checks[name] = False
except urllib.error.HTTPError as exc:
checks[name] = exc.code == 403
for host, port in (
("storage", 8333),
("storage", 9333),
("storage", 8888),
("broker", 6379),
("cache", 6379),
("api", 8080),
):
try:
with socket.create_connection((host, port), 1):
checks[f"no_plaintext_bypass_{host}_{port}"] = False
except OSError:
checks[f"no_plaintext_bypass_{host}_{port}"] = True
checks["worker_image_has_no_root_token"] = not os.path.exists(
"/operator-private/root-token"
)
print(json.dumps({"checks": checks, "passed": all(checks.values())}, sort_keys=True))
if not all(checks.values()):
raise SystemExit(1)
finally:
storage.close()
broker.close()
store.close()
if __name__ == "__main__":
main()
+106
View File
@@ -0,0 +1,106 @@
"""Private S3 access with bounded network waits; never expose storage keys in API."""
import os
from contextlib import contextmanager
import boto3
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
from .config import secret
from .domain import Failure
from .secrets_provider import refresh
from .telemetry import operation
class ObjectStorage:
def __init__(self, admin=False):
self.bucket = os.getenv("S3_BUCKET", "messenger")
prefix = "S3_ADMIN" if admin else "S3"
self.client = boto3.client(
"s3",
endpoint_url=os.getenv("S3_ENDPOINT", "http://s3:8333"),
aws_access_key_id=secret(prefix + "_ACCESS_KEY"),
aws_secret_access_key=secret(prefix + "_SECRET_KEY"),
region_name="us-east-1",
verify=os.getenv("S3_CA_FILE", "/run/tls/ca.pem"),
config=Config(
client_cert=("/run/tls/identity/cert.pem", "/run/tls/identity/key.pem"),
connect_timeout=1,
read_timeout=2,
retries={"total_max_attempts": 1},
s3={"addressing_style": "path"},
request_checksum_calculation="when_required",
response_checksum_validation="when_required",
),
)
@contextmanager
def boundary(self):
refresh()
try:
with operation("s3.request", {"rpc.system": "aws-api", "rpc.service": "S3"}):
yield
except (BotoCoreError, ClientError):
raise Failure(503, "Хранилище файлов временно недоступно") from None
def put(self, key, fileobj, size, content_type):
fileobj.seek(0)
with self.boundary():
self.client.put_object(
Bucket=self.bucket,
Key=key,
Body=fileobj,
ContentLength=size,
ContentType=content_type,
)
def get(self, key):
refresh()
try:
with operation("s3.get", {"rpc.system": "aws-api", "rpc.service": "S3"}):
return self.client.get_object(Bucket=self.bucket, Key=key)
except ClientError as exc:
if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404"}:
raise Failure(404, "Объект не найден") from None
raise Failure(503, "Хранилище файлов временно недоступно") from None
except BotoCoreError:
raise Failure(503, "Хранилище файлов временно недоступно") from None
def head(self, key):
refresh()
try:
with operation("s3.head", {"rpc.system": "aws-api", "rpc.service": "S3"}):
return self.client.head_object(Bucket=self.bucket, Key=key)
except ClientError as exc:
if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404", "NotFound"}:
return None
raise Failure(503, "Хранилище файлов временно недоступно") from None
except BotoCoreError:
raise Failure(503, "Хранилище файлов временно недоступно") from None
def exists(self, key):
return self.head(key) is not None
def delete(self, key):
with self.boundary():
self.client.delete_object(Bucket=self.bucket, Key=key)
def objects(self, page_size=100):
with self.boundary():
paginator = self.client.get_paginator("list_objects_v2")
for page in paginator.paginate(
Bucket=self.bucket, PaginationConfig={"PageSize": page_size}
):
yield from page.get("Contents", [])
def ready(self):
try:
with self.boundary():
self.client.head_bucket(Bucket=self.bucket)
return True
except Failure:
return False
def close(self):
self.client.close()
+30
View File
@@ -0,0 +1,30 @@
"""Idempotent private bucket bootstrap; this process alone receives admin keys."""
import time
from botocore.exceptions import BotoCoreError, ClientError
from .storage import ObjectStorage
def main():
storage = ObjectStorage(admin=True)
for attempt in range(30):
try:
try:
storage.client.head_bucket(Bucket=storage.bucket)
except ClientError as exc:
if exc.response["Error"]["Code"] not in {"404", "NoSuchBucket", "NotFound"}:
raise
storage.client.create_bucket(Bucket=storage.bucket)
print("Private bucket ready; anonymous identity is not configured.")
storage.close()
return
except (ClientError, BotoCoreError):
if attempt == 29:
raise RuntimeError("S3 bootstrap timeout") from None
time.sleep(1)
if __name__ == "__main__":
main()
+357
View File
@@ -0,0 +1,357 @@
"""Bounded OTLP telemetry; technical attributes only, no payload/credential capture."""
import json
import logging
import os
import threading
from contextlib import contextmanager
from datetime import UTC, datetime
from uuid import uuid4
import psutil
import requests
from opentelemetry import metrics, propagate, trace
from opentelemetry._logs import set_logger_provider
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import SpanProcessor, TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExportResult
from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
from opentelemetry.trace import SpanKind, Status, StatusCode
HTTP_BUCKETS = (0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10)
JOB_BUCKETS = (0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 20, 45, 60, 120)
INSTANCE_ID = str(uuid4())
_runtime = None
class Accounting(dict):
def __init__(self, keys):
super().__init__((key, 0) for key in keys)
self.lock = threading.Lock()
def add(self, key, value):
with self.lock:
self[key] += value
def snapshot(self):
with self.lock:
return dict(self)
class RecordedSpans(SpanProcessor):
def __init__(self, stats):
self.stats = stats
def on_start(self, span, parent_context=None):
pass
def on_end(self, span):
if span.context.trace_flags.sampled:
self.stats.add("spans_recorded", 1)
class CountingSession(requests.Session):
"""Count actual protobuf HTTP sends, including transport retries, without retaining data."""
def __init__(self, stats, signal):
super().__init__()
self.stats, self.signal = stats, signal
def send(self, request, **kwargs):
self.stats.add(self.signal + "_wire_bytes", len(request.body or b""))
self.stats.add(self.signal + "_http_sends", 1)
return super().send(request, **kwargs)
class ObservedSpanExporter(OTLPSpanExporter):
def __init__(self, stats, **kwargs):
self.stats = stats
super().__init__(**kwargs)
def export(self, spans):
self.stats.add("spans_in_flight", len(spans))
try:
result = super().export(spans)
key = "spans_exported" if result == SpanExportResult.SUCCESS else "spans_export_failed"
self.stats.add(key, len(spans))
return result
except Exception:
self.stats.add("spans_export_failed", len(spans))
raise
finally:
self.stats.add("spans_in_flight", -len(spans))
class TechnicalLogs(logging.Filter):
def filter(self, record):
return record.name.startswith("messenger")
class JsonFormatter(logging.Formatter):
def format(self, record):
try:
event = json.loads(record.getMessage())
if not isinstance(event, dict):
event = {"event": "application_event"}
except ValueError:
event = {"event": "application_event"}
context = trace.get_current_span().get_span_context()
return json.dumps(
{
**event,
"timestamp": datetime.now(UTC).isoformat(),
"severity": record.levelname,
"service.name": os.getenv("OTEL_SERVICE_NAME", "messenger-api"),
"trace_id": format(context.trace_id, "032x") if context.is_valid else None,
"span_id": format(context.span_id, "016x") if context.is_valid else None,
}
)
class Telemetry:
def __init__(self, service):
self.service = service
self.enabled = os.getenv("OTEL_ENABLED", "1") == "1"
self.busy = 0
self.providers = []
self.export_stats = Accounting(
(
"spans_recorded",
"spans_in_flight",
"spans_exported",
"spans_export_failed",
"traces_wire_bytes",
"traces_http_sends",
"metrics_wire_bytes",
"metrics_http_sends",
"logs_wire_bytes",
"logs_http_sends",
)
)
self.stop = threading.Event()
self.thread = None
resource = Resource.create(
{
"service.name": service,
"service.version": os.getenv("SERVICE_VERSION", "lab7-v1"),
"service.instance.id": INSTANCE_ID,
"deployment.environment.name": "lab",
}
)
os.environ["OTEL_SERVICE_NAME"] = service
root = logging.getLogger()
root.setLevel(logging.INFO)
for handler in root.handlers:
handler.setFormatter(JsonFormatter())
if not root.handlers:
handler = logging.StreamHandler()
handler.setFormatter(JsonFormatter())
root.addHandler(handler)
if self.enabled:
endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector:4318").rstrip("/")
common = {"timeout": 1.0}
# Standard OTLP environment variables also support HTTPS/mTLS in lab7.
trace_provider = TracerProvider(
resource=resource,
sampler=ParentBased(TraceIdRatioBased(float(os.getenv("OTEL_SAMPLE_RATIO", "1")))),
)
trace_provider.add_span_processor(RecordedSpans(self.export_stats))
self.batch_spans = BatchSpanProcessor(
ObservedSpanExporter(
self.export_stats,
endpoint=endpoint + "/v1/traces",
session=CountingSession(self.export_stats, "traces"),
**common,
),
max_queue_size=256,
max_export_batch_size=64,
schedule_delay_millis=500,
export_timeout_millis=1500,
)
trace_provider.add_span_processor(self.batch_spans)
trace.set_tracer_provider(trace_provider)
reader = PeriodicExportingMetricReader(
OTLPMetricExporter(
endpoint=endpoint + "/v1/metrics",
session=CountingSession(self.export_stats, "metrics"),
**common,
),
export_interval_millis=int(os.getenv("OTEL_METRIC_INTERVAL_MS", "5000")),
export_timeout_millis=1500,
)
meter_provider = MeterProvider(
resource=resource,
metric_readers=[reader],
views=[
View(
instrument_name="messenger.http.duration",
aggregation=ExplicitBucketHistogramAggregation(HTTP_BUCKETS),
),
View(
instrument_name="messenger.job.duration",
aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS),
),
View(
instrument_name="messenger.attachment.ready.duration",
aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS),
),
],
)
metrics.set_meter_provider(meter_provider)
logger_provider = LoggerProvider(resource=resource)
logger_provider.add_log_record_processor(
BatchLogRecordProcessor(
OTLPLogExporter(
endpoint=endpoint + "/v1/logs",
session=CountingSession(self.export_stats, "logs"),
**common,
),
max_queue_size=256,
max_export_batch_size=64,
schedule_delay_millis=500,
export_timeout_millis=1500,
)
)
set_logger_provider(logger_provider)
otlp_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
otlp_handler.addFilter(TechnicalLogs())
# Keep machine-readable JSON in OTLP log body, preserving request ID.
otlp_handler.setFormatter(JsonFormatter())
root.addHandler(otlp_handler)
self.providers = [trace_provider, meter_provider, logger_provider]
self.tracer = trace.get_tracer("messenger", "1.0")
meter = metrics.get_meter("messenger", "1.0")
self.requests = meter.create_counter("messenger.http.requests", unit="{request}")
self.http_duration = meter.create_histogram("messenger.http.duration", unit="s")
self.sampling = meter.create_counter("messenger.http.trace.decisions", unit="{request}")
self.jobs = meter.create_counter("messenger.jobs", unit="{job}")
self.job_duration = meter.create_histogram("messenger.job.duration", unit="s")
self.ready_duration = meter.create_histogram(
"messenger.attachment.ready.duration", unit="s"
)
self.queue_depth = meter.create_gauge("messenger.queue.depth", unit="{job}")
self.oldest_age = meter.create_gauge("messenger.queue.oldest.age", unit="s")
self.pool_used = meter.create_gauge("messenger.db.pool.used", unit="{connection}")
self.pool_waiting = meter.create_gauge("messenger.db.pool.waiting", unit="{request}")
self.worker_busy = meter.create_gauge("messenger.worker.busy", unit="{worker}")
self.worker_capacity = meter.create_gauge("messenger.worker.capacity", unit="{worker}")
self.process_cpu = meter.create_gauge("messenger.process.cpu", unit="%")
self.process_memory = meter.create_gauge("messenger.process.memory", unit="By")
self.export_observation = meter.create_gauge("messenger.telemetry.export", unit="{item}")
def start_background(self, store):
process = psutil.Process()
process.cpu_percent()
def collect():
while not self.stop.wait(2):
counts = self.export_stats.snapshot()
# Read-only queue observation of pinned SDK 1.45.2. Conservation
# is evaluated after quiescence, not during concurrent export.
pending = len(self.batch_spans._batch_processor._queue)
counts["spans_pending"] = pending
counts["span_queue_capacity"] = 256
counts["spans_queue_lost"] = max(
0,
counts["spans_recorded"]
- counts["spans_exported"]
- counts["spans_export_failed"]
- counts["spans_in_flight"]
- pending,
)
for name, value in counts.items():
self.export_observation.set(value, {"observation": name})
stats = store.pool.get_stats()
self.pool_used.set(
max(0, stats.get("pool_size", 0) - stats.get("pool_available", 0))
)
self.pool_waiting.set(stats.get("requests_waiting", 0))
self.process_cpu.set(process.cpu_percent())
self.process_memory.set(process.memory_info().rss)
if self.service == "messenger-worker":
self.worker_busy.set(self.busy)
self.worker_capacity.set(1)
if self.service == "messenger-dispatcher":
try:
with store.connection() as conn:
row = conn.execute(
"SELECT count(*) AS depth, COALESCE(EXTRACT(EPOCH FROM now()-min(created_at)),0) AS age FROM attachment_jobs WHERE state IN ('queued','processing')"
).fetchone()
self.queue_depth.set(row["depth"], {"queue": "attachments"})
self.oldest_age.set(float(row["age"]), {"queue": "attachments"})
except Exception:
logging.getLogger("messenger.telemetry").warning(
json.dumps({"event": "queue_observation_unavailable"})
)
if self.enabled:
self.thread = threading.Thread(
target=collect, name="telemetry-observations", daemon=True
)
self.thread.start()
def close(self):
self.stop.set()
if self.thread:
self.thread.join(timeout=4)
# Business shutdown has a fixed budget even if telemetry endpoints are down.
def shutdown_providers():
for provider in self.providers:
provider.shutdown()
shutdown = threading.Thread(
target=shutdown_providers, name="telemetry-shutdown", daemon=True
)
shutdown.start()
shutdown.join(timeout=2)
def setup(service):
global _runtime
if _runtime is None:
_runtime = Telemetry(service)
return _runtime
def current():
return _runtime
def carrier():
result = {}
propagate.inject(result)
return {key: result[key] for key in ("traceparent", "tracestate") if key in result}
@contextmanager
def operation(name, attributes=None, context=None, kind=SpanKind.CLIENT):
if (
kind == SpanKind.CLIENT
and context is None
and not trace.get_current_span().get_span_context().is_valid
):
yield trace.get_current_span()
return
tracer = trace.get_tracer("messenger", "1.0")
with tracer.start_as_current_span(
name,
attributes=attributes or {},
context=context,
kind=kind,
record_exception=False,
set_status_on_exception=False,
) as span:
try:
yield span
except BaseException as exc:
span.set_status(Status(StatusCode.ERROR, type(exc).__name__))
raise
+272
View File
@@ -0,0 +1,272 @@
"""Small TLS boundary: exact DNS SAN allowlist, revocation, bounded TCP forwarding.
The upstream is always loopback in the same network namespace. HTTP mode is used
only for S3 prefix authorization; AWS signatures are still checked by S3 itself.
"""
import argparse
import http.client
import json
import os
import selectors
import socket
import socketserver
import ssl
import threading
from contextlib import closing
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from urllib.parse import unquote, urlsplit
MAX_BODY = 10 * 1024 * 1024
def settings():
return json.loads(Path(os.environ["TLS_GATEWAY_CONFIG"]).read_text())
def role(cert, allowed):
names = {v for k, v in cert.get("subjectAltName", ()) if k == "DNS"}
found = names.intersection(allowed)
if len(found) != 1:
raise ssl.SSLError("service_identity_denied")
revoked = Path(os.getenv("TLS_REVOKED_FILE", "/run/tls/revoked.json"))
serials = json.loads(revoked.read_text()) if revoked.exists() else []
if cert.get("serialNumber", "").lstrip("0").lower() in serials:
raise ssl.SSLError("certificate_revoked")
return next(iter(found))
def context(cfg):
ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
ctx.set_alpn_protocols(["http/1.1"] if cfg.get("http") else ["h2", "http/1.1"])
ctx.minimum_version = ssl.TLSVersion.TLSv1_2
ctx.load_cert_chain(cfg["cert"], cfg["key"])
if cfg.get("allowed"):
ctx.verify_mode = ssl.CERT_REQUIRED
ctx.load_verify_locations(cfg["ca"])
ctx.options |= ssl.OP_NO_TICKET
return ctx
class Boundary:
def process_request(self, request, address):
if not self.slots.acquire(blocking=False):
request.close()
return
try:
super().process_request(request, address)
except Exception:
self.slots.release()
raise
def process_request_thread(self, request, address):
try:
request.settimeout(2)
wrapped = context(self.cfg).wrap_socket(request, server_side=True)
wrapped.identity = (
role(wrapped.getpeercert(), self.cfg["allowed"])
if self.cfg.get("allowed")
else "public"
)
wrapped.settimeout(15)
super().process_request_thread(wrapped, address)
except (OSError, ValueError):
request.close()
finally:
self.slots.release()
class TCPHandler(socketserver.BaseRequestHandler):
def handle(self):
try:
with socket.create_connection(
("127.0.0.1", self.server.cfg["upstream"]), 2
) as upstream:
self.request.setblocking(False)
upstream.setblocking(False)
with selectors.DefaultSelector() as selector:
selector.register(self.request, selectors.EVENT_READ, upstream)
selector.register(upstream, selectors.EVENT_READ, self.request)
while True:
role(self.request.getpeercert(), self.server.cfg["allowed"])
pending = self.request.pending()
events = selector.select(timeout=0 if pending else 30)
if pending and not any(key.fileobj is self.request for key, _ in events):
events.append((selector.get_key(self.request), selectors.EVENT_READ))
if not events:
return
for key, _ in events:
# Recheck after the bounded wait: a revoked peer must
# not forward one extra command when it wakes select.
role(self.request.getpeercert(), self.server.cfg["allowed"])
try:
chunk = key.fileobj.recv(65536)
except ssl.SSLWantReadError:
continue
if not chunk:
return
# Small writes with finite socket timeout prevent unbounded
# per-connection queues when a peer stops consuming.
destination = key.data
destination.settimeout(5)
destination.sendall(chunk)
destination.setblocking(False)
except (OSError, ssl.SSLError):
return
class TCPServer(Boundary, socketserver.ThreadingTCPServer):
allow_reuse_address = True
daemon_threads = True
request_queue_size = 64
class HTTPHandler(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
def handle_expect_100(self):
return True
def log_message(self, *_):
pass
def do_GET(self):
self.forward()
do_HEAD = do_GET
do_PUT = do_GET
do_POST = do_GET
do_DELETE = do_GET
def forward(self):
self.close_connection = True
cfg = self.server.cfg
identity = self.connection.identity
parsed = urlsplit(self.path)
path = unquote(parsed.path)
parts = path.split("/")
if (
parsed.scheme
or parsed.netloc
or parsed.fragment
or any(p in {".", ".."} for p in parts)
or "\\" in path
or "%" in parsed.path
or any(ord(c) < 32 for c in path)
):
self.send_error(403)
return
if cfg.get("backend") and identity != "operator":
allowed = (
self.command == "GET"
if identity == "grafana"
else (
identity == "collector"
and cfg["backend"] == "loki"
and self.command == "POST"
and parsed.path == "/otlp/v1/logs"
)
)
if not allowed:
self.send_error(403)
return
if cfg.get("s3"):
if (
self.headers.get("Host", "").lower() != "storage:8443"
or (identity != "operator" and parsed.query)
or self.headers.get("x-amz-copy-source")
):
self.send_error(403)
return
# A worker can read its sources, write versioned results and remove
# staging. It cannot list buckets, overwrite staging or touch others.
permitted = identity == "operator"
if identity == "api":
permitted = (
path == "/messenger"
and self.command == "HEAD"
or (
path.startswith("/messenger/staging/")
and self.command in {"PUT", "GET", "HEAD", "DELETE"}
)
or (
path.startswith("/messenger/attachments/")
and self.command in {"GET", "HEAD"}
)
)
elif identity == "worker":
permitted = (
path.startswith("/messenger/staging/")
and self.command in {"GET", "HEAD", "DELETE"}
) or (
path.startswith("/messenger/attachments/")
and self.command in {"PUT", "GET", "HEAD"}
)
if not permitted:
self.send_error(403)
return
try:
length = int(self.headers.get("Content-Length", "0"))
body_limit = MAX_BODY if cfg.get("s3") else 1024 * 1024
if length < 0 or length > body_limit or self.headers.get("Transfer-Encoding"):
self.send_error(413)
return
if self.headers.get("Expect", "").lower() == "100-continue":
self.wfile.write(b"HTTP/1.1 100 Continue\r\n\r\n")
self.wfile.flush()
body = self.rfile.read(length)
if len(body) != length:
self.send_error(400)
return
headers = {
k: v
for k, v in self.headers.items()
if k.lower() not in {"connection", "proxy-connection", "expect"}
}
with closing(
http.client.HTTPConnection("127.0.0.1", cfg["upstream"], timeout=5)
) as upstream:
upstream.request(self.command, self.path, body=body, headers=headers)
response = upstream.getresponse()
self.send_response(response.status)
for key, value in response.getheaders():
if key.lower() not in {"connection", "transfer-encoding", "server", "date"}:
self.send_header(key, value)
self.send_header("Connection", "close")
self.end_headers()
if self.command != "HEAD":
while chunk := response.read(65536):
self.wfile.write(chunk)
except (OSError, ValueError, http.client.HTTPException):
self.close_connection = True
class HTTPServer(Boundary, ThreadingHTTPServer):
daemon_threads = True
def serve(cfg):
cls = HTTPServer if cfg.get("http") else TCPServer
handler = HTTPHandler if cfg.get("http") else TCPHandler
with cls(("0.0.0.0", cfg["port"]), handler) as server:
server.cfg = cfg
server.slots = threading.BoundedSemaphore(2 if cfg.get("http") else 32)
server.serve_forever(poll_interval=0.2)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--config")
args = parser.parse_args()
cfg = json.loads(Path(args.config).read_text()) if args.config else settings()
listeners = cfg if isinstance(cfg, list) else [cfg]
threads = [threading.Thread(target=serve, args=(item,), daemon=True) for item in listeners]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
if __name__ == "__main__":
main()
+132
View File
@@ -0,0 +1,132 @@
"""Operator-only network probe: real TLS, exact role authorization and closed raw ports."""
import json
import socket
import ssl
from pathlib import Path
CA = "/run/tls/ca.pem"
IDENTITIES = Path("/run/audit-identities")
def exchange(host, port, role, payload, *, hostname=None, trust=True):
context = (
ssl.create_default_context(cafile=CA) if trust else ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
)
if role:
context.load_cert_chain(IDENTITIES / role / "cert.pem", IDENTITIES / role / "key.pem")
context.set_alpn_protocols(["h2"] if port == 44317 else ["http/1.1"])
try:
with socket.create_connection((host, port), 2) as raw:
with context.wrap_socket(raw, server_hostname=hostname or host) as secured:
secured.settimeout(3)
secured.sendall(payload)
result = secured.recv(4096)
return bool(result), result
except (OSError, ssl.SSLError):
return False, b""
def main():
checks = {}
# Every gateway is tested positively with its intended role and negatively
# with another valid certificate from the SAME CA, no certificate, foreign
# CA certificate, empty trust and wrong server hostname.
targets = [
(
"api",
9443,
"lb",
b"GET /health/ready HTTP/1.1\r\nHost: api\r\nConnection: close\r\n\r\n",
),
("broker", 6380, "worker", b"*1\r\n$4\r\nPING\r\n"),
("cache", 6380, "api", b"*1\r\n$4\r\nPING\r\n"),
(
"storage",
8443,
"worker",
b"HEAD /messenger HTTP/1.1\r\nHost: storage:8443\r\nConnection: close\r\n\r\n",
),
(
"collector",
4318,
"api",
b"POST /v1/traces HTTP/1.1\r\nHost: collector\r\nContent-Type: application/json\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}",
),
(
"collector",
9464,
"prometheus",
b"GET /metrics HTTP/1.1\r\nHost: collector\r\nConnection: close\r\n\r\n",
),
(
"collector",
8888,
"prometheus",
b"GET /metrics HTTP/1.1\r\nHost: collector\r\nConnection: close\r\n\r\n",
),
(
"prometheus",
9443,
"grafana",
b"GET /api/v1/query?query=up HTTP/1.1\r\nHost: prometheus\r\nConnection: close\r\n\r\n",
),
(
"loki",
9443,
"grafana",
b"GET /loki/api/v1/labels HTTP/1.1\r\nHost: loki\r\nConnection: close\r\n\r\n",
),
(
"tempo",
9443,
"grafana",
b"GET /api/search HTTP/1.1\r\nHost: tempo\r\nConnection: close\r\n\r\n",
),
]
for host, port, role, payload in targets:
key = f"{host}_{port}"
ok, response = exchange(host, port, role, payload)
checks[key + "_allowed_identity"] = ok and (
response.startswith(b"HTTP/1.1") or response.startswith(b"-NOAUTH")
)
wrong = "worker" if host == "api" else "lb"
for label, identity, hostname, trust in (
("wrong_trusted_identity", wrong, None, True),
("no_client_cert", None, None, True),
("foreign_ca_client", "foreign", None, True),
("wrong_server_hostname", role, "invalid.example", True),
("empty_trust", role, None, False),
):
ok, _ = exchange(host, port, identity, payload, hostname=hostname, trust=trust)
checks[key + "_" + label + "_denied"] = not ok
otlp = b"POST /v1/traces HTTP/1.1\r\nHost: tempo\r\nContent-Type: application/json\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}"
checks["tempo_otlp_collector_allowed"] = exchange("tempo", 44318, "collector", otlp)[0]
checks["tempo_otlp_wrong_role_denied"] = not exchange("tempo", 44318, "worker", otlp)[0]
for host, ports in (
("api", [8080]),
("storage", [8333, 8888, 9333]),
("broker", [6379]),
("cache", [6379]),
("collector", [14318, 19464, 18888, 13133]),
("prometheus", [9090]),
("loki", [3100]),
("tempo", [3200, 4317, 4318]),
):
for port in ports:
try:
with socket.create_connection((host, port), 1):
checks[f"raw_{host}_{port}_closed"] = False
except OSError:
checks[f"raw_{host}_{port}_closed"] = True
# Datasource roles cannot submit arbitrary telemetry or mutate Prometheus.
forbidden = b"POST /otlp/v1/logs HTTP/1.1\r\nHost: loki\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}"
ok, body = exchange("loki", 9443, "grafana", forbidden)
checks["grafana_loki_write_denied"] = ok and b" 403 " in body.split(b"\r\n", 1)[0]
print(json.dumps({"checks": checks, "passed": all(checks.values())}, sort_keys=True))
if not all(checks.values()):
raise SystemExit(1)
if __name__ == "__main__":
main()
+292
View File
@@ -0,0 +1,292 @@
"""Separate serial worker (prefetch=1) and transactional-outbox dispatcher."""
import argparse
import hashlib
import json
import logging
import re
import signal
import subprocess
import sys
import tempfile
import time
from datetime import datetime
from pathlib import Path
from uuid import UUID, uuid4
import redis
from opentelemetry import propagate, trace
from opentelemetry.trace import SpanKind, Status, StatusCode
from . import telemetry
from .domain import Failure
from .jobs import GROUP, STREAM, Jobs, broker_client, fault
from .postgres import PostgresStore
from .secrets_provider import refresh
from .storage import ObjectStorage
logger = logging.getLogger("messenger.worker")
logging.basicConfig(level=logging.INFO, format="%(message)s")
stopping = False
MAX_JOB_BYTES = 8192
def stop(signum, frame):
global stopping
stopping = True
def process(job, jobs, storage):
fault(jobs.store, "before_effect", job["attachment_id"])
with tempfile.TemporaryDirectory(prefix="image-job-") as directory:
root = Path(directory)
source = root / "source"
response = storage.get(job["source_key"])
try:
if response["ContentLength"] > 10_485_760:
raise ValueError("invalid_image")
with source.open("wb") as target:
while block := response["Body"].read(64 * 1024):
target.write(block)
if target.tell() > 10_485_760:
raise ValueError("invalid_image")
finally:
response["Body"].close()
with source.open("rb") as file:
digest = hashlib.file_digest(file, "sha256").hexdigest()
if digest != job["source_sha256"] or source.stat().st_size != job["source_size"]:
raise ValueError("invalid_image")
# Resize executes outside the API and is bounded in wall time, CPU and address space.
result = subprocess.run(
[
sys.executable,
"-m",
"messenger.image_pipeline",
str(source),
directory,
str(job["pipeline_version"]),
],
timeout=20,
capture_output=True,
check=False,
)
if result.returncode:
raise ValueError("invalid_image")
variants = json.loads((root / "variants.json").read_text())
prefix = f"attachments/{job['attachment_id']}/v{job['pipeline_version']}"
with source.open("rb") as file:
storage.put(
prefix + "/original", file, source.stat().st_size, job["source_content_type"]
)
for variant in variants:
path = root / variant["name"]
with path.open("rb") as file:
storage.put(
prefix + "/" + variant["name"],
file,
variant["size_bytes"],
variant["content_type"],
)
fault(jobs.store, "after_effect", job["attachment_id"])
published = jobs.complete(job, prefix + "/original", variants)
if published:
fault(jobs.store, "after_result_commit", job["attachment_id"])
cleanup_ready(jobs.store, storage, job["attachment_id"])
return published
def cleanup_ready(store, storage, attachment_id):
# Same row lock as reprocess/replay: cannot delete a source chosen by a new job.
with store.connection() as conn, conn.transaction():
a = conn.execute(
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,)
).fetchone()
if (
a
and a["status"] == "ready"
and a["stage_key"]
and a["stage_key"].startswith("staging/")
):
storage.delete(a["stage_key"])
conn.execute("UPDATE attachments SET stage_key=NULL WHERE id=%s", (attachment_id,))
def validate_delivery(body):
if not isinstance(body, (str, bytes, bytearray)):
raise ValueError("invalid_job")
encoded = body.encode("utf-8") if isinstance(body, str) else body
if len(encoded) > MAX_JOB_BYTES:
raise ValueError("invalid_job")
try:
data = json.loads(encoded)
except RecursionError:
raise ValueError("invalid_job") from None
expected = {
"schema_version",
"job_id",
"type",
"attachment_id",
"pipeline_version",
"attempt",
"created_at",
}
optional = {"traceparent", "tracestate"}
if not isinstance(data, dict) or not expected <= set(data) or set(data) - expected - optional:
raise ValueError("invalid_job")
if (
type(data["schema_version"]) is not int
or data["schema_version"] != 1
or data["type"] != "attachment.process.v1"
):
raise ValueError("invalid_job")
if not isinstance(data["job_id"], str) or not isinstance(data["attachment_id"], str):
raise ValueError("invalid_job")
for field in ("job_id", "attachment_id"):
canonical = str(UUID(data[field]))
if data[field].lower() != canonical:
raise ValueError("invalid_job")
# RFC UUID hex is case-insensitive; PostgreSQL and state comparisons use
# the same canonical spelling, so a valid uppercase delivery is not lost.
data[field] = canonical
if type(data["pipeline_version"]) is not int or data["pipeline_version"] < 1:
raise ValueError("invalid_job")
if type(data["attempt"]) is not int or data["attempt"] < 1:
raise ValueError("invalid_job")
created = data["created_at"]
if (
not isinstance(created, str)
or len(created) > 64
or not re.fullmatch(
r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]+)?Z", created
)
):
raise ValueError("invalid_job")
datetime.fromisoformat(created[:-1] + "+00:00")
if "traceparent" in data and (
not isinstance(data["traceparent"], str)
or not re.fullmatch(r"00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}", data["traceparent"])
):
raise ValueError("invalid_job")
if "tracestate" in data and (
not isinstance(data["tracestate"], str) or len(data["tracestate"]) > 512
):
raise ValueError("invalid_job")
return data
def consume_one(store, jobs, storage, broker, entry):
entry_id, fields = entry
try:
body = validate_delivery(fields.get("body", fields.get(b"body")))
except (KeyError, ValueError, TypeError):
logger.error(json.dumps({"event": "invalid_delivery"}))
broker.xack(STREAM, GROUP, entry_id)
broker.xdel(STREAM, entry_id)
return
with telemetry.operation(
"attachments consume", context=propagate.extract(body), kind=SpanKind.CONSUMER
):
consume_job(store, jobs, storage, broker, body)
# Durable state owns recovery even when notification acknowledgements repeat.
broker.xack(STREAM, GROUP, entry_id)
broker.xdel(STREAM, entry_id)
def consume_job(store, jobs, storage, broker, body):
job = jobs.claim(body["job_id"], body["attachment_id"], body["pipeline_version"])
t = telemetry.current()
started = time.perf_counter()
outcome = "duplicate"
if job:
try:
if t:
t.busy = 1
outcome = "success" if process(job, jobs, storage) else "superseded"
except ValueError:
jobs.fail(job, "invalid_image", permanent=True)
outcome = "failed"
except (Failure, subprocess.TimeoutExpired, OSError, RuntimeError) as exc:
logger.warning(
json.dumps(
{"event": "job_retry", "job_id": str(job["id"]), "kind": type(exc).__name__}
)
)
jobs.fail(job, "processing_failed")
outcome = "retry" if job["attempt"] < job["attempt_limit"] else "failed"
finally:
if t:
t.busy = 0
attrs = {"job_type": "attachment.process.v1", "outcome": outcome}
t.jobs.add(1, attrs)
t.job_duration.record(time.perf_counter() - started, attrs)
span = trace.get_current_span()
span.set_attributes({"job.type": "attachment.process.v1", "job.outcome": outcome})
if outcome in {"failed", "retry"}:
span.set_status(Status(StatusCode.ERROR, "processing_failed"))
logger.info(json.dumps({"event": "job_finished", "outcome": outcome}))
else:
trace.get_current_span().set_attributes(
{"job.type": "attachment.process.v1", "job.outcome": "duplicate"}
)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--dispatcher", action="store_true")
args = parser.parse_args()
signal.signal(signal.SIGTERM, stop)
signal.signal(signal.SIGINT, stop)
t = telemetry.setup("messenger-dispatcher" if args.dispatcher else "messenger-worker")
store = PostgresStore()
t.start_background(store)
jobs = Jobs(store)
broker = broker_client()
storage = None if args.dispatcher else ObjectStorage()
consumer = "worker-" + str(uuid4())
last_claim = 0
try:
while not stopping:
try:
refresh()
if args.dispatcher:
jobs.recover_one()
if not jobs.publish_one(broker):
time.sleep(0.2)
continue
try:
broker.xgroup_create(STREAM, GROUP, id="0", mkstream=True)
except redis.ResponseError as exc:
if "BUSYGROUP" not in str(exc):
raise
entries = []
if time.monotonic() - last_claim >= 5:
claimed = broker.xautoclaim(
STREAM,
GROUP,
consumer,
min_idle_time=jobs.lease_seconds * 1000,
start_id="0-0",
count=1,
)
entries = claimed[1]
last_claim = time.monotonic()
if not entries:
batches = broker.xreadgroup(GROUP, consumer, {STREAM: ">"}, count=1, block=500)
entries = batches[0][1] if batches else []
for entry in entries:
consume_one(store, jobs, storage, broker, entry)
except (redis.RedisError, Failure) as exc:
logger.warning(
json.dumps({"event": "dependency_retry", "kind": type(exc).__name__})
)
time.sleep(0.5)
finally:
broker.close()
if storage is not None:
storage.close()
t.close()
store.close()
if __name__ == "__main__":
main()
+201
View File
@@ -0,0 +1,201 @@
import base64
import os
import threading
import uuid
from concurrent.futures import ThreadPoolExecutor
from urllib.parse import quote
import pytest
from client import LAB, Client
@pytest.fixture
def scenario():
c = Client(os.environ.get("BASE_URL", "http://127.0.0.1:8085"))
return c, *c.bootstrap()
@pytest.mark.parametrize(
"body",
[
{"text": ""},
{"text": " \n\t"},
{"text": 1},
{"text": True},
{"text": None},
{"text": "x", "sender_id": str(uuid.uuid4())},
{"text": "x", "unknown": 1},
{"text": "x" * 4001},
{},
[],
],
)
def test_invalid_message(scenario, body):
c, alice, _, _, chat = scenario
c.call(
"POST",
f"/api/v1/chats/{chat['id']}/messages",
400,
actor=alice,
body=body,
headers={"Idempotency-Key": uuid.uuid4().hex},
)
def test_malformed_json(scenario):
c, alice, _, _, chat = scenario
c.call(
"POST",
f"/api/v1/chats/{chat['id']}/messages",
400,
actor=alice,
raw=b'{"text":',
headers={"Content-Type": "application/json", "Idempotency-Key": uuid.uuid4().hex},
)
def test_acl_and_unknown_chat(scenario):
c, alice, _, eve, chat = scenario
for cid in [chat["id"], str(uuid.uuid4())]:
for suffix in ["", "/messages"]:
c.call("GET", f"/api/v1/chats/{cid}{suffix}", 404, actor=eve)
c.call(
"POST",
f"/api/v1/chats/{cid}/messages",
404,
actor=eve,
body={"text": "x"},
headers={"Idempotency-Key": uuid.uuid4().hex},
)
c.call("GET", "/api/v1/chats/not-a-uuid", 400, actor=alice)
@pytest.mark.parametrize("name", ["ab", "UPPER", "a-b", "a" * 33, "имя", "abc\n"])
def test_username_validation(scenario, name):
c, alice, *_ = scenario
body = {**alice["registration"], "username": name}
c.call("POST", "/api/v1/users", 400, body=body)
def test_unicode_and_limits(scenario):
c, alice, _, _, chat = scenario
text = "Ё👩‍💻漢字" * 500
assert c.message(alice, chat, text)["json"]["text"] == text
c.call(
"POST",
"/api/v1/users",
400,
body={
**alice["registration"],
"username": "u_" + uuid.uuid4().hex[:12],
"display_name": " ",
},
)
@pytest.mark.parametrize("value", ["0", "101", "1.5", "true", "-1", "", "1&limit=2"])
def test_invalid_limit(scenario, value):
c, alice, _, _, chat = scenario
c.call("GET", f"/api/v1/chats/{chat['id']}/messages?limit={value}", 400, actor=alice)
def test_pagination_and_cursor_scope(scenario):
c, alice, bob, _, chat = scenario
expected = [c.message(alice, chat, str(i))["json"] for i in range(13)]
path = f"/api/v1/chats/{chat['id']}/messages"
result, cursor, first = [], None, None
for i in range(20):
url = path + f"?limit={2 if i == 0 else 3}"
if cursor:
url += "&cursor=" + quote(cursor, safe="")
data = c.call("GET", url, actor=bob, schema="MessagePage")["json"]
result += data["items"]
cursor = data["next_cursor"]
first = first or cursor
if cursor is None:
break
assert result == sorted(expected, key=lambda m: (m["created_at"], m["id"]))
other = c.call(
"POST",
"/api/v1/chats",
201,
actor=alice,
body={"title": "другой", "member_ids": [alice["id"]]},
schema="Chat",
)["json"]
c.call("GET", f"/api/v1/chats/{other['id']}/messages?cursor={first}", 400, actor=alice)
c.call("GET", path + "?cursor=broken", 400, actor=alice)
def test_chat_validation_and_atomicity(scenario):
c, alice, bob, _, _ = scenario
before = c.call("GET", "/api/v1/chats?limit=100", actor=alice, schema="ChatPage")["json"]
for members in [
[bob["id"]],
[alice["id"], alice["id"]],
[alice["id"], str(uuid.uuid4())],
[],
[123],
]:
c.call(
"POST",
"/api/v1/chats",
400,
actor=alice,
body={"title": "не должен появиться", "member_ids": members},
)
after = c.call("GET", "/api/v1/chats?limit=100", actor=alice, schema="ChatPage")["json"]
assert before == after
def test_concurrent_messages(scenario):
c, alice, bob, _, chat = scenario
barrier = threading.Barrier(10)
def sender(i):
local = Client(c.base)
barrier.wait()
return [local.message(alice, chat, f"{i}:{j}")["json"] for j in range(10)]
with ThreadPoolExecutor(max_workers=10) as pool:
sent = [m for batch in pool.map(sender, range(10)) for m in batch]
data = c.call(
"GET", f"/api/v1/chats/{chat['id']}/messages?limit=100", actor=bob, schema="MessagePage"
)["json"]
assert len(sent) == len({m["id"] for m in sent}) == len(data["items"]) == 100
assert {m["id"] for m in sent} == {m["id"] for m in data["items"]}
assert data["next_cursor"] is None
def test_registration_race(scenario):
c, alice, *_ = scenario
body = {**alice["registration"], "username": "race_" + uuid.uuid4().hex[:12]}
barrier = threading.Barrier(10)
def register(_):
local = Client(c.base)
barrier.wait()
return local.call("POST", "/api/v1/users", [201, 409], body=body)["status"]
with ThreadPoolExecutor(max_workers=10) as pool:
statuses = list(pool.map(register, range(10)))
assert statuses.count(201) == 1
assert statuses.count(409) == 9
@pytest.mark.skipif(LAB < 2, reason="Сессии появляются в ЛР2")
def test_sessions(scenario):
c, alice, _, eve, _ = scenario
credential = base64.b64encode(
(alice["registration"]["username"] + ":wrong-password").encode()
).decode()
c.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": "Basic " + credential})
c.call("GET", "/api/v1/users/me", 401, headers={"X-User-Id": alice["id"]})
assert (
c.call(
"GET", "/api/v1/users/me", actor=alice, headers={"X-User-Id": eve["id"]}, schema="User"
)["json"]
== alice["user"]
)
c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=alice)
c.call("GET", "/api/v1/users/me", 401, actor=alice)
+203
View File
@@ -0,0 +1,203 @@
"""Black-box regressions derived from the contract and teacher review comments."""
import hashlib
import os
import struct
import uuid
import zlib
from io import BytesIO
from urllib.parse import quote
import pytest
from client import Client
from PIL import Image
@pytest.fixture(scope="module")
def scenario():
c = Client(os.environ.get("BASE_URL", "http://127.0.0.1:8085"))
return c, *c.bootstrap()
def image_bytes(fmt="PNG", size=(32, 20)):
stream = BytesIO()
Image.new("RGB", size, (100, 130, 200)).save(stream, fmt)
return stream.getvalue()
def upload(scenario, data=None, name="sample.png", mime="image/png", actor=None, status=202):
c, alice, _, _, chat = scenario
return c.call(
"POST",
f"/api/v1/chats/{chat['id']}/attachments?filename={quote(name, safe='')}",
status,
actor=actor or alice,
raw=data if data is not None else image_bytes(),
headers={"Content-Type": mime},
schema="Attachment" if status == 202 else None,
)
def test_roundtrip_acl_and_variant(scenario):
c, alice, bob, eve, chat = scenario
data = image_bytes()
a = upload(scenario, data)["json"]
assert a["sha256"] == hashlib.sha256(data).hexdigest()
assert a["status"] == "queued"
a = c.wait_ready(alice, a)
assert {v["name"] for v in a["variants"]} == {"thumbnail", "crop", "optimized"}
path = f"/api/v1/attachments/{a['id']}"
assert c.call("GET", path + "/content", 200, actor=bob)["bytes"] == data
c.call("GET", path, 404, actor=eve)
c.call("GET", path + "/content", 404, actor=eve)
c.call("GET", path + "/content?variant=thumbnail", 200, actor=alice)
c.call("GET", path + "/content?variant=unknown", 400, actor=alice)
message = c.call(
"POST",
f"/api/v1/chats/{chat['id']}/messages",
201,
actor=alice,
body={"text": "", "attachment_ids": [a["id"]]},
headers={"Idempotency-Key": uuid.uuid4().hex},
schema="Message",
)["json"]
assert message["text"] == "" and message["attachment_ids"] == [a["id"]]
@pytest.mark.parametrize(
"name", [" ", "../x.png", "a\\b.png", "a\0.png", "a\x7f.png", "a\x85.png", "a\n.png"]
)
def test_bad_filename(scenario, name):
upload(scenario, name=name, status=400)
def chunk(kind, data):
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data))
def png_with_data(data, width=1, height=1, color=2, palette=b""):
return (
b"\x89PNG\r\n\x1a\n"
+ chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, color, 0, 0, 0))
+ (chunk(b"PLTE", palette) if palette else b"")
+ chunk(b"IDAT", data)
+ chunk(b"IEND", b"")
)
@pytest.mark.parametrize(
"data,mime",
[
(image_bytes()[:33], "image/png"),
(image_bytes()[:-15], "image/png"),
(png_with_data(b"broken-zlib"), "image/png"),
(png_with_data(zlib.compress(b"\x05\0\0\0")), "image/png"),
(png_with_data(zlib.compress(b"\0\0"), color=3, palette=bytes(771)), "image/png"),
(bytes.fromhex("ffd8ffc0000b080001000101011100ffda0008010100003f00ffd9"), "image/jpeg"),
(bytes.fromhex("ffd8ffc00008080001000101ffda0002ffd9"), "image/jpeg"),
(image_bytes(), "image/jpeg"),
(b"not-an-image", "image/png"),
],
)
def test_corrupt_or_spoofed_image(scenario, data, mime):
upload(scenario, data=data, mime=mime, status=415)
@pytest.mark.parametrize("size", [(8000, 5001), (20000, 10000)])
def test_pixel_limit_before_decode(scenario, size):
data = png_with_data(b"", width=size[0], height=size[1])
upload(scenario, data=data, status=413)
def test_byte_limit_and_exact_boundary(scenario):
data = image_bytes()
exact = data + bytes(10 * 1024 * 1024 - len(data))
a = upload(scenario, data=exact)["json"]
assert a["size_bytes"] == len(exact)
upload(scenario, data=exact + b"x", status=413)
@pytest.mark.parametrize(
"body",
[
{"attachment_ids": []},
{"text": None},
{"text": "x", "attachment_ids": None},
{"text": "x", "attachment_ids": [123]},
],
)
def test_strict_message_fields(scenario, body):
c, alice, _, _, chat = scenario
c.call(
"POST",
f"/api/v1/chats/{chat['id']}/messages",
400,
actor=alice,
body=body,
headers={"Idempotency-Key": uuid.uuid4().hex},
)
def test_foreign_unknown_and_ordered_attachments(scenario):
c, alice, _, _, chat = scenario
first, second = upload(scenario)["json"], upload(scenario)["json"]
other = c.call(
"POST",
"/api/v1/chats",
201,
actor=alice,
body={"title": "other", "member_ids": [alice["id"]]},
schema="Chat",
)["json"]
for aid in [first["id"], str(uuid.uuid4())]:
c.call(
"POST",
f"/api/v1/chats/{other['id']}/messages",
404,
actor=alice,
body={"text": "", "attachment_ids": [aid]},
headers={"Idempotency-Key": uuid.uuid4().hex},
)
key = uuid.uuid4().hex
body = {"text": "with files", "attachment_ids": [second["id"], first["id"]]}
p = f"/api/v1/chats/{chat['id']}/messages"
sent = c.call(
"POST", p, 201, actor=alice, body=body, headers={"Idempotency-Key": key}, schema="Message"
)["json"]
repeat = c.call(
"POST", p, 201, actor=alice, body=body, headers={"Idempotency-Key": key}, schema="Message"
)["json"]
assert repeat == sent
c.call(
"POST",
p,
409,
actor=alice,
body={**body, "attachment_ids": list(reversed(body["attachment_ids"]))},
headers={"Idempotency-Key": key},
)
history = c.call("GET", p, 200, actor=alice, schema="MessagePage")["json"]["items"]
assert next(m for m in history if m["id"] == sent["id"]) == sent
def test_nul_message_roundtrip(scenario):
c, alice, _, _, chat = scenario
assert c.message(alice, chat, "abc\0def")["json"]["text"] == "abc\0def"
def test_truncated_jpeg_with_forged_eoi(scenario):
import random
stream = BytesIO()
Image.frombytes("RGB", (512, 512), random.Random(42).randbytes(512 * 512 * 3)).save(
stream, "JPEG", quality=95
)
original = stream.getvalue()
upload(scenario, data=original, name="real.jpg", mime="image/jpeg")
upload(
scenario,
data=original[: len(original) // 2] + b"\xff\xd9",
name="broken.jpg",
mime="image/jpeg",
status=415,
)
+39
View File
@@ -0,0 +1,39 @@
import base64
import json
from uuid import uuid4
import pytest
from messenger.domain import Failure, boundary
@pytest.mark.parametrize(
"version,date",
[
(True, "2026-01-01T00:00:00Z"),
(1.0, "2026-01-01T00:00:00Z"),
(1, "2026-W01-1T00:00:00Z"),
(1, "20260101T000000Z"),
(1, "2026-01-01 00:00:00Z"),
(1, "2026-01-01X00:00:00Z"),
(1, "2026-02-30T00:00:00Z"),
(1, "2026-01-01T00:00:00.1234567Z"),
],
)
def test_boundary_rejects_python_iso_extensions(version, date):
payload = [version, "scope", date, str(uuid4())]
cursor = base64.urlsafe_b64encode(json.dumps(payload).encode()).decode().rstrip("=")
with pytest.raises(Failure) as error:
boundary("scope", cursor)
assert error.value.status == 400
@pytest.mark.parametrize(
"date", ["2026-01-01T00:00:00Z", "2026-01-01T00:00:00.1Z", "2026-01-01T00:00:00.123456Z"]
)
def test_boundary_preserves_generated_format(date):
uid = str(uuid4())
cursor = (
base64.urlsafe_b64encode(json.dumps([1, "scope", date, uid]).encode()).decode().rstrip("=")
)
assert boundary("scope", cursor) == (date, uid)
+49
View File
@@ -0,0 +1,49 @@
import base64
import json
import os
import uuid
import pytest
from client import Client
@pytest.fixture(scope="module")
def shared():
client = Client(os.getenv("BASE_URL", "http://127.0.0.1:8085"))
return client, *client.bootstrap()
@pytest.mark.parametrize(
"date",
[
"2026-W01-1T00:00:00Z",
"20260101T000000Z",
"2026-01-01 00:00:00Z",
"2026-01-01X00:00:00Z",
"2026-01-01T00:00:00+00:00",
"2026-02-30T00:00:00Z",
"2026-01-01T24:00:00Z",
"2026-01-01T00:00:00.1234567Z",
],
)
def test_cursor_rejects_noncanonical_dates_over_http(shared, date):
c, alice, _, _, chat = shared
for path, scope in [
("/api/v1/chats", "chats:" + alice["id"]),
(f"/api/v1/chats/{chat['id']}/messages", "messages:" + chat["id"]),
]:
payload = [1, scope, date, str(uuid.uuid4())]
cursor = base64.urlsafe_b64encode(json.dumps(payload).encode()).decode().rstrip("=")
c.call("GET", path + "?cursor=" + cursor, 400, actor=alice)
@pytest.mark.parametrize("version", [True, 1.0])
def test_cursor_rejects_noninteger_version_over_http(shared, version):
c, alice, _, _, chat = shared
for path, scope in [
("/api/v1/chats", "chats:" + alice["id"]),
(f"/api/v1/chats/{chat['id']}/messages", "messages:" + chat["id"]),
]:
payload = [version, scope, "2026-01-01T00:00:00Z", str(uuid.uuid4())]
cursor = base64.urlsafe_b64encode(json.dumps(payload).encode()).decode().rstrip("=")
c.call("GET", path + "?cursor=" + cursor, 400, actor=alice)
+99
View File
@@ -0,0 +1,99 @@
import base64
import json
import os
import uuid
from urllib.parse import quote
import pytest
from client import LAB, Client
@pytest.fixture
def edge_scenario():
client = Client(os.environ.get("BASE_URL", "http://127.0.0.1:8085"))
return client, *client.bootstrap()
def encoded_cursor(value):
return base64.urlsafe_b64encode(json.dumps(value).encode()).decode().rstrip("=")
@pytest.mark.parametrize("identity", ["missing", "malformed", "unknown"])
def test_invalid_identity(edge_scenario, identity):
client, _, _, _, chat = edge_scenario
headers = {}
if identity == "malformed":
headers["X-User-Id"] = "not-a-uuid"
elif identity == "unknown":
headers["X-User-Id"] = str(uuid.uuid4())
for path in ["/api/v1/users/me", "/api/v1/chats", f"/api/v1/chats/{chat['id']}/messages"]:
client.call("GET", path, 401, headers=headers)
@pytest.mark.parametrize("uid", [None, [], {}, 123, True, "not-a-uuid"])
def test_cursor_rejects_non_uuid_boundary(edge_scenario, uid):
client, alice, _, _, chat = edge_scenario
payload = [1, "messages:" + chat["id"], "2026-10-30T12:00:00.000000Z", uid]
cursor = quote(encoded_cursor(payload), safe="")
client.call("GET", f"/api/v1/chats/{chat['id']}/messages?cursor={cursor}", 400, actor=alice)
@pytest.mark.parametrize(
"payload", [None, {}, [], [1], [1, "wrong-scope", "2026-10-30T12:00:00Z", str(uuid.uuid4())]]
)
def test_cursor_rejects_wrong_shape(edge_scenario, payload):
client, alice, _, _, chat = edge_scenario
cursor = quote(encoded_cursor(payload), safe="")
client.call("GET", f"/api/v1/chats/{chat['id']}/messages?cursor={cursor}", 400, actor=alice)
def test_request_id_is_present_and_not_reused(edge_scenario):
client, alice, _, _, _ = edge_scenario
first = client.call("GET", "/health/live", schema="Health")
second = client.call("GET", "/health/ready", schema="Health")
error = client.call("GET", "/api/v1/chats/malformed", 400, actor=alice)
values = [r["headers"]["X-Request-Id"] for r in [first, second, error]]
assert len(set(values)) == 3
assert error["json"]["error"]["request_id"] == values[-1]
def test_unicode_codepoint_boundaries(edge_scenario):
client, alice, _, _, chat = edge_scenario
username = "unicode_" + uuid.uuid4().hex[:12]
registration = {"username": username, "display_name": "😀" * 100}
if LAB >= 2:
registration["password"] = "unicode-test-" + uuid.uuid4().hex
user = client.call(
"POST",
"/api/v1/users",
201,
body=registration,
schema="User",
)["json"]
assert user["display_name"] == "😀" * 100
client.call(
"POST", "/api/v1/users", 400, body={"username": username + "x", "display_name": "😀" * 101}
)
message = client.message(alice, chat, "😀" * 4000)["json"]
assert message["text"] == "😀" * 4000
client.call(
"POST", f"/api/v1/chats/{chat['id']}/messages", 400, actor=alice, body={"text": "😀" * 4001}
)
def test_empty_history_and_cursor_exhaustion(edge_scenario):
client, alice, _, _, chat = edge_scenario
path = f"/api/v1/chats/{chat['id']}/messages"
empty = client.call("GET", path, actor=alice, schema="MessagePage")["json"]
assert empty == {"items": [], "next_cursor": None}
message = client.message(alice, chat, "одна запись")["json"]
data = client.call("GET", path + "?limit=1", actor=alice, schema="MessagePage")["json"]
assert data == {"items": [message], "next_cursor": None}
def test_missing_chat_is_hidden_like_foreign_chat(edge_scenario):
client, _, _, eve, chat = edge_scenario
foreign = client.call("GET", f"/api/v1/chats/{chat['id']}", 404, actor=eve)["json"]
missing = client.call("GET", f"/api/v1/chats/{uuid.uuid4()}", 404, actor=eve)["json"]
assert foreign["error"]["code"] == missing["error"]["code"] == "not_found"
assert foreign["error"]["message"] == missing["error"]["message"]
+229
View File
@@ -0,0 +1,229 @@
"""Full-range JPEG and bounded multi-frame PNG regressions."""
import os
import struct
import zlib
from io import BytesIO
import pytest
from client import Client
from PIL import Image
@pytest.fixture(scope="module")
def scenario():
c = Client(os.getenv("BASE_URL", "http://127.0.0.1:8085"))
return c, *c.bootstrap()
def upload(scenario, data, mime, expected):
c, alice, _, _, chat = scenario
result = c.call(
"POST",
f"/api/v1/chats/{chat['id']}/attachments?filename=edge.png",
202 if expected == 201 else expected,
actor=alice,
raw=data,
headers={"Content-Type": mime},
schema="Attachment" if expected == 201 else None,
)
if expected == 201:
result["json"] = c.wait_ready(alice, result["json"])
return result
def progressive_jpeg():
output = BytesIO()
image = Image.new("RGB", (6000, 6000), "blue")
try:
image.save(output, "JPEG", quality=90, progressive=True, subsampling=0)
finally:
image.close()
return output.getvalue()
def apng(size=(16, 16), default_image=False):
output = BytesIO()
first, second = Image.new("RGB", size, "red"), Image.new("RGB", size, "blue")
try:
first.save(
output,
"PNG",
save_all=True,
append_images=[second],
duration=100,
loop=0,
default_image=default_image,
)
finally:
first.close()
second.close()
return output.getvalue()
def broken_later_frame(data):
result, position = bytearray(data[:8]), 8
while position < len(data):
length = struct.unpack(">I", data[position : position + 4])[0]
kind = data[position + 4 : position + 8]
payload = data[position + 8 : position + 8 + length]
if kind == b"fdAT":
payload = payload[:4] + b"not-a-valid-deflate-stream"
result += (
struct.pack(">I", len(payload))
+ kind
+ payload
+ struct.pack(">I", zlib.crc32(kind + payload))
)
position += length + 12
return bytes(result)
def test_36m_progressive_jpeg_is_valid_and_unchanged(scenario):
data = progressive_jpeg()
attachment = upload(scenario, data, "image/jpeg", 201)["json"]
c, _, bob, _, _ = scenario
assert (
c.call("GET", f"/api/v1/attachments/{attachment['id']}/content", actor=bob)["bytes"] == data
)
def test_apng_corrupt_later_frame_is_rejected(scenario):
upload(scenario, broken_later_frame(apng()), "image/png", 415)
def test_valid_small_apng_is_accepted_and_unchanged(scenario):
data = apng()
attachment = upload(scenario, data, "image/png", 201)["json"]
c, _, bob, _, _ = scenario
assert (
c.call("GET", f"/api/v1/attachments/{attachment['id']}/content", actor=bob)["bytes"] == data
)
def test_apng_cumulative_canvas_limit(scenario):
# Two 25M-pixel canvases exceed the total 40M decode budget.
upload(scenario, apng((5000, 5000)), "image/png", 413)
def test_apng_forged_frame_count_cannot_hide_bad_frame(scenario):
data = bytearray(broken_later_frame(apng()))
position = 8
while position < len(data):
length = struct.unpack(">I", data[position : position + 4])[0]
kind = bytes(data[position + 4 : position + 8])
if kind == b"acTL":
data[position + 8 : position + 12] = struct.pack(">I", 1)
payload = bytes(data[position + 8 : position + 8 + length])
data[position + 8 + length : position + 12 + length] = struct.pack(
">I", zlib.crc32(kind + payload)
)
break
position += length + 12
upload(scenario, bytes(data), "image/png", 415)
def test_apng_zero_count_with_static_fallback_is_rejected(scenario):
# Pillow warns and falls back to one valid static PNG frame. The raw acTL
# count must still be rejected rather than trusting Pillow's n_frames=1.
data = apng()
result, position, controls = bytearray(data[:8]), 8, 0
while position < len(data):
length = int.from_bytes(data[position : position + 4], "big")
kind = data[position + 4 : position + 8]
payload = data[position + 8 : position + 8 + length]
position += length + 12
if kind == b"acTL":
payload = bytes(4) + payload[4:]
if kind == b"fcTL":
controls += 1
if controls > 1:
continue
if kind == b"fdAT":
continue
result += (
struct.pack(">I", len(payload))
+ kind
+ payload
+ struct.pack(">I", zlib.crc32(kind + payload))
)
upload(scenario, bytes(result), "image/png", 415)
def test_apng_exact_cumulative_limit_is_accepted(scenario):
upload(scenario, apng((4000, 5000)), "image/png", 201)
def test_40m_cmyk_progressive_jpeg_is_accepted(scenario):
output = BytesIO()
image = Image.new("CMYK", (8000, 5000), (0, 128, 128, 0))
try:
image.save(output, "JPEG", quality=90, progressive=True, subsampling=0)
finally:
image.close()
data = output.getvalue()
attachment = upload(scenario, data, "image/jpeg", 201)["json"]
c, _, bob, _, _ = scenario
assert (
c.call("GET", f"/api/v1/attachments/{attachment['id']}/content", actor=bob)["bytes"] == data
)
def test_repeated_large_image_sequence_stays_available(scenario):
# Formerly retained APNG canvases could overlap the next CMYK decoder.
c, _, bob, _, _ = scenario
data = apng((4000, 5000))
attachment = upload(scenario, data, "image/png", 201)["json"]
assert (
c.call("GET", f"/api/v1/attachments/{attachment['id']}/content", actor=bob)["bytes"] == data
)
del data
test_40m_cmyk_progressive_jpeg_is_accepted(scenario)
test_36m_progressive_jpeg_is_valid_and_unchanged(scenario)
data = BytesIO()
image = Image.new("RGB", (16, 16), "green")
image.save(data, "PNG")
image.close()
attachment = upload(scenario, data.getvalue(), "image/png", 201)["json"]
assert (
c.call("GET", f"/api/v1/attachments/{attachment['id']}/content", actor=bob)["bytes"]
== data.getvalue()
)
@pytest.mark.parametrize("progressive", [False, True])
def test_small_cmyk_jpeg(progressive, scenario):
output = BytesIO()
image = Image.new("CMYK", (24, 18), (0, 128, 128, 0))
image.save(output, "JPEG", progressive=progressive)
image.close()
upload(scenario, output.getvalue(), "image/jpeg", 201)
def test_invalid_zero_frame_count_is_rejected(scenario):
data = apng()
result, position, controls = bytearray(data[:8]), 8, 0
while position < len(data):
length = struct.unpack(">I", data[position : position + 4])[0]
kind = data[position + 4 : position + 8]
payload = data[position + 8 : position + 8 + length]
position += length + 12
if kind == b"fcTL":
controls += 1
if controls > 1:
continue
if kind == b"fdAT":
continue
if kind == b"acTL":
payload = bytes(4) + payload[4:]
result += (
struct.pack(">I", len(payload))
+ kind
+ payload
+ struct.pack(">I", zlib.crc32(kind + payload))
)
upload(scenario, bytes(result), "image/png", 415)
def test_valid_apng_default_image(scenario):
upload(scenario, apng(default_image=True), "image/png", 201)
+72
View File
@@ -0,0 +1,72 @@
import json
import pytest
from messenger.worker import validate_delivery
VALID = {
"schema_version": 1,
"job_id": "11111111-1111-4111-8111-111111111111",
"type": "attachment.process.v1",
"attachment_id": "22222222-2222-4222-8222-222222222222",
"pipeline_version": 1,
"attempt": 1,
"created_at": "2026-11-06T00:00:00Z",
}
@pytest.mark.parametrize(
"key,value",
[
("job_id", 123),
("job_id", []),
("attachment_id", {}),
("schema_version", True),
("pipeline_version", True),
("attempt", 0),
("created_at", 123),
("created_at", "2026-11-06"),
("created_at", "2026-11-06T12Z"),
("created_at", "2026-99-99T00:00:00Z"),
("traceparent", 123),
("tracestate", []),
],
)
def test_invalid_envelopes_are_bounded_validation_errors(key, value):
with pytest.raises(ValueError):
validate_delivery(json.dumps({**VALID, key: value}))
@pytest.mark.parametrize("body", [b"\xff", "null", "[]", "{}", "not json"])
def test_invalid_json(body):
with pytest.raises(ValueError):
validate_delivery(body)
def test_valid_trace_context():
body = {
**VALID,
"traceparent": "00-" + "a" * 32 + "-" + "b" * 16 + "-01",
"tracestate": "test=value",
}
assert validate_delivery(json.dumps(body)) == body
@pytest.mark.parametrize(
"body", ["[" * 2000 + "]" * 2000, "[" * 20000 + "]" * 20000, b" " * 8193 + b"{}"]
)
def test_deep_and_oversize_envelopes_reject_safely(body):
with pytest.raises(ValueError):
validate_delivery(body)
def test_uuid_uppercase_is_normalized_before_storage():
value = "AAAAAAAA-AAAA-4AAA-8AAA-AAAAAAAAAAAA"
parsed = validate_delivery(json.dumps({**VALID, "job_id": value, "attachment_id": value}))
assert parsed["job_id"] == parsed["attachment_id"] == value.lower()
@pytest.mark.parametrize("field", ["job_id", "attachment_id"])
def test_uuid_urn_is_rejected_before_storage(field):
with pytest.raises(ValueError):
validate_delivery(json.dumps({**VALID, field: "urn:uuid:" + VALID[field]}))
+169
View File
@@ -0,0 +1,169 @@
import base64
import codecs
import json
import os
import uuid
from uuid import uuid4
import pytest
from client import Client
ENCODINGS = [
"utf-16",
"utf-16-le",
"utf-16-be",
"utf-16-be-bom",
"utf-32",
"utf-32-le",
"utf-32-be",
"utf-32-be-bom",
]
def encoded(body, encoding):
text = json.dumps(body, ensure_ascii=False)
if encoding == "utf-16-be-bom":
return codecs.BOM_UTF16_BE + text.encode("utf-16-be")
if encoding == "utf-32-be-bom":
return codecs.BOM_UTF32_BE + text.encode("utf-32-be")
return text.encode(encoding)
@pytest.fixture(scope="module")
def encoding_scenario():
c = Client(os.environ["BASE_URL"])
return c, *c.bootstrap()
@pytest.mark.parametrize("encoding", ENCODINGS)
def test_non_utf8_registration_has_no_write(encoding_scenario, encoding):
c, *_ = encoding_scenario
body = {
"username": "enc_" + uuid.uuid4().hex[:14],
"display_name": "Никита\u0000имя",
"password": "Пароль:тестовый-12345",
}
c.call(
"POST",
"/api/v1/users",
400,
raw=encoded(body, encoding),
headers={"Content-Type": "application/json"},
)
# UNIQUE username proves the rejected request did not insert a user.
created = c.call("POST", "/api/v1/users", 201, body=body, schema="User")["json"]
assert created["display_name"] == body["display_name"]
@pytest.mark.parametrize("encoding", ENCODINGS)
def test_non_utf8_chat_has_no_write(encoding_scenario, encoding):
c, alice, *_ = encoding_scenario
before = c.call("GET", "/api/v1/chats?limit=100", actor=alice, schema="ChatPage")["json"]
body = {"title": "Чат\u0000" + uuid.uuid4().hex, "member_ids": [alice["id"]]}
c.call(
"POST",
"/api/v1/chats",
400,
actor=alice,
raw=encoded(body, encoding),
headers={"Content-Type": "application/json"},
)
after = c.call("GET", "/api/v1/chats?limit=100", actor=alice, schema="ChatPage")["json"]
assert before == after
assert (
c.call("POST", "/api/v1/chats", 201, actor=alice, body=body, schema="Chat")["json"]["title"]
== body["title"]
)
@pytest.mark.parametrize("encoding", ENCODINGS)
def test_non_utf8_message_has_no_write(encoding_scenario, encoding):
c, alice, _, _, chat = encoding_scenario
path = f"/api/v1/chats/{chat['id']}/messages"
key = uuid.uuid4().hex
body = {"text": "Сообщение\u0000" + key}
c.call(
"POST",
path,
400,
actor=alice,
raw=encoded(body, encoding),
headers={"Content-Type": "application/json", "Idempotency-Key": key},
)
history = c.call("GET", path + "?limit=100", actor=alice, schema="MessagePage")["json"]["items"]
assert all(item["text"] != body["text"] for item in history)
good = c.call(
"POST",
path,
201,
actor=alice,
body=body,
headers={"Idempotency-Key": key},
schema="Message",
)["json"]
repeat = c.call(
"POST",
path,
201,
actor=alice,
body=body,
headers={"Idempotency-Key": key},
schema="Message",
)["json"]
assert good == repeat and good["text"] == body["text"]
@pytest.mark.parametrize("encoding", ["utf-8", "utf-8-sig"])
def test_valid_utf8_registration_and_login(encoding_scenario, encoding):
c, *_ = encoding_scenario
body = {
"username": "utf8_" + uuid.uuid4().hex[:14],
"display_name": "Юникод\u0000😀",
"password": "Пароль:тестовый-12345",
}
user = c.call(
"POST",
"/api/v1/users",
201,
raw=encoded(body, encoding),
headers={"Content-Type": "application/json"},
schema="User",
)["json"]
assert user["display_name"] == body["display_name"]
credential = base64.b64encode((body["username"] + ":" + body["password"]).encode()).decode()
session = c.call(
"POST",
"/api/v1/auth/sessions",
201,
headers={"Authorization": "Basic " + credential},
schema="Session",
)["json"]
assert session["user"] == user
def requests_for(alice, chat):
return [
(
"/api/v1/users",
{
"username": "utf_" + uuid4().hex[:12],
"display_name": "Имя\0",
"password": "synthetic-password-123",
},
),
("/api/v1/chats", {"title": "Чат\0", "member_ids": [alice["id"]]}),
(f"/api/v1/chats/{chat['id']}/messages", {"text": "Сообщение\0"}),
]
def test_utf8_and_escaped_nul_remain_valid(encoding_scenario):
c, alice, _, _, chat = encoding_scenario
for path, body in requests_for(alice, chat):
c.call(
"POST",
path,
201,
actor=alice,
raw=json.dumps(body, ensure_ascii=False).encode("utf-8"),
headers={"Content-Type": "application/json", "Idempotency-Key": uuid4().hex},
)
+46
View File
@@ -0,0 +1,46 @@
from io import BytesIO
import pytest
from PIL import Image
from messenger.image_pipeline import fitted_size, render
@pytest.mark.parametrize(
"source,expected",
[((319, 200), (256, 160)), ((320, 200), (256, 160)), ((10, 4), (10, 4)), ((1, 1000), (1, 256))],
)
def test_fits_contract(source, expected):
assert fitted_size(*source, 256) == expected
def test_variants_alpha_and_metadata(tmp_path):
source = BytesIO()
Image.new("RGBA", (319, 200), (255, 0, 0, 0)).save(source, "PNG")
source.seek(0)
variants = render(source, tmp_path)
assert {v["name"] for v in variants} == {"thumbnail", "crop", "optimized"}
with Image.open(tmp_path / "thumbnail") as thumb:
assert thumb.format == "JPEG"
assert thumb.size == (256, 160)
assert all(channel > 245 for channel in thumb.getpixel((0, 0)))
assert not thumb.getexif()
with Image.open(tmp_path / "crop") as crop:
assert crop.size == (128, 128)
with Image.open(tmp_path / "optimized") as optimized:
assert optimized.format == "WEBP"
assert optimized.size == (319, 200)
def test_exif_orientation_without_upscale(tmp_path):
source = BytesIO()
exif = Image.Exif()
exif[274] = 6
Image.new("RGB", (80, 40), "blue").save(source, "JPEG", exif=exif)
original = source.getvalue()
source.seek(0)
render(source, tmp_path)
assert source.getvalue() == original
with Image.open(tmp_path / "thumbnail") as thumbnail:
assert thumbnail.size == (40, 80)
assert not thumbnail.getexif()
+62
View File
@@ -0,0 +1,62 @@
import base64
import uuid
import pytest
from client import Client
@pytest.mark.parametrize("password", ["x" * 11, "x" * 129, "😀" * 129, 123, None, True])
def test_password_contract(password):
import os
c = Client(os.environ["BASE_URL"])
c.call(
"POST",
"/api/v1/users",
400,
body={
"username": "pwd_" + uuid.uuid4().hex[:12],
"display_name": "Тест",
"password": password,
},
)
def test_full_unicode_password_and_colon():
import os
c = Client(os.environ["BASE_URL"])
username = "pwd_" + uuid.uuid4().hex[:12]
password = "😀" * 126 + ":Я"
user = c.call(
"POST",
"/api/v1/users",
201,
body={"username": username, "display_name": "Пароль128", "password": password},
schema="User",
)["json"]
good = base64.b64encode((username + ":" + password).encode()).decode()
session = c.call(
"POST",
"/api/v1/auth/sessions",
201,
headers={"Authorization": "bAsIc " + good},
schema="Session",
)["json"]
actor = {"id": user["id"], "token": session["token"]}
assert session["user"] == user
bad = base64.b64encode((username + ":" + password[:-1] + "Б").encode()).decode()
c.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": "Basic " + bad})
c.call("GET", "/api/v1/users/me", 401, headers={"Authorization": "Basic " + good})
c.call("GET", "/api/v1/users/me", actor=actor, schema="User")
c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=actor)
c.call("GET", "/api/v1/users/me", 401, actor=actor)
@pytest.mark.parametrize("credential", ["Basic !invalid", "Basic", "Bearer abc", "", "Basic YTpi"])
def test_invalid_basic(credential):
import os
Client(os.environ["BASE_URL"]).call(
"POST", "/api/v1/auth/sessions", 401, headers={"Authorization": credential}
)
+231
View File
@@ -0,0 +1,231 @@
"""Identity and prefix negative probes use a real local TLS server, no mocked TLS."""
import http.client
import json
import socket
import socketserver
import ssl
import threading
from pathlib import Path
import pytest
from messenger.tls_gateway import HTTPHandler, HTTPServer, TCPHandler, TCPServer, role
ROOT = Path(__file__).resolve().parents[1]
def cfg(name, allowed, upstream, **extra):
return dict(
cert=str(ROOT / f"secrets/pki/{name}/cert.pem"),
key=str(ROOT / f"secrets/pki/{name}/key.pem"),
ca=str(ROOT / "secrets/ca.pem"),
allowed=allowed,
upstream=upstream,
**extra,
)
def client(identity=None):
ctx = ssl.create_default_context(cafile=str(ROOT / "secrets/ca.pem"))
if identity:
ctx.load_cert_chain(
ROOT / f"secrets/pki/{identity}/cert.pem", ROOT / f"secrets/pki/{identity}/key.pem"
)
return ctx
class Echo(socketserver.BaseRequestHandler):
def handle(self):
while data := self.request.recv(1024):
self.request.sendall(data)
@pytest.fixture
def tcp_boundary(monkeypatch, tmp_path):
revoked = tmp_path / "revoked.json"
revoked.write_text("[]")
monkeypatch.setenv("TLS_REVOKED_FILE", str(revoked))
with socketserver.ThreadingTCPServer(("127.0.0.1", 0), Echo) as echo:
threading.Thread(target=echo.serve_forever, daemon=True).start()
with TCPServer(("127.0.0.1", 0), TCPHandler) as proxy:
proxy.cfg = cfg("api", ["lb"], echo.server_address[1])
proxy.slots = threading.BoundedSemaphore(32)
threading.Thread(target=proxy.serve_forever, daemon=True).start()
yield proxy.server_address[1], revoked
proxy.shutdown()
echo.shutdown()
def exchange(port, ctx, hostname="api"):
with socket.create_connection(("127.0.0.1", port), timeout=2) as raw:
with ctx.wrap_socket(raw, server_hostname=hostname) as conn:
conn.sendall(b"positive")
return conn.recv(1024)
def test_allowed_lb_identity(tcp_boundary):
assert exchange(tcp_boundary[0], client("lb")) == b"positive"
@pytest.mark.parametrize("identity", [None, "worker", "operator"])
def test_missing_or_trusted_wrong_identity_denied(tcp_boundary, identity):
try:
result = exchange(tcp_boundary[0], client(identity))
except (OSError, ssl.SSLError):
result = b""
assert result == b""
def test_wrong_hostname(tcp_boundary):
with pytest.raises(ssl.SSLCertVerificationError):
exchange(tcp_boundary[0], client("lb"), "wrong-service")
def test_empty_ca(tcp_boundary):
ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
ctx.load_cert_chain(ROOT / "secrets/pki/lb/cert.pem", ROOT / "secrets/pki/lb/key.pem")
with pytest.raises(ssl.SSLCertVerificationError):
exchange(tcp_boundary[0], ctx)
def test_revocation_new_connection(tcp_boundary):
port, revoked = tcp_boundary
assert exchange(port, client("lb")) == b"positive"
cert = ssl._ssl._test_decode_cert(str(ROOT / "secrets/pki/lb/cert.pem"))
revoked.write_text(json.dumps([cert["serialNumber"].lstrip("0").lower()]))
try:
result = exchange(port, client("lb"))
except (OSError, ssl.SSLError):
result = b""
assert result == b""
@pytest.mark.parametrize(
"method,path,headers",
[
("PUT", "/messenger/private/secret", {}),
("PUT", "/messenger/staging/new", {}),
("GET", "/other-bucket/secret", {}),
("GET", "/messenger/staging/../private/secret", {}),
("GET", "/messenger/staging/%2e%2e/private/secret", {}),
("GET", "/messenger/staging/%252e%252e/private/secret", {}),
(
"PUT",
"/messenger/attachments/id/v1/original",
{"x-amz-copy-source": "/messenger/private/secret"},
),
("GET", "/messenger?list-type=2", {}),
("GET", "https://storage:8443/messenger/staging/id", {}),
("GET", "/messenger/staging/id#fragment", {}),
],
)
def test_worker_prefix_and_copy_bypasses_denied(tmp_path, monkeypatch, method, path, headers):
revoked = tmp_path / "revoke.json"
revoked.write_text("[]")
monkeypatch.setenv("TLS_REVOKED_FILE", str(revoked))
with HTTPServer(("127.0.0.1", 0), HTTPHandler) as proxy:
proxy.cfg = cfg("storage", ["api", "worker", "operator"], 1, http=True, s3=True)
proxy.slots = threading.BoundedSemaphore(32)
threading.Thread(target=proxy.serve_forever, daemon=True).start()
conn = http.client.HTTPSConnection(
"127.0.0.1", proxy.server_address[1], context=client("worker"), timeout=2
)
conn.request(method, path, headers={"Host": "storage:8443", **headers})
assert conn.getresponse().status == 403
conn.close()
proxy.shutdown()
def test_exactly_one_allowed_san():
with pytest.raises(ssl.SSLError):
role({"subjectAltName": [("DNS", "worker"), ("DNS", "lb")]}, ["lb", "worker"])
def test_expired_certificate_rejected(tcp_boundary, tmp_path):
import subprocess
import time
cert = tmp_path / "expired.pem"
subprocess.run(
[
"openssl",
"x509",
"-req",
"-in",
str(ROOT / "operator-private/worker.csr"),
"-CA",
str(ROOT / "operator-private/ca.pem"),
"-CAkey",
str(ROOT / "operator-private/ca-key.pem"),
"-set_serial",
"999999",
"-days",
"0",
"-extfile",
str(ROOT / "operator-private/worker.ext"),
"-out",
str(cert),
],
check=True,
capture_output=True,
)
time.sleep(1)
ctx = client()
ctx.load_cert_chain(cert, ROOT / "secrets/pki/worker/key.pem")
try:
result = exchange(tcp_boundary[0], ctx)
except (OSError, ssl.SSLError):
result = b""
assert result == b""
def test_http_forward_preserves_object_bytes(tmp_path, monkeypatch):
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
class Object(BaseHTTPRequestHandler):
def do_GET(self):
body = b"original-bytes"
self.send_response(200)
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def log_message(self, *_):
pass
revoked = tmp_path / "revoked.json"
revoked.write_text("[]")
monkeypatch.setenv("TLS_REVOKED_FILE", str(revoked))
with ThreadingHTTPServer(("127.0.0.1", 0), Object) as upstream:
threading.Thread(target=upstream.serve_forever, daemon=True).start()
with HTTPServer(("127.0.0.1", 0), HTTPHandler) as proxy:
proxy.cfg = cfg("storage", ["worker"], upstream.server_address[1], http=True, s3=True)
proxy.slots = threading.BoundedSemaphore(32)
threading.Thread(target=proxy.serve_forever, daemon=True).start()
conn = http.client.HTTPSConnection(
"127.0.0.1", proxy.server_address[1], context=client("worker"), timeout=2
)
conn.request("GET", "/messenger/staging/id", headers={"Host": "storage:8443"})
response = conn.getresponse()
assert response.status == 200
assert response.read() == b"original-bytes"
conn.close()
proxy.shutdown()
upstream.shutdown()
def test_revocation_existing_connection_before_next_command(tcp_boundary):
port, revoked = tcp_boundary
with socket.create_connection(("127.0.0.1", port), timeout=2) as raw:
with client("lb").wrap_socket(raw, server_hostname="api") as conn:
conn.sendall(b"before-revocation")
assert conn.recv(1024) == b"before-revocation"
cert = ssl._ssl._test_decode_cert(str(ROOT / "secrets/pki/lb/cert.pem"))
revoked.write_text(json.dumps([cert["serialNumber"].lstrip("0").lower()]))
try:
conn.sendall(b"must-not-forward")
data = conn.recv(1024)
except OSError:
data = b""
assert data == b""
Generated
+1001
View File
File diff suppressed because it is too large Load Diff