Implement TLS identities, scoped OpenBao access and worker security study
This commit is contained in:
+10
-19
@@ -1,20 +1,11 @@
|
||||
.git
|
||||
.github
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
*
|
||||
!pyproject.toml
|
||||
!uv.lock
|
||||
!src/
|
||||
!src/**
|
||||
!migrations/
|
||||
!migrations/**
|
||||
|
||||
operator-private
|
||||
secrets
|
||||
certs
|
||||
volumes
|
||||
data
|
||||
backups
|
||||
*.key
|
||||
*.pem
|
||||
*.p12
|
||||
*.pfx
|
||||
*.jks
|
||||
.venv
|
||||
__pycache__
|
||||
node_modules
|
||||
evidence
|
||||
.DS_Store
|
||||
.env
|
||||
|
||||
@@ -30,3 +30,9 @@ coverage/
|
||||
.vscode/
|
||||
# Keep small, sanitized evidence summaries in evidence/ under version control.
|
||||
# Inspect git diff --cached: ignore rules are not a secret scanner.
|
||||
.pytest_cache/
|
||||
.ruff_cache/
|
||||
|
||||
operator-private/
|
||||
secrets/
|
||||
.env
|
||||
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
FROM python:3.12-slim@sha256:646fb0bca3dd3ea1bcc6feb72c17ed16eed6e10cffc732fcc1478bd3e7f02d7b AS build
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.11.7 /uv /usr/local/bin/uv
|
||||
WORKDIR /app
|
||||
COPY pyproject.toml uv.lock ./
|
||||
RUN uv sync --locked --no-dev --no-install-project --python /usr/local/bin/python
|
||||
|
||||
FROM python:3.12-slim@sha256:646fb0bca3dd3ea1bcc6feb72c17ed16eed6e10cffc732fcc1478bd3e7f02d7b
|
||||
RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends libjpeg-turbo-progs && rm -rf /var/lib/apt/lists/*
|
||||
RUN python -m pip uninstall --yes pip
|
||||
ARG VCS_REF=uncommitted
|
||||
LABEL org.opencontainers.image.revision=$VCS_REF
|
||||
ENV PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=/app/src PATH=/app/.venv/bin:$PATH
|
||||
WORKDIR /app
|
||||
COPY --from=build /app/.venv /app/.venv
|
||||
COPY src ./src
|
||||
COPY migrations ./migrations
|
||||
USER 10001:10001
|
||||
EXPOSE 8080
|
||||
HEALTHCHECK --interval=3s --timeout=2s --start-period=5s --retries=5 CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=1)"]
|
||||
CMD ["python", "-m", "messenger"]
|
||||
@@ -31,3 +31,28 @@ test-failover:
|
||||
|
||||
test-security:
|
||||
$(PYTHON) tests/security.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
# Student implementation; teacher acceptance targets remain unchanged.
|
||||
PUBLIC_PORT ?= 8447
|
||||
BASE_URL = https://localhost:$(PUBLIC_PORT)
|
||||
CA_FILE = $(CURDIR)/secrets/ca.pem
|
||||
bootstrap:
|
||||
python3 scripts/bootstrap.py
|
||||
build:
|
||||
VCS_REF=$$(git rev-parse HEAD) docker compose build
|
||||
init: bootstrap build
|
||||
docker compose up -d postgres openbao cache cache-tls broker broker-tls storage s3 storage-tls
|
||||
python3 scripts/bao_operator.py init
|
||||
up: init
|
||||
VCS_REF=$$(git rev-parse HEAD) docker compose up -d --wait
|
||||
unseal:
|
||||
python3 scripts/bao_operator.py unseal
|
||||
migrate:
|
||||
docker compose run --rm migrate
|
||||
down:
|
||||
docker compose down
|
||||
student-test:
|
||||
BASE_URL=$(BASE_URL) SSL_CERT_FILE=$(CA_FILE) uv run --locked pytest -q
|
||||
lint:
|
||||
uv run --locked ruff check src student-tests scripts
|
||||
.PHONY: bootstrap build init up unseal migrate down student-test lint
|
||||
|
||||
+755
@@ -0,0 +1,755 @@
|
||||
name: mtusi-vasilev-lab7
|
||||
x-hardening:
|
||||
cpus: 0.25
|
||||
mem_limit: 128m
|
||||
pids_limit: 96
|
||||
cap_drop: &id001
|
||||
- ALL
|
||||
security_opt: &id002
|
||||
- no-new-privileges:true
|
||||
read_only: true
|
||||
tmpfs: &id003
|
||||
- /tmp:size=16m,noexec,nosuid
|
||||
x-environment:
|
||||
BAO_ADDR: https://openbao:8200
|
||||
S3_ENDPOINT: https://storage:8443
|
||||
S3_BUCKET: messenger
|
||||
S3_CA_FILE: /run/tls/ca.pem
|
||||
SSL_CERT_FILE: /run/tls/ca.pem
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-45}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: https://collector:4318
|
||||
OTEL_EXPORTER_OTLP_CERTIFICATE: /run/tls/ca.pem
|
||||
OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/tls/identity/cert.pem
|
||||
OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/tls/identity/key.pem
|
||||
OTEL_ENABLED: '1'
|
||||
OTEL_METRIC_INTERVAL_MS: '5000'
|
||||
x-application:
|
||||
cpus: 0.5
|
||||
mem_limit: 512m
|
||||
pids_limit: 96
|
||||
cap_drop: *id001
|
||||
security_opt: *id002
|
||||
read_only: true
|
||||
tmpfs: *id003
|
||||
image: ${API_IMAGE:-mtusi-vasilev:lab7}
|
||||
build: &id004
|
||||
context: .
|
||||
args:
|
||||
VCS_REF: ${VCS_REF:-uncommitted}
|
||||
stop_grace_period: 8s
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17.10-alpine
|
||||
entrypoint:
|
||||
- /etc/pg-entrypoint.sh
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets:
|
||||
- postgres_password
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
- ./secrets/pki/postgres:/run/tls/identity:ro
|
||||
- ./secrets/ca.pem:/run/tls/ca.pem:ro
|
||||
- ./infra/pg-entrypoint.sh:/etc/pg-entrypoint.sh:ro
|
||||
- ./infra/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro
|
||||
- ./secrets/db_roles.sql:/docker-entrypoint-initdb.d/01_roles.sql:ro
|
||||
cpus: 0.75
|
||||
mem_limit: 768m
|
||||
command:
|
||||
- postgres
|
||||
- -c
|
||||
- fsync=on
|
||||
- -c
|
||||
- ssl=on
|
||||
- -c
|
||||
- ssl_cert_file=/run/tls/identity/cert.pem
|
||||
- -c
|
||||
- ssl_key_file=/tmp/postgres-key.pem
|
||||
- -c
|
||||
- ssl_ca_file=/run/tls/ca.pem
|
||||
- -c
|
||||
- hba_file=/etc/postgresql/pg_hba.conf
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- pg_isready -U messenger -d messenger
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
openbao:
|
||||
cpus: 0.25
|
||||
mem_limit: 192m
|
||||
pids_limit: 96
|
||||
cap_drop: *id001
|
||||
security_opt: *id002
|
||||
read_only: true
|
||||
tmpfs: *id003
|
||||
image: ghcr.io/openbao/openbao:2.5.2
|
||||
user: 100:1000
|
||||
command:
|
||||
- server
|
||||
- -config=/etc/openbao.hcl
|
||||
ports:
|
||||
- 127.0.0.1:18207:8200
|
||||
volumes:
|
||||
- bao_data:/bao/data
|
||||
- ./infra/openbao.hcl:/etc/openbao.hcl:ro
|
||||
- ./secrets/pki/openbao:/run/tls/identity:ro
|
||||
- ./secrets/ca.pem:/run/tls/ca.pem:ro
|
||||
depends_on:
|
||||
state-init:
|
||||
condition: service_completed_successfully
|
||||
cache:
|
||||
cpus: 0.25
|
||||
mem_limit: 128m
|
||||
pids_limit: 96
|
||||
cap_drop: *id001
|
||||
security_opt: *id002
|
||||
read_only: true
|
||||
tmpfs: *id003
|
||||
image: valkey/valkey:8.1.3-alpine
|
||||
command:
|
||||
- /run/secrets/cache_config
|
||||
secrets:
|
||||
- cache_config
|
||||
user: 1000:1000
|
||||
entrypoint:
|
||||
- valkey-server
|
||||
cache-tls:
|
||||
cpus: 0.5
|
||||
mem_limit: 64m
|
||||
pids_limit: 96
|
||||
cap_drop: *id001
|
||||
security_opt: *id002
|
||||
read_only: true
|
||||
tmpfs: *id003
|
||||
image: ${API_IMAGE:-mtusi-vasilev:lab7}
|
||||
build: *id004
|
||||
stop_grace_period: 8s
|
||||
network_mode: service:cache
|
||||
command:
|
||||
- python
|
||||
- -m
|
||||
- messenger.tls_gateway
|
||||
- --config
|
||||
- /run/tls/gateway.json
|
||||
volumes:
|
||||
- ./secrets/pki/cache:/run/tls/identity:ro
|
||||
- ./secrets/ca.pem:/run/tls/ca.pem:ro
|
||||
- ./secrets/revoked.json:/run/tls/revoked.json:ro
|
||||
- ./secrets/cache_gateway.json:/run/tls/gateway.json:ro
|
||||
healthcheck:
|
||||
disable: true
|
||||
broker:
|
||||
cpus: 0.25
|
||||
mem_limit: 192m
|
||||
pids_limit: 96
|
||||
cap_drop: *id001
|
||||
security_opt: *id002
|
||||
read_only: true
|
||||
tmpfs: *id003
|
||||
image: valkey/valkey:8.1.3-alpine
|
||||
command:
|
||||
- /run/secrets/broker_config
|
||||
secrets:
|
||||
- broker_config
|
||||
volumes:
|
||||
- broker_data:/data
|
||||
depends_on:
|
||||
state-init:
|
||||
condition: service_completed_successfully
|
||||
user: 1000:1000
|
||||
entrypoint:
|
||||
- valkey-server
|
||||
broker-tls:
|
||||
cpus: 0.5
|
||||
mem_limit: 64m
|
||||
pids_limit: 96
|
||||
cap_drop: *id001
|
||||
security_opt: *id002
|
||||
read_only: true
|
||||
tmpfs: *id003
|
||||
image: ${API_IMAGE:-mtusi-vasilev:lab7}
|
||||
build: *id004
|
||||
stop_grace_period: 8s
|
||||
network_mode: service:broker
|
||||
command:
|
||||
- python
|
||||
- -m
|
||||
- messenger.tls_gateway
|
||||
- --config
|
||||
- /run/tls/gateway.json
|
||||
volumes:
|
||||
- ./secrets/pki/broker:/run/tls/identity:ro
|
||||
- ./secrets/ca.pem:/run/tls/ca.pem:ro
|
||||
- ./secrets/revoked.json:/run/tls/revoked.json:ro
|
||||
- ./secrets/broker_gateway.json:/run/tls/gateway.json:ro
|
||||
healthcheck:
|
||||
disable: true
|
||||
storage:
|
||||
image: chrislusf/seaweedfs:4.15@sha256:1593bd2a98964f5d4df83fa7db2b3d4ee7b20190813cc5bf8c3200e0e70d346e
|
||||
command:
|
||||
- server
|
||||
- -dir=/data
|
||||
- -ip=127.0.0.1
|
||||
- -ip.bind=127.0.0.1
|
||||
- -filer
|
||||
- -volume.max=32
|
||||
- -volume.preStopSeconds=0
|
||||
- -master.volumeSizeLimitMB=64
|
||||
volumes:
|
||||
- s3_data:/data
|
||||
- ./infra/filer.toml:/etc/seaweedfs/filer.toml:ro
|
||||
cpus: 0.5
|
||||
mem_limit: 512m
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- wget -q -O /dev/null http://127.0.0.1:9333/dir/status
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 25
|
||||
depends_on:
|
||||
state-init:
|
||||
condition: service_completed_successfully
|
||||
user: 1000:1000
|
||||
entrypoint:
|
||||
- /usr/bin/weed
|
||||
s3:
|
||||
image: chrislusf/seaweedfs:4.15@sha256:1593bd2a98964f5d4df83fa7db2b3d4ee7b20190813cc5bf8c3200e0e70d346e
|
||||
network_mode: service:storage
|
||||
command:
|
||||
- s3
|
||||
- -filer=127.0.0.1:8888
|
||||
- -ip.bind=127.0.0.1
|
||||
- -config=/run/secrets/s3_config
|
||||
- -port=8333
|
||||
- -port.iceberg=0
|
||||
secrets:
|
||||
- s3_config
|
||||
depends_on:
|
||||
storage:
|
||||
condition: service_healthy
|
||||
cpus: 0.25
|
||||
mem_limit: 256m
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
user: 1000:1000
|
||||
entrypoint:
|
||||
- /usr/bin/weed
|
||||
storage-tls:
|
||||
cpus: 0.5
|
||||
mem_limit: 96m
|
||||
pids_limit: 96
|
||||
cap_drop: *id001
|
||||
security_opt: *id002
|
||||
read_only: true
|
||||
tmpfs: *id003
|
||||
image: ${API_IMAGE:-mtusi-vasilev:lab7}
|
||||
build: *id004
|
||||
stop_grace_period: 8s
|
||||
network_mode: service:storage
|
||||
command:
|
||||
- python
|
||||
- -m
|
||||
- messenger.tls_gateway
|
||||
- --config
|
||||
- /run/tls/gateway.json
|
||||
volumes:
|
||||
- ./secrets/pki/storage:/run/tls/identity:ro
|
||||
- ./secrets/ca.pem:/run/tls/ca.pem:ro
|
||||
- ./secrets/revoked.json:/run/tls/revoked.json:ro
|
||||
- ./secrets/storage_gateway.json:/run/tls/gateway.json:ro
|
||||
healthcheck:
|
||||
disable: true
|
||||
migrate:
|
||||
cpus: 0.5
|
||||
mem_limit: 512m
|
||||
pids_limit: 96
|
||||
cap_drop: *id001
|
||||
security_opt: *id002
|
||||
read_only: true
|
||||
tmpfs: *id003
|
||||
image: ${API_IMAGE:-mtusi-vasilev:lab7}
|
||||
build: *id004
|
||||
stop_grace_period: 8s
|
||||
command:
|
||||
- python
|
||||
- -m
|
||||
- messenger.migrate
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/admin_database_url
|
||||
secrets:
|
||||
- admin_database_url
|
||||
volumes:
|
||||
- ./secrets/ca.pem:/run/tls/ca.pem:ro
|
||||
healthcheck:
|
||||
disable: true
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
s3-bootstrap:
|
||||
cpus: 0.5
|
||||
mem_limit: 512m
|
||||
pids_limit: 96
|
||||
cap_drop: *id001
|
||||
security_opt: *id002
|
||||
read_only: true
|
||||
tmpfs: *id003
|
||||
image: ${API_IMAGE:-mtusi-vasilev:lab7}
|
||||
build: *id004
|
||||
stop_grace_period: 8s
|
||||
command:
|
||||
- python
|
||||
- -m
|
||||
- messenger.storage_bootstrap
|
||||
environment:
|
||||
S3_ENDPOINT: https://storage:8443
|
||||
S3_ADMIN_ACCESS_KEY_FILE: /run/secrets/admin_s3_access_key
|
||||
S3_ADMIN_SECRET_KEY_FILE: /run/secrets/admin_s3_secret_key
|
||||
secrets:
|
||||
- admin_s3_access_key
|
||||
- admin_s3_secret_key
|
||||
volumes:
|
||||
- ./secrets/pki/operator:/run/tls/identity:ro
|
||||
- ./secrets/ca.pem:/run/tls/ca.pem:ro
|
||||
healthcheck:
|
||||
disable: true
|
||||
depends_on:
|
||||
storage-tls:
|
||||
condition: service_started
|
||||
api:
|
||||
cpus: 0.5
|
||||
mem_limit: 512m
|
||||
pids_limit: 96
|
||||
cap_drop: *id001
|
||||
security_opt: *id002
|
||||
read_only: true
|
||||
tmpfs: *id003
|
||||
image: ${API_IMAGE:-mtusi-vasilev:lab7}
|
||||
build: *id004
|
||||
stop_grace_period: 8s
|
||||
command:
|
||||
- python
|
||||
- -m
|
||||
- messenger.secure_entry
|
||||
environment:
|
||||
BAO_ADDR: https://openbao:8200
|
||||
S3_ENDPOINT: https://storage:8443
|
||||
S3_BUCKET: messenger
|
||||
S3_CA_FILE: /run/tls/ca.pem
|
||||
SSL_CERT_FILE: /run/tls/ca.pem
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-45}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: https://collector:4318
|
||||
OTEL_EXPORTER_OTLP_CERTIFICATE: /run/tls/ca.pem
|
||||
OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/tls/identity/cert.pem
|
||||
OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/tls/identity/key.pem
|
||||
BAO_ROLE: api
|
||||
TLS_GATEWAY_CONFIG: /run/tls/gateway.json
|
||||
OTEL_ENABLED: '1'
|
||||
OTEL_METRIC_INTERVAL_MS: '5000'
|
||||
volumes:
|
||||
- ./secrets/pki/api:/run/tls/identity:ro
|
||||
- ./secrets/ca.pem:/run/tls/ca.pem:ro
|
||||
- ./secrets/revoked.json:/run/tls/revoked.json:ro
|
||||
- ./secrets/api_gateway.json:/run/tls/gateway.json:ro
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
cache-tls:
|
||||
condition: service_started
|
||||
dispatcher:
|
||||
cpus: 0.5
|
||||
mem_limit: 192m
|
||||
pids_limit: 96
|
||||
cap_drop: *id001
|
||||
security_opt: *id002
|
||||
read_only: true
|
||||
tmpfs: *id003
|
||||
image: ${API_IMAGE:-mtusi-vasilev:lab7}
|
||||
build: *id004
|
||||
stop_grace_period: 8s
|
||||
command:
|
||||
- python
|
||||
- -m
|
||||
- messenger.worker
|
||||
- --dispatcher
|
||||
environment:
|
||||
BAO_ADDR: https://openbao:8200
|
||||
S3_ENDPOINT: https://storage:8443
|
||||
S3_BUCKET: messenger
|
||||
S3_CA_FILE: /run/tls/ca.pem
|
||||
SSL_CERT_FILE: /run/tls/ca.pem
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-45}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: https://collector:4318
|
||||
OTEL_EXPORTER_OTLP_CERTIFICATE: /run/tls/ca.pem
|
||||
OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/tls/identity/cert.pem
|
||||
OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/tls/identity/key.pem
|
||||
BAO_ROLE: dispatcher
|
||||
OTEL_ENABLED: '1'
|
||||
OTEL_METRIC_INTERVAL_MS: '5000'
|
||||
volumes:
|
||||
- ./secrets/pki/dispatcher:/run/tls/identity:ro
|
||||
- ./secrets/ca.pem:/run/tls/ca.pem:ro
|
||||
healthcheck:
|
||||
disable: true
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
broker-tls:
|
||||
condition: service_started
|
||||
worker:
|
||||
cpus: 0.75
|
||||
mem_limit: 896m
|
||||
pids_limit: 96
|
||||
cap_drop: *id001
|
||||
security_opt: *id002
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp:size=32m,noexec,nosuid
|
||||
image: ${API_IMAGE:-mtusi-vasilev:lab7}
|
||||
build: *id004
|
||||
stop_grace_period: 30s
|
||||
command:
|
||||
- python
|
||||
- -m
|
||||
- messenger.worker
|
||||
environment:
|
||||
BAO_ADDR: https://openbao:8200
|
||||
S3_ENDPOINT: https://storage:8443
|
||||
S3_BUCKET: messenger
|
||||
S3_CA_FILE: /run/tls/ca.pem
|
||||
SSL_CERT_FILE: /run/tls/ca.pem
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-45}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: https://collector:4318
|
||||
OTEL_EXPORTER_OTLP_CERTIFICATE: /run/tls/ca.pem
|
||||
OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/tls/identity/cert.pem
|
||||
OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/tls/identity/key.pem
|
||||
BAO_ROLE: worker
|
||||
OTEL_ENABLED: '1'
|
||||
OTEL_METRIC_INTERVAL_MS: '5000'
|
||||
volumes:
|
||||
- ./secrets/pki/worker:/run/tls/identity:ro
|
||||
- ./secrets/ca.pem:/run/tls/ca.pem:ro
|
||||
healthcheck:
|
||||
disable: true
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
broker-tls:
|
||||
condition: service_started
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
lb:
|
||||
cpus: 0.25
|
||||
mem_limit: 128m
|
||||
pids_limit: 96
|
||||
cap_drop: *id001
|
||||
security_opt: *id002
|
||||
read_only: true
|
||||
tmpfs: *id003
|
||||
image: haproxy:3.2-alpine@sha256:96ec3c75a9264ba94e733eeacb44051ed6e7b79bb77a73c2c36f236b6423f1f0
|
||||
ports:
|
||||
- 127.0.0.1:${PUBLIC_PORT:-8447}:8443
|
||||
volumes:
|
||||
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
- ./secrets/pki/lb:/run/tls/identity:ro
|
||||
- ./secrets/ca.pem:/run/tls/ca.pem:ro
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- test -S /tmp/haproxy.sock
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
collector:
|
||||
cpus: 0.25
|
||||
mem_limit: 256m
|
||||
pids_limit: 96
|
||||
cap_drop: *id001
|
||||
security_opt: *id002
|
||||
read_only: true
|
||||
tmpfs: *id003
|
||||
image: otel/opentelemetry-collector-contrib:0.123.0
|
||||
command:
|
||||
- --config=/etc/otelcol/config.yaml
|
||||
volumes:
|
||||
- ./infra/observability/collector.yaml:/etc/otelcol/config.yaml:ro
|
||||
- ./secrets/pki/collector:/run/tls/identity:ro
|
||||
- ./secrets/ca.pem:/run/tls/ca.pem:ro
|
||||
prometheus:
|
||||
cpus: 0.25
|
||||
mem_limit: 256m
|
||||
pids_limit: 96
|
||||
cap_drop: *id001
|
||||
security_opt: *id002
|
||||
read_only: true
|
||||
tmpfs: *id003
|
||||
image: prom/prometheus:v3.2.1
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time=24h
|
||||
- --web.listen-address=127.0.0.1:9090
|
||||
volumes:
|
||||
- ./infra/observability/prometheus.yaml:/etc/prometheus/prometheus.yml:ro
|
||||
- ./infra/observability/alerts.yaml:/etc/prometheus/alerts.yaml:ro
|
||||
- prometheus_data:/prometheus
|
||||
- ./secrets/pki/prometheus:/run/tls/identity:ro
|
||||
- ./secrets/ca.pem:/run/tls/ca.pem:ro
|
||||
ports:
|
||||
- 127.0.0.1:19097:9443
|
||||
depends_on:
|
||||
state-init:
|
||||
condition: service_completed_successfully
|
||||
loki:
|
||||
cpus: 0.25
|
||||
mem_limit: 384m
|
||||
pids_limit: 96
|
||||
cap_drop: *id001
|
||||
security_opt: *id002
|
||||
read_only: true
|
||||
tmpfs: *id003
|
||||
image: grafana/loki:3.4.2
|
||||
command:
|
||||
- -config.file=/etc/loki/config.yaml
|
||||
volumes:
|
||||
- ./infra/observability/loki.yaml:/etc/loki/config.yaml:ro
|
||||
- loki_data:/loki
|
||||
depends_on:
|
||||
state-init:
|
||||
condition: service_completed_successfully
|
||||
tempo:
|
||||
cpus: 0.25
|
||||
mem_limit: 768m
|
||||
environment:
|
||||
GOMEMLIMIT: 512MiB
|
||||
pids_limit: 96
|
||||
cap_drop: *id001
|
||||
security_opt: *id002
|
||||
read_only: true
|
||||
tmpfs: *id003
|
||||
image: grafana/tempo:2.7.2
|
||||
command:
|
||||
- -config.file=/etc/tempo/config.yaml
|
||||
volumes:
|
||||
- ./infra/observability/tempo.yaml:/etc/tempo/config.yaml:ro
|
||||
- tempo_data:/var/tempo
|
||||
depends_on:
|
||||
state-init:
|
||||
condition: service_completed_successfully
|
||||
grafana:
|
||||
cpus: 0.25
|
||||
mem_limit: 256m
|
||||
pids_limit: 96
|
||||
cap_drop: *id001
|
||||
security_opt: *id002
|
||||
read_only: true
|
||||
tmpfs: *id003
|
||||
image: grafana/grafana:11.6.0
|
||||
environment:
|
||||
GF_AUTH_ANONYMOUS_ENABLED: 'false'
|
||||
GF_AUTH_DISABLE_LOGIN_FORM: 'false'
|
||||
GF_USERS_ALLOW_SIGN_UP: 'false'
|
||||
GF_ANALYTICS_REPORTING_ENABLED: 'false'
|
||||
GF_ANALYTICS_CHECK_FOR_UPDATES: 'false'
|
||||
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_password
|
||||
GF_SERVER_PROTOCOL: https
|
||||
GF_SERVER_CERT_FILE: /run/tls/identity/cert.pem
|
||||
GF_SERVER_CERT_KEY: /run/tls/identity/key.pem
|
||||
volumes:
|
||||
- ./secrets/grafana-provisioning/datasources:/etc/grafana/provisioning/datasources:ro
|
||||
- ./infra/grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro
|
||||
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
- grafana_data:/var/lib/grafana
|
||||
- ./secrets/pki/grafana:/run/tls/identity:ro
|
||||
- ./secrets/ca.pem:/run/tls/ca.pem:ro
|
||||
ports:
|
||||
- 127.0.0.1:13007:3000
|
||||
secrets:
|
||||
- grafana_password
|
||||
depends_on:
|
||||
state-init:
|
||||
condition: service_completed_successfully
|
||||
collector-tls:
|
||||
cpus: 0.5
|
||||
mem_limit: 64m
|
||||
pids_limit: 96
|
||||
cap_drop: *id001
|
||||
security_opt: *id002
|
||||
read_only: true
|
||||
tmpfs: *id003
|
||||
image: ${API_IMAGE:-mtusi-vasilev:lab7}
|
||||
build: *id004
|
||||
stop_grace_period: 8s
|
||||
network_mode: service:collector
|
||||
command:
|
||||
- python
|
||||
- -m
|
||||
- messenger.tls_gateway
|
||||
- --config
|
||||
- /run/tls/gateway.json
|
||||
volumes:
|
||||
- ./secrets/pki/collector:/run/tls/identity:ro
|
||||
- ./secrets/ca.pem:/run/tls/ca.pem:ro
|
||||
- ./secrets/revoked.json:/run/tls/revoked.json:ro
|
||||
- ./secrets/collector_gateway.json:/run/tls/gateway.json:ro
|
||||
healthcheck:
|
||||
disable: true
|
||||
prometheus-tls:
|
||||
cpus: 0.5
|
||||
mem_limit: 64m
|
||||
pids_limit: 96
|
||||
cap_drop: *id001
|
||||
security_opt: *id002
|
||||
read_only: true
|
||||
tmpfs: *id003
|
||||
image: ${API_IMAGE:-mtusi-vasilev:lab7}
|
||||
build: *id004
|
||||
stop_grace_period: 8s
|
||||
network_mode: service:prometheus
|
||||
command:
|
||||
- python
|
||||
- -m
|
||||
- messenger.tls_gateway
|
||||
- --config
|
||||
- /run/tls/gateway.json
|
||||
volumes:
|
||||
- ./secrets/pki/prometheus:/run/tls/identity:ro
|
||||
- ./secrets/ca.pem:/run/tls/ca.pem:ro
|
||||
- ./secrets/revoked.json:/run/tls/revoked.json:ro
|
||||
- ./secrets/prometheus_gateway.json:/run/tls/gateway.json:ro
|
||||
healthcheck:
|
||||
disable: true
|
||||
loki-tls:
|
||||
cpus: 0.5
|
||||
mem_limit: 64m
|
||||
pids_limit: 96
|
||||
cap_drop: *id001
|
||||
security_opt: *id002
|
||||
read_only: true
|
||||
tmpfs: *id003
|
||||
image: ${API_IMAGE:-mtusi-vasilev:lab7}
|
||||
build: *id004
|
||||
stop_grace_period: 8s
|
||||
network_mode: service:loki
|
||||
command:
|
||||
- python
|
||||
- -m
|
||||
- messenger.tls_gateway
|
||||
- --config
|
||||
- /run/tls/gateway.json
|
||||
volumes:
|
||||
- ./secrets/pki/loki:/run/tls/identity:ro
|
||||
- ./secrets/ca.pem:/run/tls/ca.pem:ro
|
||||
- ./secrets/revoked.json:/run/tls/revoked.json:ro
|
||||
- ./secrets/loki_gateway.json:/run/tls/gateway.json:ro
|
||||
healthcheck:
|
||||
disable: true
|
||||
tempo-tls:
|
||||
cpus: 0.5
|
||||
mem_limit: 64m
|
||||
pids_limit: 96
|
||||
cap_drop: *id001
|
||||
security_opt: *id002
|
||||
read_only: true
|
||||
tmpfs: *id003
|
||||
image: ${API_IMAGE:-mtusi-vasilev:lab7}
|
||||
build: *id004
|
||||
stop_grace_period: 8s
|
||||
network_mode: service:tempo
|
||||
command:
|
||||
- python
|
||||
- -m
|
||||
- messenger.tls_gateway
|
||||
- --config
|
||||
- /run/tls/gateway.json
|
||||
volumes:
|
||||
- ./secrets/pki/tempo:/run/tls/identity:ro
|
||||
- ./secrets/ca.pem:/run/tls/ca.pem:ro
|
||||
- ./secrets/revoked.json:/run/tls/revoked.json:ro
|
||||
- ./secrets/tempo_gateway.json:/run/tls/gateway.json:ro
|
||||
healthcheck:
|
||||
disable: true
|
||||
state-init:
|
||||
cpus: 0.5
|
||||
mem_limit: 64m
|
||||
pids_limit: 96
|
||||
cap_drop: *id001
|
||||
security_opt: *id002
|
||||
read_only: true
|
||||
tmpfs: *id003
|
||||
image: ${API_IMAGE:-mtusi-vasilev:lab7}
|
||||
build: *id004
|
||||
stop_grace_period: 8s
|
||||
user: 0:0
|
||||
command:
|
||||
- python
|
||||
- -c
|
||||
- import os; pairs=[('/state/bao',100),('/state/broker',1000),('/state/storage',1000),('/state/prometheus',65534),('/state/loki',10001),('/state/tempo',10001),('/state/grafana',472)];
|
||||
[(os.chown(root,uid,uid),[os.chown(os.path.join(root,n),uid,uid) for n in dirs+files])
|
||||
for path,uid in pairs for root,dirs,files in os.walk(path)]
|
||||
cap_add:
|
||||
- CHOWN
|
||||
- DAC_OVERRIDE
|
||||
healthcheck:
|
||||
disable: true
|
||||
volumes:
|
||||
- bao_data:/state/bao
|
||||
- broker_data:/state/broker
|
||||
- s3_data:/state/storage
|
||||
- prometheus_data:/state/prometheus
|
||||
- loki_data:/state/loki
|
||||
- tempo_data:/state/tempo
|
||||
- grafana_data:/state/grafana
|
||||
volumes:
|
||||
postgres_data: null
|
||||
s3_data: null
|
||||
broker_data: null
|
||||
bao_data: null
|
||||
prometheus_data: {}
|
||||
loki_data: {}
|
||||
tempo_data: {}
|
||||
grafana_data: {}
|
||||
secrets:
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
cache_config:
|
||||
file: ./secrets/cache_config
|
||||
broker_config:
|
||||
file: ./secrets/broker_config
|
||||
s3_config:
|
||||
file: ./secrets/s3_config
|
||||
admin_database_url:
|
||||
file: ./secrets/admin_database_url
|
||||
admin_s3_access_key:
|
||||
file: ./secrets/admin_s3_access_key
|
||||
admin_s3_secret_key:
|
||||
file: ./secrets/admin_s3_secret_key
|
||||
grafana_password:
|
||||
file: ./secrets/grafana_password
|
||||
@@ -0,0 +1,3 @@
|
||||
[leveldb2]
|
||||
enabled = true
|
||||
dir = "/data/filerldb2"
|
||||
@@ -0,0 +1,450 @@
|
||||
{
|
||||
"uid": "messenger-overview",
|
||||
"title": "Messenger: API and image pipeline",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"refresh": "5s",
|
||||
"time": {
|
||||
"from": "now-15m",
|
||||
"to": "now"
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "Business RPS",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(rate(messenger_http_requests_total[1m]))"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "Business 5xx ratio",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(rate(messenger_http_requests_total{status_class=\"5xx\"}[1m])) / clamp_min(sum(rate(messenger_http_requests_total[1m])),0.001)"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 0
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "HTTP latency quantiles",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))",
|
||||
"legendFormat": "p50"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))",
|
||||
"legendFormat": "p95"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))",
|
||||
"legendFormat": "p99"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "Job outcomes",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (outcome) (rate(messenger_jobs_total[1m]))"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "Job p95 processing",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket[1m])))"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 16
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "End-to-end p95 to ready",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_ready_duration_seconds_bucket[1m])))"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 16
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "Outstanding jobs",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "max(messenger_queue_depth)"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 24
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"title": "Oldest outstanding age",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "max(messenger_queue_oldest_age_seconds)"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 24
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"title": "DB pool used",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "messenger_db_pool_used"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 32
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"title": "DB pool waiting",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "messenger_db_pool_waiting"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 32
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"title": "Worker busy/capacity",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "messenger_worker_busy / messenger_worker_capacity"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 40
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"title": "Process resident memory",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "messenger_process_memory_bytes"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 40
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 13,
|
||||
"title": "Process CPU (100%=1 core)",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "messenger_process_cpu_percent"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 48
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 14,
|
||||
"title": "Sampling decisions",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (decision) (rate(messenger_http_trace_decisions_total[1m]))"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 48
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 15,
|
||||
"title": "Collector failed span sends",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "rate(otelcol_exporter_send_failed_spans_total[1m])"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 56
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 99,
|
||||
"title": "Structured application logs (trace_id and request_id)",
|
||||
"type": "logs",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "{service_name=~\"messenger-api|messenger-worker\"}"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 64
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
apiVersion: 1
|
||||
providers:
|
||||
- name: messenger
|
||||
folder: Messenger
|
||||
type: file
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
@@ -0,0 +1,27 @@
|
||||
apiVersion: 1
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
uid: prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
- name: Loki
|
||||
uid: loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
editable: false
|
||||
jsonData:
|
||||
derivedFields:
|
||||
- datasourceUid: tempo
|
||||
matcherRegex: '"trace_id": "([a-f0-9]{32})"'
|
||||
name: TraceID
|
||||
url: '$${__value.raw}'
|
||||
- name: Tempo
|
||||
uid: tempo
|
||||
type: tempo
|
||||
access: proxy
|
||||
url: http://tempo:3200
|
||||
editable: false
|
||||
@@ -0,0 +1,38 @@
|
||||
global
|
||||
log stdout format raw local0
|
||||
maxconn 256
|
||||
stats socket /tmp/haproxy.sock mode 600 level user
|
||||
ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets
|
||||
ssl-default-server-options ssl-min-ver TLSv1.2 no-tls-tickets
|
||||
|
||||
defaults
|
||||
mode http
|
||||
log global
|
||||
timeout connect 1s
|
||||
timeout client 20s
|
||||
timeout server 15s
|
||||
timeout check 2s
|
||||
timeout queue 2s
|
||||
retries 0
|
||||
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 1s
|
||||
hold obsolete 1s
|
||||
hold nx 1s
|
||||
|
||||
frontend public
|
||||
bind :8443 ssl crt /run/tls/identity/combined.pem
|
||||
http-request set-var(txn.lb_request_id) uuid()
|
||||
http-request return status 503 content-type application/json lf-string '{"error":{"code":"unavailable","message":"No ready API","request_id":"%[var(txn.lb_request_id)]"}}' hdr X-Request-Id '%[var(txn.lb_request_id)]' if { nbsrv(apis) eq 0 }
|
||||
default_backend apis
|
||||
|
||||
backend apis
|
||||
balance roundrobin
|
||||
option httpchk GET /health/ready
|
||||
http-check expect status 200
|
||||
default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 2
|
||||
server-template api 1-8 api:9443 ssl verify required ca-file /run/tls/ca.pem crt /run/tls/identity/combined.pem verifyhost api sni str(api) check check-sni api resolvers docker resolve-prefer ipv4 init-addr libc,none
|
||||
@@ -0,0 +1,24 @@
|
||||
groups:
|
||||
- name: messenger-slo
|
||||
rules:
|
||||
- alert: MessengerHighErrorRate
|
||||
expr: sum(rate(messenger_http_requests_total{status_class="5xx"}[1m])) / clamp_min(sum(rate(messenger_http_requests_total[1m])), 0.001) > 0.05
|
||||
for: 10s
|
||||
labels: {severity: warning}
|
||||
annotations:
|
||||
summary: Business HTTP 5xx ratio above 5% over 1 minute
|
||||
runbook_url: file:RUNBOOK.md#api-errors
|
||||
- alert: MessengerQueueLag
|
||||
expr: max(messenger_queue_oldest_age_seconds) > 15 and max(messenger_queue_depth) > 0
|
||||
for: 10s
|
||||
labels: {severity: warning}
|
||||
annotations:
|
||||
summary: Outstanding attachment job age above 15 seconds
|
||||
runbook_url: file:RUNBOOK.md#queue-lag
|
||||
- alert: MessengerTelemetryUnavailable
|
||||
expr: up{job="messenger"} == 0
|
||||
for: 10s
|
||||
labels: {severity: warning}
|
||||
annotations:
|
||||
summary: Collector metrics unavailable
|
||||
runbook_url: file:RUNBOOK.md#telemetry-unavailable
|
||||
@@ -0,0 +1,87 @@
|
||||
extensions:
|
||||
health_check:
|
||||
endpoint: 127.0.0.1:13133
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 127.0.0.1:14318
|
||||
processors:
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 160
|
||||
spike_limit_mib: 32
|
||||
batch:
|
||||
timeout: 1s
|
||||
send_batch_size: 128
|
||||
send_batch_max_size: 256
|
||||
exporters:
|
||||
prometheus:
|
||||
endpoint: 127.0.0.1:19464
|
||||
metric_expiration: 15m
|
||||
resource_to_telemetry_conversion:
|
||||
enabled: true
|
||||
otlphttp/tempo:
|
||||
endpoint: https://tempo:44318
|
||||
tls:
|
||||
ca_file: /run/tls/ca.pem
|
||||
cert_file: /run/tls/identity/cert.pem
|
||||
key_file: /run/tls/identity/key.pem
|
||||
timeout: 2s
|
||||
sending_queue:
|
||||
queue_size: 128
|
||||
num_consumers: 1
|
||||
retry_on_failure:
|
||||
initial_interval: 1s
|
||||
max_interval: 2s
|
||||
max_elapsed_time: 5s
|
||||
otlphttp/loki:
|
||||
endpoint: https://loki:9443/otlp
|
||||
timeout: 2s
|
||||
sending_queue:
|
||||
queue_size: 128
|
||||
num_consumers: 1
|
||||
retry_on_failure:
|
||||
initial_interval: 1s
|
||||
max_interval: 2s
|
||||
max_elapsed_time: 5s
|
||||
tls:
|
||||
ca_file: /run/tls/ca.pem
|
||||
cert_file: /run/tls/identity/cert.pem
|
||||
key_file: /run/tls/identity/key.pem
|
||||
service:
|
||||
extensions:
|
||||
- health_check
|
||||
telemetry:
|
||||
metrics:
|
||||
readers:
|
||||
- pull:
|
||||
exporter:
|
||||
prometheus:
|
||||
host: 127.0.0.1
|
||||
port: 18888
|
||||
pipelines:
|
||||
traces:
|
||||
receivers:
|
||||
- otlp
|
||||
processors:
|
||||
- memory_limiter
|
||||
- batch
|
||||
exporters:
|
||||
- otlphttp/tempo
|
||||
metrics:
|
||||
receivers:
|
||||
- otlp
|
||||
processors:
|
||||
- memory_limiter
|
||||
- batch
|
||||
exporters:
|
||||
- prometheus
|
||||
logs:
|
||||
receivers:
|
||||
- otlp
|
||||
processors:
|
||||
- memory_limiter
|
||||
- batch
|
||||
exporters:
|
||||
- otlphttp/loki
|
||||
@@ -0,0 +1,34 @@
|
||||
auth_enabled: false
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
http_listen_address: 127.0.0.1
|
||||
grpc_listen_address: 127.0.0.1
|
||||
common:
|
||||
instance_addr: 127.0.0.1
|
||||
path_prefix: /loki
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2024-01-01
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
limits_config:
|
||||
allow_structured_metadata: true
|
||||
retention_period: 24h
|
||||
compactor:
|
||||
working_directory: /loki/compactor
|
||||
retention_enabled: true
|
||||
delete_request_store: filesystem
|
||||
analytics:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,26 @@
|
||||
global:
|
||||
scrape_interval: 5s
|
||||
evaluation_interval: 5s
|
||||
rule_files:
|
||||
- /etc/prometheus/alerts.yaml
|
||||
scrape_configs:
|
||||
- job_name: messenger
|
||||
static_configs:
|
||||
- targets:
|
||||
- collector:9464
|
||||
scheme: https
|
||||
tls_config:
|
||||
ca_file: /run/tls/ca.pem
|
||||
cert_file: /run/tls/identity/cert.pem
|
||||
key_file: /run/tls/identity/key.pem
|
||||
server_name: collector
|
||||
- job_name: collector
|
||||
static_configs:
|
||||
- targets:
|
||||
- collector:8888
|
||||
scheme: https
|
||||
tls_config:
|
||||
ca_file: /run/tls/ca.pem
|
||||
cert_file: /run/tls/identity/cert.pem
|
||||
key_file: /run/tls/identity/key.pem
|
||||
server_name: collector
|
||||
@@ -0,0 +1,24 @@
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
http_listen_address: 127.0.0.1
|
||||
grpc_listen_address: 127.0.0.1
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 127.0.0.1:4318
|
||||
ingester:
|
||||
max_block_duration: 1m
|
||||
compactor:
|
||||
compaction:
|
||||
block_retention: 24h
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
local:
|
||||
path: /var/tempo/blocks
|
||||
usage_report:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,12 @@
|
||||
ui = true
|
||||
disable_mlock = true
|
||||
storage "file" { path = "/bao/data" }
|
||||
listener "tcp" {
|
||||
address = "0.0.0.0:8200"
|
||||
tls_cert_file = "/run/tls/identity/cert.pem"
|
||||
tls_key_file = "/run/tls/identity/key.pem"
|
||||
tls_client_ca_file = "/run/tls/ca.pem"
|
||||
tls_require_and_verify_client_cert = true
|
||||
tls_min_version = "tls12"
|
||||
}
|
||||
api_addr = "https://openbao:8200"
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cp /run/tls/identity/key.pem /tmp/postgres-key.pem
|
||||
chown postgres:postgres /tmp/postgres-key.pem
|
||||
chmod 600 /tmp/postgres-key.pem
|
||||
exec docker-entrypoint.sh "$@"
|
||||
@@ -0,0 +1,3 @@
|
||||
local all all trust
|
||||
hostnossl all all 0.0.0.0/0 reject
|
||||
hostssl messenger api,worker,dispatcher,messenger 0.0.0.0/0 scram-sha-256
|
||||
@@ -0,0 +1,35 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
CREATE TABLE members (
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
position integer NOT NULL CHECK (position >= 0),
|
||||
PRIMARY KEY(chat_id,user_id),
|
||||
UNIQUE(chat_id,position)
|
||||
);
|
||||
CREATE INDEX members_user_idx ON members(user_id,chat_id);
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL,
|
||||
sender_id uuid NOT NULL,
|
||||
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||
created_at timestamptz NOT NULL,
|
||||
FOREIGN KEY(chat_id,sender_id) REFERENCES members(chat_id,user_id)
|
||||
);
|
||||
CREATE INDEX messages_history_idx ON messages(chat_id,created_at,id);
|
||||
CREATE TABLE sessions (
|
||||
token_digest text PRIMARY KEY,
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
expires_at timestamptz NOT NULL
|
||||
);
|
||||
CREATE INDEX sessions_expiry_idx ON sessions(expires_at);
|
||||
@@ -0,0 +1,10 @@
|
||||
CREATE TABLE message_requests (
|
||||
actor_id uuid NOT NULL REFERENCES users(id),
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
key text NOT NULL,
|
||||
body_digest text NOT NULL,
|
||||
response jsonb NOT NULL,
|
||||
expires_at timestamptz NOT NULL,
|
||||
PRIMARY KEY(actor_id,chat_id,key)
|
||||
);
|
||||
CREATE INDEX message_requests_expiry_idx ON message_requests(expires_at);
|
||||
@@ -0,0 +1,13 @@
|
||||
-- PostgreSQL text rejects U+0000, while the API accepts Unicode code points.
|
||||
-- Preserve UTF-8 losslessly. API validation enforces code point limits; the DB
|
||||
-- additionally bounds byte lengths, without stripping or replacing characters.
|
||||
ALTER TABLE users DROP CONSTRAINT users_display_name_check;
|
||||
ALTER TABLE users ALTER COLUMN display_name TYPE bytea USING convert_to(display_name,'UTF8');
|
||||
ALTER TABLE users ADD CHECK (octet_length(display_name) BETWEEN 1 AND 400);
|
||||
ALTER TABLE chats DROP CONSTRAINT chats_title_check;
|
||||
ALTER TABLE chats ALTER COLUMN title TYPE bytea USING convert_to(title,'UTF8');
|
||||
ALTER TABLE chats ADD CHECK (octet_length(title) BETWEEN 1 AND 400);
|
||||
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
|
||||
ALTER TABLE messages ALTER COLUMN text TYPE bytea USING convert_to(text,'UTF8');
|
||||
ALTER TABLE messages ADD CHECK (octet_length(text) BETWEEN 1 AND 16000);
|
||||
ALTER TABLE message_requests ALTER COLUMN response TYPE bytea USING convert_to(response::text,'UTF8');
|
||||
@@ -0,0 +1,35 @@
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
owner_id uuid NOT NULL REFERENCES users(id),
|
||||
filename text NOT NULL,
|
||||
content_type text NOT NULL CHECK (content_type IN ('image/png','image/jpeg')),
|
||||
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 text NOT NULL,
|
||||
status text NOT NULL CONSTRAINT attachments_status_check CHECK (status IN ('queued','processing','ready','failed')),
|
||||
error_code text,
|
||||
created_at timestamptz NOT NULL,
|
||||
updated_at timestamptz NOT NULL,
|
||||
object_key text NOT NULL UNIQUE,
|
||||
UNIQUE(id,chat_id)
|
||||
);
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL REFERENCES messages(id),
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id),
|
||||
position integer NOT NULL CHECK (position BETWEEN 0 AND 9),
|
||||
PRIMARY KEY(message_id,attachment_id),
|
||||
UNIQUE(message_id,position)
|
||||
);
|
||||
CREATE INDEX message_attachments_attachment_idx ON message_attachments(attachment_id);
|
||||
CREATE TABLE upload_intents (
|
||||
id uuid PRIMARY KEY,
|
||||
object_key text NOT NULL UNIQUE,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
CREATE INDEX upload_intents_created_idx ON upload_intents(created_at,id);
|
||||
CREATE TABLE attachment_reconcile_audit (
|
||||
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
||||
event text NOT NULL,
|
||||
object_key text NOT NULL,
|
||||
happened_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
@@ -0,0 +1,3 @@
|
||||
-- Empty text is valid only when the API supplies at least one attachment.
|
||||
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
|
||||
ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (octet_length(text) <= 16000);
|
||||
@@ -0,0 +1,49 @@
|
||||
ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_status_check;
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_status_check
|
||||
CHECK (status IN ('queued','processing','ready','failed'));
|
||||
ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version > 0);
|
||||
ALTER TABLE attachments ADD COLUMN variants jsonb NOT NULL DEFAULT '[]';
|
||||
ALTER TABLE attachments ADD COLUMN stage_key text;
|
||||
|
||||
CREATE TABLE attachment_jobs (
|
||||
id uuid PRIMARY KEY,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id),
|
||||
pipeline_version integer NOT NULL CHECK (pipeline_version > 0),
|
||||
source_key text,
|
||||
state text NOT NULL CHECK (state IN ('queued','processing','ready','failed','superseded')),
|
||||
attempt integer NOT NULL DEFAULT 0 CHECK (attempt >= 0),
|
||||
attempt_limit integer NOT NULL CHECK (attempt_limit > 0),
|
||||
lease_id uuid,
|
||||
lease_until timestamptz,
|
||||
available_at timestamptz NOT NULL DEFAULT now(),
|
||||
error_code text,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
UNIQUE(attachment_id,pipeline_version)
|
||||
);
|
||||
CREATE INDEX jobs_recovery_idx ON attachment_jobs(state,lease_until);
|
||||
CREATE TABLE job_outbox (
|
||||
job_id uuid PRIMARY KEY REFERENCES attachment_jobs(id),
|
||||
published_at timestamptz
|
||||
);
|
||||
CREATE TABLE job_audit (
|
||||
id bigserial PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES attachment_jobs(id),
|
||||
event text NOT NULL,
|
||||
attempt integer NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE TABLE failed_jobs (
|
||||
job_id uuid PRIMARY KEY REFERENCES attachment_jobs(id),
|
||||
error_code text NOT NULL,
|
||||
failed_at timestamptz NOT NULL DEFAULT now(),
|
||||
replay_count integer NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE TABLE test_faults (
|
||||
id bigserial PRIMARY KEY,
|
||||
point text NOT NULL,
|
||||
attachment_id uuid,
|
||||
action text NOT NULL CHECK (action IN ('crash','pause','error')),
|
||||
reached_at timestamptz,
|
||||
released boolean NOT NULL DEFAULT false
|
||||
);
|
||||
@@ -0,0 +1,2 @@
|
||||
ALTER TABLE attachment_jobs ADD COLUMN traceparent text;
|
||||
ALTER TABLE attachment_jobs ADD COLUMN tracestate text;
|
||||
@@ -0,0 +1,16 @@
|
||||
CREATE TABLE login_windows (
|
||||
bucket text PRIMARY KEY,
|
||||
window_start timestamptz NOT NULL,
|
||||
count integer NOT NULL CHECK (count > 0)
|
||||
);
|
||||
REVOKE CREATE ON SCHEMA public FROM PUBLIC;
|
||||
GRANT USAGE ON SCHEMA public TO api,worker,dispatcher;
|
||||
GRANT SELECT,INSERT,UPDATE,DELETE ON users,chats,members,messages,sessions,message_requests,attachments,message_attachments,upload_intents,attachment_reconcile_audit,attachment_jobs,job_outbox,job_audit,failed_jobs,login_windows TO api;
|
||||
GRANT SELECT ON attachments,attachment_jobs,job_outbox,failed_jobs TO worker,dispatcher;
|
||||
GRANT UPDATE(status,error_code,object_key,variants,updated_at,stage_key) ON attachments TO worker,dispatcher;
|
||||
GRANT UPDATE(state,attempt,lease_id,lease_until,available_at,error_code,updated_at) ON attachment_jobs TO worker,dispatcher;
|
||||
GRANT UPDATE(published_at) ON job_outbox TO worker,dispatcher;
|
||||
GRANT INSERT ON job_audit TO worker,dispatcher;
|
||||
GRANT INSERT,UPDATE ON failed_jobs TO worker,dispatcher;
|
||||
GRANT USAGE,SELECT ON SEQUENCE job_audit_id_seq TO api,worker,dispatcher;
|
||||
GRANT USAGE,SELECT ON SEQUENCE attachment_reconcile_audit_id_seq TO api;
|
||||
@@ -0,0 +1,4 @@
|
||||
ALTER TABLE attachment_jobs ADD COLUMN accepted_at timestamptz;
|
||||
UPDATE attachment_jobs SET accepted_at=created_at;
|
||||
ALTER TABLE attachment_jobs ALTER COLUMN accepted_at SET NOT NULL;
|
||||
ALTER TABLE attachment_jobs ALTER COLUMN accepted_at SET DEFAULT clock_timestamp();
|
||||
@@ -0,0 +1 @@
|
||||
CREATE INDEX login_windows_expiry_idx ON login_windows(window_start);
|
||||
@@ -0,0 +1,32 @@
|
||||
[project]
|
||||
name = "mtusi-messenger"
|
||||
version = "1.0.0"
|
||||
requires-python = ">=3.12,<3.13"
|
||||
dependencies = [
|
||||
"fastapi",
|
||||
"uvicorn",
|
||||
"psycopg[binary,pool]",
|
||||
"redis",
|
||||
"argon2-cffi",
|
||||
"boto3>=1.43,<2",
|
||||
"pillow>=13.0.0,<14",
|
||||
"opentelemetry-sdk==1.45.2",
|
||||
"opentelemetry-exporter-otlp-proto-http==1.45.2",
|
||||
"psutil>=7,<8",
|
||||
]
|
||||
|
||||
[dependency-groups]
|
||||
dev = ["pytest", "httpx", "ruff", "openapi-spec-validator", "jsonschema"]
|
||||
|
||||
[tool.ruff]
|
||||
target-version = "py312"
|
||||
line-length = 100
|
||||
include = ["src/**/*.py", "student-tests/**/*.py", "research/**/*.py"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["student-tests"]
|
||||
pythonpath = ["src", "tests"]
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = ["E4", "E7", "E9", "F", "I", "UP", "B"]
|
||||
ignore = ["B008"] # FastAPI dependencies are declared in endpoint signatures.
|
||||
@@ -0,0 +1,115 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Shared-replica login window and real weak-hash rehash regression."""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, check # noqa: E402
|
||||
|
||||
|
||||
def sql(query):
|
||||
result = subprocess.run(
|
||||
[
|
||||
"docker",
|
||||
"compose",
|
||||
"exec",
|
||||
"-T",
|
||||
"postgres",
|
||||
"psql",
|
||||
"-U",
|
||||
"messenger",
|
||||
"-d",
|
||||
"messenger",
|
||||
"-At",
|
||||
"-v",
|
||||
"ON_ERROR_STOP=1",
|
||||
],
|
||||
cwd=ROOT,
|
||||
input=query,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=True,
|
||||
)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def main():
|
||||
c = Client("https://localhost:8447", str(ROOT / "secrets/ca.pem"))
|
||||
actor = c.user("rehash")
|
||||
password = actor["registration"]["password"]
|
||||
weak = PasswordHasher(time_cost=1, memory_cost=8192, parallelism=1).hash(password)
|
||||
sql("UPDATE users SET password_hash='" + weak + "' WHERE id='" + actor["id"] + "';")
|
||||
authorization = (
|
||||
"Basic "
|
||||
+ base64.b64encode((actor["registration"]["username"] + ":" + password).encode()).decode()
|
||||
)
|
||||
c.call(
|
||||
"POST",
|
||||
"/api/v1/auth/sessions",
|
||||
201,
|
||||
headers={"Authorization": authorization},
|
||||
schema="Session",
|
||||
)
|
||||
upgraded = sql("SELECT password_hash FROM users WHERE id='" + actor["id"] + "';")
|
||||
check(
|
||||
upgraded != weak and "$m=65536,t=2,p=1$" in upgraded,
|
||||
"successful login did not upgrade hash",
|
||||
)
|
||||
time.sleep(10.2)
|
||||
wrong = (
|
||||
"Basic "
|
||||
+ base64.b64encode((actor["registration"]["username"] + ":incorrect").encode()).decode()
|
||||
)
|
||||
rows = []
|
||||
for _ in range(12):
|
||||
response = c.call(
|
||||
"POST", "/api/v1/auth/sessions", expected=[401, 429], headers={"Authorization": wrong}
|
||||
)
|
||||
rows.append(
|
||||
{
|
||||
"status": response["status"],
|
||||
"instance": response["headers"]["X-Instance-Id"],
|
||||
"retry_after": response["headers"].get("Retry-After"),
|
||||
"milliseconds": round(response["elapsed"] * 1000, 3),
|
||||
}
|
||||
)
|
||||
sql(
|
||||
"UPDATE sessions SET expires_at=now()-interval '1 second' WHERE user_id='"
|
||||
+ actor["id"]
|
||||
+ "';"
|
||||
)
|
||||
expiry_rows = []
|
||||
for _ in range(4):
|
||||
expired = c.call("GET", "/api/v1/users/me", expected=401, actor=actor)
|
||||
expiry_rows.append(
|
||||
{"status": expired["status"], "instance": expired["headers"]["X-Instance-Id"]}
|
||||
)
|
||||
result = {
|
||||
"weak_hash_upgraded": True,
|
||||
"expired_session_requests": expiry_rows,
|
||||
"expired_session_denied_both_replicas": len({r["instance"] for r in expiry_rows}) >= 2,
|
||||
"configured_memory_kib": 65536,
|
||||
"time_cost": 2,
|
||||
"parallelism": 1,
|
||||
"requests": rows,
|
||||
"distinct_api_instances": len({r["instance"] for r in rows}),
|
||||
"passed": len({r["instance"] for r in expiry_rows}) >= 2
|
||||
and sum(r["status"] == 401 for r in rows) == 10
|
||||
and all(r["status"] == 429 and r["retry_after"] == "10" for r in rows[10:])
|
||||
and len({r["instance"] for r in rows}) >= 2,
|
||||
}
|
||||
(ROOT / "evidence/auth-budget.json").write_text(json.dumps(result, indent=2) + "\n")
|
||||
check(result["passed"], "shared login window failed")
|
||||
print(json.dumps({k: v for k, v in result.items() if k != "requests"}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,108 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Accepted jobs survive worker identity revoke + working-secret rotation."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, check # noqa: E402
|
||||
|
||||
|
||||
def docker(*args):
|
||||
return subprocess.check_output(["docker", "compose", *args], cwd=ROOT, text=True).strip()
|
||||
|
||||
|
||||
def images():
|
||||
ids = docker("ps", "-q", "api", "worker").split()
|
||||
return sorted(
|
||||
set(
|
||||
subprocess.check_output(
|
||||
["docker", "inspect", "--format", "{{.Image}}", *ids], text=True
|
||||
).split()
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
c = Client("https://localhost:8447", str(ROOT / "secrets/ca.pem"))
|
||||
alice, bob, _, chat = c.bootstrap()
|
||||
before = images()
|
||||
docker("stop", "worker")
|
||||
accepted = [c.attachment(alice, chat) for _ in range(3)]
|
||||
start = time.monotonic()
|
||||
rotate = subprocess.run(
|
||||
[sys.executable, "scripts/rotate_identity.py"], cwd=ROOT, text=True, capture_output=True
|
||||
)
|
||||
check(rotate.returncode == 0, "operator rotation failed; inspect private operator execution")
|
||||
errors = 0
|
||||
deadline = time.monotonic() + 90
|
||||
pending = {a[0]["id"] for a in accepted}
|
||||
while pending and time.monotonic() < deadline:
|
||||
for att, _ in accepted:
|
||||
if att["id"] not in pending:
|
||||
continue
|
||||
result = c.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + att["id"],
|
||||
expected=[200, 503],
|
||||
actor=alice,
|
||||
schema=None,
|
||||
)
|
||||
if result["status"] == 503:
|
||||
errors += 1
|
||||
else:
|
||||
body = json.loads(result["bytes"])
|
||||
check(body["status"] != "failed", "accepted task became failed")
|
||||
if body["status"] == "ready":
|
||||
pending.remove(att["id"])
|
||||
time.sleep(0.2)
|
||||
check(not pending, "accepted jobs did not recover")
|
||||
rows = []
|
||||
for att, original in accepted:
|
||||
downloaded = c.call("GET", "/api/v1/attachments/" + att["id"] + "/content", actor=bob)[
|
||||
"bytes"
|
||||
]
|
||||
thumbnail = c.call(
|
||||
"GET", "/api/v1/attachments/" + att["id"] + "/content?variant=thumbnail", actor=bob
|
||||
)["bytes"]
|
||||
sha = hashlib.sha256(original).hexdigest()
|
||||
rows.append(
|
||||
{
|
||||
"attachment_id": att["id"],
|
||||
"original_sha256": sha,
|
||||
"download_sha256": hashlib.sha256(downloaded).hexdigest(),
|
||||
"thumbnail_sha256": hashlib.sha256(thumbnail).hexdigest(),
|
||||
"thumbnail_nonempty": bool(thumbnail),
|
||||
"accepted_original_sha_matches": sha == att["sha256"],
|
||||
}
|
||||
)
|
||||
after = images()
|
||||
result = {
|
||||
"accepted_jobs": 3,
|
||||
"ready_jobs": len(rows),
|
||||
"recovery_seconds": round(time.monotonic() - start, 3),
|
||||
"observed_api_503": errors,
|
||||
"images_before": before,
|
||||
"images_after": after,
|
||||
"image_unchanged": before == after,
|
||||
"rows": rows,
|
||||
"passed": before == after
|
||||
and all(
|
||||
r["original_sha256"] == r["download_sha256"]
|
||||
and r["thumbnail_nonempty"]
|
||||
and r["accepted_original_sha_matches"]
|
||||
for r in rows
|
||||
),
|
||||
}
|
||||
(ROOT / "evidence/compromise-recovery.json").write_text(json.dumps(result, indent=2) + "\n")
|
||||
check(result["passed"], "recovery invariant failed")
|
||||
print(json.dumps({k: v for k, v in result.items() if k != "rows"}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fail closed after the service-auth lease expires, then bounded recovery."""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, check # noqa: E402
|
||||
|
||||
|
||||
def operator(action):
|
||||
subprocess.run(
|
||||
[sys.executable, "scripts/bao_operator.py", action],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
c = Client("https://localhost:8447", str(ROOT / "secrets/ca.pem"))
|
||||
actor = c.user("sealed")
|
||||
c.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
||||
start = time.monotonic()
|
||||
operator("seal")
|
||||
rows = []
|
||||
try:
|
||||
while time.monotonic() - start < 33:
|
||||
live = c.call("GET", "/health/live", schema="Health")
|
||||
response = c.call(
|
||||
"GET", "/api/v1/users/me", expected=[200, 503], actor=actor
|
||||
)
|
||||
rows.append(
|
||||
{
|
||||
"seconds": round(time.monotonic() - start, 3),
|
||||
"live": live["status"],
|
||||
"business": response["status"],
|
||||
}
|
||||
)
|
||||
time.sleep(1)
|
||||
check(rows[-1]["business"] == 503, "service used credentials beyond lease")
|
||||
finally:
|
||||
unseal_start = time.monotonic()
|
||||
operator("unseal")
|
||||
for _ in range(20):
|
||||
response = c.call("GET", "/health/ready", expected=[200, 503])
|
||||
if response["status"] == 200:
|
||||
break
|
||||
time.sleep(0.25)
|
||||
c.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
||||
result = {
|
||||
"lease_seconds": 30,
|
||||
"local_cache_max_seconds": 29,
|
||||
"rows": rows,
|
||||
"first_business_503_seconds": next(
|
||||
(r["seconds"] for r in rows if r["business"] == 503), None
|
||||
),
|
||||
"recovery_seconds": round(time.monotonic() - unseal_start, 3),
|
||||
"passed": rows[-1]["business"] == 503 and response["status"] == 200,
|
||||
}
|
||||
(ROOT / "evidence/sealed-expiry.json").write_text(json.dumps(result, indent=2) + "\n")
|
||||
check(result["passed"], "recovery failed")
|
||||
print(json.dumps({k: v for k, v in result.items() if k != "rows"}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,115 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Explicit operator init/unseal/configuration. Never prints credential values."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import ssl
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PRIVATE = ROOT / "secrets"
|
||||
OPERATOR = ROOT / "operator-private"
|
||||
|
||||
|
||||
def call(path, data=None, *, method=None, token=True):
|
||||
ctx = ssl.create_default_context(cafile=str(PRIVATE / "ca.pem"))
|
||||
ctx.load_cert_chain(PRIVATE / "pki/operator/cert.pem", PRIVATE / "pki/operator/key.pem")
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if token:
|
||||
headers["X-Vault-Token"] = (OPERATOR / "root-token").read_text().strip()
|
||||
req = urllib.request.Request(
|
||||
os.getenv("BAO_OPERATOR_ADDR", "https://localhost:18207") + "/v1/" + path,
|
||||
data=json.dumps(data).encode() if data is not None else None,
|
||||
headers=headers,
|
||||
method=method,
|
||||
)
|
||||
with urllib.request.urlopen(req, context=ctx, timeout=5) as response:
|
||||
raw = response.read()
|
||||
return json.loads(raw) if raw else {}
|
||||
|
||||
|
||||
def unseal():
|
||||
for share in json.loads((OPERATOR / "unseal-shares.json").read_text())[:2]:
|
||||
result = call("sys/unseal", {"key": share}, token=False)
|
||||
if result["sealed"]:
|
||||
raise RuntimeError("OpenBao remained sealed")
|
||||
|
||||
|
||||
def configure():
|
||||
try:
|
||||
call("sys/mounts/secret", {"type": "kv", "options": {"version": "2"}})
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code != 400:
|
||||
raise
|
||||
try:
|
||||
call("sys/auth/cert", {"type": "cert"})
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code != 400:
|
||||
raise
|
||||
seed = json.loads((PRIVATE / "bao_seed.json").read_text())
|
||||
for role, values in seed.items():
|
||||
policy = (
|
||||
'path "secret/data/services/'
|
||||
+ role
|
||||
+ '" { capabilities = ["read"] }\npath "auth/token/lookup-self" { capabilities = ["read"] }\npath "auth/token/renew-self" { capabilities = ["update"] }\n'
|
||||
)
|
||||
call("sys/policies/acl/" + role, {"policy": policy}, method="PUT")
|
||||
call(
|
||||
"auth/cert/certs/" + role,
|
||||
{
|
||||
"display_name": role,
|
||||
"certificate": (PRIVATE / "pki" / role / "cert.pem").read_text(),
|
||||
"allowed_common_names": role,
|
||||
"token_policies": [role],
|
||||
"token_no_default_policy": True,
|
||||
"token_ttl": "30s",
|
||||
"token_max_ttl": "60s",
|
||||
},
|
||||
)
|
||||
call("secret/data/services/" + role, {"data": values})
|
||||
try:
|
||||
call("sys/audit/file", {"type": "file", "options": {"file_path": "/bao/data/audit.jsonl"}})
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code != 400:
|
||||
raise
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("action", choices=["init", "unseal", "configure", "seal"])
|
||||
args = parser.parse_args()
|
||||
os.umask(0o077)
|
||||
if args.action == "init":
|
||||
for _attempt in range(30):
|
||||
try:
|
||||
status = call("sys/init", token=False)
|
||||
break
|
||||
except (OSError, urllib.error.URLError):
|
||||
time.sleep(1)
|
||||
else:
|
||||
raise RuntimeError("OpenBao TLS endpoint unavailable")
|
||||
if not status["initialized"]:
|
||||
result = call(
|
||||
"sys/init", {"secret_shares": 3, "secret_threshold": 2}, method="PUT", token=False
|
||||
)
|
||||
(OPERATOR / "root-token").write_text(result["root_token"])
|
||||
(OPERATOR / "unseal-shares.json").write_text(json.dumps(result["keys_base64"]))
|
||||
(OPERATOR / "root-token").chmod(0o600)
|
||||
(OPERATOR / "unseal-shares.json").chmod(0o600)
|
||||
unseal()
|
||||
configure()
|
||||
elif args.action == "unseal":
|
||||
unseal()
|
||||
elif args.action == "configure":
|
||||
configure()
|
||||
else:
|
||||
call("sys/seal", {}, method="PUT")
|
||||
print("OpenBao operator action complete; credential values were not printed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,340 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Idempotent local PKI/secrets bootstrap. Operator material never enters containers."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PRIVATE = ROOT / "secrets"
|
||||
OPERATOR = ROOT / "operator-private"
|
||||
ROLES = (
|
||||
"api",
|
||||
"worker",
|
||||
"dispatcher",
|
||||
"lb",
|
||||
"postgres",
|
||||
"storage",
|
||||
"broker",
|
||||
"cache",
|
||||
"collector",
|
||||
"prometheus",
|
||||
"loki",
|
||||
"tempo",
|
||||
"grafana",
|
||||
"openbao",
|
||||
"operator",
|
||||
)
|
||||
|
||||
|
||||
def write(path, text, mode=0o644):
|
||||
if not path.exists():
|
||||
path.write_text(text)
|
||||
path.chmod(mode)
|
||||
|
||||
|
||||
def run(*args):
|
||||
subprocess.run(
|
||||
["openssl", *args], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
|
||||
)
|
||||
|
||||
|
||||
def certificate(name, replace=False):
|
||||
dest = PRIVATE / "pki" / name
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
dest.chmod(0o755)
|
||||
if (dest / "cert.pem").exists() and not replace:
|
||||
valid = subprocess.run(
|
||||
["openssl", "x509", "-in", str(dest / "cert.pem"), "-checkend", "3600", "-noout"],
|
||||
capture_output=True,
|
||||
)
|
||||
if valid.returncode == 0:
|
||||
return
|
||||
key, csr = dest / "key.pem", OPERATOR / (name + ".csr")
|
||||
run("genpkey", "-algorithm", "EC", "-pkeyopt", "ec_paramgen_curve:P-256", "-out", str(key))
|
||||
run("req", "-new", "-key", str(key), "-subj", f"/CN={name}", "-out", str(csr))
|
||||
ext = OPERATOR / (name + ".ext")
|
||||
names = f"DNS:{name}"
|
||||
if name in {
|
||||
"lb",
|
||||
"api",
|
||||
"postgres",
|
||||
"broker",
|
||||
"cache",
|
||||
"collector",
|
||||
"prometheus",
|
||||
"loki",
|
||||
"tempo",
|
||||
"openbao",
|
||||
"grafana",
|
||||
"storage",
|
||||
}:
|
||||
names += ",DNS:localhost,IP:127.0.0.1"
|
||||
ext.write_text(
|
||||
f"subjectAltName={names}\nbasicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth,clientAuth\n"
|
||||
)
|
||||
run(
|
||||
"x509",
|
||||
"-req",
|
||||
"-in",
|
||||
str(csr),
|
||||
"-CA",
|
||||
str(OPERATOR / "ca.pem"),
|
||||
"-CAkey",
|
||||
str(OPERATOR / "ca-key.pem"),
|
||||
"-set_serial",
|
||||
str(secrets.randbits(127) or 1),
|
||||
"-days",
|
||||
"2",
|
||||
"-extfile",
|
||||
str(ext),
|
||||
"-out",
|
||||
str(dest / "cert.pem"),
|
||||
)
|
||||
key.chmod(0o644)
|
||||
(dest / "combined.pem").write_text((dest / "cert.pem").read_text() + key.read_text())
|
||||
(dest / "combined.pem").chmod(0o644)
|
||||
|
||||
|
||||
def main():
|
||||
os.umask(0o077)
|
||||
for folder in (PRIVATE, OPERATOR):
|
||||
folder.mkdir(exist_ok=True)
|
||||
folder.chmod(0o700)
|
||||
if not (OPERATOR / "ca-key.pem").exists():
|
||||
run(
|
||||
"genpkey",
|
||||
"-algorithm",
|
||||
"EC",
|
||||
"-pkeyopt",
|
||||
"ec_paramgen_curve:P-256",
|
||||
"-out",
|
||||
str(OPERATOR / "ca-key.pem"),
|
||||
)
|
||||
run(
|
||||
"req",
|
||||
"-x509",
|
||||
"-new",
|
||||
"-key",
|
||||
str(OPERATOR / "ca-key.pem"),
|
||||
"-subj",
|
||||
"/CN=MTUSI Lab7 local CA",
|
||||
"-days",
|
||||
"365",
|
||||
"-addext",
|
||||
"basicConstraints=critical,CA:TRUE",
|
||||
"-addext",
|
||||
"keyUsage=critical,keyCertSign,cRLSign",
|
||||
"-out",
|
||||
str(OPERATOR / "ca.pem"),
|
||||
)
|
||||
(PRIVATE / "ca.pem").write_text((OPERATOR / "ca.pem").read_text())
|
||||
(PRIVATE / "ca.pem").chmod(0o644)
|
||||
(PRIVATE / "pki").mkdir(exist_ok=True)
|
||||
(PRIVATE / "pki").chmod(0o755)
|
||||
for name in ROLES:
|
||||
certificate(name)
|
||||
write(PRIVATE / "revoked.json", "[]")
|
||||
for name in (
|
||||
"postgres_password",
|
||||
"api_db_password",
|
||||
"worker_db_password",
|
||||
"dispatcher_db_password",
|
||||
"cache_password",
|
||||
"cache_worker_password",
|
||||
"cache_dispatcher_password",
|
||||
"broker_worker_password",
|
||||
"broker_dispatcher_password",
|
||||
"broker_admin_password",
|
||||
"grafana_password",
|
||||
):
|
||||
write(PRIVATE / name, secrets.token_urlsafe(32))
|
||||
for role in ("api", "worker", "admin", "study_wide"):
|
||||
write(PRIVATE / (role + "_s3_access"), role + "-" + secrets.token_hex(12))
|
||||
write(PRIVATE / (role + "_s3_secret"), secrets.token_urlsafe(32))
|
||||
|
||||
def value(name):
|
||||
return (PRIVATE / name).read_text().strip()
|
||||
|
||||
identities = []
|
||||
for role in ("api", "worker", "admin", "study_wide"):
|
||||
identities.append(
|
||||
{
|
||||
"name": role,
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": value(role + "_s3_access"),
|
||||
"secretKey": value(role + "_s3_secret"),
|
||||
}
|
||||
],
|
||||
"actions": ["Admin", "Read", "Write", "List"]
|
||||
if role == "admin"
|
||||
else (["Read:security-study", "Write:security-study", "List:security-study"] if role == "study_wide" else ["Read:messenger", "Write:messenger", "List:messenger"]),
|
||||
}
|
||||
)
|
||||
(PRIVATE / "s3_config").write_text(json.dumps({"identities": identities}))
|
||||
(PRIVATE / "s3_config").chmod(0o644)
|
||||
write(
|
||||
PRIVATE / "cache_config",
|
||||
'bind 127.0.0.1\nport 6379\nprotected-mode yes\nsave ""\nmaxmemory 64mb\nmaxmemory-policy allkeys-lru\nuser default off\nuser api on >'
|
||||
+ value("cache_password")
|
||||
+ " ~user:v1:* +get +setex +ping +hello +client|setinfo\nuser worker on >"
|
||||
+ value("cache_worker_password")
|
||||
+ " +ping +hello +client|setinfo\nuser dispatcher on >"
|
||||
+ value("cache_dispatcher_password")
|
||||
+ " +ping +hello +client|setinfo\n",
|
||||
)
|
||||
write(
|
||||
PRIVATE / "broker_config",
|
||||
'bind 127.0.0.1\nport 6379\nprotected-mode yes\nappendonly yes\nappendfsync always\nsave ""\ndir /data\nmaxmemory 128mb\nmaxmemory-policy noeviction\nuser default off\nuser operator on >'
|
||||
+ value("broker_admin_password")
|
||||
+ " ~* &* +@all\nuser dispatcher on >"
|
||||
+ value("broker_dispatcher_password")
|
||||
+ " ~attachment-jobs:v1 +xadd +ping +hello +client|setinfo\nuser worker on >"
|
||||
+ value("broker_worker_password")
|
||||
+ " ~attachment-jobs:v1 +xgroup|create +xreadgroup +xautoclaim +xack +xdel +ping +hello +client|setinfo\n",
|
||||
)
|
||||
pg = "postgresql://{role}:{password}@postgres:5432/messenger?sslmode=verify-full&sslrootcert=/run/tls/ca.pem"
|
||||
values = {}
|
||||
for role in ("api", "worker", "dispatcher"):
|
||||
values[role] = {
|
||||
"DATABASE_URL": pg.format(role=role, password=value(role + "_db_password")),
|
||||
"CACHE_URL": "rediss://"
|
||||
+ role
|
||||
+ ":"
|
||||
+ value("cache_password" if role == "api" else "cache_" + role + "_password")
|
||||
+ "@cache:6380/0?ssl_cert_reqs=required&ssl_ca_certs=/run/tls/ca.pem&ssl_certfile=/run/tls/identity/cert.pem&ssl_keyfile=/run/tls/identity/key.pem",
|
||||
}
|
||||
if role in {"worker", "dispatcher"}:
|
||||
values[role]["BROKER_URL"] = (
|
||||
"rediss://"
|
||||
+ role
|
||||
+ ":"
|
||||
+ value("broker_" + role + "_password")
|
||||
+ "@broker:6380/0?ssl_cert_reqs=required&ssl_ca_certs=/run/tls/ca.pem&ssl_certfile=/run/tls/identity/cert.pem&ssl_keyfile=/run/tls/identity/key.pem"
|
||||
)
|
||||
if role in {"api", "worker"}:
|
||||
values[role].update(
|
||||
S3_ACCESS_KEY=value(role + "_s3_access"), S3_SECRET_KEY=value(role + "_s3_secret")
|
||||
)
|
||||
write(PRIVATE / "bao_seed.json", json.dumps(values))
|
||||
write(
|
||||
PRIVATE / "admin_database_url",
|
||||
pg.format(role="messenger", password=value("postgres_password")),
|
||||
)
|
||||
write(PRIVATE / "admin_s3_access_key", value("admin_s3_access"))
|
||||
write(PRIVATE / "admin_s3_secret_key", value("admin_s3_secret"))
|
||||
write(
|
||||
PRIVATE / "db_roles.sql",
|
||||
"\n".join(
|
||||
f"CREATE ROLE {role} LOGIN PASSWORD '{value(role + '_db_password')}';"
|
||||
for role in ("api", "worker", "dispatcher")
|
||||
),
|
||||
)
|
||||
gateways = {
|
||||
"api": {"port": 9443, "upstream": 8080, "allowed": ["lb"]},
|
||||
"broker": {
|
||||
"port": 6380,
|
||||
"upstream": 6379,
|
||||
"allowed": ["worker", "dispatcher", "operator"],
|
||||
},
|
||||
"cache": {
|
||||
"port": 6380,
|
||||
"upstream": 6379,
|
||||
"allowed": ["api", "worker", "dispatcher", "operator"],
|
||||
},
|
||||
"storage": {
|
||||
"port": 8443,
|
||||
"upstream": 8333,
|
||||
"allowed": ["api", "worker", "operator"],
|
||||
"http": True,
|
||||
"s3": True,
|
||||
},
|
||||
"collector": [
|
||||
{
|
||||
"port": 4318,
|
||||
"upstream": 14318,
|
||||
"allowed": ["api", "worker", "dispatcher", "operator"],
|
||||
},
|
||||
{"port": 9464, "upstream": 19464, "allowed": ["prometheus", "operator"]},
|
||||
{"port": 8888, "upstream": 18888, "allowed": ["prometheus", "operator"]},
|
||||
],
|
||||
"prometheus": {
|
||||
"http": True,
|
||||
"backend": "prometheus",
|
||||
"port": 9443,
|
||||
"upstream": 9090,
|
||||
"allowed": ["grafana", "operator"],
|
||||
},
|
||||
"loki": {
|
||||
"http": True,
|
||||
"backend": "loki",
|
||||
"port": 9443,
|
||||
"upstream": 3100,
|
||||
"allowed": ["collector", "grafana", "operator"],
|
||||
},
|
||||
"tempo": [
|
||||
{
|
||||
"port": 9443,
|
||||
"upstream": 3200,
|
||||
"allowed": ["grafana", "operator"],
|
||||
"http": True,
|
||||
"backend": "tempo",
|
||||
},
|
||||
{"port": 44318, "upstream": 4318, "allowed": ["collector", "operator"]},
|
||||
],
|
||||
}
|
||||
for role, configs in gateways.items():
|
||||
for item in configs if isinstance(configs, list) else [configs]:
|
||||
item.update(
|
||||
cert="/run/tls/identity/cert.pem",
|
||||
key="/run/tls/identity/key.pem",
|
||||
ca="/run/tls/ca.pem",
|
||||
)
|
||||
(PRIVATE / (role + "_gateway.json")).write_text(json.dumps(configs))
|
||||
(PRIVATE / (role + "_gateway.json")).chmod(0o644)
|
||||
provisioning = PRIVATE / "grafana-provisioning" / "datasources"
|
||||
provisioning.mkdir(parents=True, exist_ok=True)
|
||||
provisioning.parent.chmod(0o755)
|
||||
provisioning.chmod(0o755)
|
||||
datasources = []
|
||||
for name in ("prometheus", "loki", "tempo"):
|
||||
item = {
|
||||
"name": name.title(),
|
||||
"uid": name,
|
||||
"type": name,
|
||||
"access": "proxy",
|
||||
"url": f"https://{name}:9443",
|
||||
"editable": False,
|
||||
"jsonData": {"tlsAuth": True, "tlsAuthWithCACert": True, "serverName": name},
|
||||
"secureJsonData": {
|
||||
"tlsCACert": (PRIVATE / "ca.pem").read_text(),
|
||||
"tlsClientCert": (PRIVATE / "pki/grafana/cert.pem").read_text(),
|
||||
"tlsClientKey": (PRIVATE / "pki/grafana/key.pem").read_text(),
|
||||
},
|
||||
}
|
||||
if name == "prometheus":
|
||||
item["isDefault"] = True
|
||||
if name == "loki":
|
||||
item["jsonData"]["derivedFields"] = [
|
||||
{
|
||||
"datasourceUid": "tempo",
|
||||
"matcherRegex": '"trace_id": "([a-f0-9]{32})"',
|
||||
"name": "TraceID",
|
||||
"url": "$${__value.raw}",
|
||||
}
|
||||
]
|
||||
datasources.append(item)
|
||||
(provisioning / "datasources.yaml").write_text(
|
||||
json.dumps({"apiVersion": 1, "datasources": datasources})
|
||||
)
|
||||
(provisioning / "datasources.yaml").chmod(0o644)
|
||||
if not (ROOT / ".env").exists():
|
||||
(ROOT / ".env").write_text("PUBLIC_PORT=8447\nSESSION_TTL_SECONDS=3600\n")
|
||||
print("Local PKI and scoped credential material ready; operator CA stays outside containers.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+9
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
# Namespace-sharing gateways are stopped before their backing services restart.
|
||||
# Unseal is an explicit operator action; shares never enter application containers.
|
||||
docker compose stop
|
||||
docker compose up -d postgres openbao cache cache-tls broker broker-tls storage s3 storage-tls
|
||||
python3 scripts/bao_operator.py unseal
|
||||
docker compose up -d --wait
|
||||
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Operator-driven worker revoke/cert renewal and DB credential rotation, no rebuild."""
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import secrets
|
||||
import ssl
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def main():
|
||||
spec = importlib.util.spec_from_file_location("bootstrap", ROOT / "scripts/bootstrap.py")
|
||||
bootstrap = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(bootstrap)
|
||||
old = ROOT / "secrets/pki/worker"
|
||||
cert = ssl._ssl._test_decode_cert(str(old / "cert.pem"))
|
||||
snapshot = ROOT / "operator-private/revoked-worker"
|
||||
snapshot.mkdir(exist_ok=True, mode=0o700)
|
||||
for name in ("cert.pem", "key.pem"):
|
||||
(snapshot / name).write_bytes((old / name).read_bytes())
|
||||
(snapshot / name).chmod(0o600)
|
||||
seed_before = json.loads((ROOT / "secrets/bao_seed.json").read_text())
|
||||
(snapshot / "material.json").write_text(json.dumps(seed_before["worker"]))
|
||||
(snapshot / "material.json").chmod(0o644)
|
||||
# The containing operator-private directory stays 0700 on the host. This
|
||||
# disposable old-material mount is used only by the explicit negative probe.
|
||||
for name in ("cert.pem", "key.pem"):
|
||||
(snapshot / name).chmod(0o644)
|
||||
snapshot.chmod(0o755)
|
||||
revoked = ROOT / "secrets/revoked.json"
|
||||
serials = json.loads(revoked.read_text())
|
||||
serials.append(cert["serialNumber"].lstrip("0").lower())
|
||||
revoked.write_text(json.dumps(sorted(set(serials))))
|
||||
subprocess.run(
|
||||
["docker", "compose", "stop", "worker"], cwd=ROOT, check=True, capture_output=True
|
||||
)
|
||||
bootstrap.certificate("worker", replace=True)
|
||||
password = secrets.token_urlsafe(32)
|
||||
# psql receives SQL through stdin, never a process argument/output containing
|
||||
# credentials. The operator/admin role is confined to this explicit command.
|
||||
sql = f"ALTER ROLE worker PASSWORD '{password}'; SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE usename='worker' AND pid<>pg_backend_pid();"
|
||||
subprocess.run(
|
||||
[
|
||||
"docker",
|
||||
"compose",
|
||||
"exec",
|
||||
"-T",
|
||||
"postgres",
|
||||
"psql",
|
||||
"-U",
|
||||
"messenger",
|
||||
"-d",
|
||||
"messenger",
|
||||
"-v",
|
||||
"ON_ERROR_STOP=1",
|
||||
],
|
||||
cwd=ROOT,
|
||||
input=sql,
|
||||
text=True,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
)
|
||||
path = ROOT / "secrets/worker_db_password"
|
||||
path.write_text(password)
|
||||
seed_path = ROOT / "secrets/bao_seed.json"
|
||||
seed = json.loads(seed_path.read_text())
|
||||
seed["worker"]["DATABASE_URL"] = (
|
||||
"postgresql://worker:"
|
||||
+ password
|
||||
+ "@postgres:5432/messenger?sslmode=verify-full&sslrootcert=/run/tls/ca.pem"
|
||||
)
|
||||
seed_path.write_text(json.dumps(seed))
|
||||
subprocess.run(
|
||||
["python3", "scripts/bao_operator.py", "configure"],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
)
|
||||
subprocess.run(
|
||||
["docker", "compose", "start", "worker"], cwd=ROOT, check=True, capture_output=True
|
||||
)
|
||||
print(
|
||||
"Worker identity revoked and replaced; database credential rotated; worker resumed without image rebuild."
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,217 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Explicit operator probes; test identities never become application mounts."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def run(args, *, output=None):
|
||||
result = subprocess.run(args, cwd=ROOT, check=True, capture_output=True, text=True)
|
||||
if output:
|
||||
(ROOT / "evidence" / output).write_text(result.stdout)
|
||||
else:
|
||||
print(result.stdout, end="")
|
||||
return result.stdout
|
||||
|
||||
|
||||
def foreign_certificate():
|
||||
path = ROOT / "secrets/pki/foreign"
|
||||
path.mkdir(exist_ok=True, mode=0o755)
|
||||
run(
|
||||
[
|
||||
"openssl",
|
||||
"req",
|
||||
"-x509",
|
||||
"-newkey",
|
||||
"ec",
|
||||
"-pkeyopt",
|
||||
"ec_paramgen_curve:P-256",
|
||||
"-nodes",
|
||||
"-keyout",
|
||||
str(path / "key.pem"),
|
||||
"-out",
|
||||
str(path / "cert.pem"),
|
||||
"-days",
|
||||
"1",
|
||||
"-subj",
|
||||
"/CN=foreign",
|
||||
"-addext",
|
||||
"subjectAltName=DNS:lb,DNS:worker,DNS:api",
|
||||
]
|
||||
)
|
||||
for item in path.iterdir():
|
||||
item.chmod(0o644)
|
||||
|
||||
|
||||
def audit_container(module, *, material=False):
|
||||
command = [
|
||||
"docker",
|
||||
"compose",
|
||||
"run",
|
||||
"--rm",
|
||||
"--no-deps",
|
||||
"-e",
|
||||
"BAO_ROLE=",
|
||||
"-e",
|
||||
"OTEL_ENABLED=0",
|
||||
"-v",
|
||||
str(ROOT / "secrets/pki") + ":/run/audit-identities:ro",
|
||||
]
|
||||
if material:
|
||||
# Only selected synthetic/admin S3 credential files, never Bao root/shares.
|
||||
path = ROOT / "operator-private/s3-study-material"
|
||||
path.mkdir(exist_ok=True, mode=0o755)
|
||||
for name in (
|
||||
"admin_s3_access",
|
||||
"admin_s3_secret",
|
||||
"study_wide_s3_access",
|
||||
"study_wide_s3_secret",
|
||||
"worker_s3_access",
|
||||
"worker_s3_secret",
|
||||
):
|
||||
(path / name).write_bytes((ROOT / "secrets" / name).read_bytes())
|
||||
(path / name).chmod(0o644)
|
||||
command += ["-v", str(path) + ":/run/audit-material:ro"]
|
||||
command += ["api", "python", "-m", module]
|
||||
return command
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("probe", choices=["transport", "policies", "baseline", "revoked", "kdf"])
|
||||
args = parser.parse_args()
|
||||
os.umask(0o077)
|
||||
if args.probe == "transport":
|
||||
foreign_certificate()
|
||||
run(audit_container("messenger.transport_probe"), output="transport-matrix.json")
|
||||
elif args.probe == "policies":
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"compose",
|
||||
"run",
|
||||
"--rm",
|
||||
"--no-deps",
|
||||
"worker",
|
||||
"python",
|
||||
"-m",
|
||||
"messenger.security_probe",
|
||||
],
|
||||
output="worker-policy-final.json",
|
||||
)
|
||||
elif args.probe == "baseline":
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"compose",
|
||||
"run",
|
||||
"--rm",
|
||||
"--no-deps",
|
||||
"migrate",
|
||||
"python",
|
||||
"-m",
|
||||
"messenger.baseline_db_probe",
|
||||
],
|
||||
output="baseline-db.json",
|
||||
)
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"compose",
|
||||
"exec",
|
||||
"-T",
|
||||
"broker",
|
||||
"valkey-server",
|
||||
"--port",
|
||||
"6378",
|
||||
"--bind",
|
||||
"127.0.0.1",
|
||||
"--save",
|
||||
"",
|
||||
"--appendonly",
|
||||
"no",
|
||||
"--daemonize",
|
||||
"yes",
|
||||
"--pidfile",
|
||||
"/tmp/security-study.pid",
|
||||
"--logfile",
|
||||
"/tmp/security-study.log",
|
||||
]
|
||||
)
|
||||
broker = run(["docker", "compose", "ps", "-q", "broker"]).strip()
|
||||
config = json.loads(
|
||||
subprocess.check_output(
|
||||
["docker", "compose", "config", "--format", "json"], cwd=ROOT, text=True
|
||||
)
|
||||
)
|
||||
app_image = config["services"]["api"]["image"]
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"run",
|
||||
"--rm",
|
||||
"--network",
|
||||
"container:" + broker,
|
||||
"--read-only",
|
||||
"--cap-drop",
|
||||
"ALL",
|
||||
"--security-opt",
|
||||
"no-new-privileges",
|
||||
"--memory",
|
||||
"128m",
|
||||
"--cpus",
|
||||
"0.5",
|
||||
app_image,
|
||||
"python",
|
||||
"-m",
|
||||
"messenger.baseline_queue_probe",
|
||||
],
|
||||
output="baseline-queue.json",
|
||||
)
|
||||
run(
|
||||
audit_container("messenger.baseline_s3_probe", material=True), output="baseline-s3.json"
|
||||
)
|
||||
elif args.probe == "revoked":
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"compose",
|
||||
"run",
|
||||
"--rm",
|
||||
"--no-deps",
|
||||
"-v",
|
||||
str(ROOT / "operator-private/revoked-worker") + ":/run/old-worker:ro",
|
||||
"worker",
|
||||
"python",
|
||||
"-m",
|
||||
"messenger.revoked_probe",
|
||||
],
|
||||
output="revoked-identity.json",
|
||||
)
|
||||
else:
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"compose",
|
||||
"run",
|
||||
"--rm",
|
||||
"--no-deps",
|
||||
"-e",
|
||||
"OTEL_ENABLED=0",
|
||||
"worker",
|
||||
"python",
|
||||
"-m",
|
||||
"messenger.kdf_probe",
|
||||
],
|
||||
output="argon2-budget.json",
|
||||
)
|
||||
print(json.dumps({"probe": args.probe, "passed": True}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
python3 scripts/stop_one.py
|
||||
@@ -0,0 +1,22 @@
|
||||
"""Map a process alias to its real container, then leave exactly it stopped."""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
wanted = os.environ["TARGET_INSTANCE_ID"]
|
||||
ids = subprocess.check_output(["docker", "compose", "ps", "-q", "api"], text=True).split()
|
||||
for cid in ids:
|
||||
result = subprocess.run(
|
||||
["docker", "exec", cid, "python", "-c",
|
||||
"import urllib.request; r=urllib.request.urlopen('http://127.0.0.1:8080/health/live'); print(r.headers['X-Instance-Id'])"],
|
||||
text=True, capture_output=True, check=False,
|
||||
)
|
||||
if result.returncode == 0 and result.stdout.strip() == wanted:
|
||||
subprocess.run(["docker", "stop", "--time", "8", cid], check=True, stdout=subprocess.DEVNULL)
|
||||
receipt = {"stopped_container": cid, "instance_id": wanted}
|
||||
Path("evidence/last-stopped.json").write_text(json.dumps(receipt))
|
||||
print(json.dumps(receipt))
|
||||
break
|
||||
else:
|
||||
raise SystemExit("Target instance not found among this project's API containers")
|
||||
@@ -0,0 +1 @@
|
||||
"""Учебный REST-мессенджер."""
|
||||
@@ -0,0 +1,12 @@
|
||||
import os
|
||||
|
||||
import uvicorn
|
||||
|
||||
uvicorn.run(
|
||||
"messenger.app:app",
|
||||
host=os.getenv("APP_HOST", "0.0.0.0"),
|
||||
port=int(os.getenv("APP_PORT", "8080")),
|
||||
workers=1,
|
||||
access_log=False,
|
||||
timeout_graceful_shutdown=5,
|
||||
)
|
||||
@@ -0,0 +1,384 @@
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from contextlib import asynccontextmanager
|
||||
from threading import BoundedSemaphore
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import Depends, FastAPI, Request, Response
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
from opentelemetry import propagate
|
||||
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
from starlette.exceptions import HTTPException
|
||||
|
||||
from . import telemetry
|
||||
from .attachments import AttachmentService
|
||||
from .domain import Failure, boundary, identifier, page
|
||||
from .models import CreateChat, CreateMessage, CreateUser
|
||||
from .postgres import PostgresStore
|
||||
from .secrets_provider import refresh
|
||||
from .storage import ObjectStorage
|
||||
|
||||
logger = logging.getLogger("messenger")
|
||||
logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO"), format="%(message)s")
|
||||
INSTANCE_ID = telemetry.INSTANCE_ID
|
||||
|
||||
CODES = {
|
||||
400: "invalid_request",
|
||||
401: "unauthorized",
|
||||
404: "not_found",
|
||||
409: "conflict",
|
||||
413: "payload_too_large",
|
||||
415: "unsupported_media_type",
|
||||
429: "rate_limited",
|
||||
503: "unavailable",
|
||||
500: "internal_error",
|
||||
}
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app):
|
||||
app.state.telemetry = telemetry.setup("messenger-api")
|
||||
app.state.store = PostgresStore()
|
||||
app.state.telemetry.start_background(app.state.store)
|
||||
app.state.storage = ObjectStorage()
|
||||
app.state.attachments = AttachmentService(app.state.store, app.state.storage)
|
||||
app.state.upload_slots = BoundedSemaphore(1)
|
||||
logger.info(json.dumps({"event": "started", "storage": "postgres"}))
|
||||
yield
|
||||
logger.info(json.dumps({"event": "stopped"}))
|
||||
app.state.telemetry.close()
|
||||
app.state.storage.close()
|
||||
app.state.store.close()
|
||||
|
||||
|
||||
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
|
||||
|
||||
|
||||
def error(request, status, message, headers=None):
|
||||
return JSONResponse(
|
||||
{
|
||||
"error": {
|
||||
"code": CODES.get(status, "invalid_request"),
|
||||
"message": message,
|
||||
"request_id": request.state.request_id,
|
||||
}
|
||||
},
|
||||
status_code=status,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
|
||||
async def invalid_json_encoding(request: Request):
|
||||
path = request.url.path
|
||||
if request.method != "POST" or not (
|
||||
path in {"/api/v1/users", "/api/v1/chats"}
|
||||
or re.fullmatch(r"/api/v1/chats/[^/]+/messages", path)
|
||||
):
|
||||
return False
|
||||
raw = await request.body()
|
||||
try:
|
||||
raw.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
return True
|
||||
# Literal NUL is invalid JSON; it also prevents json.loads(bytes) from
|
||||
# autodetecting BOM-less UTF-16/32. Escaped JSON \u0000 remains valid.
|
||||
return b"\x00" in raw
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def request_log(request: Request, call_next):
|
||||
request.state.request_id = str(uuid4())
|
||||
start = time.perf_counter()
|
||||
observed = not request.url.path.startswith("/health/")
|
||||
if not observed:
|
||||
try:
|
||||
if request.url.path != "/health/live":
|
||||
await run_in_threadpool(refresh)
|
||||
response = await call_next(request)
|
||||
except Failure as exc:
|
||||
response = error(request, exc.status, exc.message, exc.headers)
|
||||
response.headers["X-Request-Id"] = request.state.request_id
|
||||
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||
return response
|
||||
method = (
|
||||
request.method
|
||||
if request.method in {"GET", "POST", "DELETE", "PUT", "PATCH", "HEAD", "OPTIONS"}
|
||||
else "OTHER"
|
||||
)
|
||||
context = propagate.extract(dict(request.headers))
|
||||
with telemetry.operation("HTTP " + method, context=context, kind=SpanKind.SERVER) as span:
|
||||
try:
|
||||
await run_in_threadpool(refresh)
|
||||
if (
|
||||
request.method == "POST"
|
||||
and request.url.path == "/api/v1/auth/sessions"
|
||||
and await request.body()
|
||||
):
|
||||
response = error(request, 400, "Вход не принимает тело запроса")
|
||||
elif await invalid_json_encoding(request):
|
||||
response = error(request, 400, "JSON должен быть закодирован в UTF-8")
|
||||
else:
|
||||
response = await call_next(request)
|
||||
except Failure as exc:
|
||||
response = error(request, exc.status, exc.message, exc.headers)
|
||||
except Exception:
|
||||
logger.error(
|
||||
json.dumps({"event": "unexpected_error", "request_id": request.state.request_id})
|
||||
)
|
||||
response = error(request, 500, "Внутренняя ошибка")
|
||||
response.headers["X-Request-Id"] = request.state.request_id
|
||||
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||
response.headers["X-Trace-Id"] = format(span.get_span_context().trace_id, "032x")
|
||||
response.headers["X-Trace-Sampled"] = "true" if span.is_recording() else "false"
|
||||
route = getattr(request.scope.get("route"), "path", "unknown")
|
||||
elapsed = time.perf_counter() - start
|
||||
attrs = {
|
||||
"method": method,
|
||||
"route": route,
|
||||
"status_class": str(response.status_code // 100) + "xx",
|
||||
}
|
||||
span.update_name(method + " " + route)
|
||||
span.set_attributes(
|
||||
{
|
||||
"http.request.method": method,
|
||||
"http.route": route,
|
||||
"http.response.status_code": response.status_code,
|
||||
}
|
||||
)
|
||||
if response.status_code >= 500:
|
||||
span.set_status(Status(StatusCode.ERROR, "server_error"))
|
||||
t = telemetry.current()
|
||||
if observed and t:
|
||||
t.requests.add(1, attrs)
|
||||
t.http_duration.record(elapsed, attrs)
|
||||
t.sampling.add(1, {"decision": "sampled" if span.is_recording() else "not_sampled"})
|
||||
logger.info(
|
||||
json.dumps(
|
||||
{
|
||||
"event": "request",
|
||||
"request_id": request.state.request_id,
|
||||
**attrs,
|
||||
"status": response.status_code,
|
||||
"duration_ms": round(elapsed * 1000, 3),
|
||||
}
|
||||
)
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
@app.exception_handler(Failure)
|
||||
async def domain_error(request, exc):
|
||||
return error(request, exc.status, exc.message, exc.headers)
|
||||
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_error(request, exc):
|
||||
return error(request, 400, "Неверный JSON или поля запроса")
|
||||
|
||||
|
||||
@app.exception_handler(HTTPException)
|
||||
async def http_error(request, exc):
|
||||
return error(request, exc.status_code, "Неверный запрос", exc.headers)
|
||||
|
||||
|
||||
def bearer_failure():
|
||||
return Failure(
|
||||
401, "Требуется действующая сессия", {"WWW-Authenticate": 'Bearer realm="messenger"'}
|
||||
)
|
||||
|
||||
|
||||
def bearer(request):
|
||||
parts = request.headers.get("Authorization", "").split()
|
||||
if len(parts) != 2 or parts[0].lower() != "bearer":
|
||||
raise bearer_failure()
|
||||
return parts[1]
|
||||
|
||||
|
||||
def actor(request: Request):
|
||||
user = request.app.state.store.authenticate(bearer(request))
|
||||
if user is None:
|
||||
raise bearer_failure()
|
||||
return user
|
||||
|
||||
|
||||
@app.post("/api/v1/auth/sessions", status_code=201)
|
||||
def login(request: Request):
|
||||
headers = {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}
|
||||
parts = request.headers.get("Authorization", "").split()
|
||||
try:
|
||||
if len(parts) != 2 or parts[0].lower() != "basic" or len(parts[1]) > 2048:
|
||||
raise ValueError
|
||||
decoded = base64.b64decode(parts[1], validate=True).decode("utf-8")
|
||||
username, password = decoded.split(":", 1)
|
||||
if not re.fullmatch(r"[a-z0-9_]{3,32}", username) or not 12 <= len(password) <= 128:
|
||||
raise ValueError
|
||||
except (ValueError, UnicodeError, binascii.Error):
|
||||
raise Failure(401, "Неверные учётные данные", headers) from None
|
||||
session = request.app.state.store.login(username, password)
|
||||
if session is None:
|
||||
raise Failure(401, "Неверные учётные данные", headers)
|
||||
return session
|
||||
|
||||
|
||||
@app.delete("/api/v1/auth/sessions/current", status_code=204)
|
||||
def logout(request: Request, user=Depends(actor)):
|
||||
if not request.app.state.store.logout(bearer(request)):
|
||||
raise bearer_failure()
|
||||
return Response(status_code=204)
|
||||
|
||||
|
||||
def pagination(request, scope):
|
||||
allowed = {"limit", "cursor"}
|
||||
if any(k not in allowed for k in request.query_params):
|
||||
raise Failure(400, "Неизвестный параметр запроса")
|
||||
if any(len(request.query_params.getlist(k)) > 1 for k in allowed):
|
||||
raise Failure(400, "Повторный параметр запроса")
|
||||
value = request.query_params.get("limit", "50")
|
||||
if len(value) > 3 or not re.fullmatch(r"[0-9]+", value) or not 1 <= int(value) <= 100:
|
||||
raise Failure(400, "limit должен быть целым от 1 до 100")
|
||||
return int(value), boundary(scope, request.query_params.get("cursor"))
|
||||
|
||||
|
||||
@app.get("/health/live")
|
||||
def live():
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get("/health/ready")
|
||||
def ready(request: Request):
|
||||
if not request.app.state.store.ready() or not request.app.state.storage.ready():
|
||||
raise Failure(503, "Хранилище недоступно")
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.post("/api/v1/users", status_code=201)
|
||||
def register(data: CreateUser, request: Request):
|
||||
return request.app.state.store.register(data.model_dump(mode="json"))
|
||||
|
||||
|
||||
@app.get("/api/v1/users/me")
|
||||
def me(user=Depends(actor)):
|
||||
return user
|
||||
|
||||
|
||||
@app.post("/api/v1/chats", status_code=201)
|
||||
def create_chat(data: CreateChat, request: Request, user=Depends(actor)):
|
||||
return request.app.state.store.create_chat(user["id"], data.model_dump(mode="json"))
|
||||
|
||||
|
||||
@app.get("/api/v1/chats")
|
||||
def chats(request: Request, user=Depends(actor)):
|
||||
scope = "chats:" + user["id"]
|
||||
limit, after = pagination(request, scope)
|
||||
return page(request.app.state.store.list_chats(user["id"], after, limit), limit, scope)
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}")
|
||||
def chat(chat_id: str, request: Request, user=Depends(actor)):
|
||||
return request.app.state.store.chat(user["id"], identifier(chat_id))
|
||||
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
|
||||
def send(chat_id: str, data: CreateMessage, request: Request, user=Depends(actor)):
|
||||
key = request.headers.get("Idempotency-Key", "")
|
||||
if not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key):
|
||||
raise Failure(400, "Требуется корректный Idempotency-Key")
|
||||
return request.app.state.store.send(
|
||||
user["id"], identifier(chat_id), data.text, key, [str(a) for a in data.attachment_ids]
|
||||
)
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}/messages")
|
||||
def history(chat_id: str, request: Request, user=Depends(actor)):
|
||||
cid = identifier(chat_id)
|
||||
scope = "messages:" + cid
|
||||
limit, after = pagination(request, scope)
|
||||
return page(request.app.state.store.history(user["id"], cid, after, limit), limit, scope)
|
||||
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202)
|
||||
async def upload_attachment(chat_id: str, request: Request, user=Depends(actor)):
|
||||
import asyncio
|
||||
import tempfile
|
||||
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
|
||||
from .attachments import MAX_BYTES, filename_valid
|
||||
|
||||
if (
|
||||
set(request.query_params) != {"filename"}
|
||||
or len(request.query_params.getlist("filename")) != 1
|
||||
):
|
||||
raise Failure(400, "Требуется одно имя файла")
|
||||
filename = filename_valid(request.query_params["filename"])
|
||||
cid = identifier(chat_id)
|
||||
await run_in_threadpool(request.app.state.store.chat, user["id"], cid)
|
||||
content_type = request.headers.get("content-type", "").lower()
|
||||
if content_type not in {"image/png", "image/jpeg"}:
|
||||
raise Failure(415, "Ожидается PNG или JPEG")
|
||||
if not request.app.state.upload_slots.acquire(blocking=False):
|
||||
raise Failure(429, "Все места загрузки заняты", {"Retry-After": "1"})
|
||||
try:
|
||||
with tempfile.SpooledTemporaryFile(max_size=256 * 1024) as spool:
|
||||
size = 0
|
||||
try:
|
||||
async with asyncio.timeout(15):
|
||||
async for chunk in request.stream():
|
||||
size += len(chunk)
|
||||
if size > MAX_BYTES:
|
||||
raise Failure(413, "Файл больше 10 MiB")
|
||||
spool.write(chunk)
|
||||
except TimeoutError:
|
||||
raise Failure(400, "Истекло время загрузки") from None
|
||||
spool.seek(0)
|
||||
return await run_in_threadpool(
|
||||
request.app.state.attachments.upload,
|
||||
user["id"],
|
||||
cid,
|
||||
filename,
|
||||
content_type,
|
||||
spool,
|
||||
size,
|
||||
)
|
||||
finally:
|
||||
request.app.state.upload_slots.release()
|
||||
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}")
|
||||
def attachment_metadata(attachment_id: str, request: Request, user=Depends(actor)):
|
||||
return request.app.state.attachments.metadata(user["id"], identifier(attachment_id))
|
||||
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}/content")
|
||||
def attachment_content(attachment_id: str, request: Request, user=Depends(actor)):
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
if (
|
||||
any(k != "variant" for k in request.query_params)
|
||||
or len(request.query_params.getlist("variant")) > 1
|
||||
):
|
||||
raise Failure(400, "Неверный вариант")
|
||||
variant = request.query_params.get("variant", "original")
|
||||
if variant not in {"original", "thumbnail", "crop", "optimized"}:
|
||||
raise Failure(400, "Неверный вариант")
|
||||
obj = request.app.state.attachments.content(user["id"], identifier(attachment_id), variant)
|
||||
|
||||
def chunks():
|
||||
try:
|
||||
while data := obj["Body"].read(64 * 1024):
|
||||
yield data
|
||||
finally:
|
||||
obj["Body"].close()
|
||||
|
||||
return StreamingResponse(
|
||||
chunks(),
|
||||
media_type=obj["ContentType"],
|
||||
headers={"Content-Length": str(obj["ContentLength"])},
|
||||
)
|
||||
@@ -0,0 +1,176 @@
|
||||
"""Durable asynchronous intake with full bounded validation before acceptance."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import unicodedata
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from .domain import Failure, timestamp
|
||||
from .jobs import Jobs, fault
|
||||
from .postgres import public
|
||||
|
||||
MAX_BYTES = 10 * 1024 * 1024
|
||||
MAX_PIXELS = 40_000_000
|
||||
PUBLIC_FIELDS = (
|
||||
"id",
|
||||
"chat_id",
|
||||
"owner_id",
|
||||
"filename",
|
||||
"content_type",
|
||||
"size_bytes",
|
||||
"sha256",
|
||||
"status",
|
||||
"error_code",
|
||||
"created_at",
|
||||
"updated_at",
|
||||
)
|
||||
|
||||
|
||||
def filename_valid(name):
|
||||
if not isinstance(name, str) or not 1 <= len(name) <= 255 or not name.strip():
|
||||
raise Failure(400, "Неверное имя файла")
|
||||
if any(c in "/\\" or unicodedata.category(c) == "Cc" for c in name):
|
||||
raise Failure(400, "Неверное имя файла")
|
||||
return name
|
||||
|
||||
|
||||
def validate_image(fileobj, declared_type):
|
||||
if declared_type not in {"image/png", "image/jpeg"}:
|
||||
raise Failure(415, "Ожидается PNG или JPEG")
|
||||
fileobj.seek(0)
|
||||
encoded = fileobj.read(MAX_BYTES + 1)
|
||||
if len(encoded) > MAX_BYTES:
|
||||
raise Failure(413, "Файл больше 10 MiB")
|
||||
try:
|
||||
with subprocess.Popen(
|
||||
[sys.executable, "-m", "messenger.image_validation", declared_type],
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
start_new_session=True,
|
||||
env={
|
||||
"PATH": os.environ.get("PATH", os.defpath),
|
||||
"PYTHONPATH": str(Path(__file__).resolve().parents[1]),
|
||||
"PYTHONDONTWRITEBYTECODE": "1",
|
||||
"LANG": "C.UTF-8",
|
||||
},
|
||||
) as process:
|
||||
try:
|
||||
output, _ = process.communicate(input=encoded, timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
# The validator can itself run djpeg: kill its whole process group
|
||||
# before releasing the upload slot, not only the Python parent.
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass # The child may have exited at the timeout boundary.
|
||||
process.communicate()
|
||||
raise Failure(415, "Изображение не прошло ограниченную проверку") from None
|
||||
if process.returncode:
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
raise Failure(415, "Изображение не прошло ограниченную проверку")
|
||||
except OSError:
|
||||
raise Failure(503, "Проверка изображения временно недоступна") from None
|
||||
finally:
|
||||
del encoded
|
||||
try:
|
||||
outcome = json.loads(output)
|
||||
except (ValueError, UnicodeError):
|
||||
raise Failure(415, "Повреждённое изображение") from None
|
||||
if not isinstance(outcome, dict):
|
||||
raise Failure(415, "Изображение не прошло ограниченную проверку")
|
||||
if "error" in outcome:
|
||||
status = outcome["error"]
|
||||
if type(status) is not int or status not in {413, 415}:
|
||||
status = 415
|
||||
raise Failure(status, "Изображение не прошло ограниченную проверку")
|
||||
values = outcome.get("ok")
|
||||
if not isinstance(values, list) or len(values) != 3:
|
||||
raise Failure(415, "Изображение не прошло ограниченную проверку")
|
||||
fileobj.seek(0)
|
||||
return tuple(values)
|
||||
|
||||
|
||||
def metadata_public(row):
|
||||
return {**public({k: row[k] for k in PUBLIC_FIELDS}), "variants": row["variants"]}
|
||||
|
||||
|
||||
class AttachmentService:
|
||||
def __init__(self, store, storage):
|
||||
self.store = store
|
||||
self.storage = storage
|
||||
self.jobs = Jobs(store)
|
||||
|
||||
def upload(self, actor, cid, filename, content_type, fileobj, size):
|
||||
filename_valid(filename)
|
||||
if not 1 <= size <= MAX_BYTES:
|
||||
raise Failure(413 if size > MAX_BYTES else 415, "Неверный размер изображения")
|
||||
validate_image(fileobj, content_type)
|
||||
digest = hashlib.file_digest(fileobj, "sha256").hexdigest()
|
||||
aid = str(uuid4())
|
||||
key = "staging/" + aid
|
||||
now = timestamp()
|
||||
with self.store.connection() as conn:
|
||||
self.store._chat(conn, actor, cid)
|
||||
conn.execute(
|
||||
"INSERT INTO upload_intents(id,object_key,created_at) VALUES (%s,%s,%s)",
|
||||
(aid, key, now),
|
||||
)
|
||||
# Keep the intent locked throughout PUT/finalization: reconciler skips live
|
||||
# uploads even if a slow request crosses the grace boundary.
|
||||
with self.store.connection() as conn, conn.transaction():
|
||||
intent = conn.execute(
|
||||
"SELECT id FROM upload_intents WHERE id=%s FOR UPDATE", (aid,)
|
||||
).fetchone()
|
||||
if not intent:
|
||||
raise Failure(503, "Загрузка больше не активна")
|
||||
self.storage.put(key, fileobj, size, content_type)
|
||||
fault(self.store, "after_staging", aid)
|
||||
row = conn.execute(
|
||||
"INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,"
|
||||
"status,error_code,created_at,updated_at,object_key,stage_key) "
|
||||
"VALUES (%s,%s,%s,%s,%s,%s,%s,'queued',NULL,%s,%s,%s,%s) RETURNING *",
|
||||
(aid, cid, actor, filename, content_type, size, digest, now, now, key, key),
|
||||
).fetchone()
|
||||
self.jobs.enqueue(conn, aid, key)
|
||||
conn.execute("DELETE FROM upload_intents WHERE id=%s", (aid,))
|
||||
# Never delete S3 on an ambiguous COMMIT: the transaction may have committed.
|
||||
fault(self.store, "after_upload_commit", aid)
|
||||
return metadata_public(row)
|
||||
|
||||
def metadata(self, actor, aid):
|
||||
with self.store.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT a.* FROM attachments a JOIN members m ON m.chat_id=a.chat_id "
|
||||
"WHERE a.id=%s AND m.user_id=%s",
|
||||
(aid, actor),
|
||||
).fetchone()
|
||||
if not row:
|
||||
raise Failure(404, "Объект не найден")
|
||||
return metadata_public(row)
|
||||
|
||||
def content(self, actor, aid, variant="original"):
|
||||
with self.store.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT a.* FROM attachments a JOIN members m ON m.chat_id=a.chat_id "
|
||||
"WHERE a.id=%s AND m.user_id=%s",
|
||||
(aid, actor),
|
||||
).fetchone()
|
||||
if not row:
|
||||
raise Failure(404, "Объект не найден")
|
||||
if row["status"] != "ready":
|
||||
raise Failure(409, "Вложение не готово")
|
||||
key = row["object_key"]
|
||||
if variant != "original":
|
||||
if variant not in {v["name"] for v in row["variants"]}:
|
||||
raise Failure(404, "Вариант не найден")
|
||||
key = key.rsplit("/", 1)[0] + "/" + variant
|
||||
return self.storage.get(key)
|
||||
@@ -0,0 +1,81 @@
|
||||
"""Same SQL with actual PG roles, confined to a disposable synthetic schema."""
|
||||
|
||||
import json
|
||||
import secrets
|
||||
|
||||
import psycopg
|
||||
from psycopg import sql
|
||||
|
||||
from .config import secret
|
||||
|
||||
|
||||
def main():
|
||||
suffix = secrets.token_hex(5)
|
||||
schema, wide, limited = (prefix + suffix for prefix in ("study_", "wide_", "limited_"))
|
||||
admin = psycopg.connect(secret("DATABASE_URL"), autocommit=True)
|
||||
rows = []
|
||||
try:
|
||||
# SET ROLE is an explicit operator act. Both NOLOGIN test roles have no
|
||||
# grants on application tables. Wide owns ONLY the synthetic namespace.
|
||||
for role in (wide, limited):
|
||||
admin.execute(sql.SQL("CREATE ROLE {} NOLOGIN").format(sql.Identifier(role)))
|
||||
admin.execute(
|
||||
sql.SQL("CREATE SCHEMA {} AUTHORIZATION {}").format(
|
||||
sql.Identifier(schema), sql.Identifier(wide)
|
||||
)
|
||||
)
|
||||
admin.execute(sql.SQL("SET search_path TO {}").format(sql.Identifier(schema)))
|
||||
admin.execute(sql.SQL("SET ROLE {}").format(sql.Identifier(wide)))
|
||||
admin.execute("CREATE TABLE users(password_hash text); CREATE TABLE jobs(id int)")
|
||||
admin.execute(
|
||||
sql.SQL("GRANT USAGE ON SCHEMA {} TO {}; GRANT SELECT ON jobs TO {}").format(
|
||||
sql.Identifier(schema), sql.Identifier(limited), sql.Identifier(limited)
|
||||
)
|
||||
)
|
||||
admin.execute("RESET ROLE")
|
||||
for label, role in (("wide", wide), ("limited", limited)):
|
||||
admin.execute(sql.SQL("SET ROLE {}").format(sql.Identifier(role)))
|
||||
assert admin.execute("SELECT current_user").fetchone()[0] == role
|
||||
for name, query, restricted_expected in (
|
||||
("read_password_hash", "SELECT password_hash FROM users LIMIT 0", False),
|
||||
("create_table", "CREATE TABLE forbidden(id int)", False),
|
||||
("read_jobs", "SELECT id FROM jobs LIMIT 0", True),
|
||||
):
|
||||
try:
|
||||
admin.execute(query)
|
||||
allowed = True
|
||||
if name == "create_table":
|
||||
admin.execute("DROP TABLE forbidden")
|
||||
except psycopg.errors.InsufficientPrivilege:
|
||||
allowed = False
|
||||
rows.append(
|
||||
{
|
||||
"role": label,
|
||||
"operation": name,
|
||||
"allowed": allowed,
|
||||
"expected": restricted_expected if label == "limited" else True,
|
||||
}
|
||||
)
|
||||
admin.execute("RESET ROLE")
|
||||
passed = all(r["allowed"] == r["expected"] for r in rows)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"scope": "disposable synthetic schema; operator SET ROLE to NOLOGIN roles",
|
||||
"rows": rows,
|
||||
"passed": passed,
|
||||
}
|
||||
)
|
||||
)
|
||||
if not passed:
|
||||
raise SystemExit(1)
|
||||
finally:
|
||||
admin.execute("RESET ROLE")
|
||||
admin.execute(sql.SQL("DROP SCHEMA IF EXISTS {} CASCADE").format(sql.Identifier(schema)))
|
||||
for role in (limited, wide):
|
||||
admin.execute(sql.SQL("DROP ROLE IF EXISTS {}").format(sql.Identifier(role)))
|
||||
admin.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Isolated loopback-only Valkey control: same operations before/after ACL narrowing."""
|
||||
|
||||
import json
|
||||
import secrets
|
||||
|
||||
import redis
|
||||
|
||||
|
||||
def main():
|
||||
admin = redis.Redis(host="127.0.0.1", port=6378, decode_responses=True)
|
||||
wide_pass, limited_pass = secrets.token_urlsafe(24), secrets.token_urlsafe(24)
|
||||
admin.execute_command(
|
||||
"ACL", "SETUSER", "wide", "on", ">" + wide_pass, "~security-study:*", "+@all"
|
||||
)
|
||||
admin.execute_command(
|
||||
"ACL",
|
||||
"SETUSER",
|
||||
"limited",
|
||||
"on",
|
||||
">" + limited_pass,
|
||||
"~security-study:jobs",
|
||||
"+ping",
|
||||
"+xgroup|create",
|
||||
"+xreadgroup",
|
||||
"+xautoclaim",
|
||||
"+xack",
|
||||
"+xdel",
|
||||
)
|
||||
rows = []
|
||||
for role, password in (("wide", wide_pass), ("limited", limited_pass)):
|
||||
client = redis.Redis(
|
||||
host="127.0.0.1", port=6378, username=role, password=password, decode_responses=True
|
||||
)
|
||||
for name, operation in (
|
||||
("queue_admin_acl_list", lambda client=client: client.acl_list()),
|
||||
(
|
||||
"queue_publish",
|
||||
lambda client=client: client.xadd("security-study:jobs", {"body": "synthetic"}),
|
||||
),
|
||||
(
|
||||
"queue_other_key_write",
|
||||
lambda client=client: client.set("security-study:other", "synthetic"),
|
||||
),
|
||||
):
|
||||
try:
|
||||
operation()
|
||||
allowed = True
|
||||
except redis.ResponseError:
|
||||
allowed = False
|
||||
rows.append({"role": role, "operation": name, "allowed": allowed})
|
||||
client.close()
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"isolated_port": 6378,
|
||||
"rows": rows,
|
||||
"passed": all(r["allowed"] == (r["role"] == "wide") for r in rows),
|
||||
}
|
||||
)
|
||||
)
|
||||
admin.shutdown(nosave=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Disposable bucket comparison; baseline credentials cannot access messenger."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
|
||||
def client(access, secret, identity):
|
||||
return boto3.client(
|
||||
"s3",
|
||||
endpoint_url="https://storage:8443",
|
||||
aws_access_key_id=access,
|
||||
aws_secret_access_key=secret,
|
||||
region_name="us-east-1",
|
||||
verify="/run/tls/ca.pem",
|
||||
config=Config(
|
||||
client_cert=(
|
||||
f"/run/audit-identities/{identity}/cert.pem",
|
||||
f"/run/audit-identities/{identity}/key.pem",
|
||||
),
|
||||
connect_timeout=2,
|
||||
read_timeout=3,
|
||||
retries={"total_max_attempts": 1},
|
||||
s3={"addressing_style": "path"},
|
||||
request_checksum_calculation="when_required",
|
||||
response_checksum_validation="when_required",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
root = Path("/run/audit-material")
|
||||
|
||||
def read(name):
|
||||
return (root / name).read_text().strip()
|
||||
|
||||
admin = client(read("admin_s3_access"), read("admin_s3_secret"), "operator")
|
||||
wide = client(read("study_wide_s3_access"), read("study_wide_s3_secret"), "operator")
|
||||
limited = client(read("worker_s3_access"), read("worker_s3_secret"), "worker")
|
||||
bucket = "security-study"
|
||||
rows = []
|
||||
try:
|
||||
wide.get_object(Bucket="messenger", Key="private/not-a-real-user-object")
|
||||
isolated = False
|
||||
except ClientError as exc:
|
||||
isolated = exc.response["ResponseMetadata"]["HTTPStatusCode"] == 403
|
||||
admin.create_bucket(Bucket=bucket)
|
||||
admin.put_object(Bucket=bucket, Key="private/synthetic", Body=b"synthetic-not-user-data")
|
||||
try:
|
||||
for role, c in (("wide", wide), ("limited", limited)):
|
||||
for operation in ("read_private", "write_private"):
|
||||
try:
|
||||
if operation == "read_private":
|
||||
response = c.get_object(Bucket=bucket, Key="private/synthetic")
|
||||
response["Body"].read()
|
||||
response["Body"].close()
|
||||
else:
|
||||
c.put_object(Bucket=bucket, Key="private/written", Body=b"synthetic")
|
||||
allowed = True
|
||||
except ClientError as exc:
|
||||
if exc.response["ResponseMetadata"]["HTTPStatusCode"] != 403:
|
||||
raise
|
||||
allowed = False
|
||||
rows.append({"role": role, "operation": operation, "allowed": allowed})
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"scope": "disposable bucket security-study; broad role has no messenger rights",
|
||||
"rows": rows,
|
||||
"wide_role_denied_application_bucket": isolated,
|
||||
"passed": isolated
|
||||
and all(row["allowed"] == (row["role"] == "wide") for row in rows),
|
||||
}
|
||||
)
|
||||
)
|
||||
finally:
|
||||
for key in ("private/synthetic", "private/written"):
|
||||
admin.delete_object(Bucket=bucket, Key=key)
|
||||
admin.delete_bucket(Bucket=bucket)
|
||||
for c in (admin, wide, limited):
|
||||
c.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,14 @@
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def secret(name: str) -> str:
|
||||
if os.getenv("BAO_ROLE"):
|
||||
from .secrets_provider import value
|
||||
|
||||
return value(name)
|
||||
filename = os.environ.get(name + "_FILE")
|
||||
value = Path(filename).read_text().strip() if filename else os.environ.get(name, "")
|
||||
if not value:
|
||||
raise RuntimeError("Отсутствует настройка " + name)
|
||||
return value
|
||||
@@ -0,0 +1,63 @@
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
from datetime import UTC, datetime
|
||||
from uuid import UUID
|
||||
|
||||
|
||||
class Failure(Exception):
|
||||
def __init__(self, status: int, message: str, headers: dict | None = None):
|
||||
self.status = status
|
||||
self.message = message
|
||||
self.headers = headers or {}
|
||||
|
||||
|
||||
def timestamp() -> str:
|
||||
return datetime.now(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def identifier(value: str) -> str:
|
||||
try:
|
||||
return str(UUID(value))
|
||||
except (ValueError, TypeError, AttributeError):
|
||||
raise Failure(400, "Некорректный UUID") from None
|
||||
|
||||
|
||||
def boundary(scope: str, cursor: str | None) -> tuple[str, str] | None:
|
||||
if cursor is None:
|
||||
return None
|
||||
try:
|
||||
if not cursor or len(cursor) > 1024 or not re.fullmatch(r"[A-Za-z0-9_-]+", cursor):
|
||||
raise ValueError
|
||||
data = json.loads(base64.urlsafe_b64decode(cursor + "=" * (-len(cursor) % 4)))
|
||||
if (
|
||||
not isinstance(data, list)
|
||||
or len(data) != 4
|
||||
or type(data[0]) is not int
|
||||
or data[0] != 1
|
||||
or data[1] != scope
|
||||
):
|
||||
raise ValueError
|
||||
date, uid = data[2:]
|
||||
if not isinstance(date, str) or not re.fullmatch(
|
||||
r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]{1,6})?Z",
|
||||
date,
|
||||
):
|
||||
raise ValueError
|
||||
datetime.fromisoformat(date.replace("Z", "+00:00"))
|
||||
if not isinstance(uid, str) or str(UUID(uid)) != uid:
|
||||
raise ValueError
|
||||
return date, uid
|
||||
except (ValueError, TypeError, KeyError, UnicodeError):
|
||||
raise Failure(400, "Некорректный cursor или область запроса") from None
|
||||
|
||||
|
||||
def page(items: list[dict], limit: int, scope: str) -> dict:
|
||||
more = len(items) > limit
|
||||
selected = items[:limit]
|
||||
cursor = None
|
||||
if more:
|
||||
last = selected[-1]
|
||||
payload = json.dumps([1, scope, last["created_at"], last["id"]], separators=(",", ":"))
|
||||
cursor = base64.urlsafe_b64encode(payload.encode()).decode().rstrip("=")
|
||||
return {"items": selected, "next_cursor": cursor}
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Image transformations, run in a bounded child process by the worker."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image, ImageOps
|
||||
|
||||
Image.MAX_IMAGE_PIXELS = 40_000_000
|
||||
|
||||
|
||||
def fitted_size(width, height, bound):
|
||||
"""Fit without upscale; integer arithmetic deliberately rounds down."""
|
||||
longest = max(width, height)
|
||||
if longest <= bound:
|
||||
return width, height
|
||||
return max(1, width * bound // longest), max(1, height * bound // longest)
|
||||
|
||||
|
||||
def render(source, destination, version=1):
|
||||
if version not in (1, 2):
|
||||
raise ValueError("unsupported_pipeline")
|
||||
destination = Path(destination)
|
||||
with Image.open(source) as image:
|
||||
if image.width * image.height > 40_000_000:
|
||||
raise ValueError("pixel_limit")
|
||||
image.load()
|
||||
ImageOps.exif_transpose(image, in_place=True)
|
||||
if "A" in image.getbands() or (image.mode == "P" and "transparency" in image.info):
|
||||
rgba = image if image.mode == "RGBA" else image.convert("RGBA")
|
||||
rgb = Image.new("RGB", rgba.size, "white")
|
||||
alpha = rgba.getchannel("A")
|
||||
rgb.paste(rgba, mask=alpha)
|
||||
alpha.close()
|
||||
if rgba is not image:
|
||||
rgba.close()
|
||||
else:
|
||||
rgb = image if image.mode == "RGB" else image.convert("RGB")
|
||||
thumbnail = rgb.resize(fitted_size(*rgb.size, 256), Image.Resampling.LANCZOS)
|
||||
crop = ImageOps.fit(rgb, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5))
|
||||
optimized = rgb.resize(fitted_size(*rgb.size, 1280), Image.Resampling.LANCZOS)
|
||||
variants = []
|
||||
for name, variant, kind, mime, options in (
|
||||
("thumbnail", thumbnail, "JPEG", "image/jpeg", {"quality": 85}),
|
||||
("crop", crop, "JPEG", "image/jpeg", {"quality": 85}),
|
||||
(
|
||||
"optimized",
|
||||
optimized,
|
||||
"WEBP",
|
||||
"image/webp",
|
||||
{"quality": 80 if version == 1 else 75, "method": 4},
|
||||
),
|
||||
):
|
||||
# No EXIF/ICC arguments: outputs do not propagate source metadata.
|
||||
path = destination / name
|
||||
variant.save(path, format=kind, **options)
|
||||
data = path.read_bytes()
|
||||
variants.append(
|
||||
{
|
||||
"name": name,
|
||||
"content_type": mime,
|
||||
"size_bytes": len(data),
|
||||
"width": variant.width,
|
||||
"height": variant.height,
|
||||
"sha256": hashlib.sha256(data).hexdigest(),
|
||||
}
|
||||
)
|
||||
variant.close()
|
||||
if rgb is not image:
|
||||
rgb.close()
|
||||
return variants
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import resource
|
||||
|
||||
resource.setrlimit(resource.RLIMIT_AS, (700 * 1024**2, 700 * 1024**2))
|
||||
resource.setrlimit(resource.RLIMIT_CPU, (15, 15))
|
||||
source, destination, version = sys.argv[1:]
|
||||
result = render(source, destination, int(version))
|
||||
(Path(destination) / "variants.json").write_text(json.dumps(result))
|
||||
@@ -0,0 +1,146 @@
|
||||
"""Short-lived bounded pixel validation; no DB, credentials or application imports."""
|
||||
|
||||
import json
|
||||
import resource
|
||||
import subprocess
|
||||
import sys
|
||||
from io import BytesIO
|
||||
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
|
||||
from .domain import Failure
|
||||
|
||||
MAX_BYTES = 10 * 1024 * 1024
|
||||
MAX_PIXELS = 40_000_000
|
||||
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
|
||||
|
||||
|
||||
def inspect_image(fileobj, declared_type):
|
||||
if declared_type not in {"image/png", "image/jpeg"}:
|
||||
raise Failure(415, "Ожидается PNG или JPEG")
|
||||
try:
|
||||
fileobj.seek(0)
|
||||
with Image.open(fileobj) as header:
|
||||
width, height = header.size
|
||||
if width * height > MAX_PIXELS:
|
||||
raise Failure(413, "Слишком много пикселей")
|
||||
actual = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(header.format)
|
||||
if actual != declared_type:
|
||||
raise Failure(415, "Формат изображения не соответствует MIME")
|
||||
frames = getattr(header, "n_frames", 1)
|
||||
default_image = bool(getattr(header, "default_image", False))
|
||||
if width * height * frames > MAX_PIXELS:
|
||||
raise Failure(413, "Слишком много пикселей во всех кадрах")
|
||||
header.verify()
|
||||
# Drop parsed metadata before the separate strict decoder process starts.
|
||||
del header
|
||||
if actual == "image/png":
|
||||
# verify() checks CRC, but does not compare acTL's frame count with
|
||||
# every fcTL. Do not let a forged count hide a corrupt later frame.
|
||||
fileobj.seek(8)
|
||||
controls = frame_controls = 0
|
||||
declared_frames = None
|
||||
animation_data = False
|
||||
while True:
|
||||
chunk_header = fileobj.read(8)
|
||||
if len(chunk_header) != 8:
|
||||
raise ValueError("Truncated PNG chunk")
|
||||
length = int.from_bytes(chunk_header[:4], "big")
|
||||
kind = chunk_header[4:]
|
||||
controls += kind == b"acTL"
|
||||
frame_controls += kind == b"fcTL"
|
||||
animation_data |= kind == b"fdAT"
|
||||
if kind == b"acTL":
|
||||
if length != 8:
|
||||
raise ValueError("Invalid acTL length")
|
||||
declared_frames = int.from_bytes(fileobj.read(4), "big")
|
||||
fileobj.seek(length, 1) # remaining payload plus CRC
|
||||
else:
|
||||
fileobj.seek(length + 4, 1)
|
||||
if kind == b"IEND":
|
||||
break
|
||||
if controls:
|
||||
if (
|
||||
controls != 1
|
||||
or not declared_frames
|
||||
or frame_controls != declared_frames
|
||||
or declared_frames + int(default_image) != frames
|
||||
):
|
||||
raise ValueError("Invalid APNG frame count")
|
||||
elif frame_controls or animation_data:
|
||||
raise ValueError("Animation chunks without acTL")
|
||||
if actual == "image/jpeg":
|
||||
# Validate scan warnings before allocating Pillow's decoded canvas.
|
||||
# 384 MiB covers coefficient arrays for <=40M pixels and <=4 channels;
|
||||
# keeping both full decoded buffers alive could exceed the API cgroup.
|
||||
fileobj.seek(0)
|
||||
# A bounded compressed copy avoids sharing buffered file-descriptor
|
||||
# offsets with the child, and supports BytesIO callers as well.
|
||||
encoded = fileobj.read(MAX_BYTES + 1)
|
||||
if len(encoded) > MAX_BYTES:
|
||||
raise Failure(413, "Файл больше 10 MiB")
|
||||
try:
|
||||
decoded = subprocess.run(
|
||||
["djpeg", "-strict", "-maxmemory", "384m", "-outfile", "/dev/null"],
|
||||
input=encoded,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.PIPE,
|
||||
timeout=4,
|
||||
check=False,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
raise Failure(415, "Изображение не прошло ограниченную проверку") from None
|
||||
finally:
|
||||
del encoded
|
||||
if decoded.returncode:
|
||||
raise Failure(415, "Повреждённое изображение")
|
||||
# djpeg has already decoded every JPEG pixel, streaming output rows.
|
||||
# A second Pillow load would overlap full progressive coefficient
|
||||
# arrays with a 160 MiB CMYK canvas and violate the API's 512 MiB cap.
|
||||
fileobj.seek(0)
|
||||
return actual, width, height
|
||||
# Decode one frame at a time. Pillow may retain a previous APNG canvas
|
||||
# for compositing, but no list/copies of every decoded frame is collected.
|
||||
fileobj.seek(0)
|
||||
with Image.open(fileobj) as image:
|
||||
decoded_pixels = 0
|
||||
for frame in range(frames):
|
||||
if decoded_pixels + width * height > MAX_PIXELS:
|
||||
raise Failure(413, "Слишком много пикселей во всех кадрах")
|
||||
image.seek(frame)
|
||||
decoded_pixels += image.size[0] * image.size[1]
|
||||
if decoded_pixels > MAX_PIXELS:
|
||||
raise Failure(413, "Слишком много пикселей во всех кадрах")
|
||||
image.load()
|
||||
# Image's context manager closes its input handle, not the decoded core.
|
||||
# Releasing the object frees native canvases before hashing/S3 publication.
|
||||
del image
|
||||
fileobj.seek(0)
|
||||
return actual, width, height
|
||||
except Image.DecompressionBombError:
|
||||
raise Failure(413, "Слишком много пикселей") from None
|
||||
except (UnidentifiedImageError, OSError, ValueError, SyntaxError, EOFError):
|
||||
raise Failure(415, "Повреждённое изображение") from None
|
||||
|
||||
|
||||
def main():
|
||||
resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
|
||||
resource.setrlimit(resource.RLIMIT_CPU, (4, 5))
|
||||
if sys.platform == "linux":
|
||||
resource.setrlimit(resource.RLIMIT_AS, (512 * 1024 * 1024, 512 * 1024 * 1024))
|
||||
try:
|
||||
data = sys.stdin.buffer.read(MAX_BYTES + 1)
|
||||
if len(data) > MAX_BYTES:
|
||||
raise Failure(413, "Файл больше 10 MiB")
|
||||
result = inspect_image(BytesIO(data), sys.argv[1])
|
||||
print(json.dumps({"ok": result}))
|
||||
except Failure as exc:
|
||||
print(json.dumps({"error": exc.status}))
|
||||
except MemoryError:
|
||||
print(json.dumps({"error": 415}))
|
||||
except Exception:
|
||||
print(json.dumps({"error": 415}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,123 @@
|
||||
"""Local operator commands; no public HTTP replay or arbitrary object paths."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
|
||||
from .domain import Failure
|
||||
from .jobs import Jobs, fault
|
||||
from .postgres import PostgresStore
|
||||
from .storage import ObjectStorage
|
||||
from .worker import cleanup_ready
|
||||
|
||||
|
||||
def cleanup(store, storage, failed_hours=168, intent_seconds=3600):
|
||||
if failed_hours < 0 or intent_seconds < 0:
|
||||
raise ValueError("Retention must be nonnegative")
|
||||
counts = {"ready_staging": 0, "failed_sources": 0, "orphan_intents": 0}
|
||||
with store.connection() as conn:
|
||||
ready = conn.execute(
|
||||
"SELECT id FROM attachments WHERE status='ready' AND stage_key IS NOT NULL"
|
||||
).fetchall()
|
||||
for row in ready:
|
||||
cleanup_ready(store, storage, row["id"])
|
||||
counts["ready_staging"] += 1
|
||||
with store.connection() as conn:
|
||||
candidates = conn.execute(
|
||||
"SELECT id FROM attachments WHERE status='failed' AND updated_at<now()-(%s * interval '1 hour')",
|
||||
(failed_hours,),
|
||||
).fetchall()
|
||||
for row in candidates:
|
||||
with store.connection() as conn, conn.transaction():
|
||||
a = conn.execute(
|
||||
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (row["id"],)
|
||||
).fetchone()
|
||||
# Recheck under the same row lock used by replay and publication.
|
||||
if not a or a["status"] != "failed":
|
||||
continue
|
||||
current = conn.execute(
|
||||
"SELECT * FROM attachment_jobs WHERE attachment_id=%s AND pipeline_version=%s FOR UPDATE",
|
||||
(a["id"], a["pipeline_version"]),
|
||||
).fetchone()
|
||||
if not current or current["state"] != "failed" or current["source_key"] is None:
|
||||
continue
|
||||
old = conn.execute(
|
||||
"SELECT updated_at<now()-(%s * interval '1 hour') AS old FROM attachments WHERE id=%s",
|
||||
(failed_hours, a["id"]),
|
||||
).fetchone()["old"]
|
||||
if not old:
|
||||
continue
|
||||
fault(store, "cleanup_before_delete", a["id"])
|
||||
# If cleanup commits first, later replay refuses source_missing. If replay
|
||||
# commits first, status is queued and cleanup skips it. DELETE stays locked.
|
||||
source = current["source_key"]
|
||||
storage.delete(source)
|
||||
prefix = f"attachments/{a['id']}/v{a['pipeline_version']}"
|
||||
for variant in ("original", "thumbnail", "crop", "optimized"):
|
||||
storage.delete(prefix + "/" + variant)
|
||||
conn.execute("UPDATE attachment_jobs SET source_key=NULL WHERE id=%s", (current["id"],))
|
||||
conn.execute("UPDATE attachments SET stage_key=NULL WHERE id=%s", (a["id"],))
|
||||
Jobs.audit(conn, current["id"], "source_retention_expired", current["attempt"])
|
||||
counts["failed_sources"] += 1
|
||||
with store.connection() as conn:
|
||||
intents = conn.execute(
|
||||
"SELECT id FROM upload_intents WHERE created_at<now()-(%s * interval '1 second')",
|
||||
(intent_seconds,),
|
||||
).fetchall()
|
||||
for row in intents:
|
||||
with store.connection() as conn, conn.transaction():
|
||||
intent = conn.execute(
|
||||
"SELECT * FROM upload_intents WHERE id=%s FOR UPDATE SKIP LOCKED", (row["id"],)
|
||||
).fetchone()
|
||||
if intent is None:
|
||||
continue
|
||||
referenced = conn.execute(
|
||||
"SELECT 1 FROM attachments WHERE id=%s", (row["id"],)
|
||||
).fetchone()
|
||||
if referenced is None:
|
||||
storage.delete(intent["object_key"])
|
||||
counts["orphan_intents"] += 1
|
||||
conn.execute("DELETE FROM upload_intents WHERE id=%s", (row["id"],))
|
||||
return counts
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
for name in ("replay", "reprocess"):
|
||||
command = sub.add_parser(name)
|
||||
command.add_argument("attachment_id")
|
||||
if name == "reprocess":
|
||||
command.add_argument("--version", type=int, required=True)
|
||||
sub.add_parser("failed")
|
||||
command = sub.add_parser("cleanup")
|
||||
command.add_argument("--failed-hours", type=float, default=168)
|
||||
command.add_argument("--intent-seconds", type=float, default=3600)
|
||||
args = parser.parse_args()
|
||||
store, storage = PostgresStore(), ObjectStorage()
|
||||
try:
|
||||
jobs = Jobs(store)
|
||||
if args.command == "replay":
|
||||
result = {"job_id": jobs.replay(args.attachment_id, storage)}
|
||||
elif args.command == "reprocess":
|
||||
result = {"job_id": jobs.reprocess(args.attachment_id, args.version, storage)}
|
||||
elif args.command == "cleanup":
|
||||
result = cleanup(store, storage, args.failed_hours, args.intent_seconds)
|
||||
else:
|
||||
with store.connection() as conn:
|
||||
result = [
|
||||
dict(row)
|
||||
for row in conn.execute(
|
||||
"SELECT j.id,j.attachment_id,j.attempt,f.error_code,f.replay_count FROM failed_jobs f JOIN attachment_jobs j ON j.id=f.job_id WHERE j.state='failed'"
|
||||
).fetchall()
|
||||
]
|
||||
print(json.dumps(result, default=str))
|
||||
except Failure as exc:
|
||||
print(json.dumps({"status": exc.status, "error": exc.message}))
|
||||
raise SystemExit(1) from None
|
||||
finally:
|
||||
storage.close()
|
||||
store.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,345 @@
|
||||
"""PostgreSQL owns job state; broker deliveries are repeatable notifications."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from uuid import uuid4
|
||||
|
||||
import redis
|
||||
from opentelemetry import propagate
|
||||
from opentelemetry.trace import SpanKind
|
||||
|
||||
from . import telemetry
|
||||
from .config import secret
|
||||
from .domain import Failure
|
||||
from .postgres import public
|
||||
|
||||
logger = logging.getLogger("messenger.jobs")
|
||||
STREAM = "attachment-jobs:v1"
|
||||
GROUP = "image-workers"
|
||||
|
||||
|
||||
def broker_client():
|
||||
return redis.Redis.from_url(
|
||||
secret("BROKER_URL"),
|
||||
decode_responses=False,
|
||||
socket_connect_timeout=2,
|
||||
socket_timeout=3,
|
||||
)
|
||||
|
||||
|
||||
def fault(store, point, attachment_id):
|
||||
"""Only enabled in the explicit fault-test overlay; never accepts HTTP controls."""
|
||||
if os.getenv("ENABLE_FAULT_INJECTION") != "1":
|
||||
return
|
||||
with store.connection() as conn:
|
||||
row = conn.execute(
|
||||
"UPDATE test_faults SET reached_at=now() WHERE id=("
|
||||
"SELECT id FROM test_faults WHERE point=%s AND reached_at IS NULL "
|
||||
"AND (attachment_id IS NULL OR attachment_id=%s) ORDER BY id LIMIT 1) RETURNING *",
|
||||
(point, attachment_id),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return
|
||||
logger.warning(json.dumps({"event": "fault_reached", "point": point, "fault_id": row["id"]}))
|
||||
if row["action"] == "crash":
|
||||
os._exit(86)
|
||||
if row["action"] == "error":
|
||||
raise RuntimeError("injected_transient_failure")
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
with store.connection() as conn:
|
||||
released = conn.execute(
|
||||
"SELECT released FROM test_faults WHERE id=%s", (row["id"],)
|
||||
).fetchone()
|
||||
if released and released["released"]:
|
||||
return
|
||||
time.sleep(0.1)
|
||||
raise RuntimeError("fault_barrier_timeout")
|
||||
|
||||
|
||||
class Jobs:
|
||||
def __init__(self, store):
|
||||
self.store = store
|
||||
self.attempt_limit = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
|
||||
self.lease_seconds = int(os.getenv("JOB_LEASE_SECONDS", "45"))
|
||||
if not 1 <= self.attempt_limit <= 20 or not 5 <= self.lease_seconds <= 300:
|
||||
raise RuntimeError("Invalid job retry/lease configuration")
|
||||
|
||||
def enqueue(self, conn, attachment_id, source_key, version=1):
|
||||
job_id = str(uuid4())
|
||||
conn.execute(
|
||||
"INSERT INTO attachment_jobs(id,attachment_id,pipeline_version,source_key,state,attempt_limit,traceparent,tracestate,accepted_at) "
|
||||
"VALUES (%s,%s,%s,%s,'queued',%s,%s,%s,clock_timestamp())",
|
||||
(
|
||||
job_id,
|
||||
attachment_id,
|
||||
version,
|
||||
source_key,
|
||||
self.attempt_limit,
|
||||
telemetry.carrier().get("traceparent"),
|
||||
telemetry.carrier().get("tracestate"),
|
||||
),
|
||||
)
|
||||
conn.execute("INSERT INTO job_outbox(job_id) VALUES (%s)", (job_id,))
|
||||
self.audit(conn, job_id, "accepted", 0)
|
||||
return job_id
|
||||
|
||||
@staticmethod
|
||||
def audit(conn, job_id, event, attempt):
|
||||
conn.execute(
|
||||
"INSERT INTO job_audit(job_id,event,attempt) VALUES (%s,%s,%s)",
|
||||
(job_id, event, attempt),
|
||||
)
|
||||
|
||||
def publish_one(self, broker):
|
||||
with self.store.connection() as conn, conn.transaction():
|
||||
job = conn.execute(
|
||||
"SELECT j.* FROM job_outbox o JOIN attachment_jobs j ON j.id=o.job_id "
|
||||
"WHERE o.published_at IS NULL AND j.available_at<=now() "
|
||||
"ORDER BY j.available_at LIMIT 1 FOR UPDATE OF o SKIP LOCKED"
|
||||
).fetchone()
|
||||
if job is None:
|
||||
return False
|
||||
if job["state"] == "queued":
|
||||
body = {
|
||||
"schema_version": 1,
|
||||
"job_id": str(job["id"]),
|
||||
"type": "attachment.process.v1",
|
||||
"attachment_id": str(job["attachment_id"]),
|
||||
"pipeline_version": job["pipeline_version"],
|
||||
"attempt": job["attempt"] + 1,
|
||||
"created_at": public({"v": job["created_at"]})["v"],
|
||||
}
|
||||
# XADD response with appendfsync=always is the broker acceptance boundary.
|
||||
parent = {
|
||||
key: job.get(key) for key in ("traceparent", "tracestate") if job.get(key)
|
||||
}
|
||||
with telemetry.operation(
|
||||
"attachments publish", context=propagate.extract(parent), kind=SpanKind.PRODUCER
|
||||
):
|
||||
body.update(telemetry.carrier())
|
||||
broker.xadd(STREAM, {"body": json.dumps(body)})
|
||||
fault(self.store, "after_publish", job["attachment_id"])
|
||||
conn.execute("UPDATE job_outbox SET published_at=now() WHERE job_id=%s", (job["id"],))
|
||||
return True
|
||||
|
||||
def claim(self, job_id, attachment_id, version):
|
||||
with self.store.connection() as conn, conn.transaction():
|
||||
attachment = conn.execute(
|
||||
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,)
|
||||
).fetchone()
|
||||
if attachment is None:
|
||||
return None
|
||||
job = conn.execute(
|
||||
"SELECT * FROM attachment_jobs WHERE id=%s FOR UPDATE", (job_id,)
|
||||
).fetchone()
|
||||
if (
|
||||
job is None
|
||||
or str(job["attachment_id"]) != str(attachment_id)
|
||||
or job["pipeline_version"] != version
|
||||
):
|
||||
return None
|
||||
if attachment["pipeline_version"] != version:
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET state='superseded',updated_at=now() WHERE id=%s AND state NOT IN ('ready','superseded')",
|
||||
(job_id,),
|
||||
)
|
||||
return None
|
||||
if job["state"] != "queued":
|
||||
return None
|
||||
lease_id = str(uuid4())
|
||||
job = conn.execute(
|
||||
"UPDATE attachment_jobs SET state='processing',attempt=attempt+1,lease_id=%s,"
|
||||
"lease_until=now()+(%s * interval '1 second'),updated_at=now() "
|
||||
"WHERE id=%s AND available_at<=now() RETURNING *",
|
||||
(lease_id, self.lease_seconds, job_id),
|
||||
).fetchone()
|
||||
if job is None:
|
||||
return None
|
||||
conn.execute(
|
||||
"UPDATE attachments SET status='processing',updated_at=now() WHERE id=%s",
|
||||
(attachment_id,),
|
||||
)
|
||||
self.audit(conn, job_id, "claimed", job["attempt"])
|
||||
job["source_sha256"] = attachment["sha256"]
|
||||
job["source_size"] = attachment["size_bytes"]
|
||||
job["source_content_type"] = attachment["content_type"]
|
||||
return job
|
||||
|
||||
def complete(self, job, object_key, variants):
|
||||
with self.store.connection() as conn, conn.transaction():
|
||||
attachment = conn.execute(
|
||||
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (job["attachment_id"],)
|
||||
).fetchone()
|
||||
current = conn.execute(
|
||||
"SELECT * FROM attachment_jobs WHERE id=%s FOR UPDATE", (job["id"],)
|
||||
).fetchone()
|
||||
if (
|
||||
attachment["pipeline_version"] != job["pipeline_version"]
|
||||
or current["state"] != "processing"
|
||||
or current["lease_id"] != job["lease_id"]
|
||||
):
|
||||
self.audit(conn, job["id"], "stale_publish_rejected", job["attempt"])
|
||||
return False
|
||||
conn.execute(
|
||||
"UPDATE attachments SET status='ready',error_code=NULL,object_key=%s,variants=%s::jsonb,updated_at=now() WHERE id=%s",
|
||||
(object_key, json.dumps(variants), job["attachment_id"]),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET state='ready',lease_until=NULL,error_code=NULL,updated_at=now() WHERE id=%s",
|
||||
(job["id"],),
|
||||
)
|
||||
self.audit(conn, job["id"], "published", job["attempt"])
|
||||
t = telemetry.current()
|
||||
if t:
|
||||
t.ready_duration.record(
|
||||
(datetime.now(UTC) - job["accepted_at"]).total_seconds(),
|
||||
{"job_type": "attachment.process.v1"},
|
||||
)
|
||||
return True
|
||||
|
||||
def fail(self, job, code, permanent=False):
|
||||
with self.store.connection() as conn, conn.transaction():
|
||||
attachment = conn.execute(
|
||||
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (job["attachment_id"],)
|
||||
).fetchone()
|
||||
current = conn.execute(
|
||||
"SELECT * FROM attachment_jobs WHERE id=%s FOR UPDATE", (job["id"],)
|
||||
).fetchone()
|
||||
if (
|
||||
attachment["pipeline_version"] != job["pipeline_version"]
|
||||
or current["lease_id"] != job["lease_id"]
|
||||
or current["state"] != "processing"
|
||||
):
|
||||
return
|
||||
if permanent or current["attempt"] >= current["attempt_limit"]:
|
||||
final_code = code if permanent else "retry_exhausted"
|
||||
self._failed(conn, current, final_code)
|
||||
else:
|
||||
delay = min(30, 2 ** (current["attempt"] - 1)) + random.uniform(0, 0.5)
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET state='queued',error_code=%s,lease_until=NULL,"
|
||||
"available_at=now()+(%s * interval '1 second'),updated_at=now() WHERE id=%s",
|
||||
(code, delay, job["id"]),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE job_outbox SET published_at=NULL WHERE job_id=%s", (job["id"],)
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachments SET status='queued',updated_at=now() WHERE id=%s",
|
||||
(job["attachment_id"],),
|
||||
)
|
||||
self.audit(conn, job["id"], "retry_scheduled", job["attempt"])
|
||||
|
||||
def _failed(self, conn, job, code):
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET state='failed',error_code=%s,lease_until=NULL,updated_at=now() WHERE id=%s",
|
||||
(code, job["id"]),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachments SET status='failed',error_code=%s,updated_at=now() WHERE id=%s",
|
||||
(code, job["attachment_id"]),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO failed_jobs(job_id,error_code) VALUES (%s,%s) "
|
||||
"ON CONFLICT(job_id) DO UPDATE SET error_code=excluded.error_code,failed_at=now()",
|
||||
(job["id"], code),
|
||||
)
|
||||
self.audit(conn, job["id"], "failed:" + code, job["attempt"])
|
||||
|
||||
def recover_one(self):
|
||||
with self.store.connection() as conn, conn.transaction():
|
||||
candidate = conn.execute(
|
||||
"SELECT a.id FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id "
|
||||
"WHERE j.state='processing' AND j.lease_until<now() LIMIT 1 FOR UPDATE OF a SKIP LOCKED"
|
||||
).fetchone()
|
||||
if candidate is None:
|
||||
return False
|
||||
job = conn.execute(
|
||||
"SELECT j.* FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id "
|
||||
"WHERE j.attachment_id=%s AND j.state='processing' AND j.lease_until<now() "
|
||||
"AND j.pipeline_version=a.pipeline_version FOR UPDATE OF j",
|
||||
(candidate["id"],),
|
||||
).fetchone()
|
||||
if job is None:
|
||||
return False
|
||||
if job["attempt"] >= job["attempt_limit"]:
|
||||
self._failed(conn, job, "retry_exhausted")
|
||||
else:
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET state='queued',lease_until=NULL,available_at=now(),updated_at=now() WHERE id=%s",
|
||||
(job["id"],),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachments SET status='queued',updated_at=now() WHERE id=%s",
|
||||
(job["attachment_id"],),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE job_outbox SET published_at=NULL WHERE job_id=%s", (job["id"],)
|
||||
)
|
||||
self.audit(conn, job["id"], "lease_recovered", job["attempt"])
|
||||
return True
|
||||
|
||||
def replay(self, attachment_id, storage):
|
||||
with self.store.connection() as conn, conn.transaction():
|
||||
a = conn.execute(
|
||||
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,)
|
||||
).fetchone()
|
||||
if not a or a["status"] != "failed":
|
||||
raise Failure(409, "Повтор разрешён только для failed")
|
||||
job = conn.execute(
|
||||
"SELECT * FROM attachment_jobs WHERE attachment_id=%s AND pipeline_version=%s FOR UPDATE",
|
||||
(attachment_id, a["pipeline_version"]),
|
||||
).fetchone()
|
||||
if not job["source_key"] or not storage.exists(job["source_key"]):
|
||||
raise Failure(409, "Источник больше недоступен")
|
||||
fault(self.store, "replay_before_commit", attachment_id)
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET state='queued',attempt_limit=attempt+%s,available_at=now(),error_code=NULL,updated_at=now() WHERE id=%s",
|
||||
(self.attempt_limit, job["id"]),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachments SET status='queued',error_code=NULL,updated_at=now() WHERE id=%s",
|
||||
(attachment_id,),
|
||||
)
|
||||
conn.execute("UPDATE job_outbox SET published_at=NULL WHERE job_id=%s", (job["id"],))
|
||||
conn.execute(
|
||||
"UPDATE failed_jobs SET replay_count=replay_count+1 WHERE job_id=%s", (job["id"],)
|
||||
)
|
||||
self.audit(conn, job["id"], "operator_replay", job["attempt"])
|
||||
return str(job["id"])
|
||||
|
||||
def reprocess(self, attachment_id, version, storage):
|
||||
if version not in (1, 2):
|
||||
raise Failure(400, "Поддерживаются версии pipeline 1 и 2")
|
||||
with self.store.connection() as conn, conn.transaction():
|
||||
a = conn.execute(
|
||||
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,)
|
||||
).fetchone()
|
||||
if not a:
|
||||
raise Failure(404, "Вложение отсутствует")
|
||||
if version <= a["pipeline_version"]:
|
||||
raise Failure(409, "Нужна более новая версия pipeline")
|
||||
source = a["object_key"] if a["status"] == "ready" else a["stage_key"]
|
||||
if not source or not storage.exists(source):
|
||||
raise Failure(409, "Источник больше недоступен")
|
||||
if a["status"] == "ready" and a["stage_key"] and a["stage_key"].startswith("staging/"):
|
||||
# A worker can be paused after publishing ready but before its
|
||||
# staging cleanup. Do not orphan that key when choosing final v1
|
||||
# as the new source. Delete stays under the same attachment lock;
|
||||
# a storage failure leaves the ready version unchanged.
|
||||
fault(self.store, "reprocess_before_staging_delete", attachment_id)
|
||||
storage.delete(a["stage_key"])
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET state='superseded',updated_at=now() WHERE attachment_id=%s AND state IN ('queued','processing','failed')",
|
||||
(attachment_id,),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachments SET pipeline_version=%s,status='queued',variants='[]',error_code=NULL,stage_key=%s,updated_at=now() WHERE id=%s",
|
||||
(version, source, attachment_id),
|
||||
)
|
||||
return self.enqueue(conn, attachment_id, source, version)
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Measure configured Argon2 costs without keeping passwords or hashes in evidence."""
|
||||
|
||||
import json
|
||||
import resource
|
||||
import statistics
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
|
||||
|
||||
def main():
|
||||
kdf = PasswordHasher(time_cost=2, memory_cost=65536, parallelism=1)
|
||||
samples = []
|
||||
for _ in range(7):
|
||||
start = time.perf_counter()
|
||||
digest = kdf.hash("synthetic-measurement-password")
|
||||
kdf.verify(digest, "synthetic-measurement-password")
|
||||
samples.append((time.perf_counter() - start) * 1000)
|
||||
|
||||
def one(_):
|
||||
start = time.perf_counter()
|
||||
kdf.hash("synthetic-parallel-password")
|
||||
return (time.perf_counter() - start) * 1000
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
parallel = list(pool.map(one, range(4)))
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"algorithm": "argon2id",
|
||||
"memory_kib": 65536,
|
||||
"time_cost": 2,
|
||||
"parallelism": 1,
|
||||
"runs_hash_plus_verify_ms": samples,
|
||||
"median_hash_plus_verify_ms": statistics.median(samples),
|
||||
"max_hash_plus_verify_ms": max(samples),
|
||||
"two_concurrent_hash_ms": parallel,
|
||||
"process_peak_rss_kib": resource.getrusage(resource.RUSAGE_SELF).ru_maxrss,
|
||||
"kdf_slots_per_api": 2,
|
||||
"api_replicas": 2,
|
||||
"maximum_active_kdf_memory_mib": 256,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,32 @@
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
|
||||
from .config import secret
|
||||
|
||||
|
||||
def migrate():
|
||||
with psycopg.connect(secret("DATABASE_URL")) as conn:
|
||||
# Serialize independent migration processes without changing application data.
|
||||
conn.execute("SELECT pg_advisory_xact_lock(2301002)")
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS schema_migrations "
|
||||
"(name text PRIMARY KEY, checksum text NOT NULL)"
|
||||
)
|
||||
for path in sorted(Path("migrations").glob("*.sql")):
|
||||
checksum = hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
old = conn.execute(
|
||||
"SELECT checksum FROM schema_migrations WHERE name=%s", (path.name,)
|
||||
).fetchone()
|
||||
if old:
|
||||
if old[0] != checksum:
|
||||
raise RuntimeError("Изменена применённая миграция: " + path.name)
|
||||
continue
|
||||
conn.execute(path.read_text())
|
||||
conn.execute("INSERT INTO schema_migrations VALUES (%s,%s)", (path.name, checksum))
|
||||
print("Applied", path.name)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
migrate()
|
||||
@@ -0,0 +1,71 @@
|
||||
import re
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator, model_validator
|
||||
|
||||
|
||||
class RequestModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
@field_validator("*", mode="after")
|
||||
@classmethod
|
||||
def nonblank(cls, value):
|
||||
if isinstance(value, str):
|
||||
value.encode("utf-8")
|
||||
if isinstance(value, str) and not value.strip():
|
||||
raise ValueError("Строка не может состоять из пробелов")
|
||||
return value
|
||||
|
||||
|
||||
class CreateUser(RequestModel):
|
||||
username: StrictStr = Field(pattern=r"^[a-z0-9_]{3,32}$")
|
||||
display_name: StrictStr = Field(min_length=1, max_length=100)
|
||||
password: StrictStr = Field(min_length=12, max_length=128)
|
||||
|
||||
@field_validator("username")
|
||||
@classmethod
|
||||
def full_username(cls, value):
|
||||
if not re.fullmatch(r"[a-z0-9_]{3,32}", value):
|
||||
raise ValueError("Неверное имя пользователя")
|
||||
return value
|
||||
|
||||
|
||||
class CreateChat(RequestModel):
|
||||
title: StrictStr = Field(min_length=1, max_length=100)
|
||||
member_ids: list[UUID] = Field(min_length=1, max_length=100)
|
||||
|
||||
@field_validator("member_ids", mode="before")
|
||||
@classmethod
|
||||
def string_ids(cls, value):
|
||||
if not isinstance(value, list) or any(not isinstance(x, str) for x in value):
|
||||
raise ValueError("Ожидается массив UUID-строк")
|
||||
return value
|
||||
|
||||
@field_validator("member_ids")
|
||||
@classmethod
|
||||
def unique_ids(cls, value):
|
||||
if len(set(value)) != len(value):
|
||||
raise ValueError("Участники не должны повторяться")
|
||||
return value
|
||||
|
||||
|
||||
class CreateMessage(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
text: StrictStr = Field(max_length=4000)
|
||||
attachment_ids: list[UUID] = Field(default_factory=list, max_length=10)
|
||||
|
||||
@field_validator("attachment_ids", mode="before")
|
||||
@classmethod
|
||||
def string_ids(cls, value):
|
||||
if not isinstance(value, list) or any(not isinstance(x, str) for x in value):
|
||||
raise ValueError("Ожидается массив UUID-строк")
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def content(self):
|
||||
self.text.encode("utf-8")
|
||||
if len(set(self.attachment_ids)) != len(self.attachment_ids):
|
||||
raise ValueError("Вложения не должны повторяться")
|
||||
if not self.text.strip() and not self.attachment_ids:
|
||||
raise ValueError("Пустое сообщение без вложений")
|
||||
return self
|
||||
@@ -0,0 +1,365 @@
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import secrets
|
||||
from contextlib import contextmanager
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from threading import BoundedSemaphore
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import psycopg
|
||||
import redis
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import VerificationError
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool, PoolTimeout, TooManyRequests
|
||||
|
||||
from .config import secret
|
||||
from .domain import Failure, timestamp
|
||||
from .secrets_provider import refresh
|
||||
from .telemetry import operation
|
||||
|
||||
logger = logging.getLogger("messenger")
|
||||
KDF_SLOTS = BoundedSemaphore(2)
|
||||
KDF = PasswordHasher(time_cost=2, memory_cost=65536, parallelism=1)
|
||||
DUMMY_HASH = KDF.hash("invalid-user-dummy-password")
|
||||
|
||||
|
||||
class DeadlineConnection(psycopg.Connection):
|
||||
def wait(self, gen, interval=0.1, timeout=None):
|
||||
try:
|
||||
return super().wait(gen, interval=interval, timeout=timeout or 3.0)
|
||||
except psycopg.errors._WaitTimeout:
|
||||
# A paused server cannot complete cancellation. Discard an uncertain
|
||||
# connection instead of returning it to the pool or waiting forever.
|
||||
self.pgconn.finish()
|
||||
raise
|
||||
|
||||
|
||||
def public(row):
|
||||
return {
|
||||
k: (
|
||||
str(v)
|
||||
if isinstance(v, UUID)
|
||||
else v.astimezone(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
if isinstance(v, datetime)
|
||||
else bytes(v).decode("utf-8")
|
||||
if isinstance(v, (bytes, memoryview))
|
||||
else v
|
||||
)
|
||||
for k, v in row.items()
|
||||
if k not in {"password_hash", "token_digest"}
|
||||
}
|
||||
|
||||
|
||||
class PostgresStore:
|
||||
def __init__(self):
|
||||
self.pool = ConnectionPool(
|
||||
secret("DATABASE_URL"),
|
||||
connection_class=DeadlineConnection,
|
||||
min_size=1,
|
||||
max_size=int(os.getenv("POOL_MAX_SIZE", "8")),
|
||||
timeout=2,
|
||||
max_waiting=32,
|
||||
kwargs={
|
||||
"autocommit": True,
|
||||
"row_factory": dict_row,
|
||||
"connect_timeout": 2,
|
||||
"options": "-c statement_timeout=2500 -c lock_timeout=1500",
|
||||
},
|
||||
check=ConnectionPool.check_connection,
|
||||
)
|
||||
self.pool.wait(timeout=15)
|
||||
self.cache = redis.Redis.from_url(
|
||||
secret("CACHE_URL"),
|
||||
socket_connect_timeout=0.2,
|
||||
socket_timeout=0.2,
|
||||
decode_responses=True,
|
||||
)
|
||||
self.ttl = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
|
||||
if self.ttl < 1:
|
||||
raise RuntimeError("SESSION_TTL_SECONDS должен быть положительным")
|
||||
|
||||
@contextmanager
|
||||
def connection(self):
|
||||
refresh()
|
||||
try:
|
||||
with operation("postgres.transaction", {"db.system.name": "postgresql"}):
|
||||
with self.pool.connection() as conn:
|
||||
yield conn
|
||||
except (
|
||||
psycopg.OperationalError,
|
||||
psycopg.InterfaceError,
|
||||
PoolTimeout,
|
||||
TooManyRequests,
|
||||
psycopg.errors.QueryCanceled,
|
||||
psycopg.errors.LockNotAvailable,
|
||||
):
|
||||
raise Failure(503, "Хранилище временно недоступно") from None
|
||||
|
||||
def close(self):
|
||||
self.cache.close()
|
||||
self.pool.close()
|
||||
|
||||
def ready(self):
|
||||
try:
|
||||
with self.connection() as conn:
|
||||
conn.execute(
|
||||
"SELECT u.id,u.password_hash,c.id,m.chat_id,msg.id,s.token_digest,r.key,"
|
||||
"a.id,a.stage_key,a.pipeline_version,a.variants,ma.attachment_id,ui.object_key,"
|
||||
"j.id,j.source_key,j.attempt_limit,j.lease_until,j.traceparent,j.tracestate,j.accepted_at,o.job_id,o.published_at,"
|
||||
"audit.job_id,f.error_code FROM users u,chats c,members m,messages msg,"
|
||||
"sessions s,message_requests r,attachments a,message_attachments ma,"
|
||||
"upload_intents ui,attachment_jobs j,job_outbox o,job_audit audit,failed_jobs f "
|
||||
"WHERE false"
|
||||
)
|
||||
return True
|
||||
except (Failure, psycopg.Error):
|
||||
return False
|
||||
|
||||
def _cached_user(self, uid):
|
||||
key = "user:v1:" + uid
|
||||
try:
|
||||
value = self.cache.get(key)
|
||||
if value:
|
||||
cached = json.loads(value)
|
||||
if (
|
||||
isinstance(cached, dict)
|
||||
and cached.get("id") == uid
|
||||
and set(cached) == {"id", "username", "display_name", "created_at"}
|
||||
):
|
||||
return cached
|
||||
except (redis.RedisError, ValueError, TypeError):
|
||||
logger.warning(json.dumps({"event": "cache_bypass"}))
|
||||
user = self.user(uid)
|
||||
if user:
|
||||
try:
|
||||
self.cache.setex(key, 60, json.dumps(user))
|
||||
except redis.RedisError:
|
||||
logger.warning(json.dumps({"event": "cache_bypass"}))
|
||||
return user
|
||||
|
||||
def user(self, uid):
|
||||
with self.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT id,username,display_name,created_at FROM users WHERE id=%s", (uid,)
|
||||
).fetchone()
|
||||
return public(row) if row else None
|
||||
|
||||
def register(self, data):
|
||||
with KDF_SLOTS:
|
||||
hashed = KDF.hash(data.pop("password"))
|
||||
uid = str(uuid4())
|
||||
try:
|
||||
with self.connection() as conn:
|
||||
row = conn.execute(
|
||||
"INSERT INTO users VALUES (%s,%s,%s,%s,%s) "
|
||||
"RETURNING id,username,display_name,created_at",
|
||||
(
|
||||
uid,
|
||||
data["username"],
|
||||
data["display_name"].encode("utf-8"),
|
||||
hashed,
|
||||
timestamp(),
|
||||
),
|
||||
).fetchone()
|
||||
return public(row)
|
||||
except psycopg.errors.UniqueViolation:
|
||||
raise Failure(409, "Имя пользователя занято") from None
|
||||
|
||||
def login(self, username, password):
|
||||
self.login_budget(username)
|
||||
with self.connection() as conn:
|
||||
row = conn.execute("SELECT * FROM users WHERE username=%s", (username,)).fetchone()
|
||||
try:
|
||||
if not KDF_SLOTS.acquire(blocking=False):
|
||||
raise Failure(429, "Слишком много попыток входа", {"Retry-After": "1"})
|
||||
try:
|
||||
KDF.verify(row["password_hash"] if row else DUMMY_HASH, password)
|
||||
finally:
|
||||
KDF_SLOTS.release()
|
||||
except VerificationError:
|
||||
return None
|
||||
if row is None:
|
||||
return None
|
||||
if KDF.check_needs_rehash(row["password_hash"]):
|
||||
with KDF_SLOTS:
|
||||
upgraded = KDF.hash(password)
|
||||
with self.connection() as conn:
|
||||
conn.execute(
|
||||
"UPDATE users SET password_hash=%s WHERE id=%s AND password_hash=%s",
|
||||
(upgraded, row["id"], row["password_hash"]),
|
||||
)
|
||||
token = secrets.token_urlsafe(32)
|
||||
digest = hashlib.sha256(token.encode()).hexdigest()
|
||||
expires = datetime.now(UTC) + timedelta(seconds=self.ttl)
|
||||
with self.connection() as conn:
|
||||
conn.execute("INSERT INTO sessions VALUES (%s,%s,%s)", (digest, row["id"], expires))
|
||||
return {
|
||||
"token": token,
|
||||
"token_type": "Bearer",
|
||||
"expires_at": public({"v": expires})["v"],
|
||||
"user": public(row),
|
||||
}
|
||||
|
||||
def login_budget(self, username):
|
||||
key = hashlib.sha256(username.encode("utf-8")).hexdigest()
|
||||
with self.connection() as conn, conn.transaction():
|
||||
# Shared PostgreSQL window, serialized across both API replicas.
|
||||
conn.execute("SELECT pg_advisory_xact_lock(2301007)")
|
||||
conn.execute("DELETE FROM login_windows WHERE window_start<now()-interval '1 hour'")
|
||||
for bucket, maximum in (("global", 30), ("user:" + key, 10)):
|
||||
count = conn.execute(
|
||||
"INSERT INTO login_windows(bucket,window_start,count) VALUES (%s,now(),1) "
|
||||
"ON CONFLICT(bucket) DO UPDATE SET count=CASE WHEN login_windows.window_start<now()-interval '10 seconds' THEN 1 ELSE login_windows.count+1 END, "
|
||||
"window_start=CASE WHEN login_windows.window_start<now()-interval '10 seconds' THEN now() ELSE login_windows.window_start END RETURNING count",
|
||||
(bucket,),
|
||||
).fetchone()["count"]
|
||||
if count > maximum:
|
||||
raise Failure(429, "Слишком много попыток входа", {"Retry-After": "10"})
|
||||
|
||||
def authenticate(self, token):
|
||||
if not token or len(token) > 512:
|
||||
return None
|
||||
digest = hashlib.sha256(token.encode()).hexdigest()
|
||||
# Session validity always comes from PostgreSQL, including on a cache hit.
|
||||
with self.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT s.user_id FROM sessions s WHERE s.token_digest=%s AND s.expires_at>now()",
|
||||
(digest,),
|
||||
).fetchone()
|
||||
return self._cached_user(str(row["user_id"])) if row else None
|
||||
|
||||
def logout(self, token):
|
||||
with self.connection() as conn:
|
||||
row = conn.execute(
|
||||
"DELETE FROM sessions WHERE token_digest=%s AND expires_at>now() RETURNING user_id",
|
||||
(hashlib.sha256(token.encode()).hexdigest(),),
|
||||
).fetchone()
|
||||
return row is not None
|
||||
|
||||
def create_chat(self, actor, data):
|
||||
if actor not in data["member_ids"]:
|
||||
raise Failure(400, "Вызывающий должен входить в чат")
|
||||
cid = str(uuid4())
|
||||
try:
|
||||
with self.connection() as conn, conn.transaction():
|
||||
row = conn.execute(
|
||||
"INSERT INTO chats VALUES (%s,%s,%s) RETURNING *",
|
||||
(cid, data["title"].encode("utf-8"), timestamp()),
|
||||
).fetchone()
|
||||
for index, uid in enumerate(data["member_ids"]):
|
||||
conn.execute("INSERT INTO members VALUES (%s,%s,%s)", (cid, uid, index))
|
||||
return {**public(row), "member_ids": data["member_ids"]}
|
||||
except psycopg.errors.ForeignKeyViolation:
|
||||
raise Failure(400, "Неизвестный участник") from None
|
||||
|
||||
@staticmethod
|
||||
def _chat(conn, actor, cid):
|
||||
row = conn.execute(
|
||||
"SELECT c.* FROM chats c JOIN members m ON m.chat_id=c.id "
|
||||
"WHERE c.id=%s AND m.user_id=%s",
|
||||
(cid, actor),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise Failure(404, "Объект не найден")
|
||||
members = conn.execute(
|
||||
"SELECT user_id FROM members WHERE chat_id=%s ORDER BY position", (cid,)
|
||||
).fetchall()
|
||||
return {**public(row), "member_ids": [str(m["user_id"]) for m in members]}
|
||||
|
||||
def chat(self, actor, cid):
|
||||
with self.connection() as conn:
|
||||
return self._chat(conn, actor, cid)
|
||||
|
||||
def list_chats(self, actor, after, limit):
|
||||
with self.connection() as conn:
|
||||
sql = "SELECT c.* FROM chats c JOIN members m ON m.chat_id=c.id WHERE m.user_id=%s"
|
||||
params = [actor]
|
||||
if after:
|
||||
sql += " AND (c.created_at,c.id)>(%s,%s)"
|
||||
params.extend(after)
|
||||
sql += " ORDER BY c.created_at,c.id LIMIT %s"
|
||||
params.append(limit + 1)
|
||||
rows = conn.execute(sql, params).fetchall()
|
||||
return [self._chat(conn, actor, str(row["id"])) for row in rows]
|
||||
|
||||
def send(self, actor, cid, text, key, attachment_ids=None):
|
||||
attachment_ids = attachment_ids or []
|
||||
body_digest = hashlib.sha256(
|
||||
json.dumps(
|
||||
{"text": text, "attachment_ids": attachment_ids},
|
||||
sort_keys=True,
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
).encode()
|
||||
).hexdigest()
|
||||
with self.connection() as conn, conn.transaction():
|
||||
self._chat(conn, actor, cid)
|
||||
# Serializes exactly this semantic key across all API processes. Hash
|
||||
# collisions can only serialize unrelated requests, never mix results.
|
||||
scope = json.dumps([actor, "POST", cid, key], separators=(",", ":"))
|
||||
lock = int.from_bytes(hashlib.sha256(scope.encode()).digest()[:8], "big", signed=True)
|
||||
conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock,))
|
||||
old = conn.execute(
|
||||
"SELECT body_digest,response FROM message_requests "
|
||||
"WHERE actor_id=%s AND chat_id=%s AND key=%s AND expires_at>now()",
|
||||
(actor, cid, key),
|
||||
).fetchone()
|
||||
if old:
|
||||
if old["body_digest"] != body_digest:
|
||||
raise Failure(409, "Ключ уже использован для другого сообщения")
|
||||
return json.loads(bytes(old["response"]))
|
||||
conn.execute(
|
||||
"DELETE FROM message_requests WHERE actor_id=%s AND chat_id=%s AND key=%s",
|
||||
(actor, cid, key),
|
||||
)
|
||||
for aid in attachment_ids:
|
||||
attachment = conn.execute(
|
||||
"SELECT id,status FROM attachments WHERE id=%s AND chat_id=%s FOR SHARE",
|
||||
(aid, cid),
|
||||
).fetchone()
|
||||
if not attachment:
|
||||
raise Failure(404, "Объект не найден")
|
||||
if attachment["status"] == "failed":
|
||||
raise Failure(409, "Вложение недоступно")
|
||||
row = conn.execute(
|
||||
"INSERT INTO messages VALUES (%s,%s,%s,%s,%s) RETURNING *",
|
||||
(str(uuid4()), cid, actor, text.encode("utf-8"), timestamp()),
|
||||
).fetchone()
|
||||
for position, aid in enumerate(attachment_ids):
|
||||
conn.execute(
|
||||
"INSERT INTO message_attachments VALUES (%s,%s,%s)", (row["id"], aid, position)
|
||||
)
|
||||
response = {**public(row), "attachment_ids": attachment_ids}
|
||||
conn.execute(
|
||||
"INSERT INTO message_requests(actor_id,chat_id,key,body_digest,response,expires_at) "
|
||||
"VALUES (%s,%s,%s,%s,%s,now()+interval '24 hours')",
|
||||
(actor, cid, key, body_digest, json.dumps(response).encode("utf-8")),
|
||||
)
|
||||
return response
|
||||
|
||||
def history(self, actor, cid, after, limit):
|
||||
with self.connection() as conn:
|
||||
self._chat(conn, actor, cid)
|
||||
sql = "SELECT * FROM messages WHERE chat_id=%s"
|
||||
params = [cid]
|
||||
if after:
|
||||
sql += " AND (created_at,id)>(%s,%s)"
|
||||
params.extend(after)
|
||||
sql += " ORDER BY created_at,id LIMIT %s"
|
||||
params.append(limit + 1)
|
||||
result = []
|
||||
for row in conn.execute(sql, params).fetchall():
|
||||
attachments = conn.execute(
|
||||
"SELECT attachment_id FROM message_attachments WHERE message_id=%s ORDER BY position",
|
||||
(row["id"],),
|
||||
).fetchall()
|
||||
result.append(
|
||||
{
|
||||
**public(row),
|
||||
"attachment_ids": [str(a["attachment_id"]) for a in attachments],
|
||||
}
|
||||
)
|
||||
return result
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Fresh connections with deliberately revoked worker material; no fallback."""
|
||||
|
||||
import json
|
||||
import ssl
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
import redis
|
||||
|
||||
|
||||
def main():
|
||||
old = json.loads(Path("/run/old-worker/material.json").read_text())
|
||||
checks = {}
|
||||
try:
|
||||
conn = psycopg.connect(old["DATABASE_URL"], connect_timeout=2)
|
||||
conn.close()
|
||||
checks["old_database_password_denied"] = False
|
||||
except psycopg.OperationalError:
|
||||
checks["old_database_password_denied"] = True
|
||||
url = old["BROKER_URL"].replace("/run/tls/identity/", "/run/old-worker/")
|
||||
broker = redis.Redis.from_url(url, socket_connect_timeout=2, socket_timeout=2)
|
||||
try:
|
||||
broker.ping()
|
||||
checks["old_broker_client_certificate_denied"] = False
|
||||
except redis.RedisError:
|
||||
checks["old_broker_client_certificate_denied"] = True
|
||||
finally:
|
||||
broker.close()
|
||||
ctx = ssl.create_default_context(cafile="/run/tls/ca.pem")
|
||||
ctx.load_cert_chain("/run/old-worker/cert.pem", "/run/old-worker/key.pem")
|
||||
for name, url, body in (
|
||||
("old_s3_certificate_denied", "https://storage:8443/messenger/staging/probe", None),
|
||||
(
|
||||
"old_bao_certificate_auth_denied",
|
||||
"https://openbao:8200/v1/auth/cert/login",
|
||||
b'{"name":"worker"}',
|
||||
),
|
||||
):
|
||||
try:
|
||||
with urllib.request.urlopen(
|
||||
urllib.request.Request(
|
||||
url, data=body, headers={"Content-Type": "application/json"}
|
||||
),
|
||||
context=ctx,
|
||||
timeout=3,
|
||||
):
|
||||
checks[name] = False
|
||||
except urllib.error.HTTPError as exc:
|
||||
checks[name] = exc.code in {400, 403}
|
||||
except (OSError, urllib.error.URLError):
|
||||
checks[name] = True
|
||||
print(json.dumps({"checks": checks, "passed": all(checks.values())}))
|
||||
if not all(checks.values()):
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Short-lived OpenBao certificate authentication; no root token or SecretID in app."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import ssl
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
from .domain import Failure
|
||||
|
||||
_lock = threading.Lock()
|
||||
_values = {}
|
||||
_valid_until = 0.0
|
||||
|
||||
|
||||
def request(path, *, data=None, token=None):
|
||||
context = ssl.create_default_context(cafile=os.getenv("BAO_CA_FILE", "/run/tls/ca.pem"))
|
||||
context.load_cert_chain("/run/tls/identity/cert.pem", "/run/tls/identity/key.pem")
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if token:
|
||||
headers["X-Vault-Token"] = token
|
||||
payload = json.dumps(data).encode() if data is not None else None
|
||||
req = urllib.request.Request(
|
||||
os.environ["BAO_ADDR"] + "/v1/" + path, data=payload, headers=headers
|
||||
)
|
||||
with urllib.request.urlopen(req, context=context, timeout=2) as response:
|
||||
return json.load(response)
|
||||
|
||||
|
||||
def refresh():
|
||||
global _values, _valid_until
|
||||
if not os.getenv("BAO_ROLE"):
|
||||
return
|
||||
with _lock:
|
||||
if time.monotonic() < _valid_until:
|
||||
return
|
||||
try:
|
||||
role = os.environ["BAO_ROLE"]
|
||||
auth = request("auth/cert/login", data={"name": role})["auth"]
|
||||
values = request("secret/data/services/" + role, token=auth["client_token"])["data"][
|
||||
"data"
|
||||
]
|
||||
# Never keep credentials usable beyond the auth token's short lease.
|
||||
_values = values
|
||||
_valid_until = time.monotonic() + min(float(auth["lease_duration"]), 30) - 1
|
||||
except (OSError, ValueError, KeyError, urllib.error.URLError):
|
||||
_values = {}
|
||||
_valid_until = 0
|
||||
raise Failure(503, "Сервис секретов временно недоступен") from None
|
||||
|
||||
|
||||
def value(name):
|
||||
refresh()
|
||||
try:
|
||||
return _values[name]
|
||||
except KeyError:
|
||||
raise RuntimeError("Scoped secret unavailable: " + name) from None
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Run API TLS boundary and loopback business server in one namespace."""
|
||||
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
|
||||
def main():
|
||||
children = [subprocess.Popen([sys.executable, "-m", "messenger.tls_gateway"])]
|
||||
env = dict(os.environ, APP_HOST="127.0.0.1")
|
||||
children.append(subprocess.Popen([sys.executable, "-m", "messenger"], env=env))
|
||||
|
||||
def stop(*_):
|
||||
for child in children:
|
||||
child.terminate()
|
||||
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
signal.signal(signal.SIGINT, stop)
|
||||
try:
|
||||
while all(child.poll() is None for child in children):
|
||||
time.sleep(0.2)
|
||||
finally:
|
||||
stop()
|
||||
for child in children:
|
||||
try:
|
||||
child.wait(timeout=6)
|
||||
except subprocess.TimeoutExpired:
|
||||
child.kill()
|
||||
child.wait()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,108 @@
|
||||
"""Read-only/rollback permission probes plus disposable synthetic prefix objects."""
|
||||
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
|
||||
import psycopg
|
||||
import redis
|
||||
|
||||
from .domain import Failure
|
||||
from .jobs import STREAM, broker_client
|
||||
from .postgres import PostgresStore
|
||||
from .secrets_provider import request
|
||||
from .storage import ObjectStorage
|
||||
|
||||
|
||||
def denied(action):
|
||||
try:
|
||||
action()
|
||||
except (psycopg.errors.InsufficientPrivilege, redis.ResponseError, Failure):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
checks = {}
|
||||
store = PostgresStore()
|
||||
broker = broker_client()
|
||||
storage = ObjectStorage()
|
||||
try:
|
||||
with store.connection() as conn:
|
||||
checks["db_identity_worker"] = (
|
||||
conn.execute("SELECT current_user AS role").fetchone()["role"] == "worker"
|
||||
)
|
||||
checks["db_business_users_denied"] = denied(
|
||||
lambda: conn.execute("SELECT password_hash FROM users LIMIT 0")
|
||||
)
|
||||
checks["db_business_messages_denied"] = denied(
|
||||
lambda: conn.execute("SELECT text FROM messages LIMIT 0")
|
||||
)
|
||||
checks["db_create_denied"] = denied(
|
||||
lambda: conn.execute("CREATE TABLE security_forbidden(id int)")
|
||||
)
|
||||
checks["db_owner_mutation_denied"] = denied(
|
||||
lambda: conn.execute("UPDATE attachments SET owner_id=owner_id WHERE false")
|
||||
)
|
||||
checks["db_job_read_allowed"] = (
|
||||
conn.execute("SELECT count(*) AS n FROM attachment_jobs").fetchone()["n"] >= 0
|
||||
)
|
||||
checks["broker_ping_allowed"] = broker.ping()
|
||||
checks["broker_acl_admin_denied"] = denied(lambda: broker.acl_list())
|
||||
checks["broker_publish_denied"] = denied(lambda: broker.xadd(STREAM, {"body": "forbidden"}))
|
||||
checks["broker_wrong_stream_denied"] = denied(lambda: broker.xlen("other-stream"))
|
||||
checks["broker_group_destroy_denied"] = denied(
|
||||
lambda: broker.xgroup_destroy(STREAM, "nonexistent-security-probe")
|
||||
)
|
||||
key = "attachments/security-probe/v1/original"
|
||||
storage.put(key, io.BytesIO(b"synthetic"), 9, "application/octet-stream")
|
||||
checks["s3_results_prefix_write_allowed"] = storage.head(key)["ContentLength"] == 9
|
||||
checks["s3_staging_write_denied"] = denied(
|
||||
lambda: storage.put("staging/security-forbidden", io.BytesIO(b"x"), 1, "text/plain")
|
||||
)
|
||||
checks["s3_private_prefix_read_denied"] = denied(
|
||||
lambda: storage.get("private/security-probe")
|
||||
)
|
||||
auth = request("auth/cert/login", data={"name": "worker"})["auth"]["client_token"]
|
||||
checks["bao_own_path_allowed"] = bool(
|
||||
request("secret/data/services/worker", token=auth)["data"]["data"]
|
||||
)
|
||||
import urllib.error
|
||||
|
||||
for name, path, data in (
|
||||
("bao_other_path_denied", "secret/data/services/api", None),
|
||||
("bao_policy_write_denied", "sys/policies/acl/forbidden", {"policy": ""}),
|
||||
):
|
||||
try:
|
||||
request(path, data=data, token=auth)
|
||||
checks[name] = False
|
||||
except urllib.error.HTTPError as exc:
|
||||
checks[name] = exc.code == 403
|
||||
for host, port in (
|
||||
("storage", 8333),
|
||||
("storage", 9333),
|
||||
("storage", 8888),
|
||||
("broker", 6379),
|
||||
("cache", 6379),
|
||||
("api", 8080),
|
||||
):
|
||||
try:
|
||||
with socket.create_connection((host, port), 1):
|
||||
checks[f"no_plaintext_bypass_{host}_{port}"] = False
|
||||
except OSError:
|
||||
checks[f"no_plaintext_bypass_{host}_{port}"] = True
|
||||
checks["worker_image_has_no_root_token"] = not os.path.exists(
|
||||
"/operator-private/root-token"
|
||||
)
|
||||
print(json.dumps({"checks": checks, "passed": all(checks.values())}, sort_keys=True))
|
||||
if not all(checks.values()):
|
||||
raise SystemExit(1)
|
||||
finally:
|
||||
storage.close()
|
||||
broker.close()
|
||||
store.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Private S3 access with bounded network waits; never expose storage keys in API."""
|
||||
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from .config import secret
|
||||
from .domain import Failure
|
||||
from .secrets_provider import refresh
|
||||
from .telemetry import operation
|
||||
|
||||
|
||||
class ObjectStorage:
|
||||
def __init__(self, admin=False):
|
||||
self.bucket = os.getenv("S3_BUCKET", "messenger")
|
||||
prefix = "S3_ADMIN" if admin else "S3"
|
||||
self.client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=os.getenv("S3_ENDPOINT", "http://s3:8333"),
|
||||
aws_access_key_id=secret(prefix + "_ACCESS_KEY"),
|
||||
aws_secret_access_key=secret(prefix + "_SECRET_KEY"),
|
||||
region_name="us-east-1",
|
||||
verify=os.getenv("S3_CA_FILE", "/run/tls/ca.pem"),
|
||||
config=Config(
|
||||
client_cert=("/run/tls/identity/cert.pem", "/run/tls/identity/key.pem"),
|
||||
connect_timeout=1,
|
||||
read_timeout=2,
|
||||
retries={"total_max_attempts": 1},
|
||||
s3={"addressing_style": "path"},
|
||||
request_checksum_calculation="when_required",
|
||||
response_checksum_validation="when_required",
|
||||
),
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def boundary(self):
|
||||
refresh()
|
||||
try:
|
||||
with operation("s3.request", {"rpc.system": "aws-api", "rpc.service": "S3"}):
|
||||
yield
|
||||
except (BotoCoreError, ClientError):
|
||||
raise Failure(503, "Хранилище файлов временно недоступно") from None
|
||||
|
||||
def put(self, key, fileobj, size, content_type):
|
||||
fileobj.seek(0)
|
||||
with self.boundary():
|
||||
self.client.put_object(
|
||||
Bucket=self.bucket,
|
||||
Key=key,
|
||||
Body=fileobj,
|
||||
ContentLength=size,
|
||||
ContentType=content_type,
|
||||
)
|
||||
|
||||
def get(self, key):
|
||||
refresh()
|
||||
try:
|
||||
with operation("s3.get", {"rpc.system": "aws-api", "rpc.service": "S3"}):
|
||||
return self.client.get_object(Bucket=self.bucket, Key=key)
|
||||
except ClientError as exc:
|
||||
if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404"}:
|
||||
raise Failure(404, "Объект не найден") from None
|
||||
raise Failure(503, "Хранилище файлов временно недоступно") from None
|
||||
except BotoCoreError:
|
||||
raise Failure(503, "Хранилище файлов временно недоступно") from None
|
||||
|
||||
def head(self, key):
|
||||
refresh()
|
||||
try:
|
||||
with operation("s3.head", {"rpc.system": "aws-api", "rpc.service": "S3"}):
|
||||
return self.client.head_object(Bucket=self.bucket, Key=key)
|
||||
except ClientError as exc:
|
||||
if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404", "NotFound"}:
|
||||
return None
|
||||
raise Failure(503, "Хранилище файлов временно недоступно") from None
|
||||
except BotoCoreError:
|
||||
raise Failure(503, "Хранилище файлов временно недоступно") from None
|
||||
|
||||
def exists(self, key):
|
||||
return self.head(key) is not None
|
||||
|
||||
def delete(self, key):
|
||||
with self.boundary():
|
||||
self.client.delete_object(Bucket=self.bucket, Key=key)
|
||||
|
||||
def objects(self, page_size=100):
|
||||
with self.boundary():
|
||||
paginator = self.client.get_paginator("list_objects_v2")
|
||||
for page in paginator.paginate(
|
||||
Bucket=self.bucket, PaginationConfig={"PageSize": page_size}
|
||||
):
|
||||
yield from page.get("Contents", [])
|
||||
|
||||
def ready(self):
|
||||
try:
|
||||
with self.boundary():
|
||||
self.client.head_bucket(Bucket=self.bucket)
|
||||
return True
|
||||
except Failure:
|
||||
return False
|
||||
|
||||
def close(self):
|
||||
self.client.close()
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Idempotent private bucket bootstrap; this process alone receives admin keys."""
|
||||
|
||||
import time
|
||||
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from .storage import ObjectStorage
|
||||
|
||||
|
||||
def main():
|
||||
storage = ObjectStorage(admin=True)
|
||||
for attempt in range(30):
|
||||
try:
|
||||
try:
|
||||
storage.client.head_bucket(Bucket=storage.bucket)
|
||||
except ClientError as exc:
|
||||
if exc.response["Error"]["Code"] not in {"404", "NoSuchBucket", "NotFound"}:
|
||||
raise
|
||||
storage.client.create_bucket(Bucket=storage.bucket)
|
||||
print("Private bucket ready; anonymous identity is not configured.")
|
||||
storage.close()
|
||||
return
|
||||
except (ClientError, BotoCoreError):
|
||||
if attempt == 29:
|
||||
raise RuntimeError("S3 bootstrap timeout") from None
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,357 @@
|
||||
"""Bounded OTLP telemetry; technical attributes only, no payload/credential capture."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from contextlib import contextmanager
|
||||
from datetime import UTC, datetime
|
||||
from uuid import uuid4
|
||||
|
||||
import psutil
|
||||
import requests
|
||||
from opentelemetry import metrics, propagate, trace
|
||||
from opentelemetry._logs import set_logger_provider
|
||||
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
||||
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import SpanProcessor, TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExportResult
|
||||
from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
|
||||
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||
|
||||
HTTP_BUCKETS = (0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10)
|
||||
JOB_BUCKETS = (0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 20, 45, 60, 120)
|
||||
INSTANCE_ID = str(uuid4())
|
||||
_runtime = None
|
||||
|
||||
|
||||
class Accounting(dict):
|
||||
def __init__(self, keys):
|
||||
super().__init__((key, 0) for key in keys)
|
||||
self.lock = threading.Lock()
|
||||
|
||||
def add(self, key, value):
|
||||
with self.lock:
|
||||
self[key] += value
|
||||
|
||||
def snapshot(self):
|
||||
with self.lock:
|
||||
return dict(self)
|
||||
|
||||
|
||||
class RecordedSpans(SpanProcessor):
|
||||
def __init__(self, stats):
|
||||
self.stats = stats
|
||||
|
||||
def on_start(self, span, parent_context=None):
|
||||
pass
|
||||
|
||||
def on_end(self, span):
|
||||
if span.context.trace_flags.sampled:
|
||||
self.stats.add("spans_recorded", 1)
|
||||
|
||||
|
||||
class CountingSession(requests.Session):
|
||||
"""Count actual protobuf HTTP sends, including transport retries, without retaining data."""
|
||||
|
||||
def __init__(self, stats, signal):
|
||||
super().__init__()
|
||||
self.stats, self.signal = stats, signal
|
||||
|
||||
def send(self, request, **kwargs):
|
||||
self.stats.add(self.signal + "_wire_bytes", len(request.body or b""))
|
||||
self.stats.add(self.signal + "_http_sends", 1)
|
||||
return super().send(request, **kwargs)
|
||||
|
||||
|
||||
class ObservedSpanExporter(OTLPSpanExporter):
|
||||
def __init__(self, stats, **kwargs):
|
||||
self.stats = stats
|
||||
super().__init__(**kwargs)
|
||||
|
||||
def export(self, spans):
|
||||
self.stats.add("spans_in_flight", len(spans))
|
||||
try:
|
||||
result = super().export(spans)
|
||||
key = "spans_exported" if result == SpanExportResult.SUCCESS else "spans_export_failed"
|
||||
self.stats.add(key, len(spans))
|
||||
return result
|
||||
except Exception:
|
||||
self.stats.add("spans_export_failed", len(spans))
|
||||
raise
|
||||
finally:
|
||||
self.stats.add("spans_in_flight", -len(spans))
|
||||
|
||||
|
||||
class TechnicalLogs(logging.Filter):
|
||||
def filter(self, record):
|
||||
return record.name.startswith("messenger")
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
def format(self, record):
|
||||
try:
|
||||
event = json.loads(record.getMessage())
|
||||
if not isinstance(event, dict):
|
||||
event = {"event": "application_event"}
|
||||
except ValueError:
|
||||
event = {"event": "application_event"}
|
||||
context = trace.get_current_span().get_span_context()
|
||||
return json.dumps(
|
||||
{
|
||||
**event,
|
||||
"timestamp": datetime.now(UTC).isoformat(),
|
||||
"severity": record.levelname,
|
||||
"service.name": os.getenv("OTEL_SERVICE_NAME", "messenger-api"),
|
||||
"trace_id": format(context.trace_id, "032x") if context.is_valid else None,
|
||||
"span_id": format(context.span_id, "016x") if context.is_valid else None,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class Telemetry:
|
||||
def __init__(self, service):
|
||||
self.service = service
|
||||
self.enabled = os.getenv("OTEL_ENABLED", "1") == "1"
|
||||
self.busy = 0
|
||||
self.providers = []
|
||||
self.export_stats = Accounting(
|
||||
(
|
||||
"spans_recorded",
|
||||
"spans_in_flight",
|
||||
"spans_exported",
|
||||
"spans_export_failed",
|
||||
"traces_wire_bytes",
|
||||
"traces_http_sends",
|
||||
"metrics_wire_bytes",
|
||||
"metrics_http_sends",
|
||||
"logs_wire_bytes",
|
||||
"logs_http_sends",
|
||||
)
|
||||
)
|
||||
self.stop = threading.Event()
|
||||
self.thread = None
|
||||
resource = Resource.create(
|
||||
{
|
||||
"service.name": service,
|
||||
"service.version": os.getenv("SERVICE_VERSION", "lab7-v1"),
|
||||
"service.instance.id": INSTANCE_ID,
|
||||
"deployment.environment.name": "lab",
|
||||
}
|
||||
)
|
||||
os.environ["OTEL_SERVICE_NAME"] = service
|
||||
root = logging.getLogger()
|
||||
root.setLevel(logging.INFO)
|
||||
for handler in root.handlers:
|
||||
handler.setFormatter(JsonFormatter())
|
||||
if not root.handlers:
|
||||
handler = logging.StreamHandler()
|
||||
handler.setFormatter(JsonFormatter())
|
||||
root.addHandler(handler)
|
||||
if self.enabled:
|
||||
endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector:4318").rstrip("/")
|
||||
common = {"timeout": 1.0}
|
||||
# Standard OTLP environment variables also support HTTPS/mTLS in lab7.
|
||||
trace_provider = TracerProvider(
|
||||
resource=resource,
|
||||
sampler=ParentBased(TraceIdRatioBased(float(os.getenv("OTEL_SAMPLE_RATIO", "1")))),
|
||||
)
|
||||
trace_provider.add_span_processor(RecordedSpans(self.export_stats))
|
||||
self.batch_spans = BatchSpanProcessor(
|
||||
ObservedSpanExporter(
|
||||
self.export_stats,
|
||||
endpoint=endpoint + "/v1/traces",
|
||||
session=CountingSession(self.export_stats, "traces"),
|
||||
**common,
|
||||
),
|
||||
max_queue_size=256,
|
||||
max_export_batch_size=64,
|
||||
schedule_delay_millis=500,
|
||||
export_timeout_millis=1500,
|
||||
)
|
||||
trace_provider.add_span_processor(self.batch_spans)
|
||||
trace.set_tracer_provider(trace_provider)
|
||||
reader = PeriodicExportingMetricReader(
|
||||
OTLPMetricExporter(
|
||||
endpoint=endpoint + "/v1/metrics",
|
||||
session=CountingSession(self.export_stats, "metrics"),
|
||||
**common,
|
||||
),
|
||||
export_interval_millis=int(os.getenv("OTEL_METRIC_INTERVAL_MS", "5000")),
|
||||
export_timeout_millis=1500,
|
||||
)
|
||||
meter_provider = MeterProvider(
|
||||
resource=resource,
|
||||
metric_readers=[reader],
|
||||
views=[
|
||||
View(
|
||||
instrument_name="messenger.http.duration",
|
||||
aggregation=ExplicitBucketHistogramAggregation(HTTP_BUCKETS),
|
||||
),
|
||||
View(
|
||||
instrument_name="messenger.job.duration",
|
||||
aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS),
|
||||
),
|
||||
View(
|
||||
instrument_name="messenger.attachment.ready.duration",
|
||||
aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS),
|
||||
),
|
||||
],
|
||||
)
|
||||
metrics.set_meter_provider(meter_provider)
|
||||
logger_provider = LoggerProvider(resource=resource)
|
||||
logger_provider.add_log_record_processor(
|
||||
BatchLogRecordProcessor(
|
||||
OTLPLogExporter(
|
||||
endpoint=endpoint + "/v1/logs",
|
||||
session=CountingSession(self.export_stats, "logs"),
|
||||
**common,
|
||||
),
|
||||
max_queue_size=256,
|
||||
max_export_batch_size=64,
|
||||
schedule_delay_millis=500,
|
||||
export_timeout_millis=1500,
|
||||
)
|
||||
)
|
||||
set_logger_provider(logger_provider)
|
||||
otlp_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
|
||||
otlp_handler.addFilter(TechnicalLogs())
|
||||
# Keep machine-readable JSON in OTLP log body, preserving request ID.
|
||||
otlp_handler.setFormatter(JsonFormatter())
|
||||
root.addHandler(otlp_handler)
|
||||
self.providers = [trace_provider, meter_provider, logger_provider]
|
||||
self.tracer = trace.get_tracer("messenger", "1.0")
|
||||
meter = metrics.get_meter("messenger", "1.0")
|
||||
self.requests = meter.create_counter("messenger.http.requests", unit="{request}")
|
||||
self.http_duration = meter.create_histogram("messenger.http.duration", unit="s")
|
||||
self.sampling = meter.create_counter("messenger.http.trace.decisions", unit="{request}")
|
||||
self.jobs = meter.create_counter("messenger.jobs", unit="{job}")
|
||||
self.job_duration = meter.create_histogram("messenger.job.duration", unit="s")
|
||||
self.ready_duration = meter.create_histogram(
|
||||
"messenger.attachment.ready.duration", unit="s"
|
||||
)
|
||||
self.queue_depth = meter.create_gauge("messenger.queue.depth", unit="{job}")
|
||||
self.oldest_age = meter.create_gauge("messenger.queue.oldest.age", unit="s")
|
||||
self.pool_used = meter.create_gauge("messenger.db.pool.used", unit="{connection}")
|
||||
self.pool_waiting = meter.create_gauge("messenger.db.pool.waiting", unit="{request}")
|
||||
self.worker_busy = meter.create_gauge("messenger.worker.busy", unit="{worker}")
|
||||
self.worker_capacity = meter.create_gauge("messenger.worker.capacity", unit="{worker}")
|
||||
self.process_cpu = meter.create_gauge("messenger.process.cpu", unit="%")
|
||||
self.process_memory = meter.create_gauge("messenger.process.memory", unit="By")
|
||||
self.export_observation = meter.create_gauge("messenger.telemetry.export", unit="{item}")
|
||||
|
||||
def start_background(self, store):
|
||||
process = psutil.Process()
|
||||
process.cpu_percent()
|
||||
|
||||
def collect():
|
||||
while not self.stop.wait(2):
|
||||
counts = self.export_stats.snapshot()
|
||||
# Read-only queue observation of pinned SDK 1.45.2. Conservation
|
||||
# is evaluated after quiescence, not during concurrent export.
|
||||
pending = len(self.batch_spans._batch_processor._queue)
|
||||
counts["spans_pending"] = pending
|
||||
counts["span_queue_capacity"] = 256
|
||||
counts["spans_queue_lost"] = max(
|
||||
0,
|
||||
counts["spans_recorded"]
|
||||
- counts["spans_exported"]
|
||||
- counts["spans_export_failed"]
|
||||
- counts["spans_in_flight"]
|
||||
- pending,
|
||||
)
|
||||
for name, value in counts.items():
|
||||
self.export_observation.set(value, {"observation": name})
|
||||
stats = store.pool.get_stats()
|
||||
self.pool_used.set(
|
||||
max(0, stats.get("pool_size", 0) - stats.get("pool_available", 0))
|
||||
)
|
||||
self.pool_waiting.set(stats.get("requests_waiting", 0))
|
||||
self.process_cpu.set(process.cpu_percent())
|
||||
self.process_memory.set(process.memory_info().rss)
|
||||
if self.service == "messenger-worker":
|
||||
self.worker_busy.set(self.busy)
|
||||
self.worker_capacity.set(1)
|
||||
if self.service == "messenger-dispatcher":
|
||||
try:
|
||||
with store.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT count(*) AS depth, COALESCE(EXTRACT(EPOCH FROM now()-min(created_at)),0) AS age FROM attachment_jobs WHERE state IN ('queued','processing')"
|
||||
).fetchone()
|
||||
self.queue_depth.set(row["depth"], {"queue": "attachments"})
|
||||
self.oldest_age.set(float(row["age"]), {"queue": "attachments"})
|
||||
except Exception:
|
||||
logging.getLogger("messenger.telemetry").warning(
|
||||
json.dumps({"event": "queue_observation_unavailable"})
|
||||
)
|
||||
|
||||
if self.enabled:
|
||||
self.thread = threading.Thread(
|
||||
target=collect, name="telemetry-observations", daemon=True
|
||||
)
|
||||
self.thread.start()
|
||||
|
||||
def close(self):
|
||||
self.stop.set()
|
||||
if self.thread:
|
||||
self.thread.join(timeout=4)
|
||||
|
||||
# Business shutdown has a fixed budget even if telemetry endpoints are down.
|
||||
def shutdown_providers():
|
||||
for provider in self.providers:
|
||||
provider.shutdown()
|
||||
|
||||
shutdown = threading.Thread(
|
||||
target=shutdown_providers, name="telemetry-shutdown", daemon=True
|
||||
)
|
||||
shutdown.start()
|
||||
shutdown.join(timeout=2)
|
||||
|
||||
|
||||
def setup(service):
|
||||
global _runtime
|
||||
if _runtime is None:
|
||||
_runtime = Telemetry(service)
|
||||
return _runtime
|
||||
|
||||
|
||||
def current():
|
||||
return _runtime
|
||||
|
||||
|
||||
def carrier():
|
||||
result = {}
|
||||
propagate.inject(result)
|
||||
return {key: result[key] for key in ("traceparent", "tracestate") if key in result}
|
||||
|
||||
|
||||
@contextmanager
|
||||
def operation(name, attributes=None, context=None, kind=SpanKind.CLIENT):
|
||||
if (
|
||||
kind == SpanKind.CLIENT
|
||||
and context is None
|
||||
and not trace.get_current_span().get_span_context().is_valid
|
||||
):
|
||||
yield trace.get_current_span()
|
||||
return
|
||||
tracer = trace.get_tracer("messenger", "1.0")
|
||||
with tracer.start_as_current_span(
|
||||
name,
|
||||
attributes=attributes or {},
|
||||
context=context,
|
||||
kind=kind,
|
||||
record_exception=False,
|
||||
set_status_on_exception=False,
|
||||
) as span:
|
||||
try:
|
||||
yield span
|
||||
except BaseException as exc:
|
||||
span.set_status(Status(StatusCode.ERROR, type(exc).__name__))
|
||||
raise
|
||||
@@ -0,0 +1,272 @@
|
||||
"""Small TLS boundary: exact DNS SAN allowlist, revocation, bounded TCP forwarding.
|
||||
|
||||
The upstream is always loopback in the same network namespace. HTTP mode is used
|
||||
only for S3 prefix authorization; AWS signatures are still checked by S3 itself.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import selectors
|
||||
import socket
|
||||
import socketserver
|
||||
import ssl
|
||||
import threading
|
||||
from contextlib import closing
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from urllib.parse import unquote, urlsplit
|
||||
|
||||
MAX_BODY = 10 * 1024 * 1024
|
||||
|
||||
|
||||
def settings():
|
||||
return json.loads(Path(os.environ["TLS_GATEWAY_CONFIG"]).read_text())
|
||||
|
||||
|
||||
def role(cert, allowed):
|
||||
names = {v for k, v in cert.get("subjectAltName", ()) if k == "DNS"}
|
||||
found = names.intersection(allowed)
|
||||
if len(found) != 1:
|
||||
raise ssl.SSLError("service_identity_denied")
|
||||
revoked = Path(os.getenv("TLS_REVOKED_FILE", "/run/tls/revoked.json"))
|
||||
serials = json.loads(revoked.read_text()) if revoked.exists() else []
|
||||
if cert.get("serialNumber", "").lstrip("0").lower() in serials:
|
||||
raise ssl.SSLError("certificate_revoked")
|
||||
return next(iter(found))
|
||||
|
||||
|
||||
def context(cfg):
|
||||
ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
||||
ctx.set_alpn_protocols(["http/1.1"] if cfg.get("http") else ["h2", "http/1.1"])
|
||||
ctx.minimum_version = ssl.TLSVersion.TLSv1_2
|
||||
ctx.load_cert_chain(cfg["cert"], cfg["key"])
|
||||
if cfg.get("allowed"):
|
||||
ctx.verify_mode = ssl.CERT_REQUIRED
|
||||
ctx.load_verify_locations(cfg["ca"])
|
||||
ctx.options |= ssl.OP_NO_TICKET
|
||||
return ctx
|
||||
|
||||
|
||||
class Boundary:
|
||||
def process_request(self, request, address):
|
||||
if not self.slots.acquire(blocking=False):
|
||||
request.close()
|
||||
return
|
||||
try:
|
||||
super().process_request(request, address)
|
||||
except Exception:
|
||||
self.slots.release()
|
||||
raise
|
||||
|
||||
def process_request_thread(self, request, address):
|
||||
try:
|
||||
request.settimeout(2)
|
||||
wrapped = context(self.cfg).wrap_socket(request, server_side=True)
|
||||
wrapped.identity = (
|
||||
role(wrapped.getpeercert(), self.cfg["allowed"])
|
||||
if self.cfg.get("allowed")
|
||||
else "public"
|
||||
)
|
||||
wrapped.settimeout(15)
|
||||
super().process_request_thread(wrapped, address)
|
||||
except (OSError, ValueError):
|
||||
request.close()
|
||||
finally:
|
||||
self.slots.release()
|
||||
|
||||
|
||||
class TCPHandler(socketserver.BaseRequestHandler):
|
||||
def handle(self):
|
||||
try:
|
||||
with socket.create_connection(
|
||||
("127.0.0.1", self.server.cfg["upstream"]), 2
|
||||
) as upstream:
|
||||
self.request.setblocking(False)
|
||||
upstream.setblocking(False)
|
||||
with selectors.DefaultSelector() as selector:
|
||||
selector.register(self.request, selectors.EVENT_READ, upstream)
|
||||
selector.register(upstream, selectors.EVENT_READ, self.request)
|
||||
while True:
|
||||
role(self.request.getpeercert(), self.server.cfg["allowed"])
|
||||
pending = self.request.pending()
|
||||
events = selector.select(timeout=0 if pending else 30)
|
||||
if pending and not any(key.fileobj is self.request for key, _ in events):
|
||||
events.append((selector.get_key(self.request), selectors.EVENT_READ))
|
||||
if not events:
|
||||
return
|
||||
for key, _ in events:
|
||||
# Recheck after the bounded wait: a revoked peer must
|
||||
# not forward one extra command when it wakes select.
|
||||
role(self.request.getpeercert(), self.server.cfg["allowed"])
|
||||
try:
|
||||
chunk = key.fileobj.recv(65536)
|
||||
except ssl.SSLWantReadError:
|
||||
continue
|
||||
if not chunk:
|
||||
return
|
||||
# Small writes with finite socket timeout prevent unbounded
|
||||
# per-connection queues when a peer stops consuming.
|
||||
destination = key.data
|
||||
destination.settimeout(5)
|
||||
destination.sendall(chunk)
|
||||
destination.setblocking(False)
|
||||
except (OSError, ssl.SSLError):
|
||||
return
|
||||
|
||||
|
||||
class TCPServer(Boundary, socketserver.ThreadingTCPServer):
|
||||
allow_reuse_address = True
|
||||
daemon_threads = True
|
||||
request_queue_size = 64
|
||||
|
||||
|
||||
class HTTPHandler(BaseHTTPRequestHandler):
|
||||
protocol_version = "HTTP/1.1"
|
||||
|
||||
def handle_expect_100(self):
|
||||
return True
|
||||
|
||||
def log_message(self, *_):
|
||||
pass
|
||||
|
||||
def do_GET(self):
|
||||
self.forward()
|
||||
|
||||
do_HEAD = do_GET
|
||||
do_PUT = do_GET
|
||||
do_POST = do_GET
|
||||
do_DELETE = do_GET
|
||||
|
||||
def forward(self):
|
||||
self.close_connection = True
|
||||
cfg = self.server.cfg
|
||||
identity = self.connection.identity
|
||||
parsed = urlsplit(self.path)
|
||||
path = unquote(parsed.path)
|
||||
parts = path.split("/")
|
||||
if (
|
||||
parsed.scheme
|
||||
or parsed.netloc
|
||||
or parsed.fragment
|
||||
or any(p in {".", ".."} for p in parts)
|
||||
or "\\" in path
|
||||
or "%" in parsed.path
|
||||
or any(ord(c) < 32 for c in path)
|
||||
):
|
||||
self.send_error(403)
|
||||
return
|
||||
if cfg.get("backend") and identity != "operator":
|
||||
allowed = (
|
||||
self.command == "GET"
|
||||
if identity == "grafana"
|
||||
else (
|
||||
identity == "collector"
|
||||
and cfg["backend"] == "loki"
|
||||
and self.command == "POST"
|
||||
and parsed.path == "/otlp/v1/logs"
|
||||
)
|
||||
)
|
||||
if not allowed:
|
||||
self.send_error(403)
|
||||
return
|
||||
if cfg.get("s3"):
|
||||
if (
|
||||
self.headers.get("Host", "").lower() != "storage:8443"
|
||||
or (identity != "operator" and parsed.query)
|
||||
or self.headers.get("x-amz-copy-source")
|
||||
):
|
||||
self.send_error(403)
|
||||
return
|
||||
# A worker can read its sources, write versioned results and remove
|
||||
# staging. It cannot list buckets, overwrite staging or touch others.
|
||||
permitted = identity == "operator"
|
||||
if identity == "api":
|
||||
permitted = (
|
||||
path == "/messenger"
|
||||
and self.command == "HEAD"
|
||||
or (
|
||||
path.startswith("/messenger/staging/")
|
||||
and self.command in {"PUT", "GET", "HEAD", "DELETE"}
|
||||
)
|
||||
or (
|
||||
path.startswith("/messenger/attachments/")
|
||||
and self.command in {"GET", "HEAD"}
|
||||
)
|
||||
)
|
||||
elif identity == "worker":
|
||||
permitted = (
|
||||
path.startswith("/messenger/staging/")
|
||||
and self.command in {"GET", "HEAD", "DELETE"}
|
||||
) or (
|
||||
path.startswith("/messenger/attachments/")
|
||||
and self.command in {"PUT", "GET", "HEAD"}
|
||||
)
|
||||
if not permitted:
|
||||
self.send_error(403)
|
||||
return
|
||||
try:
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
body_limit = MAX_BODY if cfg.get("s3") else 1024 * 1024
|
||||
if length < 0 or length > body_limit or self.headers.get("Transfer-Encoding"):
|
||||
self.send_error(413)
|
||||
return
|
||||
if self.headers.get("Expect", "").lower() == "100-continue":
|
||||
self.wfile.write(b"HTTP/1.1 100 Continue\r\n\r\n")
|
||||
self.wfile.flush()
|
||||
body = self.rfile.read(length)
|
||||
if len(body) != length:
|
||||
self.send_error(400)
|
||||
return
|
||||
headers = {
|
||||
k: v
|
||||
for k, v in self.headers.items()
|
||||
if k.lower() not in {"connection", "proxy-connection", "expect"}
|
||||
}
|
||||
with closing(
|
||||
http.client.HTTPConnection("127.0.0.1", cfg["upstream"], timeout=5)
|
||||
) as upstream:
|
||||
upstream.request(self.command, self.path, body=body, headers=headers)
|
||||
response = upstream.getresponse()
|
||||
self.send_response(response.status)
|
||||
for key, value in response.getheaders():
|
||||
if key.lower() not in {"connection", "transfer-encoding", "server", "date"}:
|
||||
self.send_header(key, value)
|
||||
self.send_header("Connection", "close")
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
while chunk := response.read(65536):
|
||||
self.wfile.write(chunk)
|
||||
except (OSError, ValueError, http.client.HTTPException):
|
||||
self.close_connection = True
|
||||
|
||||
|
||||
class HTTPServer(Boundary, ThreadingHTTPServer):
|
||||
daemon_threads = True
|
||||
|
||||
|
||||
def serve(cfg):
|
||||
cls = HTTPServer if cfg.get("http") else TCPServer
|
||||
handler = HTTPHandler if cfg.get("http") else TCPHandler
|
||||
with cls(("0.0.0.0", cfg["port"]), handler) as server:
|
||||
server.cfg = cfg
|
||||
server.slots = threading.BoundedSemaphore(2 if cfg.get("http") else 32)
|
||||
server.serve_forever(poll_interval=0.2)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--config")
|
||||
args = parser.parse_args()
|
||||
cfg = json.loads(Path(args.config).read_text()) if args.config else settings()
|
||||
listeners = cfg if isinstance(cfg, list) else [cfg]
|
||||
threads = [threading.Thread(target=serve, args=(item,), daemon=True) for item in listeners]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,132 @@
|
||||
"""Operator-only network probe: real TLS, exact role authorization and closed raw ports."""
|
||||
|
||||
import json
|
||||
import socket
|
||||
import ssl
|
||||
from pathlib import Path
|
||||
|
||||
CA = "/run/tls/ca.pem"
|
||||
IDENTITIES = Path("/run/audit-identities")
|
||||
|
||||
|
||||
def exchange(host, port, role, payload, *, hostname=None, trust=True):
|
||||
context = (
|
||||
ssl.create_default_context(cafile=CA) if trust else ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
|
||||
)
|
||||
if role:
|
||||
context.load_cert_chain(IDENTITIES / role / "cert.pem", IDENTITIES / role / "key.pem")
|
||||
context.set_alpn_protocols(["h2"] if port == 44317 else ["http/1.1"])
|
||||
try:
|
||||
with socket.create_connection((host, port), 2) as raw:
|
||||
with context.wrap_socket(raw, server_hostname=hostname or host) as secured:
|
||||
secured.settimeout(3)
|
||||
secured.sendall(payload)
|
||||
result = secured.recv(4096)
|
||||
return bool(result), result
|
||||
except (OSError, ssl.SSLError):
|
||||
return False, b""
|
||||
|
||||
|
||||
def main():
|
||||
checks = {}
|
||||
# Every gateway is tested positively with its intended role and negatively
|
||||
# with another valid certificate from the SAME CA, no certificate, foreign
|
||||
# CA certificate, empty trust and wrong server hostname.
|
||||
targets = [
|
||||
(
|
||||
"api",
|
||||
9443,
|
||||
"lb",
|
||||
b"GET /health/ready HTTP/1.1\r\nHost: api\r\nConnection: close\r\n\r\n",
|
||||
),
|
||||
("broker", 6380, "worker", b"*1\r\n$4\r\nPING\r\n"),
|
||||
("cache", 6380, "api", b"*1\r\n$4\r\nPING\r\n"),
|
||||
(
|
||||
"storage",
|
||||
8443,
|
||||
"worker",
|
||||
b"HEAD /messenger HTTP/1.1\r\nHost: storage:8443\r\nConnection: close\r\n\r\n",
|
||||
),
|
||||
(
|
||||
"collector",
|
||||
4318,
|
||||
"api",
|
||||
b"POST /v1/traces HTTP/1.1\r\nHost: collector\r\nContent-Type: application/json\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}",
|
||||
),
|
||||
(
|
||||
"collector",
|
||||
9464,
|
||||
"prometheus",
|
||||
b"GET /metrics HTTP/1.1\r\nHost: collector\r\nConnection: close\r\n\r\n",
|
||||
),
|
||||
(
|
||||
"collector",
|
||||
8888,
|
||||
"prometheus",
|
||||
b"GET /metrics HTTP/1.1\r\nHost: collector\r\nConnection: close\r\n\r\n",
|
||||
),
|
||||
(
|
||||
"prometheus",
|
||||
9443,
|
||||
"grafana",
|
||||
b"GET /api/v1/query?query=up HTTP/1.1\r\nHost: prometheus\r\nConnection: close\r\n\r\n",
|
||||
),
|
||||
(
|
||||
"loki",
|
||||
9443,
|
||||
"grafana",
|
||||
b"GET /loki/api/v1/labels HTTP/1.1\r\nHost: loki\r\nConnection: close\r\n\r\n",
|
||||
),
|
||||
(
|
||||
"tempo",
|
||||
9443,
|
||||
"grafana",
|
||||
b"GET /api/search HTTP/1.1\r\nHost: tempo\r\nConnection: close\r\n\r\n",
|
||||
),
|
||||
]
|
||||
for host, port, role, payload in targets:
|
||||
key = f"{host}_{port}"
|
||||
ok, response = exchange(host, port, role, payload)
|
||||
checks[key + "_allowed_identity"] = ok and (
|
||||
response.startswith(b"HTTP/1.1") or response.startswith(b"-NOAUTH")
|
||||
)
|
||||
wrong = "worker" if host == "api" else "lb"
|
||||
for label, identity, hostname, trust in (
|
||||
("wrong_trusted_identity", wrong, None, True),
|
||||
("no_client_cert", None, None, True),
|
||||
("foreign_ca_client", "foreign", None, True),
|
||||
("wrong_server_hostname", role, "invalid.example", True),
|
||||
("empty_trust", role, None, False),
|
||||
):
|
||||
ok, _ = exchange(host, port, identity, payload, hostname=hostname, trust=trust)
|
||||
checks[key + "_" + label + "_denied"] = not ok
|
||||
otlp = b"POST /v1/traces HTTP/1.1\r\nHost: tempo\r\nContent-Type: application/json\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}"
|
||||
checks["tempo_otlp_collector_allowed"] = exchange("tempo", 44318, "collector", otlp)[0]
|
||||
checks["tempo_otlp_wrong_role_denied"] = not exchange("tempo", 44318, "worker", otlp)[0]
|
||||
for host, ports in (
|
||||
("api", [8080]),
|
||||
("storage", [8333, 8888, 9333]),
|
||||
("broker", [6379]),
|
||||
("cache", [6379]),
|
||||
("collector", [14318, 19464, 18888, 13133]),
|
||||
("prometheus", [9090]),
|
||||
("loki", [3100]),
|
||||
("tempo", [3200, 4317, 4318]),
|
||||
):
|
||||
for port in ports:
|
||||
try:
|
||||
with socket.create_connection((host, port), 1):
|
||||
checks[f"raw_{host}_{port}_closed"] = False
|
||||
except OSError:
|
||||
checks[f"raw_{host}_{port}_closed"] = True
|
||||
# Datasource roles cannot submit arbitrary telemetry or mutate Prometheus.
|
||||
forbidden = b"POST /otlp/v1/logs HTTP/1.1\r\nHost: loki\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}"
|
||||
ok, body = exchange("loki", 9443, "grafana", forbidden)
|
||||
checks["grafana_loki_write_denied"] = ok and b" 403 " in body.split(b"\r\n", 1)[0]
|
||||
print(json.dumps({"checks": checks, "passed": all(checks.values())}, sort_keys=True))
|
||||
if not all(checks.values()):
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,292 @@
|
||||
"""Separate serial worker (prefetch=1) and transactional-outbox dispatcher."""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import redis
|
||||
from opentelemetry import propagate, trace
|
||||
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||
|
||||
from . import telemetry
|
||||
from .domain import Failure
|
||||
from .jobs import GROUP, STREAM, Jobs, broker_client, fault
|
||||
from .postgres import PostgresStore
|
||||
from .secrets_provider import refresh
|
||||
from .storage import ObjectStorage
|
||||
|
||||
logger = logging.getLogger("messenger.worker")
|
||||
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
||||
stopping = False
|
||||
MAX_JOB_BYTES = 8192
|
||||
|
||||
|
||||
def stop(signum, frame):
|
||||
global stopping
|
||||
stopping = True
|
||||
|
||||
|
||||
def process(job, jobs, storage):
|
||||
fault(jobs.store, "before_effect", job["attachment_id"])
|
||||
with tempfile.TemporaryDirectory(prefix="image-job-") as directory:
|
||||
root = Path(directory)
|
||||
source = root / "source"
|
||||
response = storage.get(job["source_key"])
|
||||
try:
|
||||
if response["ContentLength"] > 10_485_760:
|
||||
raise ValueError("invalid_image")
|
||||
with source.open("wb") as target:
|
||||
while block := response["Body"].read(64 * 1024):
|
||||
target.write(block)
|
||||
if target.tell() > 10_485_760:
|
||||
raise ValueError("invalid_image")
|
||||
finally:
|
||||
response["Body"].close()
|
||||
with source.open("rb") as file:
|
||||
digest = hashlib.file_digest(file, "sha256").hexdigest()
|
||||
if digest != job["source_sha256"] or source.stat().st_size != job["source_size"]:
|
||||
raise ValueError("invalid_image")
|
||||
# Resize executes outside the API and is bounded in wall time, CPU and address space.
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"messenger.image_pipeline",
|
||||
str(source),
|
||||
directory,
|
||||
str(job["pipeline_version"]),
|
||||
],
|
||||
timeout=20,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
if result.returncode:
|
||||
raise ValueError("invalid_image")
|
||||
variants = json.loads((root / "variants.json").read_text())
|
||||
prefix = f"attachments/{job['attachment_id']}/v{job['pipeline_version']}"
|
||||
with source.open("rb") as file:
|
||||
storage.put(
|
||||
prefix + "/original", file, source.stat().st_size, job["source_content_type"]
|
||||
)
|
||||
for variant in variants:
|
||||
path = root / variant["name"]
|
||||
with path.open("rb") as file:
|
||||
storage.put(
|
||||
prefix + "/" + variant["name"],
|
||||
file,
|
||||
variant["size_bytes"],
|
||||
variant["content_type"],
|
||||
)
|
||||
fault(jobs.store, "after_effect", job["attachment_id"])
|
||||
published = jobs.complete(job, prefix + "/original", variants)
|
||||
if published:
|
||||
fault(jobs.store, "after_result_commit", job["attachment_id"])
|
||||
cleanup_ready(jobs.store, storage, job["attachment_id"])
|
||||
return published
|
||||
|
||||
|
||||
def cleanup_ready(store, storage, attachment_id):
|
||||
# Same row lock as reprocess/replay: cannot delete a source chosen by a new job.
|
||||
with store.connection() as conn, conn.transaction():
|
||||
a = conn.execute(
|
||||
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,)
|
||||
).fetchone()
|
||||
if (
|
||||
a
|
||||
and a["status"] == "ready"
|
||||
and a["stage_key"]
|
||||
and a["stage_key"].startswith("staging/")
|
||||
):
|
||||
storage.delete(a["stage_key"])
|
||||
conn.execute("UPDATE attachments SET stage_key=NULL WHERE id=%s", (attachment_id,))
|
||||
|
||||
|
||||
def validate_delivery(body):
|
||||
if not isinstance(body, (str, bytes, bytearray)):
|
||||
raise ValueError("invalid_job")
|
||||
encoded = body.encode("utf-8") if isinstance(body, str) else body
|
||||
if len(encoded) > MAX_JOB_BYTES:
|
||||
raise ValueError("invalid_job")
|
||||
try:
|
||||
data = json.loads(encoded)
|
||||
except RecursionError:
|
||||
raise ValueError("invalid_job") from None
|
||||
expected = {
|
||||
"schema_version",
|
||||
"job_id",
|
||||
"type",
|
||||
"attachment_id",
|
||||
"pipeline_version",
|
||||
"attempt",
|
||||
"created_at",
|
||||
}
|
||||
optional = {"traceparent", "tracestate"}
|
||||
if not isinstance(data, dict) or not expected <= set(data) or set(data) - expected - optional:
|
||||
raise ValueError("invalid_job")
|
||||
if (
|
||||
type(data["schema_version"]) is not int
|
||||
or data["schema_version"] != 1
|
||||
or data["type"] != "attachment.process.v1"
|
||||
):
|
||||
raise ValueError("invalid_job")
|
||||
if not isinstance(data["job_id"], str) or not isinstance(data["attachment_id"], str):
|
||||
raise ValueError("invalid_job")
|
||||
for field in ("job_id", "attachment_id"):
|
||||
canonical = str(UUID(data[field]))
|
||||
if data[field].lower() != canonical:
|
||||
raise ValueError("invalid_job")
|
||||
# RFC UUID hex is case-insensitive; PostgreSQL and state comparisons use
|
||||
# the same canonical spelling, so a valid uppercase delivery is not lost.
|
||||
data[field] = canonical
|
||||
if type(data["pipeline_version"]) is not int or data["pipeline_version"] < 1:
|
||||
raise ValueError("invalid_job")
|
||||
if type(data["attempt"]) is not int or data["attempt"] < 1:
|
||||
raise ValueError("invalid_job")
|
||||
created = data["created_at"]
|
||||
if (
|
||||
not isinstance(created, str)
|
||||
or len(created) > 64
|
||||
or not re.fullmatch(
|
||||
r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]+)?Z", created
|
||||
)
|
||||
):
|
||||
raise ValueError("invalid_job")
|
||||
datetime.fromisoformat(created[:-1] + "+00:00")
|
||||
if "traceparent" in data and (
|
||||
not isinstance(data["traceparent"], str)
|
||||
or not re.fullmatch(r"00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}", data["traceparent"])
|
||||
):
|
||||
raise ValueError("invalid_job")
|
||||
if "tracestate" in data and (
|
||||
not isinstance(data["tracestate"], str) or len(data["tracestate"]) > 512
|
||||
):
|
||||
raise ValueError("invalid_job")
|
||||
return data
|
||||
|
||||
|
||||
def consume_one(store, jobs, storage, broker, entry):
|
||||
entry_id, fields = entry
|
||||
try:
|
||||
body = validate_delivery(fields.get("body", fields.get(b"body")))
|
||||
except (KeyError, ValueError, TypeError):
|
||||
logger.error(json.dumps({"event": "invalid_delivery"}))
|
||||
broker.xack(STREAM, GROUP, entry_id)
|
||||
broker.xdel(STREAM, entry_id)
|
||||
return
|
||||
with telemetry.operation(
|
||||
"attachments consume", context=propagate.extract(body), kind=SpanKind.CONSUMER
|
||||
):
|
||||
consume_job(store, jobs, storage, broker, body)
|
||||
# Durable state owns recovery even when notification acknowledgements repeat.
|
||||
broker.xack(STREAM, GROUP, entry_id)
|
||||
broker.xdel(STREAM, entry_id)
|
||||
|
||||
|
||||
def consume_job(store, jobs, storage, broker, body):
|
||||
job = jobs.claim(body["job_id"], body["attachment_id"], body["pipeline_version"])
|
||||
t = telemetry.current()
|
||||
started = time.perf_counter()
|
||||
outcome = "duplicate"
|
||||
if job:
|
||||
try:
|
||||
if t:
|
||||
t.busy = 1
|
||||
outcome = "success" if process(job, jobs, storage) else "superseded"
|
||||
except ValueError:
|
||||
jobs.fail(job, "invalid_image", permanent=True)
|
||||
outcome = "failed"
|
||||
except (Failure, subprocess.TimeoutExpired, OSError, RuntimeError) as exc:
|
||||
logger.warning(
|
||||
json.dumps(
|
||||
{"event": "job_retry", "job_id": str(job["id"]), "kind": type(exc).__name__}
|
||||
)
|
||||
)
|
||||
jobs.fail(job, "processing_failed")
|
||||
outcome = "retry" if job["attempt"] < job["attempt_limit"] else "failed"
|
||||
finally:
|
||||
if t:
|
||||
t.busy = 0
|
||||
attrs = {"job_type": "attachment.process.v1", "outcome": outcome}
|
||||
t.jobs.add(1, attrs)
|
||||
t.job_duration.record(time.perf_counter() - started, attrs)
|
||||
span = trace.get_current_span()
|
||||
span.set_attributes({"job.type": "attachment.process.v1", "job.outcome": outcome})
|
||||
if outcome in {"failed", "retry"}:
|
||||
span.set_status(Status(StatusCode.ERROR, "processing_failed"))
|
||||
logger.info(json.dumps({"event": "job_finished", "outcome": outcome}))
|
||||
else:
|
||||
trace.get_current_span().set_attributes(
|
||||
{"job.type": "attachment.process.v1", "job.outcome": "duplicate"}
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--dispatcher", action="store_true")
|
||||
args = parser.parse_args()
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
signal.signal(signal.SIGINT, stop)
|
||||
t = telemetry.setup("messenger-dispatcher" if args.dispatcher else "messenger-worker")
|
||||
store = PostgresStore()
|
||||
t.start_background(store)
|
||||
jobs = Jobs(store)
|
||||
broker = broker_client()
|
||||
storage = None if args.dispatcher else ObjectStorage()
|
||||
consumer = "worker-" + str(uuid4())
|
||||
last_claim = 0
|
||||
try:
|
||||
while not stopping:
|
||||
try:
|
||||
refresh()
|
||||
if args.dispatcher:
|
||||
jobs.recover_one()
|
||||
if not jobs.publish_one(broker):
|
||||
time.sleep(0.2)
|
||||
continue
|
||||
try:
|
||||
broker.xgroup_create(STREAM, GROUP, id="0", mkstream=True)
|
||||
except redis.ResponseError as exc:
|
||||
if "BUSYGROUP" not in str(exc):
|
||||
raise
|
||||
entries = []
|
||||
if time.monotonic() - last_claim >= 5:
|
||||
claimed = broker.xautoclaim(
|
||||
STREAM,
|
||||
GROUP,
|
||||
consumer,
|
||||
min_idle_time=jobs.lease_seconds * 1000,
|
||||
start_id="0-0",
|
||||
count=1,
|
||||
)
|
||||
entries = claimed[1]
|
||||
last_claim = time.monotonic()
|
||||
if not entries:
|
||||
batches = broker.xreadgroup(GROUP, consumer, {STREAM: ">"}, count=1, block=500)
|
||||
entries = batches[0][1] if batches else []
|
||||
for entry in entries:
|
||||
consume_one(store, jobs, storage, broker, entry)
|
||||
except (redis.RedisError, Failure) as exc:
|
||||
logger.warning(
|
||||
json.dumps({"event": "dependency_retry", "kind": type(exc).__name__})
|
||||
)
|
||||
time.sleep(0.5)
|
||||
finally:
|
||||
broker.close()
|
||||
if storage is not None:
|
||||
storage.close()
|
||||
t.close()
|
||||
store.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,201 @@
|
||||
import base64
|
||||
import os
|
||||
import threading
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from urllib.parse import quote
|
||||
|
||||
import pytest
|
||||
from client import LAB, Client
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def scenario():
|
||||
c = Client(os.environ.get("BASE_URL", "http://127.0.0.1:8085"))
|
||||
return c, *c.bootstrap()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"body",
|
||||
[
|
||||
{"text": ""},
|
||||
{"text": " \n\t"},
|
||||
{"text": 1},
|
||||
{"text": True},
|
||||
{"text": None},
|
||||
{"text": "x", "sender_id": str(uuid.uuid4())},
|
||||
{"text": "x", "unknown": 1},
|
||||
{"text": "x" * 4001},
|
||||
{},
|
||||
[],
|
||||
],
|
||||
)
|
||||
def test_invalid_message(scenario, body):
|
||||
c, alice, _, _, chat = scenario
|
||||
c.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
400,
|
||||
actor=alice,
|
||||
body=body,
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||
)
|
||||
|
||||
|
||||
def test_malformed_json(scenario):
|
||||
c, alice, _, _, chat = scenario
|
||||
c.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
400,
|
||||
actor=alice,
|
||||
raw=b'{"text":',
|
||||
headers={"Content-Type": "application/json", "Idempotency-Key": uuid.uuid4().hex},
|
||||
)
|
||||
|
||||
|
||||
def test_acl_and_unknown_chat(scenario):
|
||||
c, alice, _, eve, chat = scenario
|
||||
for cid in [chat["id"], str(uuid.uuid4())]:
|
||||
for suffix in ["", "/messages"]:
|
||||
c.call("GET", f"/api/v1/chats/{cid}{suffix}", 404, actor=eve)
|
||||
c.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{cid}/messages",
|
||||
404,
|
||||
actor=eve,
|
||||
body={"text": "x"},
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||
)
|
||||
c.call("GET", "/api/v1/chats/not-a-uuid", 400, actor=alice)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", ["ab", "UPPER", "a-b", "a" * 33, "имя", "abc\n"])
|
||||
def test_username_validation(scenario, name):
|
||||
c, alice, *_ = scenario
|
||||
body = {**alice["registration"], "username": name}
|
||||
c.call("POST", "/api/v1/users", 400, body=body)
|
||||
|
||||
|
||||
def test_unicode_and_limits(scenario):
|
||||
c, alice, _, _, chat = scenario
|
||||
text = "Ё👩💻漢字" * 500
|
||||
assert c.message(alice, chat, text)["json"]["text"] == text
|
||||
c.call(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
400,
|
||||
body={
|
||||
**alice["registration"],
|
||||
"username": "u_" + uuid.uuid4().hex[:12],
|
||||
"display_name": " ",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["0", "101", "1.5", "true", "-1", "", "1&limit=2"])
|
||||
def test_invalid_limit(scenario, value):
|
||||
c, alice, _, _, chat = scenario
|
||||
c.call("GET", f"/api/v1/chats/{chat['id']}/messages?limit={value}", 400, actor=alice)
|
||||
|
||||
|
||||
def test_pagination_and_cursor_scope(scenario):
|
||||
c, alice, bob, _, chat = scenario
|
||||
expected = [c.message(alice, chat, str(i))["json"] for i in range(13)]
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
result, cursor, first = [], None, None
|
||||
for i in range(20):
|
||||
url = path + f"?limit={2 if i == 0 else 3}"
|
||||
if cursor:
|
||||
url += "&cursor=" + quote(cursor, safe="")
|
||||
data = c.call("GET", url, actor=bob, schema="MessagePage")["json"]
|
||||
result += data["items"]
|
||||
cursor = data["next_cursor"]
|
||||
first = first or cursor
|
||||
if cursor is None:
|
||||
break
|
||||
assert result == sorted(expected, key=lambda m: (m["created_at"], m["id"]))
|
||||
other = c.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
201,
|
||||
actor=alice,
|
||||
body={"title": "другой", "member_ids": [alice["id"]]},
|
||||
schema="Chat",
|
||||
)["json"]
|
||||
c.call("GET", f"/api/v1/chats/{other['id']}/messages?cursor={first}", 400, actor=alice)
|
||||
c.call("GET", path + "?cursor=broken", 400, actor=alice)
|
||||
|
||||
|
||||
def test_chat_validation_and_atomicity(scenario):
|
||||
c, alice, bob, _, _ = scenario
|
||||
before = c.call("GET", "/api/v1/chats?limit=100", actor=alice, schema="ChatPage")["json"]
|
||||
for members in [
|
||||
[bob["id"]],
|
||||
[alice["id"], alice["id"]],
|
||||
[alice["id"], str(uuid.uuid4())],
|
||||
[],
|
||||
[123],
|
||||
]:
|
||||
c.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
400,
|
||||
actor=alice,
|
||||
body={"title": "не должен появиться", "member_ids": members},
|
||||
)
|
||||
after = c.call("GET", "/api/v1/chats?limit=100", actor=alice, schema="ChatPage")["json"]
|
||||
assert before == after
|
||||
|
||||
|
||||
def test_concurrent_messages(scenario):
|
||||
c, alice, bob, _, chat = scenario
|
||||
barrier = threading.Barrier(10)
|
||||
|
||||
def sender(i):
|
||||
local = Client(c.base)
|
||||
barrier.wait()
|
||||
return [local.message(alice, chat, f"{i}:{j}")["json"] for j in range(10)]
|
||||
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
sent = [m for batch in pool.map(sender, range(10)) for m in batch]
|
||||
data = c.call(
|
||||
"GET", f"/api/v1/chats/{chat['id']}/messages?limit=100", actor=bob, schema="MessagePage"
|
||||
)["json"]
|
||||
assert len(sent) == len({m["id"] for m in sent}) == len(data["items"]) == 100
|
||||
assert {m["id"] for m in sent} == {m["id"] for m in data["items"]}
|
||||
assert data["next_cursor"] is None
|
||||
|
||||
|
||||
def test_registration_race(scenario):
|
||||
c, alice, *_ = scenario
|
||||
body = {**alice["registration"], "username": "race_" + uuid.uuid4().hex[:12]}
|
||||
barrier = threading.Barrier(10)
|
||||
|
||||
def register(_):
|
||||
local = Client(c.base)
|
||||
barrier.wait()
|
||||
return local.call("POST", "/api/v1/users", [201, 409], body=body)["status"]
|
||||
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
statuses = list(pool.map(register, range(10)))
|
||||
assert statuses.count(201) == 1
|
||||
assert statuses.count(409) == 9
|
||||
|
||||
|
||||
@pytest.mark.skipif(LAB < 2, reason="Сессии появляются в ЛР2")
|
||||
def test_sessions(scenario):
|
||||
c, alice, _, eve, _ = scenario
|
||||
credential = base64.b64encode(
|
||||
(alice["registration"]["username"] + ":wrong-password").encode()
|
||||
).decode()
|
||||
c.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": "Basic " + credential})
|
||||
c.call("GET", "/api/v1/users/me", 401, headers={"X-User-Id": alice["id"]})
|
||||
assert (
|
||||
c.call(
|
||||
"GET", "/api/v1/users/me", actor=alice, headers={"X-User-Id": eve["id"]}, schema="User"
|
||||
)["json"]
|
||||
== alice["user"]
|
||||
)
|
||||
c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=alice)
|
||||
c.call("GET", "/api/v1/users/me", 401, actor=alice)
|
||||
@@ -0,0 +1,203 @@
|
||||
"""Black-box regressions derived from the contract and teacher review comments."""
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import struct
|
||||
import uuid
|
||||
import zlib
|
||||
from io import BytesIO
|
||||
from urllib.parse import quote
|
||||
|
||||
import pytest
|
||||
from client import Client
|
||||
from PIL import Image
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def scenario():
|
||||
c = Client(os.environ.get("BASE_URL", "http://127.0.0.1:8085"))
|
||||
return c, *c.bootstrap()
|
||||
|
||||
|
||||
def image_bytes(fmt="PNG", size=(32, 20)):
|
||||
stream = BytesIO()
|
||||
Image.new("RGB", size, (100, 130, 200)).save(stream, fmt)
|
||||
return stream.getvalue()
|
||||
|
||||
|
||||
def upload(scenario, data=None, name="sample.png", mime="image/png", actor=None, status=202):
|
||||
c, alice, _, _, chat = scenario
|
||||
return c.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/attachments?filename={quote(name, safe='')}",
|
||||
status,
|
||||
actor=actor or alice,
|
||||
raw=data if data is not None else image_bytes(),
|
||||
headers={"Content-Type": mime},
|
||||
schema="Attachment" if status == 202 else None,
|
||||
)
|
||||
|
||||
|
||||
def test_roundtrip_acl_and_variant(scenario):
|
||||
c, alice, bob, eve, chat = scenario
|
||||
data = image_bytes()
|
||||
a = upload(scenario, data)["json"]
|
||||
assert a["sha256"] == hashlib.sha256(data).hexdigest()
|
||||
assert a["status"] == "queued"
|
||||
a = c.wait_ready(alice, a)
|
||||
assert {v["name"] for v in a["variants"]} == {"thumbnail", "crop", "optimized"}
|
||||
path = f"/api/v1/attachments/{a['id']}"
|
||||
assert c.call("GET", path + "/content", 200, actor=bob)["bytes"] == data
|
||||
c.call("GET", path, 404, actor=eve)
|
||||
c.call("GET", path + "/content", 404, actor=eve)
|
||||
c.call("GET", path + "/content?variant=thumbnail", 200, actor=alice)
|
||||
c.call("GET", path + "/content?variant=unknown", 400, actor=alice)
|
||||
message = c.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
201,
|
||||
actor=alice,
|
||||
body={"text": "", "attachment_ids": [a["id"]]},
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||
schema="Message",
|
||||
)["json"]
|
||||
assert message["text"] == "" and message["attachment_ids"] == [a["id"]]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"name", [" ", "../x.png", "a\\b.png", "a\0.png", "a\x7f.png", "a\x85.png", "a\n.png"]
|
||||
)
|
||||
def test_bad_filename(scenario, name):
|
||||
upload(scenario, name=name, status=400)
|
||||
|
||||
|
||||
def chunk(kind, data):
|
||||
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data))
|
||||
|
||||
|
||||
def png_with_data(data, width=1, height=1, color=2, palette=b""):
|
||||
return (
|
||||
b"\x89PNG\r\n\x1a\n"
|
||||
+ chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, color, 0, 0, 0))
|
||||
+ (chunk(b"PLTE", palette) if palette else b"")
|
||||
+ chunk(b"IDAT", data)
|
||||
+ chunk(b"IEND", b"")
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"data,mime",
|
||||
[
|
||||
(image_bytes()[:33], "image/png"),
|
||||
(image_bytes()[:-15], "image/png"),
|
||||
(png_with_data(b"broken-zlib"), "image/png"),
|
||||
(png_with_data(zlib.compress(b"\x05\0\0\0")), "image/png"),
|
||||
(png_with_data(zlib.compress(b"\0\0"), color=3, palette=bytes(771)), "image/png"),
|
||||
(bytes.fromhex("ffd8ffc0000b080001000101011100ffda0008010100003f00ffd9"), "image/jpeg"),
|
||||
(bytes.fromhex("ffd8ffc00008080001000101ffda0002ffd9"), "image/jpeg"),
|
||||
(image_bytes(), "image/jpeg"),
|
||||
(b"not-an-image", "image/png"),
|
||||
],
|
||||
)
|
||||
def test_corrupt_or_spoofed_image(scenario, data, mime):
|
||||
upload(scenario, data=data, mime=mime, status=415)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("size", [(8000, 5001), (20000, 10000)])
|
||||
def test_pixel_limit_before_decode(scenario, size):
|
||||
data = png_with_data(b"", width=size[0], height=size[1])
|
||||
upload(scenario, data=data, status=413)
|
||||
|
||||
|
||||
def test_byte_limit_and_exact_boundary(scenario):
|
||||
data = image_bytes()
|
||||
exact = data + bytes(10 * 1024 * 1024 - len(data))
|
||||
a = upload(scenario, data=exact)["json"]
|
||||
assert a["size_bytes"] == len(exact)
|
||||
upload(scenario, data=exact + b"x", status=413)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"body",
|
||||
[
|
||||
{"attachment_ids": []},
|
||||
{"text": None},
|
||||
{"text": "x", "attachment_ids": None},
|
||||
{"text": "x", "attachment_ids": [123]},
|
||||
],
|
||||
)
|
||||
def test_strict_message_fields(scenario, body):
|
||||
c, alice, _, _, chat = scenario
|
||||
c.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
400,
|
||||
actor=alice,
|
||||
body=body,
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||
)
|
||||
|
||||
|
||||
def test_foreign_unknown_and_ordered_attachments(scenario):
|
||||
c, alice, _, _, chat = scenario
|
||||
first, second = upload(scenario)["json"], upload(scenario)["json"]
|
||||
other = c.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
201,
|
||||
actor=alice,
|
||||
body={"title": "other", "member_ids": [alice["id"]]},
|
||||
schema="Chat",
|
||||
)["json"]
|
||||
for aid in [first["id"], str(uuid.uuid4())]:
|
||||
c.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{other['id']}/messages",
|
||||
404,
|
||||
actor=alice,
|
||||
body={"text": "", "attachment_ids": [aid]},
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||
)
|
||||
key = uuid.uuid4().hex
|
||||
body = {"text": "with files", "attachment_ids": [second["id"], first["id"]]}
|
||||
p = f"/api/v1/chats/{chat['id']}/messages"
|
||||
sent = c.call(
|
||||
"POST", p, 201, actor=alice, body=body, headers={"Idempotency-Key": key}, schema="Message"
|
||||
)["json"]
|
||||
repeat = c.call(
|
||||
"POST", p, 201, actor=alice, body=body, headers={"Idempotency-Key": key}, schema="Message"
|
||||
)["json"]
|
||||
assert repeat == sent
|
||||
c.call(
|
||||
"POST",
|
||||
p,
|
||||
409,
|
||||
actor=alice,
|
||||
body={**body, "attachment_ids": list(reversed(body["attachment_ids"]))},
|
||||
headers={"Idempotency-Key": key},
|
||||
)
|
||||
history = c.call("GET", p, 200, actor=alice, schema="MessagePage")["json"]["items"]
|
||||
assert next(m for m in history if m["id"] == sent["id"]) == sent
|
||||
|
||||
|
||||
def test_nul_message_roundtrip(scenario):
|
||||
c, alice, _, _, chat = scenario
|
||||
assert c.message(alice, chat, "abc\0def")["json"]["text"] == "abc\0def"
|
||||
|
||||
|
||||
def test_truncated_jpeg_with_forged_eoi(scenario):
|
||||
import random
|
||||
|
||||
stream = BytesIO()
|
||||
Image.frombytes("RGB", (512, 512), random.Random(42).randbytes(512 * 512 * 3)).save(
|
||||
stream, "JPEG", quality=95
|
||||
)
|
||||
original = stream.getvalue()
|
||||
upload(scenario, data=original, name="real.jpg", mime="image/jpeg")
|
||||
upload(
|
||||
scenario,
|
||||
data=original[: len(original) // 2] + b"\xff\xd9",
|
||||
name="broken.jpg",
|
||||
mime="image/jpeg",
|
||||
status=415,
|
||||
)
|
||||
@@ -0,0 +1,39 @@
|
||||
import base64
|
||||
import json
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from messenger.domain import Failure, boundary
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"version,date",
|
||||
[
|
||||
(True, "2026-01-01T00:00:00Z"),
|
||||
(1.0, "2026-01-01T00:00:00Z"),
|
||||
(1, "2026-W01-1T00:00:00Z"),
|
||||
(1, "20260101T000000Z"),
|
||||
(1, "2026-01-01 00:00:00Z"),
|
||||
(1, "2026-01-01X00:00:00Z"),
|
||||
(1, "2026-02-30T00:00:00Z"),
|
||||
(1, "2026-01-01T00:00:00.1234567Z"),
|
||||
],
|
||||
)
|
||||
def test_boundary_rejects_python_iso_extensions(version, date):
|
||||
payload = [version, "scope", date, str(uuid4())]
|
||||
cursor = base64.urlsafe_b64encode(json.dumps(payload).encode()).decode().rstrip("=")
|
||||
with pytest.raises(Failure) as error:
|
||||
boundary("scope", cursor)
|
||||
assert error.value.status == 400
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"date", ["2026-01-01T00:00:00Z", "2026-01-01T00:00:00.1Z", "2026-01-01T00:00:00.123456Z"]
|
||||
)
|
||||
def test_boundary_preserves_generated_format(date):
|
||||
uid = str(uuid4())
|
||||
cursor = (
|
||||
base64.urlsafe_b64encode(json.dumps([1, "scope", date, uid]).encode()).decode().rstrip("=")
|
||||
)
|
||||
assert boundary("scope", cursor) == (date, uid)
|
||||
@@ -0,0 +1,49 @@
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
from client import Client
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def shared():
|
||||
client = Client(os.getenv("BASE_URL", "http://127.0.0.1:8085"))
|
||||
return client, *client.bootstrap()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"date",
|
||||
[
|
||||
"2026-W01-1T00:00:00Z",
|
||||
"20260101T000000Z",
|
||||
"2026-01-01 00:00:00Z",
|
||||
"2026-01-01X00:00:00Z",
|
||||
"2026-01-01T00:00:00+00:00",
|
||||
"2026-02-30T00:00:00Z",
|
||||
"2026-01-01T24:00:00Z",
|
||||
"2026-01-01T00:00:00.1234567Z",
|
||||
],
|
||||
)
|
||||
def test_cursor_rejects_noncanonical_dates_over_http(shared, date):
|
||||
c, alice, _, _, chat = shared
|
||||
for path, scope in [
|
||||
("/api/v1/chats", "chats:" + alice["id"]),
|
||||
(f"/api/v1/chats/{chat['id']}/messages", "messages:" + chat["id"]),
|
||||
]:
|
||||
payload = [1, scope, date, str(uuid.uuid4())]
|
||||
cursor = base64.urlsafe_b64encode(json.dumps(payload).encode()).decode().rstrip("=")
|
||||
c.call("GET", path + "?cursor=" + cursor, 400, actor=alice)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("version", [True, 1.0])
|
||||
def test_cursor_rejects_noninteger_version_over_http(shared, version):
|
||||
c, alice, _, _, chat = shared
|
||||
for path, scope in [
|
||||
("/api/v1/chats", "chats:" + alice["id"]),
|
||||
(f"/api/v1/chats/{chat['id']}/messages", "messages:" + chat["id"]),
|
||||
]:
|
||||
payload = [version, scope, "2026-01-01T00:00:00Z", str(uuid.uuid4())]
|
||||
cursor = base64.urlsafe_b64encode(json.dumps(payload).encode()).decode().rstrip("=")
|
||||
c.call("GET", path + "?cursor=" + cursor, 400, actor=alice)
|
||||
@@ -0,0 +1,99 @@
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
from urllib.parse import quote
|
||||
|
||||
import pytest
|
||||
from client import LAB, Client
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def edge_scenario():
|
||||
client = Client(os.environ.get("BASE_URL", "http://127.0.0.1:8085"))
|
||||
return client, *client.bootstrap()
|
||||
|
||||
|
||||
def encoded_cursor(value):
|
||||
return base64.urlsafe_b64encode(json.dumps(value).encode()).decode().rstrip("=")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("identity", ["missing", "malformed", "unknown"])
|
||||
def test_invalid_identity(edge_scenario, identity):
|
||||
client, _, _, _, chat = edge_scenario
|
||||
headers = {}
|
||||
if identity == "malformed":
|
||||
headers["X-User-Id"] = "not-a-uuid"
|
||||
elif identity == "unknown":
|
||||
headers["X-User-Id"] = str(uuid.uuid4())
|
||||
for path in ["/api/v1/users/me", "/api/v1/chats", f"/api/v1/chats/{chat['id']}/messages"]:
|
||||
client.call("GET", path, 401, headers=headers)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("uid", [None, [], {}, 123, True, "not-a-uuid"])
|
||||
def test_cursor_rejects_non_uuid_boundary(edge_scenario, uid):
|
||||
client, alice, _, _, chat = edge_scenario
|
||||
payload = [1, "messages:" + chat["id"], "2026-10-30T12:00:00.000000Z", uid]
|
||||
cursor = quote(encoded_cursor(payload), safe="")
|
||||
client.call("GET", f"/api/v1/chats/{chat['id']}/messages?cursor={cursor}", 400, actor=alice)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"payload", [None, {}, [], [1], [1, "wrong-scope", "2026-10-30T12:00:00Z", str(uuid.uuid4())]]
|
||||
)
|
||||
def test_cursor_rejects_wrong_shape(edge_scenario, payload):
|
||||
client, alice, _, _, chat = edge_scenario
|
||||
cursor = quote(encoded_cursor(payload), safe="")
|
||||
client.call("GET", f"/api/v1/chats/{chat['id']}/messages?cursor={cursor}", 400, actor=alice)
|
||||
|
||||
|
||||
def test_request_id_is_present_and_not_reused(edge_scenario):
|
||||
client, alice, _, _, _ = edge_scenario
|
||||
first = client.call("GET", "/health/live", schema="Health")
|
||||
second = client.call("GET", "/health/ready", schema="Health")
|
||||
error = client.call("GET", "/api/v1/chats/malformed", 400, actor=alice)
|
||||
values = [r["headers"]["X-Request-Id"] for r in [first, second, error]]
|
||||
assert len(set(values)) == 3
|
||||
assert error["json"]["error"]["request_id"] == values[-1]
|
||||
|
||||
|
||||
def test_unicode_codepoint_boundaries(edge_scenario):
|
||||
client, alice, _, _, chat = edge_scenario
|
||||
username = "unicode_" + uuid.uuid4().hex[:12]
|
||||
registration = {"username": username, "display_name": "😀" * 100}
|
||||
if LAB >= 2:
|
||||
registration["password"] = "unicode-test-" + uuid.uuid4().hex
|
||||
user = client.call(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
201,
|
||||
body=registration,
|
||||
schema="User",
|
||||
)["json"]
|
||||
assert user["display_name"] == "😀" * 100
|
||||
client.call(
|
||||
"POST", "/api/v1/users", 400, body={"username": username + "x", "display_name": "😀" * 101}
|
||||
)
|
||||
message = client.message(alice, chat, "😀" * 4000)["json"]
|
||||
assert message["text"] == "😀" * 4000
|
||||
client.call(
|
||||
"POST", f"/api/v1/chats/{chat['id']}/messages", 400, actor=alice, body={"text": "😀" * 4001}
|
||||
)
|
||||
|
||||
|
||||
def test_empty_history_and_cursor_exhaustion(edge_scenario):
|
||||
client, alice, _, _, chat = edge_scenario
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
empty = client.call("GET", path, actor=alice, schema="MessagePage")["json"]
|
||||
assert empty == {"items": [], "next_cursor": None}
|
||||
message = client.message(alice, chat, "одна запись")["json"]
|
||||
data = client.call("GET", path + "?limit=1", actor=alice, schema="MessagePage")["json"]
|
||||
assert data == {"items": [message], "next_cursor": None}
|
||||
|
||||
|
||||
def test_missing_chat_is_hidden_like_foreign_chat(edge_scenario):
|
||||
client, _, _, eve, chat = edge_scenario
|
||||
foreign = client.call("GET", f"/api/v1/chats/{chat['id']}", 404, actor=eve)["json"]
|
||||
missing = client.call("GET", f"/api/v1/chats/{uuid.uuid4()}", 404, actor=eve)["json"]
|
||||
assert foreign["error"]["code"] == missing["error"]["code"] == "not_found"
|
||||
assert foreign["error"]["message"] == missing["error"]["message"]
|
||||
@@ -0,0 +1,229 @@
|
||||
"""Full-range JPEG and bounded multi-frame PNG regressions."""
|
||||
|
||||
import os
|
||||
import struct
|
||||
import zlib
|
||||
from io import BytesIO
|
||||
|
||||
import pytest
|
||||
from client import Client
|
||||
from PIL import Image
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def scenario():
|
||||
c = Client(os.getenv("BASE_URL", "http://127.0.0.1:8085"))
|
||||
return c, *c.bootstrap()
|
||||
|
||||
|
||||
def upload(scenario, data, mime, expected):
|
||||
c, alice, _, _, chat = scenario
|
||||
result = c.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/attachments?filename=edge.png",
|
||||
202 if expected == 201 else expected,
|
||||
actor=alice,
|
||||
raw=data,
|
||||
headers={"Content-Type": mime},
|
||||
schema="Attachment" if expected == 201 else None,
|
||||
)
|
||||
if expected == 201:
|
||||
result["json"] = c.wait_ready(alice, result["json"])
|
||||
return result
|
||||
|
||||
|
||||
def progressive_jpeg():
|
||||
output = BytesIO()
|
||||
image = Image.new("RGB", (6000, 6000), "blue")
|
||||
try:
|
||||
image.save(output, "JPEG", quality=90, progressive=True, subsampling=0)
|
||||
finally:
|
||||
image.close()
|
||||
return output.getvalue()
|
||||
|
||||
|
||||
def apng(size=(16, 16), default_image=False):
|
||||
output = BytesIO()
|
||||
first, second = Image.new("RGB", size, "red"), Image.new("RGB", size, "blue")
|
||||
try:
|
||||
first.save(
|
||||
output,
|
||||
"PNG",
|
||||
save_all=True,
|
||||
append_images=[second],
|
||||
duration=100,
|
||||
loop=0,
|
||||
default_image=default_image,
|
||||
)
|
||||
finally:
|
||||
first.close()
|
||||
second.close()
|
||||
return output.getvalue()
|
||||
|
||||
|
||||
def broken_later_frame(data):
|
||||
result, position = bytearray(data[:8]), 8
|
||||
while position < len(data):
|
||||
length = struct.unpack(">I", data[position : position + 4])[0]
|
||||
kind = data[position + 4 : position + 8]
|
||||
payload = data[position + 8 : position + 8 + length]
|
||||
if kind == b"fdAT":
|
||||
payload = payload[:4] + b"not-a-valid-deflate-stream"
|
||||
result += (
|
||||
struct.pack(">I", len(payload))
|
||||
+ kind
|
||||
+ payload
|
||||
+ struct.pack(">I", zlib.crc32(kind + payload))
|
||||
)
|
||||
position += length + 12
|
||||
return bytes(result)
|
||||
|
||||
|
||||
def test_36m_progressive_jpeg_is_valid_and_unchanged(scenario):
|
||||
data = progressive_jpeg()
|
||||
attachment = upload(scenario, data, "image/jpeg", 201)["json"]
|
||||
c, _, bob, _, _ = scenario
|
||||
assert (
|
||||
c.call("GET", f"/api/v1/attachments/{attachment['id']}/content", actor=bob)["bytes"] == data
|
||||
)
|
||||
|
||||
|
||||
def test_apng_corrupt_later_frame_is_rejected(scenario):
|
||||
upload(scenario, broken_later_frame(apng()), "image/png", 415)
|
||||
|
||||
|
||||
def test_valid_small_apng_is_accepted_and_unchanged(scenario):
|
||||
data = apng()
|
||||
attachment = upload(scenario, data, "image/png", 201)["json"]
|
||||
c, _, bob, _, _ = scenario
|
||||
assert (
|
||||
c.call("GET", f"/api/v1/attachments/{attachment['id']}/content", actor=bob)["bytes"] == data
|
||||
)
|
||||
|
||||
|
||||
def test_apng_cumulative_canvas_limit(scenario):
|
||||
# Two 25M-pixel canvases exceed the total 40M decode budget.
|
||||
upload(scenario, apng((5000, 5000)), "image/png", 413)
|
||||
|
||||
|
||||
def test_apng_forged_frame_count_cannot_hide_bad_frame(scenario):
|
||||
data = bytearray(broken_later_frame(apng()))
|
||||
position = 8
|
||||
while position < len(data):
|
||||
length = struct.unpack(">I", data[position : position + 4])[0]
|
||||
kind = bytes(data[position + 4 : position + 8])
|
||||
if kind == b"acTL":
|
||||
data[position + 8 : position + 12] = struct.pack(">I", 1)
|
||||
payload = bytes(data[position + 8 : position + 8 + length])
|
||||
data[position + 8 + length : position + 12 + length] = struct.pack(
|
||||
">I", zlib.crc32(kind + payload)
|
||||
)
|
||||
break
|
||||
position += length + 12
|
||||
upload(scenario, bytes(data), "image/png", 415)
|
||||
|
||||
|
||||
def test_apng_zero_count_with_static_fallback_is_rejected(scenario):
|
||||
# Pillow warns and falls back to one valid static PNG frame. The raw acTL
|
||||
# count must still be rejected rather than trusting Pillow's n_frames=1.
|
||||
data = apng()
|
||||
result, position, controls = bytearray(data[:8]), 8, 0
|
||||
while position < len(data):
|
||||
length = int.from_bytes(data[position : position + 4], "big")
|
||||
kind = data[position + 4 : position + 8]
|
||||
payload = data[position + 8 : position + 8 + length]
|
||||
position += length + 12
|
||||
if kind == b"acTL":
|
||||
payload = bytes(4) + payload[4:]
|
||||
if kind == b"fcTL":
|
||||
controls += 1
|
||||
if controls > 1:
|
||||
continue
|
||||
if kind == b"fdAT":
|
||||
continue
|
||||
result += (
|
||||
struct.pack(">I", len(payload))
|
||||
+ kind
|
||||
+ payload
|
||||
+ struct.pack(">I", zlib.crc32(kind + payload))
|
||||
)
|
||||
upload(scenario, bytes(result), "image/png", 415)
|
||||
|
||||
|
||||
def test_apng_exact_cumulative_limit_is_accepted(scenario):
|
||||
upload(scenario, apng((4000, 5000)), "image/png", 201)
|
||||
|
||||
|
||||
def test_40m_cmyk_progressive_jpeg_is_accepted(scenario):
|
||||
output = BytesIO()
|
||||
image = Image.new("CMYK", (8000, 5000), (0, 128, 128, 0))
|
||||
try:
|
||||
image.save(output, "JPEG", quality=90, progressive=True, subsampling=0)
|
||||
finally:
|
||||
image.close()
|
||||
data = output.getvalue()
|
||||
attachment = upload(scenario, data, "image/jpeg", 201)["json"]
|
||||
c, _, bob, _, _ = scenario
|
||||
assert (
|
||||
c.call("GET", f"/api/v1/attachments/{attachment['id']}/content", actor=bob)["bytes"] == data
|
||||
)
|
||||
|
||||
|
||||
def test_repeated_large_image_sequence_stays_available(scenario):
|
||||
# Formerly retained APNG canvases could overlap the next CMYK decoder.
|
||||
c, _, bob, _, _ = scenario
|
||||
data = apng((4000, 5000))
|
||||
attachment = upload(scenario, data, "image/png", 201)["json"]
|
||||
assert (
|
||||
c.call("GET", f"/api/v1/attachments/{attachment['id']}/content", actor=bob)["bytes"] == data
|
||||
)
|
||||
del data
|
||||
test_40m_cmyk_progressive_jpeg_is_accepted(scenario)
|
||||
test_36m_progressive_jpeg_is_valid_and_unchanged(scenario)
|
||||
data = BytesIO()
|
||||
image = Image.new("RGB", (16, 16), "green")
|
||||
image.save(data, "PNG")
|
||||
image.close()
|
||||
attachment = upload(scenario, data.getvalue(), "image/png", 201)["json"]
|
||||
assert (
|
||||
c.call("GET", f"/api/v1/attachments/{attachment['id']}/content", actor=bob)["bytes"]
|
||||
== data.getvalue()
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("progressive", [False, True])
|
||||
def test_small_cmyk_jpeg(progressive, scenario):
|
||||
output = BytesIO()
|
||||
image = Image.new("CMYK", (24, 18), (0, 128, 128, 0))
|
||||
image.save(output, "JPEG", progressive=progressive)
|
||||
image.close()
|
||||
upload(scenario, output.getvalue(), "image/jpeg", 201)
|
||||
|
||||
|
||||
def test_invalid_zero_frame_count_is_rejected(scenario):
|
||||
data = apng()
|
||||
result, position, controls = bytearray(data[:8]), 8, 0
|
||||
while position < len(data):
|
||||
length = struct.unpack(">I", data[position : position + 4])[0]
|
||||
kind = data[position + 4 : position + 8]
|
||||
payload = data[position + 8 : position + 8 + length]
|
||||
position += length + 12
|
||||
if kind == b"fcTL":
|
||||
controls += 1
|
||||
if controls > 1:
|
||||
continue
|
||||
if kind == b"fdAT":
|
||||
continue
|
||||
if kind == b"acTL":
|
||||
payload = bytes(4) + payload[4:]
|
||||
result += (
|
||||
struct.pack(">I", len(payload))
|
||||
+ kind
|
||||
+ payload
|
||||
+ struct.pack(">I", zlib.crc32(kind + payload))
|
||||
)
|
||||
upload(scenario, bytes(result), "image/png", 415)
|
||||
|
||||
|
||||
def test_valid_apng_default_image(scenario):
|
||||
upload(scenario, apng(default_image=True), "image/png", 201)
|
||||
@@ -0,0 +1,72 @@
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from messenger.worker import validate_delivery
|
||||
|
||||
VALID = {
|
||||
"schema_version": 1,
|
||||
"job_id": "11111111-1111-4111-8111-111111111111",
|
||||
"type": "attachment.process.v1",
|
||||
"attachment_id": "22222222-2222-4222-8222-222222222222",
|
||||
"pipeline_version": 1,
|
||||
"attempt": 1,
|
||||
"created_at": "2026-11-06T00:00:00Z",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"key,value",
|
||||
[
|
||||
("job_id", 123),
|
||||
("job_id", []),
|
||||
("attachment_id", {}),
|
||||
("schema_version", True),
|
||||
("pipeline_version", True),
|
||||
("attempt", 0),
|
||||
("created_at", 123),
|
||||
("created_at", "2026-11-06"),
|
||||
("created_at", "2026-11-06T12Z"),
|
||||
("created_at", "2026-99-99T00:00:00Z"),
|
||||
("traceparent", 123),
|
||||
("tracestate", []),
|
||||
],
|
||||
)
|
||||
def test_invalid_envelopes_are_bounded_validation_errors(key, value):
|
||||
with pytest.raises(ValueError):
|
||||
validate_delivery(json.dumps({**VALID, key: value}))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("body", [b"\xff", "null", "[]", "{}", "not json"])
|
||||
def test_invalid_json(body):
|
||||
with pytest.raises(ValueError):
|
||||
validate_delivery(body)
|
||||
|
||||
|
||||
def test_valid_trace_context():
|
||||
body = {
|
||||
**VALID,
|
||||
"traceparent": "00-" + "a" * 32 + "-" + "b" * 16 + "-01",
|
||||
"tracestate": "test=value",
|
||||
}
|
||||
assert validate_delivery(json.dumps(body)) == body
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"body", ["[" * 2000 + "]" * 2000, "[" * 20000 + "]" * 20000, b" " * 8193 + b"{}"]
|
||||
)
|
||||
def test_deep_and_oversize_envelopes_reject_safely(body):
|
||||
with pytest.raises(ValueError):
|
||||
validate_delivery(body)
|
||||
|
||||
|
||||
def test_uuid_uppercase_is_normalized_before_storage():
|
||||
value = "AAAAAAAA-AAAA-4AAA-8AAA-AAAAAAAAAAAA"
|
||||
parsed = validate_delivery(json.dumps({**VALID, "job_id": value, "attachment_id": value}))
|
||||
assert parsed["job_id"] == parsed["attachment_id"] == value.lower()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ["job_id", "attachment_id"])
|
||||
def test_uuid_urn_is_rejected_before_storage(field):
|
||||
with pytest.raises(ValueError):
|
||||
validate_delivery(json.dumps({**VALID, field: "urn:uuid:" + VALID[field]}))
|
||||
@@ -0,0 +1,169 @@
|
||||
import base64
|
||||
import codecs
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from client import Client
|
||||
|
||||
ENCODINGS = [
|
||||
"utf-16",
|
||||
"utf-16-le",
|
||||
"utf-16-be",
|
||||
"utf-16-be-bom",
|
||||
"utf-32",
|
||||
"utf-32-le",
|
||||
"utf-32-be",
|
||||
"utf-32-be-bom",
|
||||
]
|
||||
|
||||
|
||||
def encoded(body, encoding):
|
||||
text = json.dumps(body, ensure_ascii=False)
|
||||
if encoding == "utf-16-be-bom":
|
||||
return codecs.BOM_UTF16_BE + text.encode("utf-16-be")
|
||||
if encoding == "utf-32-be-bom":
|
||||
return codecs.BOM_UTF32_BE + text.encode("utf-32-be")
|
||||
return text.encode(encoding)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def encoding_scenario():
|
||||
c = Client(os.environ["BASE_URL"])
|
||||
return c, *c.bootstrap()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("encoding", ENCODINGS)
|
||||
def test_non_utf8_registration_has_no_write(encoding_scenario, encoding):
|
||||
c, *_ = encoding_scenario
|
||||
body = {
|
||||
"username": "enc_" + uuid.uuid4().hex[:14],
|
||||
"display_name": "Никита\u0000имя",
|
||||
"password": "Пароль:тестовый-12345",
|
||||
}
|
||||
c.call(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
400,
|
||||
raw=encoded(body, encoding),
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
# UNIQUE username proves the rejected request did not insert a user.
|
||||
created = c.call("POST", "/api/v1/users", 201, body=body, schema="User")["json"]
|
||||
assert created["display_name"] == body["display_name"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("encoding", ENCODINGS)
|
||||
def test_non_utf8_chat_has_no_write(encoding_scenario, encoding):
|
||||
c, alice, *_ = encoding_scenario
|
||||
before = c.call("GET", "/api/v1/chats?limit=100", actor=alice, schema="ChatPage")["json"]
|
||||
body = {"title": "Чат\u0000" + uuid.uuid4().hex, "member_ids": [alice["id"]]}
|
||||
c.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
400,
|
||||
actor=alice,
|
||||
raw=encoded(body, encoding),
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
after = c.call("GET", "/api/v1/chats?limit=100", actor=alice, schema="ChatPage")["json"]
|
||||
assert before == after
|
||||
assert (
|
||||
c.call("POST", "/api/v1/chats", 201, actor=alice, body=body, schema="Chat")["json"]["title"]
|
||||
== body["title"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("encoding", ENCODINGS)
|
||||
def test_non_utf8_message_has_no_write(encoding_scenario, encoding):
|
||||
c, alice, _, _, chat = encoding_scenario
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
key = uuid.uuid4().hex
|
||||
body = {"text": "Сообщение\u0000" + key}
|
||||
c.call(
|
||||
"POST",
|
||||
path,
|
||||
400,
|
||||
actor=alice,
|
||||
raw=encoded(body, encoding),
|
||||
headers={"Content-Type": "application/json", "Idempotency-Key": key},
|
||||
)
|
||||
history = c.call("GET", path + "?limit=100", actor=alice, schema="MessagePage")["json"]["items"]
|
||||
assert all(item["text"] != body["text"] for item in history)
|
||||
good = c.call(
|
||||
"POST",
|
||||
path,
|
||||
201,
|
||||
actor=alice,
|
||||
body=body,
|
||||
headers={"Idempotency-Key": key},
|
||||
schema="Message",
|
||||
)["json"]
|
||||
repeat = c.call(
|
||||
"POST",
|
||||
path,
|
||||
201,
|
||||
actor=alice,
|
||||
body=body,
|
||||
headers={"Idempotency-Key": key},
|
||||
schema="Message",
|
||||
)["json"]
|
||||
assert good == repeat and good["text"] == body["text"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("encoding", ["utf-8", "utf-8-sig"])
|
||||
def test_valid_utf8_registration_and_login(encoding_scenario, encoding):
|
||||
c, *_ = encoding_scenario
|
||||
body = {
|
||||
"username": "utf8_" + uuid.uuid4().hex[:14],
|
||||
"display_name": "Юникод\u0000😀",
|
||||
"password": "Пароль:тестовый-12345",
|
||||
}
|
||||
user = c.call(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
201,
|
||||
raw=encoded(body, encoding),
|
||||
headers={"Content-Type": "application/json"},
|
||||
schema="User",
|
||||
)["json"]
|
||||
assert user["display_name"] == body["display_name"]
|
||||
credential = base64.b64encode((body["username"] + ":" + body["password"]).encode()).decode()
|
||||
session = c.call(
|
||||
"POST",
|
||||
"/api/v1/auth/sessions",
|
||||
201,
|
||||
headers={"Authorization": "Basic " + credential},
|
||||
schema="Session",
|
||||
)["json"]
|
||||
assert session["user"] == user
|
||||
|
||||
|
||||
def requests_for(alice, chat):
|
||||
return [
|
||||
(
|
||||
"/api/v1/users",
|
||||
{
|
||||
"username": "utf_" + uuid4().hex[:12],
|
||||
"display_name": "Имя\0",
|
||||
"password": "synthetic-password-123",
|
||||
},
|
||||
),
|
||||
("/api/v1/chats", {"title": "Чат\0", "member_ids": [alice["id"]]}),
|
||||
(f"/api/v1/chats/{chat['id']}/messages", {"text": "Сообщение\0"}),
|
||||
]
|
||||
|
||||
|
||||
def test_utf8_and_escaped_nul_remain_valid(encoding_scenario):
|
||||
c, alice, _, _, chat = encoding_scenario
|
||||
for path, body in requests_for(alice, chat):
|
||||
c.call(
|
||||
"POST",
|
||||
path,
|
||||
201,
|
||||
actor=alice,
|
||||
raw=json.dumps(body, ensure_ascii=False).encode("utf-8"),
|
||||
headers={"Content-Type": "application/json", "Idempotency-Key": uuid4().hex},
|
||||
)
|
||||
@@ -0,0 +1,46 @@
|
||||
from io import BytesIO
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from messenger.image_pipeline import fitted_size, render
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"source,expected",
|
||||
[((319, 200), (256, 160)), ((320, 200), (256, 160)), ((10, 4), (10, 4)), ((1, 1000), (1, 256))],
|
||||
)
|
||||
def test_fits_contract(source, expected):
|
||||
assert fitted_size(*source, 256) == expected
|
||||
|
||||
|
||||
def test_variants_alpha_and_metadata(tmp_path):
|
||||
source = BytesIO()
|
||||
Image.new("RGBA", (319, 200), (255, 0, 0, 0)).save(source, "PNG")
|
||||
source.seek(0)
|
||||
variants = render(source, tmp_path)
|
||||
assert {v["name"] for v in variants} == {"thumbnail", "crop", "optimized"}
|
||||
with Image.open(tmp_path / "thumbnail") as thumb:
|
||||
assert thumb.format == "JPEG"
|
||||
assert thumb.size == (256, 160)
|
||||
assert all(channel > 245 for channel in thumb.getpixel((0, 0)))
|
||||
assert not thumb.getexif()
|
||||
with Image.open(tmp_path / "crop") as crop:
|
||||
assert crop.size == (128, 128)
|
||||
with Image.open(tmp_path / "optimized") as optimized:
|
||||
assert optimized.format == "WEBP"
|
||||
assert optimized.size == (319, 200)
|
||||
|
||||
|
||||
def test_exif_orientation_without_upscale(tmp_path):
|
||||
source = BytesIO()
|
||||
exif = Image.Exif()
|
||||
exif[274] = 6
|
||||
Image.new("RGB", (80, 40), "blue").save(source, "JPEG", exif=exif)
|
||||
original = source.getvalue()
|
||||
source.seek(0)
|
||||
render(source, tmp_path)
|
||||
assert source.getvalue() == original
|
||||
with Image.open(tmp_path / "thumbnail") as thumbnail:
|
||||
assert thumbnail.size == (40, 80)
|
||||
assert not thumbnail.getexif()
|
||||
@@ -0,0 +1,62 @@
|
||||
import base64
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
from client import Client
|
||||
|
||||
|
||||
@pytest.mark.parametrize("password", ["x" * 11, "x" * 129, "😀" * 129, 123, None, True])
|
||||
def test_password_contract(password):
|
||||
import os
|
||||
|
||||
c = Client(os.environ["BASE_URL"])
|
||||
c.call(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
400,
|
||||
body={
|
||||
"username": "pwd_" + uuid.uuid4().hex[:12],
|
||||
"display_name": "Тест",
|
||||
"password": password,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_full_unicode_password_and_colon():
|
||||
import os
|
||||
|
||||
c = Client(os.environ["BASE_URL"])
|
||||
username = "pwd_" + uuid.uuid4().hex[:12]
|
||||
password = "😀" * 126 + ":Я"
|
||||
user = c.call(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
201,
|
||||
body={"username": username, "display_name": "Пароль128", "password": password},
|
||||
schema="User",
|
||||
)["json"]
|
||||
good = base64.b64encode((username + ":" + password).encode()).decode()
|
||||
session = c.call(
|
||||
"POST",
|
||||
"/api/v1/auth/sessions",
|
||||
201,
|
||||
headers={"Authorization": "bAsIc " + good},
|
||||
schema="Session",
|
||||
)["json"]
|
||||
actor = {"id": user["id"], "token": session["token"]}
|
||||
assert session["user"] == user
|
||||
bad = base64.b64encode((username + ":" + password[:-1] + "Б").encode()).decode()
|
||||
c.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": "Basic " + bad})
|
||||
c.call("GET", "/api/v1/users/me", 401, headers={"Authorization": "Basic " + good})
|
||||
c.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
||||
c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=actor)
|
||||
c.call("GET", "/api/v1/users/me", 401, actor=actor)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("credential", ["Basic !invalid", "Basic", "Bearer abc", "", "Basic YTpi"])
|
||||
def test_invalid_basic(credential):
|
||||
import os
|
||||
|
||||
Client(os.environ["BASE_URL"]).call(
|
||||
"POST", "/api/v1/auth/sessions", 401, headers={"Authorization": credential}
|
||||
)
|
||||
@@ -0,0 +1,231 @@
|
||||
"""Identity and prefix negative probes use a real local TLS server, no mocked TLS."""
|
||||
|
||||
import http.client
|
||||
import json
|
||||
import socket
|
||||
import socketserver
|
||||
import ssl
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from messenger.tls_gateway import HTTPHandler, HTTPServer, TCPHandler, TCPServer, role
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def cfg(name, allowed, upstream, **extra):
|
||||
return dict(
|
||||
cert=str(ROOT / f"secrets/pki/{name}/cert.pem"),
|
||||
key=str(ROOT / f"secrets/pki/{name}/key.pem"),
|
||||
ca=str(ROOT / "secrets/ca.pem"),
|
||||
allowed=allowed,
|
||||
upstream=upstream,
|
||||
**extra,
|
||||
)
|
||||
|
||||
|
||||
def client(identity=None):
|
||||
ctx = ssl.create_default_context(cafile=str(ROOT / "secrets/ca.pem"))
|
||||
if identity:
|
||||
ctx.load_cert_chain(
|
||||
ROOT / f"secrets/pki/{identity}/cert.pem", ROOT / f"secrets/pki/{identity}/key.pem"
|
||||
)
|
||||
return ctx
|
||||
|
||||
|
||||
class Echo(socketserver.BaseRequestHandler):
|
||||
def handle(self):
|
||||
while data := self.request.recv(1024):
|
||||
self.request.sendall(data)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tcp_boundary(monkeypatch, tmp_path):
|
||||
revoked = tmp_path / "revoked.json"
|
||||
revoked.write_text("[]")
|
||||
monkeypatch.setenv("TLS_REVOKED_FILE", str(revoked))
|
||||
with socketserver.ThreadingTCPServer(("127.0.0.1", 0), Echo) as echo:
|
||||
threading.Thread(target=echo.serve_forever, daemon=True).start()
|
||||
with TCPServer(("127.0.0.1", 0), TCPHandler) as proxy:
|
||||
proxy.cfg = cfg("api", ["lb"], echo.server_address[1])
|
||||
proxy.slots = threading.BoundedSemaphore(32)
|
||||
threading.Thread(target=proxy.serve_forever, daemon=True).start()
|
||||
yield proxy.server_address[1], revoked
|
||||
proxy.shutdown()
|
||||
echo.shutdown()
|
||||
|
||||
|
||||
def exchange(port, ctx, hostname="api"):
|
||||
with socket.create_connection(("127.0.0.1", port), timeout=2) as raw:
|
||||
with ctx.wrap_socket(raw, server_hostname=hostname) as conn:
|
||||
conn.sendall(b"positive")
|
||||
return conn.recv(1024)
|
||||
|
||||
|
||||
def test_allowed_lb_identity(tcp_boundary):
|
||||
assert exchange(tcp_boundary[0], client("lb")) == b"positive"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("identity", [None, "worker", "operator"])
|
||||
def test_missing_or_trusted_wrong_identity_denied(tcp_boundary, identity):
|
||||
try:
|
||||
result = exchange(tcp_boundary[0], client(identity))
|
||||
except (OSError, ssl.SSLError):
|
||||
result = b""
|
||||
assert result == b""
|
||||
|
||||
|
||||
def test_wrong_hostname(tcp_boundary):
|
||||
with pytest.raises(ssl.SSLCertVerificationError):
|
||||
exchange(tcp_boundary[0], client("lb"), "wrong-service")
|
||||
|
||||
|
||||
def test_empty_ca(tcp_boundary):
|
||||
ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
|
||||
ctx.load_cert_chain(ROOT / "secrets/pki/lb/cert.pem", ROOT / "secrets/pki/lb/key.pem")
|
||||
with pytest.raises(ssl.SSLCertVerificationError):
|
||||
exchange(tcp_boundary[0], ctx)
|
||||
|
||||
|
||||
def test_revocation_new_connection(tcp_boundary):
|
||||
port, revoked = tcp_boundary
|
||||
assert exchange(port, client("lb")) == b"positive"
|
||||
cert = ssl._ssl._test_decode_cert(str(ROOT / "secrets/pki/lb/cert.pem"))
|
||||
revoked.write_text(json.dumps([cert["serialNumber"].lstrip("0").lower()]))
|
||||
try:
|
||||
result = exchange(port, client("lb"))
|
||||
except (OSError, ssl.SSLError):
|
||||
result = b""
|
||||
assert result == b""
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"method,path,headers",
|
||||
[
|
||||
("PUT", "/messenger/private/secret", {}),
|
||||
("PUT", "/messenger/staging/new", {}),
|
||||
("GET", "/other-bucket/secret", {}),
|
||||
("GET", "/messenger/staging/../private/secret", {}),
|
||||
("GET", "/messenger/staging/%2e%2e/private/secret", {}),
|
||||
("GET", "/messenger/staging/%252e%252e/private/secret", {}),
|
||||
(
|
||||
"PUT",
|
||||
"/messenger/attachments/id/v1/original",
|
||||
{"x-amz-copy-source": "/messenger/private/secret"},
|
||||
),
|
||||
("GET", "/messenger?list-type=2", {}),
|
||||
("GET", "https://storage:8443/messenger/staging/id", {}),
|
||||
("GET", "/messenger/staging/id#fragment", {}),
|
||||
],
|
||||
)
|
||||
def test_worker_prefix_and_copy_bypasses_denied(tmp_path, monkeypatch, method, path, headers):
|
||||
revoked = tmp_path / "revoke.json"
|
||||
revoked.write_text("[]")
|
||||
monkeypatch.setenv("TLS_REVOKED_FILE", str(revoked))
|
||||
with HTTPServer(("127.0.0.1", 0), HTTPHandler) as proxy:
|
||||
proxy.cfg = cfg("storage", ["api", "worker", "operator"], 1, http=True, s3=True)
|
||||
proxy.slots = threading.BoundedSemaphore(32)
|
||||
threading.Thread(target=proxy.serve_forever, daemon=True).start()
|
||||
conn = http.client.HTTPSConnection(
|
||||
"127.0.0.1", proxy.server_address[1], context=client("worker"), timeout=2
|
||||
)
|
||||
conn.request(method, path, headers={"Host": "storage:8443", **headers})
|
||||
assert conn.getresponse().status == 403
|
||||
conn.close()
|
||||
proxy.shutdown()
|
||||
|
||||
|
||||
def test_exactly_one_allowed_san():
|
||||
with pytest.raises(ssl.SSLError):
|
||||
role({"subjectAltName": [("DNS", "worker"), ("DNS", "lb")]}, ["lb", "worker"])
|
||||
|
||||
|
||||
def test_expired_certificate_rejected(tcp_boundary, tmp_path):
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
cert = tmp_path / "expired.pem"
|
||||
subprocess.run(
|
||||
[
|
||||
"openssl",
|
||||
"x509",
|
||||
"-req",
|
||||
"-in",
|
||||
str(ROOT / "operator-private/worker.csr"),
|
||||
"-CA",
|
||||
str(ROOT / "operator-private/ca.pem"),
|
||||
"-CAkey",
|
||||
str(ROOT / "operator-private/ca-key.pem"),
|
||||
"-set_serial",
|
||||
"999999",
|
||||
"-days",
|
||||
"0",
|
||||
"-extfile",
|
||||
str(ROOT / "operator-private/worker.ext"),
|
||||
"-out",
|
||||
str(cert),
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
)
|
||||
time.sleep(1)
|
||||
ctx = client()
|
||||
ctx.load_cert_chain(cert, ROOT / "secrets/pki/worker/key.pem")
|
||||
try:
|
||||
result = exchange(tcp_boundary[0], ctx)
|
||||
except (OSError, ssl.SSLError):
|
||||
result = b""
|
||||
assert result == b""
|
||||
|
||||
|
||||
def test_http_forward_preserves_object_bytes(tmp_path, monkeypatch):
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
class Object(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
body = b"original-bytes"
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, *_):
|
||||
pass
|
||||
|
||||
revoked = tmp_path / "revoked.json"
|
||||
revoked.write_text("[]")
|
||||
monkeypatch.setenv("TLS_REVOKED_FILE", str(revoked))
|
||||
with ThreadingHTTPServer(("127.0.0.1", 0), Object) as upstream:
|
||||
threading.Thread(target=upstream.serve_forever, daemon=True).start()
|
||||
with HTTPServer(("127.0.0.1", 0), HTTPHandler) as proxy:
|
||||
proxy.cfg = cfg("storage", ["worker"], upstream.server_address[1], http=True, s3=True)
|
||||
proxy.slots = threading.BoundedSemaphore(32)
|
||||
threading.Thread(target=proxy.serve_forever, daemon=True).start()
|
||||
conn = http.client.HTTPSConnection(
|
||||
"127.0.0.1", proxy.server_address[1], context=client("worker"), timeout=2
|
||||
)
|
||||
conn.request("GET", "/messenger/staging/id", headers={"Host": "storage:8443"})
|
||||
response = conn.getresponse()
|
||||
assert response.status == 200
|
||||
assert response.read() == b"original-bytes"
|
||||
conn.close()
|
||||
proxy.shutdown()
|
||||
upstream.shutdown()
|
||||
|
||||
|
||||
def test_revocation_existing_connection_before_next_command(tcp_boundary):
|
||||
port, revoked = tcp_boundary
|
||||
with socket.create_connection(("127.0.0.1", port), timeout=2) as raw:
|
||||
with client("lb").wrap_socket(raw, server_hostname="api") as conn:
|
||||
conn.sendall(b"before-revocation")
|
||||
assert conn.recv(1024) == b"before-revocation"
|
||||
cert = ssl._ssl._test_decode_cert(str(ROOT / "secrets/pki/lb/cert.pem"))
|
||||
revoked.write_text(json.dumps([cert["serialNumber"].lstrip("0").lower()]))
|
||||
try:
|
||||
conn.sendall(b"must-not-forward")
|
||||
data = conn.recv(1024)
|
||||
except OSError:
|
||||
data = b""
|
||||
assert data == b""
|
||||
Reference in New Issue
Block a user