Carry over the lab 4 solution

Bring the application, own tests, Dockerfile, toolchain, compose stack,
HAProxy, k6 and RustFS configuration, migrations, hooks, evidence and
scenario scripts and the CI job from the lab 4 solution commit d2744bc,
merge the lab 4 Makefile targets into the lab 5 skeleton Makefile
without touching its targets, and append the lab 4 solution variables
to the skeleton .env.example. The package and the application follow
the contract version 1.5.0, and the image and the throwaway test
containers are tagged lab5 so they do not collide with a lab 4 stand on
the same host.

This is the starting point of the lab 5 cycle: lint, own tests and the
offline skeleton check pass before any lab 5 change.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Artyom
2026-10-29 00:34:03 +03:00
co-authored by Claude Opus 5.5
parent 1210dd899c
commit 540fcb35ba
171 changed files with 12909 additions and 1 deletions
+24
View File
@@ -14,7 +14,31 @@ backups
*.pfx
*.jks
.venv
venv
__pycache__
*.pyc
*.pyo
.pytest_cache
.ruff_cache
.mypy_cache
.ty_cache
node_modules
evidence
openspec
student-tests
tests
scripts
examples
contracts
infra
*.md
!README.md
Makefile
Dockerfile*
compose*.yaml
.dockerignore
.gitignore
.DS_Store
lab.json
requirements-dev.txt
.claude
+32
View File
@@ -23,3 +23,35 @@ WORKER_CONCURRENCY=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
STAGING_RETENTION_SECONDS=86400
# Solution-specific: published LB ports, replica limits, workers and pool per replica,
# LB queue per replica, drain window before closing the listener on SIGTERM, PostgreSQL
# limits, load generator limits (profile bench), cursor key file. API_PORT/SHORTTTL_PORT of lab2 are gone:
# only the load balancer is published.
LB_HTTP_PORT=8080
LB_STATS_PORT=8404
API_CPUS=1
API_MEMORY=512m
WEB_CONCURRENCY=2
DB_POOL_MAX_SIZE=8
API_SERVER_MAXCONN=64
DRAIN_SECONDS=2
PG_MAX_CONNECTIONS=200
PG_CPUS=2
PG_MEMORY=1g
K6_CPUS=2
K6_MEMORY=512m
CURSOR_SECRET_FILE=/run/secrets/cursor_secret
# Solution-specific for lab4: S3 client timeouts (connect + read, two attempts stay below
# REQUEST_TIMEOUT_SECONDS=8), parallel image checks per worker, RustFS memory limit.
S3_CONNECT_TIMEOUT_SECONDS=1
S3_READ_TIMEOUT_SECONDS=2
IMAGE_CHECK_CONCURRENCY=2
S3_MEMORY=512m
# Solution-specific for lab4 grade 4: upload bodies buffered at once per worker, grace period
# before an abandoned pending upload is swept, sweep interval and rows per sweep pass.
UPLOAD_CONCURRENCY=3
UPLOAD_GRACE_SECONDS=900
UPLOAD_SWEEP_INTERVAL_SECONDS=30
UPLOAD_SWEEP_BATCH=100
# Fixed glibc mmap threshold for the API: buffers above 128 KiB are returned to the system on free.
MALLOC_MMAP_THRESHOLD_=131072
+88
View File
@@ -0,0 +1,88 @@
name: Application
on: [push, pull_request]
permissions:
contents: read
jobs:
quality:
name: Линт, типы и собственные тесты
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: astral-sh/setup-uv@v5
with:
enable-cache: true
- run: uv sync --locked --group dev
- run: make lint VENV=.venv
- run: make student-test VENV=.venv
acceptance:
name: Стенд Compose с RustFS за HAProxy, smoke, вложения, отказ реплики, рестарт и отказы зависимостей
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: astral-sh/setup-uv@v5
with:
enable-cache: true
- run: uv sync --locked --group dev
- name: Поднять стенд из чистого clone
run: make up
- name: Публичный smoke через балансировщик
run: make test
- name: Вложение после остановки принявшей его реплики и при недоступном S3
run: |
make scenario SCENARIO=failover-upload
make scenario SCENARIO=s3-outage
- name: Приватный bucket, ограниченный ключ API и повторный bootstrap
run: make s3-check
- name: Негативный suite вложений, компенсация, уборка после grace, обрыв и зависание загрузки
run: make upload-faults
- name: Несколько реплик на одном токене и гонка одного ключа
run: make lb-check
- name: Остановка реплики под запросами, затем возврат
run: |
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
docker compose ps -a api
make restore
- name: Рестарт LB, API и хранилищ с сохранением volumes
run: make test-persistence ACTION_SCRIPT=scripts/restart.sh
- name: Отказы Valkey и PostgreSQL, гонка и rollback
run: make test-faults
- name: Лимиты, порты, секреты, volumes и сети по политике
run: make limits
- name: Время обнаружения отказа балансировщиком (SIGTERM, kill, pause, PostgreSQL)
run: make detection
- name: Наружу опубликован только балансировщик
run: |
docker compose ps --format '{{.Service}} {{.Ports}}' | awk '$2 != "" && $1 != "lb" {print; bad=1} END {exit bad}'
- name: Процессы работают не от root
run: |
docker compose top | awk 'NR>1 && $3=="0" {print; bad=1} END {exit bad}'
- name: Корректная остановка реплик API по SIGTERM
run: |
start=$(date +%s)
docker compose kill --signal SIGTERM api
codes=$(docker wait $(docker compose ps -aq api) | sort -u)
elapsed=$(( $(date +%s) - start ))
echo "остановка за ${elapsed} с, коды завершения ${codes}"
test "$elapsed" -le 10 && test "$codes" = "0"
- name: Журналы стенда
if: always()
run: docker compose logs --tail 80
+48
View File
@@ -0,0 +1,48 @@
FROM python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS builder
COPY --from=ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142 /uv /usr/local/bin/uv
ENV UV_COMPILE_BYTECODE=1 \
UV_LINK_MODE=copy \
UV_PYTHON_DOWNLOADS=never \
UV_PROJECT_ENVIRONMENT=/opt/venv
WORKDIR /build
COPY pyproject.toml uv.lock README.md ./
RUN uv sync --locked --no-dev --no-install-project --no-editable
COPY src ./src
RUN uv sync --locked --no-dev --no-editable
FROM python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS runtime
ENV PYTHONUNBUFFERED=1 \
PYTHONDONTWRITEBYTECODE=1 \
PATH="/opt/venv/bin:$PATH" \
HOST=0.0.0.0 \
PORT=8080 \
LOG_LEVEL=INFO \
WEB_CONCURRENCY=2 \
REQUEST_TIMEOUT_SECONDS=8 \
DRAIN_SECONDS=2 \
SHUTDOWN_TIMEOUT_SECONDS=5
RUN groupadd --system --gid 10001 messenger \
&& useradd --system --uid 10001 --gid messenger --no-create-home --shell /usr/sbin/nologin messenger
COPY --from=builder --chown=root:root /opt/venv /opt/venv
WORKDIR /app
USER 10001:10001
EXPOSE 8080
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \
CMD ["python", "-c", "import urllib.request,sys;sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/live',timeout=2).status==200 else 1)"]
CMD ["python", "-m", "gunicorn", \
"--config", "python:messenger.gunicorn_conf", \
"messenger.main:create_app()"]
+140 -1
View File
@@ -4,10 +4,33 @@ CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover
VENV ?= .venv
UV ?= uv
COMPOSE ?= docker compose
RATE ?= 100
API_IMAGE ?= mtusi-messenger:lab5
FROM_REV ?=
SCRIPTS_PY = scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py \
scripts/upload_memory.py
SCENARIO ?= failover-upload
PSQL = $(COMPOSE) exec -T postgres psql -U messenger -d messenger -X -q
.PHONY: help check validate test load test-persistence test-failover \
install lint format typecheck student-test \
secrets up down reset-data migrate ps logs restore lb-check failover-evidence stack-evidence \
persistence-evidence storage-evidence cache-evidence test-faults seed explain \
limits detection failover-each rolling-update level4 \
scenario s3-check attachments-evidence upload-faults upload-memory
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@echo "install/lint/typecheck/student-test: студенческий тулчейн"
@echo "secrets/up/down/migrate/ps/logs: стенд Docker Compose; reset-data: намеренное удаление volumes"
@echo "restore: вернуть остановленные реплики; lb-check: реплики на одном токене и 20 гонок одного ключа"
@echo "level4 = limits, lb-check, detection, failover-each, rolling-update [FROM_REV=<коммит>]; RATE=$(RATE) rps"
@echo "scenario SCENARIO=failover-upload|s3-outage → evidence/<сценарий>.json; s3-check; attachments-evidence"
@echo "upload-faults: негативный suite, компенсация, уборка, обрыв и зависание загрузки против стенда"
@echo "upload-memory: пик памяти реплик при 10/20/40 одновременных загрузках 1 и 10 MiB → evidence/upload-memory/"
check:
$(PYTHON) scripts/check_contract.py
@@ -28,3 +51,119 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
install:
$(UV) sync --group dev
lint:
$(VENV)/bin/ruff check src student-tests $(SCRIPTS_PY)
$(VENV)/bin/ruff format --check src student-tests $(SCRIPTS_PY)
$(VENV)/bin/ty check src student-tests
format:
$(VENV)/bin/ruff check --fix src student-tests $(SCRIPTS_PY)
$(VENV)/bin/ruff format src student-tests $(SCRIPTS_PY)
typecheck:
$(VENV)/bin/ty check src student-tests
student-test:
$(VENV)/bin/pytest
secrets:
sh scripts/secrets.sh
up: secrets
$(COMPOSE) up -d --build --wait
down:
$(COMPOSE) down
reset-data:
@echo "Удаляются volumes с учебными данными стенда"
$(COMPOSE) down -v --remove-orphans
migrate:
$(COMPOSE) run --rm migrate
ps:
$(COMPOSE) ps
logs:
$(COMPOSE) logs --tail 200
restore:
scripts/restore.sh
lb-check:
$(PYTHON) scripts/probe.py check --rounds 20 --out evidence/lb-check.json
limits:
$(PYTHON) scripts/limits.py --out evidence/limits.txt
detection:
sh scripts/detection.sh evidence/detection
failover-each:
RATE=$(RATE) sh scripts/failover-each.sh evidence/failover-each
rolling-update:
ifneq ($(FROM_REV),)
git archive "$(FROM_REV)" | docker build -q -t "$(API_IMAGE)" -
$(COMPOSE) up -d --no-deps --force-recreate --wait api
scripts/lb-wait.sh
endif
$(COMPOSE) build -q api
RATE=$(RATE) $(PYTHON) scripts/probe.py load --expect replace --duration 90 \
--out evidence/rolling$(if $(FROM_REV),-from-$(FROM_REV)) \
-- scripts/rolling-update.sh
level4: limits lb-check detection failover-each rolling-update
scenario:
$(PYTHON) scripts/lab4_scenarios.py $(SCENARIO) --base-url "$(BASE_URL)" --restore --output evidence/$(SCENARIO).json
scripts/lb-wait.sh
s3-check:
sh scripts/s3-check.sh > evidence/s3-security.txt; status=$$?; cat evidence/s3-security.txt; exit $$status
attachments-evidence:
sh scripts/evidence-attachments.sh
upload-faults:
{ echo "date: $$(date -u +%FT%TZ) commit: $$(git rev-parse --short HEAD) BASE_URL=$(BASE_URL)"; \
BASE_URL="$(BASE_URL)" $(VENV)/bin/pytest -m stack -v -s --no-header --tb=short \
-p no:cacheprovider student-tests/stack/test_attachment_faults.py; } > evidence/upload-faults.txt; \
status=$$?; cat evidence/upload-faults.txt; exit $$status
upload-memory:
$(PYTHON) scripts/upload_memory.py --base-url "$(BASE_URL)"
failover-evidence:
sh scripts/failover-evidence.sh "$(BASE_URL)"
stack-evidence:
sh scripts/evidence-stack.sh
persistence-evidence:
sh scripts/persistence-evidence.sh "$(BASE_URL)" > evidence/persistence.txt
cat evidence/persistence.txt
storage-evidence:
sh scripts/evidence-storage.sh
cache-evidence:
sh scripts/evidence-cache.sh > evidence/cache.txt
cat evidence/cache.txt
test-faults:
BASE_URL="$(BASE_URL)" $(VENV)/bin/pytest -m stack -v -s student-tests/stack
seed:
$(PSQL) -f - < scripts/sql/seed-history.sql
explain: seed
{ echo "date: $$(date -u +%FT%TZ) commit: $$(git rev-parse --short HEAD)"; \
echo "\$$ make explain # scripts/sql/explain-history.sql"; \
$(PSQL) -f - < scripts/sql/explain-history.sql; } > evidence/explain-history.txt
cat evidence/explain-history.txt
+276
View File
@@ -0,0 +1,276 @@
x-app: &app
image: ${API_IMAGE:-mtusi-messenger:lab5}
networks: [backend]
x-s3-env: &s3-env
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
S3_CONNECT_TIMEOUT_SECONDS: ${S3_CONNECT_TIMEOUT_SECONDS:-1}
S3_READ_TIMEOUT_SECONDS: ${S3_READ_TIMEOUT_SECONDS:-2}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
IMAGE_CHECK_CONCURRENCY: ${IMAGE_CHECK_CONCURRENCY:-2}
UPLOAD_CONCURRENCY: ${UPLOAD_CONCURRENCY:-3}
UPLOAD_GRACE_SECONDS: ${UPLOAD_GRACE_SECONDS:-900}
UPLOAD_SWEEP_INTERVAL_SECONDS: ${UPLOAD_SWEEP_INTERVAL_SECONDS:-30}
UPLOAD_SWEEP_BATCH: ${UPLOAD_SWEEP_BATCH:-100}
x-s3-client: &s3-client
image: rustfs/rc:v0.1.36@sha256:ab024bfebee49a750ce886b4c70963ccd9ddaa03f491704a90710641d7a26699
read_only: true
tmpfs: ["/tmp:size=16m"]
environment:
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
RC_CONFIG_DIR: /tmp/rc
volumes: ["./infra/s3:/s3:ro"]
secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key]
cap_drop: [ALL]
security_opt: ["no-new-privileges:true"]
restart: "no"
networks: [backend]
deploy:
resources:
limits: {cpus: "0.5", memory: 64m}
services:
lb:
image: haproxy:3.2.26-alpine@sha256:5893840576071e6217cd52ebdcc14f454c6f7a631056fa6f4e0f40d3cac4215b
environment:
API_SERVER_MAXCONN: ${API_SERVER_MAXCONN:-64}
volumes: [./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro]
ports:
- "127.0.0.1:${LB_HTTP_PORT:-8080}:8080"
- "127.0.0.1:${LB_STATS_PORT:-8404}:8404"
networks: [edge, backend]
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8404/healthz"]
interval: 2s
timeout: 2s
retries: 3
start_period: 10s
sysctls:
net.ipv4.ip_local_port_range: "10240 65535"
net.ipv4.tcp_tw_reuse: "1"
stop_signal: SIGUSR1
stop_grace_period: 8s
restart: unless-stopped
read_only: true
cap_drop: [ALL]
deploy:
resources:
limits: {cpus: "${LB_CPUS:-1}", memory: 128m}
api:
<<: *app
build: .
environment:
<<: *s3-env
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
CURSOR_SECRET_FILE: /run/secrets/cursor_secret
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
WEB_CONCURRENCY: ${WEB_CONCURRENCY:-2}
DB_POOL_MAX_SIZE: ${DB_POOL_MAX_SIZE:-8}
REQUEST_TIMEOUT_SECONDS: ${REQUEST_TIMEOUT_SECONDS:-8}
DB_ACQUIRE_TIMEOUT_SECONDS: ${DB_ACQUIRE_TIMEOUT_SECONDS:-1}
DB_COMMAND_TIMEOUT_SECONDS: ${DB_COMMAND_TIMEOUT_SECONDS:-2}
DRAIN_SECONDS: ${DRAIN_SECONDS:-2}
SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-5}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
MALLOC_MMAP_THRESHOLD_: ${MALLOC_MMAP_THRESHOLD_:-131072}
secrets: [database_url, cache_url, cursor_secret, s3_access_key_id, s3_secret_access_key]
healthcheck:
test: ["CMD", "python", "-c", "import sys,urllib.request;sys.exit(urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=2).status!=200)"]
interval: 5s
timeout: 3s
retries: 3
start_period: 30s
start_interval: 1s
stop_grace_period: 8s
restart: unless-stopped
read_only: true
tmpfs: [/tmp]
sysctls:
net.ipv4.tcp_rmem: "4096 65536 262144"
net.ipv4.tcp_wmem: "4096 65536 262144"
depends_on:
migrate:
condition: service_completed_successfully
valkey:
condition: service_started
s3-init:
condition: service_completed_successfully
deploy:
replicas: ${API_REPLICAS:-2}
resources:
limits: {cpus: "${API_CPUS:-1}", memory: "${API_MEMORY:-512m}"}
migrate:
<<: *app
pull_policy: never
command: ["python", "-m", "messenger.migrate"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
LOG_LEVEL: ${LOG_LEVEL:-INFO}
secrets: [database_url]
healthcheck:
disable: true
restart: "no"
depends_on:
postgres:
condition: service_healthy
deploy:
resources:
limits: {cpus: "0.5", memory: 256m}
postgres:
image: postgres:17.11-alpine@sha256:266d3c257594e195b309ae9d1805e54c788f1d7b375f56bf6c367aac6c917e7b
user: "70:70"
command:
- postgres
- -c
- max_connections=${PG_MAX_CONNECTIONS:-200}
- -c
- shared_buffers=${PG_SHARED_BUFFERS:-256MB}
- -c
- shared_preload_libraries=pg_stat_statements
- -c
- pg_stat_statements.track=top
- -c
- track_io_timing=on
- -c
- idle_in_transaction_session_timeout=10s
environment:
POSTGRES_DB: messenger
POSTGRES_USER: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets: [postgres_password]
volumes: [pgdata:/var/lib/postgresql/data]
shm_size: 256m
healthcheck:
test: ["CMD", "pg_isready", "-h", "127.0.0.1", "-U", "messenger", "-d", "messenger"]
interval: 2s
timeout: 3s
retries: 30
start_period: 10s
stop_grace_period: 10s
restart: unless-stopped
networks: [backend]
deploy:
resources:
limits: {cpus: "${PG_CPUS:-2}", memory: "${PG_MEMORY:-1g}"}
valkey:
image: valkey/valkey:8.1.11-alpine@sha256:2602ace2be6569bc2bbb06f873776516ab9089821525be34f4305c67eb9db02a
user: "999:1000"
command:
- valkey-server
- /run/secrets/valkey.conf
- --appendonly
- "yes"
- --appendfsync
- always
- --maxmemory
- 64mb
- --maxmemory-policy
- volatile-lru
secrets: [valkey.conf]
volumes: [valkeydata:/data]
healthcheck:
test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(sed -n 's/^requirepass //p' /run/secrets/valkey.conf)\" valkey-cli ping | grep -q PONG"]
interval: 2s
timeout: 3s
retries: 30
stop_grace_period: 10s
restart: unless-stopped
networks: [backend]
deploy:
resources:
limits: {cpus: "0.5", memory: 128m}
s3:
image: rustfs/rustfs:1.0.4@sha256:ed9210d6411d77ffceb70d94b964a3cb7d225983368bc25bebe01e995293dcc2
environment:
RUSTFS_ACCESS_KEY_FILE: /run/secrets/s3_root_user
RUSTFS_SECRET_KEY_FILE: /run/secrets/s3_root_password
RUSTFS_ADDRESS: ":9000"
RUSTFS_CONSOLE_ENABLE: "false"
RUSTFS_REGION: ${S3_REGION:-us-east-1}
RUSTFS_OBS_LOG_DIRECTORY: ""
RUSTFS_OBS_LOGGER_LEVEL: ${S3_LOG_LEVEL:-warn}
secrets: [s3_root_user, s3_root_password]
volumes: [s3data:/data]
read_only: true
tmpfs: ["/tmp:size=64m"]
cap_drop: [ALL]
security_opt: ["no-new-privileges:true"]
healthcheck:
test: ["CMD", "curl", "-fsS", "-o", "/dev/null", "http://127.0.0.1:9000/health/ready"]
interval: 2s
timeout: 3s
retries: 30
start_period: 20s
stop_grace_period: 10s
restart: unless-stopped
networks: [backend]
deploy:
resources:
limits: {cpus: "1", memory: "${S3_MEMORY:-512m}"}
s3-init:
<<: *s3-client
entrypoint: ["/bin/sh", "/s3/init.sh"]
depends_on:
s3:
condition: service_healthy
s3-cli:
<<: *s3-client
profiles: [tools]
entrypoint: ["/bin/sh", "/s3/cli.sh"]
k6:
image: grafana/k6:2.3.0@sha256:2dd4756ec6e14e62778916e1eb850108e4b499304c02df56760fa0bdb0fba0b8
profiles: [bench]
environment:
BASE_URL: http://lb:8080
volumes: [./infra/k6:/scripts:ro]
networks: [edge]
deploy:
resources:
limits: {cpus: "${K6_CPUS:-2}", memory: "${K6_MEMORY:-512m}"}
secrets:
postgres_password:
file: ./secrets/postgres_password
database_url:
file: ./secrets/database_url
valkey.conf:
file: ./secrets/valkey.conf
cache_url:
file: ./secrets/cache_url
cursor_secret:
file: ./secrets/cursor_secret
s3_root_user:
file: ./secrets/s3_root_user
s3_root_password:
file: ./secrets/s3_root_password
s3_access_key_id:
file: ./secrets/s3_access_key_id
s3_secret_access_key:
file: ./secrets/s3_secret_access_key
volumes:
pgdata:
valkeydata:
s3data:
networks:
edge:
backend:
internal: true
+71
View File
@@ -0,0 +1,71 @@
global
log stdout format raw local0 info
maxconn 4096
hard-stop-after 5s
tune.bufsize 32768
tune.rcvbuf.client 65536
tune.rcvbuf.server 65536
tune.sndbuf.client 65536
tune.sndbuf.server 65536
resolvers docker
nameserver dns 127.0.0.11:53
accepted_payload_size 8192
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold nx 2s
hold other 2s
hold refused 2s
hold timeout 2s
defaults
mode http
log global
option dontlognull
option dontlog-normal
log-format "%ci [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq rid=%[var(txn.request_id)] %HM %HPO"
option http-server-close
option redispatch 1
retries 2
retry-on conn-failure empty-response
timeout connect 500ms
timeout client 30s
timeout server 10s
timeout http-request 5s
timeout http-keep-alive 5s
timeout queue 3s
timeout check 1s
http-error status 502 content-type application/json hdr X-Request-Id "%[var(txn.request_id)]" lf-string '{"error":{"code":"unavailable","message":"Upstream replica failed","request_id":"%[var(txn.request_id)]"}}'
http-error status 503 content-type application/json hdr X-Request-Id "%[var(txn.request_id)]" lf-string '{"error":{"code":"unavailable","message":"No ready API replica","request_id":"%[var(txn.request_id)]"}}'
http-error status 504 content-type application/json hdr X-Request-Id "%[var(txn.request_id)]" lf-string '{"error":{"code":"unavailable","message":"Upstream timeout","request_id":"%[var(txn.request_id)]"}}'
frontend public
bind :8080
option forwardfor
http-request set-var(txn.request_id) req.fhdr(X-Request-Id) if { req.fhdr(X-Request-Id) -m len 1:200 }
http-request set-var(txn.request_id) uuid(4) unless { var(txn.request_id) -m found }
http-request set-header X-Request-Id %[var(txn.request_id)]
default_backend api
backend api
balance roundrobin
option httpchk
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host lb
http-check expect status 200
acl message_path path_reg ^/api/v1/chats/[^/]+/messages$
acl has_key req.hdr(Idempotency-Key) -m found
http-request set-var(txn.replay_safe) bool(1) if METH_GET
http-request set-var(txn.replay_safe) bool(1) if METH_POST message_path has_key
http-request disable-l7-retry unless { var(txn.replay_safe) -m bool }
default-server check init-state fully-down inter 1s fastinter 500ms downinter 2s fall 2 rise 2 resolvers docker resolve-prefer ipv4 init-addr none maxconn "${API_SERVER_MAXCONN-64}"
server-template api 8 api:8080
frontend stats
bind :8404
monitor-uri /healthz
monitor fail if { nbsrv(api) lt 1 }
http-request use-service prometheus-exporter if { path /metrics }
stats enable
stats uri /stats
stats refresh 5s
+106
View File
@@ -0,0 +1,106 @@
import http from 'k6/http';
import { check } from 'k6';
import encoding from 'k6/encoding';
import exec from 'k6/execution';
const BASE = __ENV.BASE_URL || 'http://lb:8080';
const RATE = Number(__ENV.RATE || 50);
const WARMUP = Number(__ENV.WARMUP || 10);
const DURATION = Number(__ENV.DURATION || 60);
const USERS = Math.max(2, Number(__ENV.USERS || 20));
const READ_RATIO = Number(__ENV.READ_RATIO || 0.2);
const VUS = Number(__ENV.VUS || Math.max(10, Math.ceil(RATE / 10)));
const MAX_VUS = Number(__ENV.MAX_VUS || Math.max(VUS, Math.min(VUS * 4, 1000)));
const RUN = __ENV.RUN_ID || Date.now().toString(36);
const SUMMARY = __ENV.SUMMARY || '/out/summary.json';
const JSON_HEADERS = { 'Content-Type': 'application/json' };
const phase = (startTime, duration) => ({
executor: 'constant-arrival-rate',
rate: RATE,
timeUnit: '1s',
startTime: `${startTime}s`,
duration: `${duration}s`,
preAllocatedVUs: VUS,
maxVUs: MAX_VUS,
gracefulStop: '5s',
});
const measured = ['http_reqs', 'http_req_failed', 'http_req_duration', 'dropped_iterations', 'iterations'];
export const options = {
setupTimeout: '180s',
summaryTrendStats: ['avg', 'min', 'med', 'p(90)', 'p(95)', 'p(99)', 'max'],
scenarios: Object.assign(WARMUP > 0 ? { warmup: phase(0, WARMUP) } : {}, { measure: phase(WARMUP, DURATION) }),
thresholds: Object.fromEntries([
...measured.map((metric) => [`${metric}{scenario:measure}`, []]),
['http_req_duration{scenario:measure,name:post_message}', []],
['http_req_duration{scenario:measure,name:get_history}', []],
]),
};
function created(response, what) {
if (response.status !== 201) {
throw new Error(`${what}: HTTP ${response.status}`);
}
return response.json();
}
const bearer = (token, extra) => Object.assign({ Authorization: `Bearer ${token}` }, extra || {});
export function setup() {
const actors = [];
for (let i = 0; i < USERS; i++) {
const username = `k6_${RUN}_${i}`.slice(0, 32);
const password = `K6-${RUN}-${i}-synthetic`;
const user = created(
http.post(`${BASE}/api/v1/users`, JSON.stringify({ username, display_name: `k6 ${i}`, password }), { headers: JSON_HEADERS }),
'register',
);
const basic = encoding.b64encode(`${username}:${password}`);
const session = created(
http.post(`${BASE}/api/v1/auth/sessions`, null, { headers: { Authorization: `Basic ${basic}` } }),
'login',
);
actors.push({ id: user.id, token: session.token });
}
const chats = [];
for (let i = 0; i < USERS; i += 2) {
const [a, b] = [actors[i], actors[(i + 1) % USERS]];
const body = JSON.stringify({ title: `k6 ${RUN} ${i}`, member_ids: [a.id, b.id] });
const chat = created(http.post(`${BASE}/api/v1/chats`, body, { headers: bearer(a.token, JSON_HEADERS) }), 'chat');
chats.push({ id: chat.id, tokens: [a.token, b.token] });
}
console.log('LOAD-START');
return { chats };
}
export default function (data) {
const n = exec.scenario.iterationInTest;
const chat = data.chats[n % data.chats.length];
const token = chat.tokens[Math.floor(n / data.chats.length) % 2];
if (Math.random() < READ_RATIO) {
const response = http.get(`${BASE}/api/v1/chats/${chat.id}/messages?limit=50`, {
headers: bearer(token),
tags: { name: 'get_history' },
timeout: '10s',
});
check(response, { 'history 200': (r) => r.status === 200 });
return;
}
const key = `${RUN}-${exec.scenario.name}-${n}`;
const response = http.post(`${BASE}/api/v1/chats/${chat.id}/messages`, JSON.stringify({ text: `k6 ${key}` }), {
headers: bearer(token, Object.assign({ 'Idempotency-Key': key }, JSON_HEADERS)),
tags: { name: 'post_message' },
timeout: '10s',
});
check(response, { 'message 201': (r) => r.status === 201 });
}
export function handleSummary(data) {
delete data.setup_data;
const reqs = data.metrics['http_reqs{scenario:measure}'];
const failed = data.metrics['http_req_failed{scenario:measure}'];
const line = `measure: ${reqs ? reqs.values.count : 0} requests, failed ratio ${failed ? failed.values.rate : 'n/a'}\n`;
return { [SUMMARY]: JSON.stringify(data, null, 1), stdout: line };
}
+23
View File
@@ -0,0 +1,23 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "MessengerApiObjects",
"Effect": "Allow",
"Action": [
"s3:GetObject",
"s3:PutObject",
"s3:DeleteObject",
"s3:AbortMultipartUpload",
"s3:ListMultipartUploadParts"
],
"Resource": ["arn:aws:s3:::messenger-attachments/*"]
},
{
"Sid": "MessengerApiBucket",
"Effect": "Allow",
"Action": ["s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:GetBucketLocation"],
"Resource": ["arn:aws:s3:::messenger-attachments"]
}
]
}
+12
View File
@@ -0,0 +1,12 @@
#!/bin/sh
set -eu
secret() { tr -d '\r\n' < "/run/secrets/$1"; }
endpoint="${S3_ENDPOINT:-http://s3:9000}"
RC_HOST_root="${endpoint%%://*}://$(secret s3_root_user):$(secret s3_root_password)@${endpoint#*://}"
RC_HOST_api="${endpoint%%://*}://$(secret s3_access_key_id):$(secret s3_secret_access_key)@${endpoint#*://}"
export RC_HOST_root RC_HOST_api
rc alias set anon "$endpoint" --anonymous >/dev/null
case "${1:-}" in
sh|wget) exec "$@" ;;
esac
exec rc "$@"
+20
View File
@@ -0,0 +1,20 @@
#!/bin/sh
set -eu
secret() { tr -d '\r\n' < "/run/secrets/$1"; }
bucket="${S3_BUCKET:-messenger-attachments}"
endpoint="${S3_ENDPOINT:-http://s3:9000}"
policy="${S3_API_POLICY:-messenger-api}"
RC_HOST_root="${endpoint%%://*}://$(secret s3_root_user):$(secret s3_root_password)@${endpoint#*://}"
export RC_HOST_root
api_key=$(secret s3_access_key_id)
rc ready root --timeout 10 >/dev/null
rc bucket create --ignore-existing "root/$bucket" >/dev/null
rc bucket anonymous set private "root/$bucket" >/dev/null
sed "s/messenger-attachments/$bucket/g" /s3/api-policy.json > /tmp/api-policy.json
rc admin policy create root "$policy" /tmp/api-policy.json >/dev/null
rc admin user add root "$api_key" "$(secret s3_secret_access_key)" >/dev/null
rc admin policy attach root "$policy" --user "$api_key" >/dev/null
rc bucket lifecycle rule import "root/$bucket" /s3/lifecycle.json >/dev/null
rc bucket anonymous get "root/$bucket" --json | grep -q '"permission": *"private"'
rc admin user info root "$api_key" --json | grep -q "\"$policy\""
echo "s3-init: bucket=$bucket anonymous=private policy=$policy lifecycle=abort-incomplete-uploads"
+10
View File
@@ -0,0 +1,10 @@
{
"rules": [
{
"id": "abort-incomplete-uploads",
"status": "Enabled",
"prefix": "",
"abortIncompleteMultipartUploadDays": 1
}
]
}
+117
View File
@@ -0,0 +1,117 @@
[project]
name = "mtusi-messenger"
version = "1.5.0"
description = "Лабораторная 5 «Очереди и асинхронная обработка изображений», МТУСИ"
requires-python = ">=3.14,<3.15"
dependencies = [
"fastapi>=0.118.0",
"uvicorn[standard]>=0.37.0",
"pydantic>=2.11.0",
"pydantic-settings>=2.11.0",
"dishka>=1.7.0",
"structlog>=26.1.0",
"gunicorn>=26.2.0",
"orjson>=3.12.0",
"asyncpg>=0.32.0",
"argon2-cffi>=25.1.0",
"uvicorn-worker>=0.4.0",
"redis>=8.1.0",
"aiobotocore>=3.10.0",
"pillow>=12.0.0",
]
[dependency-groups]
dev = [
"pytest>=8.4.0",
"pytest-asyncio>=1.2.0",
"httpx>=0.28.0",
"ruff>=0.14.0",
"ty>=0.0.1a21",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[tool.hatch.build.targets.wheel]
packages = ["src/messenger"]
[tool.ruff]
line-length = 100
target-version = "py314"
src = ["src", "student-tests"]
extend-exclude = ["tests", "scripts/check_contract.py", "scripts/validate_openapi.py"]
per-file-target-version = { "scripts/*.py" = "py39" }
[tool.ruff.format]
indent-style = "space"
quote-style = "double"
[tool.ruff.lint]
select = [
"E",
"W",
"F",
"I",
"N",
"UP",
"B",
"A",
"C4",
"DTZ",
"T20",
"SIM",
"TID",
"PTH",
"RUF",
"ASYNC",
"S",
"ARG",
"PL",
]
ignore = [
"PLR0913",
"RUF001",
"RUF002",
"RUF003",
"S104",
"BLE001",
]
[tool.ruff.lint.per-file-ignores]
"student-tests/**" = ["S101", "S105", "S106", "S603", "S607", "PLR2004", "ARG001", "T20", "TID251"]
"scripts/*.py" = ["T20", "S603", "S607", "S310", "PLR2004", "TID251"]
"src/messenger/*/infrastructure/**" = ["TID251", "ARG002"]
"src/messenger/*/domain/errors.py" = ["N818"]
"src/messenger/*/application/errors.py" = ["N818"]
"src/messenger/*/entrypoints/**" = ["TID251", "N818", "PLR0917", "S105"]
"src/messenger/shared/config.py" = ["ARG002", "ARG003"]
"src/messenger/main.py" = ["TID251"]
"src/messenger/migrate.py" = ["TID251"]
"src/messenger/gunicorn_conf.py" = ["TID251", "ARG002", "RUF012"]
[tool.ruff.lint.flake8-tidy-imports.banned-api]
"messenger.shared.infrastructure".msg = "Домен и слой приложения не импортируют инфраструктуру: зависимость идёт через порт."
"messenger.identity.infrastructure".msg = "Домен и слой приложения не импортируют инфраструктуру: зависимость идёт через порт."
"messenger.messaging.infrastructure".msg = "Домен и слой приложения не импортируют инфраструктуру: зависимость идёт через порт."
"messenger.attachments.infrastructure".msg = "Домен и слой приложения не импортируют инфраструктуру: зависимость идёт через порт."
[tool.ruff.lint.isort]
known-first-party = ["messenger"]
[tool.ty.src]
include = ["src", "student-tests"]
[tool.ty.rules]
unresolved-import = "error"
invalid-argument-type = "error"
possibly-missing-attribute = "warn"
[tool.pytest.ini_options]
testpaths = ["student-tests"]
asyncio_mode = "auto"
asyncio_default_fixture_loop_scope = "function"
addopts = "-q --strict-markers -m 'not stack'"
markers = [
"stack: black-box проверки против поднятого compose-стенда (make test-faults)",
]
+20
View File
@@ -0,0 +1,20 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
out=${1:-evidence/detection}
mkdir -p "$out"
scripts/restore.sh
replica() { docker compose ps -q "${API_SERVICE:-api}" | head -1; }
detect() { python3 scripts/probe.py detect "$@"; }
failed=0
detect --max 1.5 --within-action --restore scripts/restore.sh --out "$out/sigterm.json" \
-- docker stop -t "${STOP_TIMEOUT:-6}" "$(replica)" || failed=1
detect --max 1.5 --restore scripts/restore.sh --out "$out/kill.json" \
-- docker kill "$(replica)" || failed=1
hung=$(replica)
detect --max 3.5 --restore "docker unpause $hung && scripts/lb-wait.sh" --out "$out/pause.json" \
-- docker pause "$hung" || failed=1
db=$(docker compose ps -q postgres)
detect --max 1.5 --until 0 --lb-status 503 --restore "docker start $db && scripts/lb-wait.sh" \
--out "$out/postgres.json" -- docker stop "$db" || failed=1
exit "$failed"
+40
View File
@@ -0,0 +1,40 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
psql() { docker compose exec -T postgres psql -U messenger -d messenger -X -c "$1"; }
value() { docker compose exec -T postgres psql -U messenger -d messenger -X -tA -c "$1"; }
cli() { docker compose run --rm --no-deps -T s3-cli "$@"; }
bucket=${S3_BUCKET:-messenger-attachments}
{
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"
echo "\$ docker compose ps --format '{{.Service}} {{.Status}} {{.Ports}}'"
docker compose ps -a --format '{{.Service}} {{.Status}} {{.Ports}}'
for table in attachments message_attachments; do
echo "\$ psql -c '\\d $table'"
psql "\\d $table"
done
echo "\$ psql: столбцы bytea во всей схеме (хеши токенов и отпечатки, не файлы)"
psql "SELECT table_name, column_name FROM information_schema.columns WHERE table_schema = 'public' AND data_type = 'bytea' ORDER BY 1, 2"
echo "\$ psql: вложения по статусу и максимальный размер строки attachments"
psql "SELECT status, count(*), max(pg_column_size(a.*)) AS max_row_bytes FROM attachments a GROUP BY status"
echo "\$ rc object list --recursive root/$bucket/attachments/ | count"
cli object list --recursive "root/$bucket/attachments/" --json | grep -c '"key"' || true
echo "\$ rc object list root/$bucket/ (первые 3 объекта, ключи назначены сервером)"
cli object list --recursive "root/$bucket/attachments/" | head -3
row=$(value "SELECT object_key || ' ' || sha256 || ' ' || size_bytes FROM attachments WHERE status = 'ready' ORDER BY created_at DESC LIMIT 1")
key=${row%% *}
echo "\$ сверка вне API: sha256 и размер последнего ready-вложения в PostgreSQL против байтов объекта в bucket"
echo "postgres: ${row#* }"
echo "s3: $(cli sh -c "rc object show root/$bucket/$key | sha256sum | cut -d' ' -f1") $(cli sh -c "rc object show root/$bucket/$key | wc -c" | tr -d ' ')"
echo "\$ docker inspect api: ReadonlyRootfs, tmpfs, volumes/bind mounts"
for id in $(docker compose ps -q api); do
docker inspect -f '{{.Config.Hostname}} ro={{.HostConfig.ReadonlyRootfs}} tmpfs={{json .HostConfig.Tmpfs}} mounts={{range .Mounts}}{{.Type}}:{{.Destination}} {{end}}' "$id"
done
echo "\$ docker inspect s3: volume данных"
docker inspect -f '{{range .Mounts}}{{.Type}}:{{.Name}}:{{.Destination}} {{end}}' "$(docker compose ps -q s3)"
echo "\$ секреты сервисов: root-ключи S3 только у s3/s3-init/s3-cli"
docker compose config --format json | python3 -c 'import json,sys
for name, service in sorted(json.load(sys.stdin)["services"].items()):
print(name, sorted(s["source"] for s in service.get("secrets", [])))'
} > evidence/storage-layout.txt
cat evidence/storage-layout.txt
+24
View File
@@ -0,0 +1,24 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
cli() { docker compose exec -T valkey sh -c "REDISCLI_AUTH=\"\$(sed -n 's/^requirepass //p' /run/secrets/valkey.conf)\" valkey-cli $1"; }
psql() { docker compose exec -T postgres psql -U messenger -d messenger -tA -c "$1"; }
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"
echo "\$ valkey-cli config get appendonly appendfsync maxmemory maxmemory-policy"
cli "config get appendonly" ; cli "config get appendfsync"; cli "config get maxmemory"; cli "config get maxmemory-policy"
echo "\$ valkey-cli info persistence | grep aof_enabled,aof_last_write_status"
cli "info persistence" | grep -E '^aof_enabled|^aof_last_write_status'
echo "\$ valkey-cli dbsize; info stats | grep keyspace"
cli dbsize
cli "info stats" | grep -E '^keyspace_(hits|misses)'
key=$(cli "--scan --pattern session:* --count 1000" | head -1)
hash=${key#session:}
echo "\$ сравнение срока ключа кэша и срока сессии для одного ключа (значение и токен не печатаются)"
echo "valkey pexpiretime, ms: $(cli "pexpiretime $key")"
echo "postgres expires_at, ms: $(psql "SELECT (extract(epoch FROM expires_at) * 1000)::bigint FROM sessions WHERE token_hash = decode('$hash', 'hex')")"
echo "\$ количество отметок отзыва и закэшированных сессий"
revoked=0; cached=0
for k in $(cli "--scan --pattern session:* --count 1000"); do
if [ "$(cli "get $k")" = "-" ]; then revoked=$((revoked + 1)); else cached=$((cached + 1)); fi
done
echo "tombstones: $revoked, cached sessions: $cached"
+49
View File
@@ -0,0 +1,49 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
base=${1:-http://$(docker compose port lb 8080)}
stats=http://$(docker compose port lb 8404)/stats\;csv
header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"; }
{
header
echo "\$ docker compose --env-file .env config --quiet && echo valid"
docker compose --env-file .env config --quiet && echo valid
echo "\$ docker compose ps --format '{{.Service}} {{.Name}} {{.Status}} {{.Ports}}'"
docker compose ps --format '{{.Service}} {{.Name}} {{.Status}} {{.Ports}}'
echo "\$ docker compose config | grep -c container_name"
docker compose config | grep -c container_name || true
echo "\$ docker port <каждая реплика api> # реплики не публикуют порт"
for id in $(docker compose ps -q api); do echo "$(docker inspect -f '{{.Config.Hostname}}' "$id"): [$(docker port "$id")]"; done
echo "\$ docker inspect: hostname = X-Instance-Id, сети, лимиты CPU (nano) и памяти"
for id in $(docker compose ps -q); do
docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}} {{.Config.Hostname}} {{.Name}} nets={{range $k, $v := .NetworkSettings.Networks}}{{$k}},{{end}} cpus={{.HostConfig.NanoCpus}} mem={{.HostConfig.Memory}} user={{.Config.User}} ro={{.HostConfig.ReadonlyRootfs}}' "$id"
done | sort
echo "\$ curl LB stats: backend api, server status check"
curl -fsS "$stats" | python3 -c 'import csv,sys
for r in csv.DictReader(sys.stdin.read().lstrip("# ").splitlines()):
if r["pxname"] == "api" and r["status"] != "MAINT (resolution)":
print(r["svname"], r["status"], r["check_status"], r["check_code"])'
echo "\$ for i in 1..6: curl -D - $base/health/live | X-Instance-Id"
for _ in 1 2 3 4 5 6; do curl -fsS -D - -o /dev/null "$base/health/live" | tr -d '\r' | grep -i '^x-instance-id'; done
} > evidence/stack.txt
{
header
echo "host: $(sysctl -n machdep.cpu.brand_string 2>/dev/null || uname -m), $(sysctl -n hw.ncpu 2>/dev/null) CPU, $(($(sysctl -n hw.memsize 2>/dev/null || echo 0) / 1073741824)) GiB RAM, $(sw_vers -productName 2>/dev/null) $(sw_vers -productVersion 2>/dev/null)"
echo "docker: $(docker version -f '{{.Server.Version}}'), VM: $(docker info -f '{{.NCPU}}') CPU, $(docker info -f '{{.MemTotal}}') bytes"
echo "compose: $(docker compose version --short)"
echo "images:"
grep -h -E 'image:|@sha256' compose.yaml Dockerfile | sed 's/^ */ /'
echo "python in image: $(docker exec "$(docker compose ps -q api | head -1)" python -V)"
echo "haproxy: $(docker compose exec -T lb haproxy -v | head -1)"
echo "rustfs: $(docker compose exec -T s3 rustfs --version 2>/dev/null | head -1)"
echo "packages (uv.lock):"
for name in fastapi uvicorn uvicorn-worker gunicorn asyncpg argon2-cffi pydantic pydantic-settings dishka structlog redis aiobotocore botocore aiohttp pillow; do
printf ' %s %s\n' "$name" "$(sed -n "/^name = \"$name\"$/{n;s/version = \"\(.*\)\"/\1/p;}" uv.lock)"
done
echo "process users (service uid command):"
for id in $(docker compose ps -q); do
service=$(docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}}' "$id")
docker top "$id" -eo pid,uid,comm | awk -v s="$service" 'NR > 1 {print " " s, $2, $3}'
done | sort -u
} > evidence/environment.txt
cat evidence/stack.txt evidence/environment.txt
+44
View File
@@ -0,0 +1,44 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
psql() { docker compose exec -T postgres psql -U messenger -d messenger -c "$1"; }
header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"; }
counts="SELECT (SELECT count(*) FROM users) users, (SELECT count(*) FROM sessions) sessions, (SELECT count(*) FROM chats) chats, (SELECT count(*) FROM chat_members) members, (SELECT count(*) FROM messages) messages, (SELECT count(*) FROM idempotency_keys) keys, (SELECT count(*) FROM attachments) attachments, (SELECT count(*) FROM message_attachments) links, (SELECT count(*) FROM schema_migrations) versions"
kdf="SELECT substring(password_hash from '^[$]argon2id[$]v=[0-9]+[$]m=[0-9]+,t=[0-9]+,p=[0-9]+[$]') AS kdf_prefix, count(*) AS users, min(length(password_hash)) AS hash_length FROM users GROUP BY 1"
tokens="SELECT octet_length(token_hash) AS stored_bytes, count(*) AS sessions FROM sessions GROUP BY 1"
{
header
for table in users sessions chats chat_members messages idempotency_keys attachments message_attachments schema_migrations; do
echo "\$ psql -c '\\d+ $table'"
psql "\\d+ $table"
done
echo "\$ psql -c 'SELECT version, left(checksum, 12), applied_at FROM schema_migrations'"
psql "SELECT version, left(checksum, 12) AS checksum, applied_at FROM schema_migrations"
} > evidence/schema.txt
{
header
echo "\$ psql -c \"$counts\""
psql "$counts"
echo "\$ make migrate"
docker compose run --rm migrate 2>/dev/null
echo "\$ psql -c \"$counts\""
psql "$counts"
} > evidence/migrations.txt
{
header
echo "\$ psql -c \"$kdf\""
psql "$kdf"
echo "\$ psql -c \"$tokens\""
psql "$tokens"
} > evidence/kdf.txt
{
header
echo "\$ git ls-files secrets .env backups volumes"
git ls-files secrets .env backups volumes
echo "(пусто: секретов, .env, дампов и volumes в Git нет)"
echo "\$ git check-ignore -v secrets/database_url secrets/postgres_password secrets/s3_root_password secrets/s3_secret_access_key .env backups/"
git check-ignore -v secrets/database_url secrets/postgres_password secrets/s3_root_password secrets/s3_secret_access_key .env backups/
echo "\$ docker volume ls --filter name=mtusi2026lab5"
docker volume ls --filter name=mtusi2026lab5
} > evidence/secrets.txt
echo "evidence/{schema,migrations,kdf,secrets}.txt обновлены"
+19
View File
@@ -0,0 +1,19 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
out=${1:-evidence/failover-each}
rm -rf "$out"
mkdir -p "$out"
scripts/restore.sh
hosts=$(docker inspect -f '{{.Config.Hostname}}' $(docker compose ps -q "${API_SERVICE:-api}"))
failed=0
set --
for host in $hosts; do
TARGET_INSTANCE_ID=$host python3 scripts/probe.py load --expect dip --out "$out/$host" \
-- sh -c "scripts/stop-one.sh && sleep ${HOLD:-10} && scripts/restore.sh" || failed=1
scripts/restore.sh
set -- "$@" "$out/$host/result.json"
done
python3 -c 'import json,sys; print(json.dumps([json.load(open(p)) for p in sys.argv[1:]], ensure_ascii=False, indent=2))' "$@" > "$out/summary.tmp"
mv "$out/summary.tmp" "$out/summary.json"
exit "$failed"
+28
View File
@@ -0,0 +1,28 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
replicas() {
docker inspect -f '{{.Config.Hostname}} {{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}' $(docker compose ps -aq "${API_SERVICE:-api}")
}
tmp=evidence/failover.tmp
since=$(date -u +%FT%TZ)
{
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"
echo "== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt"
replicas
echo "\$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh"
make --no-print-directory test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL="${1:-http://localhost:8080}"
echo "== after: api replicas"
replicas
} > "$tmp" 2>&1
stopped=$(replicas | awk '$2 == "exited" {print $1}')
{
echo "== stopped replica: $stopped; shutdown: signal, drain window with readiness 503, close, exit"
docker logs --since "$since" "$stopped" 2>&1 | grep -E 'Handling signal|Draining|"status":503|Shutting down|Waiting for conn|Finished server|service.stopped|SIGTERM' | tail -16
echo "\$ scripts/restore.sh"
scripts/restore.sh
echo "== after restore"
replicas
} >> "$tmp" 2>&1
mv "$tmp" "evidence/failover-$stopped.txt"
cat "evidence/failover-$stopped.txt"
Executable
+10
View File
@@ -0,0 +1,10 @@
#!/bin/sh
set -eu
out=${1:?usage: k6.sh <output dir> [k6 run args]}
shift
mkdir -p "$out"
out=$(cd "$out" && pwd)
cd "$(dirname "$0")/.."
exec docker compose --profile bench run --rm --no-deps -T -u "$(id -u):$(id -g)" -v "$out:/out" \
-e RATE -e WARMUP -e DURATION -e USERS -e READ_RATIO -e VUS -e MAX_VUS -e RUN_ID \
k6 run --quiet "$@" "/scripts/${K6_SCRIPT:-messages.js}"
+230
View File
@@ -0,0 +1,230 @@
#!/usr/bin/env python3
import argparse
import hashlib
import http.client
import json
import os
import subprocess
import sys
import time
import uuid
from pathlib import Path
from urllib.parse import urlsplit
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
client = __import__("client")
Client, Failure, check = client.Client, client.Failure, client.check
FIXTURE = (ROOT / "tests/fixtures/sample.png").read_bytes()
def compose(*args, stdin=None):
cmd = ["docker", "compose", *args]
return subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, text=True, check=False)
def upload(c, actor, chat, expected=201):
path = f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png"
schema = "Attachment" if expected == 201 else None
return c.call(
"POST",
path,
expected,
actor=actor,
raw=FIXTURE,
headers={"Content-Type": "image/png"},
schema=schema,
)
def content(c, actor, attachment, expected=200):
return c.call("GET", f"/api/v1/attachments/{attachment['id']}/content", expected, actor=actor)
def timed(fn):
started = time.monotonic()
try:
response = fn()
except Failure as exc:
return {
"status": "failure",
"seconds": round(time.monotonic() - started, 3),
"error": str(exc),
}
seconds = round(time.monotonic() - started, 3)
return {
"status": response["status"],
"seconds": seconds,
"instance": response["headers"].get("X-Instance-Id"),
}
def wait(fn, seconds):
deadline = time.monotonic() + seconds
while True:
try:
return fn()
except Failure:
if time.monotonic() > deadline:
raise
time.sleep(0.2)
def failover_upload(c, args):
alice, bob, _, chat = c.bootstrap()
sent = upload(c, alice, chat)
att, target = sent["json"], sent["headers"]["X-Instance-Id"]
started = time.monotonic()
hook = subprocess.run(
[str(ROOT / "scripts/stop-one.sh")],
cwd=ROOT,
env=dict(os.environ, TARGET_INSTANCE_ID=target),
capture_output=True,
text=True,
check=False,
)
check(hook.returncode == 0, f"stop-one.sh failed: {hook.stderr.strip()}")
hook_seconds = time.monotonic() - started
got = wait(lambda: content(c, bob, att), 15)
recovered = time.monotonic() - started
served = got["headers"].get("X-Instance-Id")
check(served != target, "download served by the stopped replica")
digest = hashlib.sha256(got["bytes"]).hexdigest()
check(
digest == att["sha256"] == hashlib.sha256(FIXTURE).hexdigest(),
"sha256 mismatch after failover",
)
if args.restore:
compose("up", "-d", "--wait", "--no-recreate", "--no-deps", args.service)
return {
"scenario": "failover-upload",
"upload_instance": target,
"download_instance": served,
"sha256_equal": True,
"hook_seconds": round(hook_seconds, 3),
"download_ok_after_seconds": round(recovered, 3),
"restored": args.restore,
}
def s3_outage(c, args):
alice, bob, _, chat = c.bootstrap()
att = upload(c, alice, chat)["json"]
check(
compose("stop", "-t", "10", args.s3_service).returncode == 0, "docker compose stop failed"
)
try:
during = {
"upload": timed(lambda: upload(c, alice, chat, [503])),
"metadata": timed(
lambda: c.call("GET", f"/api/v1/attachments/{att['id']}", [200, 503], actor=bob)
),
"download": timed(lambda: content(c, bob, att, 503)),
"text_message": timed(lambda: c.message(alice, chat, "S3 недоступен, текст работает")),
"health_ready": timed(lambda: c.call("GET", "/health/ready", [200, 503])),
}
finally:
started = time.monotonic()
compose("up", "-d", "--wait", "--no-deps", "--no-recreate", args.s3_service)
recovered = round(time.monotonic() - started, 3)
got = wait(lambda: content(c, bob, att), 30)
check(got["bytes"] == FIXTURE, "attachment changed across S3 outage")
expected = {"upload": 503, "download": 503, "text_message": 201}
return {
"scenario": "s3-outage",
"during_outage": during,
"expected": expected,
"as_expected": all(during[k]["status"] == v for k, v in expected.items()),
"s3_back_healthy_seconds": recovered,
"download_after_recovery": "sha256 equal",
}
def abort_upload(c, args):
alice, _, _, chat = c.bootstrap()
name = f"abort-{uuid.uuid4().hex}.png"
parts = urlsplit(c.base)
factory = http.client.HTTPSConnection if parts.scheme == "https" else http.client.HTTPConnection
conn = factory(parts.netloc, timeout=10)
conn.putrequest("POST", f"/api/v1/chats/{chat['id']}/attachments?filename={name}")
headers = {
"Authorization": f"Bearer {alice['token']}",
"Content-Type": "image/png",
"Content-Length": str(args.declared),
"Accept": "application/json",
}
for key, value in headers.items():
conn.putheader(key, value)
conn.endheaders()
conn.send(FIXTURE[:8] + os.urandom(args.sent - 8))
conn.close()
time.sleep(args.settle)
user, db = os.environ.get("PGUSER", "messenger"), os.environ.get("PGDATABASE", "messenger")
sql = "SELECT status, count(*) FROM attachments WHERE filename = :'name' GROUP BY status;\n"
out = compose(
"exec",
"-T",
"postgres",
"psql",
"-qAt",
"-F",
" ",
"-U",
user,
"-d",
db,
"-v",
f"name={name}",
stdin=sql,
)
rows = out.stdout.split()
counts = dict(zip(rows[::2], (int(x) for x in rows[1::2])))
alive = c.call("GET", "/api/v1/users/me", actor=alice, schema="User")["status"]
return {
"scenario": "abort-upload",
"declared_bytes": args.declared,
"sent_bytes": args.sent,
"settle_seconds": args.settle,
"rows_by_status": counts,
"ready_rows": counts.get("ready", 0),
"api_still_serves": alive == 200,
}
def main():
p = argparse.ArgumentParser(
description="Lab4 failure scenarios over the public HTTP contract plus docker compose"
)
p.add_argument("scenario", choices=["failover-upload", "s3-outage", "abort-upload"])
p.add_argument("--base-url", default=None)
p.add_argument("--ca-file", default=None)
p.add_argument("--timeout", type=float, default=15)
p.add_argument("--service", default="api")
p.add_argument("--s3-service", default="s3")
p.add_argument(
"--restore", action="store_true", help="failover-upload: start the stopped replica again"
)
p.add_argument("--declared", type=int, default=5 * 1024 * 1024)
p.add_argument("--sent", type=int, default=256 * 1024)
p.add_argument("--settle", type=float, default=3)
p.add_argument("--output", metavar="FILE")
args = p.parse_args()
c = Client(args.base_url, args.ca_file, timeout=args.timeout)
run = {
"failover-upload": failover_upload,
"s3-outage": s3_outage,
"abort-upload": abort_upload,
}[args.scenario]
try:
report = {"ok": True, **run(c, args)}
except Failure as exc:
report = {"scenario": args.scenario, "ok": False, "error": str(exc)}
text = json.dumps(report, ensure_ascii=False, indent=2)
if args.output:
Path(args.output).write_text(text + "\n")
print(text)
sys.exit(0 if report["ok"] else 1)
if __name__ == "__main__":
main()
+352
View File
@@ -0,0 +1,352 @@
#!/usr/bin/env python3
import argparse
import base64
import hashlib
import http.client
import json
import os
import re
import struct
import subprocess
import tempfile
import threading
import time
import uuid
import zlib
from pathlib import Path
from urllib.parse import urlsplit
ROOT = Path(__file__).resolve().parents[1]
MAX_UPLOAD = 10 * 1024 * 1024
COLUMNS = (
"id",
"chat_id",
"owner_id",
"filename",
"content_type",
"size_bytes",
"sha256",
"width",
"height",
"object_key",
"status",
)
def chunk(tag, data):
return (
struct.pack(">I", len(data))
+ tag
+ data
+ struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF)
)
def png(width, height):
raw = b"".join(b"\x00" + os.urandom(width * 3) for _ in range(height))
header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
return (
b"\x89PNG\r\n\x1a\n"
+ chunk(b"IHDR", header)
+ chunk(b"IDAT", zlib.compress(raw, 1))
+ chunk(b"IEND", b"")
)
def sized_png(target):
height = max(1, target // (1024 * 3 + 1))
body = png(1024, height)
while len(body) > min(target, MAX_UPLOAD) and height > 1:
height -= max(1, (len(body) - target) // 3073 + 1)
body = png(1024, height)
return body
def run(cmd, stdin=None):
out = subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, check=False)
if out.returncode:
raise SystemExit(
f"command failed: {' '.join(cmd[:6])}\n{out.stderr.decode(errors='replace')[-500:]}"
)
return out.stdout.decode()
def psql(sql, stdin=None):
user, db = os.environ.get("PGUSER", "messenger"), os.environ.get("PGDATABASE", "messenger")
psql = [
"docker",
"compose",
"exec",
"-T",
"postgres",
"psql",
"-v",
"ON_ERROR_STOP=1",
"-qAt",
"-F",
" ",
]
return run([*psql, "-U", user, "-d", db, "-c", sql], stdin)
def direct(args):
bucket = os.environ.get("S3_BUCKET", "messenger-attachments")
prefix = os.environ.get("S3_PREFIX", "attachments/")
if args.chat_id and args.owner_id:
chat_id, owner_id = args.chat_id, args.owner_id
else:
found = psql(
"SELECT chat_id, owner_id FROM attachments ORDER BY created_at LIMIT 1"
).split()
if len(found) != 2:
raise SystemExit(
"no attachment rows yet: upload one via API or pass --chat-id and --owner-id"
)
chat_id, owner_id = found
rows = []
started = time.monotonic()
with tempfile.TemporaryDirectory(prefix="lab4-seed-") as tmp:
Path(tmp).chmod(0o755)
for i in range(args.count):
aid = str(uuid.uuid4())
body = png(8, 8)
path = Path(tmp) / aid
path.write_bytes(body)
path.chmod(0o644)
digest = hashlib.sha256(body).hexdigest()
rows.append(
f"{aid},{chat_id},{owner_id},seed-{i}.png,image/png,{len(body)},{digest},8,8,{prefix}{aid},ready"
)
generated = time.monotonic()
cli = [
"docker",
"compose",
"run",
"--rm",
"--no-deps",
"-T",
"-v",
f"{tmp}:/seed:ro",
"s3-cli",
]
run([*cli, "put", "--recursive", "--no-progress", "/seed/", f"root/{bucket}/{prefix}"])
uploaded = time.monotonic()
columns = ", ".join(COLUMNS)
psql(
f"COPY attachments ({columns}) FROM STDIN WITH (FORMAT csv)",
"".join(f"{r}\n" for r in rows).encode(),
)
seconds = {
"generate": round(generated - started, 3),
"s3_put": round(uploaded - generated, 3),
"pg_copy": round(time.monotonic() - uploaded, 3),
}
return {"mode": "direct", "count": args.count, "seconds": seconds}
class Http:
def __init__(self, base, timeout):
parts = urlsplit(base)
self.cls = (
http.client.HTTPSConnection if parts.scheme == "https" else http.client.HTTPConnection
)
self.host, self.timeout, self.local = parts.netloc, timeout, threading.local()
def call(self, method, path, body=None, headers=None, token=None):
conn = getattr(self.local, "conn", None) or self.cls(self.host, timeout=self.timeout)
self.local.conn = conn
h = {"Accept": "application/json"}
if token:
h["Authorization"] = "Bearer " + token
if isinstance(body, dict):
body, h["Content-Type"] = json.dumps(body).encode(), "application/json"
h.update(headers or {})
try:
conn.request(method, path, body=body, headers=h)
response = conn.getresponse()
data = response.read()
except (OSError, http.client.HTTPException):
conn.close()
self.local.conn = None
return 0, {}, b""
return response.status, {k.lower(): v for k, v in response.getheaders()}, data
def login(api, label):
name = f"seed_{label}_{uuid.uuid4().hex[:12]}"
password = "Seed-" + uuid.uuid4().hex
body = {"username": name, "display_name": label, "password": password}
status, _, data = api.call("POST", "/api/v1/users", body)
if status != 201:
raise SystemExit(f"user create failed: {status}")
user = json.loads(data)
basic = base64.b64encode(f"{name}:{password}".encode()).decode()
status, _, data = api.call(
"POST", "/api/v1/auth/sessions", headers={"Authorization": f"Basic {basic}"}
)
if status != 201:
raise SystemExit(f"login failed: {status}")
return user["id"], json.loads(data)["token"]
def mib(text):
m = re.match(r"([\d.]+)\s*([KMG]i?B)", text)
if not m:
return 0.0
scale = {"KiB": 1 / 1024, "KB": 1 / 1024, "GiB": 1024, "GB": 1024}
return float(m.group(1)) * scale.get(m.group(2), 1)
def sampler(ids, peaks, stop):
while not stop.is_set():
cmd = ["docker", "stats", "--no-stream", "--format", "{{.ID}} {{.MemUsage}}", *ids]
out = subprocess.run(cmd, capture_output=True, text=True, check=False).stdout
for line in out.splitlines():
cid, usage = line.split(" ", 1)
peaks[cid] = max(peaks.get(cid, 0.0), mib(usage))
def cgroup(cid, name):
cmd = ["docker", "exec", cid, "cat", f"/sys/fs/cgroup/{name}"]
return subprocess.run(cmd, capture_output=True, text=True, check=False).stdout.strip()
def percentile(values, q):
return (
round(sorted(values)[min(len(values) - 1, int(q * len(values)))] * 1000, 1)
if values
else None
)
def over_http(args):
api = Http(args.base_url, args.timeout)
alice, token = login(api, "alice")
bob, _ = login(api, "bob")
status, _, data = api.call(
"POST", "/api/v1/chats", {"title": "seed", "member_ids": [alice, bob]}, token=token
)
if status != 201:
raise SystemExit(f"chat create failed: {status}")
chat = json.loads(data)["id"]
fixed = sized_png(args.size) if args.size else None
counter, lock, results = iter(range(args.count)), threading.Lock(), []
ids = run(["docker", "compose", "ps", "-q", args.service]).split() if args.sample else []
idle = {cid: round(int(cgroup(cid, "memory.current") or 0) / 1048576, 1) for cid in ids}
peaks, stop = {}, threading.Event()
watcher = threading.Thread(target=sampler, args=(ids, peaks, stop), daemon=True)
if ids:
watcher.start()
def worker():
while True:
with lock:
i = next(counter, None)
if i is None:
return
body = fixed or png(8, 8)
began = time.monotonic()
path = f"/api/v1/chats/{chat}/attachments?filename=seed-{i}.png"
status, headers, data = api.call(
"POST", path, body, {"Content-Type": "image/png"}, token
)
created = json.loads(data)["id"] if status == 201 else None
results.append(
(status, time.monotonic() - began, headers.get("x-instance-id", ""), created)
)
started = time.monotonic()
threads = [threading.Thread(target=worker) for _ in range(args.concurrency)]
for t in threads:
t.start()
for t in threads:
t.join()
elapsed = time.monotonic() - started
stop.set()
if ids:
watcher.join()
created = [r[3] for r in results if r[3]]
for start in range(0, min(args.link, len(created)), 10):
body = {"text": "", "attachment_ids": created[start : min(start + 10, args.link)]}
api.call(
"POST",
f"/api/v1/chats/{chat}/messages",
body,
{"Idempotency-Key": uuid.uuid4().hex},
token,
)
if args.ids_out:
Path(args.ids_out).write_text("".join(f"{i}\n" for i in created))
codes, instances = {}, {}
for status, _, instance, _ in results:
codes[str(status)] = codes.get(str(status), 0) + 1
instances[instance] = instances.get(instance, 0) + 1
ok = [r[1] for r in results if r[0] == 201]
memory = {}
for cid in ids:
peak = cgroup(cid, "memory.peak")
events = dict(line.split() for line in cgroup(cid, "memory.events").splitlines())
memory[cid[:12]] = {
"idle_mib": idle[cid],
"docker_stats_peak_mib": round(peaks.get(cid[:12], 0.0), 1),
"cgroup_memory_peak_mib": round(int(peak) / 1048576, 1) if peak.isdigit() else None,
"oom_kill": int(events.get("oom_kill", 0)),
}
return {
"mode": "http",
"count": args.count,
"concurrency": args.concurrency,
"body_bytes": len(fixed) if fixed else "~200 (unique 8x8 PNG)",
"status_codes": codes,
"instances": len([i for i in instances if i]),
"linked_in_messages": min(args.link, len(created)),
"elapsed_seconds": round(elapsed, 3),
"successful_rps": round(len(ok) / elapsed, 2) if elapsed else None,
"latency_ms": {
"p50": percentile(ok, 0.5),
"p95": percentile(ok, 0.95),
"p99": percentile(ok, 0.99),
"max": percentile(ok, 1.0),
},
"memory": memory,
}
def main():
p = argparse.ArgumentParser(
description="Lab4 attachment volume generator: HTTP uploads or direct S3+PG seeding"
)
p.add_argument("--mode", choices=["http", "direct"], default="http")
p.add_argument("--count", type=int, default=1000)
p.add_argument("--concurrency", type=int, default=16)
p.add_argument(
"--size",
type=int,
default=0,
help="http: bytes per upload (one PNG reused); 0 = unique 8x8 PNGs",
)
p.add_argument(
"--link", type=int, default=0, help="http: reference the first N attachments from messages"
)
p.add_argument(
"--sample",
action="store_true",
help="http: sample docker stats and memory.peak of API containers",
)
p.add_argument("--service", default="api")
p.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080"))
p.add_argument("--timeout", type=float, default=60)
p.add_argument("--ids-out", metavar="FILE")
p.add_argument("--chat-id")
p.add_argument("--owner-id")
p.add_argument("--output", metavar="FILE")
args = p.parse_args()
report = direct(args) if args.mode == "direct" else over_http(args)
text = json.dumps(report, ensure_ascii=False, indent=2)
if args.output:
Path(args.output).write_text(text + "\n")
print(text)
if __name__ == "__main__":
main()
+11
View File
@@ -0,0 +1,11 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
service=${API_SERVICE:-api}
want=${1:-$(docker compose ps -q "$service" | wc -l | tr -d ' ')}
url=${LB_STATS_URL:-http://$(docker compose port lb 8404)/stats;csv}
deadline=$(($(date +%s) + ${WAIT_TIMEOUT:-60}))
until [ "$(curl -fsS "$url" | awk -F, -v b="$service" '$1 == b && $2 != "BACKEND" && $18 == "UP"' | wc -l)" -ge "$want" ]; do
[ "$(date +%s)" -lt "$deadline" ] || { echo "LB does not see $want UP $service servers" >&2; exit 4; }
sleep 0.5
done
+156
View File
@@ -0,0 +1,156 @@
#!/usr/bin/env python3
import argparse
import json
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
S3_CLIENT_SECRETS = ["s3_access_key_id", "s3_root_password", "s3_root_user", "s3_secret_access_key"]
POLICY = {
"lb": {"secrets": [], "volumes": [], "networks": ["backend", "edge"], "ports": True},
"api": {
"secrets": [
"cache_url",
"cursor_secret",
"database_url",
"s3_access_key_id",
"s3_secret_access_key",
],
"volumes": [],
"networks": ["backend"],
},
"migrate": {"secrets": ["database_url"], "volumes": [], "networks": ["backend"]},
"postgres": {"secrets": ["postgres_password"], "volumes": ["pgdata"], "networks": ["backend"]},
"valkey": {"secrets": ["valkey.conf"], "volumes": ["valkeydata"], "networks": ["backend"]},
"k6": {"secrets": [], "volumes": [], "networks": ["edge"]},
"s3": {
"secrets": ["s3_root_password", "s3_root_user"],
"volumes": ["s3data"],
"networks": ["backend"],
},
"s3-init": {"secrets": S3_CLIENT_SECRETS, "volumes": [], "networks": ["backend"]},
"s3-cli": {"secrets": S3_CLIENT_SECRETS, "volumes": [], "networks": ["backend"]},
}
def run(*command):
done = subprocess.run(command, cwd=str(ROOT), capture_output=True, text=True, check=False)
return done.returncode, done.stdout.strip()
def declared(name, service, networks, problems):
limits = service.get("deploy", {}).get("resources", {}).get("limits", {})
secrets = sorted(s["source"] for s in service.get("secrets", []))
volumes = sorted(v["source"] for v in service.get("volumes", []) if v["type"] == "volume")
nets = sorted(service.get("networks", {}))
ports = [
f"{p.get('host_ip', '0.0.0.0')}:{p.get('published')}" for p in service.get("ports", [])
]
want = POLICY.get(name)
if want is None:
problems.append(f"{name}: service is not covered by the policy")
want = {"secrets": [], "volumes": [], "networks": []}
if not limits.get("cpus") or not limits.get("memory"):
problems.append(f"{name}: no CPU or memory limit")
if "container_name" in service:
problems.append(f"{name}: container_name prevents scaling")
for kind, actual in (("secrets", secrets), ("volumes", volumes), ("networks", nets)):
if actual != want[kind]:
problems.append(f"{name}: {kind} {actual}, policy {want[kind]}")
if ports and not want.get("ports"):
problems.append(f"{name}: publishes {ports} in bypass of the balancer")
if any(not port.startswith("127.0.0.1:") for port in ports):
problems.append(f"{name}: publishes {ports} beyond loopback")
if name == "api" and not service.get("read_only"):
problems.append("api: root filesystem is writable")
internal = [n for n in nets if networks.get(n, {}).get("internal")]
return [
name,
f"{limits.get('cpus', '-')}/{int(limits.get('memory') or 0) >> 20}MiB",
",".join(ports) or "-",
",".join(secrets) or "-",
",".join(volumes) or "-",
",".join(f"{n}{'(internal)' if n in internal else ''}" for n in nets),
str(service.get("read_only", False)).lower(),
]
def running(problems):
_, ids = run("docker", "compose", "ps", "-q")
template = (
'{{index .Config.Labels "com.docker.compose.service"}} {{.Config.Hostname}} '
"{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}} {{json .HostConfig.PortBindings}}"
)
_, found = run("docker", "inspect", "-f", template, *ids.split())
rows = []
for line in found.splitlines():
service, host, cpus, memory, bindings = line.split(" ", 4)
published = [
f"{b['HostIp']}:{b['HostPort']}->{port}"
for port, items in (json.loads(bindings) or {}).items()
for b in items or []
]
if int(cpus) <= 0 or int(memory) <= 0:
problems.append(f"container {service}/{host}: no CPU or memory limit applied")
if published and service != "lb":
problems.append(f"container {service}/{host}: publishes {published}")
rows.append(
[service, host, f"{int(cpus) / 1e9:g}", f"{int(memory) >> 20}MiB", ",".join(published)]
)
return sorted(rows)
def table(header, rows):
widths = [max(len(str(r[i])) for r in [header, *rows]) for i in range(len(header))]
lines = [header, ["-" * w for w in widths], *rows]
return [" ".join(str(c).ljust(w) for c, w in zip(r, widths)).rstrip() for r in lines]
def main():
parser = argparse.ArgumentParser(description="limits, ports, secrets, volumes and networks")
parser.add_argument("--out", help="text report, written only after a complete check")
args = parser.parse_args()
problems = []
_, raw = run(
"docker",
"compose",
"--profile",
"bench",
"--profile",
"tools",
"config",
"--format",
"json",
)
config = json.loads(raw)
services = [
declared(name, service, config.get("networks", {}), problems)
for name, service in sorted(config["services"].items())
]
containers = running(problems)
if not any(row[0] == "api" for row in containers):
problems.append("no running api replicas to inspect")
lines = [
f"date: {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}",
"$ docker compose --profile bench --profile tools config --format json (declared)",
*table(["service", "cpus/mem", "ports", "secrets", "volumes", "networks", "ro"], services),
"",
"$ docker inspect (running containers: hostname = X-Instance-Id, applied limits)",
*table(["service", "hostname", "cpus", "memory", "published"], containers),
"",
*(f"FAIL: {p}" for p in problems),
"PASS: limits, ports, secrets, volumes and networks match the policy"
if not problems
else f"FAIL: {len(problems)} violation(s)",
]
text = "\n".join(lines) + "\n"
if args.out:
Path(args.out).write_text(text)
print(text, end="")
return 1 if problems else 0
if __name__ == "__main__":
sys.exit(main())
+14
View File
@@ -0,0 +1,14 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
started() {
for id in $(docker compose ps -aq); do
docker inspect --format '{{index .Config.Labels "com.docker.compose.service"}} {{.Config.Hostname}} {{.State.StartedAt}} {{.State.Status}}' "$id"
done | sort
}
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"
echo "== before: service hostname StartedAt status"
started
make --no-print-directory test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL="${1:-http://localhost:8080}"
echo "== after: service hostname StartedAt status"
started
+436
View File
@@ -0,0 +1,436 @@
#!/usr/bin/env python3
import argparse
import base64
import collections
import functools
import json
import os
import subprocess
import sys
import threading
import time
import urllib.error
import urllib.request
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
COUNTERS = ("wretr", "wredis", "econ", "eresp", "hrsp_5xx")
@functools.cache
def published(port, fallback):
command = ["docker", "compose", "port", "lb", str(port)]
found = subprocess.run(command, cwd=str(ROOT), capture_output=True, text=True, check=False)
return found.stdout.strip() or fallback
def base_url():
return os.environ.get("BASE_URL") or "http://" + published(8080, "localhost:8080")
def stats_url():
return os.environ.get("LB_STATS_URL") or f"http://{published(8404, '127.0.0.1:8404')}/stats;csv"
def call(method, path, body=None, headers=None, timeout=2.0):
data = None if body is None else json.dumps(body, ensure_ascii=False).encode()
head = {"Accept": "application/json"}
if data is not None:
head["Content-Type"] = "application/json"
head.update(headers or {})
request = urllib.request.Request(
base_url().rstrip("/") + path, data=data, headers=head, method=method
)
try:
with urllib.request.urlopen(request, timeout=timeout) as response:
payload = json.loads(response.read() or b"null")
return response.status, response.headers.get("X-Instance-Id", ""), payload
except urllib.error.HTTPError as error:
try:
payload = json.loads(error.read() or b"null")
except (OSError, ValueError):
payload = None
return error.code, error.headers.get("X-Instance-Id", ""), payload
except (OSError, ValueError):
return 0, "", None
def emit(report, out=None):
text = json.dumps(report, ensure_ascii=False, indent=2)
if out:
Path(out).parent.mkdir(parents=True, exist_ok=True)
Path(out).write_text(text + "\n")
print(text)
def created(result, what):
if result[0] != 201:
raise SystemExit(f"{what} failed: HTTP {result[0]}")
return result[2]
def actor(label):
username = f"p_{label}_{uuid.uuid4().hex[:12]}"
password = "Probe-" + uuid.uuid4().hex
body = {"username": username, "display_name": "probe " + label, "password": password}
user = created(call("POST", "/api/v1/users", body, timeout=10), "register")
basic = base64.b64encode(f"{username}:{password}".encode()).decode()
login = call(
"POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}, timeout=10
)
return {
"id": user["id"],
"auth": {"Authorization": "Bearer " + created(login, "login")["token"]},
}
def chat_of(owner, other):
body = {"title": "probe", "member_ids": [owner["id"], other["id"]]}
return created(call("POST", "/api/v1/chats", body, owner["auth"], timeout=10), "chat")["id"]
def send(owner, chat, text, key):
headers = dict(owner["auth"], **{"Idempotency-Key": key})
return call("POST", f"/api/v1/chats/{chat}/messages", {"text": text}, headers)
def lb_rows(backend="api"):
try:
with urllib.request.urlopen(stats_url(), timeout=2) as response:
lines = response.read().decode().splitlines()
except OSError:
return None
header = lines[0].lstrip("# ").split(",")
rows = [dict(zip(header, line.split(","))) for line in lines[1:] if line]
return [r for r in rows if r["pxname"] == backend]
def lb_up():
rows = lb_rows()
if rows is None:
return None
return sum(1 for r in rows if r["svname"] != "BACKEND" and r["status"].startswith("UP"))
def lb_counters():
row = next((r for r in lb_rows() or [] if r["svname"] == "BACKEND"), {})
return {name: int(row.get(name) or 0) for name in COUNTERS}
def run_k6(out, env, on_start=None):
Path(out).mkdir(parents=True, exist_ok=True)
started = []
process = subprocess.Popen(
[str(ROOT / "scripts/k6.sh"), str(out)],
env=dict(os.environ, **env),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
)
with (Path(out) / "k6.txt").open("w") as log:
for line in process.stdout:
log.write(line)
if not started and "LOAD-START" in line:
started.append(time.time())
if on_start:
on_start()
return process.wait(), started[0] if started else None
def k6_metrics(path, duration):
metrics = json.loads(Path(path).read_text())["metrics"]
def value(name, field):
return metrics.get(name + "{scenario:measure}", {}).get("values", {}).get(field, 0.0)
count = value("http_reqs", "count")
return {
"requests": int(count),
"rps": round(count / duration, 2),
"ok_rps": round(count * (1 - value("http_req_failed", "rate")) / duration, 2),
"failed_ratio": round(value("http_req_failed", "rate"), 5),
"dropped_iterations": int(value("dropped_iterations", "count")),
"p50_ms": round(value("http_req_duration", "med"), 2),
"p95_ms": round(value("http_req_duration", "p(95)"), 2),
"p99_ms": round(value("http_req_duration", "p(99)"), 2),
}
def race(alice, chat, parallel):
key, text = uuid.uuid4().hex, "same key " + uuid.uuid4().hex
barrier, results = threading.Barrier(parallel), []
def worker():
barrier.wait()
results.append(send(alice, chat, text, key))
threads = [threading.Thread(target=worker) for _ in range(parallel)]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
bodies = [r[2] for r in results if r[0] == 201]
identical = bool(bodies) and all(b == bodies[0] for b in bodies)
return text, results, identical, send(alice, chat, text + " changed", key)[0]
def check(args):
alice, bob = actor("alice"), actor("bob")
chat = chat_of(alice, bob)
seen = collections.Counter(
call("GET", "/api/v1/users/me", headers=alice["auth"])[1] for _ in range(args.samples)
)
rounds = [race(alice, chat, args.parallel) for _ in range(min(args.rounds, 100))]
page = call("GET", f"/api/v1/chats/{chat}/messages?limit=100", headers=bob["auth"])[2]
copies = collections.Counter(m.get("text") for m in (page or {"items": []})["items"])
statuses = collections.Counter(r[0] for _, results, _, _ in rounds for r in results)
served = collections.Counter(r[1] for _, results, _, _ in rounds for r in results)
same = {
"rounds": len(rounds),
"parallel": args.parallel,
"statuses": dict(statuses),
"instances": dict(served),
"identical_bodies_rounds": sum(1 for _, _, identical, _ in rounds if identical),
"single_copy_rounds": sum(1 for text, _, _, _ in rounds if copies[text] == 1),
"conflict_rounds": sum(1 for _, _, _, other in rounds if other == 409),
}
passed = (
len(seen) >= 2
and statuses == {201: len(rounds) * args.parallel}
and len(served) >= 2
and same["identical_bodies_rounds"] == len(rounds)
and same["single_copy_rounds"] == len(rounds)
and same["conflict_rounds"] == len(rounds)
)
emit({"instances_one_token": dict(seen), "same_key": same, "pass": passed}, args.out)
return 0 if passed else 1
def record(stop, alice, chat):
probes, trace = [], []
def prober():
key = uuid.uuid4().hex
while not stop.is_set():
started = time.time()
status, instance, _ = send(alice, chat, "probe " + key, key)
probes.append((started, status, round(1000 * (time.time() - started), 1), instance))
if status == 201:
key = uuid.uuid4().hex
time.sleep(0.1)
def poller():
while not stop.is_set():
trace.append((time.time(), lb_up()))
time.sleep(0.1)
threads = [threading.Thread(target=t, daemon=True) for t in (prober, poller)]
for thread in threads:
thread.start()
return probes, trace, threads
def write_csv(path, header, rows):
with path.open("w") as handle:
handle.write(header + "\n")
handle.writelines(",".join(str(v) for v in row) + "\n" for row in rows)
def summarize(args, probes, trace, action, expected):
t0, end = action["start"], action["end"]
label = args.label or os.environ.get("TARGET_INSTANCE_ID", "action")
failures = [t - t0 for t, s, _, _ in probes if s != 201 and t >= t0]
down = [t for t, up in trace if t >= t0 and up is not None and up < expected]
back = [t for t, up in trace if down and t > down[0] and up is not None and up >= expected]
outage = [t for t in down if not back or t < back[0]] or [end]
served = [(t, i) for t, s, _, i in probes if s == 201]
before = collections.Counter(i for t, i in served if t < t0)
during = collections.Counter(i for t, i in served if outage[0] <= t <= outage[-1])
after = collections.Counter(i for t, i in served if t > end)
return {
"label": label,
"action": args.command,
"action_exit": action["exit"],
"action_seconds": round(end - t0, 3),
"action_finished_before_load_end": end < action["load_end"],
"lb_up_before": expected,
"lb_up_min": min((up for _, up in trace if up is not None), default=None),
"detection_seconds": round(down[0] - t0, 3) if down else None,
"rejoin_seconds": round(back[0] - t0, 3) if back else None,
"probe": {
"attempts": len(probes),
"failed_attempts": sum(1 for _, s, _, _ in probes if s != 201),
"first_failure_s": round(failures[0], 3) if failures else None,
"recovery_s": round(failures[-1], 3) if failures else 0.0,
"instances_before": dict(before),
"instances_during_outage": dict(during),
"instances_after": dict(after),
},
}
def verdict(args, result):
probe, k6 = result["probe"], result["k6"]
checks = {
"action_ok": result["action_exit"] == 0 and result["action_finished_before_load_end"],
"load_errors_ok": k6["exit"] == 0 and k6["failed_ratio"] <= args.max_failed_ratio,
"write_recovery_ok": probe["recovery_s"] <= args.max_recovery,
}
if args.expect == "dip":
checks["detected"] = result["detection_seconds"] is not None
checks["target_excluded"] = probe["instances_during_outage"].get(result["label"], 0) == 0
checks["rejoined"] = result["rejoin_seconds"] is not None and bool(
probe["instances_after"].get(result["label"])
)
if args.expect == "replace":
checks["capacity_kept"] = (result["lb_up_min"] or 0) >= result["lb_up_before"]
old, new = set(probe["instances_before"]), set(probe["instances_after"])
checks["all_replaced"] = bool(new) and not old & new
return checks
def load(args):
out = Path(args.out)
out.mkdir(parents=True, exist_ok=True)
expected = lb_up() or 0
alice, bob = actor("alice"), actor("bob")
stop, action = threading.Event(), {}
probes, trace, threads = record(stop, alice, chat_of(alice, bob))
def act():
time.sleep(args.warmup + args.lead)
action["start"] = time.time()
with (out / "action.txt").open("w") as log:
action["exit"] = subprocess.call(
args.command, cwd=str(ROOT), stdout=log, stderr=subprocess.STDOUT
)
action["end"] = time.time()
runner = threading.Thread(target=act)
env = {"RATE": str(args.rate), "WARMUP": str(args.warmup), "DURATION": str(args.duration)}
counters = lb_counters()
code, load_start = run_k6(out / "k6", env, runner.start)
if load_start is not None:
runner.join()
action["load_end"] = load_start + args.warmup + args.duration
stop.set()
for thread in threads:
thread.join()
if "start" not in action:
raise SystemExit(f"k6 failed before the action (exit {code}), see {out / 'k6/k6.txt'}")
t0 = action["start"]
write_csv(
out / "probe.csv",
"t_rel_s,status,latency_ms,instance",
((round(t - t0, 3), *r) for t, *r in probes),
)
write_csv(out / "lb_up.csv", "t_rel_s,up", ((round(t - t0, 3), up) for t, up in trace))
result = summarize(args, probes, trace, action, expected)
result["lb_counters"] = {k: v - counters[k] for k, v in lb_counters().items()}
result["k6"] = dict(
k6_metrics(out / "k6/summary.json", args.duration), exit=code, rate=args.rate
)
result["checks"] = verdict(args, result)
result["pass"] = all(result["checks"].values())
(out / "result.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
print(json.dumps(result, ensure_ascii=False))
return 0 if result["pass"] else 1
def detect(args):
before = lb_up() or 0
until = before - 1 if args.until is None else args.until
started = time.time()
process = subprocess.Popen(args.command, cwd=str(ROOT))
detected = acted = None
while time.time() - started < args.timeout and (detected is None or acted is None):
if acted is None and process.poll() is not None:
acted = time.time() - started
if detected is None and (lb_up() or 0) <= until:
detected = time.time() - started
time.sleep(0.05)
code = process.wait()
acted = time.time() - started if acted is None else acted
status, instance, payload = call("GET", "/health/live")
restored = None
if args.restore:
begun = time.time()
subprocess.call(["sh", "-c", args.restore], cwd=str(ROOT))
while (lb_up() or 0) < before and time.time() - begun < 3 * args.timeout:
time.sleep(0.1)
restored = round(time.time() - begun, 3) if (lb_up() or 0) >= before else None
checks = {
"action_ok": code == 0,
"detected_in_time": detected is not None and detected <= args.max + args.slack,
"lb_answer_ok": status == args.lb_status,
}
if args.within_action:
checks["excluded_before_action_end"] = detected is not None and detected < acted
if args.restore:
checks["restored"] = restored is not None
report = {
"action": args.command,
"action_exit": code,
"action_seconds": round(acted, 3),
"lb_up_before": before,
"lb_up_counted_as_detected": until,
"detection_seconds": None if detected is None else round(detected, 3),
"declared_max_seconds": args.max,
"measurement_slack_seconds": args.slack,
"lb_answer_after_detection": {
"path": "/health/live",
"status": status,
"x_instance_id_present": bool(instance),
"error_code": ((payload or {}).get("error") or {}).get("code"),
},
"restore": args.restore,
"restore_seconds": restored,
"lb_up_after_restore": lb_up(),
"checks": checks,
"pass": all(checks.values()),
}
emit(report, args.out)
return 0 if report["pass"] else 1
def main():
parser = argparse.ArgumentParser(description="LB probes over the public HTTP contract")
sub = parser.add_subparsers(dest="mode", required=True)
one = sub.add_parser("check", help="instances behind one token and same-key races")
one.add_argument("--samples", type=int, default=40)
one.add_argument("--parallel", type=int, default=8)
one.add_argument("--rounds", type=int, default=1)
one.add_argument("--out", help="JSON file, written only after a complete report")
under = sub.add_parser("load", help="run an action under k6 load and measure recovery")
under.add_argument("--out", required=True)
under.add_argument("--label")
under.add_argument("--expect", choices=("dip", "replace"), default="dip")
under.add_argument("--rate", type=int, default=int(os.environ.get("RATE", "50")))
under.add_argument("--warmup", type=int, default=10)
under.add_argument("--duration", type=int, default=int(os.environ.get("DURATION", "60")))
under.add_argument("--lead", type=int, default=15)
under.add_argument("--max-failed-ratio", type=float, default=0.01)
under.add_argument("--max-recovery", type=float, default=10.0)
under.add_argument("command", nargs=argparse.REMAINDER)
watch = sub.add_parser("detect", help="time until the LB takes replicas out of rotation")
watch.add_argument("--max", type=float, required=True, help="declared worst detection, s")
watch.add_argument("--slack", type=float, default=0.25, help="docker CLI and polling, s")
watch.add_argument("--until", type=int, help="UP servers counted as detected")
watch.add_argument("--within-action", action="store_true")
watch.add_argument("--lb-status", type=int, default=200)
watch.add_argument("--restore", help="shell command that brings replicas back")
watch.add_argument("--timeout", type=float, default=20)
watch.add_argument("--out", help="JSON file, written only after a complete report")
watch.add_argument("command", nargs=argparse.REMAINDER)
args = parser.parse_args()
if args.mode == "check":
return check(args)
args.command = [c for c in args.command if c != "--"]
if not args.command:
parser.error(f"{args.mode} needs an action command after --")
return load(args) if args.mode == "load" else detect(args)
if __name__ == "__main__":
sys.exit(main())
+12
View File
@@ -0,0 +1,12 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
defined=$(docker compose config --services)
stores=""
for service in ${STATE_SERVICES:-postgres valkey s3}; do
if printf '%s\n' "$defined" | grep -qx "$service"; then stores="$stores $service"; fi
done
[ -n "$stores" ] || { echo "no state services found" >&2; exit 3; }
docker compose stop lb "${API_SERVICE:-api}"
docker compose restart $stores
docker compose up -d --no-recreate --wait --wait-timeout "${WAIT_TIMEOUT:-75}"
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
docker compose up -d --no-deps --no-recreate --wait --wait-timeout "${WAIT_TIMEOUT:-60}" "${API_SERVICE:-api}"
scripts/lb-wait.sh
+20
View File
@@ -0,0 +1,20 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
service=${API_SERVICE:-api}
show() { docker inspect -f '{{.Config.Hostname}} image={{slice .Image 7 19}}' "$1"; }
echo "$(date -u +%T) migrate once before the first replacement"
docker compose run --rm --no-deps -T migrate 2>&1
old=$(docker compose ps -q "$service")
count=$(printf '%s\n' "$old" | grep -c .)
for id in $old; do
running=$(docker compose ps -q "$service")
docker compose up -d --no-deps --no-recreate --wait --wait-timeout "${WAIT_TIMEOUT:-90}" --scale "$service=$((count + 1))" "$service" 2>&1
scripts/lb-wait.sh $((count + 1))
new=$(docker compose ps -q "$service" | grep -v -F "$running")
echo "$(date -u +%T) replace $(show "$id") -> $(show "$new")"
docker stop -t "${STOP_TIMEOUT:-6}" "$id" >/dev/null
docker rm "$id" >/dev/null
done
scripts/lb-wait.sh "$count"
echo "$(date -u +%T) done: $count replicas replaced"
+66
View File
@@ -0,0 +1,66 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
cli() { docker compose run --rm --no-deps -T s3-cli "$@"; }
snapshot() {
cli sh -s <<'EOF' | grep -v '_date'
ak=$(cat /run/secrets/s3_access_key_id)
for c in "bucket list root" "bucket anonymous get root/$S3_BUCKET" "admin policy info root messenger-api" "admin user info root $ak" "bucket lifecycle rule export root/$S3_BUCKET"; do
rc $c --json
done | sed "s/$ak/<api-key>/g"
EOF
}
failed=0
verdict() { if [ "$2" = "$3" ]; then echo "PASS $1: $3"; else echo "FAIL $1: ожидалось $2, получено $3"; failed=1; fi; }
echo "== $(date -u +%FT%TZ) S3 privacy and bootstrap checks"
verdict "s3: опубликованные host-порты" "нет" "$(docker inspect -f '{{json .NetworkSettings.Ports}}' "$(docker compose ps -q s3)" | grep -q HostPort && echo есть || echo нет)"
verdict "s3: консоль :9001 внутри контейнера" "000" "$(docker compose exec -T s3 curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:9001/ || true)"
before=$(snapshot)
first=$(docker compose run --rm --no-deps -T s3-init >/dev/null 2>&1 && echo 0 || echo 1)
second=$(docker compose run --rm --no-deps -T s3-init 2>/dev/null || echo FAILED)
after=$(snapshot)
verdict "s3-init: повторный запуск 1" "0" "$first"
verdict "s3-init: повторный запуск 2" "0" "$(echo "$second" | grep -q '^s3-init:' && echo 0 || echo 1)"
verdict "s3-init: состояние IAM/bucket не изменилось" "same" "$([ "$before" = "$after" ] && echo same || echo changed)"
cli sh -s <<'EOF' || failed=1
b=$S3_BUCKET
key="attachments/s3-check-$(date +%s)-$$"
other="s3-check-$(date +%s)-$$"
head -c 4096 /dev/urandom > /tmp/probe
result=0
expect() {
want=$1; name=$2; shift 2
if "$@" >/tmp/out 2>&1; then got=allow; elif grep -qE '403|AccessDenied|set_bucket_policy: service error' /tmp/out; then got=deny; else got=error; fi
if [ "$got" = "$want" ]; then echo "PASS $name: $got"; else echo "FAIL $name: ожидалось $want, получено $got"; result=1; fi
}
expect allow "api: PutObject" rc put /tmp/probe "api/$b/$key"
expect allow "api: HeadObject" rc object stat "api/$b/$key"
expect allow "api: GetObject" rc object show "api/$b/$key"
expect allow "api: ListBucket" rc object list "api/$b/attachments/"
expect deny "api: CreateBucket" rc bucket create "api/$other"
expect deny "api: DeleteBucket" rc bucket remove "api/$b"
expect deny "api: PutBucketPolicy (public)" rc bucket anonymous set public "api/$b"
expect deny "api: PutBucketLifecycle" rc bucket lifecycle rule import "api/$b" /s3/lifecycle.json
expect deny "api: admin ListUsers" rc admin user list api
rc bucket create --ignore-existing "root/$other" >/dev/null
expect deny "api: PutObject в чужой bucket" rc put /tmp/probe "api/$other/x"
expect deny "api: ListBucket чужого bucket" rc object list "api/$other"
rc bucket remove "root/$other" >/dev/null
expect deny "anonymous: HeadObject" rc object stat "anon/$b/$key"
expect deny "anonymous: ListBucket" rc object list "anon/$b"
expect deny "anonymous: PutObject" rc put /tmp/probe "anon/$b/attachments/anonymous"
status=$(wget -S -O /dev/null "$S3_ENDPOINT/$b/$key" 2>&1 | sed -n 's/^ *HTTP\/[0-9.]* \([0-9]*\).*/\1/p' | head -n 1)
if [ "$status" = 403 ]; then echo "PASS anonymous: прямой HTTP GET объекта из отдельного контейнера: $status"; else echo "FAIL anonymous: прямой HTTP GET: $status"; result=1; fi
permission=$(rc bucket anonymous get "root/$b" --json | sed -n 's/.*"permission": *"\([a-z]*\)".*/\1/p')
if [ "$permission" = private ]; then echo "PASS bucket: anonymous policy: $permission"; else echo "FAIL bucket: anonymous policy: $permission"; result=1; fi
expect allow "api: DeleteObject" rc object remove --force "api/$b/$key"
host=$(echo "$S3_ENDPOINT" | sed 's#^[a-z]*://##; s#:.*##')
if wget -T 2 -O /dev/null "http://$host:9001/" >/tmp/out 2>&1; then echo "FAIL s3: консоль :9001 отвечает из сети стенда"; result=1
elif grep -q refused /tmp/out; then echo "PASS s3: консоль :9001 из отдельного контейнера сети стенда: connection refused"
else echo "FAIL s3: консоль :9001 из сети стенда: $(tail -n 1 /tmp/out)"; result=1; fi
exit $result
EOF
echo "== IAM/bucket snapshot (без секретов)"
echo "$after"
if [ "$failed" != 0 ]; then echo "RESULT: FAIL"; exit 1; fi
echo "RESULT: PASS"
+24
View File
@@ -0,0 +1,24 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
umask 077
mkdir -p secrets
random() { od -An -N32 -tx1 /dev/urandom | tr -d ' \n'; }
put() {
if [ ! -s "secrets/$1" ]; then
printf '%s' "$2" > "secrets/$1"
chmod 644 "secrets/$1"
echo "создан secrets/$1"
fi
}
put postgres_password "$(random)"
put database_url "postgresql://messenger:$(cat secrets/postgres_password)@postgres:5432/messenger"
put cursor_secret "$(random)"
put valkey_password "$(random)"
put valkey.conf "requirepass $(cat secrets/valkey_password)"
put cache_url "redis://:$(cat secrets/valkey_password)@valkey:6379/0"
put s3_root_user "root$(random | cut -c1-16)"
put s3_root_password "$(random)"
put s3_access_key_id "api$(random | cut -c1-16)"
put s3_secret_access_key "$(random)"
[ -f .env ] || { cp .env.example .env; echo "создан .env из .env.example"; }
+59
View File
@@ -0,0 +1,59 @@
\set ON_ERROR_STOP on
\set big '''00000000-0000-7000-8000-00000000c001'''
\set cursor_at '''2026-01-01T02:46:40Z'''
\set cursor_id '''00000000-0000-7000-8001-000000002710'''
PREPARE history_first (uuid, int) AS
SELECT id, chat_id, sender_id, text, created_at
FROM messages
WHERE chat_id = $1
ORDER BY created_at, id
LIMIT $2;
PREPARE history_after (uuid, timestamptz, uuid, int) AS
SELECT id, chat_id, sender_id, text, created_at
FROM messages
WHERE chat_id = $1 AND (created_at, id) > ($2, $3)
ORDER BY created_at, id
LIMIT $4;
PREPARE history_offset (uuid, int, int) AS
SELECT id, chat_id, sender_id, text, created_at
FROM messages
WHERE chat_id = $1
ORDER BY created_at, id
LIMIT $2 OFFSET $3;
SELECT count(*) AS chat_messages FROM messages WHERE chat_id = :big;
SELECT count(*) AS all_messages FROM messages;
SELECT pg_size_pretty(pg_relation_size('messages_history_idx')) AS history_index_size;
\echo '== Первая страница, limit=50 (запрос HISTORY_FIRST приложения, LIMIT limit+1)'
EXPLAIN (ANALYZE, BUFFERS) EXECUTE history_first(:big, 51);
\echo '== Глубокая страница: keyset после 10 000-го сообщения (запрос HISTORY_AFTER приложения)'
EXPLAIN (ANALYZE, BUFFERS) EXECUTE history_after(:big, :cursor_at, :cursor_id, 51);
\echo '== Та же глубокая страница через OFFSET 10000 (не используется приложением, для сравнения)'
EXPLAIN (ANALYZE, BUFFERS) EXECUTE history_offset(:big, 51, 10000);
\echo '== Проверка: keyset и OFFSET возвращают одну и ту же страницу'
SELECT (
SELECT array_agg(id ORDER BY created_at, id) FROM (
SELECT id, created_at FROM messages
WHERE chat_id = :big AND (created_at, id) > (:cursor_at, :cursor_id)
ORDER BY created_at, id LIMIT 51
) AS keyset
) = (
SELECT array_agg(id ORDER BY created_at, id) FROM (
SELECT id, created_at FROM messages
WHERE chat_id = :big
ORDER BY created_at, id LIMIT 51 OFFSET 10000
) AS skipped
) AS same_page;
\echo '== Контрольный план без индекса истории (DROP INDEX внутри транзакции, затем ROLLBACK)'
BEGIN;
DROP INDEX messages_history_idx;
EXPLAIN (ANALYZE, BUFFERS) EXECUTE history_after(:big, :cursor_at, :cursor_id, 51);
ROLLBACK;
+40
View File
@@ -0,0 +1,40 @@
\set ON_ERROR_STOP on
\set author '''00000000-0000-7000-8000-00000000a001'''
\set big '''00000000-0000-7000-8000-00000000c001'''
\set epoch '''2026-01-01T00:00:00Z'''
INSERT INTO users (id, username, display_name, password_hash, created_at)
VALUES (:author, 'seed_author', 'Синтетический автор', '$argon2id$v=19$m=19456,t=2,p=1$c2VlZA$c2VlZA', :epoch)
ON CONFLICT DO NOTHING;
INSERT INTO chats (id, title, created_at)
SELECT ('00000000-0000-7000-8000-00000000c' || lpad(to_hex(chat), 3, '0'))::uuid,
'Синтетический чат ' || chat, :epoch
FROM generate_series(1, 21) AS chat
ON CONFLICT DO NOTHING;
INSERT INTO chat_members (chat_id, user_id, position, chat_created_at)
SELECT id, :author, 1, created_at FROM chats WHERE title LIKE 'Синтетический чат %'
ON CONFLICT DO NOTHING;
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
SELECT ('00000000-0000-7000-8001-' || lpad(to_hex(n), 12, '0'))::uuid, :big, :author,
'синтетическое сообщение ' || n, :epoch::timestamptz + n * interval '1 second'
FROM generate_series(1, 20000) AS n
ON CONFLICT DO NOTHING;
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
SELECT ('00000000-0000-7000-8002-' || lpad(to_hex(chat * 1000 + n), 12, '0'))::uuid,
('00000000-0000-7000-8000-00000000c' || lpad(to_hex(chat), 3, '0'))::uuid, :author,
'фоновое сообщение ' || n, :epoch::timestamptz + n * interval '1 second'
FROM generate_series(2, 21) AS chat, generate_series(1, 500) AS n
ON CONFLICT DO NOTHING;
UPDATE chats
SET last_message_at = (SELECT max(created_at) FROM messages WHERE messages.chat_id = chats.id)
WHERE title LIKE 'Синтетический чат %';
ANALYZE;
SELECT count(*) FILTER (WHERE chat_id = :big) AS big_chat_messages, count(*) AS all_messages
FROM messages;
+9
View File
@@ -0,0 +1,9 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
cd "$(dirname "$0")/.."
ids=$(docker compose ps -q "${API_SERVICE:-api}")
[ -n "$ids" ] || { echo "no running ${API_SERVICE:-api} replicas" >&2; exit 3; }
target=$(docker inspect -f '{{.Id}} {{.Config.Hostname}}' $ids | awk -v t="$TARGET_INSTANCE_ID" '$2 == t {print $1}')
[ -n "$target" ] || { echo "no running ${API_SERVICE:-api} replica answers as $TARGET_INSTANCE_ID" >&2; exit 3; }
docker stop -t "${STOP_TIMEOUT:-6}" "$target" >/dev/null
+117
View File
@@ -0,0 +1,117 @@
#!/usr/bin/env python3
import argparse
import json
import os
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
seed = __import__("lab4_seed")
FORMAT = "{:>5} {:>5} {:>4} {:<18} {:>7} {:>7} {:>8} {:>8} {:>7} {:>8} {:>6}"
COLUMNS = FORMAT.format(
"slots",
"size",
"conc",
"codes",
"p50_ms",
"p95_ms",
"api_idle",
"api_peak",
"api_oom",
"lb_peak",
"lb_oom",
)
OVERRIDES = ("UPLOAD_CONCURRENCY", "MALLOC_MMAP_THRESHOLD_")
def recreate(slots):
env = dict(os.environ, UPLOAD_CONCURRENCY=str(slots))
if not slots:
env = {k: v for k, v in os.environ.items() if k not in OVERRIDES}
cmd = ["docker", "compose", "up", "-d", "--no-deps", "--force-recreate", "--wait", "api", "lb"]
subprocess.run(cmd, cwd=ROOT, env=env, check=True, capture_output=True)
subprocess.run([str(ROOT / "scripts/lb-wait.sh")], cwd=ROOT, check=True)
def balancer():
cid = seed.run(["docker", "compose", "ps", "-q", "lb"]).strip()
events = dict(row.split() for row in seed.cgroup(cid, "memory.events").splitlines())
peak = seed.cgroup(cid, "memory.peak")
return {
"cgroup_memory_peak_mib": round(int(peak) / 1048576, 1),
"oom_kill": int(events["oom_kill"]),
}
def line(report):
memory = report["memory"].values()
codes = ",".join(f"{k}x{v}" for k, v in sorted(report["status_codes"].items()))
return FORMAT.format(
report["upload_concurrency"],
f"{report['body_bytes'] / 1048576:.1f}M",
report["concurrency"],
codes,
report["latency_ms"]["p50"],
report["latency_ms"]["p95"],
max(m["idle_mib"] for m in memory),
max(m["cgroup_memory_peak_mib"] or 0 for m in memory),
sum(m["oom_kill"] for m in memory),
report["lb"]["cgroup_memory_peak_mib"],
report["lb"]["oom_kill"],
)
def main():
p = argparse.ArgumentParser(description="Peak memory of API replicas under concurrent uploads")
p.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080"))
p.add_argument("--slots", type=int, nargs="+", default=[64, 3])
p.add_argument("--sizes", type=int, nargs="+", default=[1 << 20, 10 << 20])
p.add_argument("--levels", type=int, nargs="+", default=[10, 20, 40])
p.add_argument("--rounds", type=int, default=2, help="uploads per concurrent client")
p.add_argument("--out", default="evidence/upload-memory")
args = p.parse_args()
out = ROOT / args.out
out.mkdir(parents=True, exist_ok=True)
commit = seed.run(["git", "rev-parse", "--short", "HEAD"]).strip()
threshold = os.environ.get("MALLOC_MMAP_THRESHOLD_", "131072")
lines = [
f"# {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} commit {commit}; api x2: 1 CPU,"
f" 512 MiB, 2 workers; MALLOC_MMAP_THRESHOLD_={threshold}; {args.rounds} uploads"
" per client; peaks from cgroup memory.peak, MiB",
COLUMNS,
]
print("\n".join(lines), flush=True)
try:
for slots in args.slots:
for size in args.sizes:
for level in args.levels:
recreate(slots)
run = argparse.Namespace(
base_url=args.base_url,
timeout=60,
size=size,
count=level * args.rounds,
concurrency=level,
sample=True,
service="api",
link=0,
ids_out=None,
)
report = {"upload_concurrency": slots, **seed.over_http(run)}
report["lb"] = balancer()
name = f"slots{slots}-{size >> 20}mib-c{level}.json"
(out / name).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
lines.append(line(report))
print(lines[-1], flush=True)
finally:
recreate(None)
(out / "summary.txt").write_text("\n".join(lines) + "\n")
if __name__ == "__main__":
main()
View File
@@ -0,0 +1,5 @@
from messenger.shared.application.messagebus import CommandBus
class AttachmentsCommandBus(CommandBus):
__slots__ = ()
@@ -0,0 +1,21 @@
from collections.abc import AsyncIterator
from dataclasses import dataclass
from datetime import timedelta
from uuid import UUID
from messenger.shared.application.commands import Command
@dataclass(frozen=True, slots=True)
class UploadAttachment(Command):
owner_id: UUID
chat_id: UUID
filename: str | None
content_type: str | None
body: AsyncIterator[bytes]
@dataclass(frozen=True, slots=True)
class SweepAbandonedUploads(Command):
grace: timedelta
limit: int
@@ -0,0 +1,2 @@
class ObjectMissingError(Exception):
pass
@@ -0,0 +1,150 @@
import asyncio
from collections.abc import AsyncIterator
from hashlib import sha256
from messenger.attachments.application.commands import SweepAbandonedUploads, UploadAttachment
from messenger.attachments.application.ports import (
AttachmentsUnitOfWork,
ChatMembership,
ImageInspector,
ObjectStorage,
)
from messenger.attachments.domain.errors import ChatNotFound, PayloadTooLarge
from messenger.attachments.domain.models import Attachment, AttachmentStatus
from messenger.attachments.domain.values import Filename, ImageKind
from messenger.shared.application.ports import Clock, IdGenerator, Logger
class UploadSlots(asyncio.Semaphore):
pass
async def read_limited(body: AsyncIterator[bytes], limit: int) -> tuple[bytes, str]:
chunks: list[bytes] = []
digest = sha256()
size = 0
async for chunk in body:
size += len(chunk)
if size > limit:
raise PayloadTooLarge(f"тело больше {limit} байт")
digest.update(chunk)
chunks.append(chunk)
return b"".join(chunks), digest.hexdigest()
async def drop(
uow: AttachmentsUnitOfWork, storage: ObjectStorage, attachment: Attachment, reason: str
) -> None:
await storage.delete(attachment.object_key)
attachment.discard(reason)
await uow.attachments.remove(attachment)
await uow.commit()
class UploadAttachmentHandler:
__slots__ = (
"_clock",
"_ids",
"_images",
"_limit",
"_logger",
"_membership",
"_slots",
"_storage",
"_uow",
)
def __init__(
self,
uow: AttachmentsUnitOfWork,
membership: ChatMembership,
images: ImageInspector,
storage: ObjectStorage,
*,
clock: Clock,
ids: IdGenerator,
limit: int,
slots: UploadSlots,
logger: Logger,
) -> None:
self._uow = uow
self._membership = membership
self._images = images
self._storage = storage
self._clock = clock
self._ids = ids
self._limit = limit
self._slots = slots
self._logger = logger
async def __call__(self, command: UploadAttachment) -> Attachment:
if not await self._membership.is_member(command.chat_id, command.owner_id):
raise ChatNotFound("чат не найден")
filename = Filename(command.filename or "")
declared = ImageKind.declared(command.content_type)
async with self._slots:
attachment = await self._write(command, filename, declared)
attachment.store(self._clock.now())
try:
async with self._uow.transaction():
await self._uow.attachments.save(attachment)
await self._uow.commit()
except Exception:
await self._compensate(attachment)
raise
return attachment
async def _write(
self, command: UploadAttachment, filename: Filename, declared: ImageKind
) -> Attachment:
data, digest = await read_limited(command.body, self._limit)
image = await self._images.inspect(data, declared)
attachment = Attachment.stage(
attachment_id=self._ids.new_id(),
chat_id=command.chat_id,
owner_id=command.owner_id,
filename=filename,
image=image,
size_bytes=len(data),
sha256=digest,
now=self._clock.now(),
)
async with self._uow.transaction():
await self._uow.attachments.add(attachment)
await self._uow.commit()
await self._storage.put(attachment.object_key, data, attachment.content_type)
return attachment
async def _compensate(self, staged: Attachment) -> None:
try:
async with self._uow.transaction():
current = await self._uow.attachments.lock(staged.id)
if current is None:
await self._storage.delete(staged.object_key)
elif current.status is AttachmentStatus.PENDING:
await drop(self._uow, self._storage, current, "compensation")
except Exception as error:
self._logger.warning(
"attachment.compensation_failed",
attachment_id=str(staged.id),
error=type(error).__name__,
)
class SweepAbandonedUploadsHandler:
__slots__ = ("_clock", "_storage", "_uow")
def __init__(self, uow: AttachmentsUnitOfWork, storage: ObjectStorage, clock: Clock) -> None:
self._uow = uow
self._storage = storage
self._clock = clock
async def __call__(self, command: SweepAbandonedUploads) -> int:
before = self._clock.now() - command.grace
for swept in range(command.limit):
async with self._uow.transaction():
abandoned = await self._uow.attachments.next_abandoned(before)
if abandoned is None:
return swept
await drop(self._uow, self._storage, abandoned, "abandoned")
return command.limit
@@ -0,0 +1,58 @@
from collections.abc import AsyncIterator
from dataclasses import dataclass
from datetime import datetime
from typing import Protocol, runtime_checkable
from uuid import UUID
from messenger.attachments.domain.models import Attachment
from messenger.attachments.domain.values import ImageInfo, ImageKind
from messenger.shared.application.ports import UnitOfWork
@dataclass(frozen=True, slots=True)
class ObjectContent:
size: int
chunks: AsyncIterator[bytes]
@runtime_checkable
class ObjectStorage(Protocol):
async def put(self, key: str, data: bytes, content_type: str) -> None: ...
async def open(self, key: str) -> ObjectContent: ...
async def delete(self, key: str) -> None: ...
@runtime_checkable
class ImageInspector(Protocol):
async def inspect(self, data: bytes, declared: ImageKind) -> ImageInfo: ...
@runtime_checkable
class ChatMembership(Protocol):
async def is_member(self, chat_id: UUID, user_id: UUID) -> bool: ...
@runtime_checkable
class AttachmentRepository(Protocol):
async def add(self, attachment: Attachment) -> None: ...
async def save(self, attachment: Attachment) -> None: ...
async def lock(self, attachment_id: UUID) -> Attachment | None: ...
async def next_abandoned(self, before: datetime) -> Attachment | None: ...
async def remove(self, attachment: Attachment) -> None: ...
@runtime_checkable
class AttachmentsUnitOfWork(UnitOfWork, Protocol):
@property
def attachments(self) -> AttachmentRepository: ...
@runtime_checkable
class AttachmentDirectory(Protocol):
async def ready(self, attachment_id: UUID) -> Attachment | None: ...
@@ -0,0 +1,46 @@
from uuid import UUID
from messenger.attachments.application.errors import ObjectMissingError
from messenger.attachments.application.ports import (
AttachmentDirectory,
ChatMembership,
ObjectContent,
ObjectStorage,
)
from messenger.attachments.domain.errors import AttachmentNotFound
from messenger.attachments.domain.models import Attachment
from messenger.shared.application.ports import Logger
class GetAttachment:
__slots__ = ("_attachments", "_membership")
def __init__(self, attachments: AttachmentDirectory, membership: ChatMembership) -> None:
self._attachments = attachments
self._membership = membership
async def __call__(self, attachment_id: UUID, user_id: UUID) -> Attachment:
attachment = await self._attachments.ready(attachment_id)
if attachment is None or not await self._membership.is_member(attachment.chat_id, user_id):
raise AttachmentNotFound("вложение не найдено")
return attachment
class OpenAttachment:
__slots__ = ("_get", "_logger", "_storage")
def __init__(self, get: GetAttachment, storage: ObjectStorage, logger: Logger) -> None:
self._get = get
self._storage = storage
self._logger = logger
async def __call__(
self, attachment_id: UUID, user_id: UUID
) -> tuple[Attachment, ObjectContent]:
attachment = await self._get(attachment_id, user_id)
try:
content = await self._storage.open(attachment.object_key)
except ObjectMissingError as error:
self._logger.warning("attachment.object_missing", attachment_id=str(attachment.id))
raise AttachmentNotFound("вложение не найдено") from error
return attachment, content
+24
View File
@@ -0,0 +1,24 @@
from pydantic import Field, SecretStr
from messenger.shared.config import Settings
UPLOAD_BYTES_MAX = 10_485_760
IMAGE_PIXELS_MAX = 40_000_000
class AttachmentsSettings(Settings):
s3_endpoint: str = "http://s3:9000"
s3_region: str = "us-east-1"
s3_bucket: str = Field(default="messenger-attachments", min_length=3)
s3_access_key_id: SecretStr = Field(min_length=3)
s3_secret_access_key: SecretStr = Field(min_length=8)
s3_connect_timeout_seconds: float = Field(default=1.0, gt=0, le=30)
s3_read_timeout_seconds: float = Field(default=2.0, gt=0, le=60)
s3_max_attempts: int = Field(default=2, ge=1, le=5)
max_upload_bytes: int = Field(default=UPLOAD_BYTES_MAX, ge=1, le=UPLOAD_BYTES_MAX)
max_image_pixels: int = Field(default=IMAGE_PIXELS_MAX, ge=1, le=IMAGE_PIXELS_MAX)
image_check_concurrency: int = Field(default=2, ge=1, le=16)
upload_concurrency: int = Field(default=3, ge=1, le=64)
upload_grace_seconds: int = Field(default=900, ge=60)
upload_sweep_interval_seconds: float = Field(default=30.0, gt=0)
upload_sweep_batch: int = Field(default=100, ge=1, le=10_000)
@@ -0,0 +1,29 @@
from messenger.shared.domain.errors import DomainError
class AttachmentError(DomainError):
pass
class InvalidFilename(AttachmentError):
pass
class UnsupportedImage(AttachmentError):
pass
class PayloadTooLarge(AttachmentError):
pass
class ChatNotFound(AttachmentError):
pass
class AttachmentNotFound(AttachmentError):
pass
class UploadAborted(AttachmentError):
pass
@@ -0,0 +1,21 @@
from dataclasses import dataclass, field
from uuid import UUID
from messenger.shared.domain.events import DomainEvent
@dataclass(frozen=True, slots=True)
class AttachmentStored(DomainEvent):
attachment_id: UUID
chat_id: UUID
owner_id: UUID
size_bytes: int
name: str = field(init=False, default="attachment.stored")
@dataclass(frozen=True, slots=True)
class AttachmentDiscarded(DomainEvent):
attachment_id: UUID
chat_id: UUID
reason: str
name: str = field(init=False, default="attachment.discarded")
@@ -0,0 +1,83 @@
from dataclasses import dataclass, field
from datetime import datetime
from enum import StrEnum
from uuid import UUID
from messenger.attachments.domain.events import AttachmentDiscarded, AttachmentStored
from messenger.attachments.domain.values import Filename, ImageInfo
from messenger.shared.domain.events import DomainEvent
OBJECT_PREFIX = "attachments/"
class AttachmentStatus(StrEnum):
PENDING = "pending"
READY = "ready"
@dataclass(slots=True)
class Attachment:
id: UUID
chat_id: UUID
owner_id: UUID
filename: Filename
image: ImageInfo
size_bytes: int
sha256: str
object_key: str
status: AttachmentStatus
created_at: datetime
updated_at: datetime
_events: list[DomainEvent] = field(init=False, default_factory=list, repr=False, compare=False)
@classmethod
def stage(
cls,
*,
attachment_id: UUID,
chat_id: UUID,
owner_id: UUID,
filename: Filename,
image: ImageInfo,
size_bytes: int,
sha256: str,
now: datetime,
) -> Attachment:
return cls(
id=attachment_id,
chat_id=chat_id,
owner_id=owner_id,
filename=filename,
image=image,
size_bytes=size_bytes,
sha256=sha256,
object_key=f"{OBJECT_PREFIX}{attachment_id}",
status=AttachmentStatus.PENDING,
created_at=now,
updated_at=now,
)
@property
def content_type(self) -> str:
return self.image.kind.value
def store(self, now: datetime) -> None:
self.status = AttachmentStatus.READY
self.updated_at = max(now, self.created_at)
self._events.append(
AttachmentStored(
attachment_id=self.id,
chat_id=self.chat_id,
owner_id=self.owner_id,
size_bytes=self.size_bytes,
)
)
def discard(self, reason: str) -> None:
self._events.append(
AttachmentDiscarded(attachment_id=self.id, chat_id=self.chat_id, reason=reason)
)
def collect_events(self) -> list[DomainEvent]:
events, self._events = self._events, []
return events
@@ -0,0 +1,46 @@
import unicodedata
from dataclasses import dataclass
from enum import StrEnum
from messenger.attachments.domain.errors import InvalidFilename, UnsupportedImage
FILENAME_MAX = 255
SEPARATORS = frozenset("/\\")
@dataclass(frozen=True, slots=True)
class Filename:
value: str
def __post_init__(self) -> None:
if not self.value.strip():
raise InvalidFilename("параметр filename обязателен и не может быть пустым")
if len(self.value) > FILENAME_MAX:
raise InvalidFilename(f"параметр filename длиннее {FILENAME_MAX} символов Unicode")
if any(c in SEPARATORS or unicodedata.category(c) == "Cc" for c in self.value):
raise InvalidFilename(
"параметр filename содержит разделитель пути или управляющий символ"
)
def __str__(self) -> str:
return self.value
class ImageKind(StrEnum):
PNG = "image/png"
JPEG = "image/jpeg"
@classmethod
def declared(cls, content_type: str | None) -> ImageKind:
media = (content_type or "").partition(";")[0].strip().lower()
try:
return cls(media)
except ValueError as error:
raise UnsupportedImage("Content-Type должен быть image/png или image/jpeg") from error
@dataclass(frozen=True, slots=True)
class ImageInfo:
kind: ImageKind
width: int
height: int
@@ -0,0 +1,18 @@
from messenger.attachments.domain.errors import (
AttachmentNotFound,
ChatNotFound,
InvalidFilename,
PayloadTooLarge,
UnsupportedImage,
UploadAborted,
)
from messenger.shared.domain.errors import DomainError
ERROR_CODES: dict[type[DomainError], tuple[str, int]] = {
InvalidFilename: ("invalid_request", 400),
UploadAborted: ("invalid_request", 400),
ChatNotFound: ("not_found", 404),
AttachmentNotFound: ("not_found", 404),
PayloadTooLarge: ("payload_too_large", 413),
UnsupportedImage: ("unsupported_media_type", 415),
}
@@ -0,0 +1,77 @@
import re
from collections.abc import AsyncIterator
from typing import Annotated, Literal
from uuid import UUID
from dishka.integrations.fastapi import FromDishka, inject
from fastapi import APIRouter, Query, Request, status
from fastapi.responses import StreamingResponse
from starlette.requests import ClientDisconnect
from messenger.attachments.application.bus import AttachmentsCommandBus
from messenger.attachments.application.commands import UploadAttachment
from messenger.attachments.application.queries import GetAttachment, OpenAttachment
from messenger.attachments.domain.errors import UploadAborted
from messenger.attachments.entrypoints.schemas import AttachmentOut
from messenger.shared.entrypoints.actor import Actor
UPLOAD_PATH = re.compile(r"/api/v1/chats/[^/]+/attachments/?")
CONTENT_HEADERS = {"X-Content-Type-Options": "nosniff", "Cache-Control": "private, no-store"}
uploads_router = APIRouter(prefix="/api/v1/chats/{chat_id}/attachments", tags=["attachments"])
attachments_router = APIRouter(prefix="/api/v1/attachments", tags=["attachments"])
async def body_of(request: Request) -> AsyncIterator[bytes]:
try:
async for chunk in request.stream():
yield chunk
except ClientDisconnect as error:
raise UploadAborted("клиент оборвал загрузку до конца тела") from error
@uploads_router.post("", status_code=status.HTTP_201_CREATED)
@inject
async def upload_attachment(
chat_id: UUID,
actor: Actor,
request: Request,
bus: FromDishka[AttachmentsCommandBus],
filename: Annotated[str | None, Query()] = None,
) -> AttachmentOut:
attachment = await bus.execute(
UploadAttachment(
owner_id=actor,
chat_id=chat_id,
filename=filename,
content_type=request.headers.get("content-type"),
body=body_of(request),
)
)
return AttachmentOut.of(attachment)
@attachments_router.get("/{attachment_id}")
@inject
async def read_attachment(
attachment_id: UUID,
actor: Actor,
query: FromDishka[GetAttachment],
) -> AttachmentOut:
return AttachmentOut.of(await query(attachment_id, actor))
@attachments_router.get("/{attachment_id}/content")
@inject
async def download_attachment(
attachment_id: UUID,
actor: Actor,
query: FromDishka[OpenAttachment],
_variant: Annotated[Literal["original"], Query(alias="variant")] = "original",
) -> StreamingResponse:
attachment, content = await query(attachment_id, actor)
return StreamingResponse(
content.chunks,
media_type=attachment.content_type,
headers={**CONTENT_HEADERS, "Content-Length": str(content.size)},
)
@@ -0,0 +1,112 @@
from collections.abc import AsyncIterator
from dishka import Provider, Scope, provide
from messenger.attachments.application.bus import AttachmentsCommandBus
from messenger.attachments.application.commands import SweepAbandonedUploads, UploadAttachment
from messenger.attachments.application.handlers import (
SweepAbandonedUploadsHandler,
UploadAttachmentHandler,
UploadSlots,
)
from messenger.attachments.application.ports import (
AttachmentDirectory,
AttachmentsUnitOfWork,
ChatMembership,
ImageInspector,
ObjectStorage,
)
from messenger.attachments.application.queries import GetAttachment, OpenAttachment
from messenger.attachments.config import AttachmentsSettings
from messenger.attachments.infrastructure.images import PillowImageInspector
from messenger.attachments.infrastructure.postgres import (
PostgresAttachmentDirectory,
PostgresAttachmentsUnitOfWork,
)
from messenger.attachments.infrastructure.s3 import S3ObjectStorage, s3_client
from messenger.shared.application.commands import Command
from messenger.shared.application.messagebus import CommandHandler
from messenger.shared.application.ports import Clock, EventPublisher, IdGenerator, Logger
from messenger.shared.infrastructure.postgres import Database
class AttachmentsProvider(Provider):
def __init__(self, settings: AttachmentsSettings) -> None:
super().__init__()
self._settings = settings
@provide(scope=Scope.APP)
def settings(self) -> AttachmentsSettings:
return self._settings
@provide(scope=Scope.APP)
async def storage(self, settings: AttachmentsSettings) -> AsyncIterator[ObjectStorage]:
async with s3_client(
endpoint=settings.s3_endpoint,
region=settings.s3_region,
access_key_id=settings.s3_access_key_id.get_secret_value(),
secret_access_key=settings.s3_secret_access_key.get_secret_value(),
connect_timeout=settings.s3_connect_timeout_seconds,
read_timeout=settings.s3_read_timeout_seconds,
max_attempts=settings.s3_max_attempts,
) as client:
yield S3ObjectStorage(client, settings.s3_bucket)
@provide(scope=Scope.APP)
def images(self, settings: AttachmentsSettings) -> ImageInspector:
return PillowImageInspector(settings.max_image_pixels, settings.image_check_concurrency)
@provide(scope=Scope.APP)
def slots(self, settings: AttachmentsSettings) -> UploadSlots:
return UploadSlots(settings.upload_concurrency)
@provide(scope=Scope.APP)
def directory(self, database: Database) -> AttachmentDirectory:
return PostgresAttachmentDirectory(database)
@provide(scope=Scope.REQUEST)
def unit_of_work(self, database: Database) -> AttachmentsUnitOfWork:
return PostgresAttachmentsUnitOfWork(database)
@provide(scope=Scope.REQUEST)
def attachment(
self, directory: AttachmentDirectory, membership: ChatMembership
) -> GetAttachment:
return GetAttachment(directory, membership)
@provide(scope=Scope.REQUEST)
def content(
self, attachment: GetAttachment, storage: ObjectStorage, logger: Logger
) -> OpenAttachment:
return OpenAttachment(attachment, storage, logger)
@provide(scope=Scope.REQUEST)
def command_bus(
self,
uow: AttachmentsUnitOfWork,
membership: ChatMembership,
images: ImageInspector,
storage: ObjectStorage,
publisher: EventPublisher,
clock: Clock,
ids: IdGenerator,
settings: AttachmentsSettings,
slots: UploadSlots,
logger: Logger,
) -> AttachmentsCommandBus:
upload = UploadAttachmentHandler(
uow,
membership,
images,
storage,
clock=clock,
ids=ids,
limit=settings.max_upload_bytes,
slots=slots,
logger=logger,
)
handlers: dict[type[Command], CommandHandler] = {
UploadAttachment: upload,
SweepAbandonedUploads: SweepAbandonedUploadsHandler(uow, storage, clock),
}
return AttachmentsCommandBus(handlers, uow, publisher)
@@ -0,0 +1,28 @@
from messenger.attachments.domain.events import AttachmentDiscarded, AttachmentStored
from messenger.shared.application.ports import Logger
LOG_FIELDS = frozenset({"attachment_id", "chat_id", "actor_id", "size_bytes", "count"})
class AttachmentsJournal:
__slots__ = ("_logger",)
def __init__(self, logger: Logger) -> None:
self._logger = logger
async def on_attachment_stored(self, event: AttachmentStored) -> None:
self._logger.info(
"audit.attachment_stored",
attachment_id=str(event.attachment_id),
chat_id=str(event.chat_id),
actor_id=str(event.owner_id),
size_bytes=event.size_bytes,
)
async def on_attachment_discarded(self, event: AttachmentDiscarded) -> None:
self._logger.info(
"audit.attachment_discarded",
attachment_id=str(event.attachment_id),
chat_id=str(event.chat_id),
reason=event.reason,
)
@@ -0,0 +1,36 @@
from datetime import datetime
from typing import Literal
from uuid import UUID
from pydantic import BaseModel
from messenger.attachments.domain.models import Attachment
class AttachmentOut(BaseModel):
id: UUID
chat_id: UUID
owner_id: UUID
filename: str
content_type: str
size_bytes: int
sha256: str
status: Literal["ready"] = "ready"
variants: tuple[()] = ()
error_code: None = None
created_at: datetime
updated_at: datetime
@classmethod
def of(cls, attachment: Attachment) -> AttachmentOut:
return cls(
id=attachment.id,
chat_id=attachment.chat_id,
owner_id=attachment.owner_id,
filename=attachment.filename.value,
content_type=attachment.content_type,
size_bytes=attachment.size_bytes,
sha256=attachment.sha256,
created_at=attachment.created_at,
updated_at=attachment.updated_at,
)
@@ -0,0 +1,53 @@
import asyncio
from contextlib import suppress
from datetime import timedelta
from dishka import AsyncContainer
from messenger.attachments.application.bus import AttachmentsCommandBus
from messenger.attachments.application.commands import SweepAbandonedUploads
from messenger.attachments.config import AttachmentsSettings
from messenger.shared.application.ports import Logger
class UploadSweeper:
__slots__ = ("_command", "_container", "_interval", "_logger", "_task")
def __init__(
self, container: AsyncContainer, settings: AttachmentsSettings, logger: Logger
) -> None:
self._container = container
self._logger = logger
self._interval = settings.upload_sweep_interval_seconds
self._command = SweepAbandonedUploads(
grace=timedelta(seconds=settings.upload_grace_seconds),
limit=settings.upload_sweep_batch,
)
self._task: asyncio.Task[None] | None = None
def start(self) -> None:
self._task = asyncio.create_task(self._run())
async def stop(self) -> None:
if self._task is None:
return
self._task.cancel()
with suppress(asyncio.CancelledError):
await self._task
self._task = None
async def sweep(self) -> int:
async with self._container() as scope:
bus = await scope.get(AttachmentsCommandBus)
return await bus.execute(self._command)
async def _run(self) -> None:
while True:
await asyncio.sleep(self._interval)
try:
swept = await self.sweep()
except Exception as error:
self._logger.warning("attachments.sweep_failed", error=type(error).__name__)
continue
if swept:
self._logger.info("attachments.swept", count=swept)
@@ -0,0 +1,49 @@
import asyncio
import warnings
from io import BytesIO
from PIL import Image
from messenger.attachments.domain.errors import PayloadTooLarge, UnsupportedImage
from messenger.attachments.domain.values import ImageInfo, ImageKind
KINDS = {"PNG": ImageKind.PNG, "JPEG": ImageKind.JPEG}
BOMBS = (Image.DecompressionBombError, Image.DecompressionBombWarning)
JPEG_SCALE = 8
class PillowImageInspector:
__slots__ = ("_max_pixels", "_slots")
def __init__(self, max_pixels: int, concurrency: int) -> None:
self._max_pixels = max_pixels
self._slots = asyncio.Semaphore(concurrency)
warnings.simplefilter("error", Image.DecompressionBombWarning)
async def inspect(self, data: bytes, declared: ImageKind) -> ImageInfo:
async with self._slots:
return await asyncio.to_thread(self._inspect, data, declared)
def _inspect(self, data: bytes, declared: ImageKind) -> ImageInfo:
try:
image = Image.open(BytesIO(data), formats=tuple(KINDS))
except BOMBS as error:
raise PayloadTooLarge("изображение больше допустимого числа пикселей") from error
except Exception as error:
raise UnsupportedImage("содержимое не является изображением PNG или JPEG") from error
with image:
kind = KINDS[str(image.format)]
width, height = image.size
if kind is not declared:
raise UnsupportedImage("содержимое не совпадает с заявленным Content-Type")
if width * height > self._max_pixels:
raise PayloadTooLarge("изображение больше допустимого числа пикселей")
try:
if kind is ImageKind.JPEG:
image.draft("RGB", (width // JPEG_SCALE or 1, height // JPEG_SCALE or 1))
image.load()
else:
image.verify()
except Exception as error:
raise UnsupportedImage("изображение повреждено или обрезано") from error
return ImageInfo(kind, width, height)
@@ -0,0 +1,121 @@
from datetime import datetime
from uuid import UUID
from asyncpg import ForeignKeyViolationError, Record
from messenger.attachments.domain.errors import ChatNotFound
from messenger.attachments.domain.models import Attachment, AttachmentStatus
from messenger.attachments.domain.values import Filename, ImageInfo, ImageKind
from messenger.shared.application.errors import Unavailable
from messenger.shared.infrastructure.postgres import (
Connection,
Database,
EventSource,
PostgresUnitOfWork,
)
INSERT = """
INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
width, height, object_key, status, created_at, updated_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
"""
MARK = "UPDATE attachments SET status = $2, updated_at = $3 WHERE id = $1 AND status = 'pending'"
COLUMNS = """
SELECT id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height,
object_key, status, created_at, updated_at
FROM attachments
"""
READY = COLUMNS + "WHERE id = $1 AND status = 'ready'"
LOCK = COLUMNS + "WHERE id = $1 FOR UPDATE"
ABANDONED = (
COLUMNS
+ """WHERE status = 'pending' AND created_at < $1
ORDER BY created_at LIMIT 1 FOR UPDATE SKIP LOCKED"""
)
REMOVE = "DELETE FROM attachments WHERE id = $1 AND status = 'pending'"
def attachment_of(row: Record) -> Attachment:
return Attachment(
id=row["id"],
chat_id=row["chat_id"],
owner_id=row["owner_id"],
filename=Filename(row["filename"]),
image=ImageInfo(ImageKind(row["content_type"]), row["width"], row["height"]),
size_bytes=row["size_bytes"],
sha256=row["sha256"],
object_key=row["object_key"],
status=AttachmentStatus(row["status"]),
created_at=row["created_at"],
updated_at=row["updated_at"],
)
class PostgresAttachmentRepository:
__slots__ = ("_connection", "_seen")
def __init__(self, connection: Connection, seen: list[EventSource]) -> None:
self._connection = connection
self._seen = seen
async def add(self, attachment: Attachment) -> None:
try:
await self._connection.execute(
INSERT,
attachment.id,
attachment.chat_id,
attachment.owner_id,
attachment.filename.value,
attachment.content_type,
attachment.size_bytes,
attachment.sha256,
attachment.image.width,
attachment.image.height,
attachment.object_key,
attachment.status.value,
attachment.created_at,
attachment.updated_at,
)
except ForeignKeyViolationError as error:
raise ChatNotFound("чат не найден") from error
self._seen.append(attachment)
async def save(self, attachment: Attachment) -> None:
done = await self._connection.execute(
MARK, attachment.id, attachment.status.value, attachment.updated_at
)
if done == "UPDATE 0":
raise Unavailable("намерение загрузки уже снято уборкой")
self._seen.append(attachment)
async def lock(self, attachment_id: UUID) -> Attachment | None:
row = await self._connection.fetchrow(LOCK, attachment_id)
return None if row is None else attachment_of(row)
async def next_abandoned(self, before: datetime) -> Attachment | None:
row = await self._connection.fetchrow(ABANDONED, before)
return None if row is None else attachment_of(row)
async def remove(self, attachment: Attachment) -> None:
await self._connection.execute(REMOVE, attachment.id)
self._seen.append(attachment)
class PostgresAttachmentsUnitOfWork(PostgresUnitOfWork):
__slots__ = ()
@property
def attachments(self) -> PostgresAttachmentRepository:
return PostgresAttachmentRepository(self.connection, self.seen)
class PostgresAttachmentDirectory:
__slots__ = ("_database",)
def __init__(self, database: Database) -> None:
self._database = database
async def ready(self, attachment_id: UUID) -> Attachment | None:
async with self._database.connection() as connection:
row = await connection.fetchrow(READY, attachment_id)
return None if row is None else attachment_of(row)
@@ -0,0 +1,89 @@
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from typing import Any
from aiobotocore.session import get_session
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
from botocore.handlers import add_expect_header
from messenger.attachments.application.errors import ObjectMissingError
from messenger.attachments.application.ports import ObjectContent
from messenger.shared.application.errors import Unavailable
CHUNK_BYTES = 64 * 1024
MISSING = frozenset({"NoSuchKey", "404", "NotFound"})
FAILURES = (BotoCoreError, ClientError, OSError, TimeoutError)
def missing(error: Exception) -> bool:
return isinstance(error, ClientError) and error.response.get("Error", {}).get("Code") in MISSING
@asynccontextmanager
async def s3_client(
*,
endpoint: str,
region: str,
access_key_id: str,
secret_access_key: str,
connect_timeout: float,
read_timeout: float,
max_attempts: int,
) -> AsyncIterator[Any]:
config = Config(
region_name=region,
signature_version="s3v4",
s3={"addressing_style": "path"},
connect_timeout=connect_timeout,
read_timeout=read_timeout,
retries={"mode": "standard", "max_attempts": max_attempts},
max_pool_connections=32,
request_checksum_calculation="when_required",
response_checksum_validation="when_required",
)
async with get_session().create_client(
"s3",
endpoint_url=endpoint,
aws_access_key_id=access_key_id,
aws_secret_access_key=secret_access_key,
config=config,
) as client:
client.meta.events.unregister("before-call.s3", add_expect_header)
yield client
class S3ObjectStorage:
__slots__ = ("_bucket", "_client")
def __init__(self, client: Any, bucket: str) -> None:
self._client = client
self._bucket = bucket
async def put(self, key: str, data: bytes, content_type: str) -> None:
try:
await self._client.put_object(
Bucket=self._bucket, Key=key, Body=data, ContentType=content_type
)
except FAILURES as error:
raise Unavailable("S3 недоступно") from error
async def open(self, key: str) -> ObjectContent:
try:
response = await self._client.get_object(Bucket=self._bucket, Key=key)
except FAILURES as error:
if missing(error):
raise ObjectMissingError(key) from error
raise Unavailable("S3 недоступно") from error
return ObjectContent(size=response["ContentLength"], chunks=self._chunks(response["Body"]))
async def delete(self, key: str) -> None:
try:
await self._client.delete_object(Bucket=self._bucket, Key=key)
except FAILURES as error:
raise Unavailable("S3 недоступно") from error
async def _chunks(self, body: Any) -> AsyncIterator[bytes]:
async with body as stream:
async for chunk in stream.iter_chunks(CHUNK_BYTES):
yield chunk
+66
View File
@@ -0,0 +1,66 @@
import asyncio
import logging
import os
import sys
from typing import Any
from gunicorn.arbiter import Arbiter
from gunicorn.glogging import Logger as GunicornBaseLogger
from uvicorn.server import Server
from uvicorn_worker import UvicornWorker
from messenger.main import LOG_FIELDS
from messenger.shared.entrypoints.health import DRAINING
from messenger.shared.infrastructure.logging import configure
LOG_LEVEL = os.environ.get("LOG_LEVEL", "INFO")
SHUTDOWN_SECONDS = int(os.environ.get("SHUTDOWN_TIMEOUT_SECONDS", "5"))
DRAIN_SECONDS = float(os.environ.get("DRAIN_SECONDS", "2"))
class StructlogGunicornLogger(GunicornBaseLogger):
def setup(self, cfg: object) -> None:
configure(LOG_LEVEL, LOG_FIELDS)
def access(self, resp: object, req: object, environ: object, request_time: object) -> None:
return
class DrainingServer(Server):
async def main_loop(self) -> None:
await super().main_loop()
if not self.force_exit:
DRAINING.set()
logging.getLogger("uvicorn.error").info("Draining for %.1f s", DRAIN_SECONDS)
await asyncio.sleep(DRAIN_SECONDS)
class MessengerWorker(UvicornWorker):
CONFIG_KWARGS: dict[str, Any] = {
"access_log": False,
"server_header": False,
"date_header": False,
"timeout_graceful_shutdown": max(1, int(SHUTDOWN_SECONDS - DRAIN_SECONDS)),
}
async def _serve(self) -> None:
self.config.app = self.wsgi
server = DrainingServer(config=self.config)
self._install_sigquit_handler()
await server.serve(sockets=self.sockets)
if not server.started:
sys.exit(Arbiter.WORKER_BOOT_ERROR)
bind = f"{os.environ.get('HOST', '0.0.0.0')}:{os.environ.get('PORT', '8080')}"
workers = int(os.environ.get("WEB_CONCURRENCY", "2"))
worker_class = "messenger.gunicorn_conf.MessengerWorker"
logger_class = StructlogGunicornLogger
loglevel = LOG_LEVEL.lower()
accesslog = None
errorlog = "-"
timeout = int(float(os.environ.get("REQUEST_TIMEOUT_SECONDS", "8")) + 5)
graceful_timeout = SHUTDOWN_SECONDS
keepalive = 5
preload_app = False
control_socket_disable = True
View File
@@ -0,0 +1,5 @@
from messenger.shared.application.messagebus import CommandBus
class IdentityCommandBus(CommandBus):
__slots__ = ()
@@ -0,0 +1,22 @@
from dataclasses import dataclass, field
from uuid import UUID
from messenger.identity.domain.models import Session
from messenger.shared.application.commands import Command
@dataclass(frozen=True, slots=True)
class RegisterUser(Command):
username: str
display_name: str
password: str = field(repr=False)
@dataclass(frozen=True, slots=True)
class OpenSession(Command):
user_id: UUID
@dataclass(frozen=True, slots=True)
class CloseSession(Command):
session: Session
@@ -0,0 +1,76 @@
from datetime import timedelta
from messenger.identity.application.commands import CloseSession, OpenSession, RegisterUser
from messenger.identity.application.ports import (
IdentityUnitOfWork,
PasswordHasher,
SessionDirectory,
)
from messenger.identity.domain.events import SessionClosed, SessionOpened
from messenger.identity.domain.models import Session, User
from messenger.identity.domain.values import DisplayName, Password, Username
from messenger.shared.application.ports import Clock, IdGenerator
class RegisterUserHandler:
__slots__ = ("_clock", "_hasher", "_ids", "_uow")
def __init__(
self, uow: IdentityUnitOfWork, clock: Clock, ids: IdGenerator, hasher: PasswordHasher
) -> None:
self._uow = uow
self._clock = clock
self._ids = ids
self._hasher = hasher
async def __call__(self, command: RegisterUser) -> User:
username = Username(command.username)
display_name = DisplayName(command.display_name)
password = Password(command.password)
password_hash = await self._hasher.hash(password.value)
user, event = User.register(
user_id=self._ids.new_id(),
username=username,
display_name=display_name,
password_hash=password_hash,
registered_at=self._clock.now(),
)
async with self._uow.transaction():
await self._uow.users.add(user)
self._uow.record(event)
await self._uow.commit()
return user
class OpenSessionHandler:
__slots__ = ("_clock", "_ttl", "_uow")
def __init__(self, uow: IdentityUnitOfWork, clock: Clock, ttl: timedelta) -> None:
self._uow = uow
self._clock = clock
self._ttl = ttl
async def __call__(self, command: OpenSession) -> tuple[str, Session]:
token, session = Session.issue(
user_id=command.user_id, issued_at=self._clock.now(), ttl=self._ttl
)
async with self._uow.transaction():
await self._uow.sessions.add(session)
self._uow.record(SessionOpened(user_id=session.user_id, expires_at=session.expires_at))
await self._uow.commit()
return token, session
class CloseSessionHandler:
__slots__ = ("_sessions", "_uow")
def __init__(self, uow: IdentityUnitOfWork, sessions: SessionDirectory) -> None:
self._uow = uow
self._sessions = sessions
async def __call__(self, command: CloseSession) -> None:
await self._sessions.invalidate(command.session)
async with self._uow.transaction():
await self._uow.sessions.remove(command.session.token_hash)
self._uow.record(SessionClosed(user_id=command.session.user_id))
await self._uow.commit()
@@ -0,0 +1,48 @@
from datetime import datetime
from typing import Protocol, runtime_checkable
from uuid import UUID
from messenger.identity.domain.models import Session, User
from messenger.shared.application.ports import UnitOfWork
@runtime_checkable
class UserRepository(Protocol):
async def add(self, user: User) -> None: ...
@runtime_checkable
class SessionRepository(Protocol):
async def add(self, session: Session) -> None: ...
async def remove(self, token_hash: bytes) -> None: ...
@runtime_checkable
class IdentityUnitOfWork(UnitOfWork, Protocol):
@property
def users(self) -> UserRepository: ...
@property
def sessions(self) -> SessionRepository: ...
@runtime_checkable
class UserDirectory(Protocol):
async def find(self, user_id: UUID) -> User | None: ...
async def find_by_username(self, username: str) -> User | None: ...
@runtime_checkable
class SessionDirectory(Protocol):
async def find(self, token_hash: bytes, now: datetime) -> Session | None: ...
async def invalidate(self, session: Session) -> None: ...
@runtime_checkable
class PasswordHasher(Protocol):
async def hash(self, password: str) -> str: ...
async def verify(self, password_hash: str | None, password: str) -> bool: ...
@@ -0,0 +1,43 @@
from uuid import UUID
from messenger.identity.application.ports import PasswordHasher, SessionDirectory, UserDirectory
from messenger.identity.domain.errors import ActorNotIdentified
from messenger.identity.domain.models import Session, User, token_digest
from messenger.shared.application.ports import Clock
class GetCurrentUser:
__slots__ = ("_directory",)
def __init__(self, directory: UserDirectory) -> None:
self._directory = directory
async def __call__(self, user_id: UUID) -> User:
user = await self._directory.find(user_id)
if user is None:
raise ActorNotIdentified("идентификация не подтверждена")
return user
class VerifyCredentials:
__slots__ = ("_directory", "_hasher")
def __init__(self, directory: UserDirectory, hasher: PasswordHasher) -> None:
self._directory = directory
self._hasher = hasher
async def __call__(self, username: str, password: str) -> User | None:
user = await self._directory.find_by_username(username)
valid = await self._hasher.verify(user.password_hash if user else None, password)
return user if valid else None
class Authenticate:
__slots__ = ("_clock", "_sessions")
def __init__(self, sessions: SessionDirectory, clock: Clock) -> None:
self._sessions = sessions
self._clock = clock
async def __call__(self, token: str) -> Session | None:
return await self._sessions.find(token_digest(token), self._clock.now())
+12
View File
@@ -0,0 +1,12 @@
from pydantic import Field, SecretStr
from messenger.shared.config import Settings
class IdentitySettings(Settings):
session_ttl_seconds: int = Field(default=3600, ge=1, le=30 * 24 * 3600)
password_hash_concurrency: int = Field(default=2, ge=1, le=64)
cache_url: SecretStr | None = None
session_cache_ttl_seconds: int = Field(default=300, ge=1, le=24 * 3600)
cache_timeout_seconds: float = Field(default=0.2, gt=0, le=5)
cache_cooldown_seconds: float = Field(default=1.0, ge=0, le=60)
+17
View File
@@ -0,0 +1,17 @@
from messenger.shared.domain.errors import DomainError
class IdentityError(DomainError):
pass
class InvalidUserData(IdentityError):
pass
class UsernameAlreadyTaken(IdentityError):
pass
class ActorNotIdentified(IdentityError):
pass
+25
View File
@@ -0,0 +1,25 @@
from dataclasses import dataclass, field
from datetime import datetime
from uuid import UUID
from messenger.shared.domain.events import DomainEvent
@dataclass(frozen=True, slots=True)
class UserRegistered(DomainEvent):
user_id: UUID
registered_at: datetime
name: str = field(init=False, default="user.registered")
@dataclass(frozen=True, slots=True)
class SessionOpened(DomainEvent):
user_id: UUID
expires_at: datetime
name: str = field(init=False, default="session.opened")
@dataclass(frozen=True, slots=True)
class SessionClosed(DomainEvent):
user_id: UUID
name: str = field(init=False, default="session.closed")
+62
View File
@@ -0,0 +1,62 @@
import secrets
from dataclasses import dataclass, field
from datetime import datetime, timedelta
from hashlib import sha256
from uuid import UUID
from messenger.identity.domain.events import UserRegistered
from messenger.identity.domain.values import DisplayName, Username
from messenger.shared.domain.events import DomainEvent
TOKEN_BYTES = 32
def token_digest(token: str) -> bytes:
return sha256(token.encode()).digest()
@dataclass(frozen=True, slots=True)
class User:
id: UUID
username: Username
display_name: DisplayName
created_at: datetime
password_hash: str = field(repr=False)
@classmethod
def register(
cls,
*,
user_id: UUID,
username: Username,
display_name: DisplayName,
password_hash: str,
registered_at: datetime,
) -> tuple[User, DomainEvent]:
user = cls(
id=user_id,
username=username,
display_name=display_name,
created_at=registered_at,
password_hash=password_hash,
)
return user, UserRegistered(user_id=user.id, registered_at=registered_at)
@dataclass(frozen=True, slots=True)
class Session:
token_hash: bytes = field(repr=False)
user_id: UUID
created_at: datetime
expires_at: datetime
@classmethod
def issue(cls, *, user_id: UUID, issued_at: datetime, ttl: timedelta) -> tuple[str, Session]:
token = secrets.token_urlsafe(TOKEN_BYTES)
session = cls(
token_hash=token_digest(token),
user_id=user_id,
created_at=issued_at,
expires_at=issued_at + ttl,
)
return token, session
+54
View File
@@ -0,0 +1,54 @@
import re
from dataclasses import dataclass, field
from messenger.identity.domain.errors import InvalidUserData
USERNAME_PATTERN = re.compile(r"\A[a-z0-9_]{3,32}\Z")
DISPLAY_NAME_MAX = 100
PASSWORD_MIN = 12
PASSWORD_MAX = 128
@dataclass(frozen=True, slots=True)
class Username:
value: str
def __post_init__(self) -> None:
if not isinstance(self.value, str):
raise InvalidUserData("поле username должно быть строкой")
if USERNAME_PATTERN.match(self.value) is None:
raise InvalidUserData("поле username должно соответствовать [a-z0-9_]{3,32}")
def __str__(self) -> str:
return self.value
@dataclass(frozen=True, slots=True)
class DisplayName:
value: str
def __post_init__(self) -> None:
if not isinstance(self.value, str):
raise InvalidUserData("поле display_name должно быть строкой")
if not self.value.strip():
raise InvalidUserData("поле display_name не может быть пустым или состоять из пробелов")
if len(self.value) > DISPLAY_NAME_MAX:
raise InvalidUserData(f"поле display_name длиннее {DISPLAY_NAME_MAX} символов Unicode")
def __str__(self) -> str:
return self.value
@dataclass(frozen=True, slots=True)
class Password:
value: str = field(repr=False)
def __post_init__(self) -> None:
if not isinstance(self.value, str):
raise InvalidUserData("поле password должно быть строкой")
if not self.value.strip():
raise InvalidUserData("поле password не может состоять из пробелов")
if not PASSWORD_MIN <= len(self.value) <= PASSWORD_MAX:
raise InvalidUserData(
f"поле password: от {PASSWORD_MIN} до {PASSWORD_MAX} символов Unicode"
)
@@ -0,0 +1,98 @@
import re
from base64 import b64decode
from binascii import Error as Base64Error
from typing import Annotated
from fastapi import Depends, Request
from starlette.types import Scope
from messenger.identity.application.queries import Authenticate, VerifyCredentials
from messenger.identity.domain.models import Session, User
from messenger.shared.entrypoints.errors import Unauthenticated
from messenger.shared.entrypoints.pipeline import Rejection, header
LOGIN_PATH = "/api/v1/auth/sessions"
PROTECTED_PREFIX = "/api/v1/"
PUBLIC_ROUTES = frozenset({("POST", "/api/v1/users")})
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
BEARER_CHALLENGE = 'Bearer realm="messenger"'
AUTHORIZATION = b"authorization"
TOKEN = re.compile(r"[A-Za-z0-9\-._~+/]{1,512}=*")
def normalized(path: str) -> str:
return path.rstrip("/") or "/"
def challenge_for(path: str) -> str:
return BASIC_CHALLENGE if normalized(path) == LOGIN_PATH else BEARER_CHALLENGE
def basic_credentials(value: str) -> tuple[str, str] | None:
scheme, _, encoded = value.partition(" ")
if scheme.lower() != "basic":
return None
try:
decoded = b64decode(encoded.strip(), validate=True).decode("utf-8")
except Base64Error, UnicodeDecodeError:
return None
username, separator, password = decoded.partition(":")
if not separator or not username.isascii():
return None
return username, password
def bearer_token(value: str) -> str | None:
scheme, _, token = value.partition(" ")
token = token.strip()
if scheme.lower() != "bearer" or TOKEN.fullmatch(token) is None:
return None
return token
def rejection(path: str) -> Rejection:
return Rejection(401, "unauthorized", (("www-authenticate", challenge_for(path)),))
async def guard(scope: Scope) -> Rejection | None:
method, path = scope["method"], normalized(scope["path"])
if not path.startswith(PROTECTED_PREFIX) or (method, path) in PUBLIC_ROUTES:
return None
container = scope["app"].state.dishka_container
state = scope.setdefault("state", {})
raw = header(scope, AUTHORIZATION)
if (method, path) == ("POST", LOGIN_PATH):
credentials = basic_credentials(raw)
verify = await container.get(VerifyCredentials)
user = await verify(*credentials) if credentials else None
if user is None:
return rejection(path)
state["user"] = user
state["actor"] = user.id
return None
token = bearer_token(raw)
authenticate = await container.get(Authenticate)
session = await authenticate(token) if token else None
if session is None:
return rejection(path)
state["session"] = session
state["actor"] = session.user_id
return None
async def verified_user(request: Request) -> User:
user = getattr(request.state, "user", None)
if not isinstance(user, User):
raise Unauthenticated("учётные данные не подтверждены")
return user
async def current_session(request: Request) -> Session:
session = getattr(request.state, "session", None)
if not isinstance(session, Session):
raise Unauthenticated("сессия не подтверждена")
return session
type VerifiedUser = Annotated[User, Depends(verified_user)]
type CurrentSession = Annotated[Session, Depends(current_session)]
@@ -0,0 +1,12 @@
from messenger.identity.domain.errors import (
ActorNotIdentified,
InvalidUserData,
UsernameAlreadyTaken,
)
from messenger.shared.domain.errors import DomainError
ERROR_CODES: dict[type[DomainError], tuple[str, int]] = {
InvalidUserData: ("invalid_request", 400),
UsernameAlreadyTaken: ("conflict", 409),
ActorNotIdentified: ("unauthorized", 401),
}
@@ -0,0 +1,57 @@
from dishka.integrations.fastapi import FromDishka, inject
from fastapi import APIRouter, Request, Response, status
from fastapi.exceptions import RequestValidationError
from messenger.identity.application.bus import IdentityCommandBus
from messenger.identity.application.commands import CloseSession, OpenSession, RegisterUser
from messenger.identity.application.queries import GetCurrentUser
from messenger.identity.entrypoints.authentication import CurrentSession, VerifiedUser
from messenger.identity.entrypoints.schemas import CreateUserRequest, SessionOut, UserOut
from messenger.shared.entrypoints.actor import Actor
router = APIRouter(prefix="/api/v1/users", tags=["users"])
sessions_router = APIRouter(prefix="/api/v1/auth/sessions", tags=["sessions"])
@router.post("", status_code=status.HTTP_201_CREATED)
@inject
async def create_user(
body: CreateUserRequest,
bus: FromDishka[IdentityCommandBus],
) -> UserOut:
user = await bus.execute(
RegisterUser(username=body.username, display_name=body.display_name, password=body.password)
)
return UserOut.of(user)
@router.get("/me")
@inject
async def current_user(
actor: Actor,
query: FromDishka[GetCurrentUser],
) -> UserOut:
return UserOut.of(await query(actor))
@sessions_router.post("", status_code=status.HTTP_201_CREATED)
@inject
async def open_session(
request: Request,
user: VerifiedUser,
bus: FromDishka[IdentityCommandBus],
) -> SessionOut:
if await request.body():
raise RequestValidationError([])
token, session = await bus.execute(OpenSession(user_id=user.id))
return SessionOut.of(token, session, user)
@sessions_router.delete("/current", status_code=status.HTTP_204_NO_CONTENT)
@inject
async def close_session(
session: CurrentSession,
bus: FromDishka[IdentityCommandBus],
) -> Response:
await bus.execute(CloseSession(session=session))
return Response(status_code=status.HTTP_204_NO_CONTENT)
+112
View File
@@ -0,0 +1,112 @@
from collections.abc import AsyncIterator
from datetime import timedelta
from dishka import Provider, Scope, provide
from redis.asyncio import Redis
from redis.asyncio.retry import Retry
from redis.backoff import NoBackoff
from messenger.identity.application.bus import IdentityCommandBus
from messenger.identity.application.commands import CloseSession, OpenSession, RegisterUser
from messenger.identity.application.handlers import (
CloseSessionHandler,
OpenSessionHandler,
RegisterUserHandler,
)
from messenger.identity.application.ports import (
IdentityUnitOfWork,
PasswordHasher,
SessionDirectory,
UserDirectory,
)
from messenger.identity.application.queries import Authenticate, GetCurrentUser, VerifyCredentials
from messenger.identity.config import IdentitySettings
from messenger.identity.infrastructure.argon2 import Argon2PasswordHasher
from messenger.identity.infrastructure.postgres import (
PostgresIdentityUnitOfWork,
PostgresSessionDirectory,
PostgresUserDirectory,
)
from messenger.identity.infrastructure.valkey import CachedSessionDirectory
from messenger.shared.application.commands import Command
from messenger.shared.application.messagebus import CommandHandler
from messenger.shared.application.ports import Clock, EventPublisher, IdGenerator, Logger
from messenger.shared.infrastructure.postgres import Database
class IdentityProvider(Provider):
def __init__(self, settings: IdentitySettings) -> None:
super().__init__()
self._settings = settings
@provide(scope=Scope.APP)
def settings(self) -> IdentitySettings:
return self._settings
@provide(scope=Scope.APP)
def hasher(self, settings: IdentitySettings) -> PasswordHasher:
return Argon2PasswordHasher(settings.password_hash_concurrency)
@provide(scope=Scope.APP)
def users(self, database: Database) -> UserDirectory:
return PostgresUserDirectory(database)
@provide(scope=Scope.APP)
async def sessions(
self, database: Database, settings: IdentitySettings, logger: Logger
) -> AsyncIterator[SessionDirectory]:
origin = PostgresSessionDirectory(database)
if settings.cache_url is None:
yield origin
return
client = Redis.from_url(
settings.cache_url.get_secret_value(),
socket_connect_timeout=settings.cache_timeout_seconds,
socket_timeout=settings.cache_timeout_seconds,
retry=Retry(NoBackoff(), 1),
)
try:
yield CachedSessionDirectory(
origin,
client,
logger,
limit=timedelta(seconds=settings.session_cache_ttl_seconds),
cooldown=settings.cache_cooldown_seconds,
)
finally:
await client.aclose()
@provide(scope=Scope.APP)
def verify(self, users: UserDirectory, hasher: PasswordHasher) -> VerifyCredentials:
return VerifyCredentials(users, hasher)
@provide(scope=Scope.APP)
def authenticate(self, sessions: SessionDirectory, clock: Clock) -> Authenticate:
return Authenticate(sessions, clock)
@provide(scope=Scope.REQUEST)
def unit_of_work(self, database: Database) -> IdentityUnitOfWork:
return PostgresIdentityUnitOfWork(database)
@provide(scope=Scope.REQUEST)
def current_user(self, users: UserDirectory) -> GetCurrentUser:
return GetCurrentUser(users)
@provide(scope=Scope.REQUEST)
def command_bus(
self,
uow: IdentityUnitOfWork,
publisher: EventPublisher,
clock: Clock,
ids: IdGenerator,
hasher: PasswordHasher,
sessions: SessionDirectory,
settings: IdentitySettings,
) -> IdentityCommandBus:
ttl = timedelta(seconds=settings.session_ttl_seconds)
handlers: dict[type[Command], CommandHandler] = {
RegisterUser: RegisterUserHandler(uow, clock, ids, hasher),
OpenSession: OpenSessionHandler(uow, clock, ttl),
CloseSession: CloseSessionHandler(uow, sessions),
}
return IdentityCommandBus(handlers, uow, publisher)
@@ -0,0 +1,20 @@
from messenger.identity.domain.events import SessionClosed, SessionOpened, UserRegistered
from messenger.shared.application.ports import Logger
LOG_FIELDS = frozenset({"actor_id"})
class IdentityJournal:
__slots__ = ("_logger",)
def __init__(self, logger: Logger) -> None:
self._logger = logger
async def on_user_registered(self, event: UserRegistered) -> None:
self._logger.info("audit.user_registered", actor_id=str(event.user_id))
async def on_session_opened(self, event: SessionOpened) -> None:
self._logger.info("audit.session_opened", actor_id=str(event.user_id))
async def on_session_closed(self, event: SessionClosed) -> None:
self._logger.info("audit.session_closed", actor_id=str(event.user_id))
@@ -0,0 +1,42 @@
from datetime import datetime
from typing import Literal
from uuid import UUID
from pydantic import BaseModel, ConfigDict
from messenger.identity.domain.models import Session, User
class CreateUserRequest(BaseModel):
model_config = ConfigDict(extra="forbid", strict=True)
username: str
display_name: str
password: str
class UserOut(BaseModel):
id: UUID
username: str
display_name: str
created_at: datetime
@classmethod
def of(cls, user: User) -> UserOut:
return cls(
id=user.id,
username=user.username.value,
display_name=user.display_name.value,
created_at=user.created_at,
)
class SessionOut(BaseModel):
token: str
token_type: Literal["Bearer"] = "Bearer"
expires_at: datetime
user: UserOut
@classmethod
def of(cls, token: str, session: Session, user: User) -> SessionOut:
return cls(token=token, expires_at=session.expires_at, user=UserOut.of(user))
@@ -0,0 +1,35 @@
import asyncio
import secrets
from argon2 import PasswordHasher, Type
from argon2.exceptions import InvalidHashError, VerificationError
TIME_COST = 2
MEMORY_COST_KIB = 19_456
PARALLELISM = 1
class Argon2PasswordHasher:
__slots__ = ("_dummy", "_hasher", "_slots")
def __init__(self, concurrency: int) -> None:
self._hasher = PasswordHasher(
time_cost=TIME_COST,
memory_cost=MEMORY_COST_KIB,
parallelism=PARALLELISM,
type=Type.ID,
)
self._slots = asyncio.Semaphore(concurrency)
self._dummy = self._hasher.hash(secrets.token_urlsafe(16))
async def hash(self, password: str) -> str:
async with self._slots:
return await asyncio.to_thread(self._hasher.hash, password)
async def verify(self, password_hash: str | None, password: str) -> bool:
async with self._slots:
try:
await asyncio.to_thread(self._hasher.verify, password_hash or self._dummy, password)
except VerificationError, InvalidHashError:
return False
return password_hash is not None
@@ -0,0 +1,124 @@
from datetime import datetime
from uuid import UUID
from asyncpg import Record
from messenger.identity.domain.errors import UsernameAlreadyTaken
from messenger.identity.domain.models import Session, User
from messenger.identity.domain.values import DisplayName, Username
from messenger.shared.infrastructure.postgres import Connection, Database, PostgresUnitOfWork
INSERT_USER = """
INSERT INTO users (id, username, display_name, password_hash, created_at)
VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (username) DO NOTHING
RETURNING true
"""
USER_BY_ID = """
SELECT id, username, display_name, password_hash, created_at FROM users WHERE id = $1
"""
USER_BY_USERNAME = """
SELECT id, username, display_name, password_hash, created_at FROM users WHERE username = $1
"""
INSERT_SESSION = """
WITH expired AS (DELETE FROM sessions WHERE user_id = $2 AND expires_at <= $3)
INSERT INTO sessions (token_hash, user_id, created_at, expires_at) VALUES ($1, $2, $3, $4)
"""
DELETE_SESSION = "DELETE FROM sessions WHERE token_hash = $1"
ACTIVE_SESSION = """
SELECT token_hash, user_id, created_at, expires_at
FROM sessions
WHERE token_hash = $1 AND expires_at > $2
"""
def user_of(row: Record | None) -> User | None:
if row is None:
return None
return User(
id=row["id"],
username=Username(row["username"]),
display_name=DisplayName(row["display_name"]),
created_at=row["created_at"],
password_hash=row["password_hash"],
)
class PostgresUserRepository:
__slots__ = ("_connection",)
def __init__(self, connection: Connection) -> None:
self._connection = connection
async def add(self, user: User) -> None:
inserted = await self._connection.fetchval(
INSERT_USER,
user.id,
user.username.value,
user.display_name.value,
user.password_hash,
user.created_at,
)
if inserted is None:
raise UsernameAlreadyTaken("username уже занят")
class PostgresSessionRepository:
__slots__ = ("_connection",)
def __init__(self, connection: Connection) -> None:
self._connection = connection
async def add(self, session: Session) -> None:
await self._connection.execute(
INSERT_SESSION,
session.token_hash,
session.user_id,
session.created_at,
session.expires_at,
)
async def remove(self, token_hash: bytes) -> None:
await self._connection.execute(DELETE_SESSION, token_hash)
class PostgresIdentityUnitOfWork(PostgresUnitOfWork):
__slots__ = ()
@property
def users(self) -> PostgresUserRepository:
return PostgresUserRepository(self.connection)
@property
def sessions(self) -> PostgresSessionRepository:
return PostgresSessionRepository(self.connection)
class PostgresUserDirectory:
__slots__ = ("_database",)
def __init__(self, database: Database) -> None:
self._database = database
async def find(self, user_id: UUID) -> User | None:
async with self._database.connection() as connection:
return user_of(await connection.fetchrow(USER_BY_ID, user_id))
async def find_by_username(self, username: str) -> User | None:
async with self._database.connection() as connection:
return user_of(await connection.fetchrow(USER_BY_USERNAME, username))
class PostgresSessionDirectory:
__slots__ = ("_database",)
def __init__(self, database: Database) -> None:
self._database = database
async def find(self, token_hash: bytes, now: datetime) -> Session | None:
async with self._database.connection() as connection:
row = await connection.fetchrow(ACTIVE_SESSION, token_hash, now)
return None if row is None else Session(**dict(row))
async def invalidate(self, session: Session) -> None:
return None
@@ -0,0 +1,102 @@
import time
from collections.abc import Awaitable
from datetime import datetime, timedelta
from uuid import UUID
from redis.asyncio import Redis
from redis.exceptions import RedisError
from messenger.identity.application.ports import SessionDirectory
from messenger.identity.domain.models import Session
from messenger.shared.application.errors import Unavailable
from messenger.shared.application.ports import Logger
REVOKED = "-"
FAILURES = (RedisError, OSError)
def key_of(token_hash: bytes) -> str:
return f"session:{token_hash.hex()}"
def milliseconds(moment: datetime) -> int:
return int(moment.timestamp() * 1000)
def encode(session: Session) -> str:
return f"{session.user_id} {session.created_at.isoformat()} {session.expires_at.isoformat()}"
def decode(token_hash: bytes, value: str) -> Session:
user_id, created_at, expires_at = value.split(" ")
return Session(
token_hash=token_hash,
user_id=UUID(user_id),
created_at=datetime.fromisoformat(created_at),
expires_at=datetime.fromisoformat(expires_at),
)
class CachedSessionDirectory:
__slots__ = ("_client", "_cooldown", "_limit", "_logger", "_origin", "_resume_at")
def __init__(
self,
origin: SessionDirectory,
client: Redis,
logger: Logger,
*,
limit: timedelta,
cooldown: float,
) -> None:
self._origin = origin
self._client = client
self._logger = logger
self._limit = limit
self._cooldown = cooldown
self._resume_at = 0.0
async def find(self, token_hash: bytes, now: datetime) -> Session | None:
key = key_of(token_hash)
cached = await self._read(key)
if cached == REVOKED:
return None
if cached is not None:
session = decode(token_hash, cached)
return session if session.expires_at > now else None
session = await self._origin.find(token_hash, now)
if session is not None and self._closed():
until = milliseconds(min(session.expires_at, now + self._limit))
await self._quietly(self._client.set(key, encode(session), pxat=until, nx=True))
return session
async def invalidate(self, session: Session) -> None:
if not self._closed():
raise Unavailable("кэш сессий недоступен")
try:
await self._client.set(
key_of(session.token_hash), REVOKED, pxat=milliseconds(session.expires_at)
)
except FAILURES as error:
self._trip(error)
raise Unavailable("кэш сессий недоступен") from error
async def _read(self, key: str) -> str | None:
if not self._closed():
return None
value = await self._quietly(self._client.get(key))
return value.decode() if isinstance(value, bytes) else value
async def _quietly[ResultT](self, call: Awaitable[ResultT]) -> ResultT | None:
try:
return await call
except FAILURES as error:
self._trip(error)
return None
def _closed(self) -> bool:
return time.monotonic() >= self._resume_at
def _trip(self, error: Exception) -> None:
self._resume_at = time.monotonic() + self._cooldown
self._logger.warning("cache.bypassed", error=type(error).__name__, limit=self._cooldown)
+151
View File
@@ -0,0 +1,151 @@
from collections.abc import AsyncIterator, Sequence
from contextlib import asynccontextmanager
from uuid import UUID
from dishka import Provider, Scope, make_async_container, provide
from dishka.integrations.fastapi import FastapiProvider, setup_dishka
from fastapi import FastAPI
from messenger.attachments.application.ports import ChatMembership
from messenger.attachments.config import AttachmentsSettings
from messenger.attachments.domain.events import AttachmentDiscarded, AttachmentStored
from messenger.attachments.entrypoints.errors import ERROR_CODES as ATTACHMENT_ERRORS
from messenger.attachments.entrypoints.http import UPLOAD_PATH, attachments_router, uploads_router
from messenger.attachments.entrypoints.ioc import AttachmentsProvider
from messenger.attachments.entrypoints.journal import LOG_FIELDS as ATTACHMENT_FIELDS
from messenger.attachments.entrypoints.journal import AttachmentsJournal
from messenger.attachments.entrypoints.sweeper import UploadSweeper
from messenger.identity.config import IdentitySettings
from messenger.identity.domain.events import SessionClosed, SessionOpened, UserRegistered
from messenger.identity.entrypoints.authentication import challenge_for, guard
from messenger.identity.entrypoints.errors import ERROR_CODES as IDENTITY_ERRORS
from messenger.identity.entrypoints.http import router as users_router
from messenger.identity.entrypoints.http import sessions_router
from messenger.identity.entrypoints.ioc import IdentityProvider
from messenger.identity.entrypoints.journal import LOG_FIELDS as IDENTITY_FIELDS
from messenger.identity.entrypoints.journal import IdentityJournal
from messenger.messaging.application.ports import ChatDirectory
from messenger.messaging.config import MessagingSettings
from messenger.messaging.domain.events import ChatCreated, MessagePosted
from messenger.messaging.entrypoints.errors import ERROR_CODES as MESSAGING_ERRORS
from messenger.messaging.entrypoints.http import chats_router, messages_router
from messenger.messaging.entrypoints.ioc import MessagingProvider
from messenger.messaging.entrypoints.journal import LOG_FIELDS as MESSAGING_FIELDS
from messenger.messaging.entrypoints.journal import MessagingJournal
from messenger.shared.application.messagebus import EventBus
from messenger.shared.application.ports import Logger
from messenger.shared.config import PlatformSettings
from messenger.shared.entrypoints import errors, health
from messenger.shared.entrypoints.config import TransportSettings
from messenger.shared.entrypoints.ioc import PlatformProvider
from messenger.shared.entrypoints.pipeline import RequestPipeline
from messenger.shared.infrastructure.logging import PLATFORM_FIELDS, StructlogLogger
TRANSPORT_FIELDS = frozenset({"dropped_total", "queue", "instance"})
LOG_FIELDS = (
PLATFORM_FIELDS | TRANSPORT_FIELDS | IDENTITY_FIELDS | MESSAGING_FIELDS | ATTACHMENT_FIELDS
)
ERROR_CODES = {**IDENTITY_ERRORS, **MESSAGING_ERRORS, **ATTACHMENT_ERRORS}
class MessagingMembership:
__slots__ = ("_chats",)
def __init__(self, chats: ChatDirectory) -> None:
self._chats = chats
async def is_member(self, chat_id: UUID, user_id: UUID) -> bool:
return await self._chats.chat_for_member(chat_id, user_id) is not None
class ZoneLinks(Provider):
@provide(scope=Scope.APP)
def membership(self, chats: ChatDirectory) -> ChatMembership:
return MessagingMembership(chats)
def wire_events(bus: EventBus, logger: Logger) -> None:
identity = IdentityJournal(logger)
bus.subscribe(UserRegistered, identity.on_user_registered)
bus.subscribe(SessionOpened, identity.on_session_opened)
bus.subscribe(SessionClosed, identity.on_session_closed)
messaging = MessagingJournal(logger)
bus.subscribe(ChatCreated, messaging.on_chat_created)
bus.subscribe(MessagePosted, messaging.on_message_posted)
attachments = AttachmentsJournal(logger)
bus.subscribe(AttachmentStored, attachments.on_attachment_stored)
bus.subscribe(AttachmentDiscarded, attachments.on_attachment_discarded)
def create_app(
*,
platform: PlatformSettings | None = None,
transport: TransportSettings | None = None,
identity: IdentitySettings | None = None,
messaging: MessagingSettings | None = None,
attachments: AttachmentsSettings | None = None,
overrides: Sequence[Provider] = (),
) -> FastAPI:
platform = platform or PlatformSettings()
transport = transport or TransportSettings()
identity = identity or IdentitySettings()
messaging = messaging or MessagingSettings()
attachments = attachments or AttachmentsSettings()
logger = StructlogLogger(level=platform.log_level, allowed_fields=LOG_FIELDS)
container = make_async_container(
PlatformProvider(platform, logger),
IdentityProvider(identity),
MessagingProvider(messaging),
AttachmentsProvider(attachments),
ZoneLinks(),
FastapiProvider(),
*overrides,
)
sweeper = UploadSweeper(container, attachments, logger)
@asynccontextmanager
async def lifespan(_app: FastAPI) -> AsyncIterator[None]:
bus = await container.get(EventBus)
wire_events(bus, logger)
bus.start()
sweeper.start()
logger.info("service.started")
try:
yield
finally:
await sweeper.stop()
await bus.stop()
await container.close()
logger.info("service.stopped")
app = FastAPI(
title="MTUSI Messenger",
version="1.5.0",
lifespan=lifespan,
docs_url="/docs" if transport.docs else None,
redoc_url=None,
openapi_url="/openapi.json" if transport.docs else None,
)
app.add_middleware(
RequestPipeline,
logger=logger,
max_bytes=transport.max_body_bytes,
timeout=transport.request_timeout_seconds,
guard=guard,
instance=platform.instance_id,
limits=((UPLOAD_PATH, attachments.max_upload_bytes),),
)
errors.install(app, logger, ERROR_CODES, challenge_for, platform.instance_id)
app.include_router(health.router)
app.include_router(users_router)
app.include_router(sessions_router)
app.include_router(chats_router)
app.include_router(messages_router)
app.include_router(uploads_router)
app.include_router(attachments_router)
setup_dishka(container, app)
app.state.sweeper = sweeper
return app
View File
@@ -0,0 +1,5 @@
from messenger.shared.application.messagebus import CommandBus
class MessagingCommandBus(CommandBus):
__slots__ = ()
@@ -0,0 +1,22 @@
from collections.abc import Sequence
from dataclasses import dataclass
from uuid import UUID
from messenger.shared.application.commands import Command
@dataclass(frozen=True, slots=True)
class CreateChat(Command):
creator_id: UUID
title: str
member_ids: Sequence[UUID]
@dataclass(frozen=True, slots=True)
class PostMessage(Command):
sender_id: UUID
chat_id: UUID
text: str
key: str
fingerprint: bytes
attachment_ids: tuple[UUID, ...] = ()
@@ -0,0 +1,63 @@
from datetime import timedelta
from messenger.messaging.application.commands import CreateChat, PostMessage
from messenger.messaging.application.ports import MessagingUnitOfWork
from messenger.messaging.domain.errors import ChatNotFound
from messenger.messaging.domain.models import Chat, Message, Receipt
from messenger.shared.application.ports import Clock, IdGenerator
class CreateChatHandler:
__slots__ = ("_clock", "_ids", "_uow")
def __init__(self, uow: MessagingUnitOfWork, clock: Clock, ids: IdGenerator) -> None:
self._uow = uow
self._clock = clock
self._ids = ids
async def __call__(self, command: CreateChat) -> Chat:
chat = Chat.create(
chat_id=self._ids.new_id(),
title=command.title,
member_ids=command.member_ids,
creator_id=command.creator_id,
created_at=self._clock.now(),
)
async with self._uow.transaction():
await self._uow.chats.add(chat)
await self._uow.commit()
return chat
class PostMessageHandler:
__slots__ = ("_clock", "_ids", "_ttl", "_uow")
def __init__(
self, uow: MessagingUnitOfWork, clock: Clock, ids: IdGenerator, ttl: timedelta
) -> None:
self._uow = uow
self._clock = clock
self._ids = ids
self._ttl = ttl
async def __call__(self, command: PostMessage) -> Message:
async with self._uow.transaction():
chat = await self._uow.chats.get(command.chat_id)
if chat is None:
raise ChatNotFound("чат не найден")
now = self._clock.now()
message = chat.post(
message_id=self._ids.new_id(),
sender_id=command.sender_id,
text=command.text,
now=now,
attachment_ids=command.attachment_ids,
)
known = await self._uow.chats.receipt(
chat.id, command.sender_id, command.key, now - self._ttl
)
if known is not None:
return known.replay(command.fingerprint)
await self._uow.chats.append(Receipt(command.key, command.fingerprint, message))
await self._uow.commit()
return message
@@ -0,0 +1,46 @@
from datetime import datetime
from typing import Protocol, runtime_checkable
from uuid import UUID
from messenger.messaging.domain.keys import HistoryKey
from messenger.messaging.domain.models import Chat, Message, Receipt
from messenger.shared.application.ports import UnitOfWork
@runtime_checkable
class CursorCodec(Protocol):
def encode(self, scope: str, key: HistoryKey) -> str: ...
def decode(self, scope: str, raw: str) -> HistoryKey: ...
@runtime_checkable
class ChatRepository(Protocol):
async def add(self, chat: Chat) -> None: ...
async def get(self, chat_id: UUID) -> Chat | None: ...
async def receipt(
self, chat_id: UUID, sender_id: UUID, key: str, since: datetime
) -> Receipt | None: ...
async def append(self, receipt: Receipt) -> None: ...
@runtime_checkable
class MessagingUnitOfWork(UnitOfWork, Protocol):
@property
def chats(self) -> ChatRepository: ...
@runtime_checkable
class ChatDirectory(Protocol):
async def chat_for_member(self, chat_id: UUID, user_id: UUID) -> Chat | None: ...
async def chats_of(
self, user_id: UUID, *, after: HistoryKey | None, limit: int
) -> tuple[list[Chat], bool]: ...
async def history(
self, chat_id: UUID, *, after: HistoryKey | None, limit: int
) -> tuple[list[Message], bool]: ...
@@ -0,0 +1,73 @@
from dataclasses import dataclass
from uuid import UUID
from messenger.messaging.application.ports import ChatDirectory, CursorCodec
from messenger.messaging.domain.errors import ChatNotFound
from messenger.messaging.domain.keys import HistoryKey
from messenger.messaging.domain.models import Chat, Message
@dataclass(frozen=True, slots=True)
class Page[ItemT]:
items: list[ItemT]
next_cursor: str | None
def chats_scope(user_id: UUID) -> str:
return f"chats:{user_id}"
def messages_scope(chat_id: UUID) -> str:
return f"messages:{chat_id}"
class GetChat:
__slots__ = ("_chats",)
def __init__(self, chats: ChatDirectory) -> None:
self._chats = chats
async def __call__(self, chat_id: UUID, user_id: UUID) -> Chat:
chat = await self._chats.chat_for_member(chat_id, user_id)
if chat is None:
raise ChatNotFound("чат не найден")
return chat
class ListChats:
__slots__ = ("_chats", "_cursors")
def __init__(self, chats: ChatDirectory, cursors: CursorCodec) -> None:
self._chats = chats
self._cursors = cursors
async def __call__(self, user_id: UUID, *, limit: int, cursor: str | None) -> Page[Chat]:
scope = chats_scope(user_id)
after: HistoryKey | None = self._cursors.decode(scope, cursor) if cursor else None
items, has_more = await self._chats.chats_of(user_id, after=after, limit=limit)
next_cursor = self._cursors.encode(scope, items[-1].key) if has_more and items else None
return Page(items=items, next_cursor=next_cursor)
class ReadHistory:
__slots__ = ("_chats", "_cursors")
def __init__(self, chats: ChatDirectory, cursors: CursorCodec) -> None:
self._chats = chats
self._cursors = cursors
async def __call__(
self,
chat_id: UUID,
user_id: UUID,
*,
limit: int,
cursor: str | None,
) -> Page[Message]:
if await self._chats.chat_for_member(chat_id, user_id) is None:
raise ChatNotFound("чат не найден")
scope = messages_scope(chat_id)
after: HistoryKey | None = self._cursors.decode(scope, cursor) if cursor else None
items, has_more = await self._chats.history(chat_id, after=after, limit=limit)
next_cursor = self._cursors.encode(scope, items[-1].key) if has_more and items else None
return Page(items=items, next_cursor=next_cursor)
+8
View File
@@ -0,0 +1,8 @@
from pydantic import Field, SecretStr
from messenger.shared.config import Settings
class MessagingSettings(Settings):
cursor_secret: SecretStr = Field(min_length=16)
idempotency_ttl_seconds: int = Field(default=86_400, ge=1)
+33
View File
@@ -0,0 +1,33 @@
from messenger.shared.domain.errors import DomainError
class MessagingError(DomainError):
pass
class InvalidChatData(MessagingError):
pass
class InvalidMessageText(MessagingError):
pass
class InvalidPageRequest(MessagingError):
pass
class ChatNotFound(MessagingError):
pass
class IdempotencyConflict(MessagingError):
pass
class InvalidAttachmentList(MessagingError):
pass
class UnknownAttachment(MessagingError):
pass
+24
View File
@@ -0,0 +1,24 @@
from dataclasses import dataclass, field
from datetime import datetime
from uuid import UUID
from messenger.shared.domain.events import DomainEvent
@dataclass(frozen=True, slots=True)
class ChatCreated(DomainEvent):
chat_id: UUID
creator_id: UUID
member_count: int
created_at: datetime
name: str = field(init=False, default="chat.created")
@dataclass(frozen=True, slots=True)
class MessagePosted(DomainEvent):
message_id: UUID
chat_id: UUID
sender_id: UUID
text_length: int
created_at: datetime
name: str = field(init=False, default="message.posted")
+3
View File
@@ -0,0 +1,3 @@
from datetime import datetime
type HistoryKey = tuple[datetime, int]

Some files were not shown because too many files have changed in this diff Show More