ЛР 5: асинхронная обработка изображений и идемпотентность pipeline
This commit is contained in:
+231
@@ -0,0 +1,231 @@
|
||||
name: mtusi-lab5-local
|
||||
x-image-service: &image-service
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
DB_POOL_MAX: 4
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_worker_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_worker_secret_access_key
|
||||
BROKER_URL_FILE: ${BROKER_URL_FILE:-/run/secrets/broker_url}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-30}
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-1}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
GOMEMLIMIT: 96MiB
|
||||
secrets: [database_url, s3_worker_access_key_id, s3_worker_secret_access_key, broker_url]
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
s3-init: {condition: service_completed_successfully}
|
||||
broker: {condition: service_started}
|
||||
read_only: true
|
||||
cap_drop: [ALL]
|
||||
security_opt: [no-new-privileges:true]
|
||||
healthcheck: {disable: true}
|
||||
logging:
|
||||
options: {max-size: 10m, max-file: "2"}
|
||||
|
||||
services:
|
||||
db:
|
||||
image: postgres:17.6-bookworm@sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets: [postgres_password]
|
||||
volumes: [pgdata:/var/lib/postgresql/data]
|
||||
healthcheck:
|
||||
test: [CMD-SHELL, "pg_isready -h 127.0.0.1 -U messenger -d messenger"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
mem_limit: 512m
|
||||
cpus: ${DB_CPUS:-1.0}
|
||||
valkey:
|
||||
image: valkey/valkey:8.1.3-bookworm@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571
|
||||
command: [valkey-server, /run/secrets/valkey_config]
|
||||
secrets: [valkey_config, valkey_password]
|
||||
volumes: [sessions:/data]
|
||||
healthcheck:
|
||||
test: [CMD-SHELL, 'REDISCLI_AUTH="$$(cat /run/secrets/valkey_password)" valkey-cli ping | grep -q PONG']
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
mem_limit: 256m
|
||||
cpus: 0.5
|
||||
broker:
|
||||
image: valkey/valkey:8.1.3-bookworm@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571
|
||||
command: [valkey-server, /run/secrets/broker_config]
|
||||
secrets: [broker_config, broker_password]
|
||||
volumes: [brokerdata:/data]
|
||||
healthcheck:
|
||||
test: [CMD-SHELL, 'REDISCLI_AUTH="$$(cat /run/secrets/broker_password)" valkey-cli ping | grep -q PONG']
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
mem_limit: 256m
|
||||
cpus: 0.5
|
||||
migrate:
|
||||
image: mtusi-messenger:lab5
|
||||
build: .
|
||||
command: [migrate]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
secrets: [database_url]
|
||||
depends_on:
|
||||
db: {condition: service_healthy}
|
||||
mem_limit: 256m
|
||||
cpus: 0.5
|
||||
read_only: true
|
||||
cap_drop: [ALL]
|
||||
security_opt: [no-new-privileges:true]
|
||||
api:
|
||||
image: mtusi-messenger:lab5
|
||||
build: .
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
environment:
|
||||
APP_HOST: ${APP_HOST:-0.0.0.0}
|
||||
APP_PORT: 8080
|
||||
APP_ENV: ${APP_ENV:-lab}
|
||||
SESSION_STORE: ${SESSION_STORE:-valkey}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
DB_POOL_MAX: ${DB_POOL_MAX:-8}
|
||||
MAX_INFLIGHT: ${MAX_INFLIGHT:-32}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
GOMEMLIMIT: ${GOMEMLIMIT:-512MiB}
|
||||
GODEBUG: madvdontneed=0
|
||||
POOL_METRICS: ${POOL_METRICS:-0}
|
||||
REQUEST_LOG: ${REQUEST_LOG:-1}
|
||||
DATABASE_URL_FILE: ${DATABASE_URL_FILE:-/run/secrets/database_url}
|
||||
CACHE_URL_FILE: ${CACHE_URL_FILE:-/run/secrets/cache_url}
|
||||
CURSOR_SECRET_FILE: ${CURSOR_SECRET_FILE:-/run/secrets/cursor_secret}
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
secrets: [database_url, cache_url, cursor_secret, s3_access_key_id, s3_secret_access_key]
|
||||
tmpfs: ["/tmp:size=64m,mode=1777,noexec,nosuid,nodev"]
|
||||
depends_on:
|
||||
db: {condition: service_healthy}
|
||||
valkey: {condition: service_healthy}
|
||||
migrate: {condition: service_completed_successfully}
|
||||
s3-init: {condition: service_completed_successfully}
|
||||
read_only: true
|
||||
cap_drop: [ALL]
|
||||
security_opt: [no-new-privileges:true]
|
||||
mem_limit: ${API_MEMORY:-768m}
|
||||
cpus: ${API_CPUS:-1.0}
|
||||
pids_limit: 128
|
||||
logging:
|
||||
options: {max-size: 10m, max-file: "2"}
|
||||
stop_grace_period: 10s
|
||||
s3:
|
||||
image: mtusi-minio:RELEASE.2025-10-15T17-29-55Z
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.minio
|
||||
command: [server, /data, --console-address, "127.0.0.1:9001"]
|
||||
environment:
|
||||
MINIO_ROOT_USER_FILE: /run/secrets/s3_root_user
|
||||
MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_root_password
|
||||
MINIO_BROWSER: "off"
|
||||
secrets: [s3_root_user, s3_root_password]
|
||||
volumes: [objects:/data]
|
||||
networks:
|
||||
object_store:
|
||||
aliases: [minio-backend]
|
||||
cap_drop: [ALL]
|
||||
security_opt: [no-new-privileges:true]
|
||||
mem_limit: 1g
|
||||
cpus: 1.0
|
||||
healthcheck:
|
||||
test: [CMD, /healthcheck]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 60
|
||||
s3-init:
|
||||
image: mtusi-minio-tools:RELEASE.2025-08-13T08-35-41Z
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.minio
|
||||
target: tools
|
||||
entrypoint: [/bin/sh, /bootstrap/s3-init.sh]
|
||||
secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key, s3_worker_access_key_id, s3_worker_secret_access_key]
|
||||
volumes: [./infra/s3-init.sh:/bootstrap/s3-init.sh:ro, ./infra/s3-policy.json:/bootstrap/policy.json:ro, ./infra/s3-worker-policy.json:/bootstrap/worker-policy.json:ro]
|
||||
depends_on:
|
||||
s3: {condition: service_healthy}
|
||||
lb: {condition: service_started}
|
||||
tmpfs: ["/tmp:size=16m,mode=1777,noexec,nosuid,nodev"]
|
||||
environment: {MC_CONFIG_DIR: /tmp/mc}
|
||||
read_only: true
|
||||
cap_drop: [ALL]
|
||||
security_opt: [no-new-privileges:true]
|
||||
mem_limit: 128m
|
||||
cpus: 0.5
|
||||
dispatcher:
|
||||
<<: *image-service
|
||||
image: mtusi-messenger:lab5
|
||||
build: .
|
||||
command: [dispatcher]
|
||||
mem_limit: 192m
|
||||
cpus: 0.5
|
||||
worker:
|
||||
<<: *image-service
|
||||
image: mtusi-messenger-worker:lab5
|
||||
build: {context: ., target: worker}
|
||||
command: [worker]
|
||||
tmpfs: ["/tmp:size=32m,mode=1777,noexec,nosuid,nodev"]
|
||||
mem_limit: 640m
|
||||
cpus: 1.0
|
||||
pids_limit: 64
|
||||
lb:
|
||||
image: haproxy:3.2.6-alpine
|
||||
ports: ["127.0.0.1:${APP_PORT:-8080}:8080"]
|
||||
volumes: [./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro]
|
||||
depends_on:
|
||||
s3: {condition: service_healthy}
|
||||
networks:
|
||||
default:
|
||||
aliases: [s3]
|
||||
object_store: {}
|
||||
read_only: true
|
||||
cap_drop: [ALL]
|
||||
security_opt: [no-new-privileges:true]
|
||||
mem_limit: 128m
|
||||
cpus: 0.5
|
||||
stop_grace_period: 10s
|
||||
logging:
|
||||
options: {max-size: 10m, max-file: "2"}
|
||||
healthcheck:
|
||||
test: [CMD-SHELL, "wget -q -O /dev/null http://127.0.0.1:8080/health/ready"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
networks:
|
||||
object_store:
|
||||
internal: true
|
||||
volumes:
|
||||
pgdata:
|
||||
sessions:
|
||||
brokerdata:
|
||||
objects:
|
||||
secrets:
|
||||
postgres_password: {file: ./secrets/postgres_password}
|
||||
database_url: {file: ./secrets/database_url}
|
||||
valkey_password: {file: ./secrets/valkey_password}
|
||||
valkey_config: {file: ./secrets/valkey.conf}
|
||||
cache_url: {file: ./secrets/cache_url}
|
||||
cursor_secret: {file: ./secrets/cursor_secret}
|
||||
s3_root_user: {file: ./secrets/s3_root_user}
|
||||
s3_root_password: {file: ./secrets/s3_root_password}
|
||||
s3_access_key_id: {file: ./secrets/s3_access_key_id}
|
||||
s3_secret_access_key: {file: ./secrets/s3_secret_access_key}
|
||||
broker_url: {file: ./secrets/broker_url}
|
||||
broker_password: {file: ./secrets/broker_password}
|
||||
broker_config: {file: ./secrets/broker.conf}
|
||||
s3_worker_access_key_id: {file: ./secrets/s3_worker_access_key_id}
|
||||
s3_worker_secret_access_key: {file: ./secrets/s3_worker_secret_access_key}
|
||||
Reference in New Issue
Block a user