diff --git a/Dockerfile b/Dockerfile index f602f140..e903874c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,12 @@ FROM golang:1.26.5-alpine3.24 AS build WORKDIR /src +ENV GOMAXPROCS=2 GOMEMLIMIT=1GiB +COPY go.mod go.sum ./ +RUN go mod download COPY . . -RUN go build -o /out/messenger ./cmd/messenger +RUN go build -p=1 -o /out/messenger ./cmd/messenger +RUN go build -p=1 -o /out/lab4experiment ./cmd/lab4experiment FROM build AS test @@ -30,6 +34,7 @@ FROM alpine:3.24.2 RUN addgroup -S app && adduser -S -G app -u 10001 app COPY --from=build --chown=app:app /out/messenger /app/messenger +COPY --from=build --chown=app:app /out/lab4experiment /app/lab4experiment USER app WORKDIR /app diff --git a/compose.yaml b/compose.yaml index 2192f643..26eff747 100644 --- a/compose.yaml +++ b/compose.yaml @@ -32,6 +32,55 @@ services: mem_limit: ${VALKEY_MEMORY} restart: unless-stopped + s3: + build: + context: . + dockerfile: infra/minio/server.Dockerfile + args: + MINIO_VERSION: RELEASE.2025-10-15T17-29-55Z + image: mtusi-minio:RELEASE.2025-10-15T17-29-55Z + command: ["server", "/data", "--address", ":9000"] + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} + MINIO_BROWSER: "off" + MINIO_PROMETHEUS_AUTH_TYPE: public + volumes: + - s3_data:/data + expose: + - "9000" + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9000/minio/health/ready"] + interval: 5s + timeout: 3s + retries: 20 + cpus: ${S3_CPUS} + mem_limit: ${S3_MEMORY} + restart: unless-stopped + + s3-bootstrap: + build: + context: . + dockerfile: infra/minio/client.Dockerfile + args: + MC_VERSION: RELEASE.2025-08-13T08-35-41Z + image: mtusi-mc:RELEASE.2025-08-13T08-35-41Z + entrypoint: ["/bin/sh", "/usr/local/bin/bootstrap.sh"] + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} + S3_BUCKET: ${S3_BUCKET} + S3_ACCESS_KEY: ${S3_ACCESS_KEY} + S3_SECRET_KEY: ${S3_SECRET_KEY} + depends_on: + s3: + condition: service_healthy + healthcheck: + disable: true + cpus: ${S3_BOOTSTRAP_CPUS} + mem_limit: ${S3_BOOTSTRAP_MEMORY} + restart: "no" + migrate: image: ${API_IMAGE:-mtusi-messenger:lab4} command: ["/app/messenger", "migrate"] @@ -82,6 +131,16 @@ services: PASSWORD_SALT_LENGTH: ${PASSWORD_SALT_LENGTH} PASSWORD_KEY_LENGTH: ${PASSWORD_KEY_LENGTH} SWAGGER_PORT: ${SWAGGER_PORT} + S3_ENDPOINT: ${S3_ENDPOINT} + S3_REGION: ${S3_REGION} + S3_BUCKET: ${S3_BUCKET} + S3_ACCESS_KEY: ${S3_ACCESS_KEY} + S3_SECRET_KEY: ${S3_SECRET_KEY} + S3_USE_TLS: ${S3_USE_TLS} + S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS} + ATTACHMENT_MAX_BYTES: ${ATTACHMENT_MAX_BYTES} + ATTACHMENT_MAX_PIXELS: ${ATTACHMENT_MAX_PIXELS} + ATTACHMENT_DECODE_MEMORY_BYTES: ${ATTACHMENT_DECODE_MEMORY_BYTES} expose: - "8080" depends_on: @@ -89,6 +148,8 @@ services: condition: service_completed_successfully valkey: condition: service_healthy + s3-bootstrap: + condition: service_completed_successfully healthcheck: test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:${APP_PORT}/health/ready || exit 1"] interval: 2s @@ -211,6 +272,7 @@ services: - nginx-exporter - postgres-exporter - valkey-exporter + - s3 cpus: ${PROMETHEUS_CPUS} mem_limit: ${PROMETHEUS_MEMORY} restart: unless-stopped @@ -239,3 +301,4 @@ volumes: valkey_data: prometheus_data: grafana_data: + s3_data: diff --git a/infra/minio/bootstrap.sh b/infra/minio/bootstrap.sh new file mode 100644 index 00000000..68a28884 --- /dev/null +++ b/infra/minio/bootstrap.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +set -eu + +alias_name=bootstrap +policy_name=messenger-attachments +policy_file=/tmp/messenger-attachments-policy.json + +mc alias set "$alias_name" http://s3:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" +mc mb --ignore-existing "$alias_name/$S3_BUCKET" +mc anonymous set none "$alias_name/$S3_BUCKET" + +cat >"$policy_file" </dev/null diff --git a/infra/minio/client.Dockerfile b/infra/minio/client.Dockerfile new file mode 100644 index 00000000..fe8ee0a4 --- /dev/null +++ b/infra/minio/client.Dockerfile @@ -0,0 +1,16 @@ +FROM golang:1.24.8-alpine3.22 AS build + +ARG MC_VERSION +RUN apk add --no-cache git +RUN git clone --depth 1 --branch "${MC_VERSION}" https://github.com/minio/mc.git /src/mc +WORKDIR /src/mc +RUN go mod download +RUN LDFLAGS="$(MC_RELEASE=RELEASE go run buildscripts/gen-ldflags.go)" && \ + GOMAXPROCS=2 GOMEMLIMIT=1GiB CGO_ENABLED=0 go build -p=1 -tags kqueue -trimpath -ldflags "$LDFLAGS" -o /out/mc . + +FROM alpine:3.22.2 + +COPY --from=build /out/mc /usr/local/bin/mc +COPY infra/minio/bootstrap.sh /usr/local/bin/bootstrap.sh + +ENTRYPOINT ["mc"] diff --git a/infra/minio/server.Dockerfile b/infra/minio/server.Dockerfile new file mode 100644 index 00000000..44b75c4d --- /dev/null +++ b/infra/minio/server.Dockerfile @@ -0,0 +1,18 @@ +FROM golang:1.24.8-alpine3.22 AS build + +ARG MINIO_VERSION +RUN apk add --no-cache git +RUN git clone --depth 1 --branch "${MINIO_VERSION}" https://github.com/minio/minio.git /src/minio +WORKDIR /src/minio +RUN go mod download +RUN LDFLAGS="$(MINIO_RELEASE=RELEASE go run buildscripts/gen-ldflags.go)" && \ + GOMAXPROCS=2 GOMEMLIMIT=1GiB CGO_ENABLED=0 go build -p=1 -tags kqueue -trimpath -ldflags "$LDFLAGS" -o /out/minio . + +FROM alpine:3.22.2 + +RUN addgroup -S minio && adduser -S -G minio minio && mkdir /data && chown minio:minio /data +COPY --from=build /out/minio /usr/local/bin/minio + +USER minio +VOLUME ["/data"] +ENTRYPOINT ["minio"] diff --git a/lab4.env.example b/lab4.env.example index 447b3b72..11cbdb52 100644 --- a/lab4.env.example +++ b/lab4.env.example @@ -26,6 +26,22 @@ DATABASE_URL=postgres://messenger:local-postgres-password@postgres:5432/messenge VALKEY_PASSWORD=local-valkey-password CACHE_URL=redis://default:local-valkey-password@valkey:6379/0 +# Административные данные MinIO используются только при bootstrap. +MINIO_ROOT_USER=local-minio-admin +MINIO_ROOT_PASSWORD=local-minio-admin-password + +# API использует отдельного пользователя с доступом только к bucket вложений. +S3_ENDPOINT=s3:9000 +S3_REGION=us-east-1 +S3_BUCKET=messenger-attachments +S3_ACCESS_KEY=messenger-app +S3_SECRET_KEY=local-messenger-s3-password +S3_USE_TLS=false +S3_TIMEOUT_SECONDS=5 +ATTACHMENT_MAX_BYTES=10485760 +ATTACHMENT_MAX_PIXELS=40000000 +ATTACHMENT_DECODE_MEMORY_BYTES=402653184 + # Размер пула и обслуживание соединений PostgreSQL. DB_MIN_CONNS=1 DB_MAX_CONNS=10 @@ -55,7 +71,7 @@ SWAGGER_PORT=8081 # Ограничения ресурсов контейнеров. API_CPUS=1 -API_MEMORY=256m +API_MEMORY=512m MIGRATE_CPUS=0.5 MIGRATE_MEMORY=128m NGINX_CPUS=0.5 @@ -74,6 +90,10 @@ POSTGRES_CPUS=2 POSTGRES_MEMORY=1g VALKEY_CPUS=1 VALKEY_MEMORY=256m +S3_CPUS=1 +S3_MEMORY=512m +S3_BOOTSTRAP_CPUS=0.25 +S3_BOOTSTRAP_MEMORY=256m # Пароль локального администратора Grafana. GRAFANA_ADMIN_PASSWORD=local-grafana-password diff --git a/monitoring/grafana/dashboards/messenger.json b/monitoring/grafana/dashboards/messenger.json index f6af0cd2..0478e84b 100644 --- a/monitoring/grafana/dashboards/messenger.json +++ b/monitoring/grafana/dashboards/messenger.json @@ -85,6 +85,50 @@ {"expr": "sum(redis_db_keys)", "legendFormat": "keys", "refId": "A"}, {"expr": "rate(redis_commands_processed_total[1m])", "legendFormat": "commands/s", "refId": "B"} ] + }, + { + "type": "timeseries", "title": "Attachment uploads by result", "id": 15, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 56}, + "targets": [{"expr": "sum by (result) (rate(messenger_attachment_uploads_total[1m]))", "legendFormat": "{{result}}", "refId": "A"}] + }, + { + "type": "timeseries", "title": "Attachment upload latency p95", "id": 16, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 56}, + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, + "targets": [{"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_upload_duration_seconds_bucket[5m])))", "legendFormat": "p95", "refId": "A"}] + }, + { + "type": "timeseries", "title": "Attachment upload traffic", "id": 17, + "gridPos": {"h": 8, "w": 8, "x": 0, "y": 64}, + "fieldConfig": {"defaults": {"unit": "Bps"}, "overrides": []}, + "targets": [{"expr": "sum(rate(messenger_attachment_upload_bytes_total[1m]))", "legendFormat": "bytes/s", "refId": "A"}] + }, + { + "type": "timeseries", "title": "MinIO objects", "id": 18, + "gridPos": {"h": 8, "w": 8, "x": 8, "y": 64}, + "targets": [{"expr": "sum(minio_cluster_usage_object_total)", "legendFormat": "objects", "refId": "A"}] + }, + { + "type": "timeseries", "title": "MinIO requests and errors", "id": 19, + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 64}, + "targets": [ + {"expr": "sum(rate(minio_s3_requests_total[1m]))", "legendFormat": "requests/s", "refId": "A"}, + {"expr": "sum(rate(minio_s3_requests_errors_total[1m]))", "legendFormat": "errors/s", "refId": "B"} + ] + }, + { + "type": "timeseries", "title": "MinIO availability", "id": 20, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 72}, + "targets": [{"expr": "up{job=\"minio\"}", "legendFormat": "up", "refId": "A"}] + }, + { + "type": "timeseries", "title": "MinIO storage", "id": 21, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 72}, + "fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []}, + "targets": [ + {"expr": "sum(minio_cluster_usage_total_bytes)", "legendFormat": "used", "refId": "A"}, + {"expr": "sum(minio_cluster_capacity_usable_free_bytes)", "legendFormat": "free", "refId": "B"} + ] } ], "refresh": "5s", @@ -93,7 +137,7 @@ "templating": {"list": []}, "time": {"from": "now-15m", "to": "now"}, "timezone": "browser", - "title": "Messenger lab3 overview", + "title": "Messenger lab4 overview", "uid": "messenger-overview", - "version": 2 + "version": 3 } diff --git a/monitoring/prometheus.yml b/monitoring/prometheus.yml index 2c0881ea..83721c48 100644 --- a/monitoring/prometheus.yml +++ b/monitoring/prometheus.yml @@ -1,6 +1,6 @@ global: - scrape_interval: 5s - evaluation_interval: 5s + scrape_interval: 1s + evaluation_interval: 1s scrape_configs: - job_name: messenger @@ -21,3 +21,8 @@ scrape_configs: - job_name: valkey static_configs: - targets: ["valkey-exporter:9121"] + + - job_name: minio + metrics_path: /minio/v2/metrics/cluster + static_configs: + - targets: ["s3:9000"] diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 9dcebe8f..3dde36ac 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -26,6 +26,7 @@ http { server { listen 8080; + client_max_body_size 11m; location / { proxy_pass http://messenger; @@ -34,6 +35,7 @@ http { proxy_set_header Host $host; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; + proxy_request_buffering off; proxy_connect_timeout 1s; proxy_send_timeout 10s; diff --git a/scripts/restart.sh b/scripts/restart.sh index cacb5215..a724583d 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -2,7 +2,8 @@ set -eu -docker compose stop nginx api valkey postgres -docker compose start postgres valkey +docker compose stop nginx api valkey postgres s3 +docker compose start postgres valkey s3 +docker compose run --rm s3-bootstrap docker compose run --rm migrate docker compose up -d --wait --wait-timeout 90 diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh index e32e2973..8d1e4369 100755 --- a/scripts/rolling-update.sh +++ b/scripts/rolling-update.sh @@ -34,7 +34,8 @@ for container in $old_containers; do done docker compose build -docker compose up -d postgres valkey +docker compose up -d postgres valkey s3 +docker compose run --rm s3-bootstrap docker compose run --rm migrate if [ "$old_count" -lt 2 ]; then