ops: add private object storage

This commit is contained in:
2026-10-02 20:16:39 +03:00
parent b3d5d76302
commit 110236d01f
11 changed files with 220 additions and 9 deletions
+6 -1
View File
@@ -1,8 +1,12 @@
FROM golang:1.26.5-alpine3.24 AS build
WORKDIR /src
ENV GOMAXPROCS=2 GOMEMLIMIT=1GiB
COPY go.mod go.sum ./
RUN go mod download
COPY . .
RUN go build -o /out/messenger ./cmd/messenger
RUN go build -p=1 -o /out/messenger ./cmd/messenger
RUN go build -p=1 -o /out/lab4experiment ./cmd/lab4experiment
FROM build AS test
@@ -30,6 +34,7 @@ FROM alpine:3.24.2
RUN addgroup -S app && adduser -S -G app -u 10001 app
COPY --from=build --chown=app:app /out/messenger /app/messenger
COPY --from=build --chown=app:app /out/lab4experiment /app/lab4experiment
USER app
WORKDIR /app
+63
View File
@@ -32,6 +32,55 @@ services:
mem_limit: ${VALKEY_MEMORY}
restart: unless-stopped
s3:
build:
context: .
dockerfile: infra/minio/server.Dockerfile
args:
MINIO_VERSION: RELEASE.2025-10-15T17-29-55Z
image: mtusi-minio:RELEASE.2025-10-15T17-29-55Z
command: ["server", "/data", "--address", ":9000"]
environment:
MINIO_ROOT_USER: ${MINIO_ROOT_USER}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
MINIO_BROWSER: "off"
MINIO_PROMETHEUS_AUTH_TYPE: public
volumes:
- s3_data:/data
expose:
- "9000"
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9000/minio/health/ready"]
interval: 5s
timeout: 3s
retries: 20
cpus: ${S3_CPUS}
mem_limit: ${S3_MEMORY}
restart: unless-stopped
s3-bootstrap:
build:
context: .
dockerfile: infra/minio/client.Dockerfile
args:
MC_VERSION: RELEASE.2025-08-13T08-35-41Z
image: mtusi-mc:RELEASE.2025-08-13T08-35-41Z
entrypoint: ["/bin/sh", "/usr/local/bin/bootstrap.sh"]
environment:
MINIO_ROOT_USER: ${MINIO_ROOT_USER}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
S3_BUCKET: ${S3_BUCKET}
S3_ACCESS_KEY: ${S3_ACCESS_KEY}
S3_SECRET_KEY: ${S3_SECRET_KEY}
depends_on:
s3:
condition: service_healthy
healthcheck:
disable: true
cpus: ${S3_BOOTSTRAP_CPUS}
mem_limit: ${S3_BOOTSTRAP_MEMORY}
restart: "no"
migrate:
image: ${API_IMAGE:-mtusi-messenger:lab4}
command: ["/app/messenger", "migrate"]
@@ -82,6 +131,16 @@ services:
PASSWORD_SALT_LENGTH: ${PASSWORD_SALT_LENGTH}
PASSWORD_KEY_LENGTH: ${PASSWORD_KEY_LENGTH}
SWAGGER_PORT: ${SWAGGER_PORT}
S3_ENDPOINT: ${S3_ENDPOINT}
S3_REGION: ${S3_REGION}
S3_BUCKET: ${S3_BUCKET}
S3_ACCESS_KEY: ${S3_ACCESS_KEY}
S3_SECRET_KEY: ${S3_SECRET_KEY}
S3_USE_TLS: ${S3_USE_TLS}
S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS}
ATTACHMENT_MAX_BYTES: ${ATTACHMENT_MAX_BYTES}
ATTACHMENT_MAX_PIXELS: ${ATTACHMENT_MAX_PIXELS}
ATTACHMENT_DECODE_MEMORY_BYTES: ${ATTACHMENT_DECODE_MEMORY_BYTES}
expose:
- "8080"
depends_on:
@@ -89,6 +148,8 @@ services:
condition: service_completed_successfully
valkey:
condition: service_healthy
s3-bootstrap:
condition: service_completed_successfully
healthcheck:
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:${APP_PORT}/health/ready || exit 1"]
interval: 2s
@@ -211,6 +272,7 @@ services:
- nginx-exporter
- postgres-exporter
- valkey-exporter
- s3
cpus: ${PROMETHEUS_CPUS}
mem_limit: ${PROMETHEUS_MEMORY}
restart: unless-stopped
@@ -239,3 +301,4 @@ volumes:
valkey_data:
prometheus_data:
grafana_data:
s3_data:
+36
View File
@@ -0,0 +1,36 @@
#!/bin/sh
set -eu
alias_name=bootstrap
policy_name=messenger-attachments
policy_file=/tmp/messenger-attachments-policy.json
mc alias set "$alias_name" http://s3:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"
mc mb --ignore-existing "$alias_name/$S3_BUCKET"
mc anonymous set none "$alias_name/$S3_BUCKET"
cat >"$policy_file" <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["s3:GetBucketLocation", "s3:ListBucket"],
"Resource": ["arn:aws:s3:::$S3_BUCKET"]
},
{
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
"Resource": ["arn:aws:s3:::$S3_BUCKET/*"]
}
]
}
EOF
mc admin policy create "$alias_name" "$policy_name" "$policy_file"
mc admin user add "$alias_name" "$S3_ACCESS_KEY" "$S3_SECRET_KEY"
mc admin policy attach "$alias_name" "$policy_name" --user "$S3_ACCESS_KEY"
mc alias set application http://s3:9000 "$S3_ACCESS_KEY" "$S3_SECRET_KEY"
mc ls "application/$S3_BUCKET" >/dev/null
+16
View File
@@ -0,0 +1,16 @@
FROM golang:1.24.8-alpine3.22 AS build
ARG MC_VERSION
RUN apk add --no-cache git
RUN git clone --depth 1 --branch "${MC_VERSION}" https://github.com/minio/mc.git /src/mc
WORKDIR /src/mc
RUN go mod download
RUN LDFLAGS="$(MC_RELEASE=RELEASE go run buildscripts/gen-ldflags.go)" && \
GOMAXPROCS=2 GOMEMLIMIT=1GiB CGO_ENABLED=0 go build -p=1 -tags kqueue -trimpath -ldflags "$LDFLAGS" -o /out/mc .
FROM alpine:3.22.2
COPY --from=build /out/mc /usr/local/bin/mc
COPY infra/minio/bootstrap.sh /usr/local/bin/bootstrap.sh
ENTRYPOINT ["mc"]
+18
View File
@@ -0,0 +1,18 @@
FROM golang:1.24.8-alpine3.22 AS build
ARG MINIO_VERSION
RUN apk add --no-cache git
RUN git clone --depth 1 --branch "${MINIO_VERSION}" https://github.com/minio/minio.git /src/minio
WORKDIR /src/minio
RUN go mod download
RUN LDFLAGS="$(MINIO_RELEASE=RELEASE go run buildscripts/gen-ldflags.go)" && \
GOMAXPROCS=2 GOMEMLIMIT=1GiB CGO_ENABLED=0 go build -p=1 -tags kqueue -trimpath -ldflags "$LDFLAGS" -o /out/minio .
FROM alpine:3.22.2
RUN addgroup -S minio && adduser -S -G minio minio && mkdir /data && chown minio:minio /data
COPY --from=build /out/minio /usr/local/bin/minio
USER minio
VOLUME ["/data"]
ENTRYPOINT ["minio"]
+21 -1
View File
@@ -26,6 +26,22 @@ DATABASE_URL=postgres://messenger:local-postgres-password@postgres:5432/messenge
VALKEY_PASSWORD=local-valkey-password
CACHE_URL=redis://default:local-valkey-password@valkey:6379/0
# Административные данные MinIO используются только при bootstrap.
MINIO_ROOT_USER=local-minio-admin
MINIO_ROOT_PASSWORD=local-minio-admin-password
# API использует отдельного пользователя с доступом только к bucket вложений.
S3_ENDPOINT=s3:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY=messenger-app
S3_SECRET_KEY=local-messenger-s3-password
S3_USE_TLS=false
S3_TIMEOUT_SECONDS=5
ATTACHMENT_MAX_BYTES=10485760
ATTACHMENT_MAX_PIXELS=40000000
ATTACHMENT_DECODE_MEMORY_BYTES=402653184
# Размер пула и обслуживание соединений PostgreSQL.
DB_MIN_CONNS=1
DB_MAX_CONNS=10
@@ -55,7 +71,7 @@ SWAGGER_PORT=8081
# Ограничения ресурсов контейнеров.
API_CPUS=1
API_MEMORY=256m
API_MEMORY=512m
MIGRATE_CPUS=0.5
MIGRATE_MEMORY=128m
NGINX_CPUS=0.5
@@ -74,6 +90,10 @@ POSTGRES_CPUS=2
POSTGRES_MEMORY=1g
VALKEY_CPUS=1
VALKEY_MEMORY=256m
S3_CPUS=1
S3_MEMORY=512m
S3_BOOTSTRAP_CPUS=0.25
S3_BOOTSTRAP_MEMORY=256m
# Пароль локального администратора Grafana.
GRAFANA_ADMIN_PASSWORD=local-grafana-password
+46 -2
View File
@@ -85,6 +85,50 @@
{"expr": "sum(redis_db_keys)", "legendFormat": "keys", "refId": "A"},
{"expr": "rate(redis_commands_processed_total[1m])", "legendFormat": "commands/s", "refId": "B"}
]
},
{
"type": "timeseries", "title": "Attachment uploads by result", "id": 15,
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 56},
"targets": [{"expr": "sum by (result) (rate(messenger_attachment_uploads_total[1m]))", "legendFormat": "{{result}}", "refId": "A"}]
},
{
"type": "timeseries", "title": "Attachment upload latency p95", "id": 16,
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 56},
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
"targets": [{"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_upload_duration_seconds_bucket[5m])))", "legendFormat": "p95", "refId": "A"}]
},
{
"type": "timeseries", "title": "Attachment upload traffic", "id": 17,
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 64},
"fieldConfig": {"defaults": {"unit": "Bps"}, "overrides": []},
"targets": [{"expr": "sum(rate(messenger_attachment_upload_bytes_total[1m]))", "legendFormat": "bytes/s", "refId": "A"}]
},
{
"type": "timeseries", "title": "MinIO objects", "id": 18,
"gridPos": {"h": 8, "w": 8, "x": 8, "y": 64},
"targets": [{"expr": "sum(minio_cluster_usage_object_total)", "legendFormat": "objects", "refId": "A"}]
},
{
"type": "timeseries", "title": "MinIO requests and errors", "id": 19,
"gridPos": {"h": 8, "w": 8, "x": 16, "y": 64},
"targets": [
{"expr": "sum(rate(minio_s3_requests_total[1m]))", "legendFormat": "requests/s", "refId": "A"},
{"expr": "sum(rate(minio_s3_requests_errors_total[1m]))", "legendFormat": "errors/s", "refId": "B"}
]
},
{
"type": "timeseries", "title": "MinIO availability", "id": 20,
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 72},
"targets": [{"expr": "up{job=\"minio\"}", "legendFormat": "up", "refId": "A"}]
},
{
"type": "timeseries", "title": "MinIO storage", "id": 21,
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 72},
"fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []},
"targets": [
{"expr": "sum(minio_cluster_usage_total_bytes)", "legendFormat": "used", "refId": "A"},
{"expr": "sum(minio_cluster_capacity_usable_free_bytes)", "legendFormat": "free", "refId": "B"}
]
}
],
"refresh": "5s",
@@ -93,7 +137,7 @@
"templating": {"list": []},
"time": {"from": "now-15m", "to": "now"},
"timezone": "browser",
"title": "Messenger lab3 overview",
"title": "Messenger lab4 overview",
"uid": "messenger-overview",
"version": 2
"version": 3
}
+7 -2
View File
@@ -1,6 +1,6 @@
global:
scrape_interval: 5s
evaluation_interval: 5s
scrape_interval: 1s
evaluation_interval: 1s
scrape_configs:
- job_name: messenger
@@ -21,3 +21,8 @@ scrape_configs:
- job_name: valkey
static_configs:
- targets: ["valkey-exporter:9121"]
- job_name: minio
metrics_path: /minio/v2/metrics/cluster
static_configs:
- targets: ["s3:9000"]
+2
View File
@@ -26,6 +26,7 @@ http {
server {
listen 8080;
client_max_body_size 11m;
location / {
proxy_pass http://messenger;
@@ -34,6 +35,7 @@ http {
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_request_buffering off;
proxy_connect_timeout 1s;
proxy_send_timeout 10s;
+3 -2
View File
@@ -2,7 +2,8 @@
set -eu
docker compose stop nginx api valkey postgres
docker compose start postgres valkey
docker compose stop nginx api valkey postgres s3
docker compose start postgres valkey s3
docker compose run --rm s3-bootstrap
docker compose run --rm migrate
docker compose up -d --wait --wait-timeout 90
+2 -1
View File
@@ -34,7 +34,8 @@ for container in $old_containers; do
done
docker compose build
docker compose up -d postgres valkey
docker compose up -d postgres valkey s3
docker compose run --rm s3-bootstrap
docker compose run --rm migrate
if [ "$old_count" -lt 2 ]; then