forked from mtusi2026/mtusi2026lab4
ops: add private object storage
This commit is contained in:
+6
-1
@@ -1,8 +1,12 @@
|
||||
FROM golang:1.26.5-alpine3.24 AS build
|
||||
|
||||
WORKDIR /src
|
||||
ENV GOMAXPROCS=2 GOMEMLIMIT=1GiB
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
RUN go build -o /out/messenger ./cmd/messenger
|
||||
RUN go build -p=1 -o /out/messenger ./cmd/messenger
|
||||
RUN go build -p=1 -o /out/lab4experiment ./cmd/lab4experiment
|
||||
|
||||
FROM build AS test
|
||||
|
||||
@@ -30,6 +34,7 @@ FROM alpine:3.24.2
|
||||
|
||||
RUN addgroup -S app && adduser -S -G app -u 10001 app
|
||||
COPY --from=build --chown=app:app /out/messenger /app/messenger
|
||||
COPY --from=build --chown=app:app /out/lab4experiment /app/lab4experiment
|
||||
|
||||
USER app
|
||||
WORKDIR /app
|
||||
|
||||
@@ -32,6 +32,55 @@ services:
|
||||
mem_limit: ${VALKEY_MEMORY}
|
||||
restart: unless-stopped
|
||||
|
||||
s3:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: infra/minio/server.Dockerfile
|
||||
args:
|
||||
MINIO_VERSION: RELEASE.2025-10-15T17-29-55Z
|
||||
image: mtusi-minio:RELEASE.2025-10-15T17-29-55Z
|
||||
command: ["server", "/data", "--address", ":9000"]
|
||||
environment:
|
||||
MINIO_ROOT_USER: ${MINIO_ROOT_USER}
|
||||
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
|
||||
MINIO_BROWSER: "off"
|
||||
MINIO_PROMETHEUS_AUTH_TYPE: public
|
||||
volumes:
|
||||
- s3_data:/data
|
||||
expose:
|
||||
- "9000"
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9000/minio/health/ready"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
cpus: ${S3_CPUS}
|
||||
mem_limit: ${S3_MEMORY}
|
||||
restart: unless-stopped
|
||||
|
||||
s3-bootstrap:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: infra/minio/client.Dockerfile
|
||||
args:
|
||||
MC_VERSION: RELEASE.2025-08-13T08-35-41Z
|
||||
image: mtusi-mc:RELEASE.2025-08-13T08-35-41Z
|
||||
entrypoint: ["/bin/sh", "/usr/local/bin/bootstrap.sh"]
|
||||
environment:
|
||||
MINIO_ROOT_USER: ${MINIO_ROOT_USER}
|
||||
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
|
||||
S3_BUCKET: ${S3_BUCKET}
|
||||
S3_ACCESS_KEY: ${S3_ACCESS_KEY}
|
||||
S3_SECRET_KEY: ${S3_SECRET_KEY}
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
disable: true
|
||||
cpus: ${S3_BOOTSTRAP_CPUS}
|
||||
mem_limit: ${S3_BOOTSTRAP_MEMORY}
|
||||
restart: "no"
|
||||
|
||||
migrate:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab4}
|
||||
command: ["/app/messenger", "migrate"]
|
||||
@@ -82,6 +131,16 @@ services:
|
||||
PASSWORD_SALT_LENGTH: ${PASSWORD_SALT_LENGTH}
|
||||
PASSWORD_KEY_LENGTH: ${PASSWORD_KEY_LENGTH}
|
||||
SWAGGER_PORT: ${SWAGGER_PORT}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT}
|
||||
S3_REGION: ${S3_REGION}
|
||||
S3_BUCKET: ${S3_BUCKET}
|
||||
S3_ACCESS_KEY: ${S3_ACCESS_KEY}
|
||||
S3_SECRET_KEY: ${S3_SECRET_KEY}
|
||||
S3_USE_TLS: ${S3_USE_TLS}
|
||||
S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS}
|
||||
ATTACHMENT_MAX_BYTES: ${ATTACHMENT_MAX_BYTES}
|
||||
ATTACHMENT_MAX_PIXELS: ${ATTACHMENT_MAX_PIXELS}
|
||||
ATTACHMENT_DECODE_MEMORY_BYTES: ${ATTACHMENT_DECODE_MEMORY_BYTES}
|
||||
expose:
|
||||
- "8080"
|
||||
depends_on:
|
||||
@@ -89,6 +148,8 @@ services:
|
||||
condition: service_completed_successfully
|
||||
valkey:
|
||||
condition: service_healthy
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:${APP_PORT}/health/ready || exit 1"]
|
||||
interval: 2s
|
||||
@@ -211,6 +272,7 @@ services:
|
||||
- nginx-exporter
|
||||
- postgres-exporter
|
||||
- valkey-exporter
|
||||
- s3
|
||||
cpus: ${PROMETHEUS_CPUS}
|
||||
mem_limit: ${PROMETHEUS_MEMORY}
|
||||
restart: unless-stopped
|
||||
@@ -239,3 +301,4 @@ volumes:
|
||||
valkey_data:
|
||||
prometheus_data:
|
||||
grafana_data:
|
||||
s3_data:
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -eu
|
||||
|
||||
alias_name=bootstrap
|
||||
policy_name=messenger-attachments
|
||||
policy_file=/tmp/messenger-attachments-policy.json
|
||||
|
||||
mc alias set "$alias_name" http://s3:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"
|
||||
mc mb --ignore-existing "$alias_name/$S3_BUCKET"
|
||||
mc anonymous set none "$alias_name/$S3_BUCKET"
|
||||
|
||||
cat >"$policy_file" <<EOF
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetBucketLocation", "s3:ListBucket"],
|
||||
"Resource": ["arn:aws:s3:::$S3_BUCKET"]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
||||
"Resource": ["arn:aws:s3:::$S3_BUCKET/*"]
|
||||
}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
|
||||
mc admin policy create "$alias_name" "$policy_name" "$policy_file"
|
||||
mc admin user add "$alias_name" "$S3_ACCESS_KEY" "$S3_SECRET_KEY"
|
||||
mc admin policy attach "$alias_name" "$policy_name" --user "$S3_ACCESS_KEY"
|
||||
|
||||
mc alias set application http://s3:9000 "$S3_ACCESS_KEY" "$S3_SECRET_KEY"
|
||||
mc ls "application/$S3_BUCKET" >/dev/null
|
||||
@@ -0,0 +1,16 @@
|
||||
FROM golang:1.24.8-alpine3.22 AS build
|
||||
|
||||
ARG MC_VERSION
|
||||
RUN apk add --no-cache git
|
||||
RUN git clone --depth 1 --branch "${MC_VERSION}" https://github.com/minio/mc.git /src/mc
|
||||
WORKDIR /src/mc
|
||||
RUN go mod download
|
||||
RUN LDFLAGS="$(MC_RELEASE=RELEASE go run buildscripts/gen-ldflags.go)" && \
|
||||
GOMAXPROCS=2 GOMEMLIMIT=1GiB CGO_ENABLED=0 go build -p=1 -tags kqueue -trimpath -ldflags "$LDFLAGS" -o /out/mc .
|
||||
|
||||
FROM alpine:3.22.2
|
||||
|
||||
COPY --from=build /out/mc /usr/local/bin/mc
|
||||
COPY infra/minio/bootstrap.sh /usr/local/bin/bootstrap.sh
|
||||
|
||||
ENTRYPOINT ["mc"]
|
||||
@@ -0,0 +1,18 @@
|
||||
FROM golang:1.24.8-alpine3.22 AS build
|
||||
|
||||
ARG MINIO_VERSION
|
||||
RUN apk add --no-cache git
|
||||
RUN git clone --depth 1 --branch "${MINIO_VERSION}" https://github.com/minio/minio.git /src/minio
|
||||
WORKDIR /src/minio
|
||||
RUN go mod download
|
||||
RUN LDFLAGS="$(MINIO_RELEASE=RELEASE go run buildscripts/gen-ldflags.go)" && \
|
||||
GOMAXPROCS=2 GOMEMLIMIT=1GiB CGO_ENABLED=0 go build -p=1 -tags kqueue -trimpath -ldflags "$LDFLAGS" -o /out/minio .
|
||||
|
||||
FROM alpine:3.22.2
|
||||
|
||||
RUN addgroup -S minio && adduser -S -G minio minio && mkdir /data && chown minio:minio /data
|
||||
COPY --from=build /out/minio /usr/local/bin/minio
|
||||
|
||||
USER minio
|
||||
VOLUME ["/data"]
|
||||
ENTRYPOINT ["minio"]
|
||||
+21
-1
@@ -26,6 +26,22 @@ DATABASE_URL=postgres://messenger:local-postgres-password@postgres:5432/messenge
|
||||
VALKEY_PASSWORD=local-valkey-password
|
||||
CACHE_URL=redis://default:local-valkey-password@valkey:6379/0
|
||||
|
||||
# Административные данные MinIO используются только при bootstrap.
|
||||
MINIO_ROOT_USER=local-minio-admin
|
||||
MINIO_ROOT_PASSWORD=local-minio-admin-password
|
||||
|
||||
# API использует отдельного пользователя с доступом только к bucket вложений.
|
||||
S3_ENDPOINT=s3:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY=messenger-app
|
||||
S3_SECRET_KEY=local-messenger-s3-password
|
||||
S3_USE_TLS=false
|
||||
S3_TIMEOUT_SECONDS=5
|
||||
ATTACHMENT_MAX_BYTES=10485760
|
||||
ATTACHMENT_MAX_PIXELS=40000000
|
||||
ATTACHMENT_DECODE_MEMORY_BYTES=402653184
|
||||
|
||||
# Размер пула и обслуживание соединений PostgreSQL.
|
||||
DB_MIN_CONNS=1
|
||||
DB_MAX_CONNS=10
|
||||
@@ -55,7 +71,7 @@ SWAGGER_PORT=8081
|
||||
|
||||
# Ограничения ресурсов контейнеров.
|
||||
API_CPUS=1
|
||||
API_MEMORY=256m
|
||||
API_MEMORY=512m
|
||||
MIGRATE_CPUS=0.5
|
||||
MIGRATE_MEMORY=128m
|
||||
NGINX_CPUS=0.5
|
||||
@@ -74,6 +90,10 @@ POSTGRES_CPUS=2
|
||||
POSTGRES_MEMORY=1g
|
||||
VALKEY_CPUS=1
|
||||
VALKEY_MEMORY=256m
|
||||
S3_CPUS=1
|
||||
S3_MEMORY=512m
|
||||
S3_BOOTSTRAP_CPUS=0.25
|
||||
S3_BOOTSTRAP_MEMORY=256m
|
||||
|
||||
# Пароль локального администратора Grafana.
|
||||
GRAFANA_ADMIN_PASSWORD=local-grafana-password
|
||||
|
||||
@@ -85,6 +85,50 @@
|
||||
{"expr": "sum(redis_db_keys)", "legendFormat": "keys", "refId": "A"},
|
||||
{"expr": "rate(redis_commands_processed_total[1m])", "legendFormat": "commands/s", "refId": "B"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries", "title": "Attachment uploads by result", "id": 15,
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 56},
|
||||
"targets": [{"expr": "sum by (result) (rate(messenger_attachment_uploads_total[1m]))", "legendFormat": "{{result}}", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "timeseries", "title": "Attachment upload latency p95", "id": 16,
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 56},
|
||||
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
|
||||
"targets": [{"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_upload_duration_seconds_bucket[5m])))", "legendFormat": "p95", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "timeseries", "title": "Attachment upload traffic", "id": 17,
|
||||
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 64},
|
||||
"fieldConfig": {"defaults": {"unit": "Bps"}, "overrides": []},
|
||||
"targets": [{"expr": "sum(rate(messenger_attachment_upload_bytes_total[1m]))", "legendFormat": "bytes/s", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "timeseries", "title": "MinIO objects", "id": 18,
|
||||
"gridPos": {"h": 8, "w": 8, "x": 8, "y": 64},
|
||||
"targets": [{"expr": "sum(minio_cluster_usage_object_total)", "legendFormat": "objects", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "timeseries", "title": "MinIO requests and errors", "id": 19,
|
||||
"gridPos": {"h": 8, "w": 8, "x": 16, "y": 64},
|
||||
"targets": [
|
||||
{"expr": "sum(rate(minio_s3_requests_total[1m]))", "legendFormat": "requests/s", "refId": "A"},
|
||||
{"expr": "sum(rate(minio_s3_requests_errors_total[1m]))", "legendFormat": "errors/s", "refId": "B"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries", "title": "MinIO availability", "id": 20,
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 72},
|
||||
"targets": [{"expr": "up{job=\"minio\"}", "legendFormat": "up", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "timeseries", "title": "MinIO storage", "id": 21,
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 72},
|
||||
"fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []},
|
||||
"targets": [
|
||||
{"expr": "sum(minio_cluster_usage_total_bytes)", "legendFormat": "used", "refId": "A"},
|
||||
{"expr": "sum(minio_cluster_capacity_usable_free_bytes)", "legendFormat": "free", "refId": "B"}
|
||||
]
|
||||
}
|
||||
],
|
||||
"refresh": "5s",
|
||||
@@ -93,7 +137,7 @@
|
||||
"templating": {"list": []},
|
||||
"time": {"from": "now-15m", "to": "now"},
|
||||
"timezone": "browser",
|
||||
"title": "Messenger lab3 overview",
|
||||
"title": "Messenger lab4 overview",
|
||||
"uid": "messenger-overview",
|
||||
"version": 2
|
||||
"version": 3
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
global:
|
||||
scrape_interval: 5s
|
||||
evaluation_interval: 5s
|
||||
scrape_interval: 1s
|
||||
evaluation_interval: 1s
|
||||
|
||||
scrape_configs:
|
||||
- job_name: messenger
|
||||
@@ -21,3 +21,8 @@ scrape_configs:
|
||||
- job_name: valkey
|
||||
static_configs:
|
||||
- targets: ["valkey-exporter:9121"]
|
||||
|
||||
- job_name: minio
|
||||
metrics_path: /minio/v2/metrics/cluster
|
||||
static_configs:
|
||||
- targets: ["s3:9000"]
|
||||
|
||||
@@ -26,6 +26,7 @@ http {
|
||||
|
||||
server {
|
||||
listen 8080;
|
||||
client_max_body_size 11m;
|
||||
|
||||
location / {
|
||||
proxy_pass http://messenger;
|
||||
@@ -34,6 +35,7 @@ http {
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_request_buffering off;
|
||||
|
||||
proxy_connect_timeout 1s;
|
||||
proxy_send_timeout 10s;
|
||||
|
||||
+3
-2
@@ -2,7 +2,8 @@
|
||||
|
||||
set -eu
|
||||
|
||||
docker compose stop nginx api valkey postgres
|
||||
docker compose start postgres valkey
|
||||
docker compose stop nginx api valkey postgres s3
|
||||
docker compose start postgres valkey s3
|
||||
docker compose run --rm s3-bootstrap
|
||||
docker compose run --rm migrate
|
||||
docker compose up -d --wait --wait-timeout 90
|
||||
|
||||
@@ -34,7 +34,8 @@ for container in $old_containers; do
|
||||
done
|
||||
|
||||
docker compose build
|
||||
docker compose up -d postgres valkey
|
||||
docker compose up -d postgres valkey s3
|
||||
docker compose run --rm s3-bootstrap
|
||||
docker compose run --rm migrate
|
||||
|
||||
if [ "$old_count" -lt 2 ]; then
|
||||
|
||||
Reference in New Issue
Block a user