feat: secure messenger ingress and rate-limit logins

This commit is contained in:
Dmitrii Grishko
2026-11-14 19:24:51 +03:00
parent e3d7df4e72
commit 48e6a6ac0d
16 changed files with 337 additions and 76 deletions
+4 -4
View File
@@ -12,20 +12,23 @@ COMPOSE = docker compose --project-name highload-lab7 --env-file .env -f compose
.PHONY: app-init app-build app-up app-start app-stop app-down app-restart app-logs app-status
.PHONY: app-migrate app-s3-bootstrap worker-start worker-stop queue-status attachments-cleanup
.PHONY: jobs-failed jobs-replay jobs-upgrade jobs-cleanup-staging
.PHONY: test-backend test-pipeline frontend-build
.PHONY: test-backend test-pipeline
app-init:
$(PYTHON) scripts/init_env.py
./scripts/bootstrap_tls.sh
app-build: app-init
$(COMPOSE) build
app-up: app-init
$(COMPOSE) up -d --build --wait
./scripts/wait_ingress.sh
app-start: app-init
$(COMPOSE) up -d --no-build --wait
./scripts/wait_ingress.sh
app-stop:
$(COMPOSE) stop
@@ -80,6 +83,3 @@ test-backend: app-init
test-pipeline: app-init
$(COMPOSE) --profile test run --rm --build backend-tests go test -race -count=1 -v ./backend/tests ./backend/workers -run 'TestPipeline|TestVersionedRendering|TestThumbnail|TestOptimized'
frontend-build:
$(COMPOSE) exec -T frontend npm run build
+20 -56
View File
@@ -121,6 +121,9 @@ services:
# Go 1.22 sees host CPUs; keep parallelism suitable for this 0.5 CPU limit.
GOMAXPROCS: ${API_GOMAXPROCS:-1}
GOMEMLIMIT: 384MiB
TLS_CA_FILE: /run/tls/ca.crt
TLS_CERT_FILE: /run/tls/api.crt
TLS_KEY_FILE: /run/tls/api.key
APP_HOST: ${APP_HOST:-0.0.0.0}
APP_PORT: ${APP_PORT:-8080}
APP_ENV: ${APP_ENV:-lab}
@@ -132,12 +135,15 @@ services:
DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}?sslmode=disable
tmpfs:
- /tmp/uploads:rw,noexec,nosuid,size=16m,mode=1777
volumes: &api-volumes
- ./secrets/s3_api_access_key_id:${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id}:ro
- ./secrets/s3_api_secret_access_key:${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key}:ro
- ./secrets/broker_url:/run/secrets/broker_url:ro
- ./secrets/database_url:${DATABASE_URL_FILE:-/run/secrets/database_url}:ro
- ./secrets/cache_url:${CACHE_URL_FILE:-/run/secrets/cache_url}:ro
volumes:
- ./secrets/tls/lab7-ca.crt:/run/tls/ca.crt:ro
- ./secrets/tls/api.crt:/run/tls/api.crt:ro
- ./secrets/tls/api.key:/run/tls/api.key:ro
- &s3-api-id ./secrets/s3_api_access_key_id:${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id}:ro
- &s3-api-key ./secrets/s3_api_secret_access_key:${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key}:ro
- &broker-url ./secrets/broker_url:/run/secrets/broker_url:ro
- &db-url ./secrets/database_url:${DATABASE_URL_FILE:-/run/secrets/database_url}:ro
- &cache-url ./secrets/cache_url:${CACHE_URL_FILE:-/run/secrets/cache_url}:ro
depends_on:
s3-bootstrap:
condition: service_completed_successfully
@@ -148,10 +154,7 @@ services:
redis:
condition: service_healthy
healthcheck:
test: ["CMD", "./messenger", "healthcheck"]
interval: 5s
timeout: 3s
retries: 20
disable: true
broker:
image: rabbitmq:4.2.0-management
hostname: broker
@@ -187,7 +190,7 @@ services:
mem_limit: 512m
networks: [data, telemetry]
environment: *api-environment
volumes: *api-volumes
volumes: [*s3-api-id, *s3-api-key, *broker-url, *db-url, *cache-url]
command: ["./messenger", "worker"]
healthcheck:
disable: true
@@ -205,7 +208,7 @@ services:
mem_limit: 128m
networks: [data, telemetry]
environment: *api-environment
volumes: *api-volumes
volumes: [*s3-api-id, *s3-api-key, *broker-url, *db-url, *cache-url]
command: ["./messenger", "dispatcher"]
healthcheck:
disable: true
@@ -246,26 +249,15 @@ services:
networks: [edge, proxy]
image: haproxy:3.2.25-alpine
ports:
- "127.0.0.1:${API_HTTP_PORT:-8080}:8080"
- "127.0.0.1:8404:8404"
- "127.0.0.1:${API_HTTPS_PORT:-8443}:8443"
volumes:
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
- ./secrets/tls/lab7-ca.crt:/run/tls/ca.crt:ro
- ./secrets/tls/ingress.pem:/run/tls/ingress.pem:ro
- ./secrets/tls/lb.pem:/run/tls/lb.pem:ro
depends_on:
api:
condition: service_healthy
healthcheck:
test:
[
"CMD",
"wget",
"-q",
"-O",
"/dev/null",
"http://127.0.0.1:8080/health/ready",
]
interval: 2s
timeout: 3s
retries: 20
condition: service_started
backend-tests:
cpus: 2.0
mem_limit: 1g
@@ -292,34 +284,6 @@ services:
redis:
condition: service_healthy
command: ["go", "test", "-race", "-count=1", "-v", "./backend/..."]
frontend:
cpus: 0.5
mem_limit: 512m
networks: [edge]
build:
context: ./src/frontend
environment:
API_PROXY_TARGET: http://haproxy:8080
ports:
- "127.0.0.1:5173:5173"
volumes:
- ./src/frontend:/app
- /app/node_modules
depends_on:
haproxy:
condition: service_healthy
healthcheck:
test:
[
"CMD",
"node",
"-e",
"fetch('http://127.0.0.1:5173/').then(r => process.exit(r.ok ? 0 : 1)).catch(() => process.exit(1))",
]
interval: 5s
timeout: 3s
retries: 20
otel-collector:
image: otel/opentelemetry-collector-contrib:0.120.0
networks: [telemetry]
+3 -10
View File
@@ -30,7 +30,8 @@ resolvers docker
hold other 1s
frontend api_entry
bind :8080
bind :8443 ssl crt /run/tls/ingress.pem ssl-min-ver TLSv1.2
http-request set-header X-Forwarded-For %[src]
default_backend api_replicas
backend api_replicas
@@ -38,12 +39,4 @@ backend api_replicas
option httpchk GET /health/ready
http-check expect status 200
# Separate slots receive distinct Docker DNS addresses. No sticky sessions.
server-template api "${API_REPLICAS}" "api:${APP_PORT}" check inter 1s fall 2 rise 2 resolvers docker resolve-prefer ipv4 init-addr libc,none
# Local teaching aid: backend addresses, UP/DOWN state and request counters.
listen statistics
bind :8404
stats enable
stats show-legends
stats uri /stats
stats refresh 2s
server-template api "${API_REPLICAS}" "api:${APP_PORT}" ssl verify required ca-file /run/tls/ca.crt crt /run/tls/lb.pem verifyhost api ssl-min-ver TLSv1.2 check inter 1s fall 2 rise 2 resolvers docker resolve-prefer ipv4 init-addr libc,none
+45
View File
@@ -0,0 +1,45 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
umask 077
dir=secrets/tls
mkdir -p "$dir"
chmod 700 "$dir"
make_ca() {
name=$1
if [ ! -e "$dir/$name.crt" ]; then
openssl req -x509 -newkey ed25519 -nodes -days 365 \
-subj "/CN=$name" -addext "basicConstraints=critical,CA:TRUE,pathlen:0" \
-addext "keyUsage=critical,keyCertSign,cRLSign" \
-keyout "$dir/$name.key" -out "$dir/$name.crt"
fi
}
make_leaf() {
name=$1 ca=$2 usage=$3 san=$4
if [ ! -e "$dir/$name.crt" ]; then
openssl req -new -newkey ed25519 -nodes -subj "/CN=$name" \
-keyout "$dir/$name.key" -out "$dir/$name.csr"
printf 'basicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature\nextendedKeyUsage=%s\nsubjectAltName=%s\nsubjectKeyIdentifier=hash\nauthorityKeyIdentifier=keyid:always\n' \
"$usage" "$san" > "$dir/$name.ext"
openssl x509 -req -in "$dir/$name.csr" -CA "$dir/$ca.crt" -CAkey "$dir/$ca.key" \
-CAcreateserial -days 30 -extfile "$dir/$name.ext" -out "$dir/$name.crt"
rm "$dir/$name.ext" "$dir/$name.csr"
fi
# Containers can only read files individually mounted into them.
chmod 644 "$dir/$name.crt" "$dir/$name.key"
}
make_ca lab7-ca
make_ca untrusted-ca
chmod 644 "$dir/lab7-ca.crt"
make_leaf ingress lab7-ca serverAuth DNS:localhost,IP:127.0.0.1
make_leaf api lab7-ca serverAuth DNS:api
make_leaf lb lab7-ca clientAuth URI:urn:lab7:load-balancer
make_leaf wrong-role lab7-ca clientAuth URI:urn:lab7:probe
make_leaf wrong-ca untrusted-ca clientAuth URI:urn:lab7:load-balancer
for name in ingress lb; do
cat "$dir/$name.crt" "$dir/$name.key" > "$dir/$name.pem"
chmod 644 "$dir/$name.pem"
done
+56
View File
@@ -0,0 +1,56 @@
#!/usr/bin/env python3
"""Probe each API replica directly from the Docker host on Linux."""
import http.client
import json
import ssl
import socket
import subprocess
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
TLS = ROOT / "secrets/tls"
def check(address, port, identity, error):
context = ssl.create_default_context(cafile=str(TLS / "lab7-ca.crt"))
if identity:
context.load_cert_chain(TLS / f"{identity}.crt", TLS / f"{identity}.key")
with socket.create_connection((address, port), timeout=5) as tcp:
try:
with context.wrap_socket(tcp, server_hostname="api") as connection:
connection.sendall(b"GET /health/live HTTP/1.1\r\nHost: api\r\nConnection: close\r\n\r\n")
response = http.client.HTTPResponse(connection)
response.begin()
assert response.status == 200, response.status
except ssl.SSLError as exc:
if not error or error not in str(exc).lower():
raise
return
if error:
raise AssertionError(f"{identity or 'no certificate'} unexpectedly accepted")
def main():
config = subprocess.check_output(
["docker", "compose", "config", "--format", "json"], cwd=ROOT, text=True
)
port = int(json.loads(config)["services"]["api"]["environment"]["APP_PORT"])
replicas = subprocess.check_output(
["docker", "compose", "ps", "-q", "api"], cwd=ROOT, text=True
).split()
assert len(replicas) >= 2, "Expected at least two running API replicas"
for replica in replicas:
address = subprocess.check_output(
["docker", "inspect", "--format",
'{{(index .NetworkSettings.Networks "highload-lab7_proxy").IPAddress}}', replica],
text=True,
).strip()
check(address, port, "lb", None)
check(address, port, None, "certificate required")
check(address, port, "wrong-ca", "unknown ca")
check(address, port, "wrong-role", "bad certificate")
print(f"PASS {replica[:12]}: LB permitted; absent, foreign CA and disallowed SAN rejected")
if __name__ == "__main__":
main()
+40
View File
@@ -0,0 +1,40 @@
#!/usr/bin/env python3
"""Run against a quiet test ingress after at least 60 seconds without logins."""
import argparse
import ssl
from urllib.error import HTTPError
from urllib.request import Request, urlopen
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="https://localhost:8443")
parser.add_argument("--ca-file", default="secrets/tls/lab7-ca.crt")
args = parser.parse_args()
context = ssl.create_default_context(cafile=args.ca_file)
instances = set()
for attempt in range(1, 14):
request = Request(
args.base_url.rstrip("/") + "/api/v1/auth/sessions",
data=b"",
headers={"X-Forwarded-For": f"192.0.2.{attempt}"},
)
try:
with urlopen(request, context=context, timeout=5):
raise AssertionError("Missing credentials unexpectedly accepted")
except HTTPError as error:
with error:
instance = error.headers["X-Instance-Id"]
assert instance, "Missing API instance identity"
instances.add(instance)
if attempt <= 12:
assert error.code == 401, f"Attempt {attempt}: expected 401, got {error.code}"
else:
assert error.code == 429, f"Expected 429, got {error.code}"
assert 0 < int(error.headers["Retry-After"]) <= 60
assert len(instances) >= 2, "Expected requests to reach multiple API replicas"
print(f"PASS: request 13 returned 429 with Retry-After across {len(instances)} API replicas")
if __name__ == "__main__":
main()
+1 -1
View File
@@ -26,7 +26,7 @@ def main():
POSTGRES_USER="messenger",
POSTGRES_PASSWORD=secrets.token_hex(24),
)
defaults.update(API_HTTP_PORT="8080", S3_ENDPOINT="http://s3:9000",
defaults.update(API_HTTPS_PORT="8443", S3_ENDPOINT="http://s3:9000",
OTEL_EXPORTER_OTLP_ENDPOINT="http://otel-collector:4318", UPLOAD_CLEANUP_GRACE_SECONDS="600", UPLOAD_TMP_DIR="/tmp/uploads",
S3_PUBLIC_ENDPOINT="http://127.0.0.1:9000",
S3_CORS_ORIGINS="http://127.0.0.1:5173,http://localhost:5173")
+13
View File
@@ -0,0 +1,13 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
port=$(docker compose port haproxy 8443)
for attempt in $(seq 1 30); do
if curl --silent --fail --max-time 2 --cacert secrets/tls/lab7-ca.crt \
"https://localhost:${port##*:}/health/ready" >/dev/null; then
exit 0
fi
sleep 1
done
printf 'HTTPS ingress did not become ready\n' >&2
exit 1
+57
View File
@@ -0,0 +1,57 @@
# HTTPS и mTLS на входе
Для локального стенда нужны Docker Compose, OpenSSL, curl и Python 3.
`make -f app.mk app-up` создаёт сертификаты в `secrets/tls/`,
запускает два API и ждёт ответа `/health/ready` по HTTPS.
| Связь | Доверие и идентичность |
| --- | --- |
| Клиент → HAProxy, localhost:8443 | TLS 1.2+, локальный CA, SAN localhost и 127.0.0.1. Для бизнес-операций действует Bearer-сессия. |
| HAProxy → API, api:8080 | TLS 1.2+, тот же CA. HAProxy проверяет имя api. API проверяет цепочку, назначение clientAuth и URI SAN urn:lab7:load-balancer. |
| Readiness | HAProxy проверяет обе реплики через тот же mTLS listener. На хосте опубликован только порт балансировщика для API. |
Локальный CA действует 365 дней, сертификаты стенда — 30 дней.
Bootstrap сохраняет уже выпущенные ключи и сертификаты. CA key доступен
оператору в закрытом каталоге. В контейнеры монтируются отдельные нужные
файлы. Тестовый чужой CA и клиентские ключи остаются на хосте.
```sh
make test test-security CA_FILE=secrets/tls/lab7-ca.crt
python3 scripts/check_ingress_mtls.py
make test-persistence CA_FILE=secrets/tls/lab7-ca.crt ACTION_SCRIPT=scripts/restart.sh
make test-failover CA_FILE=secrets/tls/lab7-ca.crt ACTION_SCRIPT=scripts/stop-one.sh
docker compose start api
```
Проверка mTLS запускается на Linux-хосте с доступом к Docker bridge и
проверяет отдельно каждую реплику. Сертификат с правильной ролью и чужим CA,
сертификат со своим CA и чужой ролью и отсутствие сертификата приводят
к ошибкам TLS.
HAProxy перезаписывает `X-Forwarded-For` фактическим адресом соединения.
Redis атомарно считает попытки входа по этому адресу для обеих реплик.
После 12 запросов за окно в 60 секунд API возвращает 429 с `Retry-After`.
На тихом стенде после 60 секунд без входов выполните
`python3 scripts/check_login_rate.py`. Проверка исчерпывает лимит адреса
до конца текущего окна. Пользователи за одним NAT делят лимит.
Использованные настройки TLS описаны в руководстве
[HAProxy 3.2](https://docs.haproxy.org/3.2/configuration.html).
## Стоимость проверки пароля
Перенесённый Argon2id сохраняет параметры в PHC-строке вместе с солью.
Параметры текущей реализации: 64 МиБ, 3 прохода, параллелизм 2.
Среднее время в каждом из трёх прогонов по три операции составило
205, 223 и 290 мс. Go выделял около 64 МиБ на операцию. Замер выполнен
на AMD Ryzen 9 PRO 7940HS с ограничением контейнера 0,5 CPU и 512 МиБ,
`GOMAXPROCS=1`, на одном синтетическом пароле без HTTP и обращения к БД.
Это исходный замер для выбора параметров. Он не измеряет пропускную
способность при конкурентных входах.
```sh
docker compose --profile test build backend-tests
docker run --rm --network none --cpus 0.5 --memory 512m -e GOMAXPROCS=1 \
highload-lab7-backend-tests go test ./backend/services -run '^$' \
-bench '^BenchmarkPasswordVerification$' -benchtime=3x -count=3
```
-3
View File
@@ -34,9 +34,6 @@ ENV APP_PORT=8080
EXPOSE 8080
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \
CMD ["./messenger", "healthcheck"]
USER app:app
CMD ["./messenger"]
+7 -2
View File
@@ -35,6 +35,10 @@ func serverAddress() string {
}
func RunHTTP() error {
tlsConfig, err := ingressTLS()
if err != nil {
return err
}
if _, err := repositories.UploadCleanupGrace(); err != nil {
return err
}
@@ -100,6 +104,7 @@ func RunHTTP() error {
server := &http.Server{
Addr: serverAddress(),
TLSConfig: tlsConfig,
Handler: signals.HTTP(mux),
ReadHeaderTimeout: 5 * time.Second,
ReadTimeout: 30 * time.Second,
@@ -119,7 +124,7 @@ func RunHTTP() error {
go func() {
log.Printf("server is listening on %s", server.Addr)
if err := server.ListenAndServe(); err != nil &&
if err := server.ListenAndServeTLS(os.Getenv("TLS_CERT_FILE"), os.Getenv("TLS_KEY_FILE")); err != nil &&
!errors.Is(err, http.ErrServerClosed) {
serverErrors <- err
}
@@ -127,7 +132,7 @@ func RunHTTP() error {
select {
case err := <-serverErrors:
log.Printf("server error: %v", err)
return err
case <-shutdownContext.Done():
log.Printf("shutdown signal received")
+32
View File
@@ -0,0 +1,32 @@
package app
import (
"crypto/tls"
"crypto/x509"
"errors"
"os"
)
func ingressTLS() (*tls.Config, error) {
ca, err := os.ReadFile(os.Getenv("TLS_CA_FILE"))
if err != nil {
return nil, err
}
roots := x509.NewCertPool()
if !roots.AppendCertsFromPEM(ca) {
return nil, errors.New("invalid ingress client CA")
}
return &tls.Config{
MinVersion: tls.VersionTLS12,
ClientAuth: tls.RequireAndVerifyClientCert,
ClientCAs: roots,
VerifyConnection: func(state tls.ConnectionState) error {
for _, identity := range state.PeerCertificates[0].URIs {
if identity.String() == "urn:lab7:load-balancer" {
return nil
}
}
return errors.New("client certificate does not identify the load balancer")
},
}, nil
}
+20
View File
@@ -3,7 +3,9 @@ package handlers
import (
"context"
"errors"
"net"
"net/http"
"strconv"
"strings"
"time"
@@ -57,6 +59,24 @@ func sessionHandler(sessions *services.SessionService) http.HandlerFunc {
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
return
}
// HAProxy overwrites this header and is the only mTLS identity allowed to reach the API.
if r.TLS != nil {
source := net.ParseIP(r.Header.Get("X-Forwarded-For"))
if source == nil {
writeError(w, 503, "unavailable", "missing ingress client address", requestID)
return
}
retryAfter, err := sessions.LoginRetryAfter(r.Context(), source.String())
if err != nil {
writeError(w, 503, "unavailable", "dependency unavailable", requestID)
return
}
if retryAfter > 0 {
w.Header().Set("Retry-After", strconv.Itoa(retryAfter))
writeError(w, 429, "rate_limited", "too many login attempts", requestID)
return
}
}
username, password, ok := r.BasicAuth()
if !ok || r.ContentLength > 0 {
unauthorized(w, "Basic", requestID)
+16
View File
@@ -54,3 +54,19 @@ func (s *SessionStore) Revoke(ctx context.Context, tokenHash []byte) error {
func (s *SessionStore) Ping(ctx context.Context) error {
return s.client.Ping(ctx).Err()
}
// Apply the expiry and increment atomically across replicas.
var loginWindow = redis.NewScript(`
local count = redis.call('INCR', KEYS[1])
if count == 1 then
redis.call('EXPIRE', KEYS[1], 60)
end
if count <= 12 then
return 0
end
return math.max(1, redis.call('TTL', KEYS[1]))
`)
func (s *SessionStore) LoginRetryAfter(ctx context.Context, source string) (int, error) {
return loginWindow.Run(ctx, s.client, []string{"messenger:login:" + source}).Int()
}
@@ -0,0 +1,19 @@
package services
import "testing"
func BenchmarkPasswordVerification(b *testing.B) {
const password = "synthetic-benchmark-password"
hash, err := HashPassword(password)
if err != nil {
b.Fatal(err)
}
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
ok, err := VerifyPassword(password, hash)
if err != nil || !ok {
b.Fatalf("verification failed: %v", err)
}
}
}
+4
View File
@@ -81,3 +81,7 @@ func (s *SessionService) Revoke(ctx context.Context, token string) error {
hash := sha256.Sum256([]byte(token))
return s.sessions.Revoke(ctx, hash[:])
}
func (s *SessionService) LoginRetryAfter(ctx context.Context, source string) (int, error) {
return s.sessions.LoginRetryAfter(ctx, source)
}