forked from mtusi2026/mtusi2026lab3
feat:level4
This commit is contained in:
+1
-1
@@ -26,6 +26,6 @@ USER app
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=10s --timeout=3s --start-period=20s --retries=5 \
|
||||
CMD wget -qO- http://127.0.0.1:8080/health/live >/dev/null || exit 1
|
||||
CMD wget -qO- http://127.0.0.1:8080/health/ready >/dev/null || exit 1
|
||||
|
||||
ENTRYPOINT ["/app/messenger"]
|
||||
@@ -4,7 +4,7 @@ CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover
|
||||
.PHONY: help check validate test load test-persistence test-failover my_tests
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@@ -28,3 +28,9 @@ test-persistence:
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
my_tests:
|
||||
$(PYTHON) student_tests/test_limits.py
|
||||
$(PYTHON) student_tests/test_idempotency.py
|
||||
$(PYTHON) student_tests/test_failover_both.py
|
||||
$(PYTHON) student_tests/test_rolling.py
|
||||
@@ -50,7 +50,7 @@ func main() {
|
||||
|
||||
sessions := domain.NewSessionStore(store, cache, cfg.SessionTTL)
|
||||
handler := &api.Handler{Store: store, Sessions: sessions}
|
||||
router := api.NewRouter(handler)
|
||||
router := api.NewRouter(handler, cfg.InstanceID)
|
||||
|
||||
srv := &http.Server{
|
||||
Addr: fmt.Sprintf("%s:%d", cfg.Host, cfg.Port),
|
||||
@@ -66,6 +66,7 @@ func main() {
|
||||
|
||||
go func() {
|
||||
log.Printf("messenger listening on %s (env=%s, session_ttl=%s)", srv.Addr, cfg.Env, cfg.SessionTTL)
|
||||
log.Printf("instance_id=%s", cfg.InstanceID)
|
||||
if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
log.Fatalf("server error: %v", err)
|
||||
}
|
||||
@@ -73,7 +74,7 @@ func main() {
|
||||
|
||||
<-stop
|
||||
log.Println("shutting down...")
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
_ = srv.Shutdown(ctx)
|
||||
log.Println("stopped")
|
||||
|
||||
+79
-17
@@ -1,3 +1,34 @@
|
||||
x-api-base: &api-base
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
env_file:
|
||||
- .env
|
||||
environment:
|
||||
APP_HOST: "0.0.0.0"
|
||||
APP_PORT: "8080"
|
||||
APP_ENV: "lab"
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
SESSION_STORE: "postgres"
|
||||
SESSION_TTL_SECONDS: "3600"
|
||||
secrets:
|
||||
- database_url
|
||||
- cache_url
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
networks: [backend]
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.75"
|
||||
memory: 256M
|
||||
restart_policy:
|
||||
condition: on-failure
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16.3-alpine
|
||||
@@ -15,6 +46,11 @@ services:
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
networks: [backend]
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 512M
|
||||
|
||||
redis:
|
||||
image: redis:7.2.5-alpine
|
||||
@@ -27,30 +63,35 @@ services:
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
networks: [backend]
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 256M
|
||||
|
||||
api:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
env_file:
|
||||
- .env
|
||||
api1:
|
||||
<<: *api-base
|
||||
environment:
|
||||
APP_HOST: "0.0.0.0"
|
||||
APP_PORT: "8080"
|
||||
APP_ENV: "lab"
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
secrets:
|
||||
- database_url
|
||||
- cache_url
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
ports:
|
||||
- "8080:8080"
|
||||
networks: [backend]
|
||||
SESSION_STORE: "postgres"
|
||||
SESSION_TTL_SECONDS: "3600"
|
||||
INSTANCE_ID: "api-1"
|
||||
|
||||
api2:
|
||||
<<: *api-base
|
||||
environment:
|
||||
APP_HOST: "0.0.0.0"
|
||||
APP_PORT: "8080"
|
||||
APP_ENV: "lab"
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
SESSION_STORE: "postgres"
|
||||
SESSION_TTL_SECONDS: "3600"
|
||||
INSTANCE_ID: "api-2"
|
||||
|
||||
migrate:
|
||||
build:
|
||||
@@ -67,6 +108,27 @@ services:
|
||||
networks: [backend]
|
||||
restart: "no"
|
||||
|
||||
lb:
|
||||
image: nginx:1.27.0-alpine
|
||||
volumes:
|
||||
- ./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
|
||||
ports:
|
||||
- "8080:8080"
|
||||
depends_on:
|
||||
- api1
|
||||
- api2
|
||||
networks: [backend]
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8080/health/live"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 128M
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
redisdata:
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
worker_processes auto;
|
||||
events { worker_connections 1024; }
|
||||
|
||||
http {
|
||||
upstream api_backend {
|
||||
server api1:8080 max_fails=2 fail_timeout=5s;
|
||||
server api2:8080 max_fails=2 fail_timeout=5s;
|
||||
keepalive 32;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 8080;
|
||||
|
||||
location / {
|
||||
proxy_pass http://api_backend;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "";
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
|
||||
proxy_connect_timeout 2s;
|
||||
proxy_send_timeout 15s;
|
||||
proxy_read_timeout 15s;
|
||||
|
||||
proxy_next_upstream error timeout http_502 http_503 http_504;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
CREATE TABLE IF NOT EXISTS idempotency_keys (
|
||||
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
method TEXT NOT NULL,
|
||||
path TEXT NOT NULL,
|
||||
key TEXT NOT NULL,
|
||||
request_hash TEXT NOT NULL,
|
||||
status_code INTEGER NOT NULL,
|
||||
response JSONB NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (user_id, method, path, key)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_idempotency_created
|
||||
ON idempotency_keys (created_at);
|
||||
@@ -0,0 +1,14 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Перезапуск API + всех state stores, volumes сохраняются.
|
||||
docker compose restart postgres redis api1 api2 lb
|
||||
# Ждём готовности.
|
||||
for i in $(seq 1 60); do
|
||||
if docker compose exec -T lb wget -qO- http://127.0.0.1:8080/health/ready >/dev/null 2>&1; then
|
||||
echo "ready after ${i}s"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "stack did not become ready in 60s" >&2
|
||||
exit 1
|
||||
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?}"
|
||||
case "$TARGET_INSTANCE_ID" in
|
||||
api-1) SERVICE=api1 ;;
|
||||
api-2) SERVICE=api2 ;;
|
||||
*) exit 2 ;;
|
||||
esac
|
||||
docker compose start "$SERVICE"
|
||||
@@ -0,0 +1,15 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# TARGET_INSTANCE_ID приходит от теста: alias обслужившей реплики.
|
||||
# В compose alias = INSTANCE_ID = "api-1" / "api-2".
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
|
||||
case "$TARGET_INSTANCE_ID" in
|
||||
api-1) SERVICE=api1 ;;
|
||||
api-2) SERVICE=api2 ;;
|
||||
*) echo "unknown instance: $TARGET_INSTANCE_ID" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
# Останавливаем ТОЛЬКО эту реплику, оставляем остановленной.
|
||||
docker compose stop "$SERVICE"
|
||||
echo "stopped $SERVICE ($TARGET_INSTANCE_ID)"
|
||||
+43
-3
@@ -1,8 +1,10 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strconv"
|
||||
@@ -12,6 +14,7 @@ import (
|
||||
)
|
||||
|
||||
var usernameRe = regexp.MustCompile(`^[a-z0-9_]{3,32}$`)
|
||||
var idempotencyKeyRe = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`)
|
||||
|
||||
type Handler struct {
|
||||
Store *domain.Store
|
||||
@@ -224,12 +227,31 @@ func (h *Handler) CreateMessage(w http.ResponseWriter, r *http.Request) {
|
||||
h.bearerAuth(func(w http.ResponseWriter, r *http.Request) {
|
||||
me, _ := userFrom(r)
|
||||
chatID := r.PathValue("chat_id")
|
||||
|
||||
// Idempotency-Key обязателен.
|
||||
key := r.Header.Get("Idempotency-Key")
|
||||
if key == "" {
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "Idempotency-Key required")
|
||||
return
|
||||
}
|
||||
if !idempotencyKeyRe.MatchString(key) {
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid Idempotency-Key")
|
||||
return
|
||||
}
|
||||
|
||||
if _, err := h.Store.GetChat(r.Context(), chatID, me.ID); err != nil {
|
||||
writeError(w, r, http.StatusNotFound, "not_found", "chat not found")
|
||||
return
|
||||
}
|
||||
|
||||
// Читаем тело один раз и хешируем нормализованный JSON.
|
||||
rawBody, err := io.ReadAll(io.LimitReader(r.Body, 64*1024))
|
||||
if err != nil {
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "cannot read body")
|
||||
return
|
||||
}
|
||||
var req createMessageReq
|
||||
dec := json.NewDecoder(r.Body)
|
||||
dec := json.NewDecoder(bytes.NewReader(rawBody))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&req); err != nil {
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "malformed JSON")
|
||||
@@ -243,13 +265,23 @@ func (h *Handler) CreateMessage(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "empty message")
|
||||
return
|
||||
}
|
||||
m, err := h.Store.AddMessage(r.Context(), chatID, me.ID, req.Text)
|
||||
bodyHash, err := domain.HashBody(rawBody)
|
||||
if err != nil {
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "malformed JSON")
|
||||
return
|
||||
}
|
||||
|
||||
msg, rec, replay, err := h.Store.AddMessageIdempotent(
|
||||
r.Context(), chatID, me.ID, req.Text, key, bodyHash)
|
||||
switch {
|
||||
case err == nil:
|
||||
// ok
|
||||
case errors.Is(err, domain.ErrNotFound):
|
||||
writeError(w, r, http.StatusNotFound, "not_found", "chat not found")
|
||||
return
|
||||
case errors.Is(err, domain.ErrConflict):
|
||||
writeError(w, r, http.StatusConflict, "conflict", "idempotency key reused with different body")
|
||||
return
|
||||
case errors.Is(err, domain.ErrTimeout):
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database timeout")
|
||||
return
|
||||
@@ -257,7 +289,15 @@ func (h *Handler) CreateMessage(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, m)
|
||||
|
||||
if replay {
|
||||
// Возвращаем сохранённый 201 и то же Message.
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(rec.StatusCode)
|
||||
_, _ = w.Write(rec.Response)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, msg)
|
||||
})(w, r)
|
||||
}
|
||||
|
||||
|
||||
@@ -33,3 +33,14 @@ func withRequestID(next http.Handler) http.Handler {
|
||||
next.ServeHTTP(w, r.WithContext(ctx))
|
||||
})
|
||||
}
|
||||
|
||||
// withInstanceID ставит X-Instance-Id на КАЖДЫЙ ответ (включая ошибки и health).
|
||||
// ID генерируется один раз при старте процесса — стабилен в пределах жизни.
|
||||
func withInstanceID(instanceID string) func(http.Handler) http.Handler {
|
||||
return func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("X-Instance-Id", instanceID)
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+4
-2
@@ -2,7 +2,7 @@ package api
|
||||
|
||||
import "net/http"
|
||||
|
||||
func NewRouter(h *Handler) http.Handler {
|
||||
func NewRouter(h *Handler, instanceID string) http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("GET /health/live", h.Live)
|
||||
mux.HandleFunc("GET /health/ready", h.Ready)
|
||||
@@ -15,5 +15,7 @@ func NewRouter(h *Handler) http.Handler {
|
||||
mux.HandleFunc("GET /api/v1/chats/{chat_id}", h.GetChat)
|
||||
mux.HandleFunc("POST /api/v1/chats/{chat_id}/messages", h.CreateMessage)
|
||||
mux.HandleFunc("GET /api/v1/chats/{chat_id}/messages", h.ListMessages)
|
||||
return withRequestID(mux)
|
||||
|
||||
// Порядок: instanceID снаружи, requestID внутри.
|
||||
return withInstanceID(instanceID)(withRequestID(mux))
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
@@ -15,6 +17,7 @@ type Config struct {
|
||||
SessionTTL time.Duration
|
||||
SessionStore string // "postgres" | "redis"
|
||||
RedisRequired bool
|
||||
InstanceID string
|
||||
}
|
||||
|
||||
func getenv(key, def string) string {
|
||||
@@ -59,6 +62,11 @@ func Load() Config {
|
||||
cacheURL = readSecretFromFile(os.Getenv("CACHE_URL_FILE"))
|
||||
}
|
||||
|
||||
instanceID := os.Getenv("INSTANCE_ID")
|
||||
if instanceID == "" {
|
||||
instanceID = "api-" + randomShortID()
|
||||
}
|
||||
|
||||
store := getenv("SESSION_STORE", "postgres")
|
||||
return Config{
|
||||
Host: getenv("APP_HOST", "0.0.0.0"),
|
||||
@@ -69,5 +77,14 @@ func Load() Config {
|
||||
SessionTTL: time.Duration(ttl) * time.Second,
|
||||
SessionStore: store,
|
||||
RedisRequired: store == "redis",
|
||||
InstanceID: instanceID,
|
||||
}
|
||||
}
|
||||
|
||||
func randomShortID() string {
|
||||
b := make([]byte, 6)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
return "unknown"
|
||||
}
|
||||
return hex.EncodeToString(b)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// IdempotencyRecord — сохранённый результат POST.
|
||||
type IdempotencyRecord struct {
|
||||
StatusCode int
|
||||
Response json.RawMessage
|
||||
}
|
||||
|
||||
// LookupIdempotency возвращает запись, если ключ уже использован.
|
||||
// Второй аргумент — совпадает ли хеш тела с сохранённым.
|
||||
func (s *Store) LookupIdempotency(ctx context.Context, userID, method, path, key, bodyHash string) (*IdempotencyRecord, bool, error) {
|
||||
var (
|
||||
storedHash string
|
||||
rec IdempotencyRecord
|
||||
)
|
||||
err := s.pool.QueryRow(ctx,
|
||||
`SELECT request_hash, status_code, response
|
||||
FROM idempotency_keys
|
||||
WHERE user_id=$1 AND method=$2 AND path=$3 AND key=$4`,
|
||||
userID, method, path, key,
|
||||
).Scan(&storedHash, &rec.StatusCode, &rec.Response)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, false, classifyDBError(err)
|
||||
}
|
||||
if storedHash != bodyHash {
|
||||
return nil, true, ErrConflict
|
||||
}
|
||||
return &rec, true, nil
|
||||
}
|
||||
|
||||
// SaveIdempotency пишет результат. Конфликт PK означает, что параллельный
|
||||
// запрос успел раньше — вызывающий должен перечитать через LookupIdempotency.
|
||||
func (s *Store) SaveIdempotency(ctx context.Context, userID, method, path, key, bodyHash string, status int, resp json.RawMessage) error {
|
||||
_, err := s.pool.Exec(ctx,
|
||||
`INSERT INTO idempotency_keys (user_id, method, path, key, request_hash, status_code, response)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7)`,
|
||||
userID, method, path, key, bodyHash, status, resp)
|
||||
return classifyDBError(err)
|
||||
}
|
||||
|
||||
// HashBody — стабильный хеш JSON независимо от порядка ключей.
|
||||
func HashBody(raw []byte) (string, error) {
|
||||
var v any
|
||||
if err := json.Unmarshal(raw, &v); err != nil {
|
||||
return "", err
|
||||
}
|
||||
normalized, err := json.Marshal(v) // map keys сортируются
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
sum := sha256.Sum256(normalized)
|
||||
return hex.EncodeToString(sum[:]), nil
|
||||
}
|
||||
|
||||
// CleanupIdempotency — можно звать из фонового тикера (опционально).
|
||||
func (s *Store) CleanupIdempotency(ctx context.Context, olderThan time.Duration) error {
|
||||
_, err := s.pool.Exec(ctx,
|
||||
`DELETE FROM idempotency_keys WHERE created_at < now() - $1::interval`,
|
||||
olderThan.String())
|
||||
return err
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"crypto/rand"
|
||||
"crypto/subtle"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
@@ -443,3 +444,95 @@ func classifyDBError(err error) error {
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// AddMessageIdempotent — атомарно проверяет членство, вставляет сообщение
|
||||
// и сохраняет результат идемпотентности в одной транзакции.
|
||||
// Если ключ уже есть с тем же body — возвращает сохранённый ответ и replay=true.
|
||||
func (s *Store) AddMessageIdempotent(
|
||||
ctx context.Context,
|
||||
chatID, senderID, text, key, bodyHash string,
|
||||
) (msg *Message, rec *IdempotencyRecord, replay bool, err error) {
|
||||
path := "/api/v1/chats/" + chatID + "/messages"
|
||||
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.Serializable})
|
||||
if err != nil {
|
||||
return nil, nil, false, classifyDBError(err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
// 1) Членство.
|
||||
var ok bool
|
||||
if err := tx.QueryRow(ctx,
|
||||
`SELECT true FROM chat_members WHERE chat_id=$1 AND user_id=$2`,
|
||||
chatID, senderID,
|
||||
).Scan(&ok); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, nil, false, ErrNotFound
|
||||
}
|
||||
return nil, nil, false, classifyDBError(err)
|
||||
}
|
||||
|
||||
// 2) Уже был такой ключ?
|
||||
var storedHash string
|
||||
var storedStatus int
|
||||
var storedResp json.RawMessage
|
||||
err = tx.QueryRow(ctx,
|
||||
`SELECT request_hash, status_code, response
|
||||
FROM idempotency_keys
|
||||
WHERE user_id=$1 AND method='POST' AND path=$2 AND key=$3`,
|
||||
senderID, path, key,
|
||||
).Scan(&storedHash, &storedStatus, &storedResp)
|
||||
switch {
|
||||
case err == nil:
|
||||
if storedHash != bodyHash {
|
||||
return nil, nil, false, ErrConflict
|
||||
}
|
||||
return nil, &IdempotencyRecord{StatusCode: storedStatus, Response: storedResp}, true, nil
|
||||
case !errors.Is(err, pgx.ErrNoRows):
|
||||
return nil, nil, false, classifyDBError(err)
|
||||
}
|
||||
|
||||
// 3) Вставка сообщения.
|
||||
var m Message
|
||||
err = tx.QueryRow(ctx,
|
||||
`INSERT INTO messages (id, chat_id, sender_id, text)
|
||||
VALUES ($1,$2,$3,$4)
|
||||
RETURNING id, chat_id, sender_id, text, created_at`,
|
||||
uuid.NewString(), chatID, senderID, text,
|
||||
).Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt)
|
||||
if err != nil {
|
||||
// Гонка на одном ключе в разных репликах: unique violation в idempotency_keys
|
||||
// маловероятен здесь, но serialization failure возможен.
|
||||
return nil, nil, false, classifyDBError(err)
|
||||
}
|
||||
|
||||
// 4) Сохранение ответа.
|
||||
respJSON, _ := json.Marshal(m)
|
||||
_, err = tx.Exec(ctx,
|
||||
`INSERT INTO idempotency_keys (user_id, method, path, key, request_hash, status_code, response)
|
||||
VALUES ($1,'POST',$2,$3,$4,201,$5)`,
|
||||
senderID, path, key, bodyHash, respJSON)
|
||||
if err != nil {
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) && pgErr.Code == "23505" {
|
||||
// Кто-то параллельно успел вставить: перечитываем в этой же tx.
|
||||
if err2 := tx.QueryRow(ctx,
|
||||
`SELECT request_hash, status_code, response
|
||||
FROM idempotency_keys
|
||||
WHERE user_id=$1 AND method='POST' AND path=$2 AND key=$3`,
|
||||
senderID, path, key,
|
||||
).Scan(&storedHash, &storedStatus, &storedResp); err2 == nil {
|
||||
if storedHash != bodyHash {
|
||||
return nil, nil, false, ErrConflict
|
||||
}
|
||||
return nil, &IdempotencyRecord{StatusCode: storedStatus, Response: storedResp}, true, nil
|
||||
}
|
||||
}
|
||||
return nil, nil, false, classifyDBError(err)
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return nil, nil, false, classifyDBError(err)
|
||||
}
|
||||
return &m, nil, false, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
"""Общие утилиты для студенческих тестов."""
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
|
||||
from client import Client, Failure, check # noqa: E402
|
||||
|
||||
BASE_URL = os.environ.get('BASE_URL', 'http://localhost:8080')
|
||||
COMPOSE_CMD = os.environ.get('COMPOSE_CMD', 'docker compose')
|
||||
COMPOSE_PROJECT = os.environ.get('COMPOSE_PROJECT', '')
|
||||
|
||||
|
||||
def compose(*args, timeout=120, check_rc=True):
|
||||
"""Вызов docker compose. Возвращает CompletedProcess."""
|
||||
cmd = COMPOSE_CMD.split()
|
||||
if COMPOSE_PROJECT:
|
||||
cmd += ['-p', COMPOSE_PROJECT]
|
||||
cmd += list(args)
|
||||
proc = subprocess.run(
|
||||
cmd, cwd=str(ROOT), capture_output=True, text=True, timeout=timeout)
|
||||
if check_rc and proc.returncode != 0:
|
||||
raise Failure(
|
||||
'compose %s failed rc=%d\nstdout=%s\nstderr=%s'
|
||||
% (' '.join(args), proc.returncode, proc.stdout[-2000:], proc.stderr[-2000:]))
|
||||
return proc
|
||||
|
||||
|
||||
def wait_ready(deadline_seconds=90):
|
||||
c = Client(BASE_URL, timeout=2)
|
||||
start = time.monotonic()
|
||||
while time.monotonic() - start < deadline_seconds:
|
||||
try:
|
||||
c.call('GET', '/health/ready', schema='Health')
|
||||
return
|
||||
except Failure:
|
||||
time.sleep(0.5)
|
||||
raise Failure('LB did not become ready in %ds' % deadline_seconds)
|
||||
|
||||
|
||||
def observe_instances(c, actor, n=40):
|
||||
"""Собираем X-Instance-Id через /users/me."""
|
||||
seen = {}
|
||||
for _ in range(n):
|
||||
r = c.call('GET', '/api/v1/users/me', actor=actor, schema='User')
|
||||
iid = r['headers'].get('X-Instance-Id')
|
||||
check(iid, 'X-Instance-Id missing')
|
||||
seen[iid] = seen.get(iid, 0) + 1
|
||||
return seen
|
||||
|
||||
|
||||
def run_file(path, *args, timeout=180):
|
||||
proc = subprocess.run(
|
||||
[sys.executable, str(path), *args],
|
||||
cwd=str(ROOT), capture_output=True, text=True, timeout=timeout)
|
||||
return proc
|
||||
|
||||
|
||||
def print_pass(name, extra=''):
|
||||
print('PASS: ' + name + (' ' + extra if extra else ''))
|
||||
@@ -0,0 +1,87 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Останавливает КАЖДУЮ реплику по очереди и проверяет восстановление."""
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from common import (
|
||||
BASE_URL, Client, Failure, check, compose, observe_instances, print_pass,
|
||||
wait_ready,
|
||||
)
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def stop_one(instance_alias):
|
||||
env = {'TARGET_INSTANCE_ID': instance_alias, 'PATH': __import__('os').environ['PATH']}
|
||||
proc = subprocess.run(
|
||||
[str(ROOT / 'scripts/stop-one.sh')],
|
||||
cwd=str(ROOT), env={**__import__('os').environ, **env},
|
||||
capture_output=True, text=True, timeout=30)
|
||||
check(proc.returncode == 0,
|
||||
'stop-one.sh для %s упал rc=%d\nstderr=%s'
|
||||
% (instance_alias, proc.returncode, proc.stderr[-1000:]))
|
||||
|
||||
|
||||
def restore_all():
|
||||
"""Возвращает все реплики в running."""
|
||||
for svc in ('api1', 'api2'):
|
||||
compose('start', svc, check_rc=False)
|
||||
wait_ready(90)
|
||||
|
||||
|
||||
def main():
|
||||
c = Client(BASE_URL)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
seen = observe_instances(c, alice, n=40)
|
||||
check(len(seen) >= 2, 'видна одна реплика: %r' % seen)
|
||||
print(' instances:', seen)
|
||||
|
||||
for alias in sorted(seen):
|
||||
print('--- failover test for %s' % alias)
|
||||
target_alias = alias
|
||||
stop_one(target_alias)
|
||||
time.sleep(1)
|
||||
|
||||
# Ни одна последующая успешная запись не должна обслуживаться target.
|
||||
first_ok = None
|
||||
after = []
|
||||
deadline = time.monotonic() + 15
|
||||
key = 'fo-' + __import__('uuid').uuid4().hex
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
r = c.message(alice, chat, 'after-stop-' + key, key=key)
|
||||
if first_ok is None:
|
||||
first_ok = time.monotonic()
|
||||
after.append(r['headers']['X-Instance-Id'])
|
||||
key = 'fo-' + __import__('uuid').uuid4().hex
|
||||
except Failure:
|
||||
pass
|
||||
time.sleep(0.2)
|
||||
check(first_ok is not None, 'запись не восстановилась за 15с')
|
||||
check(len(after) >= 5, 'слишком мало успешных записей после отказа: %d' % len(after))
|
||||
check(target_alias not in after[-5:],
|
||||
'остановленная реплика %s всё ещё отвечает' % target_alias)
|
||||
print(' after stop %s: %d writes, последние instances=%r'
|
||||
% (target_alias, len(after), after[-3:]))
|
||||
|
||||
# Восстановить и подождать, пока LB снова начнёт её использовать.
|
||||
restore_all()
|
||||
time.sleep(2)
|
||||
seen2 = observe_instances(c, alice, n=60)
|
||||
print(' после restore видно:', seen2)
|
||||
|
||||
# В конце — обе реплики снова живы.
|
||||
seen3 = observe_instances(c, alice, n=60)
|
||||
check(len(seen3) >= 2, 'после restore видна одна реплика: %r' % seen3)
|
||||
print_pass('failover both replicas')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except Failure as e:
|
||||
print('FAIL:', e)
|
||||
raise SystemExit(1)
|
||||
@@ -0,0 +1,88 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Проверки идемпотентности и X-Instance-Id (лаб3)."""
|
||||
import json
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from common import (
|
||||
BASE_URL, Client, Failure, check, observe_instances, print_pass,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
c = Client(BASE_URL)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
path = '/api/v1/chats/' + chat['id'] + '/messages'
|
||||
|
||||
# 1) Через LB видно ≥2 реплики.
|
||||
seen = observe_instances(c, alice, n=40)
|
||||
check(len(seen) >= 2, 'менее двух X-Instance-Id через LB: %r' % seen)
|
||||
print(' instances:', seen)
|
||||
|
||||
# 2) Повтор с тем же ключом и тем же телом → тот же Message.
|
||||
key = 'idem-' + uuid.uuid4().hex
|
||||
body = {'text': 'idem-verify-' + uuid.uuid4().hex}
|
||||
r1 = c.call('POST', path, 201, actor=alice, body=body,
|
||||
headers={'Idempotency-Key': key}, schema='Message')
|
||||
r2 = c.call('POST', path, 201, actor=alice, body=body,
|
||||
headers={'Idempotency-Key': key}, schema='Message')
|
||||
check(r1['json']['id'] == r2['json']['id'], 'replay дал другой id')
|
||||
check(r1['json'] == r2['json'], 'replay дал другое тело')
|
||||
|
||||
# 3) Тот же ключ, другое тело → 409.
|
||||
c.call('POST', path, 409, actor=alice, body={'text': 'other'},
|
||||
headers={'Idempotency-Key': key})
|
||||
|
||||
# 4) Нет ключа → 400.
|
||||
c.call('POST', path, 400, actor=alice, body={'text': 'no key'})
|
||||
|
||||
# 5) Невалидный ключ (пробел) → 400.
|
||||
c.call('POST', path, 400, actor=alice, body={'text': 'bad key'},
|
||||
headers={'Idempotency-Key': 'has space'})
|
||||
|
||||
# 6) Разные пользователи с одинаковым ключом — независимы.
|
||||
key_shared = 'shared-' + uuid.uuid4().hex
|
||||
ra = c.message(alice, chat, 'alice-' + uuid.uuid4().hex, key=key_shared)
|
||||
rb = c.message(bob, chat, 'bob-' + uuid.uuid4().hex, key=key_shared)
|
||||
check(ra['json']['id'] != rb['json']['id'],
|
||||
'одинаковый ключ у разных пользователей склеил сообщения')
|
||||
|
||||
# 7) Конкурентный повтор: 20 параллельных POST с одним ключом.
|
||||
key_c = 'conc-' + uuid.uuid4().hex
|
||||
text_c = 'concurrent-' + uuid.uuid4().hex
|
||||
|
||||
def send(_):
|
||||
local = Client(BASE_URL, timeout=15)
|
||||
# Разрешаем 201 или 503 (transient serialization failure).
|
||||
r = local.call('POST', path, (201, 503), actor=alice,
|
||||
body={'text': text_c},
|
||||
headers={'Idempotency-Key': key_c})
|
||||
payload = r['json']
|
||||
if payload is None and r['status'] == 201:
|
||||
try:
|
||||
payload = json.loads(r['bytes'])
|
||||
except ValueError:
|
||||
payload = None
|
||||
return {'status': r['status'], 'json': payload}
|
||||
|
||||
with ThreadPoolExecutor(max_workers=20) as pool:
|
||||
results = list(pool.map(send, range(20)))
|
||||
|
||||
ok = [r for r in results if r['status'] == 201 and r['json']]
|
||||
check(len(ok) >= 15, 'слишком много не-201 при конкурентном повторе: %d/20'
|
||||
% len(ok))
|
||||
ids = {r['json']['id'] for r in ok}
|
||||
check(len(ids) == 1,
|
||||
'конкурентный повтор создал %d разных сообщений: %r'
|
||||
% (len(ids), ids))
|
||||
print(' concurrent: %d×201, один id' % len(ok))
|
||||
|
||||
print_pass('idempotency')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except Failure as e:
|
||||
print('FAIL:', e)
|
||||
raise SystemExit(1)
|
||||
@@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Проверяет, что контейнеры имеют CPU/RAM лимиты и api не публикует порт."""
|
||||
import json
|
||||
|
||||
from common import Failure, check, compose, print_pass
|
||||
|
||||
|
||||
def main():
|
||||
raw = compose('config', '--format', 'json').stdout
|
||||
cfg = json.loads(raw)
|
||||
services = cfg['services']
|
||||
|
||||
for svc in ('api1', 'api2'):
|
||||
check(svc in services, 'сервис %s отсутствует' % svc)
|
||||
ports = services[svc].get('ports', [])
|
||||
check(not ports, '%s публикует порты наружу: %r — должно быть только через LB'
|
||||
% (svc, ports))
|
||||
deploy = services[svc].get('deploy', {})
|
||||
res = deploy.get('resources', {}).get('limits', {})
|
||||
check('cpus' in res and 'memory' in res,
|
||||
'%s без deploy.resources.limits cpus/memory: %r' % (svc, res))
|
||||
|
||||
check('lb' in services, 'нет сервиса lb')
|
||||
lb_ports = services['lb'].get('ports', [])
|
||||
check(any('8080' in str(p) for p in lb_ports),
|
||||
'lb не публикует 8080: %r' % lb_ports)
|
||||
|
||||
check('container_name' not in services.get('api1', {}),
|
||||
'api1 задаёт container_name — ломает масштабирование')
|
||||
check('container_name' not in services.get('api2', {}),
|
||||
'api2 задаёт container_name — ломает масштабирование')
|
||||
|
||||
print_pass('limits & topology')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except Failure as e:
|
||||
print('FAIL:', e)
|
||||
raise SystemExit(1)
|
||||
@@ -0,0 +1,79 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Rolling update одной реплики под нагрузкой (уровень 4)."""
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from common import (
|
||||
BASE_URL, Client, Failure, check, compose, print_pass,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
c = Client(BASE_URL)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
path = '/api/v1/chats/' + chat['id'] + '/messages'
|
||||
|
||||
stop = threading.Event()
|
||||
stats = {'ok': 0, 'err': 0, 'instances': {}}
|
||||
|
||||
def load():
|
||||
local = Client(BASE_URL, timeout=5)
|
||||
while not stop.is_set():
|
||||
try:
|
||||
key = uuid.uuid4().hex
|
||||
r = local.call('POST', path, 201, actor=alice,
|
||||
body={'text': 'roll-' + key},
|
||||
headers={'Idempotency-Key': key},
|
||||
schema='Message')
|
||||
stats['ok'] += 1
|
||||
iid = r['headers'].get('X-Instance-Id')
|
||||
stats['instances'][iid] = stats['instances'].get(iid, 0) + 1
|
||||
except Failure:
|
||||
stats['err'] += 1
|
||||
time.sleep(0.05)
|
||||
|
||||
t = threading.Thread(target=load, daemon=True)
|
||||
t.start()
|
||||
time.sleep(10)
|
||||
base_ok = stats['ok']
|
||||
base_err = stats['err']
|
||||
print(' base: ok=%d err=%d' % (base_ok, base_err))
|
||||
|
||||
# Rolling update api1.
|
||||
compose('up', '-d', '--no-deps', '--force-recreate', 'api1', timeout=180)
|
||||
time.sleep(15)
|
||||
|
||||
stop.set()
|
||||
t.join(timeout=10)
|
||||
|
||||
delta_ok = stats['ok'] - base_ok
|
||||
delta_err = stats['err'] - base_err
|
||||
print(' after rolling: ok=+%d err=+%d instances=%r'
|
||||
% (delta_ok, delta_err, stats['instances']))
|
||||
|
||||
check(delta_ok > 0, 'rolling update остановил запись полностью')
|
||||
# Ошибки допустимы кратковременно, но не должны доминировать.
|
||||
check(delta_err <= max(5, delta_ok // 3),
|
||||
'слишком много ошибок при rolling update: +%d ok / +%d err'
|
||||
% (delta_ok, delta_err))
|
||||
|
||||
# После rolling обе реплики должны снова обслуживать.
|
||||
seen_final = {}
|
||||
for _ in range(40):
|
||||
r = c.call('GET', '/api/v1/users/me', actor=alice, schema='User')
|
||||
iid = r['headers']['X-Instance-Id']
|
||||
seen_final[iid] = seen_final.get(iid, 0) + 1
|
||||
check(len(seen_final) >= 2,
|
||||
'после rolling видна одна реплика: %r' % seen_final)
|
||||
print(' final instances:', seen_final)
|
||||
print_pass('rolling update under load')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except Failure as e:
|
||||
print('FAIL:', e)
|
||||
raise SystemExit(1)
|
||||
Reference in New Issue
Block a user