feat:level4

This commit is contained in:
Оля Мороз
2026-10-20 18:35:44 +03:00
parent 4349a539d9
commit ae53d39ccc
20 changed files with 775 additions and 26 deletions
+1 -1
View File
@@ -26,6 +26,6 @@ USER app
EXPOSE 8080
HEALTHCHECK --interval=10s --timeout=3s --start-period=20s --retries=5 \
CMD wget -qO- http://127.0.0.1:8080/health/live >/dev/null || exit 1
CMD wget -qO- http://127.0.0.1:8080/health/ready >/dev/null || exit 1
ENTRYPOINT ["/app/messenger"]
+7 -1
View File
@@ -4,7 +4,7 @@ CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover
.PHONY: help check validate test load test-persistence test-failover my_tests
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@@ -28,3 +28,9 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
my_tests:
$(PYTHON) student_tests/test_limits.py
$(PYTHON) student_tests/test_idempotency.py
$(PYTHON) student_tests/test_failover_both.py
$(PYTHON) student_tests/test_rolling.py
+3 -2
View File
@@ -50,7 +50,7 @@ func main() {
sessions := domain.NewSessionStore(store, cache, cfg.SessionTTL)
handler := &api.Handler{Store: store, Sessions: sessions}
router := api.NewRouter(handler)
router := api.NewRouter(handler, cfg.InstanceID)
srv := &http.Server{
Addr: fmt.Sprintf("%s:%d", cfg.Host, cfg.Port),
@@ -66,6 +66,7 @@ func main() {
go func() {
log.Printf("messenger listening on %s (env=%s, session_ttl=%s)", srv.Addr, cfg.Env, cfg.SessionTTL)
log.Printf("instance_id=%s", cfg.InstanceID)
if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
log.Fatalf("server error: %v", err)
}
@@ -73,7 +74,7 @@ func main() {
<-stop
log.Println("shutting down...")
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
_ = srv.Shutdown(ctx)
log.Println("stopped")
+79 -17
View File
@@ -1,3 +1,34 @@
x-api-base: &api-base
build:
context: .
dockerfile: Dockerfile
env_file:
- .env
environment:
APP_HOST: "0.0.0.0"
APP_PORT: "8080"
APP_ENV: "lab"
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
SESSION_STORE: "postgres"
SESSION_TTL_SECONDS: "3600"
secrets:
- database_url
- cache_url
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
networks: [backend]
deploy:
resources:
limits:
cpus: "0.75"
memory: 256M
restart_policy:
condition: on-failure
services:
postgres:
image: postgres:16.3-alpine
@@ -15,6 +46,11 @@ services:
timeout: 3s
retries: 20
networks: [backend]
deploy:
resources:
limits:
cpus: "1.0"
memory: 512M
redis:
image: redis:7.2.5-alpine
@@ -27,30 +63,35 @@ services:
timeout: 3s
retries: 20
networks: [backend]
deploy:
resources:
limits:
cpus: "0.5"
memory: 256M
api:
build:
context: .
dockerfile: Dockerfile
env_file:
- .env
api1:
<<: *api-base
environment:
APP_HOST: "0.0.0.0"
APP_PORT: "8080"
APP_ENV: "lab"
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
secrets:
- database_url
- cache_url
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
ports:
- "8080:8080"
networks: [backend]
SESSION_STORE: "postgres"
SESSION_TTL_SECONDS: "3600"
INSTANCE_ID: "api-1"
api2:
<<: *api-base
environment:
APP_HOST: "0.0.0.0"
APP_PORT: "8080"
APP_ENV: "lab"
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
SESSION_STORE: "postgres"
SESSION_TTL_SECONDS: "3600"
INSTANCE_ID: "api-2"
migrate:
build:
@@ -67,6 +108,27 @@ services:
networks: [backend]
restart: "no"
lb:
image: nginx:1.27.0-alpine
volumes:
- ./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
ports:
- "8080:8080"
depends_on:
- api1
- api2
networks: [backend]
healthcheck:
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8080/health/live"]
interval: 5s
timeout: 3s
retries: 10
deploy:
resources:
limits:
cpus: "0.5"
memory: 128M
volumes:
pgdata:
redisdata:
+28
View File
@@ -0,0 +1,28 @@
worker_processes auto;
events { worker_connections 1024; }
http {
upstream api_backend {
server api1:8080 max_fails=2 fail_timeout=5s;
server api2:8080 max_fails=2 fail_timeout=5s;
keepalive 32;
}
server {
listen 8080;
location / {
proxy_pass http://api_backend;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_connect_timeout 2s;
proxy_send_timeout 15s;
proxy_read_timeout 15s;
proxy_next_upstream error timeout http_502 http_503 http_504;
}
}
}
+14
View File
@@ -0,0 +1,14 @@
CREATE TABLE IF NOT EXISTS idempotency_keys (
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
method TEXT NOT NULL,
path TEXT NOT NULL,
key TEXT NOT NULL,
request_hash TEXT NOT NULL,
status_code INTEGER NOT NULL,
response JSONB NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (user_id, method, path, key)
);
CREATE INDEX IF NOT EXISTS idx_idempotency_created
ON idempotency_keys (created_at);
+14
View File
@@ -0,0 +1,14 @@
#!/bin/sh
set -eu
# Перезапуск API + всех state stores, volumes сохраняются.
docker compose restart postgres redis api1 api2 lb
# Ждём готовности.
for i in $(seq 1 60); do
if docker compose exec -T lb wget -qO- http://127.0.0.1:8080/health/ready >/dev/null 2>&1; then
echo "ready after ${i}s"
exit 0
fi
sleep 1
done
echo "stack did not become ready in 60s" >&2
exit 1
+9
View File
@@ -0,0 +1,9 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?}"
case "$TARGET_INSTANCE_ID" in
api-1) SERVICE=api1 ;;
api-2) SERVICE=api2 ;;
*) exit 2 ;;
esac
docker compose start "$SERVICE"
+15
View File
@@ -0,0 +1,15 @@
#!/bin/sh
set -eu
# TARGET_INSTANCE_ID приходит от теста: alias обслужившей реплики.
# В compose alias = INSTANCE_ID = "api-1" / "api-2".
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
case "$TARGET_INSTANCE_ID" in
api-1) SERVICE=api1 ;;
api-2) SERVICE=api2 ;;
*) echo "unknown instance: $TARGET_INSTANCE_ID" >&2; exit 2 ;;
esac
# Останавливаем ТОЛЬКО эту реплику, оставляем остановленной.
docker compose stop "$SERVICE"
echo "stopped $SERVICE ($TARGET_INSTANCE_ID)"
+43 -3
View File
@@ -1,8 +1,10 @@
package api
import (
"bytes"
"encoding/json"
"errors"
"io"
"net/http"
"regexp"
"strconv"
@@ -12,6 +14,7 @@ import (
)
var usernameRe = regexp.MustCompile(`^[a-z0-9_]{3,32}$`)
var idempotencyKeyRe = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`)
type Handler struct {
Store *domain.Store
@@ -224,12 +227,31 @@ func (h *Handler) CreateMessage(w http.ResponseWriter, r *http.Request) {
h.bearerAuth(func(w http.ResponseWriter, r *http.Request) {
me, _ := userFrom(r)
chatID := r.PathValue("chat_id")
// Idempotency-Key обязателен.
key := r.Header.Get("Idempotency-Key")
if key == "" {
writeError(w, r, http.StatusBadRequest, "invalid_request", "Idempotency-Key required")
return
}
if !idempotencyKeyRe.MatchString(key) {
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid Idempotency-Key")
return
}
if _, err := h.Store.GetChat(r.Context(), chatID, me.ID); err != nil {
writeError(w, r, http.StatusNotFound, "not_found", "chat not found")
return
}
// Читаем тело один раз и хешируем нормализованный JSON.
rawBody, err := io.ReadAll(io.LimitReader(r.Body, 64*1024))
if err != nil {
writeError(w, r, http.StatusBadRequest, "invalid_request", "cannot read body")
return
}
var req createMessageReq
dec := json.NewDecoder(r.Body)
dec := json.NewDecoder(bytes.NewReader(rawBody))
dec.DisallowUnknownFields()
if err := dec.Decode(&req); err != nil {
writeError(w, r, http.StatusBadRequest, "invalid_request", "malformed JSON")
@@ -243,13 +265,23 @@ func (h *Handler) CreateMessage(w http.ResponseWriter, r *http.Request) {
writeError(w, r, http.StatusBadRequest, "invalid_request", "empty message")
return
}
m, err := h.Store.AddMessage(r.Context(), chatID, me.ID, req.Text)
bodyHash, err := domain.HashBody(rawBody)
if err != nil {
writeError(w, r, http.StatusBadRequest, "invalid_request", "malformed JSON")
return
}
msg, rec, replay, err := h.Store.AddMessageIdempotent(
r.Context(), chatID, me.ID, req.Text, key, bodyHash)
switch {
case err == nil:
// ok
case errors.Is(err, domain.ErrNotFound):
writeError(w, r, http.StatusNotFound, "not_found", "chat not found")
return
case errors.Is(err, domain.ErrConflict):
writeError(w, r, http.StatusConflict, "conflict", "idempotency key reused with different body")
return
case errors.Is(err, domain.ErrTimeout):
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database timeout")
return
@@ -257,7 +289,15 @@ func (h *Handler) CreateMessage(w http.ResponseWriter, r *http.Request) {
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
return
}
writeJSON(w, http.StatusCreated, m)
if replay {
// Возвращаем сохранённый 201 и то же Message.
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(rec.StatusCode)
_, _ = w.Write(rec.Response)
return
}
writeJSON(w, http.StatusCreated, msg)
})(w, r)
}
+11
View File
@@ -33,3 +33,14 @@ func withRequestID(next http.Handler) http.Handler {
next.ServeHTTP(w, r.WithContext(ctx))
})
}
// withInstanceID ставит X-Instance-Id на КАЖДЫЙ ответ (включая ошибки и health).
// ID генерируется один раз при старте процесса — стабилен в пределах жизни.
func withInstanceID(instanceID string) func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("X-Instance-Id", instanceID)
next.ServeHTTP(w, r)
})
}
}
+4 -2
View File
@@ -2,7 +2,7 @@ package api
import "net/http"
func NewRouter(h *Handler) http.Handler {
func NewRouter(h *Handler, instanceID string) http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("GET /health/live", h.Live)
mux.HandleFunc("GET /health/ready", h.Ready)
@@ -15,5 +15,7 @@ func NewRouter(h *Handler) http.Handler {
mux.HandleFunc("GET /api/v1/chats/{chat_id}", h.GetChat)
mux.HandleFunc("POST /api/v1/chats/{chat_id}/messages", h.CreateMessage)
mux.HandleFunc("GET /api/v1/chats/{chat_id}/messages", h.ListMessages)
return withRequestID(mux)
// Порядок: instanceID снаружи, requestID внутри.
return withInstanceID(instanceID)(withRequestID(mux))
}
+17
View File
@@ -1,6 +1,8 @@
package config
import (
"crypto/rand"
"encoding/hex"
"os"
"strconv"
"time"
@@ -15,6 +17,7 @@ type Config struct {
SessionTTL time.Duration
SessionStore string // "postgres" | "redis"
RedisRequired bool
InstanceID string
}
func getenv(key, def string) string {
@@ -59,6 +62,11 @@ func Load() Config {
cacheURL = readSecretFromFile(os.Getenv("CACHE_URL_FILE"))
}
instanceID := os.Getenv("INSTANCE_ID")
if instanceID == "" {
instanceID = "api-" + randomShortID()
}
store := getenv("SESSION_STORE", "postgres")
return Config{
Host: getenv("APP_HOST", "0.0.0.0"),
@@ -69,5 +77,14 @@ func Load() Config {
SessionTTL: time.Duration(ttl) * time.Second,
SessionStore: store,
RedisRequired: store == "redis",
InstanceID: instanceID,
}
}
func randomShortID() string {
b := make([]byte, 6)
if _, err := rand.Read(b); err != nil {
return "unknown"
}
return hex.EncodeToString(b)
}
+75
View File
@@ -0,0 +1,75 @@
package domain
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"time"
"github.com/jackc/pgx/v5"
)
// IdempotencyRecord — сохранённый результат POST.
type IdempotencyRecord struct {
StatusCode int
Response json.RawMessage
}
// LookupIdempotency возвращает запись, если ключ уже использован.
// Второй аргумент — совпадает ли хеш тела с сохранённым.
func (s *Store) LookupIdempotency(ctx context.Context, userID, method, path, key, bodyHash string) (*IdempotencyRecord, bool, error) {
var (
storedHash string
rec IdempotencyRecord
)
err := s.pool.QueryRow(ctx,
`SELECT request_hash, status_code, response
FROM idempotency_keys
WHERE user_id=$1 AND method=$2 AND path=$3 AND key=$4`,
userID, method, path, key,
).Scan(&storedHash, &rec.StatusCode, &rec.Response)
if errors.Is(err, pgx.ErrNoRows) {
return nil, false, nil
}
if err != nil {
return nil, false, classifyDBError(err)
}
if storedHash != bodyHash {
return nil, true, ErrConflict
}
return &rec, true, nil
}
// SaveIdempotency пишет результат. Конфликт PK означает, что параллельный
// запрос успел раньше — вызывающий должен перечитать через LookupIdempotency.
func (s *Store) SaveIdempotency(ctx context.Context, userID, method, path, key, bodyHash string, status int, resp json.RawMessage) error {
_, err := s.pool.Exec(ctx,
`INSERT INTO idempotency_keys (user_id, method, path, key, request_hash, status_code, response)
VALUES ($1,$2,$3,$4,$5,$6,$7)`,
userID, method, path, key, bodyHash, status, resp)
return classifyDBError(err)
}
// HashBody — стабильный хеш JSON независимо от порядка ключей.
func HashBody(raw []byte) (string, error) {
var v any
if err := json.Unmarshal(raw, &v); err != nil {
return "", err
}
normalized, err := json.Marshal(v) // map keys сортируются
if err != nil {
return "", err
}
sum := sha256.Sum256(normalized)
return hex.EncodeToString(sum[:]), nil
}
// CleanupIdempotency — можно звать из фонового тикера (опционально).
func (s *Store) CleanupIdempotency(ctx context.Context, olderThan time.Duration) error {
_, err := s.pool.Exec(ctx,
`DELETE FROM idempotency_keys WHERE created_at < now() - $1::interval`,
olderThan.String())
return err
}
+93
View File
@@ -10,6 +10,7 @@ import (
"crypto/rand"
"crypto/subtle"
"encoding/base64"
"encoding/json"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
@@ -443,3 +444,95 @@ func classifyDBError(err error) error {
}
return err
}
// AddMessageIdempotent — атомарно проверяет членство, вставляет сообщение
// и сохраняет результат идемпотентности в одной транзакции.
// Если ключ уже есть с тем же body — возвращает сохранённый ответ и replay=true.
func (s *Store) AddMessageIdempotent(
ctx context.Context,
chatID, senderID, text, key, bodyHash string,
) (msg *Message, rec *IdempotencyRecord, replay bool, err error) {
path := "/api/v1/chats/" + chatID + "/messages"
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.Serializable})
if err != nil {
return nil, nil, false, classifyDBError(err)
}
defer tx.Rollback(ctx)
// 1) Членство.
var ok bool
if err := tx.QueryRow(ctx,
`SELECT true FROM chat_members WHERE chat_id=$1 AND user_id=$2`,
chatID, senderID,
).Scan(&ok); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return nil, nil, false, ErrNotFound
}
return nil, nil, false, classifyDBError(err)
}
// 2) Уже был такой ключ?
var storedHash string
var storedStatus int
var storedResp json.RawMessage
err = tx.QueryRow(ctx,
`SELECT request_hash, status_code, response
FROM idempotency_keys
WHERE user_id=$1 AND method='POST' AND path=$2 AND key=$3`,
senderID, path, key,
).Scan(&storedHash, &storedStatus, &storedResp)
switch {
case err == nil:
if storedHash != bodyHash {
return nil, nil, false, ErrConflict
}
return nil, &IdempotencyRecord{StatusCode: storedStatus, Response: storedResp}, true, nil
case !errors.Is(err, pgx.ErrNoRows):
return nil, nil, false, classifyDBError(err)
}
// 3) Вставка сообщения.
var m Message
err = tx.QueryRow(ctx,
`INSERT INTO messages (id, chat_id, sender_id, text)
VALUES ($1,$2,$3,$4)
RETURNING id, chat_id, sender_id, text, created_at`,
uuid.NewString(), chatID, senderID, text,
).Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt)
if err != nil {
// Гонка на одном ключе в разных репликах: unique violation в idempotency_keys
// маловероятен здесь, но serialization failure возможен.
return nil, nil, false, classifyDBError(err)
}
// 4) Сохранение ответа.
respJSON, _ := json.Marshal(m)
_, err = tx.Exec(ctx,
`INSERT INTO idempotency_keys (user_id, method, path, key, request_hash, status_code, response)
VALUES ($1,'POST',$2,$3,$4,201,$5)`,
senderID, path, key, bodyHash, respJSON)
if err != nil {
var pgErr *pgconn.PgError
if errors.As(err, &pgErr) && pgErr.Code == "23505" {
// Кто-то параллельно успел вставить: перечитываем в этой же tx.
if err2 := tx.QueryRow(ctx,
`SELECT request_hash, status_code, response
FROM idempotency_keys
WHERE user_id=$1 AND method='POST' AND path=$2 AND key=$3`,
senderID, path, key,
).Scan(&storedHash, &storedStatus, &storedResp); err2 == nil {
if storedHash != bodyHash {
return nil, nil, false, ErrConflict
}
return nil, &IdempotencyRecord{StatusCode: storedStatus, Response: storedResp}, true, nil
}
}
return nil, nil, false, classifyDBError(err)
}
if err := tx.Commit(ctx); err != nil {
return nil, nil, false, classifyDBError(err)
}
return &m, nil, false, nil
}
+67
View File
@@ -0,0 +1,67 @@
"""Общие утилиты для студенческих тестов."""
import base64
import json
import os
import subprocess
import sys
import time
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, check # noqa: E402
BASE_URL = os.environ.get('BASE_URL', 'http://localhost:8080')
COMPOSE_CMD = os.environ.get('COMPOSE_CMD', 'docker compose')
COMPOSE_PROJECT = os.environ.get('COMPOSE_PROJECT', '')
def compose(*args, timeout=120, check_rc=True):
"""Вызов docker compose. Возвращает CompletedProcess."""
cmd = COMPOSE_CMD.split()
if COMPOSE_PROJECT:
cmd += ['-p', COMPOSE_PROJECT]
cmd += list(args)
proc = subprocess.run(
cmd, cwd=str(ROOT), capture_output=True, text=True, timeout=timeout)
if check_rc and proc.returncode != 0:
raise Failure(
'compose %s failed rc=%d\nstdout=%s\nstderr=%s'
% (' '.join(args), proc.returncode, proc.stdout[-2000:], proc.stderr[-2000:]))
return proc
def wait_ready(deadline_seconds=90):
c = Client(BASE_URL, timeout=2)
start = time.monotonic()
while time.monotonic() - start < deadline_seconds:
try:
c.call('GET', '/health/ready', schema='Health')
return
except Failure:
time.sleep(0.5)
raise Failure('LB did not become ready in %ds' % deadline_seconds)
def observe_instances(c, actor, n=40):
"""Собираем X-Instance-Id через /users/me."""
seen = {}
for _ in range(n):
r = c.call('GET', '/api/v1/users/me', actor=actor, schema='User')
iid = r['headers'].get('X-Instance-Id')
check(iid, 'X-Instance-Id missing')
seen[iid] = seen.get(iid, 0) + 1
return seen
def run_file(path, *args, timeout=180):
proc = subprocess.run(
[sys.executable, str(path), *args],
cwd=str(ROOT), capture_output=True, text=True, timeout=timeout)
return proc
def print_pass(name, extra=''):
print('PASS: ' + name + (' ' + extra if extra else ''))
+87
View File
@@ -0,0 +1,87 @@
#!/usr/bin/env python3
"""Останавливает КАЖДУЮ реплику по очереди и проверяет восстановление."""
import json
import subprocess
import sys
import time
from pathlib import Path
from common import (
BASE_URL, Client, Failure, check, compose, observe_instances, print_pass,
wait_ready,
)
ROOT = Path(__file__).resolve().parents[1]
def stop_one(instance_alias):
env = {'TARGET_INSTANCE_ID': instance_alias, 'PATH': __import__('os').environ['PATH']}
proc = subprocess.run(
[str(ROOT / 'scripts/stop-one.sh')],
cwd=str(ROOT), env={**__import__('os').environ, **env},
capture_output=True, text=True, timeout=30)
check(proc.returncode == 0,
'stop-one.sh для %s упал rc=%d\nstderr=%s'
% (instance_alias, proc.returncode, proc.stderr[-1000:]))
def restore_all():
"""Возвращает все реплики в running."""
for svc in ('api1', 'api2'):
compose('start', svc, check_rc=False)
wait_ready(90)
def main():
c = Client(BASE_URL)
alice, bob, eve, chat = c.bootstrap()
seen = observe_instances(c, alice, n=40)
check(len(seen) >= 2, 'видна одна реплика: %r' % seen)
print(' instances:', seen)
for alias in sorted(seen):
print('--- failover test for %s' % alias)
target_alias = alias
stop_one(target_alias)
time.sleep(1)
# Ни одна последующая успешная запись не должна обслуживаться target.
first_ok = None
after = []
deadline = time.monotonic() + 15
key = 'fo-' + __import__('uuid').uuid4().hex
while time.monotonic() < deadline:
try:
r = c.message(alice, chat, 'after-stop-' + key, key=key)
if first_ok is None:
first_ok = time.monotonic()
after.append(r['headers']['X-Instance-Id'])
key = 'fo-' + __import__('uuid').uuid4().hex
except Failure:
pass
time.sleep(0.2)
check(first_ok is not None, 'запись не восстановилась за 15с')
check(len(after) >= 5, 'слишком мало успешных записей после отказа: %d' % len(after))
check(target_alias not in after[-5:],
'остановленная реплика %s всё ещё отвечает' % target_alias)
print(' after stop %s: %d writes, последние instances=%r'
% (target_alias, len(after), after[-3:]))
# Восстановить и подождать, пока LB снова начнёт её использовать.
restore_all()
time.sleep(2)
seen2 = observe_instances(c, alice, n=60)
print(' после restore видно:', seen2)
# В конце — обе реплики снова живы.
seen3 = observe_instances(c, alice, n=60)
check(len(seen3) >= 2, 'после restore видна одна реплика: %r' % seen3)
print_pass('failover both replicas')
if __name__ == '__main__':
try:
main()
except Failure as e:
print('FAIL:', e)
raise SystemExit(1)
+88
View File
@@ -0,0 +1,88 @@
#!/usr/bin/env python3
"""Проверки идемпотентности и X-Instance-Id (лаб3)."""
import json
import uuid
from concurrent.futures import ThreadPoolExecutor
from common import (
BASE_URL, Client, Failure, check, observe_instances, print_pass,
)
def main():
c = Client(BASE_URL)
alice, bob, eve, chat = c.bootstrap()
path = '/api/v1/chats/' + chat['id'] + '/messages'
# 1) Через LB видно ≥2 реплики.
seen = observe_instances(c, alice, n=40)
check(len(seen) >= 2, 'менее двух X-Instance-Id через LB: %r' % seen)
print(' instances:', seen)
# 2) Повтор с тем же ключом и тем же телом → тот же Message.
key = 'idem-' + uuid.uuid4().hex
body = {'text': 'idem-verify-' + uuid.uuid4().hex}
r1 = c.call('POST', path, 201, actor=alice, body=body,
headers={'Idempotency-Key': key}, schema='Message')
r2 = c.call('POST', path, 201, actor=alice, body=body,
headers={'Idempotency-Key': key}, schema='Message')
check(r1['json']['id'] == r2['json']['id'], 'replay дал другой id')
check(r1['json'] == r2['json'], 'replay дал другое тело')
# 3) Тот же ключ, другое тело → 409.
c.call('POST', path, 409, actor=alice, body={'text': 'other'},
headers={'Idempotency-Key': key})
# 4) Нет ключа → 400.
c.call('POST', path, 400, actor=alice, body={'text': 'no key'})
# 5) Невалидный ключ (пробел) → 400.
c.call('POST', path, 400, actor=alice, body={'text': 'bad key'},
headers={'Idempotency-Key': 'has space'})
# 6) Разные пользователи с одинаковым ключом — независимы.
key_shared = 'shared-' + uuid.uuid4().hex
ra = c.message(alice, chat, 'alice-' + uuid.uuid4().hex, key=key_shared)
rb = c.message(bob, chat, 'bob-' + uuid.uuid4().hex, key=key_shared)
check(ra['json']['id'] != rb['json']['id'],
'одинаковый ключ у разных пользователей склеил сообщения')
# 7) Конкурентный повтор: 20 параллельных POST с одним ключом.
key_c = 'conc-' + uuid.uuid4().hex
text_c = 'concurrent-' + uuid.uuid4().hex
def send(_):
local = Client(BASE_URL, timeout=15)
# Разрешаем 201 или 503 (transient serialization failure).
r = local.call('POST', path, (201, 503), actor=alice,
body={'text': text_c},
headers={'Idempotency-Key': key_c})
payload = r['json']
if payload is None and r['status'] == 201:
try:
payload = json.loads(r['bytes'])
except ValueError:
payload = None
return {'status': r['status'], 'json': payload}
with ThreadPoolExecutor(max_workers=20) as pool:
results = list(pool.map(send, range(20)))
ok = [r for r in results if r['status'] == 201 and r['json']]
check(len(ok) >= 15, 'слишком много не-201 при конкурентном повторе: %d/20'
% len(ok))
ids = {r['json']['id'] for r in ok}
check(len(ids) == 1,
'конкурентный повтор создал %d разных сообщений: %r'
% (len(ids), ids))
print(' concurrent: %d×201, один id' % len(ok))
print_pass('idempotency')
if __name__ == '__main__':
try:
main()
except Failure as e:
print('FAIL:', e)
raise SystemExit(1)
+41
View File
@@ -0,0 +1,41 @@
#!/usr/bin/env python3
"""Проверяет, что контейнеры имеют CPU/RAM лимиты и api не публикует порт."""
import json
from common import Failure, check, compose, print_pass
def main():
raw = compose('config', '--format', 'json').stdout
cfg = json.loads(raw)
services = cfg['services']
for svc in ('api1', 'api2'):
check(svc in services, 'сервис %s отсутствует' % svc)
ports = services[svc].get('ports', [])
check(not ports, '%s публикует порты наружу: %r — должно быть только через LB'
% (svc, ports))
deploy = services[svc].get('deploy', {})
res = deploy.get('resources', {}).get('limits', {})
check('cpus' in res and 'memory' in res,
'%s без deploy.resources.limits cpus/memory: %r' % (svc, res))
check('lb' in services, 'нет сервиса lb')
lb_ports = services['lb'].get('ports', [])
check(any('8080' in str(p) for p in lb_ports),
'lb не публикует 8080: %r' % lb_ports)
check('container_name' not in services.get('api1', {}),
'api1 задаёт container_name — ломает масштабирование')
check('container_name' not in services.get('api2', {}),
'api2 задаёт container_name — ломает масштабирование')
print_pass('limits & topology')
if __name__ == '__main__':
try:
main()
except Failure as e:
print('FAIL:', e)
raise SystemExit(1)
+79
View File
@@ -0,0 +1,79 @@
#!/usr/bin/env python3
"""Rolling update одной реплики под нагрузкой (уровень 4)."""
import json
import threading
import time
import uuid
from common import (
BASE_URL, Client, Failure, check, compose, print_pass,
)
def main():
c = Client(BASE_URL)
alice, bob, eve, chat = c.bootstrap()
path = '/api/v1/chats/' + chat['id'] + '/messages'
stop = threading.Event()
stats = {'ok': 0, 'err': 0, 'instances': {}}
def load():
local = Client(BASE_URL, timeout=5)
while not stop.is_set():
try:
key = uuid.uuid4().hex
r = local.call('POST', path, 201, actor=alice,
body={'text': 'roll-' + key},
headers={'Idempotency-Key': key},
schema='Message')
stats['ok'] += 1
iid = r['headers'].get('X-Instance-Id')
stats['instances'][iid] = stats['instances'].get(iid, 0) + 1
except Failure:
stats['err'] += 1
time.sleep(0.05)
t = threading.Thread(target=load, daemon=True)
t.start()
time.sleep(10)
base_ok = stats['ok']
base_err = stats['err']
print(' base: ok=%d err=%d' % (base_ok, base_err))
# Rolling update api1.
compose('up', '-d', '--no-deps', '--force-recreate', 'api1', timeout=180)
time.sleep(15)
stop.set()
t.join(timeout=10)
delta_ok = stats['ok'] - base_ok
delta_err = stats['err'] - base_err
print(' after rolling: ok=+%d err=+%d instances=%r'
% (delta_ok, delta_err, stats['instances']))
check(delta_ok > 0, 'rolling update остановил запись полностью')
# Ошибки допустимы кратковременно, но не должны доминировать.
check(delta_err <= max(5, delta_ok // 3),
'слишком много ошибок при rolling update: +%d ok / +%d err'
% (delta_ok, delta_err))
# После rolling обе реплики должны снова обслуживать.
seen_final = {}
for _ in range(40):
r = c.call('GET', '/api/v1/users/me', actor=alice, schema='User')
iid = r['headers']['X-Instance-Id']
seen_final[iid] = seen_final.get(iid, 0) + 1
check(len(seen_final) >= 2,
'после rolling видна одна реплика: %r' % seen_final)
print(' final instances:', seen_final)
print_pass('rolling update under load')
if __name__ == '__main__':
try:
main()
except Failure as e:
print('FAIL:', e)
raise SystemExit(1)