diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 0000000..bf7cd07 --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,91 @@ +# Изменения lab5 относительно lab4 + +## API + +- `POST /chats/{chat_id}/attachments` возвращает `202` со статусом `queued` вместо `201 ready`. Проверки при приёме те же, что в lab4. +- Оригинал сначала сохраняется в приватный S3-prefix `staging/`, а не сразу в конечный `attachments/`. +- `GET /attachments/{id}` возвращает статусы `queued`, `processing`, `ready`, `failed`, а также `error_code` и `variants`. +- `GET /attachments/{id}/content`: + - принимает `variant=original|thumbnail|crop|optimized`, неизвестное имя → 400; + - для не-ready вложения → 409; + - отсутствующий вариант ready-вложения → 404. +- `POST /chats/{chat_id}/messages`: прикрепить `failed`-вложение нельзя (409), `queued`/`processing` можно. + +## База данных + +Новая миграция `migrations/0004_jobs.sql`: + +- В `attachments` добавлены `status`, `error_code`, `staging_key`, `updated_at`. `object_key` теперь заполняет worker. +- `attachment_variants`: варианты с первичным ключом `(attachment_id, pipeline_version, name)`. +- `jobs`: задача обработки (статус, попытка, лимит попыток, время следующей попытки, lease, последняя ошибка). +- `outbox`: задания на публикацию в брокер. +- `job_replays`: аудит операторских повторов. + +## Новые компоненты + +- `broker`: Valkey Streams, отдельный сервис с volume, `appendfsync always`, `noeviction`, паролем из secrets. Stream `attachments.jobs`, группа `workers`. +- `worker`: отдельный процесс (`messenger worker`), 2 потребителя, prefetch 1, лимит 768 MiB. +- `broker.go`: формат сообщения по `jobs.schema.json`, публикация, outbox relay (работает в каждой API-реплике). +- `worker.go`: + - захват задачи с lease и fencing по `lease_owner` и `attempt`; + - обработка и фиксация результата; + - retry с backoff и jitter; + - reaper для задач с истёкшим lease; + - повторное чтение своих pending и `XAUTOCLAIM`; + - команды `failed` и `replay`. +- `imaging.go`: + - thumbnail JPEG 256, crop JPEG 128×128, optimized WebP quality 80 (вписан в 1280); + - применение EXIF orientation, alpha на белом; + - предварительное уменьшение больших изображений для экономии памяти. + +## Изменения существующего кода + +- `attachments.go`: приём в staging, транзакция `attachments` + `jobs` + `outbox`, ответ 202, загрузка вариантов, скачивание по `variant`. Janitor дополнительно чистит `staging/`. +- `handlers.go`: проверка failed-вложений при создании сообщения, точка внедрения отказа. +- `main.go`: подкоманды `worker`, `failed`, `replay`; запуск relay в API; новые переменные окружения. +- `s3.go`: listing по произвольному prefix. +- `Dockerfile`: сборка с тегом `nodynamic` (WebP без системных библиотек). +- `go.mod`: добавлены `golang.org/x/image`, `github.com/gen2brain/webp`, `github.com/rwcarlsen/goexif`. + +## Инфраструктура и скрипты + +- `compose.yaml`: сервисы `broker` и `worker`, volume `broker_data`, secrets `broker_conf` и `broker_url`, образ `mtusi-messenger:lab5`, переменные для точек внедрения отказа (по умолчанию пусты). +- `scripts/mksecrets.sh`: генерирует пароль брокера, `broker.conf` и `broker_url`. +- `scripts/restart.sh`: перезапускает также `broker` и `worker`. +- `scripts/rolling-update.sh`: обновляет также `worker`. +- `.env.example`: `WORKER_CONCURRENCY`, `JOB_MAX_ATTEMPTS`, `JOB_TIMEOUT_SECONDS`, `JOB_BACKOFF_BASE_MS`, `JOB_BACKOFF_MAX_MS`, `STAGING_RETENTION_SECONDS`. + +## Надёжность + +- Работа не теряется после ответа 202: оригинал в S3, задача и outbox в одной транзакции PostgreSQL. +- Разрыв COMMIT/publish закрыт transactional outbox. +- Повторная доставка и повторы не создают второго набора вариантов: детерминированные S3-ключи и upsert по ключу эффекта. +- Зависшие задачи возвращаются в очередь по истечении lease. +- Постоянные ошибки (`invalid_image`, `processing_failed`) уходят в failed сразу, временные повторяются до `retry_exhausted`. +- Failed-задачи можно повторить через `replay`. Повтор уже готовой задачи ничего не делает. +- Повреждение внутри PNG, которое lab4 пропускал (проверялись только заголовок и конец), теперь ловит worker при полном декодировании. + +## Тесты + +Новые: +- `tests/stack.py`: общие хелперы. +- `tests/variants.py`: размеры, форматы, sha256, без увеличения, alpha, EXIF. +- `tests/queue_recovery.py`: сценарий на 3 и повторная доставка. +- `tests/faults.py`: 5 сценариев сбоев на 4. +- `tests/worker_load.py`: конкуренция, память и timeout worker. + +Адаптированы под 202: `tests/attachments.py`, `tests/s3_checks.py` (плюс проверка очистки staging), `tests/upload_memory.py`. + +Без изменений: `tests/idempotency.py`, `tests/scenario.py`, `tests/image_gen.py`. + +## Отчёт и доказательства + +- `REPORT.md` переписан под lab5. +- Новые файлы в `evidence/`: `variants.txt`, `queue-recovery.txt`, `faults.txt`, `worker-load.json`. +- Остальные файлы evidence получены заново на стенде lab5. + +## Исправления после ревью + +- Авторизация проверяет сессию только в PostgreSQL одним запросом (`sessions` ⋈ `users`, `expires_at > now()`); запись кэша не принимается за доказательство действительности токена. Код кэша сессий (`cache.go`) удалён. +- Добавлен `tests/session_revocation.py`: Valkey остановлен → logout 204 → Valkey запущен → прежний токен 401. +- Тесты `image_gen.py`, `idempotency.py`, `s3_checks.py`, `scenario.py`, `upload_memory.py` были не добавлены в git, из-за чего `attachments.py` и `worker_load.py` падали с `ModuleNotFoundError: image_gen`. diff --git a/REPORT.md b/REPORT.md index e8ef995..bc4e398 100644 --- a/REPORT.md +++ b/REPORT.md @@ -42,6 +42,7 @@ python3 tests/attachments.py python3 tests/s3_checks.py python3 tests/upload_memory.py --output evidence/upload-memory.json python3 tests/idempotency.py +python3 tests/session_revocation.py python3 tests/scenario.py --mode failover-each --output evidence/failover-each.json python3 tests/scenario.py --mode rolling --output evidence/rolling.json ``` @@ -65,7 +66,7 @@ worker ──XREADGROUP──→ claim (lease) → S3 attachments//... → C | broker | `broker` | valkey/valkey:8.1-alpine@sha256:081c2f5c… | `valkey-cli ping` с паролем | broker_data | backend | broker.conf (`requirepass`) | | s3 | `s3` | chrislusf/seaweedfs@sha256:c2658820… | `/dir/assign` выдаёт fid | s3_data | storage | s3_config.json | | load_balancer | `lb` | haproxy:3.0-alpine@sha256:cf3af0fa… | проверки `/health/ready` бэкендов | нет | backend, 127.0.0.1:8080 | нет | -| кэш сессий | `valkey` | valkey/valkey:8.1-alpine | `valkey-cli ping` | valkey_data | backend | нет | +| Valkey (сессии проверяются только в PostgreSQL) | `valkey` | valkey/valkey:8.1-alpine | `valkey-cli ping` | valkey_data | backend | нет | Bootstrap: `migrate` применяет `migrations/*.sql` под advisory lock, повторный запуск пропускает применённые. `s3-init` создаёт bucket, если его нет. Consumer group `workers` создают API (перед первой публикацией) и worker при старте (`XGROUP CREATE … 0 MKSTREAM`, ошибка BUSYGROUP игнорируется). @@ -128,6 +129,7 @@ Bootstrap: `migrate` применяет `migrations/*.sql` под advisory lock, | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | | Публичный smoke (202 → ready, оригинал, настоящий JPEG 256×160) | `make check && make test` | PASS | evidence/smoke.txt | +| Отозванная сессия не воскресает после перезапуска Valkey | `tests/session_revocation.py` | Valkey остановлен → logout 204 → Valkey запущен → прежний токен 401 | evidence/session-revocation.txt | | Минимум lab4 сохранён (валидация, доступ, приватный bucket, отказ S3, идемпотентность) | `tests/attachments.py`, `tests/s3_checks.py`, `tests/idempotency.py` | PASS: 413/415/400/404, анонимный доступ 403, 503 при остановленном S3 | evidence/attachments.txt, evidence/s3-checks.txt, evidence/idempotency.txt | | Restart всех stores с volumes | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS; перезапущены postgres, valkey, broker, s3, api, lb, janitor, worker (состояния до/после в файле) | evidence/persistence.txt | | Failover каждой API-реплики | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | PASS для api-1 и api-2, 0–1 неуспешная попытка, восстановление 3.5 s | evidence/failover-1.txt, evidence/failover-2.txt | diff --git a/cache.go b/cache.go deleted file mode 100644 index 9afc5e3..0000000 --- a/cache.go +++ /dev/null @@ -1,104 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "log/slog" - "time" - - "github.com/redis/go-redis/v9" -) - -type cachedSession struct { - UserID string `json:"user_id"` - ExpiresAt time.Time `json:"expires_at"` -} - -type Cache struct { - rdb *redis.Client - log *slog.Logger -} - -func openCache(log *slog.Logger) *Cache { - url, err := readSecret("CACHE_URL", "CACHE_URL_FILE") - if err != nil { - log.Warn("cache disabled", "err", err.Error()) - return &Cache{log: log} - } - opt, err := redis.ParseURL(url) - if err != nil { - log.Warn("cache parse failed", "err", err.Error()) - return &Cache{log: log} - } - opt.DialTimeout = 200 * time.Millisecond - opt.ReadTimeout = 200 * time.Millisecond - opt.WriteTimeout = 200 * time.Millisecond - opt.PoolTimeout = 200 * time.Millisecond - rdb := redis.NewClient(opt) - ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) - defer cancel() - if err := rdb.Ping(ctx).Err(); err != nil { - log.Warn("cache ping failed, will bypass", "err", err.Error()) - } - return &Cache{rdb: rdb, log: log} -} - -func (c *Cache) Close() { - if c.rdb != nil { - _ = c.rdb.Close() - } -} - -func sessKey(hashHex string) string { - return "sess:" + hashHex -} - -func (c *Cache) GetSession(ctx context.Context, hashHex string) (*cachedSession, bool) { - if c.rdb == nil { - return nil, false - } - ctx, cancel := context.WithTimeout(ctx, 200*time.Millisecond) - defer cancel() - val, err := c.rdb.Get(ctx, sessKey(hashHex)).Result() - if err != nil { - return nil, false - } - var s cachedSession - if err := json.Unmarshal([]byte(val), &s); err != nil { - return nil, false - } - if time.Now().UTC().After(s.ExpiresAt) { - return nil, false - } - return &s, true -} - -func (c *Cache) SetSession(ctx context.Context, hashHex string, userID string, expiresAt time.Time) { - if c.rdb == nil { - return - } - ttl := time.Until(expiresAt) - if ttl <= 0 { - return - } - body, err := json.Marshal(cachedSession{UserID: userID, ExpiresAt: expiresAt.UTC()}) - if err != nil { - return - } - ctx, cancel := context.WithTimeout(ctx, 200*time.Millisecond) - defer cancel() - if err := c.rdb.Set(ctx, sessKey(hashHex), body, ttl).Err(); err != nil { - c.log.Warn("cache set failed", "err", err.Error()) - } -} - -func (c *Cache) DelSession(ctx context.Context, hashHex string) { - if c.rdb == nil { - return - } - ctx, cancel := context.WithTimeout(ctx, 200*time.Millisecond) - defer cancel() - if err := c.rdb.Del(ctx, sessKey(hashHex)).Err(); err != nil { - c.log.Warn("cache del failed", "err", err.Error()) - } -} diff --git a/evidence/session-revocation.txt b/evidence/session-revocation.txt new file mode 100644 index 0000000..38dab27 --- /dev/null +++ b/evidence/session-revocation.txt @@ -0,0 +1 @@ +PASS: token revoked while valkey was stopped stays revoked after valkey restart (401) diff --git a/handlers.go b/handlers.go index 7fc5c80..0dadcec 100644 --- a/handlers.go +++ b/handlers.go @@ -49,7 +49,6 @@ type errorBody struct { type App struct { db *pgxpool.Pool - cache *Cache log *slog.Logger ready atomic.Bool sessionTTL time.Duration @@ -60,8 +59,8 @@ type App struct { fault string } -func NewApp(db *pgxpool.Pool, cache *Cache, log *slog.Logger, sessionTTL, idempotencyTTL time.Duration, store *Store, lim limits) *App { - a := &App{db: db, cache: cache, log: log, sessionTTL: sessionTTL, idempotencyTTL: idempotencyTTL, store: store, limits: lim, uploads: make(chan struct{}, lim.uploadSlots), fault: os.Getenv("FAULT_POINT")} +func NewApp(db *pgxpool.Pool, log *slog.Logger, sessionTTL, idempotencyTTL time.Duration, store *Store, lim limits) *App { + a := &App{db: db, log: log, sessionTTL: sessionTTL, idempotencyTTL: idempotencyTTL, store: store, limits: lim, uploads: make(chan struct{}, lim.uploadSlots), fault: os.Getenv("FAULT_POINT")} a.ready.Store(true) return a } @@ -268,23 +267,6 @@ func userJSON(u *User) map[string]any { } } -func (a *App) getUserByID(ctx context.Context, id string) (*User, error) { - ctx, cancel := dbCtx(ctx) - defer cancel() - var u User - err := a.db.QueryRow(ctx, ` - select id::text, username, display_name, password_hash, created_at - from users where id = $1::uuid`, id).Scan( - &u.ID, &u.Username, &u.DisplayName, &u.PasswordHash, &u.CreatedAt) - if errors.Is(err, pgx.ErrNoRows) { - return nil, nil - } - if err != nil { - return nil, err - } - return &u, nil -} - func (a *App) authUser(w http.ResponseWriter, r *http.Request) (*User, string) { token, ok := parseBearer(r.Header.Get("Authorization")) if !ok { @@ -293,26 +275,15 @@ func (a *App) authUser(w http.ResponseWriter, r *http.Request) (*User, string) { } hash, hashHex := tokenHash(token) - if cs, hit := a.cache.GetSession(r.Context(), hashHex); hit { - u, err := a.getUserByID(r.Context(), cs.UserID) - if err != nil { - a.writeErr(w, reqIDFrom(r), 503, "unavailable", "database unavailable") - return nil, "" - } - if u == nil { - a.writeUnauthorized(w, r, "Bearer") - return nil, "" - } - return u, hashHex - } - ctx, cancel := dbCtx(r.Context()) defer cancel() - var userID string + var u User var expiresAt time.Time err := a.db.QueryRow(ctx, ` - select user_id::text, expires_at from sessions - where token_hash = $1 and expires_at > now()`, hash).Scan(&userID, &expiresAt) + select u.id::text, u.username, u.display_name, u.password_hash, u.created_at, s.expires_at + from sessions s join users u on u.id = s.user_id + where s.token_hash = $1 and s.expires_at > now()`, hash).Scan( + &u.ID, &u.Username, &u.DisplayName, &u.PasswordHash, &u.CreatedAt, &expiresAt) if errors.Is(err, pgx.ErrNoRows) { a.writeUnauthorized(w, r, "Bearer") return nil, "" @@ -321,17 +292,7 @@ func (a *App) authUser(w http.ResponseWriter, r *http.Request) (*User, string) { a.writeErr(w, reqIDFrom(r), 503, "unavailable", "database unavailable") return nil, "" } - u, err := a.getUserByID(r.Context(), userID) - if err != nil { - a.writeErr(w, reqIDFrom(r), 503, "unavailable", "database unavailable") - return nil, "" - } - if u == nil { - a.writeUnauthorized(w, r, "Bearer") - return nil, "" - } - a.cache.SetSession(r.Context(), hashHex, userID, expiresAt) - return u, hashHex + return &u, hashHex } func (a *App) handleCreateUser(w http.ResponseWriter, r *http.Request) { @@ -423,7 +384,7 @@ func (a *App) handleCreateSession(w http.ResponseWriter, r *http.Request) { a.writeUnauthorized(w, r, "Basic") return } - token, hash, hashHex, err := newToken() + token, hash, _, err := newToken() if err != nil { a.writeErr(w, reqID, 500, "internal_error", "internal error") return @@ -437,7 +398,6 @@ func (a *App) handleCreateSession(w http.ResponseWriter, r *http.Request) { a.writeErr(w, reqID, 503, "unavailable", "database unavailable") return } - a.cache.SetSession(r.Context(), hashHex, u.ID, expiresAt) a.writeJSON(w, 201, map[string]any{ "token": token, "token_type": "Bearer", @@ -453,7 +413,7 @@ func (a *App) handleRevokeSession(w http.ResponseWriter, r *http.Request) { a.writeUnauthorized(w, r, "Bearer") return } - hash, hashHex := tokenHash(token) + hash, _ := tokenHash(token) ctx, cancel := dbCtx(r.Context()) defer cancel() tag, err := a.db.Exec(ctx, ` @@ -462,7 +422,6 @@ func (a *App) handleRevokeSession(w http.ResponseWriter, r *http.Request) { a.writeErr(w, reqID, 503, "unavailable", "database unavailable") return } - a.cache.DelSession(r.Context(), hashHex) if tag.RowsAffected() == 0 { a.writeUnauthorized(w, r, "Bearer") return diff --git a/main.go b/main.go index e72de1f..a338371 100644 --- a/main.go +++ b/main.go @@ -125,11 +125,8 @@ func main() { return } - cache := openCache(log) - defer cache.Close() - slots := envInt("UPLOAD_CONCURRENCY", 10) - app := NewApp(db, cache, log, time.Duration(ttlSec)*time.Second, time.Duration(idemSec)*time.Second, store, + app := NewApp(db, log, time.Duration(ttlSec)*time.Second, time.Duration(idemSec)*time.Second, store, limits{maxBytes: maxBytes, maxPixels: maxPixels, uploadSlots: slots, maxAttempts: maxAttempts}) relayCtx, stopRelay := context.WithCancel(ctx) defer stopRelay() diff --git a/tests/idempotency.py b/tests/idempotency.py new file mode 100644 index 0000000..951d438 --- /dev/null +++ b/tests/idempotency.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +from concurrent.futures import ThreadPoolExecutor +import threading +import uuid +from client import Client, Failure, arguments, check, run + + +def main(): + args = arguments('Concurrent Idempotency-Key check through the load balancer') + c = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = c.bootstrap() + path = '/api/v1/chats/' + chat['id'] + '/messages' + parallel = 32 + key = 'idem-' + uuid.uuid4().hex + barrier = threading.Barrier(parallel) + + def send(_): + local = Client(args.base_url, args.ca_file) + barrier.wait() + return local.call('POST', path, 201, actor=alice, body={'text': 'same'}, headers={'Idempotency-Key': key}, schema='Message') + + with ThreadPoolExecutor(max_workers=parallel) as pool: + results = list(pool.map(send, range(parallel))) + ids = {r['json']['id'] for r in results} + instances = {r['headers']['X-Instance-Id'] for r in results} + check(len(ids) == 1, 'parallel requests produced %d messages' % len(ids)) + check(len(instances) >= 2, 'requests were served by one instance only') + check(len({r['json']['created_at'] for r in results}) == 1, 'created_at differs between replays') + + listing = c.call('GET', path, actor=alice, schema='MessagePage')['json']['items'] + check(sum(1 for m in listing if m['text'] == 'same') == 1, 'duplicate message in history') + + c.call('POST', path, 409, actor=alice, body={'text': 'other'}, headers={'Idempotency-Key': key}) + c.call('POST', path, 400, actor=alice, body={'text': 'x'}) + c.call('POST', path, 400, actor=alice, body={'text': 'x'}, headers={'Idempotency-Key': 'bad key!'}) + other = c.call('POST', path, 201, actor=bob, body={'text': 'same'}, headers={'Idempotency-Key': key}, schema='Message')['json'] + check(other['id'] not in ids, 'key must be scoped per user') + print('PASS: %d parallel requests, 1 message, %d instances' % (parallel, len(instances))) + + +if __name__ == '__main__': + run(main) diff --git a/tests/image_gen.py b/tests/image_gen.py new file mode 100644 index 0000000..aeb7de0 --- /dev/null +++ b/tests/image_gen.py @@ -0,0 +1,23 @@ +import os +import struct +import zlib + + +def chunk(kind, data): + body = kind + data + return struct.pack('>I', len(data)) + body + struct.pack('>I', zlib.crc32(body) & 0xffffffff) + + +def png(width, height, noise=False): + header = b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)) + raw = bytearray() + row = width * 3 + 1 + for _ in range(height): + raw += b'\x00' + (os.urandom(row - 1) if noise else bytes(row - 1)) + data = zlib.compress(bytes(raw), 0 if noise else 6) + return header + chunk(b'IDAT', data) + chunk(b'IEND', b'') + + +def png_header_only(width, height): + header = b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)) + return header + chunk(b'IDAT', zlib.compress(b'\x00')) + chunk(b'IEND', b'') diff --git a/tests/s3_checks.py b/tests/s3_checks.py new file mode 100644 index 0000000..2541390 --- /dev/null +++ b/tests/s3_checks.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +import hashlib +import subprocess +import time +import uuid +from pathlib import Path +from client import Client, Failure, arguments, check, run +from stack import wait_status + +ROOT = Path(__file__).resolve().parent.parent +SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes() +BUCKET = 'messenger-attachments' + + +def compose(*args, check_rc=True, stdin=None): + return subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT, check=check_rc, + capture_output=True, text=True, input=stdin) + + +def secret(name): + return (ROOT / 'secrets' / name).read_text().strip() + + +def s3_status(method, path, user=None, data=None): + cmd = ['exec', '-T', 's3', 'curl', '-s', '-o', '/dev/null', '-w', '%{http_code}', '-X', method] + if user: + cmd += ['--aws-sigv4', 'aws:amz:us-east-1:s3', '--user', user[0] + ':' + user[1]] + if data is not None: + cmd += ['--data-binary', data] + cmd.append('http://127.0.0.1:9000' + path) + return compose(*cmd).stdout.strip() + + +def upload(c, actor, chat, expected): + return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=a.png', expected, actor=actor, + raw=SAMPLE, headers={'Content-Type': 'image/png'}, schema='Attachment' if expected == 202 else None) + + +def wait_api(c): + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + try: + c.call('GET', '/health/ready') + return + except Failure: + time.sleep(1) + raise Failure('api did not recover') + + +def wait_s3(timeout=90): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if compose('ps', 's3', '--format', '{{.Status}}').stdout.count('(healthy)'): + return + time.sleep(1) + raise Failure('s3 did not recover') + + +def object_key(attachment_id): + out = compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', + "select object_key from attachments where id='%s'" % attachment_id).stdout.strip() + check(out.startswith('attachments/'), 'object key not found') + return out + + +def main(): + args = arguments('S3 privacy, outage and orphan cleanup checks') + c = Client(args.base_url, args.ca_file, timeout=15) + alice, bob, eve, chat = c.bootstrap() + att = upload(c, alice, chat, 202)['json'] + wait_status(c, bob, att['id'], ('ready',), 60) + key = object_key(att['id']) + check(key != 'attachments/' + att['filename'] and att['filename'] not in key, 'object key derived from filename') + + api = (secret('s3_access_key_id'), secret('s3_secret_access_key')) + admin = (secret('s3_admin_access_key_id'), secret('s3_admin_secret_access_key')) + path = '/' + BUCKET + '/' + key + check(s3_status('GET', path) == '403', 'anonymous GET must be denied') + check(s3_status('GET', '/' + BUCKET + '/') == '403', 'anonymous listing must be denied') + check(s3_status('PUT', path + '-anon', data='x') == '403', 'anonymous PUT must be denied') + check(s3_status('GET', path, ('wrong', 'wrong')) == '403', 'wrong credentials must be denied') + check(s3_status('GET', path, api) == '200', 'api credentials cannot read') + check(s3_status('PUT', '/other-bucket-' + uuid.uuid4().hex[:8], api) == '403', 'api credentials can create buckets') + ports = compose('ps', '--format', '{{.Service}} {{.Ports}}').stdout + published = [line.split()[0] for line in ports.splitlines() if '->' in line] + check(published == ['lb'], 'only lb may publish ports, found %s' % published) + + compose('stop', 's3') + try: + upload(c, alice, chat, 503) + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 503, actor=bob) + c.call('GET', '/api/v1/attachments/' + att['id'], 200, actor=bob, schema='Attachment') + c.message(alice, chat, 'text works while s3 is down') + finally: + compose('start', 's3') + wait_s3() + wait_api(c) + deadline = time.monotonic() + 60 + while True: + try: + body = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes'] + check(body == SAMPLE, 'content differs after s3 restart') + break + except Failure: + check(time.monotonic() < deadline, 'download did not recover') + time.sleep(1) + upload(c, alice, chat, 202) + + orphan = '/' + BUCKET + '/attachments/' + uuid.uuid4().hex + check(s3_status('PUT', orphan, admin, 'orphan') == '200', 'cannot create orphan') + compose('run', '--rm', '--no-deps', '-e', 'CLEANUP_GRACE_SECONDS=3600', 'janitor', 'cleanup') + check(s3_status('GET', orphan, admin) == '200', 'fresh object was removed inside grace period') + time.sleep(1) + compose('run', '--rm', '--no-deps', '-e', 'CLEANUP_GRACE_SECONDS=0', 'janitor', 'cleanup') + check(s3_status('GET', orphan, admin) == '404', 'orphan was not removed') + check(c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes'] == SAMPLE, 'referenced object was removed') + + check(s3_status('GET', '/' + BUCKET + '/staging/' + att['id'], admin) == '404', 'staging kept after ready') + stale = '/' + BUCKET + '/staging/' + str(uuid.uuid4()) + check(s3_status('PUT', stale, admin, 'stale') == '200', 'cannot create stale staging object') + compose('stop', 'worker') + try: + queued = upload(c, alice, chat, 202)['json'] + compose('run', '--rm', '--no-deps', '-e', 'CLEANUP_GRACE_SECONDS=0', 'janitor', 'cleanup') + check(s3_status('GET', '/' + BUCKET + '/staging/' + queued['id'], admin) == '200', 'staging of queued job was removed') + check(s3_status('GET', stale, admin) == '404', 'unreferenced staging object was not removed') + finally: + compose('up', '-d', '--no-deps', 'worker') + check(wait_status(c, bob, queued['id'], ('ready', 'failed'), 60)['status'] == 'ready', 'queued job lost staging') + print('PASS: private bucket, limited credentials, s3 outage, orphan and staging cleanup') + + +if __name__ == '__main__': + run(main) diff --git a/tests/scenario.py b/tests/scenario.py new file mode 100644 index 0000000..f7ddd70 --- /dev/null +++ b/tests/scenario.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +from concurrent.futures import ThreadPoolExecutor +import json +import subprocess +import threading +import time +import uuid +from pathlib import Path +from client import Client, Failure, arguments, check, run + +ROOT = Path(__file__).resolve().parent.parent + + +def sh(*cmd): + return subprocess.run(cmd, cwd=ROOT, check=True, capture_output=True, text=True).stdout.strip() + + +def api_containers(): + out = sh('docker', 'compose', '--env-file', '.env', 'ps', '-q', 'api') + return [x[:12] for x in out.split()] + + +def wait_healthy(count, timeout=90): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + out = sh('docker', 'compose', '--env-file', '.env', 'ps', 'api', '--format', '{{.Status}}') + if out.count('(healthy)') >= count: + return + time.sleep(1) + raise Failure('replicas did not become healthy') + + +class Load: + def __init__(self, base, ca, concurrency): + c = Client(base, ca) + self.alice, _, _, self.chat = c.bootstrap() + self.base, self.ca, self.n = base, ca, concurrency + self.events = [] + self.lock = threading.Lock() + self.stop = threading.Event() + self.t0 = time.monotonic() + + def worker(self): + c = Client(self.base, self.ca, timeout=3) + while not self.stop.is_set(): + key = uuid.uuid4().hex + first_error = None + while not self.stop.is_set(): + try: + r = c.message(self.alice, self.chat, 'k-' + key, key=key) + t = time.monotonic() - self.t0 + with self.lock: + self.events.append((t, True, r['headers']['X-Instance-Id'], first_error)) + break + except Failure: + t = time.monotonic() - self.t0 + first_error = first_error or t + with self.lock: + self.events.append((t, False, None, None)) + time.sleep(0.05) + + def start(self): + self.pool = ThreadPoolExecutor(max_workers=self.n) + self.futures = [self.pool.submit(self.worker) for _ in range(self.n)] + + def finish(self): + self.stop.set() + self.pool.shutdown(wait=True) + + def summary(self, begin, end): + ev = [e for e in self.events if begin <= e[0] < end] + ok = [e for e in ev if e[1]] + bad = [e for e in ev if not e[1]] + span = max(end - begin, 1e-9) + return { + 'window_seconds': round(span, 2), + 'successful_requests': len(ok), + 'failed_attempts': len(bad), + 'successful_rps': round(len(ok) / span, 1), + 'last_error_offset_seconds': round(max(e[0] for e in bad) - begin, 2) if bad else None, + 'instances': sorted({e[2] for e in ok}), + } + + def instances_between(self, begin, end=float('inf')): + return sorted({e[2] for e in self.events if e[1] and begin <= e[0] < end}) + + +def failover_each(args): + ids = api_containers() + check(len(ids) >= 2, 'need at least two replicas') + load = Load(args.base_url, args.ca_file, args.concurrency) + load.start() + time.sleep(args.warmup) + results = [] + base = load.summary(2, args.warmup) + for target in ids: + before = time.monotonic() - load.t0 + sh('docker', 'stop', '-t', '10', target) + stopped = time.monotonic() - load.t0 + time.sleep(args.hold) + after_stop = time.monotonic() - load.t0 + window = load.summary(before, after_stop) + sh('docker', 'start', target) + wait_healthy(len(ids)) + restored = time.monotonic() - load.t0 + time.sleep(args.warmup) + results.append({ + 'stopped_replica': target, + 'during_failure': window, + 'seen_stopped_replica_after_stop': target in load.instances_between(stopped + 0.5, after_stop), + 'stop_command_seconds': round(stopped - before, 2), + 'restore_seconds': round(restored - after_stop, 2), + 'after_restore': load.summary(restored + 1, restored + args.warmup), + }) + load.finish() + return {'baseline': base, 'replicas': results} + + +def rolling(args): + old = api_containers() + check(len(old) >= 2, 'need at least two replicas') + n = len(old) + load = Load(args.base_url, args.ca_file, args.concurrency) + load.start() + time.sleep(args.warmup) + begin = time.monotonic() - load.t0 + sh('scripts/rolling-update.sh') + end = time.monotonic() - load.t0 + time.sleep(args.warmup) + load.finish() + return { + 'old_replicas': old, + 'new_replicas': api_containers(), + 'update_window': load.summary(begin, end), + 'old_replicas_served_after_update': [i for i in load.instances_between(end) if i in old], + } + + +def main(): + def configure(p): + p.add_argument('--mode', choices=['failover-each', 'rolling'], required=True) + p.add_argument('--concurrency', type=int, default=8) + p.add_argument('--warmup', type=float, default=8) + p.add_argument('--hold', type=float, default=12) + p.add_argument('--output') + args = arguments('Failure and rolling update under load', configure) + report = failover_each(args) if args.mode == 'failover-each' else rolling(args) + text = json.dumps(report, indent=2) + if args.output: + Path(args.output).write_text(text + '\n') + print(text) + + +if __name__ == '__main__': + run(main) diff --git a/tests/session_revocation.py b/tests/session_revocation.py new file mode 100644 index 0000000..2e2a80d --- /dev/null +++ b/tests/session_revocation.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +import time +from client import Client, arguments, check, run +from stack import compose + + +def wait_valkey(timeout=60): + deadline = time.monotonic() + timeout + while '(healthy)' not in compose('ps', 'valkey', '--format', '{{.Status}}').stdout: + check(time.monotonic() < deadline, 'valkey did not recover') + time.sleep(1) + + +def main(): + args = arguments('Session revocation must not depend on the cache') + c = Client(args.base_url, args.ca_file, timeout=15) + alice, bob, eve, chat = c.bootstrap() + c.call('GET', '/api/v1/chats', 200, actor=alice) + compose('stop', 'valkey') + try: + c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=alice) + finally: + compose('start', 'valkey') + wait_valkey() + c.call('GET', '/api/v1/chats', 401, actor=alice) + c.call('GET', '/api/v1/chats', 200, actor=bob) + print('PASS: token revoked while valkey was stopped stays revoked after valkey restart (401)') + + +if __name__ == '__main__': + run(main) diff --git a/tests/upload_memory.py b/tests/upload_memory.py new file mode 100644 index 0000000..aa88929 --- /dev/null +++ b/tests/upload_memory.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +from concurrent.futures import ThreadPoolExecutor +import hashlib +import json +import re +import subprocess +import threading +import time +from pathlib import Path +from client import Client, Failure, arguments, check, run +from image_gen import png + +ROOT = Path(__file__).resolve().parent.parent +UNITS = {'B': 1, 'KiB': 1024, 'MiB': 1024 ** 2, 'GiB': 1024 ** 3} + + +def to_mib(text): + value, unit = re.match(r'([\d.]+)\s*([A-Za-z]+)', text).groups() + return float(value) * UNITS[unit] / 1024 ** 2 + + +def sample(stop, peaks): + proc = subprocess.Popen(['docker', 'stats', '--format', '{{.Name}} {{.MemUsage}}'], cwd=ROOT, + stdout=subprocess.PIPE, text=True) + try: + for line in proc.stdout: + line = re.sub(r'\x1b\[[0-9;]*[A-Za-z]', '', line).strip() + m = re.match(r'(\S+-api-\d+)\s+(\S+ ?\S+) /', line) + if m: + peaks[m.group(1)] = max(peaks.get(m.group(1), 0), to_mib(m.group(2))) + if stop.is_set(): + break + finally: + proc.terminate() + + +def main(): + def configure(p): + p.add_argument('--concurrency', type=int, default=12) + p.add_argument('--rounds', type=int, default=3) + p.add_argument('--output') + args = arguments('Concurrent upload memory experiment', configure) + check(args.concurrency >= 10, 'use at least 10 concurrent uploads') + c = Client(args.base_url, args.ca_file, timeout=60) + alice, bob, eve, chat = c.bootstrap() + data = png(1800, 1800, noise=True) + digest = hashlib.sha256(data).hexdigest() + path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=load.png' + + idle, stop = {}, threading.Event() + t = threading.Thread(target=sample, args=(stop, idle)) + t.start() + time.sleep(4) + stop.set() + t.join() + + peaks, stop = {}, threading.Event() + t = threading.Thread(target=sample, args=(stop, peaks)) + t.start() + time.sleep(2) + barrier = threading.Barrier(args.concurrency) + results = [] + + def worker(_): + local = Client(args.base_url, args.ca_file, timeout=60) + barrier.wait() + out = [] + for _ in range(args.rounds): + try: + r = local.call('POST', path, 202, actor=alice, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment') + out.append(r['json']['sha256'] == digest) + except Failure: + out.append(False) + return out + + started = time.monotonic() + with ThreadPoolExecutor(max_workers=args.concurrency) as pool: + for out in pool.map(worker, range(args.concurrency)): + results.extend(out) + elapsed = time.monotonic() - started + time.sleep(2) + stop.set() + t.join() + limit = 192 + report = { + 'file_bytes': len(data), + 'concurrent_uploads': args.concurrency, + 'rounds': args.rounds, + 'uploads_ok': sum(results), + 'uploads_failed': len(results) - sum(results), + 'elapsed_seconds': round(elapsed, 2), + 'api_memory_limit_mib': limit, + 'idle_memory_mib': {k: round(v, 1) for k, v in sorted(idle.items())}, + 'peak_memory_mib': {k: round(v, 1) for k, v in sorted(peaks.items())}, + 'buffering_all_files_would_need_mib': round(args.concurrency * len(data) / 1024 ** 2, 1), + } + text = json.dumps(report, indent=2) + if args.output: + Path(args.output).write_text(text + '\n') + print(text) + check(report['uploads_failed'] == 0, 'some uploads failed') + check(max(peaks.values()) < limit, 'peak memory reached the limit') + + +if __name__ == '__main__': + run(main)