48 changed files with 3555 additions and 41 deletions
+3 -3
View File
@@ -7,7 +7,7 @@ SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
SESSION_STORE=valkey
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
@@ -19,7 +19,7 @@ S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
WORKER_CONCURRENCY=2
WORKER_CONCURRENCY=1
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
JOB_TIMEOUT_SECONDS=30
STAGING_RETENTION_SECONDS=86400
+2
View File
@@ -0,0 +1,2 @@
*.sh text eol=lf
*.sql text eol=lf
+11 -15
View File
@@ -1,18 +1,14 @@
## Лабораторная 5
Автор / группа: TODO
Целевая оценка (3/4/5): TODO
Трек 5, если заявлен: TODO
- Автор и группа — Ситосенко, БВТ2303.
- Целевая оценка — 5. Трек 2, идемпотентность версионированного pipeline.
- Основа lab4 дополнена S3 staging, транзакционным outbox, Valkey Streams, worker, ограниченными повторами и операторским replay.
- Запуск и результаты проверок приведены в [REPORT.md](../REPORT.md).
- Эксперимент использует один учебный хост и сохраняемые volumes.
- Источники и применение Codex описаны в отчёте.
Что реализовано и что перенесено: TODO
Команды запуска из чистого clone: TODO
Результаты проверок и ссылка на REPORT.md: TODO
Ограничения / известные ошибки: TODO
Источники и использование ИИ: TODO
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
- [ ] Контракт и публичные тесты не ослаблены.
- [ ] make check и make test выполнены; приложен реальный вывод.
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
- [x] Обязательный минимум сохранён с объявленным переходом upload на 202.
- [x] Контракт и публичные тесты сохранены.
- [x] Вывод публичных проверок и эксперимента приложен.
- [x] Секреты создаются локально, отдельно от Git.
- [x] README задания сохранён, инструкция запуска находится в отчёте.
+23
View File
@@ -0,0 +1,23 @@
FROM golang:1.27.3-bookworm@sha256:d9b6a1beb9474adc2a1ce842573639b44b2a0dd49b6669f51d3821baf9ec37c9 AS build
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
COPY *.go ./
COPY migrations ./migrations
RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /messenger .
FROM python:3.13.18-slim-bookworm AS worker
RUN pip install --no-cache-dir --only-binary=:all: Pillow==13.0.0
COPY --from=build /messenger /messenger
COPY pipeline.py /pipeline.py
USER 65532:65532
STOPSIGNAL SIGTERM
ENTRYPOINT ["/messenger"]
FROM scratch AS api
COPY --from=build /messenger /messenger
USER 65532:65532
ENV APP_HOST=0.0.0.0 APP_PORT=8080 GOMEMLIMIT=256MiB
EXPOSE 8080
HEALTHCHECK --interval=5s --timeout=3s --start-period=10s --retries=3 CMD ["/messenger","healthcheck"]
STOPSIGNAL SIGTERM
ENTRYPOINT ["/messenger"]
+15
View File
@@ -0,0 +1,15 @@
# Build official sources: upstream registry images are no longer public.
FROM golang:1.27.3-bookworm@sha256:d9b6a1beb9474adc2a1ce842573639b44b2a0dd49b6669f51d3821baf9ec37c9 AS build
ENV GOTOOLCHAIN=local CGO_ENABLED=0
RUN go install -p 4 github.com/minio/minio@v0.0.0-20251015172955-9e49d5e7a648
RUN go install -p 4 github.com/minio/mc@v0.0.0-20250813083541-7394ce0dd2a8
COPY infra/s3-health.go /health.go
RUN go build -trimpath -o /healthcheck /health.go
FROM alpine:3.23.3@sha256:25109184c71bdad752c8312a8623239686a9a2071e8825f20acb8f2198c3f659 AS tools
COPY --from=build /go/bin/mc /usr/bin/mc
ENTRYPOINT ["/usr/bin/mc"]
FROM scratch AS server
COPY --from=build /go/bin/minio /usr/bin/minio
COPY --from=build /healthcheck /healthcheck
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
ENTRYPOINT ["/usr/bin/minio"]
+88 -23
View File
@@ -1,37 +1,102 @@
# Отчёт — лабораторная 5
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор и группа — Ситосенко, БВТ2303.
- Целевая оценка — **5**, трек 2 «Доказанная идемпотентность pipeline».
- Реализация основана на собственном коде lab4. Codex использовался для переноса кода, реализации очереди, подготовки эксперимента и проверки.
- Commit/PR — ветка `solution/bvt2303-sitosenko`. [МР](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab5/pulls/24).
- Источники — [Valkey Streams](https://valkey.io/commands/xautoclaim/), [Valkey AOF](https://valkey.io/topics/persistence/), [условная запись S3](https://docs.aws.amazon.com/AmazonS3/latest/userguide/conditional-writes.html), [Pillow](https://pypi.org/project/pillow/).
## Запуск из чистого clone
## Запуск
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Нужны Docker с Linux-контейнерами, Compose v2 и Python 3.9+ для клиентских проверок. Для сборки нужен доступ к реестру образов, Go modules и PyPI. Команды запуска в PowerShell из корня clone.
## Архитектура и решения
```powershell
python -X utf8 scripts/bootstrap.py
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build --wait --wait-timeout 150
$env:BASE_URL = "http://127.0.0.1:8080"
python -X utf8 scripts/check_contract.py
python -X utf8 tests/smoke.py
python -X utf8 student-tests/image_rules.py
python -X utf8 student-tests/pipeline_faults.py
```
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
На Unix эквиваленты проверки контракта и публичного smoke — `make check` и `make test`. Проверки перезапуска выполняются последовательно на выделенном учебном стенде.
## Доказательства критериев
```sh
make test-persistence ACTION_SCRIPT=scripts/restart.sh
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
docker compose start api
```
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
Bootstrap генерирует локальные секреты и `.env` при их отсутствии. Миграции `001`–`005` применяет одноразовый сервис `migrate`. Старые записи lab4 с исходником в `original/<id>` читаются по этому ключу с `result_version=0`. Сервис `s3-init` создаёт приватный бакет и ограниченные учётные записи. API доступны через HAProxy на `127.0.0.1:8080`. `APP_PORT` задаёт внешний порт. При его изменении задайте тестам соответствующий `BASE_URL`.
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
`docker compose down` останавливает стенд и сохраняет volumes. `docker compose down --volumes` предназначена для намеренного удаления данных **своего учебного проекта** после завершения проверок.
## Эксперименты и отказы
Стек использует Go 1.27.3, PostgreSQL 17.6, Valkey 8.1.3, HAProxy 3.2.6, MinIO из фиксированного коммита `9e49d5e7a648`, Python 3.13.18 и Pillow 13.0.0. `compose.yaml` задаёт лимиты сервисов. Worker получает 1 vCPU, 640 MiB памяти, 32 MiB tmpfs и обрабатывает одно задание за раз с prefetch 1. Локальный HTTP используется для синтетических данных.
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
## Архитектура
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
1. API проверяет Bearer-сессию, членство в чате, filename, MIME, целостность изображения и лимиты 10 MiB и 40 млн пикселей. Исходные байты и SHA-256 передаются в приватный S3 staging под серверным UUID. Принятие до `202 queued` включает завершившийся S3 PUT и commit строки вложения вместе с записью задания в PostgreSQL.
2. `image_jobs` служит реестром заданий и transactional outbox. Dispatcher отправляет JSON одним полем `job` через XADD, затем фиксирует признак публикации. JSON несёт поля `schema_version`, `job_id`, `type`, `attachment_id`, `pipeline_version`, `attempt`, `created_at` по `jobs.schema.json`. Valkey для очереди имеет собственный volume, AOF с `appendfsync always` и `noeviction`.
3. Worker читает Streams через consumer group, захватывает задание с lease и сверяет ID, версию и номер попытки с PostgreSQL. Истёкшие lease восстанавливает dispatcher. `XAUTOCLAIM` возвращает потерянные pending delivery. После устойчивого завершения либо записи retry/failed worker подтверждает доставку и удаляет запись из Streams одной командой Lua. Дубли завершённых и устаревших попыток подтверждаются по сохранённому состоянию.
4. Worker проверяет SHA-256 и длину исходника и запускает ограниченный по времени процесс Pillow. Thumbnail — JPEG качества 85, вписанный в 256×256, с округлением вниз и без увеличения. Crop — JPEG качества 85, центрированный квадрат 128×128. Optimized — WebP качества 80 в v1 и 90 в v2, до 1280×1280 без увеличения. Для вариантов применяется EXIF orientation, прозрачность компонуется на белый фон, а EXIF, ICC и текстовые метаданные удаляются. Исходник сохраняется побайтно.
5. Эффект привязан к `(attachment_id, pipeline_version, variant)`. S3 PUT с `If-None-Match: *` исключает повторную запись уже существующего объекта. При 412 сверяются размер, MIME и SHA-256 из метаданных объекта. После записи всех объектов одна транзакция фиксирует результат с проверкой активной версии и lease. GET использует атомарно опубликованную версию. Читатели видят v1 в ходе подготовки v2 и переключаются на v2 после commit.
6. Невалидное изображение и нарушение целостности считаются постоянными ошибками. Временный отказ возвращает задание в очередь с экспоненциальной задержкой и jitter от 100% до 200% базового интервала, с базой от 1 до 32 секунд. По исчерпании лимита выдаётся `retry_exhausted`. Dispatcher очищает staging после публикации результата, для failed — после retention, для orphan — после часа и повторной проверки строки под тем же lock, что использует upload.
7. Разрешения S3 разделены для API и фоновой обработки. API пишет staging и читает опубликованные варианты, фоновая учётная запись может записывать результаты и удалять staging. API сохраняет регистрацию, Argon2id, постоянные сессии Valkey, права чата, keyset-пагинацию и идемпотентность сообщений из lab4. Broker, worker и dispatcher работают независимо от синхронной readiness API, которая проверяет PostgreSQL, Valkey сессий и S3.
## Ограничения и следующие шаги
`JOB_MAX_ATTEMPTS` задаёт число попыток за цикл (5 по умолчанию), `JOB_TIMEOUT_SECONDS` — время обработки (30 секунд). Lease длиннее на 5 секунд. `STAGING_RETENTION_SECONDS` — срок хранения failed-исходника для replay (86400 секунд). `WORKER_CONCURRENCY=1` закрепляет один слот в контейнере. Лимиты байт и пикселей зафиксированы в коде по контракту. Настройки endpoint, region и bucket в Compose относятся к встроенному локальному S3.
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Команды оператора доступны через Compose и используют доступ к PostgreSQL.
```sh
docker compose run --rm --no-deps dispatcher jobs failed
docker compose run --rm --no-deps dispatcher jobs replay <job-id>
docker compose run --rm --no-deps dispatcher jobs upgrade <attachment-id>
```
Replay разрешён для текущего failed-задания, пока staging может использоваться. Он сохраняет `job_id`, увеличивает attempt, начинает новый цикл и добавляет запись в аудит. Для v2 источником служит опубликованный оригинал v1. Upgrade переводит готовый результат v1 на v2 без изменения публичного HTTP API.
## Эксперимент
**Гипотеза.** Подтверждённая загрузка доходит до ready после каждого моделируемого сбоя. Переотправка или переупорядочивание заданий сохраняет по одному набору объектов на версию и оставляет опубликованную v2 неизменной.
Скрипт `student-tests/pipeline_faults.py` использует синтетический PNG 320×200 и последовательно проводит сценарии на одном стенде. Каждый сценарий получает новый attachment ID. Для сбоя запускается отдельный контейнер нужной роли с failpoint, привязанным к этому ID. Скрипт проверяет код завершения 86 и маркер точки сбоя. Через HTTP, PostgreSQL, Streams и S3 проверяются статусы, попытки, опубликованные версии, размеры, контрольные суммы и время записи объектов. Время в JSON отсчитывается от получения 202 и включает запуск контейнеров и ожидание подтверждений. Это последовательная проверка корректности, без нагрузочного прогона.
| Критерий | Проверка | Доказательство |
| --- | --- | --- |
| Публичный контракт и smoke | `scripts/check_contract.py`, `tests/smoke.py` | [contract.txt](evidence/contract.txt), [openapi.txt](evidence/openapi.txt), [smoke.txt](evidence/smoke.txt) |
| Отправка сообщения с queued-вложением, перезапуск API и broker при остановленном worker | Сценарий `api_broker_restart_with_worker_stopped`, сверка StartedAt, записи Streams и оригинала | [pipeline.json](evidence/pipeline.json) |
| Геометрия, EXIF, alpha, форматы и лимиты | `student-tests/image_rules.py` | [image-rules.txt](evidence/image-rules.txt) |
| COMMIT до XADD и сбой после XADD | Сценарии `before_publish` и `after_publish` | [pipeline.json](evidence/pipeline.json) |
| Lease и сбой до/после записи S3 и после commit до ack | `before_effect`, `after_effect`, `after_commit`, проверка pending и LastModified/ETag | [pipeline.json](evidence/pipeline.json) |
| Ограниченный retry, failed и операторский replay | `invalid_image_replay`, `temporary_replay`, проверка backoff и записи аудита | [pipeline.json](evidence/pipeline.json) |
| Reorder и поздняя доставка старой версии | `reorder_and_stale_version`, проверка версии и неизменности S3 | [pipeline.json](evidence/pipeline.json) |
Прогон выполнялся в Docker Desktop на Windows, движок Docker 29.3.0, Compose 5.1.0, Linux amd64. Docker VM предоставляла 12 логических CPU и 3,68 GiB памяти. Дополнительного прогрева не было, перед экспериментом прошли smoke и проверка форматов. Для каждого воздействия использовалась одна загрузка и отдельный набор ожидаемых переходов.
## Результаты прогона
| Воздействие | Наблюдение | Время сценария |
| --- | --- | --- |
| Перезапуск API и брокера | Та же запись Streams сохранилась после перезапуска трёх контейнеров. Сообщение было создано с queued-вложением. Исходник и варианты доступны после запуска worker. | 39,9 с |
| Сбой до XADD | В Streams было 0 записей. Dispatcher восстановил задание из outbox, после чего загрузка стала ready. | 29,0 с |
| Сбой после XADD | Dispatcher опубликовал тот же job_id повторно. Две доставки завершились одним набором вариантов и одной попыткой. | 29,0 с |
| Сбой до записи S3 | В S3 результата ещё не было. Истечение lease запустило вторую попытку, вложение стало ready. | 60,8 с |
| Сбой после записи S3 | До сбоя был записан оригинал. Вторая попытка сохранила его ETag и LastModified, дописала варианты и опубликовала результат. | 88,9 с |
| Сбой после commit до ack | Была pending-доставка при ready. Повтор подтвердил её без дополнительной попытки и перезаписи объекта. | 49,0 с |
| Постоянная ошибка | Одна попытка, затем invalid_image. Операторский replay завершился ready с одной записью аудита. | 42,2 с |
| Временная ошибка | После 5 попыток получен retry_exhausted. После replay вложение стало ready. | 73,7 с |
| Переупорядочивание и поздний дубль | Последовательность версий 2, 1, 2, 1, 2 сохранила один результат на версию и итоговую v2. Проверены контрольные суммы, ETag и LastModified. | 62,4 с |
Во всех девяти сценариях подтверждённая загрузка дошла до ready. Сырые результаты с ID и длительностью сохранены в [pipeline.json](evidence/pipeline.json). Прогон выполнен с настройками из `.env.example` и именем проекта `mtusi-lab5`.
Для сборки Linux amd64 `govulncheck` не нашёл достижимых уязвимостей в Go-коде. Проверка охватывает Go-пакеты приложения. [go-vuln.txt](evidence/go-vuln.txt)
Публичные persistence и failover прошли с неизменёнными тестами и сессией. Сценарии сохранили подтверждённые сообщения и оригинал вложения. Прогон failover через localhost на Windows дал около 2 с задержки клиента на запрос и исчерпал окно записи. При первом прогоне по IPv4 холодный запуск Argon2id превысил таймаут регистрации. Зачётный прогон через 127.0.0.1 после двух синтетических регистраций и входов дал 115 успешных записей, 4 переходных ошибки и первую успешную запись после остановки через 3,881 с. [persistence.txt](evidence/persistence.txt), [failover.txt](evidence/failover.txt), [до перезапуска](evidence/restart-before.txt), [после перезапуска](evidence/restart-after.txt), [до остановки реплики](evidence/stopped-before.txt), [после остановки реплики](evidence/stopped-after.txt)
## Границы
XADD и commit в PostgreSQL — отдельные операции. Подтверждение XADD означает приём брокером. Consumer ack относится к отдельной доставке и может потеряться после успешной фиксации результата. Поэтому Streams доставляет at least once, а единственность эффекта обеспечивают условная запись S3 и проверка состояния в PostgreSQL.
Эксперимент проверяет отказы процессов и рестарт broker с сохранённым volume. Потеря хоста или постоянного хранилища, сетевой split-brain и максимальная пропускная способность за пределами методики. Старые конечные версии сохраняются для чтения и анализа. MinIO CE унаследован из lab4 и ограничен внутренним сегментом и фильтрацией HAProxy. Стенд рассчитан на локальный учебный запуск.
+322
View File
@@ -0,0 +1,322 @@
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"image"
_ "image/jpeg"
_ "image/png"
"io"
"mime"
"net"
"net/http"
"net/url"
"os"
"strconv"
"strings"
"time"
"unicode"
"unicode/utf8"
"github.com/jackc/pgx/v5"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
)
const maxUploadBytes int64 = 10 * 1024 * 1024
const maxImagePixels int64 = 40000000
type Variant struct {
Name string `json:"name"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
Width int `json:"width"`
Height int `json:"height"`
SHA256 string `json:"sha256"`
}
type Attachment struct {
ID string `json:"id"`
ChatID string `json:"chat_id"`
OwnerID string `json:"owner_id"`
Filename string `json:"filename"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
SHA256 string `json:"sha256"`
Status string `json:"status"`
Variants []Variant `json:"variants"`
ErrorCode *string `json:"error_code"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
resultVersion int
}
func s3Client() (*minio.Client, string, error) {
endpoint, err := url.Parse(env("S3_ENDPOINT", "http://s3:9000"))
if err != nil || (endpoint.Scheme != "http" && endpoint.Scheme != "https") || endpoint.Host == "" || endpoint.User != nil || endpoint.RawQuery != "" || (endpoint.Path != "" && endpoint.Path != "/") {
return nil, "", errors.New("invalid S3 endpoint")
}
access, err := secretFile("S3_ACCESS_KEY_ID_FILE")
if err != nil {
return nil, "", err
}
secret, err := secretFile("S3_SECRET_ACCESS_KEY_FILE")
if err != nil {
return nil, "", err
}
transport := &http.Transport{Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{Timeout: 2 * time.Second, KeepAlive: 30 * time.Second}).DialContext, MaxIdleConns: 16, MaxIdleConnsPerHost: 8, MaxConnsPerHost: 8, ResponseHeaderTimeout: 5 * time.Second, TLSHandshakeTimeout: 3 * time.Second, IdleConnTimeout: 30 * time.Second, DisableCompression: true}
client, err := minio.New(endpoint.Host, &minio.Options{Creds: credentials.NewStaticV4(access, secret, ""), Secure: endpoint.Scheme == "https", Region: env("S3_REGION", "us-east-1"), Transport: transport, MaxRetries: 1, BucketLookup: minio.BucketLookupPath})
return client, env("S3_BUCKET", "messenger-attachments"), err
}
func validFilename(s string) bool {
if !utf8.ValidString(s) || strings.TrimSpace(s) == "" || utf8.RuneCountInString(s) > 255 {
return false
}
for _, c := range s {
if c == '/' || c == '\\' || unicode.IsControl(c) {
return false
}
}
return true
}
func messageFields(body map[string]json.RawMessage) (string, []string, error) {
var text string
if string(body["text"]) == "null" || json.Unmarshal(body["text"], &text) != nil || utf8.RuneCountInString(text) > 4000 {
return "", nil, invalid()
}
ids := []string{}
if raw, ok := body["attachment_ids"]; ok {
if string(raw) == "null" || json.Unmarshal(raw, &ids) != nil || len(ids) > 10 {
return "", nil, invalid()
}
}
seen := map[string]bool{}
for i, id := range ids {
normalized, err := normalizeID(id)
if err != nil || seen[normalized] {
return "", nil, invalid()
}
ids[i] = normalized
seen[normalized] = true
}
if strings.TrimSpace(text) == "" && len(ids) == 0 {
return "", nil, invalid()
}
return text, ids, nil
}
func (a *App) s3Ready(ctx context.Context) bool {
check, cancel := context.WithTimeout(ctx, time.Second)
defer cancel()
ok, err := a.s3.BucketExists(check, a.bucket)
return err == nil && ok
}
func mediaError() error {
return apiError{415, "unsupported_media_type", "Expected a valid PNG or JPEG matching Content-Type"}
}
func tooLarge() error { return apiError{413, "payload_too_large", "Image exceeds byte or pixel limit"} }
// The encoded file lives only in a bounded tmpfs during validation/PUT. At most
// five uploads and one decoded image per process; no bytes enter PostgreSQL.
func (a *App) upload(ctx context.Context, w http.ResponseWriter, r *http.Request, chat, user string) (Attachment, error) {
result := Attachment{}
query, err := url.ParseQuery(r.URL.RawQuery)
if err != nil || len(query) != 1 || len(query["filename"]) != 1 || !validFilename(query.Get("filename")) {
return result, invalid()
}
contentType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
if err != nil || len(params) != 0 || (contentType != "image/png" && contentType != "image/jpeg") {
return result, mediaError()
}
if r.ContentLength > maxUploadBytes {
return result, tooLarge()
}
select {
case a.uploadSlots <- struct{}{}:
defer func() { <-a.uploadSlots }()
default:
return result, unavailable()
}
f, err := os.CreateTemp("", "upload-*")
if err != nil {
return result, unavailable()
}
defer os.Remove(f.Name())
defer f.Close()
digest := sha256.New()
size, err := io.Copy(io.MultiWriter(f, digest), http.MaxBytesReader(w, r.Body, maxUploadBytes))
if err != nil {
var big *http.MaxBytesError
if errors.As(err, &big) {
return result, tooLarge()
}
return result, invalid()
}
if size == 0 {
return result, mediaError()
}
if _, err = f.Seek(0, io.SeekStart); err != nil {
return result, unavailable()
}
cfg, format, err := image.DecodeConfig(f)
if err != nil || (format != "png" && format != "jpeg") || "image/"+format != contentType {
return result, mediaError()
}
if cfg.Width <= 0 || cfg.Height <= 0 || int64(cfg.Width) > maxImagePixels/int64(cfg.Height) {
return result, tooLarge()
}
select {
case a.decodeSlot <- struct{}{}:
case <-ctx.Done():
return result, unavailable()
}
validation := func() error {
defer func() { <-a.decodeSlot }()
if _, e := f.Seek(0, io.SeekStart); e != nil {
return unavailable()
}
decoded, actual, e := image.Decode(f)
if e != nil || actual != format || decoded.Bounds().Dx() != cfg.Width || decoded.Bounds().Dy() != cfg.Height {
return mediaError()
}
return nil
}
if err = validation(); err != nil {
return result, err
}
if ctx.Err() != nil {
return result, unavailable()
}
if _, err = f.Seek(0, io.SeekStart); err != nil {
return result, unavailable()
}
result = Attachment{ID: uuid(), ChatID: chat, OwnerID: user, Filename: query.Get("filename"), ContentType: contentType, SizeBytes: size, SHA256: hex.EncodeToString(digest.Sum(nil)), Status: "queued", Variants: []Variant{}}
tx, err := a.db.Begin(ctx)
if err != nil {
return result, unavailable()
}
defer rollbackAttachment(tx)
if err = attachmentLock(ctx, tx, result.ID); err != nil {
return result, unavailable()
}
key := "original/" + result.ID
_, err = a.s3.PutObject(ctx, a.bucket, stagingKey(result.ID), f, size, minio.PutObjectOptions{ContentType: contentType, DisableMultipart: true, UserMetadata: map[string]string{"sha256": result.SHA256}})
if err != nil {
return result, unavailable()
}
var created, updated time.Time
err = tx.QueryRow(ctx, `INSERT INTO attachments(id,chat_id,owner_id,object_key,filename,content_type,size_bytes,sha256,status,active_version)
VALUES($1,$2,$3,$4,$5,$6,$7,$8,'queued',1) RETURNING created_at,updated_at`, result.ID, chat, user, key, result.Filename, contentType, size, result.SHA256).Scan(&created, &updated)
if err != nil {
return result, unavailable()
}
if err = enqueueImage(ctx, tx, result.ID, 1); err != nil {
return result, unavailable()
}
// COMMIT may have succeeded even when its acknowledgement is lost. Never
// blindly delete the blob here: reconciler checks durable metadata + grace.
if err = tx.Commit(ctx); err != nil {
return result, unavailable()
}
result.CreatedAt = stamp(created)
result.UpdatedAt = stamp(updated)
return result, nil
}
func attachmentLock(ctx context.Context, tx pgx.Tx, id string) error {
_, err := tx.Exec(ctx, "SELECT pg_advisory_xact_lock(hashtextextended($1,0))", "attachment:"+id)
return err
}
func rollbackAttachment(tx pgx.Tx) {
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
_ = tx.Rollback(ctx)
}
func (a *App) attachment(ctx context.Context, id, user string) (Attachment, error) {
x := Attachment{}
var created, updated time.Time
var missingSince *time.Time
err := a.db.QueryRow(ctx, `SELECT a.id::text,a.chat_id::text,a.owner_id::text,a.filename,a.content_type,a.size_bytes,a.sha256,a.created_at,a.updated_at,a.missing_since,a.status,a.variants,a.error_code,a.result_version
FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id AND m.user_id=$2 WHERE a.id=$1`, id, user).Scan(&x.ID, &x.ChatID, &x.OwnerID, &x.Filename, &x.ContentType, &x.SizeBytes, &x.SHA256, &created, &updated, &missingSince, &x.Status, &x.Variants, &x.ErrorCode, &x.resultVersion)
if err != nil {
return x, dbError(err)
}
if missingSince != nil {
return x, unavailable()
}
x.CreatedAt = stamp(created)
x.UpdatedAt = stamp(updated)
return x, nil
}
type binaryContent struct {
attachment Attachment
object *minio.Object
}
func (a *App) content(ctx context.Context, r *http.Request, x Attachment) (binaryContent, error) {
q, err := url.ParseQuery(r.URL.RawQuery)
if err != nil || len(q) > 1 || len(q["variant"]) > 1 {
return binaryContent{}, invalid()
}
for k := range q {
if k != "variant" {
return binaryContent{}, invalid()
}
}
name := "original"
if values, ok := q["variant"]; ok {
name = values[0]
switch name {
case "original", "thumbnail", "crop", "optimized":
default:
return binaryContent{}, invalid()
}
}
if x.Status != "ready" {
return binaryContent{}, apiError{409, "conflict", "Attachment is not ready"}
}
if name != "original" {
found := false
for _, variant := range x.Variants {
if variant.Name == name {
x.ContentType, x.SizeBytes = variant.ContentType, variant.SizeBytes
found = true
break
}
}
if !found {
return binaryContent{}, missing()
}
}
object, err := a.s3.GetObject(ctx, a.bucket, resultKey(x.ID, x.resultVersion, name), minio.GetObjectOptions{})
if err != nil {
return binaryContent{}, unavailable()
}
info, err := object.Stat()
if err != nil || info.Size != x.SizeBytes {
object.Close()
return binaryContent{}, unavailable()
}
return binaryContent{x, object}, nil
}
func writeContent(w http.ResponseWriter, b binaryContent) {
defer b.object.Close()
w.Header().Set("Content-Type", b.attachment.ContentType)
w.Header().Set("Content-Length", strconv.FormatInt(b.attachment.SizeBytes, 10))
w.Header().Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": b.attachment.Filename}))
w.Header().Set("X-Content-Type-Options", "nosniff")
w.Header().Set("Cache-Control", "private, no-store")
w.WriteHeader(200)
if _, err := io.CopyN(w, b.object, b.attachment.SizeBytes); err != nil {
panic(http.ErrAbortHandler)
}
}
+147
View File
@@ -0,0 +1,147 @@
package main
import (
"context"
"crypto/rand"
"crypto/sha256"
"crypto/subtle"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"net/http"
"strings"
"time"
"unicode/utf8"
"github.com/jackc/pgx/v5"
"github.com/redis/go-redis/v9"
"golang.org/x/crypto/argon2"
)
func password(body map[string]json.RawMessage) (string, error) {
var p string
if json.Unmarshal(body["password"], &p) != nil || utf8.RuneCountInString(p) < 12 || utf8.RuneCountInString(p) > 128 {
return "", invalid()
}
return p, nil
}
func hashPassword(p string) string {
salt := make([]byte, 16)
if _, err := rand.Read(salt); err != nil {
panic(err)
}
key := argon2.IDKey([]byte(p), salt, 3, 64*1024, 1, 32)
return "$argon2id$v=19$m=65536,t=3,p=1$" + base64.RawStdEncoding.EncodeToString(salt) + "$" + base64.RawStdEncoding.EncodeToString(key)
}
func verifyPassword(hash, p string) bool {
parts := strings.Split(hash, "$")
if len(parts) != 6 || parts[1] != "argon2id" || parts[2] != "v=19" || parts[3] != "m=65536,t=3,p=1" {
return false
}
salt, e := base64.RawStdEncoding.DecodeString(parts[4])
if e != nil || len(salt) != 16 {
return false
}
want, e := base64.RawStdEncoding.DecodeString(parts[5])
if e != nil || len(want) != 32 {
return false
}
actual := argon2.IDKey([]byte(p), salt, 3, 64*1024, 1, 32)
return subtle.ConstantTimeCompare(want, actual) == 1
}
func (a *App) kdfSlot(ctx context.Context) error {
select {
case a.kdf <- struct{}{}:
return nil
case <-ctx.Done():
return unavailable()
}
}
type storedSession struct {
UserID string `json:"user_id"`
ExpiresAt time.Time `json:"expires_at"`
}
func sessionKey(token string) string {
h := sha256.Sum256([]byte(token))
return "sess:" + hex.EncodeToString(h[:])
}
func (a *App) authenticate(ctx context.Context, r *http.Request) (string, string, error) {
values := r.Header.Values("Authorization")
if len(values) != 1 {
return "", "", unauthorized()
}
parts := strings.Fields(values[0])
if len(parts) != 2 || !strings.EqualFold(parts[0], "Bearer") {
return "", "", unauthorized()
}
token, e := base64.RawURLEncoding.DecodeString(parts[1])
if e != nil || len(token) != 32 {
return "", "", unauthorized()
}
key := sessionKey(parts[1])
raw, err := a.sessions.Get(ctx, key).Bytes()
if errors.Is(err, redis.Nil) {
return "", "", unauthorized()
}
if err != nil {
return "", "", unavailable()
}
var session storedSession
if json.Unmarshal(raw, &session) != nil {
return "", "", unavailable()
}
if !time.Now().Before(session.ExpiresAt) {
return "", "", unauthorized()
}
if _, err = a.user(ctx, session.UserID); err != nil {
var ae apiError
if errors.As(err, &ae) && ae.status == 404 {
return "", "", unauthorized()
}
return "", "", err
}
return session.UserID, key, nil
}
func (a *App) login(ctx context.Context, r *http.Request) (int, any, error) {
if r.ContentLength > 0 || len(r.TransferEncoding) > 0 {
return 0, nil, invalid()
}
username, p, ok := r.BasicAuth()
if len(r.Header.Values("Authorization")) != 1 || !ok || !usernamePattern.MatchString(username) || !utf8.ValidString(p) || utf8.RuneCountInString(p) < 12 || utf8.RuneCountInString(p) > 128 {
return 0, nil, unauthorized()
}
var hash string
var u User
var created time.Time
err := a.db.QueryRow(ctx, "SELECT id::text,username,display_name,created_at,password_hash FROM users WHERE username=$1", username).Scan(&u.ID, &u.Username, utf8String{&u.DisplayName}, &created, &hash)
absent := errors.Is(err, pgx.ErrNoRows)
if err != nil && !absent {
return 0, nil, unavailable()
}
if absent {
hash = a.dummyHash
}
if err = a.kdfSlot(ctx); err != nil {
return 0, nil, err
}
valid := verifyPassword(hash, p)
<-a.kdf
if !valid || absent {
return 0, nil, unauthorized()
}
u.CreatedAt = stamp(created)
tokenBytes := make([]byte, 32)
if _, err = rand.Read(tokenBytes); err != nil {
return 0, nil, unavailable()
}
token := base64.RawURLEncoding.EncodeToString(tokenBytes)
expires := time.Now().UTC().Truncate(time.Second).Add(a.ttl)
raw, _ := json.Marshal(storedSession{u.ID, expires})
if err = a.sessions.SetArgs(ctx, sessionKey(token), raw, redis.SetArgs{Mode: "NX", ExpireAt: expires}).Err(); err != nil {
return 0, nil, unavailable()
}
return 201, map[string]any{"token": token, "token_type": "Bearer", "expires_at": expires.Format(time.RFC3339), "user": u}, nil
}
+231
View File
@@ -0,0 +1,231 @@
name: mtusi-lab5-local
x-image-service: &image-service
restart: unless-stopped
environment:
DATABASE_URL_FILE: /run/secrets/database_url
DB_POOL_MAX: 4
S3_ENDPOINT: http://s3:9000
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_worker_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_worker_secret_access_key
BROKER_URL_FILE: ${BROKER_URL_FILE:-/run/secrets/broker_url}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-30}
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-1}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
GOMEMLIMIT: 96MiB
secrets: [database_url, s3_worker_access_key_id, s3_worker_secret_access_key, broker_url]
depends_on:
migrate: {condition: service_completed_successfully}
s3-init: {condition: service_completed_successfully}
broker: {condition: service_started}
read_only: true
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
healthcheck: {disable: true}
logging:
options: {max-size: 10m, max-file: "2"}
services:
db:
image: postgres:17.6-bookworm@sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets: [postgres_password]
volumes: [pgdata:/var/lib/postgresql/data]
healthcheck:
test: [CMD-SHELL, "pg_isready -h 127.0.0.1 -U messenger -d messenger"]
interval: 2s
timeout: 2s
retries: 30
mem_limit: 512m
cpus: ${DB_CPUS:-1.0}
valkey:
image: valkey/valkey:8.1.3-bookworm@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571
command: [valkey-server, /run/secrets/valkey_config]
secrets: [valkey_config, valkey_password]
volumes: [sessions:/data]
healthcheck:
test: [CMD-SHELL, 'REDISCLI_AUTH="$$(cat /run/secrets/valkey_password)" valkey-cli ping | grep -q PONG']
interval: 2s
timeout: 2s
retries: 30
mem_limit: 256m
cpus: 0.5
broker:
image: valkey/valkey:8.1.3-bookworm@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571
command: [valkey-server, /run/secrets/broker_config]
secrets: [broker_config, broker_password]
volumes: [brokerdata:/data]
healthcheck:
test: [CMD-SHELL, 'REDISCLI_AUTH="$$(cat /run/secrets/broker_password)" valkey-cli ping | grep -q PONG']
interval: 2s
timeout: 2s
retries: 30
mem_limit: 256m
cpus: 0.5
migrate:
image: mtusi-messenger:lab5
build: .
command: [migrate]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
secrets: [database_url]
depends_on:
db: {condition: service_healthy}
mem_limit: 256m
cpus: 0.5
read_only: true
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
api:
image: mtusi-messenger:lab5
build: .
deploy:
replicas: ${API_REPLICAS:-2}
environment:
APP_HOST: ${APP_HOST:-0.0.0.0}
APP_PORT: 8080
APP_ENV: ${APP_ENV:-lab}
SESSION_STORE: ${SESSION_STORE:-valkey}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
DB_POOL_MAX: ${DB_POOL_MAX:-8}
MAX_INFLIGHT: ${MAX_INFLIGHT:-32}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
GOMEMLIMIT: ${GOMEMLIMIT:-512MiB}
GODEBUG: madvdontneed=0
POOL_METRICS: ${POOL_METRICS:-0}
REQUEST_LOG: ${REQUEST_LOG:-1}
DATABASE_URL_FILE: ${DATABASE_URL_FILE:-/run/secrets/database_url}
CACHE_URL_FILE: ${CACHE_URL_FILE:-/run/secrets/cache_url}
CURSOR_SECRET_FILE: ${CURSOR_SECRET_FILE:-/run/secrets/cursor_secret}
S3_ENDPOINT: http://s3:9000
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
secrets: [database_url, cache_url, cursor_secret, s3_access_key_id, s3_secret_access_key]
tmpfs: ["/tmp:size=64m,mode=1777,noexec,nosuid,nodev"]
depends_on:
db: {condition: service_healthy}
valkey: {condition: service_healthy}
migrate: {condition: service_completed_successfully}
s3-init: {condition: service_completed_successfully}
read_only: true
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
mem_limit: ${API_MEMORY:-768m}
cpus: ${API_CPUS:-1.0}
pids_limit: 128
logging:
options: {max-size: 10m, max-file: "2"}
stop_grace_period: 10s
s3:
image: mtusi-minio:RELEASE.2025-10-15T17-29-55Z
build:
context: .
dockerfile: Dockerfile.minio
command: [server, /data, --console-address, "127.0.0.1:9001"]
environment:
MINIO_ROOT_USER_FILE: /run/secrets/s3_root_user
MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_root_password
MINIO_BROWSER: "off"
secrets: [s3_root_user, s3_root_password]
volumes: [objects:/data]
networks:
object_store:
aliases: [minio-backend]
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
mem_limit: 1g
cpus: 1.0
healthcheck:
test: [CMD, /healthcheck]
interval: 2s
timeout: 2s
retries: 60
s3-init:
image: mtusi-minio-tools:RELEASE.2025-08-13T08-35-41Z
build:
context: .
dockerfile: Dockerfile.minio
target: tools
entrypoint: [/bin/sh, /bootstrap/s3-init.sh]
secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key, s3_worker_access_key_id, s3_worker_secret_access_key]
volumes: [./infra/s3-init.sh:/bootstrap/s3-init.sh:ro, ./infra/s3-policy.json:/bootstrap/policy.json:ro, ./infra/s3-worker-policy.json:/bootstrap/worker-policy.json:ro]
depends_on:
s3: {condition: service_healthy}
lb: {condition: service_started}
tmpfs: ["/tmp:size=16m,mode=1777,noexec,nosuid,nodev"]
environment: {MC_CONFIG_DIR: /tmp/mc}
read_only: true
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
mem_limit: 128m
cpus: 0.5
dispatcher:
<<: *image-service
image: mtusi-messenger:lab5
build: .
command: [dispatcher]
mem_limit: 192m
cpus: 0.5
worker:
<<: *image-service
image: mtusi-messenger-worker:lab5
build: {context: ., target: worker}
command: [worker]
tmpfs: ["/tmp:size=32m,mode=1777,noexec,nosuid,nodev"]
mem_limit: 640m
cpus: 1.0
pids_limit: 64
lb:
image: haproxy:3.2.6-alpine
ports: ["127.0.0.1:${APP_PORT:-8080}:8080"]
volumes: [./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro]
depends_on:
s3: {condition: service_healthy}
networks:
default:
aliases: [s3]
object_store: {}
read_only: true
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
mem_limit: 128m
cpus: 0.5
stop_grace_period: 10s
logging:
options: {max-size: 10m, max-file: "2"}
healthcheck:
test: [CMD-SHELL, "wget -q -O /dev/null http://127.0.0.1:8080/health/ready"]
interval: 2s
timeout: 2s
retries: 30
networks:
object_store:
internal: true
volumes:
pgdata:
sessions:
brokerdata:
objects:
secrets:
postgres_password: {file: ./secrets/postgres_password}
database_url: {file: ./secrets/database_url}
valkey_password: {file: ./secrets/valkey_password}
valkey_config: {file: ./secrets/valkey.conf}
cache_url: {file: ./secrets/cache_url}
cursor_secret: {file: ./secrets/cursor_secret}
s3_root_user: {file: ./secrets/s3_root_user}
s3_root_password: {file: ./secrets/s3_root_password}
s3_access_key_id: {file: ./secrets/s3_access_key_id}
s3_secret_access_key: {file: ./secrets/s3_secret_access_key}
broker_url: {file: ./secrets/broker_url}
broker_password: {file: ./secrets/broker_password}
broker_config: {file: ./secrets/broker.conf}
s3_worker_access_key_id: {file: ./secrets/s3_worker_access_key_id}
s3_worker_secret_access_key: {file: ./secrets/s3_worker_secret_access_key}
+1
View File
@@ -0,0 +1 @@
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
+3
View File
@@ -0,0 +1,3 @@
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 115, "failed_attempts": 4, "first_recovery_seconds": 3.881}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
+9
View File
@@ -0,0 +1,9 @@
=== Symbol Results ===
No vulnerabilities found.
Your code is affected by 0 vulnerabilities.
This scan also found 0 vulnerabilities in packages you import and 3
vulnerabilities in modules you require, but your code doesn't appear to call
these vulnerabilities.
Use '-show verbose' for more details.
+3
View File
@@ -0,0 +1,3 @@
image/jpeg attachment=7fc7382c-c1ff-4606-a3cc-712eaa50fdbf dimensions=[(85, 256), (128, 128), (100, 300)]
image/png attachment=612a2488-bb20-420a-a19b-2611f27737b3 dimensions=[(8, 4), (128, 128), (8, 4)]
PASS EXIF, fit, crop, WebP, transparency, metadata removal, byte/pixel/format limits
+1
View File
@@ -0,0 +1 @@
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
+2
View File
@@ -0,0 +1,2 @@
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
+91
View File
@@ -0,0 +1,91 @@
[
{
"scenario": "api_broker_restart_with_worker_stopped",
"attachment_id": "ac9f52f0-8b17-4613-9b4e-01918c60f164",
"seconds": 39.867,
"message_id": "977f8ad3-4a55-4a2f-b5b8-7281aee9f733",
"job_id": "dec6b536-d9d5-4371-9b06-5a8ded18b3e0",
"processes_restarted": 3
},
{
"scenario": "before_publish",
"attachment_id": "fb18b732-4e70-4fb7-9808-a5f338520c00",
"seconds": 28.952,
"copies": 1,
"attempt": 1
},
{
"scenario": "after_publish",
"attachment_id": "83af5eda-3f01-44b1-ab41-2d8c3dcae9f6",
"seconds": 29.022,
"copies": 2,
"attempt": 1
},
{
"scenario": "before_effect",
"attachment_id": "1bbb2aaa-d551-443d-86b7-3bcc8a0d0ebe",
"seconds": 60.844,
"state_at_crash": "processing",
"object_at_crash": false,
"final_attempt": 2,
"staging_removed": true
},
{
"scenario": "after_effect",
"attachment_id": "c40aa0c1-8d91-44c5-a7f9-e464194164c5",
"seconds": 88.928,
"state_at_crash": "processing",
"object_at_crash": true,
"final_attempt": 2,
"staging_removed": true
},
{
"scenario": "after_commit",
"attachment_id": "193dbffc-a5ae-4d45-a2eb-01457874ad8f",
"seconds": 48.973,
"state_at_crash": "ready",
"object_at_crash": true,
"final_attempt": 1,
"staging_removed": true
},
{
"scenario": "invalid_image_replay",
"attachment_id": "e85abbd1-4069-41c6-9d23-b1aacdbcf8f2",
"seconds": 42.186,
"failed_attempt": 1,
"error_code": "invalid_image",
"retry_delays_ms": {},
"replay_audit": 1
},
{
"scenario": "temporary_replay",
"attachment_id": "5d4083f7-5649-4efe-9f6b-ad4360380194",
"seconds": 73.706,
"failed_attempt": 5,
"error_code": "retry_exhausted",
"retry_delays_ms": {
"2": 1628,
"3": 2080,
"4": 6975,
"5": 12512
},
"replay_audit": 1
},
{
"scenario": "reorder_and_stale_version",
"attachment_id": "44f672bb-b13e-4910-ada6-8826531561c4",
"seconds": 62.444,
"delivery_versions": [
2,
1,
2,
1,
2
],
"result_version": 2,
"attempts": [
1,
1
]
}
]
+9
View File
@@ -0,0 +1,9 @@
52196e225aa0c30cb3b8e137294544e553a3c711bbeccec9050dc89eb2ff7a21 api 2026-11-05T11:17:40.315162038Z running
a500b4905027d34bfd298158d98df1cac489a2809f7bde08e232df13c8a7263f api 2026-11-05T11:17:40.339787099Z running
b35f0f1b79f444793b3df4e4c68b4afc05d9c97b08c0a30392d797f3a080a9be broker 2026-11-05T11:17:39.017739902Z running
a47468ebeefaa9c3b052ac1f801cae716a109a062b4f64734219c79f7c52e8d8 db 2026-11-05T11:17:39.235121485Z running
7ecc90a34761bd4ee79d05b72cecc0138fb82e0cabbad464fcc111f6e21a6ddd dispatcher 2026-11-05T11:17:41.119753562Z running
89d66e72f743c002310046f7fd8fe7999e84a0afbf21d7d75492bf4c25f55e38 lb 2026-11-05T11:17:39.636866059Z running
73b0478b615f5b64f9b1521869ff5117cac557e3f9a99eaee0edc2c1ad50d685 s3 2026-11-05T11:17:38.846869865Z running
23c968fe12bb060c64fd9c1968f1246ca27be3d658f0a86e40076576047ff050 valkey 2026-11-05T11:17:39.102087274Z running
42f10feab1bf81d4ca84e9753fedf2777b544c874eb70cece6d1d31a0f14d9c9 worker 2026-11-05T11:17:39.842033858Z running
+9
View File
@@ -0,0 +1,9 @@
52196e225aa0c30cb3b8e137294544e553a3c711bbeccec9050dc89eb2ff7a21 api 2026-11-05T11:08:41.229724315Z running
a500b4905027d34bfd298158d98df1cac489a2809f7bde08e232df13c8a7263f api 2026-11-05T11:08:41.185233575Z running
b35f0f1b79f444793b3df4e4c68b4afc05d9c97b08c0a30392d797f3a080a9be broker 2026-11-05T11:08:39.817778853Z running
a47468ebeefaa9c3b052ac1f801cae716a109a062b4f64734219c79f7c52e8d8 db 2026-11-05T11:04:20.990065265Z running
7ecc90a34761bd4ee79d05b72cecc0138fb82e0cabbad464fcc111f6e21a6ddd dispatcher 2026-11-05T11:09:52.937079509Z running
89d66e72f743c002310046f7fd8fe7999e84a0afbf21d7d75492bf4c25f55e38 lb 2026-11-05T11:02:00.180487515Z running
73b0478b615f5b64f9b1521869ff5117cac557e3f9a99eaee0edc2c1ad50d685 s3 2026-11-05T11:01:56.746828889Z running
23c968fe12bb060c64fd9c1968f1246ca27be3d658f0a86e40076576047ff050 valkey 2026-11-05T11:01:56.735026576Z running
42f10feab1bf81d4ca84e9753fedf2777b544c874eb70cece6d1d31a0f14d9c9 worker 2026-11-05T11:16:21.654964917Z running
+1
View File
@@ -0,0 +1 @@
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
+1
View File
@@ -0,0 +1 @@
52196e225aa0c30cb3b8e137294544e553a3c711bbeccec9050dc89eb2ff7a21 exited 2026-11-05T11:22:28.153861915Z 0
+1
View File
@@ -0,0 +1 @@
52196e225aa0c30cb3b8e137294544e553a3c711bbeccec9050dc89eb2ff7a21 running 2026-11-05T11:17:40.315162038Z
+37
View File
@@ -0,0 +1,37 @@
module mtusi-messenger
go 1.26.0
toolchain go1.27.3
require (
github.com/jackc/pgx/v5 v5.9.2
github.com/minio/minio-go/v7 v7.4.1
github.com/redis/go-redis/v9 v9.12.1
golang.org/x/crypto v0.57.0
)
require (
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/klauspost/compress v1.19.2 // indirect
github.com/klauspost/cpuid/v2 v2.4.0 // indirect
github.com/klauspost/crc32 v1.3.0 // indirect
github.com/minio/crc64nvme v1.1.1 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/philhofer/fwd v1.2.0 // indirect
github.com/rs/xid v1.6.0 // indirect
github.com/tinylib/msgp v1.6.4 // indirect
github.com/zeebo/xxh3 v1.1.0 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect
golang.org/x/net v0.60.0 // indirect
golang.org/x/sync v0.23.0 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/text v0.42.0 // indirect
gopkg.in/ini.v1 v1.67.3 // indirect
)
+77
View File
@@ -0,0 +1,77 @@
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw=
github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw=
github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.4.1 h1:PtU/zIOfQRWZ+QydXAZysc+xZYHdG+BBR4Z6TUvy73w=
github.com/minio/minio-go/v7 v7.4.1/go.mod h1:nQnQi7zTfXR4lREtYXZQWqo5aw0Rqv+ZqCGrTORygRk=
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/redis/go-redis/v9 v9.12.1 h1:k5iquqv27aBtnTm2tIkROUDp8JBXhXZIVu1InSgvovg=
github.com/redis/go-redis/v9 v9.12.1/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw=
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ=
github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU=
golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw=
gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+81
View File
@@ -0,0 +1,81 @@
package main
import (
"context"
"errors"
"slices"
"sort"
"time"
"github.com/jackc/pgx/v5"
)
// The advisory lock serializes this key across processes. Message and result
// commit together; an interrupted/rolled-back transaction reserves nothing.
func (a *App) createMessage(ctx context.Context, chat, user, key, text string, ids []string) (Message, error) {
m := Message{ChatID: chat, SenderID: user, Text: text, AttachmentIDs: ids}
tx, err := a.db.Begin(ctx)
if err != nil {
return m, unavailable()
}
defer tx.Rollback(ctx)
// POST and the canonical chat path are implicit in this dedicated table.
if _, err = tx.Exec(ctx, "SELECT pg_advisory_xact_lock(hashtextextended($1,0))", user+":"+chat+":"+key); err != nil {
return m, unavailable()
}
var previous Message
var created time.Time
err = tx.QueryRow(ctx, `SELECT m.id::text,m.chat_id::text,m.sender_id::text,m.text,m.created_at,ARRAY(SELECT unnest(m.attachment_ids)::text)
FROM message_keys k JOIN messages m ON m.id=k.message_id
WHERE k.user_id=$1 AND k.chat_id=$2 AND k.key=$3 AND k.expires_at>clock_timestamp()`, user, chat, key).Scan(&previous.ID, &previous.ChatID, &previous.SenderID, utf8String{&previous.Text}, &created, &previous.AttachmentIDs)
if err == nil {
if previous.Text != text || !slices.Equal(previous.AttachmentIDs, ids) {
return m, apiError{409, "conflict", "Idempotency key has a different message"}
}
previous.CreatedAt = stamp(created)
return previous, nil
}
if !errors.Is(err, pgx.ErrNoRows) {
return m, unavailable()
}
if _, err = tx.Exec(ctx, "DELETE FROM message_keys WHERE user_id=$1 AND chat_id=$2 AND key=$3", user, chat, key); err != nil {
return m, unavailable()
}
// Serialize with attachment state transitions; stable ordering prevents cross-upload deadlocks.
locked := slices.Clone(ids)
sort.Strings(locked)
for _, id := range locked {
if err = attachmentLock(ctx, tx, id); err != nil {
return m, unavailable()
}
var missingSince *time.Time
var status string
err = tx.QueryRow(ctx, "SELECT missing_since,status FROM attachments WHERE id=$1 AND chat_id=$2", id, chat).Scan(&missingSince, &status)
if err != nil {
return m, dbError(err)
}
if status == "failed" {
return m, apiError{409, "conflict", "Attachment processing failed"}
}
if missingSince != nil {
return m, unavailable()
}
}
m.ID = uuid()
if err = tx.QueryRow(ctx, "INSERT INTO messages(id,chat_id,sender_id,text,attachment_ids) VALUES($1,$2,$3,$4,$5) RETURNING created_at", m.ID, chat, user, []byte(text), ids).Scan(&created); err != nil {
return m, dbError(err)
}
if _, err = tx.Exec(ctx, "INSERT INTO message_keys(user_id,chat_id,key,message_id,expires_at) VALUES($1,$2,$3,$4,clock_timestamp()+$5::bigint*interval '1 second')", user, chat, key, m.ID, int64(a.idempotencyTTL.Seconds())); err != nil {
return m, unavailable()
}
for _, id := range ids {
if _, err = tx.Exec(ctx, "INSERT INTO message_attachments(message_id,chat_id,attachment_id) VALUES($1,$2,$3)", m.ID, chat, id); err != nil {
return m, dbError(err)
}
}
if err = tx.Commit(ctx); err != nil {
return m, unavailable()
}
m.CreatedAt = stamp(created)
return m, nil
}
+54
View File
@@ -0,0 +1,54 @@
package main
import (
"context"
"errors"
"strings"
"time"
"github.com/jackc/pgx/v5"
"github.com/minio/minio-go/v7"
)
func (a *App) cleanStaging(ctx context.Context, retention int) error {
listCtx, cancel := context.WithCancel(ctx)
defer cancel()
for entry := range a.s3.ListObjects(listCtx, a.bucket, minio.ListObjectsOptions{Prefix: "staging/", Recursive: true}) {
if entry.Err != nil {
return entry.Err
}
id := strings.TrimPrefix(entry.Key, "staging/")
if normalized, err := normalizeID(id); err != nil || normalized != id {
continue
}
if err := a.cleanStagedImage(ctx, id, entry.LastModified, retention); err != nil {
return err
}
}
return ctx.Err()
}
func (a *App) cleanStagedImage(ctx context.Context, id string, modified time.Time, retention int) error {
tx, err := a.db.Begin(ctx)
if err != nil {
return err
}
defer rollbackAttachment(tx)
var locked bool
if err = tx.QueryRow(ctx, "SELECT pg_try_advisory_xact_lock(hashtextextended($1,0))", "attachment:"+id).Scan(&locked); err != nil || !locked {
return err
}
var removable bool
err = tx.QueryRow(ctx, `SELECT result_version>0 OR
(status='failed' AND updated_at<=clock_timestamp()-$2::bigint*interval '1 second')
FROM attachments WHERE id=$1`, id, retention).Scan(&removable)
if errors.Is(err, pgx.ErrNoRows) {
removable = time.Since(modified) > time.Hour
} else if err != nil {
return err
}
if removable {
return a.s3.RemoveObject(ctx, a.bucket, stagingKey(id), minio.RemoveObjectOptions{})
}
return nil
}
+132
View File
@@ -0,0 +1,132 @@
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"io"
"os"
"os/exec"
"path/filepath"
"strconv"
"github.com/minio/minio-go/v7"
)
type imageFailure string
func (e imageFailure) Error() string { return string(e) }
func (a *App) generateImage(ctx context.Context, j imageJob, src imageSource) ([]Variant, error) {
dir, err := os.MkdirTemp("", "image-job-")
if err != nil {
return nil, err
}
defer os.RemoveAll(dir)
key := stagingKey(j.AttachmentID)
if j.Version == 2 {
key = resultKey(j.AttachmentID, 1, "original")
}
original := filepath.Join(dir, "input")
if err = a.fetchImage(ctx, src, key, original); err != nil {
return nil, err
}
command := exec.CommandContext(ctx, "/usr/local/bin/python3", "-I", "/pipeline.py", dir, src.contentType, strconv.Itoa(j.Version))
command.Env = []string{"LC_ALL=C.UTF-8"}
manifest, err := command.Output()
if err != nil {
var exit *exec.ExitError
if errors.As(err, &exit) && exit.ExitCode() == 2 {
return nil, imageFailure("invalid_image")
}
return nil, err
}
var variants []Variant
if err = json.Unmarshal(manifest, &variants); err != nil || len(variants) != 3 {
return nil, errors.New("invalid encoder manifest")
}
if err = a.putImage(ctx, resultKey(j.AttachmentID, j.Version, "original"), original, src); err != nil {
return nil, err
}
injectImageFault("after_effect", j.AttachmentID)
for i, name := range []string{"thumbnail", "crop", "optimized"} {
v := &variants[i]
v.Name = name
v.ContentType = "image/jpeg"
if name == "optimized" {
v.ContentType = "image/webp"
}
path := filepath.Join(dir, name)
v.SizeBytes, v.SHA256, err = fileDigest(path)
if err != nil {
return nil, err
}
if err = a.putImage(ctx, resultKey(j.AttachmentID, j.Version, name), path, imageSource{v.ContentType, v.SizeBytes, v.SHA256}); err != nil {
return nil, err
}
}
return variants, nil
}
func (a *App) fetchImage(ctx context.Context, src imageSource, key, path string) error {
object, err := a.s3.GetObject(ctx, a.bucket, key, minio.GetObjectOptions{})
if err != nil {
return err
}
defer object.Close()
file, err := os.Create(path)
if err != nil {
return err
}
digest := sha256.New()
size, err := io.Copy(io.MultiWriter(file, digest), io.LimitReader(object, maxUploadBytes+1))
closeErr := file.Close()
if err != nil {
return err
}
if closeErr != nil {
return closeErr
}
if size != src.size || hex.EncodeToString(digest.Sum(nil)) != src.sha256 {
return imageFailure("source_integrity")
}
return nil
}
func fileDigest(path string) (int64, string, error) {
file, err := os.Open(path)
if err != nil {
return 0, "", err
}
defer file.Close()
digest := sha256.New()
size, err := io.Copy(digest, file)
return size, hex.EncodeToString(digest.Sum(nil)), err
}
func (a *App) putImage(ctx context.Context, key, path string, src imageSource) error {
file, err := os.Open(path)
if err != nil {
return err
}
defer file.Close()
opts := minio.PutObjectOptions{ContentType: src.contentType, DisableMultipart: true,
UserMetadata: map[string]string{"sha256": src.sha256}}
opts.SetMatchETagExcept("*")
_, err = a.s3.PutObject(ctx, a.bucket, key, file, src.size, opts)
if minio.ToErrorResponse(err).Code != "PreconditionFailed" {
return err
}
info, err := a.s3.StatObject(ctx, a.bucket, key, minio.StatObjectOptions{})
if err != nil {
return err
}
if info.Size != src.size || info.ContentType != src.contentType || info.Metadata.Get("X-Amz-Meta-Sha256") != src.sha256 {
return imageFailure("result_integrity")
}
return nil
}
+249
View File
@@ -0,0 +1,249 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"math/rand/v2"
"os"
"time"
"github.com/jackc/pgx/v5"
)
type imageJob struct {
SchemaVersion int `json:"schema_version"`
JobID string `json:"job_id"`
Type string `json:"type"`
AttachmentID string `json:"attachment_id"`
Version int `json:"pipeline_version"`
Attempt int `json:"attempt"`
CreatedAt string `json:"created_at"`
}
type imageSource struct {
contentType string
size int64
sha256 string
}
func enqueueImage(ctx context.Context, tx pgx.Tx, attachment string, version int) error {
_, err := tx.Exec(ctx, "INSERT INTO image_jobs(id,attachment_id,version) VALUES($1,$2,$3)", uuid(), attachment, version)
return err
}
func stagingKey(id string) string { return "staging/" + id }
func resultKey(id string, version int, variant string) string {
if version == 0 {
return "original/" + id
}
return fmt.Sprintf("%s/%s/v%d", variant, id, version)
}
func (a *App) claimImage(ctx context.Context, j imageJob, lease time.Duration) (imageSource, bool, error) {
var src imageSource
tx, err := a.db.Begin(ctx)
if err != nil {
return src, false, err
}
defer rollbackAttachment(tx)
if err = attachmentLock(ctx, tx, j.AttachmentID); err != nil {
return src, false, err
}
tag, err := tx.Exec(ctx, `UPDATE image_jobs j SET state='processing',
lease_until=clock_timestamp()+$5::bigint*interval '1 second', updated_at=clock_timestamp()
FROM attachments a WHERE a.id=j.attachment_id AND a.active_version=j.version
AND j.id=$1 AND j.attachment_id=$2 AND j.version=$3 AND j.attempt=$4
AND j.state='queued' AND j.next_at<=clock_timestamp()`, j.JobID, j.AttachmentID, j.Version, j.Attempt, int64(lease.Seconds()))
if err != nil || tag.RowsAffected() == 0 {
return src, false, err
}
if _, err = tx.Exec(ctx, `UPDATE attachments SET status='processing',updated_at=clock_timestamp()
WHERE id=$1 AND result_version=0`, j.AttachmentID); err != nil {
return src, false, err
}
err = tx.QueryRow(ctx, "SELECT content_type,size_bytes,sha256 FROM attachments WHERE id=$1", j.AttachmentID).
Scan(&src.contentType, &src.size, &src.sha256)
if err != nil {
return src, false, err
}
return src, true, tx.Commit(ctx)
}
func (a *App) completeImage(ctx context.Context, j imageJob, variants []Variant) error {
tx, err := a.db.Begin(ctx)
if err != nil {
return err
}
defer rollbackAttachment(tx)
if err = attachmentLock(ctx, tx, j.AttachmentID); err != nil {
return err
}
tag, err := tx.Exec(ctx, `UPDATE image_jobs SET state='ready',lease_until=NULL,error_code=NULL,updated_at=clock_timestamp()
WHERE id=$1 AND attempt=$2 AND state='processing' AND lease_until>clock_timestamp()`, j.JobID, j.Attempt)
if err != nil || tag.RowsAffected() == 0 {
return err // A lost lease is recovered by the dispatcher.
}
tag, err = tx.Exec(ctx, `UPDATE attachments SET status='ready',error_code=NULL,result_version=$2,
variants=$3,updated_at=clock_timestamp() WHERE id=$1 AND active_version=$2 AND result_version<$2`,
j.AttachmentID, j.Version, variants)
if err != nil {
return err
}
if tag.RowsAffected() != 1 {
return errors.New("image result fence rejected")
}
return tx.Commit(ctx)
}
func (a *App) failImage(ctx context.Context, j imageJob, code string, permanent, expired bool, maxAttempts int) error {
tx, err := a.db.Begin(ctx)
if err != nil {
return err
}
defer rollbackAttachment(tx)
if err = attachmentLock(ctx, tx, j.AttachmentID); err != nil {
return err
}
var cycleStart int
err = tx.QueryRow(ctx, `SELECT cycle_start FROM image_jobs WHERE id=$1 AND attempt=$2
AND state='processing' AND (lease_until<=clock_timestamp())=$3 FOR UPDATE`, j.JobID, j.Attempt, expired).Scan(&cycleStart)
if errors.Is(err, pgx.ErrNoRows) {
return nil
}
if err != nil {
return err
}
used := j.Attempt - cycleStart + 1
state := "failed"
if !permanent {
if used >= maxAttempts {
code = "retry_exhausted"
} else {
state = "queued"
}
}
attempt := j.Attempt
var delay int64
if state == "queued" {
attempt++
base := int64(1000) << min(used-1, 5)
delay = base + rand.Int64N(base)
}
_, err = tx.Exec(ctx, `UPDATE image_jobs SET state=$2,error_code=$3,lease_until=NULL,attempt=$4,
published=false,next_at=clock_timestamp()+$5::bigint*interval '1 millisecond',updated_at=clock_timestamp() WHERE id=$1`,
j.JobID, state, code, attempt, delay)
if err != nil {
return err
}
var publicCode *string
if state == "failed" {
publicCode = &code
}
if _, err = tx.Exec(ctx, `UPDATE attachments SET status=$2,error_code=$3,updated_at=clock_timestamp()
WHERE id=$1 AND active_version=$4 AND result_version=0`, j.AttachmentID, state, publicCode, j.Version); err != nil {
return err
}
return tx.Commit(ctx)
}
func (a *App) recoverImage(ctx context.Context, maxAttempts int) error {
var j imageJob
err := a.db.QueryRow(ctx, `SELECT id::text,attachment_id::text,version,attempt FROM image_jobs
WHERE state='processing' AND lease_until<=clock_timestamp() ORDER BY lease_until LIMIT 1`).
Scan(&j.JobID, &j.AttachmentID, &j.Version, &j.Attempt)
if errors.Is(err, pgx.ErrNoRows) {
return nil
}
if err != nil {
return err
}
return a.failImage(ctx, j, "lease_expired", false, true, maxAttempts)
}
func (a *App) upgradeImage(ctx context.Context, id string) error {
tx, err := a.db.Begin(ctx)
if err != nil {
return err
}
defer rollbackAttachment(tx)
if err = attachmentLock(ctx, tx, id); err != nil {
return err
}
tag, err := tx.Exec(ctx, `UPDATE attachments SET active_version=2
WHERE id=$1 AND active_version=1 AND result_version=1 AND status='ready'`, id)
if err != nil {
return err
}
if tag.RowsAffected() != 1 {
return errors.New("attachment is not eligible for upgrade")
}
if err = enqueueImage(ctx, tx, id, 2); err != nil {
return err
}
return tx.Commit(ctx)
}
func (a *App) replayImage(ctx context.Context, id string, retention int) error {
var attachment string
if err := a.db.QueryRow(ctx, "SELECT attachment_id::text FROM image_jobs WHERE id=$1", id).Scan(&attachment); err != nil {
return errors.New("job lookup failed")
}
tx, err := a.db.Begin(ctx)
if err != nil {
return err
}
defer rollbackAttachment(tx)
if err = attachmentLock(ctx, tx, attachment); err != nil {
return err
}
var attempt int
var code string
err = tx.QueryRow(ctx, `SELECT j.attempt,j.error_code FROM image_jobs j JOIN attachments a ON a.id=j.attachment_id
WHERE j.id=$1 AND j.state='failed' AND a.active_version=j.version AND a.result_version<j.version
AND (j.version=2 OR j.updated_at>clock_timestamp()-$2::bigint*interval '1 second')`, id, retention).Scan(&attempt, &code)
if errors.Is(err, pgx.ErrNoRows) {
return errors.New("job is not eligible for replay")
}
if err != nil {
return err
}
attempt++
if _, err = tx.Exec(ctx, "INSERT INTO image_job_replays(job_id,attempt,previous_error) VALUES($1,$2,$3)", id, attempt, code); err != nil {
return err
}
if _, err = tx.Exec(ctx, `UPDATE image_jobs SET state='queued',attempt=$2,cycle_start=$2,error_code=NULL,
published=false,next_at=clock_timestamp(),updated_at=clock_timestamp() WHERE id=$1`, id, attempt); err != nil {
return err
}
if _, err = tx.Exec(ctx, `UPDATE attachments SET status='queued',error_code=NULL,updated_at=clock_timestamp()
WHERE id=$1 AND result_version=0`, attachment); err != nil {
return err
}
return tx.Commit(ctx)
}
func (a *App) listFailedImages(ctx context.Context) error {
rows, err := a.db.Query(ctx, `SELECT id::text,attachment_id::text,version,attempt,error_code,updated_at FROM image_jobs
WHERE state='failed' ORDER BY updated_at DESC,id LIMIT 100`)
if err != nil {
return err
}
defer rows.Close()
out := json.NewEncoder(os.Stdout)
for rows.Next() {
var j imageJob
var code string
var updated time.Time
if err = rows.Scan(&j.JobID, &j.AttachmentID, &j.Version, &j.Attempt, &code, &updated); err != nil {
return err
}
if err = out.Encode(map[string]any{"job_id": j.JobID, "attachment_id": j.AttachmentID,
"pipeline_version": j.Version, "attempt": j.Attempt, "error_code": code, "updated_at": stamp(updated)}); err != nil {
return err
}
}
return rows.Err()
}
+263
View File
@@ -0,0 +1,263 @@
package main
import (
"context"
"encoding/json"
"errors"
"log/slog"
"os"
"os/signal"
"strings"
"syscall"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/redis/go-redis/v9"
)
const imageStream = "attachment-jobs"
const imageGroup = "image-workers"
var ackImage = redis.NewScript(`
if redis.call('XACK', KEYS[1], ARGV[1], ARGV[2]) == 1 then
redis.call('XDEL', KEYS[1], ARGV[2])
end
return 1
`)
func imageCommand(db *pgxpool.Pool, args []string) error {
a := &App{db: db}
retention := positive("STAGING_RETENTION_SECONDS", 86400)
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
if args[0] == "jobs" {
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
if len(args) == 2 && args[1] == "failed" {
return a.listFailedImages(ctx)
}
if len(args) != 3 {
return errors.New("usage: jobs failed | jobs replay <job-id> | jobs upgrade <attachment-id>")
}
id, err := normalizeID(args[2])
if err != nil {
return errors.New("invalid job or attachment ID")
}
switch args[1] {
case "replay":
return a.replayImage(ctx, id, retention)
case "upgrade":
return a.upgradeImage(ctx, id)
default:
return errors.New("unknown jobs command")
}
}
if len(args) != 1 || (args[0] != "dispatcher" && args[0] != "worker") {
return errors.New("unknown command")
}
attempts := positive("JOB_MAX_ATTEMPTS", 5)
timeoutSeconds := positive("JOB_TIMEOUT_SECONDS", 30)
if attempts > 20 || timeoutSeconds > 120 || positive("WORKER_CONCURRENCY", 1) != 1 {
return errors.New("invalid image job limits")
}
timeout := time.Duration(timeoutSeconds) * time.Second
s3, bucket, err := s3Client()
if err != nil {
return errors.New("invalid image storage configuration")
}
a.s3, a.bucket = s3, bucket
brokerURL, err := secretFile("BROKER_URL_FILE")
if err != nil {
return err
}
opts, err := redis.ParseURL(brokerURL)
if err != nil {
return errors.New("invalid broker configuration")
}
opts.MaxRetries = -1
opts.PoolSize = 2
opts.DialTimeout = time.Second
opts.ReadTimeout = 2 * time.Second
opts.WriteTimeout = time.Second
opts.ContextTimeoutEnabled = true
broker := redis.NewClient(opts)
defer broker.Close()
if args[0] == "dispatcher" {
a.dispatchImages(ctx, broker, attempts, retention)
} else {
a.consumeImages(ctx, broker, attempts, timeout)
}
return nil
}
func injectImageFault(phase, attachment string) {
if os.Getenv("LAB5_FAULT_ATTACHMENT") == attachment && os.Getenv("LAB5_FAULT") == phase {
slog.Info("test crash", "phase", phase, "attachment_id", attachment)
os.Exit(86)
}
}
func (a *App) publishImage(ctx context.Context, broker *redis.Client) (bool, error) {
tx, err := a.db.Begin(ctx)
if err != nil {
return false, err
}
defer rollbackAttachment(tx)
j := imageJob{SchemaVersion: 1, Type: "attachment.process.v1"}
var created time.Time
err = tx.QueryRow(ctx, `SELECT id::text,attachment_id::text,version,attempt,created_at FROM image_jobs
WHERE state='queued' AND NOT published AND next_at<=clock_timestamp()
ORDER BY next_at,id LIMIT 1 FOR UPDATE SKIP LOCKED`).Scan(&j.JobID, &j.AttachmentID, &j.Version, &j.Attempt, &created)
if errors.Is(err, pgx.ErrNoRows) {
return false, nil
}
if err != nil {
return false, err
}
j.CreatedAt = stamp(created)
payload, err := json.Marshal(j)
if err != nil {
return false, err
}
injectImageFault("before_publish", j.AttachmentID)
// XADD replies after the broker's AOF durability boundary. A lost reply
// or crash before the SQL commit causes another delivery of the same job.
err = broker.XAdd(ctx, &redis.XAddArgs{Stream: imageStream, Values: map[string]any{"job": string(payload)}}).Err()
if err != nil {
return false, err
}
injectImageFault("after_publish", j.AttachmentID)
if _, err = tx.Exec(ctx, "UPDATE image_jobs SET published=true WHERE id=$1", j.JobID); err != nil {
return false, err
}
return true, tx.Commit(ctx)
}
func (a *App) dispatchImages(ctx context.Context, broker *redis.Client, attempts, retention int) {
cleanup := time.Now()
for ctx.Err() == nil {
step, cancel := context.WithTimeout(ctx, 5*time.Second)
recoverErr := a.recoverImage(step, attempts)
published, publishErr := a.publishImage(step, broker)
cancel()
if recoverErr != nil || publishErr != nil {
slog.Warn("image outbox unavailable", "recover_failed", recoverErr != nil, "publish_failed", publishErr != nil)
}
if time.Now().After(cleanup) {
step, cancel = context.WithTimeout(ctx, 5*time.Second)
if err := a.cleanStaging(step, retention); err != nil && ctx.Err() == nil {
slog.Warn("staging cleanup incomplete")
}
cancel()
cleanup = time.Now().Add(15 * time.Second)
}
if !published || recoverErr != nil {
waitImage(ctx, 300*time.Millisecond)
}
}
}
func waitImage(ctx context.Context, delay time.Duration) {
timer := time.NewTimer(delay)
defer timer.Stop()
select {
case <-ctx.Done():
case <-timer.C:
}
}
func (a *App) consumeImages(ctx context.Context, broker *redis.Client, attempts int, timeout time.Duration) {
cursor := "0-0"
groupReady := false
for ctx.Err() == nil {
if !groupReady {
err := broker.XGroupCreateMkStream(ctx, imageStream, imageGroup, "0").Err()
if err != nil && !strings.HasPrefix(err.Error(), "BUSYGROUP") {
waitImage(ctx, time.Second)
continue
}
groupReady = true
}
messages, next, err := broker.XAutoClaim(ctx, &redis.XAutoClaimArgs{
Stream: imageStream, Group: imageGroup, Consumer: "worker", MinIdle: timeout + 10*time.Second, Start: cursor, Count: 1,
}).Result()
if err == nil {
cursor = next
if len(messages) == 0 {
var streams []redis.XStream
streams, err = broker.XReadGroup(ctx, &redis.XReadGroupArgs{
Group: imageGroup, Consumer: "worker", Streams: []string{imageStream, ">"}, Count: 1, Block: time.Second,
}).Result()
if err == nil && len(streams) > 0 {
messages = streams[0].Messages
}
}
}
if err != nil && !errors.Is(err, redis.Nil) {
groupReady = false
waitImage(ctx, time.Second)
continue
}
for _, message := range messages {
if err = a.handleImage(ctx, message, attempts, timeout); err != nil {
slog.Warn("image delivery left pending", "delivery_id", message.ID)
continue
}
if err = ackImage.Run(ctx, broker, []string{imageStream}, imageGroup, message.ID).Err(); err != nil {
slog.Warn("image acknowledgement unavailable", "delivery_id", message.ID)
}
}
}
}
func (a *App) handleImage(ctx context.Context, message redis.XMessage, attempts int, timeout time.Duration) error {
var j imageJob
payload, ok := message.Values["job"].(string)
if !ok || len(payload) > 2048 || json.Unmarshal([]byte(payload), &j) != nil ||
j.SchemaVersion != 1 || j.Type != "attachment.process.v1" || j.Attempt < 1 || (j.Version != 1 && j.Version != 2) {
slog.Warn("invalid image delivery", "delivery_id", message.ID)
return nil
}
for _, id := range []string{j.JobID, j.AttachmentID} {
if normalized, err := normalizeID(id); err != nil || normalized != id {
slog.Warn("invalid image delivery ID", "delivery_id", message.ID)
return nil
}
}
claimCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
src, claimed, err := a.claimImage(claimCtx, j, timeout+5*time.Second)
cancel()
if err != nil || !claimed {
return err // Stale/duplicate attempts are no-ops. A processing row has a recoverable lease.
}
injectImageFault("before_effect", j.AttachmentID)
processCtx, cancel := context.WithTimeout(ctx, timeout)
var variants []Variant
switch {
case os.Getenv("LAB5_FAULT_ATTACHMENT") == j.AttachmentID && os.Getenv("LAB5_FAULT") == "invalid_image":
err = imageFailure("invalid_image")
case os.Getenv("LAB5_FAULT_ATTACHMENT") == j.AttachmentID && os.Getenv("LAB5_FAULT") == "temporary":
err = errors.New("test transient failure")
default:
variants, err = a.generateImage(processCtx, j, src)
}
cancel()
finishCtx, cancel := context.WithTimeout(ctx, 4*time.Second)
defer cancel()
if err != nil {
code := "processing_failed"
var permanent imageFailure
if errors.As(err, &permanent) {
code = string(permanent)
}
return a.failImage(finishCtx, j, code, permanent != "", false, attempts)
}
if err = a.completeImage(finishCtx, j, variants); err != nil {
return err
}
injectImageFault("after_commit", j.AttachmentID)
return nil
}
+62
View File
@@ -0,0 +1,62 @@
global
log stdout format raw local0
maxconn 2048
defaults
log global
mode http
option httplog
timeout connect 500ms
timeout client 40s
timeout server 40s
timeout http-request 3s
timeout http-keep-alive 5s
timeout check 500ms
retries 0
resolvers docker
nameserver dns 127.0.0.11:53
accepted_payload_size 8192
resolve_retries 2
timeout resolve 1s
timeout retry 1s
hold valid 1s
hold obsolete 1s
hold nx 1s
frontend public
bind :8080
default_backend api
backend api
balance roundrobin
option httpchk
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
http-check expect status 200
server-template replica 1-4 api:8080 check inter 500ms fall 2 rise 2 resolvers docker resolve-prefer ipv4 init-addr none
listen stats
bind :8404
stats enable
stats uri /stats
# MinIO is reachable only through this internal listener. Public port 9000 is
# never published. Block unsupported vulnerable features in the archived CE.
frontend private_s3
bind :9000
no log
acl unsigned_trailer req.hdr(x-amz-content-sha256) -i STREAMING-UNSIGNED-PAYLOAD-TRAILER
http-request deny deny_status 403 if unsigned_trailer
acl internal_storage path,url_dec -m beg -i /minio/storage /minio/peer /minio/grid /minio/bootstrap
http-request deny deny_status 403 if internal_storage
acl s3_select query,url_dec -m reg -i '(^|&)select([=&]|$)'
http-request deny deny_status 403 if s3_select
acl replication_header req.hdr(x-minio-source-replication-request) -m found
http-request deny deny_status 403 if replication_header
default_backend minio_private
backend minio_private
option httpchk
http-check send meth GET uri /minio/health/ready ver HTTP/1.1 hdr Host minio-backend
http-check expect status 200
server minio minio-backend:9000 check inter 1s resolvers docker init-addr none
+20
View File
@@ -0,0 +1,20 @@
// Standalone health probe for the minimal MinIO container.
package main
import (
"net/http"
"os"
"time"
)
func main() {
client := http.Client{Timeout: time.Second}
response, err := client.Get("http://127.0.0.1:9000/minio/health/ready")
if err != nil {
os.Exit(1)
}
response.Body.Close()
if response.StatusCode != 200 {
os.Exit(1)
}
}
+17
View File
@@ -0,0 +1,17 @@
#!/bin/sh
set -eu
# No shell tracing and no credentials in persistent config or output.
for attempt in $(seq 1 30); do
if wget -q -O /dev/null http://s3:9000/minio/health/ready; then break; fi
sleep 1
done
mc alias set local http://s3:9000 "$(cat /run/secrets/s3_root_user)" "$(cat /run/secrets/s3_root_password)" >/dev/null
mc mb --ignore-existing local/messenger-attachments >/dev/null
mc anonymous set none local/messenger-attachments >/dev/null
mc admin user add local "$(cat /run/secrets/s3_access_key_id)" "$(cat /run/secrets/s3_secret_access_key)" >/dev/null
mc admin policy create local messenger-api /bootstrap/policy.json >/dev/null
mc admin policy attach local messenger-api --user "$(cat /run/secrets/s3_access_key_id)" >/dev/null
mc admin user add local "$(cat /run/secrets/s3_worker_access_key_id)" "$(cat /run/secrets/s3_worker_secret_access_key)" >/dev/null
mc admin policy create local messenger-worker /bootstrap/worker-policy.json >/dev/null
mc admin policy attach local messenger-worker --user "$(cat /run/secrets/s3_worker_access_key_id)" >/dev/null
echo 'Private bucket and restricted service accounts ready.'
+8
View File
@@ -0,0 +1,8 @@
{
"Version": "2012-10-17",
"Statement": [
{"Effect": "Allow", "Action": ["s3:GetBucketLocation", "s3:ListBucket"], "Resource": ["arn:aws:s3:::messenger-attachments"]},
{"Effect": "Allow", "Action": ["s3:PutObject"], "Resource": ["arn:aws:s3:::messenger-attachments/staging/*"]},
{"Effect": "Allow", "Action": ["s3:GetObject"], "Resource": ["arn:aws:s3:::messenger-attachments/original/*", "arn:aws:s3:::messenger-attachments/thumbnail/*", "arn:aws:s3:::messenger-attachments/crop/*", "arn:aws:s3:::messenger-attachments/optimized/*"]}
]
}
+8
View File
@@ -0,0 +1,8 @@
{
"Version": "2012-10-17",
"Statement": [
{"Effect": "Allow", "Action": ["s3:GetBucketLocation", "s3:ListBucket"], "Resource": ["arn:aws:s3:::messenger-attachments"]},
{"Effect": "Allow", "Action": ["s3:GetObject", "s3:DeleteObject"], "Resource": ["arn:aws:s3:::messenger-attachments/staging/*"]},
{"Effect": "Allow", "Action": ["s3:PutObject", "s3:GetObject"], "Resource": ["arn:aws:s3:::messenger-attachments/original/*", "arn:aws:s3:::messenger-attachments/thumbnail/*", "arn:aws:s3:::messenger-attachments/crop/*", "arn:aws:s3:::messenger-attachments/optimized/*"]}
]
}
+441
View File
@@ -0,0 +1,441 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"log/slog"
"net"
"net/http"
"os"
"os/signal"
"strconv"
"strings"
"sync/atomic"
"syscall"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/minio/minio-go/v7"
"github.com/redis/go-redis/v9"
)
type App struct {
db *pgxpool.Pool
sessions *redis.Client
secret []byte
ttl time.Duration
slots, kdf chan struct{}
stopping atomic.Bool
dummyHash string
instance string
idempotencyTTL time.Duration
s3 *minio.Client
bucket string
uploadSlots, decodeSlot chan struct{}
}
func env(key, value string) string {
if v := os.Getenv(key); v != "" {
return v
}
return value
}
func positive(key string, fallback int) int {
n, e := strconv.Atoi(env(key, strconv.Itoa(fallback)))
if e != nil || n <= 0 {
panic("invalid positive configuration: " + key)
}
return n
}
func secretFile(key string) (string, error) {
b, e := os.ReadFile(os.Getenv(key))
if e != nil {
return "", fmt.Errorf("cannot read %s", key)
}
s := strings.TrimSpace(string(b))
if s == "" {
return "", fmt.Errorf("empty %s", key)
}
return s, nil
}
func (a *App) dispatch(ctx context.Context, w http.ResponseWriter, r *http.Request) (int, any, error) {
path := r.URL.Path
if r.Method == "GET" && (path == "/health/live" || path == "/health/ready") {
if path == "/health/ready" && (a.stopping.Load() || a.db.Ping(ctx) != nil || a.sessions.Ping(ctx).Err() != nil || !a.s3Ready(ctx)) {
return 0, nil, unavailable()
}
return 200, map[string]string{"status": "ok"}, nil
}
if a.stopping.Load() {
return 0, nil, unavailable()
}
if r.Method == "POST" && path == "/api/v1/auth/sessions" {
return a.login(ctx, r)
}
if r.Method == "POST" && path == "/api/v1/users" {
body, err := readObject(w, r, 128*1024, []string{"username", "display_name", "password"})
if err != nil {
return 0, nil, err
}
name, err := textField(body, "username", 32)
if err != nil || !usernamePattern.MatchString(name) {
return 0, nil, invalid()
}
display, err := textField(body, "display_name", 100)
if err != nil {
return 0, nil, err
}
p, err := password(body)
if err != nil {
return 0, nil, err
}
if err = a.kdfSlot(ctx); err != nil {
return 0, nil, err
}
hash := hashPassword(p)
<-a.kdf
u, err := scanUser(a.db.QueryRow(ctx, "INSERT INTO users(id,username,display_name,password_hash) VALUES($1,$2,$3,$4) RETURNING id::text,username,display_name,created_at", uuid(), name, []byte(display), hash))
return 201, u, err
}
user, key, err := a.authenticate(ctx, r)
if err != nil {
return 0, nil, err
}
if r.Method == "DELETE" && path == "/api/v1/auth/sessions/current" {
count, e := a.sessions.Del(ctx, key).Result()
if e != nil {
return 0, nil, unavailable()
}
if count == 0 {
return 0, nil, unauthorized()
}
return 204, nil, nil
}
if r.Method == "GET" && path == "/api/v1/users/me" {
u, e := a.user(ctx, user)
return 200, u, e
}
if path == "/api/v1/chats" {
if r.Method == "GET" {
scope := "chats:" + user
limit, c, e := a.pagination(r, scope)
if e != nil {
return 0, nil, e
}
page, e := a.listChats(ctx, user, limit, c, scope)
return 200, page, e
}
if r.Method == "POST" {
body, e := readObject(w, r, 128*1024, []string{"title", "member_ids"})
if e != nil {
return 0, nil, e
}
title, e := textField(body, "title", 100)
if e != nil {
return 0, nil, e
}
var ids []string
if json.Unmarshal(body["member_ids"], &ids) != nil || len(ids) < 1 || len(ids) > 100 {
return 0, nil, invalid()
}
seen := map[string]bool{}
for i, id := range ids {
normalized, e := normalizeID(id)
if e != nil || seen[normalized] {
return 0, nil, invalid()
}
ids[i] = normalized
seen[normalized] = true
}
if !seen[user] {
return 0, nil, invalid()
}
chat, e := a.createChat(ctx, title, ids)
return 201, chat, e
}
}
if strings.HasPrefix(path, "/api/v1/attachments/") {
parts := strings.Split(strings.TrimPrefix(path, "/api/v1/attachments/"), "/")
id, e := normalizeID(parts[0])
if e != nil {
return 0, nil, e
}
if r.Method != "GET" || len(parts) > 2 || (len(parts) == 2 && parts[1] != "content") {
return 0, nil, missing()
}
x, e := a.attachment(ctx, id, user)
if e != nil {
return 0, nil, e
}
if len(parts) == 1 {
return 200, x, nil
}
b, e := a.content(ctx, r, x)
return 200, b, e
}
if strings.HasPrefix(path, "/api/v1/chats/") {
parts := strings.Split(strings.TrimPrefix(path, "/api/v1/chats/"), "/")
id, e := normalizeID(parts[0])
if e != nil {
return 0, nil, e
}
if len(parts) > 2 || (len(parts) == 2 && parts[1] != "messages" && parts[1] != "attachments") {
return 0, nil, missing()
}
chat, e := a.chat(ctx, id, user)
if e != nil {
return 0, nil, e
}
if len(parts) == 1 && r.Method == "GET" {
return 200, chat, nil
}
if len(parts) == 2 && parts[1] == "attachments" {
if r.Method != "POST" {
return 0, nil, missing()
}
x, e := a.upload(ctx, w, r, id, user)
return 202, x, e
}
if len(parts) == 2 {
if r.Method == "GET" {
scope := "messages:" + id
limit, c, e := a.pagination(r, scope)
if e != nil {
return 0, nil, e
}
page, e := a.history(ctx, id, limit, c, scope)
return 200, page, e
}
if r.Method == "POST" {
keys := r.Header.Values("Idempotency-Key")
if len(keys) != 1 || !idempotencyPattern.MatchString(keys[0]) {
return 0, nil, invalid()
}
body, e := readObject(w, r, 128*1024, []string{"text", "attachment_ids"})
if e != nil {
return 0, nil, e
}
text, ids, e := messageFields(body)
if e != nil {
return 0, nil, e
}
m, e := a.createMessage(ctx, id, user, keys[0], text, ids)
return 201, m, e
}
}
}
return 0, nil, missing()
}
func (a *App) ServeHTTP(w http.ResponseWriter, r *http.Request) {
start := time.Now()
requestID := uuid()
status := 500
w.Header().Set("Content-Type", "application/json")
w.Header().Set("X-Request-Id", requestID)
w.Header().Set("X-Instance-Id", a.instance)
fail := func(err error) {
var e apiError
if !errors.As(err, &e) {
e = unavailable()
}
status = e.status
if status == 401 {
scheme := `Bearer realm="messenger"`
if r.URL.Path == "/api/v1/auth/sessions" {
scheme = `Basic realm="messenger", charset="UTF-8"`
}
w.Header().Set("WWW-Authenticate", scheme)
}
if status == 503 {
w.Header().Set("Retry-After", "1")
}
w.WriteHeader(status)
json.NewEncoder(w).Encode(map[string]any{"error": map[string]string{"code": e.code, "message": e.message, "request_id": requestID}})
}
defer func() {
if recovered := recover(); recovered != nil {
if recovered == http.ErrAbortHandler {
panic(recovered)
}
fail(apiError{500, "internal_error", "Internal server error"})
}
if env("REQUEST_LOG", "1") == "1" {
slog.Info("request", "method", r.Method, "status", status, "request_id", requestID, "duration_ms", time.Since(start).Milliseconds())
}
}()
ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
defer cancel()
health := r.Method == "GET" && (r.URL.Path == "/health/live" || r.URL.Path == "/health/ready")
if !health {
select {
case a.slots <- struct{}{}:
defer func() { <-a.slots }()
default:
fail(unavailable())
return
}
}
code, value, err := a.dispatch(ctx, w, r)
if err != nil {
fail(err)
return
}
status = code
if b, ok := value.(binaryContent); ok {
writeContent(w, b)
return
}
w.WriteHeader(code)
if code != 204 {
json.NewEncoder(w).Encode(value)
}
}
func run() error {
if len(os.Args) > 1 && os.Args[1] == "memory-snapshot" {
return memorySnapshot()
}
if env("SESSION_STORE", "valkey") != "valkey" {
return errors.New("SESSION_STORE must be valkey")
}
address := net.JoinHostPort(env("APP_HOST", "0.0.0.0"), env("APP_PORT", "8080"))
if len(os.Args) > 1 && os.Args[1] == "healthcheck" {
client := http.Client{Timeout: 2 * time.Second}
res, e := client.Get("http://127.0.0.1:" + env("APP_PORT", "8080") + "/health/ready")
if e != nil {
return errors.New("not ready")
}
defer res.Body.Close()
if res.StatusCode != 200 {
return errors.New("not ready")
}
return nil
}
dbURL, err := secretFile("DATABASE_URL_FILE")
if err != nil {
return err
}
cfg, err := pgxpool.ParseConfig(dbURL)
if err != nil {
return errors.New("invalid database configuration")
}
cfg.MaxConns = int32(positive("DB_POOL_MAX", 8))
cfg.MinConns = 0
cfg.ConnConfig.ConnectTimeout = 2 * time.Second
cfg.ConnConfig.RuntimeParams["statement_timeout"] = "3000"
cfg.ConnConfig.RuntimeParams["lock_timeout"] = "1000"
cfg.ConnConfig.RuntimeParams["synchronous_commit"] = "on"
startup, cancel := context.WithTimeout(context.Background(), 45*time.Second)
defer cancel()
db, err := pgxpool.NewWithConfig(startup, cfg)
if err != nil {
return errors.New("database setup failed")
}
defer db.Close()
if len(os.Args) > 1 && os.Args[1] == "migrate" {
if err = migrate(startup, db); err != nil {
return fmt.Errorf("migration failed: %w", err)
}
slog.Info("migrations complete")
return nil
}
var schemaReady bool
if err = db.QueryRow(startup, "SELECT EXISTS(SELECT 1 FROM schema_migrations WHERE version='005_image_jobs.sql')").Scan(&schemaReady); err != nil || !schemaReady {
return errors.New("run the migrate service before starting API")
}
if len(os.Args) > 1 {
return imageCommand(db, os.Args[1:])
}
cacheURL, err := secretFile("CACHE_URL_FILE")
if err != nil {
return err
}
signing, err := secretFile("CURSOR_SECRET_FILE")
if err != nil {
return err
}
if len(signing) < 32 {
return errors.New("cursor secret too short")
}
options, err := redis.ParseURL(cacheURL)
if err != nil {
return errors.New("invalid session store configuration")
}
options.PoolSize = 8
options.MaxActiveConns = 8
options.DialTimeout = time.Second
options.ReadTimeout = time.Second
options.WriteTimeout = time.Second
options.PoolTimeout = time.Second
options.MaxRetries = -1
options.ContextTimeoutEnabled = true
sessions := redis.NewClient(options)
defer sessions.Close()
hostname, err := os.Hostname()
if err != nil {
return errors.New("cannot identify instance")
}
s3, bucket, err := s3Client()
if err != nil {
return err
}
a := &App{s3: s3, bucket: bucket, uploadSlots: make(chan struct{}, 5), decodeSlot: make(chan struct{}, 1), db: db, sessions: sessions, secret: []byte(signing), ttl: time.Duration(positive("SESSION_TTL_SECONDS", 3600)) * time.Second, slots: make(chan struct{}, positive("MAX_INFLIGHT", 32)), kdf: make(chan struct{}, 2), dummyHash: hashPassword("unknown-user-dummy-password"), instance: hostname, idempotencyTTL: time.Duration(positive("IDEMPOTENCY_TTL_SECONDS", 86400)) * time.Second}
if a.idempotencyTTL < 24*time.Hour {
return errors.New("idempotency retention must be at least 24 hours")
}
server := &http.Server{Addr: address, Handler: a, ReadHeaderTimeout: 2 * time.Second, ReadTimeout: 30 * time.Second, WriteTimeout: 35 * time.Second, IdleTimeout: 5 * time.Second, MaxHeaderBytes: 16 * 1024}
listener, err := net.Listen("tcp", address)
if err != nil {
return errors.New("listen failed")
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
if env("POOL_METRICS", "0") == "1" {
go func() {
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
s := db.Stat()
slog.Info("pool", "instance", a.instance, "acquired", s.AcquiredConns(), "max", s.MaxConns(), "empty_acquires", s.EmptyAcquireCount(), "acquire_ns", s.AcquireDuration().Nanoseconds())
}
}
}()
}
done := make(chan struct{})
go func() {
<-ctx.Done()
a.stopping.Store(true)
slog.Info("shutdown started")
// Allow two LB readiness checks to drain this backend before closing it.
time.Sleep(1500 * time.Millisecond)
shutdown, cancel := context.WithTimeout(context.Background(), 6*time.Second)
defer cancel()
err := server.Shutdown(shutdown)
if err != nil {
server.Close()
}
slog.Info("shutdown complete", "forced", err != nil)
close(done)
}()
slog.Info("listening", "address", address, "environment", env("APP_ENV", "lab"), "session_store", "valkey", "session_ttl_seconds", int(a.ttl.Seconds()))
if err = server.Serve(listener); err != nil && !errors.Is(err, http.ErrServerClosed) {
return errors.New("serve failed")
}
<-done
return nil
}
func main() {
slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, nil)))
if err := run(); err != nil {
slog.Error("startup failed", "reason", err.Error())
os.Exit(1)
}
}
+27
View File
@@ -0,0 +1,27 @@
package main
import (
"encoding/json"
"fmt"
"os"
"strconv"
"strings"
)
// Local operator command, never an HTTP endpoint. cgroup v2 records peaks
// between Docker samples too; the experiment recreates API cgroups per case.
func memorySnapshot() error {
result := map[string]uint64{}
for _, name := range []string{"memory.current", "memory.peak"} {
raw, err := os.ReadFile("/sys/fs/cgroup/" + name)
if err != nil {
return fmt.Errorf("cgroup v2 %s unavailable", name)
}
value, err := strconv.ParseUint(strings.TrimSpace(string(raw)), 10, 64)
if err != nil {
return fmt.Errorf("invalid cgroup %s", name)
}
result[name] = value
}
return json.NewEncoder(os.Stdout).Encode(result)
}
+28
View File
@@ -0,0 +1,28 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats(id),
user_id uuid NOT NULL REFERENCES users(id),
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX chat_members_user_idx ON chat_members(user_id, chat_id);
CREATE INDEX chats_page_idx ON chats(created_at, id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
sender_id uuid NOT NULL,
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id)
);
CREATE INDEX messages_history_idx ON messages(chat_id, created_at, id);
+12
View File
@@ -0,0 +1,12 @@
-- PostgreSQL text cannot contain U+0000, although it is a valid JSON string
-- code point. Keep exact UTF-8 bytes for free-form API strings. The API checks
-- Unicode length/whitespace; these checks also bound stored byte lengths.
ALTER TABLE users DROP CONSTRAINT users_display_name_check;
ALTER TABLE users ALTER COLUMN display_name TYPE bytea USING convert_to(display_name,'UTF8');
ALTER TABLE users ADD CHECK (octet_length(display_name) BETWEEN 1 AND 400);
ALTER TABLE chats DROP CONSTRAINT chats_title_check;
ALTER TABLE chats ALTER COLUMN title TYPE bytea USING convert_to(title,'UTF8');
ALTER TABLE chats ADD CHECK (octet_length(title) BETWEEN 1 AND 400);
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
ALTER TABLE messages ALTER COLUMN text TYPE bytea USING convert_to(text,'UTF8');
ALTER TABLE messages ADD CHECK (octet_length(text) BETWEEN 1 AND 16000);
+8
View File
@@ -0,0 +1,8 @@
CREATE TABLE message_keys (
user_id uuid NOT NULL REFERENCES users(id),
chat_id uuid NOT NULL REFERENCES chats(id),
key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
message_id uuid NOT NULL REFERENCES messages(id),
expires_at timestamptz NOT NULL,
PRIMARY KEY (user_id, chat_id, key)
);
+27
View File
@@ -0,0 +1,27 @@
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
owner_id uuid NOT NULL,
object_key text NOT NULL UNIQUE CHECK (object_key = 'original/' || id::text),
filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255),
content_type text NOT NULL CHECK (content_type IN ('image/png','image/jpeg')),
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
missing_since timestamptz,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
UNIQUE (chat_id,id),
FOREIGN KEY (chat_id,owner_id) REFERENCES chat_members(chat_id,user_id)
);
CREATE INDEX attachments_page_idx ON attachments(id);
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
ALTER TABLE messages ADD CHECK (octet_length(text) BETWEEN 0 AND 16000);
ALTER TABLE messages ADD COLUMN attachment_ids uuid[] NOT NULL DEFAULT '{}';
ALTER TABLE messages ADD CHECK (cardinality(attachment_ids) <= 10);
CREATE TABLE message_attachments (
message_id uuid NOT NULL REFERENCES messages(id),
chat_id uuid NOT NULL,
attachment_id uuid NOT NULL,
PRIMARY KEY (message_id,attachment_id),
FOREIGN KEY (chat_id,attachment_id) REFERENCES attachments(chat_id,id)
);
+37
View File
@@ -0,0 +1,37 @@
ALTER TABLE attachments
ADD COLUMN status text NOT NULL DEFAULT 'ready' CHECK (status IN ('queued','processing','ready','failed')),
ADD COLUMN error_code text,
ADD COLUMN active_version integer NOT NULL DEFAULT 0,
ADD COLUMN result_version integer NOT NULL DEFAULT 0,
ADD COLUMN variants jsonb NOT NULL DEFAULT '[]',
ADD CHECK ((status='failed') = (error_code IS NOT NULL)),
ADD CHECK (active_version >= result_version AND result_version >= 0);
-- A queued row is also the transactional outbox entry. Keep completed rows as
-- the deduplication ledger and failed rows as the operator's dead-letter registry.
CREATE TABLE image_jobs (
id uuid PRIMARY KEY,
attachment_id uuid NOT NULL REFERENCES attachments(id),
version integer NOT NULL CHECK (version IN (1,2)),
attempt integer NOT NULL DEFAULT 1,
cycle_start integer NOT NULL DEFAULT 1 CHECK (cycle_start BETWEEN 1 AND attempt),
state text NOT NULL DEFAULT 'queued' CHECK (state IN ('queued','processing','ready','failed')),
published boolean NOT NULL DEFAULT false,
next_at timestamptz NOT NULL DEFAULT clock_timestamp(),
lease_until timestamptz,
error_code text,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
UNIQUE (attachment_id,version),
CHECK ((state='processing') = (lease_until IS NOT NULL))
);
CREATE INDEX image_jobs_outbox ON image_jobs(next_at,id) WHERE state='queued' AND NOT published;
CREATE INDEX image_jobs_lease ON image_jobs(lease_until,id) WHERE state='processing';
CREATE TABLE image_job_replays (
job_id uuid NOT NULL REFERENCES image_jobs(id),
attempt integer NOT NULL,
previous_error text NOT NULL,
replayed_at timestamptz NOT NULL DEFAULT clock_timestamp(),
PRIMARY KEY (job_id,attempt)
);
+65
View File
@@ -0,0 +1,65 @@
"""Decode a bounded local input and write metadata-free variants for one job."""
import json
from pathlib import Path
import sys
import warnings
from PIL import Image, ImageOps, UnidentifiedImageError
Image.MAX_IMAGE_PIXELS = 40_000_000
warnings.simplefilter("error", Image.DecompressionBombWarning)
def fit(image, edge):
width, height = image.size
longest = max(width, height)
if longest <= edge:
return image
size = (max(1, width * edge // longest), max(1, height * edge // longest))
return image.resize(size, Image.Resampling.LANCZOS)
def white_rgb(image):
if image.mode == "RGBA":
background = Image.new("RGB", image.size, "white")
background.paste(image, mask=image.getchannel("A"))
return background
return image.convert("RGB")
def render(directory, mime, version):
try:
with Image.open(directory / "input", formats=("JPEG", "PNG")) as source:
if Image.MIME[source.format] != mime or source.width * source.height > Image.MAX_IMAGE_PIXELS:
return 2
source.seek(0)
# PNG may carry transparency in a tRNS chunk or in its alpha channel.
mode = "RGBA" if "A" in source.getbands() or "transparency" in source.info else "RGB"
ImageOps.exif_transpose(source, in_place=True)
image = source.convert(mode)
except (UnidentifiedImageError, OSError, ValueError, SyntaxError,
Image.DecompressionBombError, Image.DecompressionBombWarning):
return 2
square = min(image.size)
left = (image.width - square) // 2
top = (image.height - square) // 2
outputs = [
("thumbnail", fit(image, 256), "JPEG", 85),
("crop", image.resize((128, 128), Image.Resampling.LANCZOS,
box=(left, top, left + square, top + square)), "JPEG", 85),
("optimized", fit(image, 1280), "WEBP", 80 if version == "1" else 90),
]
variants = []
for name, pixels, format_name, quality in outputs:
clean = white_rgb(pixels)
clean.info.clear()
clean.save(directory / name, format=format_name, quality=quality)
variants.append({"width": clean.width, "height": clean.height})
print(json.dumps(variants))
return 0
if __name__ == "__main__":
sys.exit(render(Path(sys.argv[1]), sys.argv[2], sys.argv[3]))
+239
View File
@@ -0,0 +1,239 @@
package main
import (
"crypto/hmac"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"io"
"mime"
"net/http"
"net/url"
"regexp"
"strconv"
"strings"
"unicode/utf8"
)
type User struct {
ID string `json:"id"`
Username string `json:"username"`
DisplayName string `json:"display_name"`
CreatedAt string `json:"created_at"`
}
type Chat struct {
ID string `json:"id"`
Title string `json:"title"`
MemberIDs []string `json:"member_ids"`
CreatedAt string `json:"created_at"`
}
type Message struct {
AttachmentIDs []string `json:"attachment_ids"`
ID string `json:"id"`
ChatID string `json:"chat_id"`
SenderID string `json:"sender_id"`
Text string `json:"text"`
CreatedAt string `json:"created_at"`
}
type Page[T any] struct {
Items []T `json:"items"`
NextCursor *string `json:"next_cursor"`
}
type apiError struct {
status int
code, message string
}
func (e apiError) Error() string { return e.code }
func invalid() apiError { return apiError{400, "invalid_request", "Invalid request"} }
func unavailable() apiError { return apiError{503, "unavailable", "Dependency unavailable"} }
func missing() apiError { return apiError{404, "not_found", "Object not found"} }
func unauthorized() apiError { return apiError{401, "unauthorized", "Valid Bearer session required"} }
func uuid() string {
b := make([]byte, 16)
if _, err := rand.Read(b); err != nil {
panic(err)
}
b[6] = (b[6] & 15) | 64
b[8] = (b[8] & 63) | 128
s := hex.EncodeToString(b)
return s[:8] + "-" + s[8:12] + "-" + s[12:16] + "-" + s[16:20] + "-" + s[20:]
}
var uuidPattern = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`)
var usernamePattern = regexp.MustCompile(`^[a-z0-9_]{3,32}$`)
var idempotencyPattern = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`)
func normalizeID(s string) (string, error) {
if !uuidPattern.MatchString(s) {
return "", invalid()
}
return strings.ToLower(s), nil
}
func textField(m map[string]json.RawMessage, key string, max int) (string, error) {
var s string
b, ok := m[key]
if !ok || string(b) == "null" || json.Unmarshal(b, &s) != nil || strings.TrimSpace(s) == "" || utf8.RuneCountInString(s) > max {
return "", invalid()
}
return s, nil
}
// Reject duplicate keys, invalid UTF-8, and unpaired UTF-16 escapes rather than
// letting encoding/json silently replace malformed Unicode with U+FFFD.
func validUnicode(b []byte) bool {
if !utf8.Valid(b) {
return false
}
for i := 0; i < len(b); i++ {
if b[i] != '\\' {
continue
}
i++
if i >= len(b) {
return false
}
if b[i] != 'u' {
continue
}
if i+4 >= len(b) {
return false
}
n, e := strconv.ParseUint(string(b[i+1:i+5]), 16, 16)
if e != nil {
return false
}
i += 4
if n >= 0xDC00 && n <= 0xDFFF {
return false
}
if n >= 0xD800 && n <= 0xDBFF {
if i+6 >= len(b) || string(b[i+1:i+3]) != `\u` {
return false
}
low, e := strconv.ParseUint(string(b[i+3:i+7]), 16, 16)
if e != nil || low < 0xDC00 || low > 0xDFFF {
return false
}
i += 6
}
}
return true
}
func readObject(w http.ResponseWriter, r *http.Request, max int, fields []string) (map[string]json.RawMessage, error) {
media, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
if err != nil || media != "application/json" || (params["charset"] != "" && !strings.EqualFold(params["charset"], "utf-8")) {
return nil, apiError{415, "unsupported_media_type", "Expected application/json UTF-8"}
}
b, err := io.ReadAll(http.MaxBytesReader(w, r.Body, int64(max)))
if err != nil {
var large *http.MaxBytesError
if errors.As(err, &large) {
return nil, apiError{413, "payload_too_large", "Request body exceeds byte limit"}
}
return nil, invalid()
}
if !validUnicode(b) {
return nil, invalid()
}
d := json.NewDecoder(strings.NewReader(string(b)))
token, err := d.Token()
if err != nil || token != json.Delim('{') {
return nil, invalid()
}
m := map[string]json.RawMessage{}
for d.More() {
t, err := d.Token()
if err != nil {
return nil, invalid()
}
key, ok := t.(string)
if !ok {
return nil, invalid()
}
allowed := false
for _, f := range fields {
if key == f {
allowed = true
}
}
if !allowed || m[key] != nil {
return nil, invalid()
}
var value json.RawMessage
if d.Decode(&value) != nil {
return nil, invalid()
}
m[key] = value
}
if _, err = d.Token(); err != nil {
return nil, invalid()
}
if _, err = d.Token(); err != io.EOF {
return nil, invalid()
}
return m, nil
}
type cursor struct {
Scope string `json:"s"`
Time string `json:"t"`
ID string `json:"i"`
}
func (a *App) encodeCursor(scope, t, id string) string {
b, _ := json.Marshal(cursor{scope, t, id})
h := hmac.New(sha256.New, a.secret)
h.Write(b)
return base64.RawURLEncoding.EncodeToString(b) + "." + base64.RawURLEncoding.EncodeToString(h.Sum(nil))
}
func (a *App) pagination(r *http.Request, scope string) (int, cursor, error) {
values, e := url.ParseQuery(r.URL.RawQuery)
if e != nil {
return 0, cursor{}, invalid()
}
limit := 50
if v, ok := values["limit"]; ok {
if len(v) != 1 || v[0] == "" {
return 0, cursor{}, invalid()
}
for _, c := range v[0] {
if c < '0' || c > '9' {
return 0, cursor{}, invalid()
}
}
limit, e = strconv.Atoi(v[0])
if e != nil || limit < 1 || limit > 100 {
return 0, cursor{}, invalid()
}
}
c := cursor{}
if v, ok := values["cursor"]; ok {
if len(v) != 1 || len(v[0]) > 1024 {
return 0, c, invalid()
}
parts := strings.Split(v[0], ".")
if len(parts) != 2 {
return 0, c, invalid()
}
b, e := base64.RawURLEncoding.DecodeString(parts[0])
if e != nil {
return 0, c, invalid()
}
sig, e := base64.RawURLEncoding.DecodeString(parts[1])
if e != nil {
return 0, c, invalid()
}
h := hmac.New(sha256.New, a.secret)
h.Write(b)
if !hmac.Equal(sig, h.Sum(nil)) || json.Unmarshal(b, &c) != nil || c.Scope != scope {
return 0, c, invalid()
}
}
return limit, c, nil
}
+41
View File
@@ -0,0 +1,41 @@
"""Create local secrets once. Never print values or overwrite existing secrets."""
from pathlib import Path
import secrets
root = Path(__file__).resolve().parents[1]
folder = root / 'secrets'
folder.mkdir(exist_ok=True)
folder.chmod(0o700)
def put(name, value):
path = folder / name
if not path.exists():
with path.open('x', encoding='utf-8', newline='\n') as f:
f.write(value + '\n')
# Parent directory is private; Compose mounts individual files for non-root services.
path.chmod(0o644)
return path.read_text(encoding='utf-8').strip()
pg = put('postgres_password', secrets.token_hex(32))
vk = put('valkey_password', secrets.token_hex(32))
put('database_url', f'postgres://messenger:{pg}@db:5432/messenger?sslmode=disable')
put('cache_url', f'redis://:{vk}@valkey:6379/0')
put('cursor_secret', secrets.token_hex(32))
put('valkey.conf', '\n'.join(['bind 0.0.0.0', 'protected-mode yes', f'requirepass {vk}',
'dir /data', 'appendonly yes', 'appendfsync always', 'maxmemory 128mb',
'maxmemory-policy noeviction', 'save 60 1000']))
broker = put('broker_password', secrets.token_hex(32))
put('broker_url', f'redis://:{broker}@broker:6379/0')
put('broker.conf', '\n'.join(['bind 0.0.0.0', 'protected-mode yes', f'requirepass {broker}',
'dir /data', 'appendonly yes', 'appendfsync always', 'maxmemory 128mb',
'maxmemory-policy noeviction', 'save ""']))
if not (root / '.env').exists():
(root / '.env').write_text((root / '.env.example').read_text(encoding='utf-8'),encoding='utf-8')
put('s3_root_user', 'admin_' + secrets.token_hex(8))
put('s3_root_password', secrets.token_hex(32))
put('s3_access_key_id', 'api_' + secrets.token_hex(8))
put('s3_secret_access_key', secrets.token_hex(32))
put('s3_worker_access_key_id', 'worker_' + secrets.token_hex(8))
put('s3_worker_secret_access_key', secrets.token_hex(32))
for name in ('restart.sh', 'stop-one.sh'):
(root / 'scripts' / name).chmod(0o755)
print('Local .env and secrets ready; existing values preserved.')
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
mkdir -p evidence
ids=$(docker compose ps -q db valkey s3 api lb broker dispatcher worker)
docker inspect --format '{{.Id}} {{index .Config.Labels "com.docker.compose.service"}} {{.State.StartedAt}} {{.State.Status}}' $ids > evidence/restart-before.txt
docker compose restart db valkey s3 api lb broker dispatcher worker
docker inspect --format '{{.Id}} {{index .Config.Labels "com.docker.compose.service"}} {{.State.StartedAt}} {{.State.Status}}' $ids > evidence/restart-after.txt
+16
View File
@@ -0,0 +1,16 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}"
for id in $(docker compose ps -q api); do
hostname=$(docker inspect --format '{{.Config.Hostname}}' "$id")
if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then
mkdir -p evidence
docker inspect --format '{{.Id}} {{.State.Status}} {{.State.StartedAt}}' "$id" > evidence/stopped-before.txt
docker stop --time 9 "$id" > /dev/null
docker inspect --format '{{.Id}} {{.State.Status}} {{.State.FinishedAt}} {{.State.ExitCode}}' "$id" > evidence/stopped-after.txt
exit 0
fi
done
echo 'Target is not a running API container' >&2
exit 1
+223
View File
@@ -0,0 +1,223 @@
package main
import (
"context"
"crypto/sha256"
"embed"
"encoding/hex"
"errors"
"fmt"
"sort"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/pgxpool"
)
//go:embed migrations/*.sql
var migrations embed.FS
func migrate(ctx context.Context, db *pgxpool.Pool) error {
conn, err := db.Acquire(ctx)
if err != nil {
return err
}
defer conn.Release()
if _, err = conn.Exec(ctx, "SELECT pg_advisory_lock(202602)"); err != nil {
return err
}
defer conn.Exec(context.Background(), "SELECT pg_advisory_unlock(202602)")
if _, err = conn.Exec(ctx, "CREATE TABLE IF NOT EXISTS schema_migrations(version text PRIMARY KEY, checksum text NOT NULL, applied_at timestamptz NOT NULL DEFAULT clock_timestamp())"); err != nil {
return err
}
entries, err := migrations.ReadDir("migrations")
if err != nil {
return err
}
sort.Slice(entries, func(i, j int) bool { return entries[i].Name() < entries[j].Name() })
for _, entry := range entries {
raw, err := migrations.ReadFile("migrations/" + entry.Name())
if err != nil {
return err
}
sum := sha256.Sum256(raw)
checksum := hex.EncodeToString(sum[:])
var previous string
err = conn.QueryRow(ctx, "SELECT checksum FROM schema_migrations WHERE version=$1", entry.Name()).Scan(&previous)
if err == nil {
if previous != checksum {
return fmt.Errorf("migration checksum mismatch: %s", entry.Name())
}
continue
}
if !errors.Is(err, pgx.ErrNoRows) {
return err
}
tx, err := conn.Begin(ctx)
if err != nil {
return err
}
if _, err = tx.Exec(ctx, string(raw)); err == nil {
_, err = tx.Exec(ctx, "INSERT INTO schema_migrations(version,checksum) VALUES($1,$2)", entry.Name(), checksum)
}
if err != nil {
tx.Rollback(ctx)
return err
}
if err = tx.Commit(ctx); err != nil {
return err
}
}
return nil
}
func dbError(err error) error {
if err == nil {
return nil
}
if errors.Is(err, pgx.ErrNoRows) {
return missing()
}
var pe *pgconn.PgError
if errors.As(err, &pe) {
switch pe.Code {
case "23505":
return apiError{409, "conflict", "Username already exists"}
case "23503", "23514", "22P02", "22021":
return invalid()
}
}
return unavailable()
}
func stamp(t time.Time) string { return t.UTC().Format("2006-01-02T15:04:05.000000Z") }
// pgx BytesScanner preserves every validated UTF-8 code point, including NUL.
type utf8String struct{ dst *string }
func (s utf8String) ScanBytes(value []byte) error {
*s.dst = string(value)
return nil
}
func scanUser(row pgx.Row) (User, error) {
var u User
var t time.Time
err := row.Scan(&u.ID, &u.Username, utf8String{&u.DisplayName}, &t)
u.CreatedAt = stamp(t)
return u, dbError(err)
}
func (a *App) user(ctx context.Context, id string) (User, error) {
return scanUser(a.db.QueryRow(ctx, "SELECT id::text,username,display_name,created_at FROM users WHERE id=$1", id))
}
func (a *App) createChat(ctx context.Context, title string, ids []string) (Chat, error) {
c := Chat{ID: uuid(), Title: title, MemberIDs: ids}
var t time.Time
tx, err := a.db.Begin(ctx)
if err != nil {
return c, unavailable()
}
defer tx.Rollback(ctx)
err = tx.QueryRow(ctx, "INSERT INTO chats(id,title) VALUES($1,$2) RETURNING created_at", c.ID, []byte(title)).Scan(&t)
if err != nil {
return c, dbError(err)
}
// FK failure on any participant rolls back the chat and earlier participants.
for _, id := range ids {
if _, err = tx.Exec(ctx, "INSERT INTO chat_members(chat_id,user_id) VALUES($1,$2)", c.ID, id); err != nil {
return c, dbError(err)
}
}
if err = tx.Commit(ctx); err != nil {
return c, unavailable()
}
c.CreatedAt = stamp(t)
return c, nil
}
func (a *App) chat(ctx context.Context, id, user string) (Chat, error) {
c := Chat{}
var t time.Time
err := a.db.QueryRow(ctx, `SELECT c.id::text,c.title,c.created_at,
ARRAY(SELECT user_id::text FROM chat_members WHERE chat_id=c.id ORDER BY user_id)
FROM chats c WHERE c.id=$1 AND EXISTS(SELECT 1 FROM chat_members WHERE chat_id=c.id AND user_id=$2)`, id, user).Scan(&c.ID, utf8String{&c.Title}, &t, &c.MemberIDs)
c.CreatedAt = stamp(t)
return c, dbError(err)
}
func boundary(c cursor) (time.Time, string, error) {
if c.ID == "" {
return time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC), "00000000-0000-0000-0000-000000000000", nil
}
t, err := time.Parse(time.RFC3339Nano, c.Time)
if err != nil {
return t, "", invalid()
}
id, err := normalizeID(c.ID)
return t, id, err
}
func (a *App) history(ctx context.Context, id string, limit int, c cursor, scope string) (Page[Message], error) {
result := Page[Message]{Items: []Message{}}
t, last, err := boundary(c)
if err != nil {
return result, err
}
rows, err := a.db.Query(ctx, `SELECT id::text,chat_id::text,sender_id::text,text,created_at,ARRAY(SELECT unnest(attachment_ids)::text) FROM messages
WHERE chat_id=$1 AND (created_at,id)>($2,$3) ORDER BY messages.created_at,messages.id LIMIT $4`, id, t, last, limit+1)
if err != nil {
return result, unavailable()
}
defer rows.Close()
for rows.Next() {
var m Message
var created time.Time
if err = rows.Scan(&m.ID, &m.ChatID, &m.SenderID, utf8String{&m.Text}, &created, &m.AttachmentIDs); err != nil {
return result, unavailable()
}
m.CreatedAt = stamp(created)
result.Items = append(result.Items, m)
}
if rows.Err() != nil {
return result, unavailable()
}
if len(result.Items) > limit {
result.Items = result.Items[:limit]
m := result.Items[limit-1]
next := a.encodeCursor(scope, m.CreatedAt, m.ID)
result.NextCursor = &next
}
return result, nil
}
func (a *App) listChats(ctx context.Context, user string, limit int, c cursor, scope string) (Page[Chat], error) {
result := Page[Chat]{Items: []Chat{}}
t, last, err := boundary(c)
if err != nil {
return result, err
}
rows, err := a.db.Query(ctx, `SELECT c.id::text,c.title,c.created_at,
ARRAY(SELECT user_id::text FROM chat_members WHERE chat_id=c.id ORDER BY user_id)
FROM chats c JOIN chat_members own ON own.chat_id=c.id AND own.user_id=$1
WHERE (c.created_at,c.id)>($2,$3) ORDER BY c.created_at,c.id LIMIT $4`, user, t, last, limit+1)
if err != nil {
return result, unavailable()
}
defer rows.Close()
for rows.Next() {
var chat Chat
var created time.Time
if err = rows.Scan(&chat.ID, utf8String{&chat.Title}, &created, &chat.MemberIDs); err != nil {
return result, unavailable()
}
chat.CreatedAt = stamp(created)
result.Items = append(result.Items, chat)
}
if rows.Err() != nil {
return result, unavailable()
}
if len(result.Items) > limit {
result.Items = result.Items[:limit]
v := result.Items[limit-1]
next := a.encodeCursor(scope, v.CreatedAt, v.ID)
result.NextCursor = &next
}
return result, nil
}
+82
View File
@@ -0,0 +1,82 @@
"""Check image edge cases through the public API using the worker's pinned Pillow."""
import base64
import json
from pathlib import Path
import struct
import subprocess
import sys
import zlib
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, check
def pillow(code, data=""):
result = subprocess.run(
["docker", "compose", "exec", "-T", "worker", "python3", "-c", code],
cwd=ROOT, input=data, capture_output=True, text=True, encoding="utf-8", timeout=30, check=True,
)
return json.loads(result.stdout)
def main():
client = Client()
alice, bob, _, chat = client.bootstrap()
path = f"/api/v1/chats/{chat['id']}/attachments?filename=synthetic"
fixtures = pillow("""
import base64, io, json
from PIL import Image
jpeg = Image.new("RGB", (300, 100), "red")
exif = Image.Exif()
exif[274] = 6
exif[271] = "Synthetic EXIF"
png = Image.new("RGBA", (8, 4), (255, 0, 0, 0))
def encode(image, kind, **options):
buffer = io.BytesIO()
image.save(buffer, kind, **options)
return base64.b64encode(buffer.getvalue()).decode()
print(json.dumps([encode(jpeg, "JPEG", exif=exif), encode(png, "PNG")]))
""")
downloads = {}
for fixture, mime, dimensions in zip(fixtures, ("image/jpeg", "image/png"),
([(85, 256), (128, 128), (100, 300)],
[(8, 4), (128, 128), (8, 4)])):
raw = base64.b64decode(fixture)
att = client.call("POST", path, 202, actor=alice, raw=raw,
headers={"Content-Type": mime}, schema="Attachment")["json"]
ready = client.wait_ready(bob, att)
content = f"/api/v1/attachments/{att['id']}/content"
check(client.call("GET", content, actor=bob)["bytes"] == raw, "original changed")
for (width, height), name in zip(dimensions, ("thumbnail", "crop", "optimized")):
download = client.call("GET", content + "?variant=" + name, actor=bob)
downloads[mime + "/" + name] = [base64.b64encode(download["bytes"]).decode(), width, height]
print(f"{mime} attachment={att['id']} dimensions={dimensions}")
probes = pillow("""
import base64, io, json, sys
from PIL import Image
results = {}
for key, (body, width, height) in json.load(sys.stdin).items():
with Image.open(io.BytesIO(base64.b64decode(body))) as image:
image.load()
white = all(channel >= 250 for channel in image.convert("RGB").getpixel((0, 0)))
results[key] = image.size == (width, height) and not image.getexif() and (white or not key.startswith("image/png"))
print(json.dumps(results))
""", json.dumps(downloads))
check(all(probes.values()), "wrong actual dimensions, leaked EXIF or invalid alpha composition")
fixture = (ROOT / "tests/fixtures/sample.png").read_bytes()
ihdr = struct.pack(">II", 10_000, 4_001) + fixture[24:29]
too_many_pixels = fixture[:16] + ihdr + struct.pack(">I", zlib.crc32(b"IHDR" + ihdr)) + fixture[33:]
for body, mime, status in (
(too_many_pixels, "image/png", 413),
(b"0" * (10 * 1024 * 1024 + 1), "image/png", 413),
(fixture, "image/jpeg", 415),
(fixture[:33], "image/png", 415),
):
client.call("POST", path, status, actor=alice, raw=body, headers={"Content-Type": mime})
print("PASS EXIF, fit, crop, WebP, transparency, metadata removal, byte/pixel/format limits")
if __name__ == "__main__":
main()
+320
View File
@@ -0,0 +1,320 @@
"""Fault experiment on a dedicated lab5 Compose stand with synthetic data."""
import hashlib
import json
from pathlib import Path
import subprocess
import sys
import time
import uuid
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, check
def run(*args, expected=0, timeout=120):
proc = subprocess.run(
args, cwd=ROOT, capture_output=True, text=True,
encoding="utf-8", errors="replace", timeout=timeout,
)
if proc.returncode != expected:
raise Failure(f"{args[:5]} exit={proc.returncode}, expected={expected}")
return proc.stdout.strip()
def compose(*args, **kwargs):
return run("docker", "compose", "--ansi", "never", *args, **kwargs)
def sql(query):
return compose("exec", "-T", "db", "psql", "-XAt", "-v", "ON_ERROR_STOP=1",
"-U", "messenger", "-d", "messenger", "-c", query)
def broker(*args):
output = compose(
"exec", "-T", "broker", "/bin/sh", "-ec",
'export REDISCLI_AUTH="$(cat /run/secrets/broker_password)"; exec valkey-cli --json "$@"',
"sh", *args,
)
return json.loads(output)
def until(fn, timeout=90):
start = time.monotonic()
while time.monotonic() - start < timeout:
try:
result = fn()
except Failure:
result = None
if result:
return result
time.sleep(0.3)
raise Failure(f"condition timed out after {timeout}s")
def job(att, version=1):
return json.loads(sql(f"""
SELECT json_build_object('id',id,'state',state,'attempt',attempt,'error',error_code,
'published',published,'created',to_char(created_at AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.US"Z"'),
'updated',updated_at,'delay_ms',round(extract(epoch FROM (next_at-updated_at))*1000))
FROM image_jobs WHERE attachment_id='{att['id']}' AND version={version}
"""))
def carrier(att, version):
row = job(att, version)
return {
"schema_version": 1, "job_id": row["id"], "type": "attachment.process.v1",
"attachment_id": att["id"], "pipeline_version": version,
"attempt": row["attempt"], "created_at": row["created"],
}
def entries(att):
return [json.loads(fields[1]) for _, fields in broker("XRANGE", "attachment-jobs", "-", "+")
if json.loads(fields[1])["attachment_id"] == att["id"]]
def publish(payload):
return broker("XADD", "attachment-jobs", "*", "job", json.dumps(payload, separators=(",", ":")))
def s3stat(key):
setup = ('mc alias set local http://s3:9000 "$(cat /run/secrets/s3_root_user)" '
'"$(cat /run/secrets/s3_root_password)" >/dev/null; '
'mc stat --json "local/messenger-attachments/$1"')
# mc reports an absent object in JSON on stdout.
args = ("docker", "compose", "--ansi", "never", "run", "--rm", "--no-deps",
"--entrypoint", "/bin/sh", "s3-init", "-ec", setup, "sh", key)
proc = subprocess.run(args, cwd=ROOT, capture_output=True, text=True,
encoding="utf-8", errors="replace", timeout=60)
result = json.loads(proc.stdout.strip().splitlines()[-1])
if proc.returncode:
check("does not exist" in json.dumps(result).lower(), "unexpected object-store error")
return None
return result
def fault(service, phase, att):
out = compose("run", "--rm", "--no-deps", "-e", f"LAB5_FAULT={phase}",
"-e", f"LAB5_FAULT_ATTACHMENT={att['id']}", service, expected=86)
check('"test crash"' in out and f'"phase":"{phase}"' in out, "fault marker missing")
def generations(*services):
ids = compose("ps", "-q", *services).splitlines()
return dict(line.split("|", 1) for line in run(
"docker", "inspect", "--format", "{{.Id}}|{{.State.StartedAt}}", *ids,
).splitlines())
class Experiment:
def __init__(self):
self.client = Client()
self.alice, self.bob, self.eve, self.chat = self.client.bootstrap()
self.results = []
def record(self, scenario, att, started, **data):
row = {"scenario": scenario, "attachment_id": att["id"],
"seconds": round(time.monotonic() - started, 3), **data}
self.results.append(row)
print(json.dumps(row, ensure_ascii=False), flush=True)
def metadata(self, att):
return self.client.call("GET", "/api/v1/attachments/" + att["id"],
actor=self.bob, schema="Attachment")["json"]
def content(self, att, variant, status=200, actor=None):
return self.client.call("GET", f"/api/v1/attachments/{att['id']}/content?variant={variant}",
status, actor=actor or self.bob)
def ready(self, att, raw):
result = self.client.wait_ready(self.bob, att, wait_seconds=155)
check(self.content(att, "original")["bytes"] == raw, "original changed")
variants = {v["name"]: v for v in result["variants"]}
check(len(result["variants"]) == 3 and set(variants) == {"thumbnail", "crop", "optimized"},
"expected exactly one of each variant")
for name, (width, height, mime) in {
"thumbnail": (256, 160, "image/jpeg"), "crop": (128, 128, "image/jpeg"),
"optimized": (320, 200, "image/webp"),
}.items():
variant = variants[name]
download = self.content(att, name)
body = download["bytes"]
check((variant["width"], variant["height"], download["headers"].get_content_type())
== (width, height, mime), "wrong dimensions or MIME")
check(variant["size_bytes"] == len(body) and variant["sha256"] == hashlib.sha256(body).hexdigest(),
"wrong variant size or checksum")
check(body.startswith(b"\xff\xd8") if mime == "image/jpeg" else
body[:4] == b"RIFF" and body[8:12] == b"WEBP", "wrong encoding")
return result
def restart(self):
compose("stop", "worker")
att, raw = self.client.attachment(self.alice, self.chat)
started = time.monotonic()
check(self.metadata(att)["status"] == "queued", "job ran while worker stopped")
self.content(att, "original", 409)
self.content(att, "original", 404, self.eve)
message = self.client.message(self.alice, self.chat, "", attachments=[att["id"]])["json"]
before = until(lambda: entries(att))
services = ("api", "broker")
old = generations(*services)
check(len(old) >= 3, "expected two API replicas and broker")
compose("restart", *services)
new = generations(*services)
check(old.keys() == new.keys() and all(old[key] != new[key] for key in old), "restart not observed")
check(until(lambda: entries(att)) == before, "durable broker entry changed")
until(lambda: self.client.call("GET", "/health/ready", schema="Health"))
compose("start", "worker")
self.ready(att, raw)
until(lambda: not entries(att))
self.record("api_broker_restart_with_worker_stopped", att, started,
message_id=message["id"], job_id=before[0]["job_id"], processes_restarted=len(old))
def publish_crash(self, phase):
compose("stop", "dispatcher", "worker")
att, raw = self.client.attachment(self.alice, self.chat)
started = time.monotonic()
fault("dispatcher", phase, att)
old = job(att)
check(old["state"] == "queued" and not old["published"], "outbox lost pending intent")
observed = entries(att)
check(len(observed) == (phase == "after_publish"), "incorrect broker state at crash")
# Keep the worker stopped until both XADDs are visible in the ambiguous case.
compose("start", "dispatcher")
count = 2 if phase == "after_publish" else 1
until(lambda: len(entries(att)) == count)
check(all(entry == carrier(att, 1) for entry in entries(att)), "unexpected carrier")
compose("start", "worker")
self.ready(att, raw)
until(lambda: not entries(att))
check(job(att)["attempt"] == 1, "duplicate delivery consumed an attempt")
self.record(phase, att, started, copies=count, attempt=1)
def worker_crash(self, phase):
compose("stop", "worker")
att, raw = self.client.attachment(self.alice, self.chat)
started = time.monotonic()
fault("worker", phase, att)
old = job(att)
path = f"original/{att['id']}/v1"
written = s3stat(path)
check((written is not None) == (phase != "before_effect"), "incorrect S3 state at crash")
check(old["state"] == ("ready" if phase == "after_commit" else "processing"), "incorrect job state at crash")
check(broker("XPENDING", "attachment-jobs", "image-workers")[0] > 0, "missing unacked delivery")
compose("start", "worker")
self.ready(att, raw)
until(lambda: not entries(att), timeout=155)
if written:
final = s3stat(path)
check(final["lastModified"] == written["lastModified"] and final["etag"] == written["etag"],
"retry rewrote an existing effect")
check(job(att)["attempt"] == (1 if phase == "after_commit" else 2), "wrong attempt after recovery")
until(lambda: s3stat(f"staging/{att['id']}") is None, timeout=50)
self.record(phase, att, started, state_at_crash=old["state"], object_at_crash=written is not None,
final_attempt=job(att)["attempt"], staging_removed=True)
def replay(self, phase):
compose("stop", "worker")
att, raw = self.client.attachment(self.alice, self.chat)
started = time.monotonic()
name = f"lab5-fault-{uuid.uuid4().hex[:12]}"
compose("run", "-d", "--no-deps", "--name", name, "-e", f"LAB5_FAULT={phase}",
"-e", f"LAB5_FAULT_ATTACHMENT={att['id']}", "worker")
delays = {}
try:
def failure():
row = job(att)
if row["attempt"] > 1 and row["delay_ms"] > 0:
delays.setdefault(row["attempt"], row["delay_ms"])
return row if row["state"] == "failed" else None
failed = until(failure, timeout=155)
finally:
run("docker", "stop", name)
run("docker", "rm", name)
code = "invalid_image" if phase == "invalid_image" else "retry_exhausted"
meta = self.metadata(att)
check(meta["status"] == "failed" and meta["error_code"] == code, "incorrect final failure")
if phase == "invalid_image":
check(failed["attempt"] == 1, "permanent error was retried")
else:
config = json.loads(compose("config", "--format", "json"))
max_attempts = int(config["services"]["worker"]["environment"]["JOB_MAX_ATTEMPTS"])
check(failed["attempt"] == max_attempts, "incorrect retry limit")
check(set(delays) == set(range(2, max_attempts + 1)), "missed retry schedule")
for attempt, delay in delays.items():
base = 1000 << min(attempt - 2, 5)
check(base - 5 <= delay < 2 * base, "retry delay outside jitter window")
self.content(att, "original", 409)
self.client.call("POST", f"/api/v1/chats/{self.chat['id']}/messages", 409, actor=self.alice,
headers={"Idempotency-Key": uuid.uuid4().hex}, body={"text": "", "attachment_ids": [att["id"]]})
def operator(*args, **kwargs):
return compose("run", "--rm", "--no-deps", "dispatcher", "jobs", *args, **kwargs)
check(failed["id"] in operator("failed"), "failed registry did not contain job")
operator("replay", failed["id"])
check(self.metadata(att)["status"] == "queued", "replay failed to queue job")
compose("start", "worker")
self.ready(att, raw)
operator("replay", failed["id"], expected=1)
audit = sql(f"SELECT count(*) FROM image_job_replays WHERE job_id='{failed['id']}'")
check(audit == "1" and job(att)["attempt"] == failed["attempt"] + 1, "replay audit/attempt mismatch")
self.record(phase + "_replay", att, started, failed_attempt=failed["attempt"],
error_code=code, retry_delays_ms=delays, replay_audit=int(audit))
def version_order(self):
att, raw = self.client.attachment(self.alice, self.chat)
started = time.monotonic()
v1 = self.ready(att, raw)
old_carrier = carrier(att, 1)
old_object = s3stat(f"optimized/{att['id']}/v1")
until(lambda: not entries(att))
compose("stop", "worker")
compose("run", "--rm", "--no-deps", "dispatcher", "jobs", "upgrade", att["id"])
until(lambda: job(att, 2)["published"])
new_carrier = carrier(att, 2)
publish(old_carrier)
publish(new_carrier)
check([entry["pipeline_version"] for entry in entries(att)] == [2, 1, 2], "reorder not injected")
check(self.metadata(att) == v1, "upgrade hid the previous result")
compose("start", "worker")
until(lambda: sql(f"SELECT result_version FROM attachments WHERE id='{att['id']}'") == "2")
v2 = self.ready(att, raw)
until(lambda: not entries(att))
check(v1["variants"] != v2["variants"], "v2 was not distinct")
new_object = s3stat(f"optimized/{att['id']}/v2")
publish(old_carrier)
publish(new_carrier)
until(lambda: not entries(att))
check(self.ready(att, raw) == v2 and job(att, 2)["attempt"] == 1, "stale job changed v2")
for version, obj in ((1, old_object), (2, new_object)):
current = s3stat(f"optimized/{att['id']}/v{version}")
check(current["lastModified"] == obj["lastModified"] and current["etag"] == obj["etag"],
"duplicate or reordered job rewrote an object")
self.record("reorder_and_stale_version", att, started, delivery_versions=[2, 1, 2, 1, 2],
result_version=2, attempts=[job(att)["attempt"], job(att, 2)["attempt"]])
def main():
experiment = Experiment()
try:
experiment.restart()
for phase in ("before_publish", "after_publish"):
experiment.publish_crash(phase)
for phase in ("before_effect", "after_effect", "after_commit"):
experiment.worker_crash(phase)
for phase in ("invalid_image", "temporary"):
experiment.replay(phase)
experiment.version_order()
finally:
compose("start", "dispatcher", "worker")
(ROOT / "evidence" / "pipeline.json").write_text(
json.dumps(experiment.results, ensure_ascii=False, indent=2) + "\n", encoding="utf-8",
)
print("PASS pipeline fault experiment")
if __name__ == "__main__":
main()