diff --git a/.github/workflows/solution.yml b/.github/workflows/solution.yml new file mode 100644 index 00000000..d9414026 --- /dev/null +++ b/.github/workflows/solution.yml @@ -0,0 +1,28 @@ +name: solution +on: [push, pull_request] +permissions: {contents: read} +jobs: + application: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - name: Fresh stack and contract + run: | + make up + make check + make test + - name: Real PostgreSQL and race detector + run: make test-race + - name: Persistence + run: make test-persistence ACTION_SCRIPT=scripts/restart.sh + - name: API failover + run: make test-failover ACTION_SCRIPT=scripts/stop-one.sh + - uses: actions/upload-artifact@v4 + if: always() + with: + name: solution-evidence + path: evidence/ + - name: Dispose only this disposable CI stack + if: always() + run: docker compose --profile verify down -v --remove-orphans diff --git a/DEFENSE.md b/DEFENSE.md new file mode 100644 index 00000000..fcea3990 --- /dev/null +++ b/DEFENSE.md @@ -0,0 +1,119 @@ +# Защита ЛР5: что я сделал + +## Ответ на первый вопрос преподавателя + +«Я перенёс свой мессенджер с приватным S3 и вынес преобразование изображений в отдельный worker. API проверяет доступ, формат и лимиты, сохраняет полный оригинал в S3 staging, затем в одной транзакции PostgreSQL создаёт queued-вложение и durable job/outbox. Только после COMMIT возвращает 202. Dispatcher публикует служебное сообщение в Valkey Streams, worker читает его через consumer group. Worker публикует неизменённый оригинал, thumbnail JPEG, crop JPEG и optimized WebP, затем атомарно переводит метаданные в ready. XACK выполняется после устойчивого результата или сохранённого решения о retry/failed. + +Я проверил окна аварий, конечные повторы и явный replay. Для исследования сделал две версии pipeline и fencing результата: ключи объектов включают версию, а COMMIT проверяет текущую версию и lease token. Даже реально продолжившийся старый worker не перезаписывает новый результат. Доставка остаётся at least once; идемпотентен эффект, а не сеть». + +## Общая схема + +```mermaid +flowchart LR + C[Клиент] --> LB[HAProxy] + LB --> A[API × 2] + A --> S[S3 staging] + A --> D[(PostgreSQL: attachment + job/outbox)] + D --> P[Dispatcher] + P --> B[(Valkey Streams + AOF)] + B --> W[Worker: bounded concurrency] + W --> F[S3 final: original + 3 variants] + W --> D + C -->|poll metadata / download with ACL| LB +``` + +```mermaid +sequenceDiagram + participant A as API + participant S as S3 + participant D as PostgreSQL + participant P as Dispatcher + participant B as Streams + participant W as Worker + A->>S: PUT staging/full original + A->>D: COMMIT queued + job + A-->>A: 202 queued snapshot + P->>D: read durable unpublished job + P->>B: XADD (AOF fsync always) + P->>D: mark published + B->>W: XREADGROUP + W->>D: claim, attempt, lease token + W->>S: PUT versioned original/variants + W->>D: fenced COMMIT ready + W->>B: XACK, then XDEL +``` + +202 не означает, что worker уже завершил работу. Файл всё равно полностью проходит по сети до API и сохраняется до ответа. Staging уже находится в S3, поэтому worker читает его и отдельно публикует final objects; это не перенос из памяти API в S3. + +## Четыре вопроса из README + +**Чем publisher confirm отличается от consumer ack?** Publisher confirm подтверждает приём публикации брокером, consumer ack — устойчивую обработку получателем. В нашем Valkey Streams нет отдельного RabbitMQ confirm-протокола: успешный XADD при `appendfsync always` — граница приёма persistent брокером, XACK убирает доставку из pending. Между этими границами worker ещё может упасть. Дополнительно намерение хранится в PostgreSQL и повторно публикуется outbox. + +**Что именно уже сохранено в момент 202?** Полные проверенные bytes в приватном `staging/`; в PostgreSQL — метаданные queued и запись job в одной committed-транзакции. Публикация в broker может ещё не произойти. Именно outbox позволяет честно принять работу, пока broker недоступен. Нельзя отвечать 202 после одного запуска goroutine или записи файла в ephemeral disk API. + +**Почему повторная доставка нормальна и где ключ идемпотентности job?** При разрыве соединения невозможно всегда знать, получил ли broker XACK или producer ответ XADD. Поэтому допускаем повтор. Логическая job определяется `job_id`, эффект — `(attachment_id, pipeline_version, variant)`. Ключи S3 детерминированы внутри версии, метаданные fenced по версии и lease token; ready/stale delivery — no-op. В таблице UNIQUE `(attachment_id,pipeline_version)`. Это не exactly-once доставка. + +**Как отличить зависший worker от долго работающего и не запустить два неконтролируемых дубля?** Работа имеет persisted lease 45s и уникальный token; pipeline budget 30s, transform subprocess 20s. После истечения lease работу можно захватить снова, но старый token больше не вправе опубликовать SQL-результат. XAUTOCLAIM/redispatch используют 60s. Ограничены concurrency и чтение Count=1 на consumer. Одни часы/heartbeat без fencing не запрещают старому процессу продолжить запись после паузы. Наши разные версии пишут в разные prefixes; одинаковая версия с тем же immutable исходником и закреплёнными параметрами даёт тот же эффект. При смене алгоритма нужна новая pipeline version. + +## Состояния + +```mermaid +stateDiagram-v2 + [*] --> queued: durable stage + DB COMMIT + queued --> processing: claim + lease token + processing --> ready: all objects + fenced COMMIT + processing --> queued: transient error + persisted backoff + processing --> failed: permanent error / 5 attempts + processing --> queued: expired lease after crash + failed --> queued: explicit audited replay +``` + +Нельзя выдавать content queued/processing/failed: 409. Можно создать сообщение с queued/processing вложением своего чата. Failed прикреплять нельзя: 409. Чужой chat/attachment остаётся 404. Сессия и членство проверяются до доступа к файлу; приватный bucket и случайный UUID не заменяют ACL. + +## Окна аварии + +| Где остановились | Что осталось | Как восстанавливаемся | +|---|---|---| +| После staging до SQL COMMIT | journal/orphan, клиент не получил 202 | grace cleanup; автоматического подтверждения клиенту нет | +| После COMMIT до XADD | durable queued + job | dispatcher читает outbox | +| После XADD до mark published | возможны два сообщения | оба указывают на одну job; идемпотентный эффект | +| После claim до PUT | processing + lease | reclaim/redispatch после дедлайна | +| После PUT до result COMMIT | final objects, статус processing | повтор с теми же versioned keys, fenced COMMIT | +| После ready COMMIT до XACK | готовый результат + pending delivery | ready no-op, XACK | +| Старый v1 продолжился после v2 | старые ключи и старый lease | v2 prefix не затрагивается; SQL проверяет version/token | + +S3 не является участником SQL-транзакции. ROLLBACK не отменяет PUT. Не удаляем bytes при неопределённом COMMIT: SQL мог уже зафиксироваться. Staging удаляется только после ready COMMIT, при сбое уборки его найдёт housekeeping. + +## Изображения и ресурсы + +API: bytes ≤10 MiB, PNG/JPEG MIME и полная decode-валидация, ≤40 млн pixels до полной распаковки, filename не путь. В очереди нет bytes, URL, credentials. Worker получает storage key из доверенной БД, проверяет размер и SHA-256 исходника. Transform идёт отдельным процессом: 20s deadline, группа дочерних процессов убивается вместе. Temp directory удаляется; после SIGKILL ephemeral temp остаётся только в container layer до его удаления и не служит источником истины. + +EXIF orientation применяется до resize; alpha компонуется на белый фон. Thumbnail вписывается в 256×256 без увеличения, размеры округляются вниз, минимум 1px. 320×200 → 256×160. Crop — центрированный квадрат 128×128, допускает увеличение. Optimized — WebP ≤1280×1280 без увеличения. JPEG/WebP quality v1 = 85/80, v2 = 90/82; перекодирование убирает EXIF. Оригинал остаётся побайтно исходным. WebP маленького файла может оказаться больше исходника. + +По умолчанию 1 worker slot, CPU=1, RAM limit=1024MiB, Go memory limit=768MiB. Это реальная декодированная память, не постоянные несколько kilobytes «streaming». Concurrency 1–4 настраивается, при увеличении нужно отдельно выбирать RAM. Prefetch соответствует Count=1 на каждый consumer. Admission атомарно ограничивает незавершённые jobs 1000, затем 429 + Retry-After. Broker memory 192MiB/noeviction внутри container 256MiB: при заполнении XADD откажет, outbox остаётся в БД. Это не неограниченная очередь. + +## Retry, DLQ и уборка + +Временная S3/сетевая ошибка → queued, следующий запуск через `2^attempt` seconds + jitter [0,1)s. Всего 5 попыток. Повреждённый/несовпавший checksum исходник → failed invalid_image. Исчерпание → retry_exhausted. `failed_jobs` — SQL-реестр финальных ошибок (эквивалент DLQ), `pipeline_audit` хранит claims/errors/operator replay. Replay не разрешён для ready и требует доступного исходника. + +Housekeeping проходит каждый час, собирает только staging после 24h grace. Текущие queued/processing защищены всегда; redundant ready staging можно удалить, failed source хранится не меньше 7 days. Metadata и final objects не удаляются этим сборщиком. Старые final versions сохраняются; их GC здесь не реализован. Volume не backup; один broker/MinIO/PostgreSQL/host остаются SPOF. + +## Что показать руками + +1. `make test`: 202 → ready, download original и thumbnail. +2. Остановить worker, загрузить файл, увидеть queued и 409 content; текстовые сообщения продолжаются при доступных PG/S3. +3. Запустить worker, получить ready той же записи. +4. `pipeline.py`: реальные границы аварий и дубли; `version_fence.py`: живой старый v1 после v2. +5. `retry_backpressure.py`: 5 ошибок, финальное failed, явный replay и 429 при заполненном admission ledger. +6. Найти SQL claim/lease/token и S3 `originals//v/...` в коде, объяснить COMMIT и XACK. + +## Вопросы посложнее + +- **Почему не local queue в API?** Она теряется с процессом, не общая для реплик и не даёт durable handoff. +- **Можно ли retry upload POST?** Обязательный контракт не даёт upload idempotency: повтор может создать новое вложение. Сообщения защищены отдельным Idempotency-Key. +- **Почему DB — источник истины очереди?** Broker переносит доставку, DB связывает job, версию, состояние, lease и outbox в одной транзакции. +- **Что если пропадёт весь broker volume?** DB намерения сохраняются; незавершённые опубликованные jobs переиздаются через 60s. Это восстановление доставок, не HA broker. +- **Зачем XDEL после XACK?** Одна consumer group, завершённый delivery больше не нужен; stream не растёт от уже обработанных сообщений. Не удаляем pending до ACK. +- **Почему не просто MAXLEN stream?** Можно обрезать ещё не прочитанные/pending jobs. Мы ограничиваем приём и память, а не теряем незавершённые записи при trim. +- **Что если POST вернул timeout после COMMIT?** Клиент не знает исхода. Job не теряется, но новый upload может стать вторым объектом; нельзя обещать exactly-once приём HTTP без отдельного ключа. +- **Почему 60s recovery не нарушает smoke?** Smoke задаёт 60s для свободного исправного worker, не SLA восстановления аварии. Наблюдённая crash recovery около 60s обусловлена выбранной политикой reclaim. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..6e77e186 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,25 @@ +FROM golang:1.24.4-alpine3.22 AS build +WORKDIR /src +COPY go.mod go.sum ./ +ARG GOPROXY=https://proxy.golang.org,direct +RUN --mount=type=cache,target=/go/pkg/mod GODEBUG=http2client=0 GOPROXY="$GOPROXY" go mod download +COPY cmd/ ./cmd/ +COPY internal/ ./internal/ +RUN --mount=type=cache,target=/go/pkg/mod CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/messenger ./cmd/server + +FROM build AS tests +RUN apk add --no-cache gcc musl-dev libwebp-tools +RUN --mount=type=cache,target=/go/pkg/mod mkdir -p /opt/gomod && cp -a /go/pkg/mod/. /opt/gomod/ +ENV CGO_ENABLED=1 GOMODCACHE=/opt/gomod +CMD ["go", "test", "-race", "-count=1", "-v", "./..."] + +FROM alpine:3.22.1 +RUN apk add --no-cache libwebp-tools +RUN addgroup -g 10001 app && adduser -D -u 10001 -G app app +COPY --from=build /out/messenger /usr/local/bin/messenger +USER 10001:10001 +ENV APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab +EXPOSE 8080 +HEALTHCHECK --interval=5s --timeout=4s --start-period=3s --retries=3 CMD ["/usr/local/bin/messenger", "healthcheck"] +STOPSIGNAL SIGTERM +ENTRYPOINT ["/usr/local/bin/messenger"] diff --git a/MERGE_REQUEST.md b/MERGE_REQUEST.md new file mode 100644 index 00000000..ba8a7b8e --- /dev/null +++ b/MERGE_REQUEST.md @@ -0,0 +1,9 @@ +ЛР 5 — БВТ 2303 — Крайкин — на 5 + +Автор: Крайкин Глеб Алексеевич, БВТ2303. Цель: 5, трек2 — доказанная идемпотентность versioned pipeline. + +Приём изображения теперь сохраняет durable S3 staging и PostgreSQL job/outbox перед202. Отдельные dispatcher, persistent Valkey Streams broker и worker создают original/thumbnail/crop/optimized; реализованы bounded retry с jitter, lease/token fencing, failed registry, audited replay, backpressure и safe staging cleanup. Перенесена собственная база ЛР4; текущие контракты и публичные tests не ослаблены. + +Проверены public smoke, real PostgreSQL/race, EXIF/alpha, persistence/старые сессии, обе API/failover/rolling, реальные crash окна, duplicate/reorder, живой stale v1 после committed v2, конечные retry/replay/admission429. Evidence и воспроизводимые команды — REPORT.md; схемы, рассказ от первого лица и ответы README — DEFENSE.md. + +Из чистого clone: make up; make check; make test. Go на host не требуется. Docker volumes сохраняются; secrets генерируются отдельно и не публикуются. Ограничения: один host и shared stores без HA/backup, at-least-once доставка, старые final versions сохраняются. Использован Codex; сведения и первичные источники в REPORT.md. Remote CI не заявляется выполненным. diff --git a/Makefile b/Makefile index 5d86d171..535c3dc2 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ PYTHON ?= python3 -BASE_URL ?= http://localhost:8080 +BASE_URL ?= http://localhost:8085 CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) @@ -28,3 +28,14 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +bootstrap: + $(PYTHON) scripts/bootstrap.py +up: bootstrap + docker compose --env-file .env up -d --build --wait +down: + docker compose down +test-race: + docker compose --profile verify run --build --rm tests +clean-clone: + $(PYTHON) student-tests/clean_clone.py diff --git a/QUICKSTART.md b/QUICKSTART.md new file mode 100644 index 00000000..b56f25da --- /dev/null +++ b/QUICKSTART.md @@ -0,0 +1,38 @@ +# Запуск и демонстрация ЛР5 + +Крайкин Глеб Алексеевич, БВТ2303. Нужны Docker с Compose v2 и Python 3.9+. Go на host для запуска не нужен. + +```sh +make up +make check +make test +# API: http://localhost:8085. Только LB опубликован на loopback. +``` + +Bootstrap создаёт `.env` и случайные credentials в игнорируемом `secrets/`, существующие не меняет. Один Compose содержит две API, HAProxy, PostgreSQL, Valkey cache, MinIO, отдельный persistent Valkey Streams broker, dispatcher, worker и housekeeping. `make down` сохраняет volumes. Не удалять volumes при проверке сохранности. + +```sh +make test-race +python3 student-tests/pipeline.py +python3 student-tests/version_fence.py +python3 student-tests/retry_backpressure.py +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +# После failover остановленная реплика остаётся остановленной: +docker compose up -d --no-deps --wait api +``` + +Аварийные тесты запускать последовательно: они останавливают только компоненты этого учебного Compose-проекта и создают синтетические данные. Pipeline-тест занимает несколько минут: реальное восстановление после crash ждёт 45s lease и 60s reclaim/redispatch, timestamps не подделываются. Это не обещание ready за 60s при аварии; smoke-бюджет 60s относится к свободному worker. + +Операторский интерфейс (доступ к Docker/secrets, не публичный HTTP): + +```sh +docker compose exec -T postgres psql -U messenger -d messenger -c 'TABLE failed_jobs;' +docker compose run --rm --no-deps worker replay ATTACHMENT_UUID +docker compose run --rm --no-deps worker reprocess ATTACHMENT_UUID +# reprocess переводит поддерживаемый pipeline 1 → 2, включая supersede активного v1. +docker compose run --rm --no-deps housekeeping cleanup # dry-run +docker compose run --rm --no-deps housekeeping cleanup --apply +``` + +`replay` разрешён только failed с доступным исходником, аудит сохраняется. Для ready дубликат — no-op. `reprocess` — явное новое задание/версия, а не retry. Поддерживаются v1 и v2. Полные объяснения и ответы — `DEFENSE.md`, эксперимент — `REPORT.md`, реальные данные — `evidence/`. diff --git a/REPORT.md b/REPORT.md index e98f75e9..1a8691e9 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,106 @@ -# Отчёт — лабораторная 5 +# ЛР5 — очереди и асинхронная обработка изображений -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +Автор: Крайкин Глеб Алексеевич, БВТ2303. Целевая оценка: **5, трек 2 — доказанная идемпотентность versioned pipeline**. Дата экспериментов: 20 ноября 2026. Оценку подтверждает преподаватель после демонстрации и объяснения. -## Запуск из чистого clone +## Основа и запуск -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Skeleton текущей ЛР: `276dbdad8a34da4c5b57fbf1879ca169b581d75b`. Перенесена собственная реализация ЛР4 `1873c4c798a7d905a3247ce9351ac6da319eec98`: REST messenger, PostgreSQL, Argon2id, общие Bearer-сессии, Valkey cache-aside, HAProxy/две API, транзакционная идемпотентность сообщений и приватный MinIO. README/COURSE/contracts/публичные tests текущего задания не заменялись файлами старой лабы. -## Архитектура и решения +```sh +make up +make check +make test +make test-race +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +API `http://localhost:8085`. Запуск из этого репозитория, соседние лабы не нужны. Секреты случайные, bootstrap идемпотентный; `.env`, `secrets/`, live tokens не коммитятся. Подробные команды, replay, отказные тесты и восстановление реплик — [QUICKSTART.md](QUICKSTART.md). Разбор со схемами и **все четыре ответа из README** — [DEFENSE.md](DEFENSE.md). -## Доказательства критериев +Среда измерений: Docker Desktop Linux/aarch64, Docker 28.1.1, Compose 2.35.1-desktop.1, VM 14 CPU / 8 217 473 024 bytes RAM. Это локальная разделяемая машина, не выделенный production cluster. Лимиты: API по 1CPU/768MiB, worker 1CPU/1024MiB, PostgreSQL 2CPU/512MiB, S3 1CPU/512MiB, broker 1CPU/256MiB, dispatcher/housekeeping по .25CPU/128MiB. Версии образов закреплены в compose/Dockerfile: Go 1.24.4-alpine3.22, Alpine 3.22.1, PG 17.6, Valkey 8.1.3, HAProxy 3.2.6, MinIO RELEASE.2025-09-07T16-13-09Z, mc RELEASE.2025-08-13T08-35-41Z. Imaging v1.6.2, libwebp-tools 1.5.0-r0 наблюдался при сборке. BuildKit cache ускоряет зависимости; обычный чистый build загружает публичные модули по проверенным go.sum. При нестабильной сети для начального заполнения build cache использовалось временное loopback-зеркало уже проверенного host module cache; приложение от него не зависит, секретов в нём нет. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +## Границы durable acceptance и обработки -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +API проверяет сессию и membership, 10MiB/40MP/PNG-JPEG/filename, сохраняет capped tempfile и SHA-256; DecodeConfig ограничивает пиксели до полного decode. В S3 выполняется PUT приватного `staging/` под журналом upload_intents. В одной PG транзакции создаются attachment status=queued и job/outbox; после COMMIT возвращается **202 queued snapshot**. Bytes уже полностью переданы и устойчиво сохранены; resize выполняет только отдельный worker. SQL rollback не отменяет S3 PUT. -## Эксперименты и отказы +Отдельный Valkey Streams broker (не cache) имеет AOF, `appendfsync always`, persistent volume, maxmemory 192MiB/noeviction. Dispatcher читает job с `FOR UPDATE SKIP LOCKED`, делает XADD, затем фиксирует published_at. При неопределённом результате публикации возможен дубль. Незавершённые опубликованные jobs с истёкшим lease переиздаются после 60s, поэтому durable PG intention восстанавливает и потерянные broker deliveries. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Worker: consumer group, Count=1/consumer, concurrency по умолчанию 1 (допустимо 1–4), XAUTOCLAIM idle 60s, DB lease 45s с уникальным token, 30s pipeline budget, 20s transform child deadline. Исходник читается по key из БД, размер и checksum повторно проверяются. S3 final keys: `originals//v/{original,thumbnail,crop,optimized}`. После всех PUT готовый набор метаданных публикуется одной PG-транзакцией с проверкой version/token. Ready/stale delivery не откатывает статус. XACK после устойчивого решения; XDEL только после успешного XACK, одна consumer group. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Оригинал неизменен. EXIF orientation применяется до resize, alpha на белом. Thumbnail JPEG fit≤256×256 без upscale, floor/min1; crop JPEG center128×128; optimized WebP fit≤1280×1280 без upscale. v1 quality JPEG85/WebP80, v2 JPEG90/WebP82, metadata удаляются перекодированием. V2 — явный операторский reprocess, поддерживаются версии 1 и 2; старый namespace не затрагивает новый. Исходный 320×200 fixture даёт thumbnail256×160. WebP не обещает уменьшение каждого файла. -## Ограничения и следующие шаги +## Уровень 4 -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +| Требование | Реализация / воспроизведение | +|---|---| +| Все варианты, время и память | transform subprocess, cgroup RAM limit, concurrency1, race-тест alpha/EXIF/fit, public actual JPEG probe | +| DB/broker gap | job/outbox в той же транзакции, dispatcher; реальный API SIGKILL после committed принятия при остановленном dispatcher | +| Retry и финальная ошибка | 5 managed attempts, persisted exponential backoff 2^attempt s + jitter[0,1)s; invalid_image permanent, retry_exhausted terminal | +| DLQ/replay | SQL view failed_jobs + pipeline_audit; CLI replay только failed, HEAD исходника до изменения, аудит | +| Crash/lease/idempotency | реальные kill в before_objects/after_objects/after_commit; reclaim/redispatch; version/token fencing | +| Staging retention | после safe ready COMMIT удалить staging; hourly housekeeping, grace24h, failed retention≥7d, live work защищена | +| Backpressure | атомарный admission≤1000 unfinished jobs, затем429/Retry-After; noeviction broker не вытесняет принятую работу | + +`replay` не повторяет ready. Явный `reprocess` создаёт новую версию и помечает старую superseded; можно безопасно заменить ещё processing v1. Старый процесс не публикует метаданные из-за проверки актуальной версии/token. При одинаковой версии и immutable input параметры закреплены; эффект детерминирован. Старые final versions автоматически не собираются. + +## Исследование: гипотеза и инварианты + +Гипотеза: **durable intention + bounded lease/token + versioned deterministic effects позволяют пережить дубли и crash, не потерять принятую работу и не переписать новый результат старым**. + +Инварианты: + +1. После успешного202 существуют полный staging и committed job; остановка API не теряет bytes/намерение. +2. Одному `(attachment_id,pipeline_version)` соответствует одна логическая job и один набор effect keys. +3. ready означает существование original и всех трёх variants с сохранёнными size/checksum. +4. Старые version/token не меняют актуальные metadata, разные версии не пишут в одни object keys. +5. После 5 временных ошибок ошибка наблюдаема; очередная попытка требует явного audited replay. + +Методика: реальные синтетические HTTP upload, subprocess Docker faults, реальные lease/reclaim интервалы, проверки SQL и SHA-256 скачанного содержимого. Данные пользователя не используются. Скрипты не подделывают время. Повторять последовательно: + +```sh +python3 student-tests/pipeline.py +python3 student-tests/version_fence.py +python3 student-tests/retry_backpressure.py +``` + +| Эксперимент | Наблюдение | +|---|---| +| COMMIT до publish, API/broker restart | та же запись ready, оригинал идентичен; wait-ready0.527s после запуска сервисов | +| Kill до объектов | recovery60.062s, original +3 variants доступны | +| Kill после всех S3 PUT до result COMMIT | recovery59.747s, тот же эффект | +| Kill после ready COMMIT до ack | ready уже сохранён; GET подтвердил его за0.005s (это **не** время очистки PEL) | +| 5 повторных доставок готовой job | metadata не изменились | +| v1 reordered после ready v2 | metadata и checksums v2 не изменились | +| Живой v1 задержан перед PUT, v2 завершён, v1 отпущен | v2 ready за0.262s; поздние v1 PUT/finalization не изменили v2, unfinished jobs0 | +| checksum permanent failure + replay | failed invalid_image, запрет attachment в новом message, source восстановлен/явный replay → ready, audit сохранён | +| API SIGKILL после COMMIT до publish | принятое вложение ready после восстановления, original идентичен | +| S3 недоступен до исчерпания | 5 claims, retry_exhausted через33.17s; интервалы из реального audit2.62/4.47/8.52/16.55s плюс работа | +| Admission1000 | следующий upload429, принятая работа сохранена | + +Полные машинные данные: `evidence/pipeline.json`, `version-fence.json`, `retry-backpressure.json`, соответствующие `.txt`. Это эксперимент корректности, не исследование максимального throughput. Сохранённые времена характеризуют локальную методику, а не production SLA. Около60s после crash обусловлено политикой reclaim, не скоростью resize. + +## Проверки и доказательства + +- `make validate`: полный OpenAPI прошёл, `evidence/openapi.txt`. +- `student-tests/clean_clone.py`: настоящий чистый clone с новыми secrets/volumes; smoke и прежний токен после перезапуска API, PG, cache, S3, broker, worker и dispatcher. Изменения actual StartedAt/PID1/non-root подтверждены в `evidence/persistence-processes.json`; удалён только временный проект. +- `make check`: структура/контракт, `evidence/check.txt`. +- `make test`: публичный smoke, `evidence/smoke-final.txt`: 202, ready, ACL, исходные bytes/checksum, настоящий JPEG thumbnail. +- `make test-race`: `evidence/race.txt`: реальная PG, 100 concurrent HTTP messages + одинаковые ключи, Unicode password KDF, floor/no-upscale, alpha и EXIF orientation. +- `go vet ./...`: выполнен. +- `make test-persistence ACTION_SCRIPT=scripts/restart.sh`: прежняя сессия, сообщения и файл после restart всех application stores, `evidence/persistence.txt`. +- `make test-failover ...`: 131 успешная запись/1 failed attempt, recovery0.83s, подтверждённые данные сохранены. +- `student-tests/operations.py`: отдельно обе API recovery0.693/0.689s с1 ошибкой каждая; rolling634 успешных writes/3 ошибки. Action15.956s — длительность замены, запись продолжалась; это не15s простоя. См. `evidence/operations.json`. + +Свой smoke/fault evidence не подменяет публичные тесты. Remote CI не заявляется выполненным; GitHub workflow skeleton сам по себе не оценивает приложение. + +## Ограничения + +Один Docker host, одна PG/MinIO/broker/LB — без HA против потери host/disk; volumes не backup. S3 остаётся readiness dependency, поэтому при S3 outage HAProxy исключает API и текст через LB тоже временно недоступен. Cache outage допускает PG fallback. SDK/телеметрии в ЛР5 нет — она добавляется отдельно в ЛР6. + +Управляемый job budget5 не ограничивает количество сетевых доставок до получения claim: redelivery нормальна. Повтор job одной версии защищает эффект; exactly-once сеть/HTTP не обещается. Lease не является глобальной блокировкой S3: безопасность публикации обеспечивают SQL fencing и разделённые immutable versioned keys. Cleanup — не backup и не восстановление потерянного файла; failed source после retention требует внешнего restore. Полный hash scrub, quota на пользователя и GC старых final versions не реализованы. Runtime PG роль унаследована из учебного bootstrap, разделение ролей относится к последующей ЛР7. + +## Источники и ИИ + +Собственная база ЛР4; контракты преподавателя текущей ЛР; первичные материалы: [Valkey Streams/XAUTOCLAIM](https://valkey.io/commands/xautoclaim/), [Valkey persistence](https://valkey.io/topics/persistence/), [imaging](https://pkg.go.dev/github.com/disintegration/imaging). Использовался ИИ Codex для реализации, тестов и подготовки объяснений. Результаты получены реальным запуском; автор должен уметь объяснить код и ограничения на защите. + +Финальный encoder различает invalid image (child exit65) и временную аварию/kill/deadline: последняя проходит bounded retry, а не маркирует корректный файл invalid_image. EXIF обрабатывается для JPEG APP1 и PNG eXIf, перед resize. Финальные race/PG проверки актуальных исходников, включая PNG EXIF и аварии subprocess, — `evidence/race-final.txt`. Для проверки использован уже скачанный совместимый Go1.24.4/GCC/cwebp test toolchain с read-only bind текущих cmd/internal (в ЛР6 также текущие go.mod/go.sum); публичный `make test-race` сам собирает такой toolchain из Dockerfile, соседняя лабораторная для запуска/проверки не требуется. + +Повторная проверка финального implementation commit `8bd25a1` из настоящего чистого клона — PASS: свежие credentials/volumes, smoke, старый токен после полного restart, actual StartedAt/PID1/non-root. Вывод — `evidence/clean-clone-final.txt`, снимки — `persistence-processes.json`. Удалён только уникальный временный проект проверки. diff --git a/cmd/server/main.go b/cmd/server/main.go new file mode 100644 index 00000000..598d794d --- /dev/null +++ b/cmd/server/main.go @@ -0,0 +1,208 @@ +package main + +import ( + "context" + "encoding/base64" + "errors" + "fmt" + "github.com/redis/go-redis/v9" + "log/slog" + "mtusi/messenger/internal/database" + "mtusi/messenger/internal/messenger" + "net" + "net/http" + "os" + "os/signal" + "strconv" + "strings" + "syscall" + "time" +) + +func env(key, fallback string) string { + if v := os.Getenv(key); v != "" { + return v + } + return fallback +} +func main() { + if err := run(); err != nil { + if errors.Is(err, messenger.ErrInvalidImage) { + os.Exit(65) + } + slog.Error("server failed", "error", err) + os.Exit(1) + } +} +func run() error { + if (len(os.Args) == 4 || len(os.Args) == 5) && os.Args[1] == "transform" { + version := 1 + if len(os.Args) == 5 { + v, e := strconv.Atoi(os.Args[4]) + if e != nil { + return e + } + version = v + } + return messenger.Transform(os.Args[2], os.Args[3], version) + } + host := env("APP_HOST", "0.0.0.0") + port := env("APP_PORT", "8080") + p, err := strconv.Atoi(port) + if err != nil || p < 1 || p > 65535 { + return fmt.Errorf("invalid APP_PORT") + } + if len(os.Args) == 2 && os.Args[1] == "healthcheck" { + c := http.Client{Timeout: time.Second} + r, e := c.Get("http://127.0.0.1:" + port + "/health/ready") + if e != nil { + return e + } + defer r.Body.Close() + if r.StatusCode != 200 { + return fmt.Errorf("not ready: %d", r.StatusCode) + } + return nil + } + + log := slog.New(slog.NewJSONHandler(os.Stdout, nil)) + slog.SetDefault(log) + ttl, err := strconv.Atoi(env("SESSION_TTL_SECONDS", "3600")) + if err != nil || ttl < 1 || ttl > 86400 { + return fmt.Errorf("SESSION_TTL_SECONDS must be 1..86400") + } + databaseURL, err := secret("DATABASE_URL_FILE") + if err != nil { + return err + } + startup, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + pool, err := database.Open(startup, databaseURL) + if err != nil { + return err + } + defer pool.Close() + if len(os.Args) == 2 && os.Args[1] == "migrate" { + return database.Migrate(startup, pool) + } + storeAccess, err := secret("S3_ACCESS_KEY_ID_FILE") + if err != nil { + return err + } + storeSecret, err := secret("S3_SECRET_ACCESS_KEY_FILE") + if err != nil { + return err + } + objects, err := messenger.NewObjectStore(env("S3_ENDPOINT", "http://s3:9000"), env("S3_BUCKET", "messenger-attachments"), storeAccess, storeSecret) + if err != nil { + return err + } + if len(os.Args) > 1 { + command := os.Args[1] + if command == "worker" || command == "dispatcher" || command == "replay" || command == "reprocess" || command == "cleanup" || command == "housekeeping" { + brokerURL, e := secret("BROKER_URL_FILE") + if e != nil { + return e + } + opts, e := redis.ParseURL(brokerURL) + if e != nil { + return fmt.Errorf("invalid broker configuration") + } + opts.MaxRetries = -1 + opts.DialTimeout = time.Second + opts.ReadTimeout = 2 * time.Second + opts.WriteTimeout = 2 * time.Second + broker := redis.NewClient(opts) + defer broker.Close() + pipeline := messenger.NewPipeline(pool, broker, objects, log) + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + switch command { + case "worker": + n, e := strconv.Atoi(env("WORKER_CONCURRENCY", "1")) + if e != nil { + return e + } + return pipeline.RunWorker(ctx, n) + case "dispatcher": + return pipeline.RunDispatcher(ctx) + case "housekeeping": + return pipeline.RunHousekeeping(ctx) + case "cleanup": + return pipeline.Cleanup(ctx, len(os.Args) == 3 && os.Args[2] == "--apply") + case "replay", "reprocess": + if len(os.Args) != 3 { + return fmt.Errorf("expected attachment UUID") + } + return pipeline.Replay(ctx, os.Args[2], command == "reprocess") + } + } + } + cacheURL, err := secret("CACHE_URL_FILE") + if err != nil { + return err + } + cursorKey, err := secret("CURSOR_KEY_FILE") + if err != nil { + return err + } + key, err := base64.RawURLEncoding.DecodeString(cursorKey) + if err != nil || len(key) != 32 { + return fmt.Errorf("invalid cursor key") + } + opts, err := redis.ParseURL(cacheURL) + if err != nil { + return fmt.Errorf("invalid cache configuration") + } + opts.DialTimeout = 60 * time.Millisecond + opts.ReadTimeout = 60 * time.Millisecond + opts.WriteTimeout = 60 * time.Millisecond + opts.PoolTimeout = 60 * time.Millisecond + opts.ContextTimeoutEnabled = true + opts.MaxRetries = -1 + opts.PoolSize = 10 + cache := redis.NewClient(opts) + defer cache.Close() + api := messenger.New(pool, cache, key, ttl, log) + api.SetObjectStore(objects) + srv := &http.Server{Addr: net.JoinHostPort(host, port), Handler: api, ReadHeaderTimeout: 3 * time.Second, ReadTimeout: 35 * time.Second, WriteTimeout: 35 * time.Second, IdleTimeout: 30 * time.Second, MaxHeaderBytes: 16 << 10} + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + done := make(chan error, 1) + go func() { + log.Info("listening", "address", srv.Addr, "session_store", "postgres", "profile_cache", "valkey", "environment", env("APP_ENV", "lab")) + done <- srv.ListenAndServe() + }() + select { + case err := <-done: + return err + case <-ctx.Done(): + } + start := time.Now() + api.Stop() + log.Info("shutdown started") + shutdown, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := srv.Shutdown(shutdown); err != nil { + _ = srv.Close() + return err + } + err = <-done + if !errors.Is(err, http.ErrServerClosed) { + return err + } + log.Info("shutdown completed", "duration_ms", time.Since(start).Milliseconds()) + return nil +} + +func secret(name string) (string, error) { + path := os.Getenv(name) + if path == "" { + return "", fmt.Errorf("missing %s", name) + } + raw, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("cannot read %s", name) + } + return strings.TrimSpace(string(raw)), nil +} diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..b117ff9a --- /dev/null +++ b/compose.yaml @@ -0,0 +1,196 @@ +name: mtusi-lab5-kraykin +x-api: &api + image: mtusi-messenger:lab5 + build: . + environment: &api-env + APP_PORT: 8080 + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + CURSOR_KEY_FILE: /run/secrets/cursor_key + S3_ENDPOINT: http://s3:9000 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + GOMEMLIMIT: 600MiB + DB_POOL_MAX: ${DB_POOL_MAX:-10} + secrets: [database_url, cache_url, cursor_key, s3_access_key_id, s3_secret_access_key] + cpus: ${API_CPUS:-1} + mem_limit: ${API_MEMORY:-768m} + stop_grace_period: 8s +services: + postgres: + image: postgres:17.6-alpine + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + volumes: [postgres_data:/var/lib/postgresql/data] + cpus: 2 + mem_limit: 512m + shm_size: 128m + healthcheck: + test: [CMD-SHELL, 'pg_isready -U messenger -d messenger'] + interval: 1s + timeout: 2s + retries: 30 + valkey: + image: valkey/valkey:8.1.3-alpine + command: [valkey-server, /run/secrets/valkey_config] + secrets: [valkey_config] + cpus: 0.5 + mem_limit: 128m + s3: + image: quay.io/minio/minio:RELEASE.2025-09-07T16-13-09Z + command: [server, /data, --console-address, '127.0.0.1:9001'] + environment: + MINIO_ROOT_USER_FILE: /run/secrets/s3_root_user + MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_root_password + secrets: [s3_root_user, s3_root_password] + volumes: [s3_data:/data] + cpus: 1 + mem_limit: 512m + healthcheck: + test: [CMD, curl, -f, http://localhost:9000/minio/health/ready] + interval: 1s + timeout: 2s + retries: 30 + s3-init: + image: minio/mc:RELEASE.2025-08-13T08-35-41Z + entrypoint: [/bin/sh, /config/s3-init.sh] + volumes: ['./infra:/config:ro'] + secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key] + depends_on: + s3: {condition: service_healthy} + cpus: 0.5 + mem_limit: 128m + migrate: + <<: *api + secrets: [database_url] + command: [migrate] + depends_on: + postgres: {condition: service_healthy} + restart: 'no' + api: + <<: *api + deploy: + replicas: ${API_REPLICAS:-2} + depends_on: + s3-init: {condition: service_completed_successfully} + migrate: {condition: service_completed_successfully} + valkey: {condition: service_started} + broker: + image: valkey/valkey:8.1.3-alpine + command: [valkey-server, /run/secrets/broker_config] + secrets: [broker_config, broker_url] + volumes: [broker_data:/data] + cpus: 1 + mem_limit: 256m + dispatcher: + <<: *api + command: [dispatcher] + secrets: [database_url, broker_url, s3_access_key_id, s3_secret_access_key] + healthcheck: + test: [CMD-SHELL, 'kill -0 1'] + interval: 2s + timeout: 1s + retries: 3 + environment: + <<: *api-env + BROKER_URL_FILE: /run/secrets/broker_url + DB_POOL_MAX: 2 + cpus: 0.25 + mem_limit: 128m + depends_on: + migrate: {condition: service_completed_successfully} + broker: {condition: service_started} + restart: unless-stopped + worker: + <<: *api + command: [worker] + secrets: [database_url, broker_url, s3_access_key_id, s3_secret_access_key] + healthcheck: + test: [CMD-SHELL, 'kill -0 1'] + interval: 2s + timeout: 1s + retries: 3 + environment: + <<: *api-env + BROKER_URL_FILE: /run/secrets/broker_url + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-1} + DB_POOL_MAX: 4 + GOMEMLIMIT: 768MiB + cpus: 1 + mem_limit: 1024m + stop_grace_period: 35s + depends_on: + migrate: {condition: service_completed_successfully} + s3-init: {condition: service_completed_successfully} + broker: {condition: service_started} + restart: unless-stopped + housekeeping: + <<: *api + command: [housekeeping] + secrets: [database_url, broker_url, s3_access_key_id, s3_secret_access_key] + healthcheck: + test: [CMD-SHELL, 'kill -0 1'] + interval: 2s + timeout: 1s + retries: 3 + environment: + <<: *api-env + BROKER_URL_FILE: /run/secrets/broker_url + DB_POOL_MAX: 2 + cpus: 0.25 + mem_limit: 128m + depends_on: + migrate: {condition: service_completed_successfully} + s3-init: {condition: service_completed_successfully} + restart: unless-stopped + tests: + volumes: [go_test_cache:/root/.cache/go-build] + image: mtusi-messenger-tests:lab5 + cpus: 2 + mem_limit: 1024m + profiles: [verify] + build: + context: . + target: tests + environment: + GOMAXPROCS: "2" + GOFLAGS: -p=2 + DATABASE_TEST_URL_FILE: /run/secrets/database_url + secrets: [database_url] + depends_on: + migrate: {condition: service_completed_successfully} + lb: + image: haproxy:3.2.6-alpine + volumes: ['./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro'] + ports: ['127.0.0.1:${APP_PORT:-8085}:8080'] + cpus: 1 + mem_limit: 128m + depends_on: + api: {condition: service_healthy} + healthcheck: + test: [CMD-SHELL, 'wget -q -O /dev/null http://127.0.0.1:8080/health/ready'] + interval: 2s + timeout: 4s + retries: 20 +secrets: + broker_url: {file: ./secrets/broker_url} + broker_config: {file: ./secrets/broker_config} + s3_root_user: {file: ./secrets/s3_root_user} + s3_root_password: {file: ./secrets/s3_root_password} + s3_access_key_id: {file: ./secrets/s3_access_key_id} + s3_secret_access_key: {file: ./secrets/s3_secret_access_key} + postgres_password: {file: ./secrets/postgres_password} + database_url: {file: ./secrets/database_url} + valkey_config: {file: ./secrets/valkey_config} + cache_url: {file: ./secrets/cache_url} + cursor_key: {file: ./secrets/cursor_key} +volumes: + go_test_cache: + broker_data: + s3_data: + postgres_data: diff --git a/evidence/build-final.txt b/evidence/build-final.txt new file mode 100644 index 00000000..696e2f29 --- /dev/null +++ b/evidence/build-final.txt @@ -0,0 +1,221 @@ +Compose can now delegate builds to bake for better performance. + To do so, set COMPOSE_BAKE=true. +#0 building with "desktop-linux" instance using docker driver + +#1 [migrate internal] load build definition from Dockerfile +#1 transferring dockerfile: 1.11kB done +#1 DONE 0.0s + +#2 [migrate internal] load metadata for docker.io/library/golang:1.24.4-alpine3.22 +#2 DONE 0.0s + +#3 [migrate internal] load metadata for docker.io/library/alpine:3.22.1 +#3 DONE 1.9s + +#4 [migrate internal] load .dockerignore +#4 transferring context: 196B done +#4 DONE 0.0s + +#5 [migrate build 1/7] FROM docker.io/library/golang:1.24.4-alpine3.22@sha256:68932fa6d4d4059845c8f40ad7e654e626f3ebd3706eef7846f319293ab5cb7a +#5 resolve docker.io/library/golang:1.24.4-alpine3.22@sha256:68932fa6d4d4059845c8f40ad7e654e626f3ebd3706eef7846f319293ab5cb7a done +#5 DONE 0.0s + +#6 [migrate stage-2 1/4] FROM docker.io/library/alpine:3.22.1@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1 +#6 resolve docker.io/library/alpine:3.22.1@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1 done +#6 DONE 0.0s + +#7 [migrate internal] load build context +#7 transferring context: 28.22kB done +#7 DONE 0.0s + +#8 [migrate build 2/7] WORKDIR /src +#8 CACHED + +#9 [migrate build 3/7] COPY go.mod go.sum ./ +#9 CACHED + +#10 [migrate build 4/7] RUN --mount=type=cache,target=/go/pkg/mod GODEBUG=http2client=0 GOPROXY="https://proxy.golang.org,direct" go mod download +#10 DONE 0.1s + +#11 [migrate build 5/7] COPY cmd/ ./cmd/ +#11 DONE 0.0s + +#12 [migrate build 6/7] COPY internal/ ./internal/ +#12 DONE 0.1s + +#13 [migrate build 7/7] RUN --mount=type=cache,target=/go/pkg/mod CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/messenger ./cmd/server +#13 DONE 8.3s + +#14 [migrate stage-2 2/4] RUN apk add --no-cache libwebp-tools +#14 CACHED + +#15 [migrate stage-2 3/4] RUN addgroup -g 10001 app && adduser -D -u 10001 -G app app +#15 CACHED + +#16 [migrate stage-2 4/4] COPY --from=build /out/messenger /usr/local/bin/messenger +#16 DONE 0.0s + +#8 [migrate build 2/7] WORKDIR /src +#8 CACHED + +#17 [migrate] exporting to image +#17 exporting layers +#17 exporting layers 0.3s done +#17 exporting manifest sha256:6b6d6394a17ce66060e62a4c7b5a254b19007bde24774f34d92982110b30a331 +#17 exporting manifest sha256:6b6d6394a17ce66060e62a4c7b5a254b19007bde24774f34d92982110b30a331 done +#17 exporting config sha256:05e1e392a958982aaf1358f26a95b7d7617896a03a4fce79342e17fa2c33b5c0 done +#17 exporting attestation manifest sha256:21ce987bacdd92756093729ead847e2f1054c5ccecb0cb6e78575d83d67a3591 0.0s done +#17 exporting manifest list sha256:72d34da89387de9ae74cf6fc68e1e75a613c75ca9abf5cc91abdc9592ecb9243 done +#17 naming to docker.io/library/mtusi-messenger:lab5 done +#17 unpacking to docker.io/library/mtusi-messenger:lab5 0.1s done +#17 DONE 0.4s + +#18 [migrate] resolving provenance for metadata file +#18 DONE 0.0s + +#19 [housekeeping internal] load build definition from Dockerfile +#19 transferring dockerfile: 1.11kB done +#19 DONE 0.0s + +#20 [worker internal] load build definition from Dockerfile +#20 transferring dockerfile: 1.11kB done +#20 DONE 0.0s + +#21 [dispatcher internal] load build definition from Dockerfile +#21 transferring dockerfile: 1.11kB done +#21 DONE 0.0s + +#22 [api internal] load build definition from Dockerfile +#22 transferring dockerfile: 1.11kB done +#22 DONE 0.0s + +#2 [api internal] load metadata for docker.io/library/golang:1.24.4-alpine3.22 +#2 DONE 0.0s + +#3 [api internal] load metadata for docker.io/library/alpine:3.22.1 +#3 DONE 2.3s + +#5 [dispatcher build 1/7] FROM docker.io/library/golang:1.24.4-alpine3.22@sha256:68932fa6d4d4059845c8f40ad7e654e626f3ebd3706eef7846f319293ab5cb7a +#5 resolve docker.io/library/golang:1.24.4-alpine3.22@sha256:68932fa6d4d4059845c8f40ad7e654e626f3ebd3706eef7846f319293ab5cb7a done +#5 DONE 0.0s + +#6 [dispatcher stage-2 1/4] FROM docker.io/library/alpine:3.22.1@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1 +#6 resolve docker.io/library/alpine:3.22.1@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1 done +#6 DONE 0.0s + +#14 [dispatcher stage-2 2/4] RUN apk add --no-cache libwebp-tools +#14 CACHED + +#15 [dispatcher stage-2 3/4] RUN addgroup -g 10001 app && adduser -D -u 10001 -G app app +#15 CACHED + +#8 [dispatcher build 2/7] WORKDIR /src +#8 CACHED + +#23 [worker internal] load .dockerignore +#23 transferring context: 196B done +#23 DONE 0.0s + +#24 [api internal] load .dockerignore +#24 transferring context: 196B done +#24 DONE 0.0s + +#25 [housekeeping internal] load .dockerignore +#25 transferring context: 196B done +#25 DONE 0.0s + +#26 [dispatcher internal] load .dockerignore +#26 transferring context: 196B done +#26 DONE 0.0s + +#27 [worker internal] load build context +#27 transferring context: 1.24kB done +#27 DONE 0.0s + +#28 [api internal] load build context +#28 transferring context: 28.22kB done +#28 DONE 0.0s + +#29 [housekeeping internal] load build context +#29 transferring context: 37.71kB done +#29 DONE 0.0s + +#30 [dispatcher internal] load build context +#30 transferring context: 28.22kB done +#30 DONE 0.0s + +#31 [worker build 7/7] RUN --mount=type=cache,target=/go/pkg/mod CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/messenger ./cmd/server +#31 CACHED + +#32 [worker build 5/7] COPY cmd/ ./cmd/ +#32 CACHED + +#33 [worker build 4/7] RUN --mount=type=cache,target=/go/pkg/mod GODEBUG=http2client=0 GOPROXY="https://proxy.golang.org,direct" go mod download +#33 CACHED + +#34 [worker build 3/7] COPY go.mod go.sum ./ +#34 CACHED + +#35 [worker build 6/7] COPY internal/ ./internal/ +#35 CACHED + +#36 [api stage-2 4/4] COPY --from=build /out/messenger /usr/local/bin/messenger +#36 CACHED + +#37 [dispatcher] exporting to image +#37 exporting layers done +#37 exporting manifest sha256:c02fe9b7ebb184e6c118f4ede981fe5caa3397ffc138f731106cb7e2e4b6531d 0.0s done +#37 exporting config sha256:838b7b8a576feb3d6d5b76b3643c550e8fd71dc2a7715258cb61bb54fe21f20c 0.0s done +#37 exporting attestation manifest sha256:0ad90c403f82768961b4c47fdb5c0dcfadb9073b4db8d2e1d1bf86ae7a5e5373 0.0s done +#37 exporting manifest list sha256:b3c43cd703a7da812b351d0e37525df173df42264d6f36bea6c81dd1f9337af0 +#37 exporting manifest list sha256:b3c43cd703a7da812b351d0e37525df173df42264d6f36bea6c81dd1f9337af0 0.0s done +#37 naming to docker.io/library/mtusi-messenger:lab5 done +#37 unpacking to docker.io/library/mtusi-messenger:lab5 done +#37 DONE 0.1s + +#38 [worker] exporting to image +#38 exporting layers done +#38 exporting manifest sha256:bae712d43dc4af6c65d419a69f41938341f6bb406e6ad4cf26f3f82ad7ca9785 0.0s done +#38 exporting config sha256:405c89c05e46af771b06f7f4d671195de0e6e1b17cc37c0b0978085f5904659b 0.0s done +#38 exporting attestation manifest sha256:0b25d169e1cbe715c9af435846f658ce54cfcd8306f8c7f29acd4a6c4409db8c 0.0s done +#38 exporting manifest list sha256:ebc5e6f3c2878466f14d979e7499732080b79c9bfb71298907d8953748c79b63 0.0s done +#38 naming to docker.io/library/mtusi-messenger:lab5 done +#38 unpacking to docker.io/library/mtusi-messenger:lab5 done +#38 DONE 0.1s + +#39 [api] exporting to image +#39 exporting layers done +#39 exporting manifest sha256:514e715c3eec5b448a7b8f557d1063e312366039f81fb71c64091019003bdda1 0.0s done +#39 exporting config sha256:e4719b42b6351f1e36b2df6dd7d6bdcebdb22c675494b9bca9d9ec25a37921bd 0.0s done +#39 exporting attestation manifest sha256:d9470e40f807dfba51e6f1c3a70b53e425dfcc59bca3f4b8279290c55cfaf739 0.0s done +#39 exporting manifest list sha256:dd1e4a99fce52ee5e2d62d0bdf381a3bc7f1ead3336ef85c9dea2c6968758903 0.0s done +#39 naming to docker.io/library/mtusi-messenger:lab5 done +#39 unpacking to docker.io/library/mtusi-messenger:lab5 done +#39 DONE 0.1s + +#40 [housekeeping] exporting to image +#40 exporting layers done +#40 exporting manifest sha256:9b451c6477fbcd1831c729b6f513e5ca956b9748242d7c23ae5a7a9ea69c3542 0.0s done +#40 exporting config sha256:25a4952f9337756f86be62f3bdba3d451e0ebcc486235c1c42933d5d871c488b 0.0s done +#40 exporting attestation manifest sha256:c1c84cdd7e5dc374b60284fb70829d9ad6597f43aa69f349b1e0b9478200bb61 0.0s done +#40 exporting manifest list sha256:033d93df8f2079dcce2716e5cc7d97e22fac1092ed299c04f8002eca730a0f1e 0.0s done +#40 naming to docker.io/library/mtusi-messenger:lab5 done +#40 unpacking to docker.io/library/mtusi-messenger:lab5 done +#40 DONE 0.1s + +#41 [api] resolving provenance for metadata file +#41 DONE 0.0s + +#42 [worker] resolving provenance for metadata file +#42 DONE 0.0s + +#43 [housekeeping] resolving provenance for metadata file +#43 DONE 0.0s + +#44 [dispatcher] resolving provenance for metadata file +#44 DONE 0.0s + api Built + dispatcher Built + housekeeping Built + migrate Built + worker Built diff --git a/evidence/check.txt b/evidence/check.txt new file mode 100644 index 00000000..9a553306 --- /dev/null +++ b/evidence/check.txt @@ -0,0 +1,2 @@ +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/clean-clone-final.txt b/evidence/clean-clone-final.txt new file mode 100644 index 00000000..bfbbbae8 --- /dev/null +++ b/evidence/clean-clone-final.txt @@ -0,0 +1,389 @@ +Cloning into '/var/folders/hx/czbzpyk92ds_mnf7n9xrzn_80000gn/T/verify-lab5-23afcf3c2b-4bx3std_/repo'... +python3 scripts/bootstrap.py +Local secrets ready (values hidden); .env and secrets/ are ignored by Git. +docker compose --env-file .env up -d --build --wait +Compose can now delegate builds to bake for better performance. + To do so, set COMPOSE_BAKE=true. +#0 building with "desktop-linux" instance using docker driver + +#1 [migrate internal] load build definition from Dockerfile +#1 transferring dockerfile: 1.11kB done +#1 DONE 0.0s + +#2 [migrate internal] load metadata for docker.io/library/golang:1.24.4-alpine3.22 +#2 DONE 0.1s + +#3 [migrate internal] load metadata for docker.io/library/alpine:3.22.1 +#3 DONE 3.9s + +#4 [migrate internal] load .dockerignore +#4 transferring context: 196B done +#4 DONE 0.0s + +#5 [migrate internal] load build context +#5 DONE 0.0s + +#6 [migrate stage-2 1/4] FROM docker.io/library/alpine:3.22.1@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1 +#6 resolve docker.io/library/alpine:3.22.1@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1 done +#6 DONE 0.0s + +#7 [migrate build 1/7] FROM docker.io/library/golang:1.24.4-alpine3.22@sha256:68932fa6d4d4059845c8f40ad7e654e626f3ebd3706eef7846f319293ab5cb7a +#7 resolve docker.io/library/golang:1.24.4-alpine3.22@sha256:68932fa6d4d4059845c8f40ad7e654e626f3ebd3706eef7846f319293ab5cb7a +#7 resolve docker.io/library/golang:1.24.4-alpine3.22@sha256:68932fa6d4d4059845c8f40ad7e654e626f3ebd3706eef7846f319293ab5cb7a 1.4s done +#7 DONE 1.4s + +#5 [migrate internal] load build context +#5 transferring context: 88.77kB done +#5 DONE 0.0s + +#8 [migrate build 6/7] COPY internal/ ./internal/ +#8 CACHED + +#9 [migrate build 7/7] RUN --mount=type=cache,target=/go/pkg/mod CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/messenger ./cmd/server +#9 CACHED + +#10 [migrate build 4/7] RUN --mount=type=cache,target=/go/pkg/mod GODEBUG=http2client=0 GOPROXY="https://proxy.golang.org,direct" go mod download +#10 CACHED + +#11 [migrate stage-2 2/4] RUN apk add --no-cache libwebp-tools +#11 CACHED + +#12 [migrate build 5/7] COPY cmd/ ./cmd/ +#12 CACHED + +#13 [migrate stage-2 3/4] RUN addgroup -g 10001 app && adduser -D -u 10001 -G app app +#13 CACHED + +#14 [migrate build 2/7] WORKDIR /src +#14 CACHED + +#15 [migrate build 3/7] COPY go.mod go.sum ./ +#15 CACHED + +#16 [migrate stage-2 4/4] COPY --from=build /out/messenger /usr/local/bin/messenger +#16 CACHED + +#17 [migrate] exporting to image +#17 exporting layers done +#17 exporting manifest sha256:0e88f9ceb3bcad3e69f59aad60f9741de5df90dbeff9844fe45b73cb006a15ff done +#17 exporting config sha256:ccada027a2096dda12604dee9dda14e473761270c2b690d685ed41df10df36fb done +#17 exporting attestation manifest sha256:338551336d5d9c5c816be03e5df1ee463c6ca4bb4e315f8d476bd78fbdfd5c8a done +#17 exporting manifest list sha256:232cbb00510ce2fbb02f0065b49aca7972edc5116dd4565edd1bd746ab2fd8ac done +#17 naming to docker.io/library/mtusi-messenger:lab5 done +#17 unpacking to docker.io/library/mtusi-messenger:lab5 done +#17 DONE 0.0s + +#18 [migrate] resolving provenance for metadata file +#18 DONE 0.0s + +#19 [housekeeping internal] load build definition from Dockerfile +#19 transferring dockerfile: 1.11kB done +#19 DONE 0.0s + +#20 [worker internal] load build definition from Dockerfile +#20 transferring dockerfile: 1.11kB done +#20 DONE 0.0s + +#21 [api internal] load build definition from Dockerfile +#21 transferring dockerfile: 1.11kB done +#21 DONE 0.0s + +#2 [housekeeping internal] load metadata for docker.io/library/golang:1.24.4-alpine3.22 +#2 DONE 0.2s + +#22 [dispatcher internal] load build definition from Dockerfile +#22 transferring dockerfile: 1.11kB done +#22 DONE 0.0s + +#3 [dispatcher internal] load metadata for docker.io/library/alpine:3.22.1 +#3 DONE 4.3s + +#23 [housekeeping internal] load .dockerignore +#23 transferring context: 196B done +#23 DONE 0.0s + +#24 [worker internal] load .dockerignore +#24 transferring context: 196B done +#24 DONE 0.0s + +#25 [dispatcher internal] load .dockerignore +#25 transferring context: 196B done +#25 DONE 0.0s + +#26 [api internal] load .dockerignore +#26 transferring context: 196B done +#26 DONE 0.0s + +#7 [housekeeping build 1/7] FROM docker.io/library/golang:1.24.4-alpine3.22@sha256:68932fa6d4d4059845c8f40ad7e654e626f3ebd3706eef7846f319293ab5cb7a +#7 resolve docker.io/library/golang:1.24.4-alpine3.22@sha256:68932fa6d4d4059845c8f40ad7e654e626f3ebd3706eef7846f319293ab5cb7a 0.0s done +#7 DONE 1.4s + +#6 [worker stage-2 1/4] FROM docker.io/library/alpine:3.22.1@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1 +#6 resolve docker.io/library/alpine:3.22.1@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1 0.0s done +#6 DONE 0.0s + +#27 [api internal] load build context +#27 transferring context: 1.30kB done +#27 DONE 0.0s + +#28 [housekeeping internal] load build context +#28 transferring context: 88.77kB done +#28 DONE 0.0s + +#29 [dispatcher internal] load build context +#29 transferring context: 88.77kB done +#29 DONE 0.0s + +#30 [worker internal] load build context +#30 transferring context: 88.77kB 0.0s done +#30 DONE 0.0s + +#31 [api build 5/7] COPY cmd/ ./cmd/ +#31 CACHED + +#32 [api build 4/7] RUN --mount=type=cache,target=/go/pkg/mod GODEBUG=http2client=0 GOPROXY="https://proxy.golang.org,direct" go mod download +#32 CACHED + +#33 [api build 6/7] COPY internal/ ./internal/ +#33 CACHED + +#14 [api build 2/7] WORKDIR /src +#14 CACHED + +#34 [api build 7/7] RUN --mount=type=cache,target=/go/pkg/mod CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/messenger ./cmd/server +#34 CACHED + +#35 [api build 3/7] COPY go.mod go.sum ./ +#35 CACHED + +#11 [worker stage-2 2/4] RUN apk add --no-cache libwebp-tools +#11 CACHED + +#13 [worker stage-2 3/4] RUN addgroup -g 10001 app && adduser -D -u 10001 -G app app +#13 CACHED + +#36 [worker stage-2 4/4] COPY --from=build /out/messenger /usr/local/bin/messenger +#36 CACHED + +#37 [worker] exporting to image +#37 exporting layers done +#37 exporting manifest sha256:643b1b3edad4be21a1050fd98a6aa23a2be6ca4562cbdfebdf5b1b6a67da6629 0.0s done +#37 exporting config sha256:56b3fab89e4581a6c660a253dc89996e3c197488bb3e920333b72ce1124d252f 0.0s done +#37 exporting attestation manifest sha256:283052274cc1588ed791f69eeaf3f49f872906517fde93a1e78dd5c2837ec420 0.0s done +#37 exporting manifest list sha256:881c94e3264d9d923d0abc13670e6656d06aec6757ef828178a3a16f90614b10 0.0s done +#37 naming to docker.io/library/mtusi-messenger:lab5 done +#37 unpacking to docker.io/library/mtusi-messenger:lab5 done +#37 DONE 0.1s + +#38 [api] exporting to image +#38 exporting layers done +#38 exporting manifest sha256:b410268857f6686ac93c3c7bff8bbfc55c5f9e115057cd09326767babc159ef2 0.0s done +#38 exporting config sha256:49cd949da25f654118a65be450c944a9fc3a4c3e63e2e87176a5188649139b14 0.0s done +#38 exporting attestation manifest sha256:40ee9c282bbd408a6d7addfdf12ee055d6774b1be757d9431926623a808167a6 0.0s done +#38 exporting manifest list sha256:fa149fd8d1b38071c51051d979289bc7f2831734eda8528e1a9e8e732346d030 0.0s done +#38 naming to docker.io/library/mtusi-messenger:lab5 done +#38 unpacking to docker.io/library/mtusi-messenger:lab5 done +#38 DONE 0.1s + +#39 [dispatcher] exporting to image +#39 exporting layers done +#39 exporting manifest sha256:e948a4d3f2e757fbeb84bf30df909513559a6ae3f68683ae96fd8e9fc93dbd19 0.0s done +#39 exporting config sha256:e0bafa79dc312c35af565297280f850373fddfd2b809a150a1b79c73528ab60c 0.0s done +#39 exporting attestation manifest sha256:0bf02d807502633ae65a0a70e92807f81331897f282286188135bb611745c1f2 0.0s done +#39 exporting manifest list sha256:d88960713893b7fb69ab26b4fa15979ad10e5b32fbb4a483a08b379f07105e22 0.0s done +#39 naming to docker.io/library/mtusi-messenger:lab5 done +#39 unpacking to docker.io/library/mtusi-messenger:lab5 done +#39 DONE 0.1s + +#40 [housekeeping] exporting to image +#40 exporting layers done +#40 exporting manifest sha256:b4474ea7699e4ac88ebfd199fb8c357bf66fedcff5f57afdcaf1efafcefc9d6e 0.0s done +#40 exporting config sha256:c6e09625ef27a94cde82afadeacacd5f632288577a45f7a7853d013fbf4a2011 0.0s done +#40 exporting attestation manifest sha256:4b539c71d091143a8bed27e412efc22b8e8d1f4f37f6c9897a6e360cd14254e1 0.0s done +#40 exporting manifest list sha256:5e2f81f79525967e38b0e8e6a3ba7e4c717ffbeee9aedac115fc20a53a1f7755 0.0s done +#40 naming to docker.io/library/mtusi-messenger:lab5 done +#40 unpacking to docker.io/library/mtusi-messenger:lab5 done +#40 DONE 0.1s + +#41 [api] resolving provenance for metadata file +#41 DONE 0.0s + +#42 [dispatcher] resolving provenance for metadata file +#42 DONE 0.0s + +#43 [housekeeping] resolving provenance for metadata file +#43 DONE 0.0s + +#44 [worker] resolving provenance for metadata file +#44 DONE 0.0s + api Built + dispatcher Built + housekeeping Built + migrate Built + worker Built + Network verify-lab5-23afcf3c2b_default Creating + Network verify-lab5-23afcf3c2b_default Created + Volume "verify-lab5-23afcf3c2b_s3_data" Creating + Volume "verify-lab5-23afcf3c2b_s3_data" Created + Volume "verify-lab5-23afcf3c2b_postgres_data" Creating + Volume "verify-lab5-23afcf3c2b_postgres_data" Created + Volume "verify-lab5-23afcf3c2b_broker_data" Creating + Volume "verify-lab5-23afcf3c2b_broker_data" Created + Container verify-lab5-23afcf3c2b-broker-1 Creating + Container verify-lab5-23afcf3c2b-valkey-1 Creating + Container verify-lab5-23afcf3c2b-postgres-1 Creating + Container verify-lab5-23afcf3c2b-s3-1 Creating + Container verify-lab5-23afcf3c2b-valkey-1 Created + Container verify-lab5-23afcf3c2b-broker-1 Created + Container verify-lab5-23afcf3c2b-postgres-1 Created + Container verify-lab5-23afcf3c2b-migrate-1 Creating + Container verify-lab5-23afcf3c2b-s3-1 Created + Container verify-lab5-23afcf3c2b-s3-init-1 Creating + Container verify-lab5-23afcf3c2b-s3-init-1 Created + Container verify-lab5-23afcf3c2b-migrate-1 Created + Container verify-lab5-23afcf3c2b-dispatcher-1 Creating + Container verify-lab5-23afcf3c2b-worker-1 Creating + Container verify-lab5-23afcf3c2b-api-2 Creating + Container verify-lab5-23afcf3c2b-api-1 Creating + Container verify-lab5-23afcf3c2b-housekeeping-1 Creating + Container verify-lab5-23afcf3c2b-dispatcher-1 Created + Container verify-lab5-23afcf3c2b-api-1 Created + Container verify-lab5-23afcf3c2b-worker-1 Created + Container verify-lab5-23afcf3c2b-housekeeping-1 Created + Container verify-lab5-23afcf3c2b-api-2 Created + Container verify-lab5-23afcf3c2b-lb-1 Creating + Container verify-lab5-23afcf3c2b-lb-1 Created + Container verify-lab5-23afcf3c2b-s3-1 Starting + Container verify-lab5-23afcf3c2b-postgres-1 Starting + Container verify-lab5-23afcf3c2b-broker-1 Starting + Container verify-lab5-23afcf3c2b-valkey-1 Starting + Container verify-lab5-23afcf3c2b-valkey-1 Started + Container verify-lab5-23afcf3c2b-postgres-1 Started + Container verify-lab5-23afcf3c2b-postgres-1 Waiting + Container verify-lab5-23afcf3c2b-s3-1 Started + Container verify-lab5-23afcf3c2b-s3-1 Waiting + Container verify-lab5-23afcf3c2b-broker-1 Started + Container verify-lab5-23afcf3c2b-postgres-1 Healthy + Container verify-lab5-23afcf3c2b-migrate-1 Starting + Container verify-lab5-23afcf3c2b-s3-1 Healthy + Container verify-lab5-23afcf3c2b-s3-init-1 Starting + Container verify-lab5-23afcf3c2b-migrate-1 Started + Container verify-lab5-23afcf3c2b-migrate-1 Waiting + Container verify-lab5-23afcf3c2b-s3-init-1 Started + Container verify-lab5-23afcf3c2b-migrate-1 Waiting + Container verify-lab5-23afcf3c2b-s3-init-1 Waiting + Container verify-lab5-23afcf3c2b-migrate-1 Waiting + Container verify-lab5-23afcf3c2b-s3-init-1 Waiting + Container verify-lab5-23afcf3c2b-migrate-1 Waiting + Container verify-lab5-23afcf3c2b-s3-init-1 Waiting + Container verify-lab5-23afcf3c2b-migrate-1 Exited + Container verify-lab5-23afcf3c2b-dispatcher-1 Starting + Container verify-lab5-23afcf3c2b-migrate-1 Exited + Container verify-lab5-23afcf3c2b-migrate-1 Exited + Container verify-lab5-23afcf3c2b-migrate-1 Exited + Container verify-lab5-23afcf3c2b-dispatcher-1 Started + Container verify-lab5-23afcf3c2b-s3-init-1 Exited + Container verify-lab5-23afcf3c2b-api-2 Starting + Container verify-lab5-23afcf3c2b-s3-init-1 Exited + Container verify-lab5-23afcf3c2b-worker-1 Starting + Container verify-lab5-23afcf3c2b-s3-init-1 Exited + Container verify-lab5-23afcf3c2b-housekeeping-1 Starting + Container verify-lab5-23afcf3c2b-worker-1 Started + Container verify-lab5-23afcf3c2b-api-2 Started + Container verify-lab5-23afcf3c2b-api-1 Starting + Container verify-lab5-23afcf3c2b-housekeeping-1 Started + Container verify-lab5-23afcf3c2b-api-1 Started + Container verify-lab5-23afcf3c2b-api-2 Waiting + Container verify-lab5-23afcf3c2b-api-1 Waiting + Container verify-lab5-23afcf3c2b-api-2 Healthy + Container verify-lab5-23afcf3c2b-api-1 Healthy + Container verify-lab5-23afcf3c2b-lb-1 Starting + Container verify-lab5-23afcf3c2b-lb-1 Started + Container verify-lab5-23afcf3c2b-housekeeping-1 Waiting + Container verify-lab5-23afcf3c2b-valkey-1 Waiting + Container verify-lab5-23afcf3c2b-broker-1 Waiting + Container verify-lab5-23afcf3c2b-worker-1 Waiting + Container verify-lab5-23afcf3c2b-lb-1 Waiting + Container verify-lab5-23afcf3c2b-s3-init-1 Waiting + Container verify-lab5-23afcf3c2b-api-2 Waiting + Container verify-lab5-23afcf3c2b-api-1 Waiting + Container verify-lab5-23afcf3c2b-dispatcher-1 Waiting + Container verify-lab5-23afcf3c2b-s3-1 Waiting + Container verify-lab5-23afcf3c2b-postgres-1 Waiting + Container verify-lab5-23afcf3c2b-migrate-1 Waiting + Container verify-lab5-23afcf3c2b-migrate-1 Exited + Container verify-lab5-23afcf3c2b-dispatcher-1 Healthy + Container verify-lab5-23afcf3c2b-housekeeping-1 Healthy + Container verify-lab5-23afcf3c2b-worker-1 Healthy + Container verify-lab5-23afcf3c2b-postgres-1 Healthy + Container verify-lab5-23afcf3c2b-broker-1 Healthy + Container verify-lab5-23afcf3c2b-s3-1 Healthy + Container verify-lab5-23afcf3c2b-s3-init-1 Exited + Container verify-lab5-23afcf3c2b-valkey-1 Healthy + Container verify-lab5-23afcf3c2b-api-2 Healthy + Container verify-lab5-23afcf3c2b-api-1 Healthy + Container verify-lab5-23afcf3c2b-lb-1 Healthy +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +python3 tests/smoke.py --base-url "http://localhost:50588" +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:50588" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + Container verify-lab5-23afcf3c2b-worker-1 Stopping + Container verify-lab5-23afcf3c2b-dispatcher-1 Stopping + Container verify-lab5-23afcf3c2b-housekeeping-1 Stopping + Container verify-lab5-23afcf3c2b-lb-1 Stopping + Container verify-lab5-23afcf3c2b-dispatcher-1 Stopped + Container verify-lab5-23afcf3c2b-dispatcher-1 Removing + Container verify-lab5-23afcf3c2b-housekeeping-1 Stopped + Container verify-lab5-23afcf3c2b-housekeeping-1 Removing + Container verify-lab5-23afcf3c2b-dispatcher-1 Removed + Container verify-lab5-23afcf3c2b-lb-1 Stopped + Container verify-lab5-23afcf3c2b-lb-1 Removing + Container verify-lab5-23afcf3c2b-housekeeping-1 Removed + Container verify-lab5-23afcf3c2b-lb-1 Removed + Container verify-lab5-23afcf3c2b-api-1 Stopping + Container verify-lab5-23afcf3c2b-api-2 Stopping + Container verify-lab5-23afcf3c2b-api-1 Stopped + Container verify-lab5-23afcf3c2b-api-1 Removing + Container verify-lab5-23afcf3c2b-api-2 Stopped + Container verify-lab5-23afcf3c2b-api-2 Removing + Container verify-lab5-23afcf3c2b-api-1 Removed + Container verify-lab5-23afcf3c2b-api-2 Removed + Container verify-lab5-23afcf3c2b-valkey-1 Stopping + Container verify-lab5-23afcf3c2b-valkey-1 Stopped + Container verify-lab5-23afcf3c2b-valkey-1 Removing + Container verify-lab5-23afcf3c2b-valkey-1 Removed + Container verify-lab5-23afcf3c2b-worker-1 Stopped + Container verify-lab5-23afcf3c2b-worker-1 Removing + Container verify-lab5-23afcf3c2b-worker-1 Removed + Container verify-lab5-23afcf3c2b-s3-init-1 Stopping + Container verify-lab5-23afcf3c2b-migrate-1 Stopping + Container verify-lab5-23afcf3c2b-broker-1 Stopping + Container verify-lab5-23afcf3c2b-s3-init-1 Stopped + Container verify-lab5-23afcf3c2b-s3-init-1 Removing + Container verify-lab5-23afcf3c2b-migrate-1 Stopped + Container verify-lab5-23afcf3c2b-migrate-1 Removing + Container verify-lab5-23afcf3c2b-migrate-1 Removed + Container verify-lab5-23afcf3c2b-postgres-1 Stopping + Container verify-lab5-23afcf3c2b-s3-init-1 Removed + Container verify-lab5-23afcf3c2b-s3-1 Stopping + Container verify-lab5-23afcf3c2b-broker-1 Stopped + Container verify-lab5-23afcf3c2b-broker-1 Removing + Container verify-lab5-23afcf3c2b-broker-1 Removed + Container verify-lab5-23afcf3c2b-postgres-1 Stopped + Container verify-lab5-23afcf3c2b-postgres-1 Removing + Container verify-lab5-23afcf3c2b-postgres-1 Removed + Container verify-lab5-23afcf3c2b-s3-1 Stopped + Container verify-lab5-23afcf3c2b-s3-1 Removing + Container verify-lab5-23afcf3c2b-s3-1 Removed + Volume verify-lab5-23afcf3c2b_postgres_data Removing + Volume verify-lab5-23afcf3c2b_broker_data Removing + Volume verify-lab5-23afcf3c2b_s3_data Removing + Network verify-lab5-23afcf3c2b_default Removing + Volume verify-lab5-23afcf3c2b_s3_data Removed + Volume verify-lab5-23afcf3c2b_broker_data Removed + Volume verify-lab5-23afcf3c2b_postgres_data Removed + Network verify-lab5-23afcf3c2b_default Removed +PASS: real clean clone, fresh secrets/volumes, smoke, same-session persistence, actual store/API restart, non-root PID1 diff --git a/evidence/clean-clone.txt b/evidence/clean-clone.txt new file mode 100644 index 00000000..1cee3c03 --- /dev/null +++ b/evidence/clean-clone.txt @@ -0,0 +1,388 @@ +Cloning into '/var/folders/hx/czbzpyk92ds_mnf7n9xrzn_80000gn/T/verify-lab5-d198178375-q0ykkgab/repo'... +python3 scripts/bootstrap.py +Local secrets ready (values hidden); .env and secrets/ are ignored by Git. +docker compose --env-file .env up -d --build --wait +Compose can now delegate builds to bake for better performance. + To do so, set COMPOSE_BAKE=true. +#0 building with "desktop-linux" instance using docker driver + +#1 [migrate internal] load build definition from Dockerfile +#1 transferring dockerfile: 1.11kB done +#1 DONE 0.0s + +#2 [migrate internal] load metadata for docker.io/library/golang:1.24.4-alpine3.22 +#2 DONE 0.0s + +#3 [migrate internal] load metadata for docker.io/library/alpine:3.22.1 +#3 DONE 1.0s + +#4 [migrate internal] load .dockerignore +#4 transferring context: 196B done +#4 DONE 0.0s + +#5 [migrate build 1/7] FROM docker.io/library/golang:1.24.4-alpine3.22@sha256:68932fa6d4d4059845c8f40ad7e654e626f3ebd3706eef7846f319293ab5cb7a +#5 resolve docker.io/library/golang:1.24.4-alpine3.22@sha256:68932fa6d4d4059845c8f40ad7e654e626f3ebd3706eef7846f319293ab5cb7a 0.0s done +#5 DONE 0.0s + +#6 [migrate stage-2 1/4] FROM docker.io/library/alpine:3.22.1@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1 +#6 resolve docker.io/library/alpine:3.22.1@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1 0.0s done +#6 DONE 0.0s + +#7 [migrate internal] load build context +#7 transferring context: 84.12kB done +#7 DONE 0.0s + +#8 [migrate build 4/7] RUN --mount=type=cache,target=/go/pkg/mod GODEBUG=http2client=0 GOPROXY="https://proxy.golang.org,direct" go mod download +#8 CACHED + +#9 [migrate build 2/7] WORKDIR /src +#9 CACHED + +#10 [migrate build 3/7] COPY go.mod go.sum ./ +#10 CACHED + +#11 [migrate build 5/7] COPY cmd/ ./cmd/ +#11 CACHED + +#12 [migrate build 6/7] COPY internal/ ./internal/ +#12 DONE 0.0s + +#13 [migrate build 7/7] RUN --mount=type=cache,target=/go/pkg/mod CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/messenger ./cmd/server +#13 DONE 8.9s + +#14 [migrate stage-2 2/4] RUN apk add --no-cache libwebp-tools +#14 CACHED + +#15 [migrate stage-2 3/4] RUN addgroup -g 10001 app && adduser -D -u 10001 -G app app +#15 CACHED + +#16 [migrate stage-2 4/4] COPY --from=build /out/messenger /usr/local/bin/messenger +#16 DONE 0.0s + +#17 [migrate] exporting to image +#17 exporting layers +#17 exporting layers 0.3s done +#17 exporting manifest sha256:986770eb723cab6acce6f2e251e623d164cbbdceeea86a27d037962cc06d3852 +#17 exporting manifest sha256:986770eb723cab6acce6f2e251e623d164cbbdceeea86a27d037962cc06d3852 done +#17 exporting config sha256:581f4d15c69144d127b2948e0fca217d878c14cb5a6c1325838aad80c15786d2 done +#17 exporting attestation manifest sha256:41144cbf2c345eeb2caa9ec3ca5a9f551c5aa6af1e77059b9d869a78b6d87fc1 done +#17 exporting manifest list sha256:1bb95f6b84d5449b42084c096d487b349b33e3603291b9c0a7120a6c87133742 done +#17 naming to docker.io/library/mtusi-messenger:lab5 done +#17 unpacking to docker.io/library/mtusi-messenger:lab5 0.0s done +#17 DONE 0.4s + +#18 [migrate] resolving provenance for metadata file +#18 DONE 0.0s + +#19 [worker internal] load build definition from Dockerfile +#19 transferring dockerfile: 1.11kB done +#19 DONE 0.0s + +#20 [housekeeping internal] load build definition from Dockerfile +#20 transferring dockerfile: 1.11kB done +#20 DONE 0.0s + +#21 [api internal] load build definition from Dockerfile +#21 transferring dockerfile: 1.11kB done +#21 DONE 0.0s + +#22 [dispatcher internal] load build definition from Dockerfile +#22 transferring dockerfile: 1.11kB done +#22 DONE 0.0s + +#2 [housekeeping internal] load metadata for docker.io/library/golang:1.24.4-alpine3.22 +#2 DONE 0.0s + +#3 [worker internal] load metadata for docker.io/library/alpine:3.22.1 +#3 DONE 1.3s + +#23 [worker internal] load .dockerignore +#23 transferring context: 196B done +#23 DONE 0.0s + +#24 [api internal] load .dockerignore +#24 transferring context: 196B done +#24 DONE 0.0s + +#25 [dispatcher internal] load .dockerignore +#25 transferring context: 196B done +#25 DONE 0.0s + +#26 [housekeeping internal] load .dockerignore +#26 transferring context: 196B done +#26 DONE 0.0s + +#5 [housekeeping build 1/7] FROM docker.io/library/golang:1.24.4-alpine3.22@sha256:68932fa6d4d4059845c8f40ad7e654e626f3ebd3706eef7846f319293ab5cb7a +#5 resolve docker.io/library/golang:1.24.4-alpine3.22@sha256:68932fa6d4d4059845c8f40ad7e654e626f3ebd3706eef7846f319293ab5cb7a 0.0s done +#5 DONE 0.0s + +#6 [dispatcher stage-2 1/4] FROM docker.io/library/alpine:3.22.1@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1 +#6 resolve docker.io/library/alpine:3.22.1@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1 0.0s done +#6 DONE 0.0s + +#27 [housekeeping internal] load build context +#27 transferring context: 1.18kB done +#27 DONE 0.0s + +#28 [dispatcher internal] load build context +#28 transferring context: 84.12kB done +#28 DONE 0.0s + +#29 [housekeeping build 6/7] COPY internal/ ./internal/ +#29 CACHED + +#14 [api stage-2 2/4] RUN apk add --no-cache libwebp-tools +#14 CACHED + +#30 [housekeeping build 4/7] RUN --mount=type=cache,target=/go/pkg/mod GODEBUG=http2client=0 GOPROXY="https://proxy.golang.org,direct" go mod download +#30 CACHED + +#15 [api stage-2 3/4] RUN addgroup -g 10001 app && adduser -D -u 10001 -G app app +#15 CACHED + +#9 [housekeeping build 2/7] WORKDIR /src +#9 CACHED + +#31 [housekeeping build 3/7] COPY go.mod go.sum ./ +#31 CACHED + +#32 [housekeeping build 5/7] COPY cmd/ ./cmd/ +#32 CACHED + +#33 [housekeeping build 7/7] RUN --mount=type=cache,target=/go/pkg/mod CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/messenger ./cmd/server +#33 CACHED + +#34 [api stage-2 4/4] COPY --from=build /out/messenger /usr/local/bin/messenger +#34 CACHED + +#35 [worker internal] load build context +#35 transferring context: 84.12kB done +#35 DONE 0.0s + +#36 [api internal] load build context +#36 transferring context: 84.12kB done +#36 DONE 0.0s + +#37 [worker] exporting to image +#37 exporting layers done +#37 exporting manifest sha256:65b043d697893e885cb5893e3649648ea6e3e2df837b81d46289af134dc4ea38 0.0s done +#37 exporting config sha256:60223db5f7f8b7e8eab631667c8bdc57780b39a928c7cb845686f648922bea2c +#37 exporting config sha256:60223db5f7f8b7e8eab631667c8bdc57780b39a928c7cb845686f648922bea2c 0.0s done +#37 exporting attestation manifest sha256:06f9e3e0b2e5bbd88e7e0d9530225d8fee20a2e3075168c74acb4a7ec1dfa043 0.0s done +#37 exporting manifest list sha256:8d70fe7aacab7f9291ca42907cfacd06e70832a6bb1133300b67adc57785fe36 0.0s done +#37 naming to docker.io/library/mtusi-messenger:lab5 done +#37 unpacking to docker.io/library/mtusi-messenger:lab5 done +#37 DONE 0.1s + +#38 [housekeeping] exporting to image +#38 exporting layers done +#38 exporting manifest sha256:ee117d07acce2e5f35a059da792cfc49e1219c0c7732bb59fc287a8f6956e952 done +#38 exporting config sha256:c0dc986b49c41ed26bdfeea898184813466cb7251b70f74a4c9c2e902dff4d78 0.0s done +#38 exporting attestation manifest sha256:2668225847c35089ecc4ba9de10dc7e64191836b342a695b6d4a259709449c93 0.0s done +#38 exporting manifest list sha256:26f90c53b6f074acfe1955baed5f846db2e3e7741d9848fdd56048b973748f0f 0.0s done +#38 naming to docker.io/library/mtusi-messenger:lab5 done +#38 unpacking to docker.io/library/mtusi-messenger:lab5 0.0s done +#38 DONE 0.1s + +#39 [dispatcher] exporting to image +#39 exporting layers done +#39 exporting manifest sha256:a2c765c3caa01078989e9cd8b3f1df00d605d91737cfca8442d0378fa98d921d 0.0s done +#39 exporting config sha256:c065bd521e9ce4cdc0c5badb10dd54ec1dc8cd2c17a46651c34a6ade00186a85 0.0s done +#39 exporting attestation manifest sha256:0a48ad5a37090e5d3a62c22f3fae56d2cb2dcb9f5c5f587592517bbe34b2f5d9 0.0s done +#39 exporting manifest list sha256:d912a474c14a71a6a647788aec1b47b8320c626cddbed992be8e33ec6f804d30 0.0s done +#39 naming to docker.io/library/mtusi-messenger:lab5 done +#39 unpacking to docker.io/library/mtusi-messenger:lab5 0.0s done +#39 DONE 0.1s + +#40 [api] exporting to image +#40 exporting layers done +#40 exporting manifest sha256:9b7a2eed04ae32f20bf945fea3339e03b38aa77faddf5f49b5e706eadd893918 0.0s done +#40 exporting config sha256:cc422b5d658902cf054a8827f8eb824e8cea22418c8c69ccfd231aba3cd48983 0.0s done +#40 exporting attestation manifest sha256:c051deb5759adead65850caa6654fd72145e785c421017373262d74ba6375d88 0.0s done +#40 exporting manifest list sha256:2e2912d4d5b4607fbf19cfe6f107b30be9420fce4746218ac8718898b66aa643 0.0s done +#40 naming to docker.io/library/mtusi-messenger:lab5 done +#40 unpacking to docker.io/library/mtusi-messenger:lab5 done +#40 DONE 0.1s + +#41 [housekeeping] resolving provenance for metadata file +#41 DONE 0.0s + +#42 [worker] resolving provenance for metadata file +#42 DONE 0.0s + +#43 [dispatcher] resolving provenance for metadata file +#43 DONE 0.0s + +#44 [api] resolving provenance for metadata file +#44 DONE 0.0s + api Built + dispatcher Built + housekeeping Built + migrate Built + worker Built + Network verify-lab5-d198178375_default Creating + Network verify-lab5-d198178375_default Created + Volume "verify-lab5-d198178375_s3_data" Creating + Volume "verify-lab5-d198178375_s3_data" Created + Volume "verify-lab5-d198178375_postgres_data" Creating + Volume "verify-lab5-d198178375_postgres_data" Created + Volume "verify-lab5-d198178375_broker_data" Creating + Volume "verify-lab5-d198178375_broker_data" Created + Container verify-lab5-d198178375-postgres-1 Creating + Container verify-lab5-d198178375-s3-1 Creating + Container verify-lab5-d198178375-broker-1 Creating + Container verify-lab5-d198178375-valkey-1 Creating + Container verify-lab5-d198178375-broker-1 Created + Container verify-lab5-d198178375-valkey-1 Created + Container verify-lab5-d198178375-s3-1 Created + Container verify-lab5-d198178375-s3-init-1 Creating + Container verify-lab5-d198178375-postgres-1 Created + Container verify-lab5-d198178375-migrate-1 Creating + Container verify-lab5-d198178375-s3-init-1 Created + Container verify-lab5-d198178375-migrate-1 Created + Container verify-lab5-d198178375-api-2 Creating + Container verify-lab5-d198178375-api-1 Creating + Container verify-lab5-d198178375-housekeeping-1 Creating + Container verify-lab5-d198178375-worker-1 Creating + Container verify-lab5-d198178375-dispatcher-1 Creating + Container verify-lab5-d198178375-worker-1 Created + Container verify-lab5-d198178375-api-2 Created + Container verify-lab5-d198178375-housekeeping-1 Created + Container verify-lab5-d198178375-dispatcher-1 Created + Container verify-lab5-d198178375-api-1 Created + Container verify-lab5-d198178375-lb-1 Creating + Container verify-lab5-d198178375-lb-1 Created + Container verify-lab5-d198178375-valkey-1 Starting + Container verify-lab5-d198178375-postgres-1 Starting + Container verify-lab5-d198178375-s3-1 Starting + Container verify-lab5-d198178375-broker-1 Starting + Container verify-lab5-d198178375-valkey-1 Started + Container verify-lab5-d198178375-s3-1 Started + Container verify-lab5-d198178375-s3-1 Waiting + Container verify-lab5-d198178375-postgres-1 Started + Container verify-lab5-d198178375-postgres-1 Waiting + Container verify-lab5-d198178375-broker-1 Started + Container verify-lab5-d198178375-s3-1 Healthy + Container verify-lab5-d198178375-s3-init-1 Starting + Container verify-lab5-d198178375-postgres-1 Healthy + Container verify-lab5-d198178375-migrate-1 Starting + Container verify-lab5-d198178375-s3-init-1 Started + Container verify-lab5-d198178375-migrate-1 Started + Container verify-lab5-d198178375-migrate-1 Waiting + Container verify-lab5-d198178375-s3-init-1 Waiting + Container verify-lab5-d198178375-migrate-1 Waiting + Container verify-lab5-d198178375-s3-init-1 Waiting + Container verify-lab5-d198178375-migrate-1 Waiting + Container verify-lab5-d198178375-migrate-1 Waiting + Container verify-lab5-d198178375-s3-init-1 Waiting + Container verify-lab5-d198178375-migrate-1 Exited + Container verify-lab5-d198178375-migrate-1 Exited + Container verify-lab5-d198178375-migrate-1 Exited + Container verify-lab5-d198178375-migrate-1 Exited + Container verify-lab5-d198178375-dispatcher-1 Starting + Container verify-lab5-d198178375-dispatcher-1 Started + Container verify-lab5-d198178375-s3-init-1 Exited + Container verify-lab5-d198178375-housekeeping-1 Starting + Container verify-lab5-d198178375-s3-init-1 Exited + Container verify-lab5-d198178375-api-1 Starting + Container verify-lab5-d198178375-s3-init-1 Exited + Container verify-lab5-d198178375-worker-1 Starting + Container verify-lab5-d198178375-api-1 Started + Container verify-lab5-d198178375-api-2 Starting + Container verify-lab5-d198178375-worker-1 Started + Container verify-lab5-d198178375-housekeeping-1 Started + Container verify-lab5-d198178375-api-2 Started + Container verify-lab5-d198178375-api-1 Waiting + Container verify-lab5-d198178375-api-2 Waiting + Container verify-lab5-d198178375-api-1 Healthy + Container verify-lab5-d198178375-api-2 Healthy + Container verify-lab5-d198178375-lb-1 Starting + Container verify-lab5-d198178375-lb-1 Started + Container verify-lab5-d198178375-housekeeping-1 Waiting + Container verify-lab5-d198178375-worker-1 Waiting + Container verify-lab5-d198178375-s3-init-1 Waiting + Container verify-lab5-d198178375-postgres-1 Waiting + Container verify-lab5-d198178375-s3-1 Waiting + Container verify-lab5-d198178375-dispatcher-1 Waiting + Container verify-lab5-d198178375-lb-1 Waiting + Container verify-lab5-d198178375-broker-1 Waiting + Container verify-lab5-d198178375-valkey-1 Waiting + Container verify-lab5-d198178375-api-1 Waiting + Container verify-lab5-d198178375-api-2 Waiting + Container verify-lab5-d198178375-migrate-1 Waiting + Container verify-lab5-d198178375-migrate-1 Exited + Container verify-lab5-d198178375-valkey-1 Healthy + Container verify-lab5-d198178375-s3-init-1 Exited + Container verify-lab5-d198178375-broker-1 Healthy + Container verify-lab5-d198178375-housekeeping-1 Healthy + Container verify-lab5-d198178375-postgres-1 Healthy + Container verify-lab5-d198178375-dispatcher-1 Healthy + Container verify-lab5-d198178375-s3-1 Healthy + Container verify-lab5-d198178375-worker-1 Healthy + Container verify-lab5-d198178375-api-1 Healthy + Container verify-lab5-d198178375-api-2 Healthy + Container verify-lab5-d198178375-lb-1 Healthy +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +python3 tests/smoke.py --base-url "http://localhost:61225" +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:61225" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + Container verify-lab5-d198178375-housekeeping-1 Stopping + Container verify-lab5-d198178375-worker-1 Stopping + Container verify-lab5-d198178375-dispatcher-1 Stopping + Container verify-lab5-d198178375-lb-1 Stopping + Container verify-lab5-d198178375-housekeeping-1 Stopped + Container verify-lab5-d198178375-housekeeping-1 Removing + Container verify-lab5-d198178375-dispatcher-1 Stopped + Container verify-lab5-d198178375-dispatcher-1 Removing + Container verify-lab5-d198178375-housekeeping-1 Removed + Container verify-lab5-d198178375-lb-1 Stopped + Container verify-lab5-d198178375-lb-1 Removing + Container verify-lab5-d198178375-dispatcher-1 Removed + Container verify-lab5-d198178375-lb-1 Removed + Container verify-lab5-d198178375-api-1 Stopping + Container verify-lab5-d198178375-api-2 Stopping + Container verify-lab5-d198178375-api-2 Stopped + Container verify-lab5-d198178375-api-2 Removing + Container verify-lab5-d198178375-api-1 Stopped + Container verify-lab5-d198178375-api-1 Removing + Container verify-lab5-d198178375-api-2 Removed + Container verify-lab5-d198178375-api-1 Removed + Container verify-lab5-d198178375-valkey-1 Stopping + Container verify-lab5-d198178375-valkey-1 Stopped + Container verify-lab5-d198178375-valkey-1 Removing + Container verify-lab5-d198178375-valkey-1 Removed + Container verify-lab5-d198178375-worker-1 Stopped + Container verify-lab5-d198178375-worker-1 Removing + Container verify-lab5-d198178375-worker-1 Removed + Container verify-lab5-d198178375-migrate-1 Stopping + Container verify-lab5-d198178375-s3-init-1 Stopping + Container verify-lab5-d198178375-broker-1 Stopping + Container verify-lab5-d198178375-migrate-1 Stopped + Container verify-lab5-d198178375-migrate-1 Removing + Container verify-lab5-d198178375-s3-init-1 Stopped + Container verify-lab5-d198178375-s3-init-1 Removing + Container verify-lab5-d198178375-migrate-1 Removed + Container verify-lab5-d198178375-postgres-1 Stopping + Container verify-lab5-d198178375-s3-init-1 Removed + Container verify-lab5-d198178375-s3-1 Stopping + Container verify-lab5-d198178375-postgres-1 Stopped + Container verify-lab5-d198178375-postgres-1 Removing + Container verify-lab5-d198178375-s3-1 Stopped + Container verify-lab5-d198178375-s3-1 Removing + Container verify-lab5-d198178375-postgres-1 Removed + Container verify-lab5-d198178375-broker-1 Stopped + Container verify-lab5-d198178375-broker-1 Removing + Container verify-lab5-d198178375-s3-1 Removed + Container verify-lab5-d198178375-broker-1 Removed + Volume verify-lab5-d198178375_s3_data Removing + Volume verify-lab5-d198178375_broker_data Removing + Volume verify-lab5-d198178375_postgres_data Removing + Network verify-lab5-d198178375_default Removing + Volume verify-lab5-d198178375_postgres_data Removed + Volume verify-lab5-d198178375_s3_data Removed + Volume verify-lab5-d198178375_broker_data Removed + Network verify-lab5-d198178375_default Removed +PASS: real clean clone, fresh secrets/volumes, smoke, same-session persistence, actual store/API restart, non-root PID1 diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..b346d7d6 --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8085" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 131, "failed_attempts": 1, "first_recovery_seconds": 0.83} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/openapi.txt b/evidence/openapi.txt new file mode 100644 index 00000000..85aafc0a --- /dev/null +++ b/evidence/openapi.txt @@ -0,0 +1,4 @@ +/private/tmp/highload-verify-venv/bin/python scripts/validate_openapi.py +/private/tmp/highload-verify-venv/lib/python3.9/site-packages/urllib3/__init__.py:35: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020 + warnings.warn( +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/evidence/operations.json b/evidence/operations.json new file mode 100644 index 00000000..4d2a6c79 --- /dev/null +++ b/evidence/operations.json @@ -0,0 +1,44 @@ +{ + "failures": [ + { + "successful_writes": 270, + "errors": 1, + "successful_rps": 32.656541133158704, + "first_recovery_seconds": 0.693, + "action_seconds": 0.216, + "instances": [ + "7230a8392fd0", + "84ab044ea81c" + ], + "confirmed_preserved": true, + "stopped_alias": "84ab044ea81c" + }, + { + "successful_writes": 274, + "errors": 1, + "successful_rps": 33.094580794100544, + "first_recovery_seconds": 0.689, + "action_seconds": 0.229, + "instances": [ + "7230a8392fd0", + "84ab044ea81c" + ], + "confirmed_preserved": true, + "stopped_alias": "7230a8392fd0" + } + ], + "rolling": { + "successful_writes": 634, + "errors": 3, + "successful_rps": 33.27612468561295, + "first_recovery_seconds": 15.97, + "action_seconds": 15.956, + "instances": [ + "7230a8392fd0", + "73a213d6be45", + "84ab044ea81c", + "f2baf50a05e1" + ], + "confirmed_preserved": true + } +} diff --git a/evidence/operations.txt b/evidence/operations.txt new file mode 100644 index 00000000..9428b287 --- /dev/null +++ b/evidence/operations.txt @@ -0,0 +1,19 @@ + Container mtusi-lab5-kraykin-api-2 Running + Container mtusi-lab5-kraykin-api-3 Creating + Container mtusi-lab5-kraykin-api-3 Created + Container mtusi-lab5-kraykin-api-3 Starting + Container mtusi-lab5-kraykin-api-3 Started + Container mtusi-lab5-kraykin-api-3 Waiting + Container mtusi-lab5-kraykin-api-2 Waiting + Container mtusi-lab5-kraykin-api-3 Healthy + Container mtusi-lab5-kraykin-api-2 Healthy + Container mtusi-lab5-kraykin-api-3 Running + Container mtusi-lab5-kraykin-api-4 Creating + Container mtusi-lab5-kraykin-api-4 Created + Container mtusi-lab5-kraykin-api-4 Starting + Container mtusi-lab5-kraykin-api-4 Started + Container mtusi-lab5-kraykin-api-4 Waiting + Container mtusi-lab5-kraykin-api-3 Waiting + Container mtusi-lab5-kraykin-api-4 Healthy + Container mtusi-lab5-kraykin-api-3 Healthy +PASS: each API stopped and rolling replacement completed; confirmed writes preserved diff --git a/evidence/persistence-processes.json b/evidence/persistence-processes.json new file mode 100644 index 00000000..c730f9f2 --- /dev/null +++ b/evidence/persistence-processes.json @@ -0,0 +1,119 @@ +{ + "scope": "disposable real clean clone", + "before": { + "/verify-lab5-23afcf3c2b-api-1": { + "service": "api", + "started_at": "2026-11-20T21:58:39.719854176Z", + "image_id": "sha256:881c94e3264d9d923d0abc13670e6656d06aec6757ef828178a3a16f90614b10", + "user": "10001:10001", + "pid1": "/usr/local/bin/messenger" + }, + "/verify-lab5-23afcf3c2b-api-2": { + "service": "api", + "started_at": "2026-11-20T21:58:39.629963759Z", + "image_id": "sha256:881c94e3264d9d923d0abc13670e6656d06aec6757ef828178a3a16f90614b10", + "user": "10001:10001", + "pid1": "/usr/local/bin/messenger" + }, + "/verify-lab5-23afcf3c2b-broker-1": { + "service": "broker", + "started_at": "2026-11-20T21:58:36.970109008Z", + "image_id": "sha256:d827e7f7552cdee40cc7482dbae9da020f42bc47669af6f71182a4ef76a22773", + "user": "", + "pid1": "docker-entrypoint.sh" + }, + "/verify-lab5-23afcf3c2b-dispatcher-1": { + "service": "dispatcher", + "started_at": "2026-11-20T21:58:39.121584384Z", + "image_id": "sha256:881c94e3264d9d923d0abc13670e6656d06aec6757ef828178a3a16f90614b10", + "user": "10001:10001", + "pid1": "/usr/local/bin/messenger" + }, + "/verify-lab5-23afcf3c2b-postgres-1": { + "service": "postgres", + "started_at": "2026-11-20T21:58:36.950152716Z", + "image_id": "sha256:ef257d85f76e48da1c64832459b59fcaba1a4dac97bf5d7450c77753542eee94", + "user": "", + "pid1": "docker-entrypoint.sh" + }, + "/verify-lab5-23afcf3c2b-s3-1": { + "service": "s3", + "started_at": "2026-11-20T21:58:36.966163841Z", + "image_id": "sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e", + "user": "", + "pid1": "/usr/bin/docker-entrypoint.sh" + }, + "/verify-lab5-23afcf3c2b-valkey-1": { + "service": "valkey", + "started_at": "2026-11-20T21:58:36.962862758Z", + "image_id": "sha256:d827e7f7552cdee40cc7482dbae9da020f42bc47669af6f71182a4ef76a22773", + "user": "", + "pid1": "docker-entrypoint.sh" + }, + "/verify-lab5-23afcf3c2b-worker-1": { + "service": "worker", + "started_at": "2026-11-20T21:58:39.626740884Z", + "image_id": "sha256:881c94e3264d9d923d0abc13670e6656d06aec6757ef828178a3a16f90614b10", + "user": "10001:10001", + "pid1": "/usr/local/bin/messenger" + } + }, + "after": { + "/verify-lab5-23afcf3c2b-api-1": { + "service": "api", + "started_at": "2026-11-20T21:58:53.67220296Z", + "image_id": "sha256:881c94e3264d9d923d0abc13670e6656d06aec6757ef828178a3a16f90614b10", + "user": "10001:10001", + "pid1": "/usr/local/bin/messenger" + }, + "/verify-lab5-23afcf3c2b-api-2": { + "service": "api", + "started_at": "2026-11-20T21:58:53.592944043Z", + "image_id": "sha256:881c94e3264d9d923d0abc13670e6656d06aec6757ef828178a3a16f90614b10", + "user": "10001:10001", + "pid1": "/usr/local/bin/messenger" + }, + "/verify-lab5-23afcf3c2b-broker-1": { + "service": "broker", + "started_at": "2026-11-20T21:58:51.650668876Z", + "image_id": "sha256:d827e7f7552cdee40cc7482dbae9da020f42bc47669af6f71182a4ef76a22773", + "user": "", + "pid1": "docker-entrypoint.sh" + }, + "/verify-lab5-23afcf3c2b-dispatcher-1": { + "service": "dispatcher", + "started_at": "2026-11-20T21:58:53.594004752Z", + "image_id": "sha256:881c94e3264d9d923d0abc13670e6656d06aec6757ef828178a3a16f90614b10", + "user": "10001:10001", + "pid1": "/usr/local/bin/messenger" + }, + "/verify-lab5-23afcf3c2b-postgres-1": { + "service": "postgres", + "started_at": "2026-11-20T21:58:51.715253168Z", + "image_id": "sha256:ef257d85f76e48da1c64832459b59fcaba1a4dac97bf5d7450c77753542eee94", + "user": "", + "pid1": "docker-entrypoint.sh" + }, + "/verify-lab5-23afcf3c2b-s3-1": { + "service": "s3", + "started_at": "2026-11-20T21:58:51.624425709Z", + "image_id": "sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e", + "user": "", + "pid1": "/usr/bin/docker-entrypoint.sh" + }, + "/verify-lab5-23afcf3c2b-valkey-1": { + "service": "valkey", + "started_at": "2026-11-20T21:58:51.735570043Z", + "image_id": "sha256:d827e7f7552cdee40cc7482dbae9da020f42bc47669af6f71182a4ef76a22773", + "user": "", + "pid1": "docker-entrypoint.sh" + }, + "/verify-lab5-23afcf3c2b-worker-1": { + "service": "worker", + "started_at": "2026-11-20T21:58:53.593897752Z", + "image_id": "sha256:881c94e3264d9d923d0abc13670e6656d06aec6757ef828178a3a16f90614b10", + "user": "10001:10001", + "pid1": "/usr/local/bin/messenger" + } + } +} diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..db0959ba --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,3 @@ +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8085" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/pipeline.json b/evidence/pipeline.json new file mode 100644 index 00000000..d5312210 --- /dev/null +++ b/evidence/pipeline.json @@ -0,0 +1,46 @@ +{ + "method": "real process/container faults; no forged timestamps", + "cases": [ + { + "name": "commit_before_publish_api_broker_restart", + "attachment_id": "c983e599-6680-407b-9b1f-5b61442ca721", + "recovery_seconds": 0.527, + "original_identical": true + }, + { + "name": "crash_before_objects", + "attachment_id": "8ed9b0d7-4020-4413-8d3d-ec7b69e00924", + "status_at_crash": "processing", + "recovery_seconds": 60.062, + "versions": 3 + }, + { + "name": "crash_after_objects", + "attachment_id": "f698946e-65b6-41cf-805a-6f6a3f974c56", + "status_at_crash": "processing", + "recovery_seconds": 59.747, + "versions": 3 + }, + { + "name": "crash_after_commit", + "attachment_id": "6b415290-e617-4db2-aad7-a2f9ca1278b7", + "status_at_crash": "ready", + "recovery_seconds": 0.005, + "versions": 3 + }, + { + "name": "five_duplicate_deliveries", + "unchanged": true + }, + { + "name": "reordered_v1_after_v2", + "metadata_unchanged": true, + "checksums_unchanged": true + }, + { + "name": "failed_registry_explicit_audited_replay", + "error_code": "invalid_image", + "ready": true + } + ] +} diff --git a/evidence/pipeline.txt b/evidence/pipeline.txt new file mode 100644 index 00000000..83c4cb31 --- /dev/null +++ b/evidence/pipeline.txt @@ -0,0 +1,8 @@ +PASS: commit_before_publish_api_broker_restart +PASS: crash_before_objects +PASS: crash_after_objects +PASS: crash_after_commit +PASS: five_duplicate_deliveries +PASS: reordered_v1_after_v2 +PASS: failed_registry_explicit_audited_replay +PASS: versioned at-least-once pipeline invariants; NOT exactly-once delivery diff --git a/evidence/race-current.txt b/evidence/race-current.txt new file mode 100644 index 00000000..fcbb0449 --- /dev/null +++ b/evidence/race-current.txt @@ -0,0 +1,15 @@ +? mtusi/messenger/cmd/server [no test files] +? mtusi/messenger/internal/database [no test files] +=== RUN TestDatabaseConcurrentHTTP + database_test.go:162: 100 confirmed messages from 10 clients through real HTTP and PostgreSQL; race detector enabled +--- PASS: TestDatabaseConcurrentHTTP (1.77s) +=== RUN TestPasswordFullUnicodeRange +--- PASS: TestPasswordFullUnicodeRange (2.31s) +=== RUN TestTransformFailureClassification +--- PASS: TestTransformFailureClassification (0.00s) +=== RUN TestFitFloorNoUpscale +--- PASS: TestFitFloorNoUpscale (0.00s) +=== RUN TestTransformAlphaAndEXIF +--- PASS: TestTransformAlphaAndEXIF (0.25s) +PASS +ok mtusi/messenger/internal/messenger 5.359s diff --git a/evidence/race-final.txt b/evidence/race-final.txt new file mode 100644 index 00000000..6e090174 --- /dev/null +++ b/evidence/race-final.txt @@ -0,0 +1,19 @@ + Volume "mtusi-lab5-kraykin_go_test_cache" Creating + Volume "mtusi-lab5-kraykin_go_test_cache" Created +? mtusi/messenger/cmd/server [no test files] +? mtusi/messenger/internal/database [no test files] +=== RUN TestDatabaseConcurrentHTTP + database_test.go:162: 100 confirmed messages from 10 clients through real HTTP and PostgreSQL; race detector enabled +--- PASS: TestDatabaseConcurrentHTTP (1.78s) +=== RUN TestPasswordFullUnicodeRange +--- PASS: TestPasswordFullUnicodeRange (2.17s) +=== RUN TestTransformFailureClassification +--- PASS: TestTransformFailureClassification (0.00s) +=== RUN TestFitFloorNoUpscale +--- PASS: TestFitFloorNoUpscale (0.00s) +=== RUN TestTransformAlphaAndEXIF +--- PASS: TestTransformAlphaAndEXIF (0.21s) +=== RUN TestPNGEXIFAndMalformedIFD +--- PASS: TestPNGEXIFAndMalformedIFD (0.11s) +PASS +ok mtusi/messenger/internal/messenger 5.283s diff --git a/evidence/race.txt b/evidence/race.txt new file mode 100644 index 00000000..27fbbed9 --- /dev/null +++ b/evidence/race.txt @@ -0,0 +1,178 @@ +docker compose --profile verify run --build --rm tests +Compose can now delegate builds to bake for better performance. + To do so, set COMPOSE_BAKE=true. +#0 building with "desktop-linux" instance using docker driver + +#1 [migrate internal] load build definition from Dockerfile +#1 transferring dockerfile: 1.11kB done +#1 DONE 0.0s + +#2 [migrate internal] load metadata for docker.io/library/golang:1.24.4-alpine3.22 +#2 DONE 0.0s + +#3 [migrate internal] load metadata for docker.io/library/alpine:3.22.1 +#3 DONE 0.2s + +#4 [migrate internal] load .dockerignore +#4 transferring context: 196B done +#4 DONE 0.0s + +#5 [migrate stage-2 1/4] FROM docker.io/library/alpine:3.22.1@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1 +#5 resolve docker.io/library/alpine:3.22.1@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1 0.0s done +#5 DONE 0.0s + +#6 [migrate build 1/7] FROM docker.io/library/golang:1.24.4-alpine3.22@sha256:68932fa6d4d4059845c8f40ad7e654e626f3ebd3706eef7846f319293ab5cb7a +#6 resolve docker.io/library/golang:1.24.4-alpine3.22@sha256:68932fa6d4d4059845c8f40ad7e654e626f3ebd3706eef7846f319293ab5cb7a 0.0s done +#6 DONE 0.0s + +#7 [migrate internal] load build context +#7 transferring context: 1.25kB done +#7 DONE 0.0s + +#8 [migrate build 5/7] COPY cmd/ ./cmd/ +#8 CACHED + +#9 [migrate build 7/7] RUN --mount=type=cache,target=/go/pkg/mod CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/messenger ./cmd/server +#9 CACHED + +#10 [migrate build 4/7] RUN --mount=type=cache,target=/go/pkg/mod GODEBUG=http2client=0 GOPROXY="https://proxy.golang.org,direct" go mod download +#10 CACHED + +#11 [migrate stage-2 2/4] RUN apk add --no-cache libwebp-tools +#11 CACHED + +#12 [migrate build 3/7] COPY go.mod go.sum ./ +#12 CACHED + +#13 [migrate build 6/7] COPY internal/ ./internal/ +#13 CACHED + +#14 [migrate build 2/7] WORKDIR /src +#14 CACHED + +#15 [migrate stage-2 3/4] RUN addgroup -g 10001 app && adduser -D -u 10001 -G app app +#15 CACHED + +#16 [migrate stage-2 4/4] COPY --from=build /out/messenger /usr/local/bin/messenger +#16 CACHED + +#17 [migrate] exporting to image +#17 exporting layers done +#17 exporting manifest sha256:6b25d5f929665fec358139c521f8f1f068c1e27e16793588a46da6d6ff3e2f50 done +#17 exporting config sha256:18eeea0513813c6c8c8fa55c46714c2de772d841c886a798f1d44f6b5940e97a done +#17 exporting attestation manifest sha256:509e9ee6aeb2babc7c413c126b2bad6f9315007f7650fe6046c88a5852fc7e4a done +#17 exporting manifest list sha256:d21bed98319478b55240a42810b6b44f752bb2a01a38980ddcaad6d7a55b7e29 done +#17 naming to docker.io/library/mtusi-messenger:lab5 done +#17 unpacking to docker.io/library/mtusi-messenger:lab5 done +#17 DONE 0.0s + +#18 [migrate] resolving provenance for metadata file +#18 DONE 0.0s + migrate Built + Container mtusi-lab5-kraykin-postgres-1 Running + Container mtusi-lab5-kraykin-migrate-1 Recreate + Container mtusi-lab5-kraykin-migrate-1 Recreated + Container mtusi-lab5-kraykin-postgres-1 Waiting + Container mtusi-lab5-kraykin-postgres-1 Healthy + Container mtusi-lab5-kraykin-migrate-1 Starting + Container mtusi-lab5-kraykin-migrate-1 Started +#0 building with "desktop-linux" instance using docker driver + +#1 [tests internal] load build definition from Dockerfile +#1 transferring dockerfile: 1.11kB done +#1 DONE 0.0s + +#2 [tests internal] load metadata for docker.io/library/golang:1.24.4-alpine3.22 +#2 DONE 0.0s + +#3 [tests internal] load .dockerignore +#3 transferring context: 196B done +#3 DONE 0.0s + +#4 [tests build 1/7] FROM docker.io/library/golang:1.24.4-alpine3.22@sha256:68932fa6d4d4059845c8f40ad7e654e626f3ebd3706eef7846f319293ab5cb7a +#4 resolve docker.io/library/golang:1.24.4-alpine3.22@sha256:68932fa6d4d4059845c8f40ad7e654e626f3ebd3706eef7846f319293ab5cb7a done +#4 DONE 0.0s + +#5 [tests internal] load build context +#5 transferring context: 1.25kB done +#5 DONE 0.0s + +#6 [tests build 2/7] WORKDIR /src +#6 CACHED + +#7 [tests build 4/7] RUN --mount=type=cache,target=/go/pkg/mod GODEBUG=http2client=0 GOPROXY="https://proxy.golang.org,direct" go mod download +#7 CACHED + +#8 [tests build 6/7] COPY internal/ ./internal/ +#8 CACHED + +#9 [tests build 3/7] COPY go.mod go.sum ./ +#9 CACHED + +#10 [tests build 5/7] COPY cmd/ ./cmd/ +#10 CACHED + +#11 [tests build 7/7] RUN --mount=type=cache,target=/go/pkg/mod CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/messenger ./cmd/server +#11 CACHED + +#12 [tests tests 1/2] RUN apk add --no-cache gcc musl-dev libwebp-tools +#12 0.120 fetch https://dl-cdn.alpinelinux.org/alpine/v3.22/main/aarch64/APKINDEX.tar.gz +#12 0.856 fetch https://dl-cdn.alpinelinux.org/alpine/v3.22/community/aarch64/APKINDEX.tar.gz +#12 4.725 (1/22) Upgrading musl (1.2.5-r10 -> 1.2.5-r13) +#12 7.461 (2/22) Installing libgcc (14.2.0-r6) +#12 7.833 (3/22) Installing jansson (2.14.1-r0) +#12 8.102 (4/22) Installing libstdc++ (14.2.0-r6) +#12 12.96 (5/22) Installing zstd-libs (1.5.7-r0) +#12 15.02 (6/22) Installing binutils (2.44-r3) +#12 38.37 (7/22) Installing libgomp (14.2.0-r6) +#12 39.39 (8/22) Installing libatomic (14.2.0-r6) +#12 39.56 (9/22) Installing gmp (6.3.0-r3) +#12 41.05 (10/22) Installing isl26 (0.26-r1) +#12 47.17 (11/22) Installing mpfr4 (4.2.1_p1-r0) +#12 48.99 (12/22) Installing mpc1 (1.3.1-r1) +#12 49.27 (13/22) Installing gcc (14.2.0-r6) +#12 549.7 (14/22) Installing giflib (5.2.2-r2) +#12 549.8 (15/22) Installing libjpeg-turbo (3.1.0-r0) +#12 551.1 (16/22) Installing libpng (1.6.59-r0) +#12 551.6 (17/22) Installing libsharpyuv (1.5.0-r0) +#12 551.7 (18/22) Installing libwebp (1.5.0-r0) +#12 552.7 (19/22) Installing libwebpdemux (1.5.0-r0) +#12 552.8 (20/22) Installing libwebpmux (1.5.0-r0) +#12 553.0 (21/22) Installing libwebp-tools (1.5.0-r0) +#12 553.6 (22/22) Installing musl-dev (1.2.5-r13) +#12 566.9 Executing busybox-1.37.0-r18.trigger +#12 566.9 OK: 165 MiB in 38 packages +#12 DONE 567.0s + +#13 [tests tests 2/2] RUN --mount=type=cache,target=/go/pkg/mod mkdir -p /opt/gomod && cp -a /go/pkg/mod/. /opt/gomod/ +#13 DONE 1.6s + +#14 [tests] exporting to image +#14 exporting layers +#14 exporting layers 5.2s done +#14 exporting manifest sha256:a5c10d2aa2acc1aa68b58ccfaedf17a809a4f8fa4110e38942ece466352e58b4 done +#14 exporting config sha256:be884776181b311e560acc8aec000173c5624f9d519c4f00c8f85fc1b772e223 done +#14 exporting attestation manifest sha256:319b3e1cb5b467ca3f623126ca4de35d2d352bf4913ee20aa56a053708f026b3 0.0s done +#14 exporting manifest list sha256:32e3bd7b054feff9a5ceb8976172b23607d1f9158d9bb8d175dad01124ed7bfe done +#14 naming to docker.io/library/mtusi-messenger-tests:lab5 done +#14 unpacking to docker.io/library/mtusi-messenger-tests:lab5 +#14 unpacking to docker.io/library/mtusi-messenger-tests:lab5 2.0s done +#14 DONE 7.2s + +#15 [tests] resolving provenance for metadata file +#15 DONE 0.0s +? mtusi/messenger/cmd/server [no test files] +? mtusi/messenger/internal/database [no test files] +=== RUN TestDatabaseConcurrentHTTP + database_test.go:162: 100 confirmed messages from 10 clients through real HTTP and PostgreSQL; race detector enabled +--- PASS: TestDatabaseConcurrentHTTP (1.75s) +=== RUN TestPasswordFullUnicodeRange +--- PASS: TestPasswordFullUnicodeRange (2.12s) +=== RUN TestTransformFailureClassification +--- PASS: TestTransformFailureClassification (0.00s) +=== RUN TestFitFloorNoUpscale +--- PASS: TestFitFloorNoUpscale (0.00s) +=== RUN TestTransformAlphaAndEXIF +--- PASS: TestTransformAlphaAndEXIF (0.25s) +PASS +ok mtusi/messenger/internal/messenger 5.161s diff --git a/evidence/retry-backpressure.json b/evidence/retry-backpressure.json new file mode 100644 index 00000000..4017c8ea --- /dev/null +++ b/evidence/retry-backpressure.json @@ -0,0 +1,68 @@ +{ + "sigkill_after_commit_before_publish": { + "ready": true, + "seconds": 0.01 + }, + "retry_exhaustion": { + "attempts": 5, + "error_code": "retry_exhausted", + "seconds": 33.17, + "audit": [ + { + "event": "claimed", + "attempt": 1, + "created_at": "2026-11-20T18:31:10.961859+00:00" + }, + { + "event": "processing_error", + "attempt": 1, + "created_at": "2026-11-20T18:31:11.208323+00:00" + }, + { + "event": "claimed", + "attempt": 2, + "created_at": "2026-11-20T18:31:13.828296+00:00" + }, + { + "event": "processing_error", + "attempt": 2, + "created_at": "2026-11-20T18:31:13.878503+00:00" + }, + { + "event": "claimed", + "attempt": 3, + "created_at": "2026-11-20T18:31:18.352409+00:00" + }, + { + "event": "processing_error", + "attempt": 3, + "created_at": "2026-11-20T18:31:18.379055+00:00" + }, + { + "event": "claimed", + "attempt": 4, + "created_at": "2026-11-20T18:31:26.897115+00:00" + }, + { + "event": "processing_error", + "attempt": 4, + "created_at": "2026-11-20T18:31:26.946941+00:00" + }, + { + "event": "claimed", + "attempt": 5, + "created_at": "2026-11-20T18:31:43.49762+00:00" + }, + { + "event": "processing_error", + "attempt": 5, + "created_at": "2026-11-20T18:31:43.600166+00:00" + } + ] + }, + "admission_bound": { + "unfinished_cap": 1000, + "response": 429, + "confirmed_work_retained": true + } +} diff --git a/evidence/retry-backpressure.txt b/evidence/retry-backpressure.txt new file mode 100644 index 00000000..5521da4f --- /dev/null +++ b/evidence/retry-backpressure.txt @@ -0,0 +1 @@ +PASS: SIGKILL, finite durable retry/backoff/replay, atomic admission limit diff --git a/evidence/runtime.json b/evidence/runtime.json new file mode 100644 index 00000000..37ee3872 --- /dev/null +++ b/evidence/runtime.json @@ -0,0 +1,229 @@ +{ + "platform": "macOS-26.6.2-arm64-arm-64bit", + "containers": [ + { + "service": "api", + "name": "/mtusi-lab5-kraykin-api-3", + "alias": "9fcdf8ad0603", + "image": "mtusi-messenger:lab5", + "image_id": "sha256:c99e8f8faa79ad809301fef274802cbaac38e84e89bf91f8eaba8250edeae5ea", + "user": "10001:10001", + "pid1_path": "/usr/local/bin/messenger", + "started_at": "2026-11-20T19:45:06.310439134Z", + "memory_limit_bytes": 805306368, + "nano_cpus_limit": 1000000000, + "published_ports": { + "8080/tcp": null + }, + "health": "healthy", + "mount_targets": [ + "/run/secrets/s3_secret_access_key", + "/run/secrets/database_url", + "/run/secrets/cache_url", + "/run/secrets/cursor_key", + "/run/secrets/s3_access_key_id" + ] + }, + { + "service": "api", + "name": "/mtusi-lab5-kraykin-api-4", + "alias": "f33a60f7d635", + "image": "mtusi-messenger:lab5", + "image_id": "sha256:c99e8f8faa79ad809301fef274802cbaac38e84e89bf91f8eaba8250edeae5ea", + "user": "10001:10001", + "pid1_path": "/usr/local/bin/messenger", + "started_at": "2026-11-20T19:45:06.377709092Z", + "memory_limit_bytes": 805306368, + "nano_cpus_limit": 1000000000, + "published_ports": { + "8080/tcp": null + }, + "health": "healthy", + "mount_targets": [ + "/run/secrets/database_url", + "/run/secrets/cache_url", + "/run/secrets/cursor_key", + "/run/secrets/s3_access_key_id", + "/run/secrets/s3_secret_access_key" + ] + }, + { + "service": "broker", + "name": "/mtusi-lab5-kraykin-broker-1", + "alias": "147a5622a3a4", + "image": "valkey/valkey:8.1.3-alpine", + "image_id": "sha256:d827e7f7552cdee40cc7482dbae9da020f42bc47669af6f71182a4ef76a22773", + "user": "", + "pid1_path": "docker-entrypoint.sh", + "started_at": "2026-11-20T18:31:53.673378586Z", + "memory_limit_bytes": 268435456, + "nano_cpus_limit": 1000000000, + "published_ports": { + "6379/tcp": null + }, + "health": null, + "mount_targets": [ + "/data", + "/run/secrets/broker_config", + "/run/secrets/broker_url" + ] + }, + { + "service": "dispatcher", + "name": "/mtusi-lab5-kraykin-dispatcher-1", + "alias": "143bd2091931", + "image": "mtusi-messenger:lab5", + "image_id": "sha256:c99e8f8faa79ad809301fef274802cbaac38e84e89bf91f8eaba8250edeae5ea", + "user": "10001:10001", + "pid1_path": "/usr/local/bin/messenger", + "started_at": "2026-11-20T19:45:06.247870508Z", + "memory_limit_bytes": 134217728, + "nano_cpus_limit": 250000000, + "published_ports": { + "8080/tcp": null + }, + "health": "healthy", + "mount_targets": [ + "/run/secrets/database_url", + "/run/secrets/broker_url", + "/run/secrets/s3_access_key_id", + "/run/secrets/s3_secret_access_key" + ] + }, + { + "service": "housekeeping", + "name": "/mtusi-lab5-kraykin-housekeeping-1", + "alias": "5aaadf8c620e", + "image": "mtusi-messenger:lab5", + "image_id": "sha256:c99e8f8faa79ad809301fef274802cbaac38e84e89bf91f8eaba8250edeae5ea", + "user": "10001:10001", + "pid1_path": "/usr/local/bin/messenger", + "started_at": "2026-11-20T19:45:06.305480592Z", + "memory_limit_bytes": 134217728, + "nano_cpus_limit": 250000000, + "published_ports": { + "8080/tcp": null + }, + "health": "healthy", + "mount_targets": [ + "/run/secrets/database_url", + "/run/secrets/broker_url", + "/run/secrets/s3_access_key_id", + "/run/secrets/s3_secret_access_key" + ] + }, + { + "service": "lb", + "name": "/mtusi-lab5-kraykin-lb-1", + "alias": "b10d5a67cf79", + "image": "haproxy:3.2.6-alpine", + "image_id": "sha256:3a819392f5a6af2a9203d1c63ff0feba74f0c940623a558511be06d7d85e361c", + "user": "haproxy", + "pid1_path": "docker-entrypoint.sh", + "started_at": "2026-11-20T18:32:01.235997048Z", + "memory_limit_bytes": 134217728, + "nano_cpus_limit": 1000000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8085" + } + ] + }, + "health": "healthy", + "mount_targets": [ + "/usr/local/etc/haproxy/haproxy.cfg" + ] + }, + { + "service": "postgres", + "name": "/mtusi-lab5-kraykin-postgres-1", + "alias": "0337c63e2fbd", + "image": "postgres:17.6-alpine", + "image_id": "sha256:ef257d85f76e48da1c64832459b59fcaba1a4dac97bf5d7450c77753542eee94", + "user": "", + "pid1_path": "docker-entrypoint.sh", + "started_at": "2026-11-20T18:31:53.655077795Z", + "memory_limit_bytes": 536870912, + "nano_cpus_limit": 2000000000, + "published_ports": { + "5432/tcp": null + }, + "health": "healthy", + "mount_targets": [ + "/var/lib/postgresql/data", + "/run/secrets/postgres_password" + ] + }, + { + "service": "s3", + "name": "/mtusi-lab5-kraykin-s3-1", + "alias": "ff1758db2605", + "image": "quay.io/minio/minio:RELEASE.2025-09-07T16-13-09Z", + "image_id": "sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e", + "user": "", + "pid1_path": "/usr/bin/docker-entrypoint.sh", + "started_at": "2026-11-20T18:31:53.624743169Z", + "memory_limit_bytes": 536870912, + "nano_cpus_limit": 1000000000, + "published_ports": { + "9000/tcp": null + }, + "health": "healthy", + "mount_targets": [ + "/run/secrets/s3_root_user", + "/run/secrets/s3_root_password", + "/data" + ] + }, + { + "service": "valkey", + "name": "/mtusi-lab5-kraykin-valkey-1", + "alias": "bd4927d3dff5", + "image": "valkey/valkey:8.1.3-alpine", + "image_id": "sha256:d827e7f7552cdee40cc7482dbae9da020f42bc47669af6f71182a4ef76a22773", + "user": "", + "pid1_path": "docker-entrypoint.sh", + "started_at": "2026-11-20T18:31:53.710446336Z", + "memory_limit_bytes": 134217728, + "nano_cpus_limit": 500000000, + "published_ports": { + "6379/tcp": null + }, + "health": null, + "mount_targets": [ + "/run/secrets/valkey_config", + "/data" + ] + }, + { + "service": "worker", + "name": "/mtusi-lab5-kraykin-worker-1", + "alias": "e045cc8a7994", + "image": "mtusi-messenger:lab5", + "image_id": "sha256:c99e8f8faa79ad809301fef274802cbaac38e84e89bf91f8eaba8250edeae5ea", + "user": "10001:10001", + "pid1_path": "/usr/local/bin/messenger", + "started_at": "2026-11-20T19:45:06.306100384Z", + "memory_limit_bytes": 1073741824, + "nano_cpus_limit": 1000000000, + "published_ports": { + "8080/tcp": null + }, + "health": "healthy", + "mount_targets": [ + "/run/secrets/s3_secret_access_key", + "/run/secrets/database_url", + "/run/secrets/broker_url", + "/run/secrets/s3_access_key_id" + ] + } + ], + "postgres_durability": [ + "fsync=on", + "full_page_writes=on", + "max_connections=100", + "synchronous_commit=on" + ] +} diff --git a/evidence/smoke-final.txt b/evidence/smoke-final.txt new file mode 100644 index 00000000..2448312f --- /dev/null +++ b/evidence/smoke-final.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://localhost:8085" +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/startup-final.txt b/evidence/startup-final.txt new file mode 100644 index 00000000..cf1e5b94 --- /dev/null +++ b/evidence/startup-final.txt @@ -0,0 +1,304 @@ +Compose can now delegate builds to bake for better performance. + To do so, set COMPOSE_BAKE=true. +#0 building with "desktop-linux" instance using docker driver + +#1 [migrate internal] load build definition from Dockerfile +#1 transferring dockerfile: 1.11kB done +#1 DONE 0.0s + +#2 [migrate internal] load metadata for docker.io/library/golang:1.24.4-alpine3.22 +#2 DONE 0.0s + +#3 [migrate internal] load metadata for docker.io/library/alpine:3.22.1 +#3 DONE 0.3s + +#4 [migrate stage-2 1/4] FROM docker.io/library/alpine:3.22.1@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1 +#4 resolve docker.io/library/alpine:3.22.1@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1 done +#4 DONE 0.0s + +#5 [migrate internal] load .dockerignore +#5 transferring context: 196B done +#5 DONE 0.0s + +#6 [migrate internal] load build context +#6 DONE 0.0s + +#7 [migrate build 1/7] FROM docker.io/library/golang:1.24.4-alpine3.22@sha256:68932fa6d4d4059845c8f40ad7e654e626f3ebd3706eef7846f319293ab5cb7a +#7 resolve docker.io/library/golang:1.24.4-alpine3.22@sha256:68932fa6d4d4059845c8f40ad7e654e626f3ebd3706eef7846f319293ab5cb7a +#7 resolve docker.io/library/golang:1.24.4-alpine3.22@sha256:68932fa6d4d4059845c8f40ad7e654e626f3ebd3706eef7846f319293ab5cb7a 2.1s done +#7 DONE 2.1s + +#6 [migrate internal] load build context +#6 transferring context: 9.92kB done +#6 DONE 0.0s + +#8 [migrate build 3/7] COPY go.mod go.sum ./ +#8 CACHED + +#9 [migrate build 4/7] RUN --mount=type=cache,target=/go/pkg/mod GODEBUG=http2client=0 GOPROXY="https://proxy.golang.org,direct" go mod download +#9 CACHED + +#10 [migrate build 2/7] WORKDIR /src +#10 CACHED + +#11 [migrate build 5/7] COPY cmd/ ./cmd/ +#11 CACHED + +#12 [migrate build 6/7] COPY internal/ ./internal/ +#12 DONE 0.0s + +#13 [migrate build 7/7] RUN --mount=type=cache,target=/go/pkg/mod CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/messenger ./cmd/server +#13 DONE 11.2s + +#14 [migrate stage-2 2/4] RUN apk add --no-cache libwebp-tools +#14 CACHED + +#15 [migrate stage-2 3/4] RUN addgroup -g 10001 app && adduser -D -u 10001 -G app app +#15 CACHED + +#16 [migrate stage-2 4/4] COPY --from=build /out/messenger /usr/local/bin/messenger +#16 DONE 0.0s + +#17 [migrate] exporting to image +#17 exporting layers +#17 exporting layers 0.3s done +#17 exporting manifest sha256:74ca64f8f1ccbda510c4342ca0cb5801d0410c98c98dc583e9edab6a4fa354f2 +#17 exporting manifest sha256:74ca64f8f1ccbda510c4342ca0cb5801d0410c98c98dc583e9edab6a4fa354f2 done +#17 exporting config sha256:aaa14b42aada277f395995e7435bc78509022d32cf71468d26a3aa4d42742a7e done +#17 exporting attestation manifest sha256:d17fbf66047537fbcab6a862bba0ce258878535660b34ff1f2778c4d691500e6 0.0s done +#17 exporting manifest list sha256:f82726c3bf2e84b42cf10844dedf3e32ec03646486e571a35f9405b984c3c6fc done +#17 naming to docker.io/library/mtusi-messenger:lab5 done +#17 unpacking to docker.io/library/mtusi-messenger:lab5 0.0s done +#17 DONE 0.4s + +#18 [migrate] resolving provenance for metadata file +#18 DONE 0.0s + +#19 [worker internal] load build definition from Dockerfile +#19 transferring dockerfile: 1.11kB done +#19 DONE 0.0s + +#20 [api internal] load build definition from Dockerfile +#20 transferring dockerfile: 1.11kB done +#20 DONE 0.0s + +#21 [housekeeping internal] load build definition from Dockerfile +#21 transferring dockerfile: 1.11kB done +#21 DONE 0.0s + +#22 [dispatcher internal] load build definition from Dockerfile +#22 transferring dockerfile: 1.11kB done +#22 DONE 0.0s + +#3 [dispatcher internal] load metadata for docker.io/library/alpine:3.22.1 +#3 ... + +#2 [housekeeping internal] load metadata for docker.io/library/golang:1.24.4-alpine3.22 +#2 DONE 0.2s + +#3 [dispatcher internal] load metadata for docker.io/library/alpine:3.22.1 +#3 DONE 1.3s + +#23 [api internal] load .dockerignore +#23 transferring context: 196B done +#23 DONE 0.0s + +#24 [housekeeping internal] load .dockerignore +#24 transferring context: 196B done +#24 DONE 0.0s + +#25 [dispatcher internal] load .dockerignore +#25 transferring context: 196B done +#25 DONE 0.0s + +#26 [worker internal] load .dockerignore +#26 transferring context: 196B done +#26 DONE 0.0s + +#4 [api stage-2 1/4] FROM docker.io/library/alpine:3.22.1@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1 +#4 resolve docker.io/library/alpine:3.22.1@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1 0.0s done +#4 DONE 0.0s + +#7 [housekeeping build 1/7] FROM docker.io/library/golang:1.24.4-alpine3.22@sha256:68932fa6d4d4059845c8f40ad7e654e626f3ebd3706eef7846f319293ab5cb7a +#7 resolve docker.io/library/golang:1.24.4-alpine3.22@sha256:68932fa6d4d4059845c8f40ad7e654e626f3ebd3706eef7846f319293ab5cb7a 0.0s done +#7 DONE 2.1s + +#27 [api internal] load build context +#27 transferring context: 1.30kB done +#27 DONE 0.0s + +#28 [housekeeping internal] load build context +#28 transferring context: 9.92kB done +#28 DONE 0.0s + +#29 [worker internal] load build context +#29 transferring context: 9.92kB done +#29 DONE 0.0s + +#30 [dispatcher internal] load build context +#30 transferring context: 9.92kB done +#30 DONE 0.0s + +#31 [api build 5/7] COPY cmd/ ./cmd/ +#31 CACHED + +#32 [api build 3/7] COPY go.mod go.sum ./ +#32 CACHED + +#33 [api build 7/7] RUN --mount=type=cache,target=/go/pkg/mod CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/messenger ./cmd/server +#33 CACHED + +#34 [api build 6/7] COPY internal/ ./internal/ +#34 CACHED + +#10 [api build 2/7] WORKDIR /src +#10 CACHED + +#35 [api build 4/7] RUN --mount=type=cache,target=/go/pkg/mod GODEBUG=http2client=0 GOPROXY="https://proxy.golang.org,direct" go mod download +#35 CACHED + +#14 [dispatcher stage-2 2/4] RUN apk add --no-cache libwebp-tools +#14 CACHED + +#15 [dispatcher stage-2 3/4] RUN addgroup -g 10001 app && adduser -D -u 10001 -G app app +#15 CACHED + +#36 [dispatcher stage-2 4/4] COPY --from=build /out/messenger /usr/local/bin/messenger +#36 CACHED + +#37 [dispatcher] exporting to image +#37 exporting layers done +#37 exporting manifest sha256:37933fdac941597c967c906c4061dd8b0a8ce7b4669b2e4e2f666426545e4208 +#37 exporting manifest sha256:37933fdac941597c967c906c4061dd8b0a8ce7b4669b2e4e2f666426545e4208 0.0s done +#37 exporting config sha256:000cacf928e2b797dbb0661bfa4b1754e572779e5a4ebb9d4fd97459eee00ab8 0.0s done +#37 exporting attestation manifest sha256:eb03c27127b3592cd8ac241f7683c8ef0cfc631c9f8c01697948f064b143d073 +#37 exporting attestation manifest sha256:eb03c27127b3592cd8ac241f7683c8ef0cfc631c9f8c01697948f064b143d073 0.0s done +#37 exporting manifest list sha256:c99e8f8faa79ad809301fef274802cbaac38e84e89bf91f8eaba8250edeae5ea 0.0s done +#37 naming to docker.io/library/mtusi-messenger:lab5 done +#37 unpacking to docker.io/library/mtusi-messenger:lab5 done +#37 DONE 0.2s + +#38 [api] exporting to image +#38 exporting layers done +#38 exporting manifest sha256:6e36459b898b2aaaa269e9c92fc7990ba23deff59948882cc514cc236bc03984 done +#38 exporting config sha256:30310d416e83728d91b84a11461f00160e8401c4bf4593db2a1e0312975629e3 0.0s done +#38 exporting attestation manifest sha256:598190941da6fa0ffa671ea41fa58b20cea39672b6834e6f50ca92df38d728e8 0.0s done +#38 exporting manifest list sha256:61c0c89cb293272cb9a336ca4a8299e3aeb50e7ccf9f9519844e62f86107a3ee 0.0s done +#38 naming to docker.io/library/mtusi-messenger:lab5 done +#38 unpacking to docker.io/library/mtusi-messenger:lab5 0.0s done +#38 DONE 0.2s + +#39 [worker] exporting to image +#39 exporting layers done +#39 exporting manifest sha256:0f34703e5afafe40d98d3eab1c01b118399d8035639353de9b571b24ab612686 0.0s done +#39 exporting config sha256:d18c634d53fa1f0e8b3ef8daa5589a07ffdd1dbe35ab1f0e987aa256b566f80f 0.0s done +#39 exporting attestation manifest sha256:d0cbca3816d565931f3b73163ffb1be9fb9f06d69205348be518b637cf4cc503 0.0s done +#39 exporting manifest list sha256:c7312468c1c3e38a5699dae3bed92d992c388402193ae23b315f190f394c78a6 0.0s done +#39 naming to docker.io/library/mtusi-messenger:lab5 done +#39 unpacking to docker.io/library/mtusi-messenger:lab5 done +#39 DONE 0.2s + +#40 [housekeeping] exporting to image +#40 exporting layers done +#40 exporting manifest sha256:637d1992903591dbed41af65263b71974ded77af9aad1bee49ca2d5ec1c7039d 0.0s done +#40 exporting config sha256:c15291f20d0ece057c8c5be93cec6105efd3c44f7a20fdb05618f3e22f251370 0.0s done +#40 exporting attestation manifest sha256:7bad7626422d7f8f97fec248243e9a380754b78eeece259af5a15f325da256df 0.0s done +#40 exporting manifest list sha256:24fc59d5434ab9de6b347648b1f536b799572e83d5b7857494ae356d0b4554e0 0.0s done +#40 naming to docker.io/library/mtusi-messenger:lab5 done +#40 unpacking to docker.io/library/mtusi-messenger:lab5 done +#40 DONE 0.2s + +#41 [api] resolving provenance for metadata file +#41 DONE 0.1s + +#42 [housekeeping] resolving provenance for metadata file +#42 DONE 0.0s + +#43 [dispatcher] resolving provenance for metadata file +#43 DONE 0.1s + +#44 [worker] resolving provenance for metadata file +#44 DONE 0.0s + api Built + dispatcher Built + housekeeping Built + migrate Built + worker Built + Container mtusi-lab5-kraykin-broker-1 Running + Container mtusi-lab5-kraykin-postgres-1 Running + Container mtusi-lab5-kraykin-s3-1 Running + Container mtusi-lab5-kraykin-valkey-1 Running + Container mtusi-lab5-kraykin-migrate-1 Recreate + Container mtusi-lab5-kraykin-s3-init-1 Created + Container mtusi-lab5-kraykin-migrate-1 Recreated + Container mtusi-lab5-kraykin-worker-1 Recreate + Container mtusi-lab5-kraykin-housekeeping-1 Recreate + Container mtusi-lab5-kraykin-dispatcher-1 Recreate + Container mtusi-lab5-kraykin-api-4 Recreate + Container mtusi-lab5-kraykin-api-3 Recreate + Container mtusi-lab5-kraykin-dispatcher-1 Recreated + Container mtusi-lab5-kraykin-housekeeping-1 Recreated + Container mtusi-lab5-kraykin-api-3 Recreated + Container mtusi-lab5-kraykin-api-4 Recreated + Container mtusi-lab5-kraykin-lb-1 Running + Container mtusi-lab5-kraykin-worker-1 Recreated + Container mtusi-lab5-kraykin-postgres-1 Waiting + Container mtusi-lab5-kraykin-s3-1 Waiting + Container mtusi-lab5-kraykin-postgres-1 Healthy + Container mtusi-lab5-kraykin-migrate-1 Starting + Container mtusi-lab5-kraykin-s3-1 Healthy + Container mtusi-lab5-kraykin-s3-init-1 Starting + Container mtusi-lab5-kraykin-migrate-1 Started + Container mtusi-lab5-kraykin-migrate-1 Waiting + Container mtusi-lab5-kraykin-s3-init-1 Started + Container mtusi-lab5-kraykin-migrate-1 Waiting + Container mtusi-lab5-kraykin-s3-init-1 Waiting + Container mtusi-lab5-kraykin-migrate-1 Waiting + Container mtusi-lab5-kraykin-s3-init-1 Waiting + Container mtusi-lab5-kraykin-migrate-1 Waiting + Container mtusi-lab5-kraykin-s3-init-1 Waiting + Container mtusi-lab5-kraykin-migrate-1 Exited + Container mtusi-lab5-kraykin-dispatcher-1 Starting + Container mtusi-lab5-kraykin-migrate-1 Exited + Container mtusi-lab5-kraykin-migrate-1 Exited + Container mtusi-lab5-kraykin-migrate-1 Exited + Container mtusi-lab5-kraykin-s3-init-1 Exited + Container mtusi-lab5-kraykin-s3-init-1 Exited + Container mtusi-lab5-kraykin-api-3 Starting + Container mtusi-lab5-kraykin-worker-1 Starting + Container mtusi-lab5-kraykin-s3-init-1 Exited + Container mtusi-lab5-kraykin-housekeeping-1 Starting + Container mtusi-lab5-kraykin-dispatcher-1 Started + Container mtusi-lab5-kraykin-api-3 Started + Container mtusi-lab5-kraykin-api-4 Starting + Container mtusi-lab5-kraykin-housekeeping-1 Started + Container mtusi-lab5-kraykin-worker-1 Started + Container mtusi-lab5-kraykin-api-4 Started + Container mtusi-lab5-kraykin-api-3 Waiting + Container mtusi-lab5-kraykin-api-4 Waiting + Container mtusi-lab5-kraykin-api-3 Healthy + Container mtusi-lab5-kraykin-api-4 Healthy + Container mtusi-lab5-kraykin-broker-1 Waiting + Container mtusi-lab5-kraykin-postgres-1 Waiting + Container mtusi-lab5-kraykin-migrate-1 Waiting + Container mtusi-lab5-kraykin-housekeeping-1 Waiting + Container mtusi-lab5-kraykin-lb-1 Waiting + Container mtusi-lab5-kraykin-s3-1 Waiting + Container mtusi-lab5-kraykin-valkey-1 Waiting + Container mtusi-lab5-kraykin-dispatcher-1 Waiting + Container mtusi-lab5-kraykin-s3-init-1 Waiting + Container mtusi-lab5-kraykin-worker-1 Waiting + Container mtusi-lab5-kraykin-api-3 Waiting + Container mtusi-lab5-kraykin-api-4 Waiting + Container mtusi-lab5-kraykin-postgres-1 Healthy + Container mtusi-lab5-kraykin-broker-1 Healthy + Container mtusi-lab5-kraykin-housekeeping-1 Healthy + Container mtusi-lab5-kraykin-worker-1 Healthy + Container mtusi-lab5-kraykin-s3-init-1 Exited + Container mtusi-lab5-kraykin-lb-1 Healthy + Container mtusi-lab5-kraykin-migrate-1 Exited + Container mtusi-lab5-kraykin-dispatcher-1 Healthy + Container mtusi-lab5-kraykin-s3-1 Healthy + Container mtusi-lab5-kraykin-valkey-1 Healthy + Container mtusi-lab5-kraykin-api-3 Healthy + Container mtusi-lab5-kraykin-api-4 Healthy diff --git a/evidence/test-build.txt b/evidence/test-build.txt new file mode 100644 index 00000000..ff49d56e --- /dev/null +++ b/evidence/test-build.txt @@ -0,0 +1,88 @@ +Compose can now delegate builds to bake for better performance. + To do so, set COMPOSE_BAKE=true. +#0 building with "desktop-linux" instance using docker driver + +#1 [tests internal] load build definition from Dockerfile +#1 transferring dockerfile: 1.11kB done +#1 DONE 0.0s + +#2 [tests internal] load metadata for docker.io/library/golang:1.24.4-alpine3.22 +#2 DONE 0.0s + +#3 [tests internal] load .dockerignore +#3 transferring context: 196B done +#3 DONE 0.0s + +#4 [tests build 1/7] FROM docker.io/library/golang:1.24.4-alpine3.22@sha256:68932fa6d4d4059845c8f40ad7e654e626f3ebd3706eef7846f319293ab5cb7a +#4 resolve docker.io/library/golang:1.24.4-alpine3.22@sha256:68932fa6d4d4059845c8f40ad7e654e626f3ebd3706eef7846f319293ab5cb7a done +#4 DONE 0.0s + +#5 [tests internal] load build context +#5 transferring context: 1.24kB done +#5 DONE 0.0s + +#6 [tests build 2/7] WORKDIR /src +#6 CACHED + +#7 [tests build 4/7] RUN --mount=type=cache,target=/go/pkg/mod GODEBUG=http2client=0 GOPROXY="https://proxy.golang.org,direct" go mod download +#7 CACHED + +#8 [tests build 3/7] COPY go.mod go.sum ./ +#8 CACHED + +#9 [tests build 5/7] COPY cmd/ ./cmd/ +#9 CACHED + +#10 [tests build 6/7] COPY internal/ ./internal/ +#10 CACHED + +#11 [tests build 7/7] RUN --mount=type=cache,target=/go/pkg/mod CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/messenger ./cmd/server +#11 CACHED + +#12 [tests tests 1/2] RUN apk add --no-cache gcc musl-dev libwebp-tools +#12 0.111 fetch https://dl-cdn.alpinelinux.org/alpine/v3.22/main/aarch64/APKINDEX.tar.gz +#12 1.698 fetch https://dl-cdn.alpinelinux.org/alpine/v3.22/community/aarch64/APKINDEX.tar.gz +#12 6.015 (1/22) Upgrading musl (1.2.5-r10 -> 1.2.5-r13) +#12 6.994 (2/22) Installing libgcc (14.2.0-r6) +#12 7.617 (3/22) Installing jansson (2.14.1-r0) +#12 7.826 (4/22) Installing libstdc++ (14.2.0-r6) +#12 10.08 (5/22) Installing zstd-libs (1.5.7-r0) +#12 10.97 (6/22) Installing binutils (2.44-r3) +#12 17.96 (7/22) Installing libgomp (14.2.0-r6) +#12 18.37 (8/22) Installing libatomic (14.2.0-r6) +#12 18.57 (9/22) Installing gmp (6.3.0-r3) +#12 19.11 (10/22) Installing isl26 (0.26-r1) +#12 21.30 (11/22) Installing mpfr4 (4.2.1_p1-r0) +#12 22.01 (12/22) Installing mpc1 (1.3.1-r1) +#12 22.22 (13/22) Installing gcc (14.2.0-r6) +#12 131.5 (14/22) Installing giflib (5.2.2-r2) +#12 131.6 (15/22) Installing libjpeg-turbo (3.1.0-r0) +#12 132.3 (16/22) Installing libpng (1.6.59-r0) +#12 132.6 (17/22) Installing libsharpyuv (1.5.0-r0) +#12 132.7 (18/22) Installing libwebp (1.5.0-r0) +#12 133.3 (19/22) Installing libwebpdemux (1.5.0-r0) +#12 133.4 (20/22) Installing libwebpmux (1.5.0-r0) +#12 133.6 (21/22) Installing libwebp-tools (1.5.0-r0) +#12 134.3 (22/22) Installing musl-dev (1.2.5-r13) +#12 140.2 Executing busybox-1.37.0-r18.trigger +#12 140.2 OK: 165 MiB in 38 packages +#12 DONE 140.3s + +#13 [tests tests 2/2] RUN --mount=type=cache,target=/go/pkg/mod mkdir -p /opt/gomod && cp -a /go/pkg/mod/. /opt/gomod/ +#13 DONE 1.2s + +#14 [tests] exporting to image +#14 exporting layers +#14 exporting layers 5.1s done +#14 exporting manifest sha256:53c1c6be53ade8066c0d1baf29593f7b90b08b479a2495c099d792aac6689789 0.0s done +#14 exporting config sha256:a667c8e68024534c218d3edb4abe296b8cbc465df73dda11e38df1a74b0ac256 0.0s done +#14 exporting attestation manifest sha256:b3fd5d58c8b0990e9f0066d025f74bfe3e1f8d68d0168b7d5794a1ec093de251 0.0s done +#14 exporting manifest list sha256:67a1f68e1f56e244f1a9d6617392e0e33b69f5e8ed52a6dd0d3abcdd82261106 done +#14 naming to docker.io/library/mtusi-messenger-tests:lab5 done +#14 unpacking to docker.io/library/mtusi-messenger-tests:lab5 +#14 unpacking to docker.io/library/mtusi-messenger-tests:lab5 2.8s done +#14 DONE 8.0s + +#15 [tests] resolving provenance for metadata file +#15 DONE 0.0s + tests Built diff --git a/evidence/version-fence.json b/evidence/version-fence.json new file mode 100644 index 00000000..16cbcfe3 --- /dev/null +++ b/evidence/version-fence.json @@ -0,0 +1,14 @@ +{ + "case": "live stale v1 resumed S3 writes and SQL finalization after v2 ready", + "attachment_id": "5c747cbd-99ce-4d84-b6fb-ceb6a2f7dd7a", + "new_version": 2, + "v2_recovery_seconds": 0.262, + "metadata_unchanged": true, + "sha256_unchanged": true, + "unfinished_jobs": 0, + "variant_checksums": { + "thumbnail": "8d418e9298b45bc2ce1251edfda4cce6546220c81456917616ff8fce3fce44e6", + "crop": "25ae4d14d3c5da1ec20d41e6b9b9fd38e57e8840423d07ef361441da48801c5c", + "optimized": "b0021a56abe58c6346f8fcae3a3e9820fa230e40286de35b1c50127d8f04e49c" + } +} diff --git a/evidence/version-fence.txt b/evidence/version-fence.txt new file mode 100644 index 00000000..b6472248 --- /dev/null +++ b/evidence/version-fence.txt @@ -0,0 +1 @@ +PASS: live stale v1 cannot overwrite v2 objects or fenced SQL result diff --git a/evidence/vet.txt b/evidence/vet.txt new file mode 100644 index 00000000..e69de29b diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..eb24e5e8 --- /dev/null +++ b/go.mod @@ -0,0 +1,35 @@ +module mtusi/messenger + +go 1.24.0 + +require ( + github.com/disintegration/imaging v1.6.2 + github.com/jackc/pgx/v5 v5.7.6 + github.com/minio/minio-go/v7 v7.0.95 + github.com/redis/go-redis/v9 v9.12.1 + golang.org/x/crypto v0.41.0 +) + +require ( + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/minio/crc64nvme v1.0.2 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/tinylib/msgp v1.3.0 // indirect + golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 // indirect + golang.org/x/net v0.42.0 // indirect + golang.org/x/sync v0.16.0 // indirect + golang.org/x/sys v0.35.0 // indirect + golang.org/x/text v0.28.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..847f2a88 --- /dev/null +++ b/go.sum @@ -0,0 +1,72 @@ +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= +github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= +github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/minio/crc64nvme v1.0.2 h1:6uO1UxGAD+kwqWWp7mBFsi5gAse66C4NXO8cmcVculg= +github.com/minio/crc64nvme v1.0.2/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.95 h1:ywOUPg+PebTMTzn9VDsoFJy32ZuARN9zhB+K3IYEvYU= +github.com/minio/minio-go/v7 v7.0.95/go.mod h1:wOOX3uxS334vImCNRVyIDdXX9OsXDm89ToynKgqUKlo= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/redis/go-redis/v9 v9.12.1 h1:k5iquqv27aBtnTm2tIkROUDp8JBXhXZIVu1InSgvovg= +github.com/redis/go-redis/v9 v9.12.1/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww= +github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= +golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= +golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= +golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 h1:hVwzHzIUGRjiF7EcUjqNxk3NCfkPxbDKRdnNE1Rpg0U= +golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= +golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..4b980c4e --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,27 @@ +global + maxconn 512 + log stdout format raw local0 +defaults + log global + mode http + timeout connect 500ms + timeout client 35s + timeout server 35s + timeout check 800ms + retries 0 +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 2 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold obsolete 1s +frontend public + bind :8080 + default_backend replicas +backend replicas + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + default-server inter 1s fall 2 rise 1 + server-template api 4 api:8080 check resolvers docker init-addr libc,none resolve-prefer ipv4 diff --git a/infra/s3-init.sh b/infra/s3-init.sh new file mode 100644 index 00000000..6df43857 --- /dev/null +++ b/infra/s3-init.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -eu +# Do not enable shell tracing: mc arguments contain local credentials. +mc alias set local http://s3:9000 "$(cat /run/secrets/s3_root_user)" "$(cat /run/secrets/s3_root_password)" >/dev/null +mc mb --ignore-existing local/messenger-attachments >/dev/null +mc anonymous set none local/messenger-attachments >/dev/null +mc admin policy create local messenger-app /config/s3-policy.json >/dev/null +mc admin user add local "$(cat /run/secrets/s3_access_key_id)" "$(cat /run/secrets/s3_secret_access_key)" >/dev/null +mc admin policy attach local messenger-app --user "$(cat /run/secrets/s3_access_key_id)" >/dev/null +printf '%s\n' 'Private bucket and restricted application user ready' diff --git a/infra/s3-policy.json b/infra/s3-policy.json new file mode 100644 index 00000000..b1802e9c --- /dev/null +++ b/infra/s3-policy.json @@ -0,0 +1 @@ +{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:ListBucket","s3:GetBucketLocation"],"Resource":["arn:aws:s3:::messenger-attachments"]},{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject","s3:DeleteObject"],"Resource":["arn:aws:s3:::messenger-attachments/originals/*","arn:aws:s3:::messenger-attachments/staging/*"]}]} diff --git a/internal/database/database.go b/internal/database/database.go new file mode 100644 index 00000000..55be2d35 --- /dev/null +++ b/internal/database/database.go @@ -0,0 +1,103 @@ +package database + +import ( + "context" + "crypto/sha256" + "embed" + "fmt" + "github.com/jackc/pgx/v5/pgxpool" + "os" + "sort" + "strconv" + "time" +) + +//go:embed migrations/*.sql +var migrations embed.FS + +func Open(ctx context.Context, url string) (*pgxpool.Pool, error) { + cfg, err := pgxpool.ParseConfig(url) + if err != nil { + return nil, fmt.Errorf("invalid database configuration") + } + cfg.MaxConns = 10 + if v := os.Getenv("DB_POOL_MAX"); v != "" { + n, e := strconv.Atoi(v) + if e != nil || n < 1 || n > 64 { + return nil, fmt.Errorf("DB_POOL_MAX must be 1..64") + } + cfg.MaxConns = int32(n) + } + cfg.MinConns = 0 + cfg.MaxConnLifetime = 30 * time.Minute + cfg.MaxConnIdleTime = 5 * time.Minute + cfg.ConnConfig.ConnectTimeout = 2 * time.Second + cfg.ConnConfig.RuntimeParams["statement_timeout"] = "2000" + cfg.ConnConfig.RuntimeParams["lock_timeout"] = "1000" + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + return nil, fmt.Errorf("cannot create database pool") + } + if err = pool.Ping(ctx); err != nil { + pool.Close() + return nil, fmt.Errorf("database unavailable") + } + return pool, nil +} + +// All migrations and the version ledger commit together. An advisory lock also +// serializes bootstrap across API instances; edited applied migrations fail. +func Migrate(ctx context.Context, pool *pgxpool.Pool) error { + tx, err := pool.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(context.Background()) + if _, err = tx.Exec(ctx, "SELECT pg_advisory_xact_lock(2026002)"); err != nil { + return err + } + if _, err = tx.Exec(ctx, `CREATE TABLE IF NOT EXISTS schema_migrations(version text PRIMARY KEY, checksum text NOT NULL, applied_at timestamptz NOT NULL DEFAULT clock_timestamp())`); err != nil { + return err + } + entries, err := migrations.ReadDir("migrations") + if err != nil { + return err + } + sort.Slice(entries, func(i, j int) bool { return entries[i].Name() < entries[j].Name() }) + for _, entry := range entries { + body, err := migrations.ReadFile("migrations/" + entry.Name()) + if err != nil { + return err + } + checksum := fmt.Sprintf("%x", sha256.Sum256(body)) + var stored string + rows, err := tx.Query(ctx, "SELECT checksum FROM schema_migrations WHERE version=$1", entry.Name()) + if err != nil { + return err + } + found := rows.Next() + if found { + err = rows.Scan(&stored) + } + if err == nil { + err = rows.Err() + } + rows.Close() + if err != nil { + return err + } + if found { + if stored != checksum { + return fmt.Errorf("applied migration changed: %s", entry.Name()) + } + continue + } + if _, err = tx.Exec(ctx, string(body)); err != nil { + return err + } + if _, err = tx.Exec(ctx, "INSERT INTO schema_migrations(version,checksum) VALUES($1,$2)", entry.Name(), checksum); err != nil { + return err + } + } + return tx.Commit(ctx) +} diff --git a/internal/database/migrations/001_initial.sql b/internal/database/migrations/001_initial.sql new file mode 100644 index 00000000..005476d2 --- /dev/null +++ b/internal/database/migrations/001_initial.sql @@ -0,0 +1,36 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + last_message_at timestamptz NOT NULL DEFAULT '-infinity' +); +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id), + user_id uuid NOT NULL REFERENCES users(id), + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX chat_members_user_idx ON chat_members(user_id, chat_id); +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + sender_id uuid NOT NULL, + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL, + FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id) +); +CREATE INDEX messages_history_idx ON messages(chat_id, created_at, id); +CREATE INDEX chats_order_idx ON chats(created_at, id); +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY CHECK (octet_length(token_hash) = 32), + user_id uuid NOT NULL REFERENCES users(id), + expires_at timestamptz NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX sessions_expiry_idx ON sessions(expires_at); diff --git a/internal/database/migrations/002_idempotency.sql b/internal/database/migrations/002_idempotency.sql new file mode 100644 index 00000000..23cfaeb1 --- /dev/null +++ b/internal/database/migrations/002_idempotency.sql @@ -0,0 +1,10 @@ +CREATE TABLE message_requests ( + user_id uuid NOT NULL REFERENCES users(id), + chat_id uuid NOT NULL REFERENCES chats(id), + key text NOT NULL, + body_hash bytea NOT NULL, + response jsonb NOT NULL, + expires_at timestamptz NOT NULL, + PRIMARY KEY(user_id,chat_id,key) +); +CREATE INDEX message_requests_expiry_idx ON message_requests(expires_at); diff --git a/internal/database/migrations/003_attachments.sql b/internal/database/migrations/003_attachments.sql new file mode 100644 index 00000000..f55c2a2f --- /dev/null +++ b/internal/database/migrations/003_attachments.sql @@ -0,0 +1,29 @@ +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ADD CHECK(char_length(text) BETWEEN 0 AND 4000); +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + owner_id uuid NOT NULL REFERENCES users(id), + filename text NOT NULL, + object_key text NOT NULL UNIQUE, + content_type text NOT NULL CHECK(content_type IN ('image/png','image/jpeg')), + size_bytes bigint NOT NULL CHECK(size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL CHECK(sha256 ~ '^[a-f0-9]{64}$'), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages(id), + attachment_id uuid NOT NULL REFERENCES attachments(id), + position integer NOT NULL CHECK(position BETWEEN 0 AND 9), + PRIMARY KEY(message_id,attachment_id), + UNIQUE(message_id,position) +); +-- Durable upload journal is committed BEFORE PUT, so incomplete work remains +-- attributable after process termination. Final metadata + journal deletion +-- share one PG transaction, under the same per-key lock as the reconciler. +CREATE TABLE upload_intents ( + object_key text PRIMARY KEY, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX attachments_chat_idx ON attachments(chat_id); diff --git a/internal/database/migrations/004_pipeline.sql b/internal/database/migrations/004_pipeline.sql new file mode 100644 index 00000000..fa55d1ee --- /dev/null +++ b/internal/database/migrations/004_pipeline.sql @@ -0,0 +1,30 @@ +ALTER TABLE attachments ADD COLUMN status text NOT NULL DEFAULT 'ready' CHECK(status IN ('queued','processing','ready','failed')); +ALTER TABLE attachments ADD COLUMN error_code text; +ALTER TABLE attachments ADD COLUMN variants jsonb NOT NULL DEFAULT '[]'; +ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK(pipeline_version>0); +ALTER TABLE attachments ADD COLUMN staging_key text; +CREATE TABLE pipeline_jobs ( + job_id uuid PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments(id), + pipeline_version integer NOT NULL CHECK(pipeline_version>0), + attempt integer NOT NULL DEFAULT 0, + lease_token uuid, + lease_until timestamptz, + next_attempt_at timestamptz NOT NULL DEFAULT clock_timestamp(), + published_at timestamptz, + finished_at timestamptz, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + traceparent text NOT NULL DEFAULT '', + tracestate text NOT NULL DEFAULT '', + UNIQUE(attachment_id,pipeline_version) +); +CREATE INDEX pipeline_dispatch_idx ON pipeline_jobs(next_attempt_at) WHERE finished_at IS NULL; +CREATE TABLE pipeline_audit ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + job_id uuid NOT NULL REFERENCES pipeline_jobs(job_id), + event text NOT NULL, + attempt integer NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +-- This is the durable, queryable failed registry (DLQ equivalent). +CREATE VIEW failed_jobs AS SELECT j.job_id,j.attachment_id,j.pipeline_version,j.attempt,a.error_code,j.finished_at FROM pipeline_jobs j JOIN attachments a ON a.id=j.attachment_id AND a.pipeline_version=j.pipeline_version WHERE a.status='failed'; diff --git a/internal/messenger/attachments.go b/internal/messenger/attachments.go new file mode 100644 index 00000000..b5042d78 --- /dev/null +++ b/internal/messenger/attachments.go @@ -0,0 +1,313 @@ +package messenger + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "image" + _ "image/jpeg" + _ "image/png" + "io" + "mime" + "net/http" + "net/url" + "os" + "strings" + "time" + "unicode" + "unicode/utf8" + + "github.com/jackc/pgx/v5" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +const maxUpload = 10 << 20 +const maxPixels = 40000000 + +type ObjectStore struct { + Client *minio.Client + Bucket string +} + +func NewObjectStore(endpoint, bucket, access, secret string) (*ObjectStore, error) { + u, e := url.Parse(endpoint) + if e != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") { + return nil, fmt.Errorf("invalid S3 endpoint") + } + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.MaxConnsPerHost = 8 + transport.ResponseHeaderTimeout = 5 * time.Second + client, e := minio.New(u.Host, &minio.Options{Creds: credentials.NewStaticV4(access, secret, ""), Secure: u.Scheme == "https", Region: "us-east-1", Transport: transport, MaxRetries: 1}) + if e != nil { + return nil, fmt.Errorf("invalid S3 configuration") + } + return &ObjectStore{client, bucket}, nil +} +func (o *ObjectStore) Ready(ctx context.Context) bool { + ok, e := o.Client.BucketExists(ctx, o.Bucket) + return e == nil && ok +} +func (s *Server) SetObjectStore(o *ObjectStore) { s.objects = o; s.uploads = make(chan struct{}, 1) } + +type Attachment struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + OwnerID string `json:"owner_id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + Size int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Status string `json:"status"` + Variants []Variant `json:"variants"` + ErrorCode *string `json:"error_code"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` +} + +const attachmentColumns = `a.id::text,a.chat_id::text,a.owner_id::text,a.filename,a.content_type,a.size_bytes,a.sha256,a.created_at,a.updated_at,a.object_key,a.status,a.error_code,a.variants` + +func scanAttachment(row scanner) (Attachment, string, error) { + a := Attachment{Status: "ready", Variants: []Variant{}} + var c, u time.Time + var key string + var variants []byte + e := row.Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ContentType, &a.Size, &a.SHA256, &c, &u, &key, &a.Status, &a.ErrorCode, &variants) + if e == nil { + e = json.Unmarshal(variants, &a.Variants) + } + a.CreatedAt = stamp(c) + a.UpdatedAt = stamp(u) + return a, key, e +} +func validFilename(name string) bool { + if !validText(name, 255) { + return false + } + for _, c := range name { + if c == '/' || c == '\\' || unicode.IsControl(c) { + return false + } + } + return true +} + +// A temporary capped spool avoids holding a compressed file in RAM; it is +// removed on every exit. DecodeConfig bounds pixels before full validation. +func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request, uid, cid string) { + q, e := url.ParseQuery(r.URL.RawQuery) + if e != nil || len(q["filename"]) != 1 || !validFilename(q.Get("filename")) { + bad(w) + return + } + contentType, _, e := mime.ParseMediaType(r.Header.Get("Content-Type")) + if e != nil || (contentType != "image/png" && contentType != "image/jpeg") { + fail(w, 415, "unsupported_media_type", "Expected PNG or JPEG") + return + } + if r.ContentLength > maxUpload { + _, _ = io.Copy(io.Discard, io.LimitReader(r.Body, maxUpload+1)) + fail(w, 413, "payload_too_large", "Maximum upload is 10 MiB") + return + } + select { + case s.uploads <- struct{}{}: + defer func() { <-s.uploads }() + case <-r.Context().Done(): + unavailable(w) + return + } + f, e := os.CreateTemp("", "messenger-upload-*") + if e != nil { + unavailable(w) + return + } + defer os.Remove(f.Name()) + defer f.Close() + hash := sha256.New() + n, e := io.Copy(io.MultiWriter(f, hash), http.MaxBytesReader(w, r.Body, maxUpload)) + if e != nil { + var over *http.MaxBytesError + if errors.As(e, &over) { + fail(w, 413, "payload_too_large", "Maximum upload is 10 MiB") + } else { + bad(w) + } + return + } + _, _ = f.Seek(0, io.SeekStart) + config, format, e := image.DecodeConfig(f) + if e != nil || "image/"+format != contentType { + fail(w, 415, "unsupported_media_type", "Invalid image or MIME mismatch") + return + } + if config.Width < 1 || config.Height < 1 || int64(config.Width)*int64(config.Height) > maxPixels { + fail(w, 413, "payload_too_large", "Maximum image is 40 million pixels") + return + } + _, _ = f.Seek(0, io.SeekStart) + if !validateImage(f) { + fail(w, 415, "unsupported_media_type", "Invalid image") + return + } + _, _ = f.Seek(0, io.SeekStart) + id := uuid() + key := "staging/" + id + ctx := r.Context() + // Journal survives a crash between PUT and metadata COMMIT. + if _, e = s.db.Exec(ctx, "INSERT INTO upload_intents(object_key) VALUES($1)", key); e != nil { + unavailable(w) + return + } + tx, e := s.db.Begin(ctx) + if e != nil { + unavailable(w) + return + } + defer tx.Rollback(context.Background()) + if _, e = tx.Exec(ctx, "SELECT pg_advisory_xact_lock(hashtextextended($1,4))", key); e != nil { + unavailable(w) + return + } + // Disable multipart: the bounded spool is <=10MiB, hence one atomic object PUT. + _, e = s.objects.Client.PutObject(ctx, s.objects.Bucket, key, f, n, minio.PutObjectOptions{ContentType: contentType, DisableMultipart: true}) + if e != nil { + unavailable(w) + return + } + a, _, e := scanAttachment(tx.QueryRow(ctx, `INSERT INTO attachments AS a(id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256,status,staging_key) VALUES($1,$2,$3,$4,$5,$6,$7,$8,'queued',$9) RETURNING `+attachmentColumns, id, cid, uid, q.Get("filename"), "originals/"+id+"/v1/original", contentType, n, hex.EncodeToString(hash.Sum(nil)), key)) + if e != nil { + unavailable(w) + return + } + // Admission and intent share a transaction; no unbounded accepted backlog. + if _, e = tx.Exec(ctx, "SELECT pg_advisory_xact_lock(2026005)"); e != nil { + unavailable(w) + return + } + var pending int + if e = tx.QueryRow(ctx, "SELECT count(*) FROM pipeline_jobs WHERE finished_at IS NULL").Scan(&pending); e != nil { + unavailable(w) + return + } + if pending >= 1000 { + w.Header().Set("Retry-After", "5") + fail(w, 429, "rate_limited", "Processing backlog is full") + return + } + if _, e = tx.Exec(ctx, `INSERT INTO pipeline_jobs(job_id,attachment_id,pipeline_version) VALUES($1,$2,1)`, uuid(), id); e != nil { + unavailable(w) + return + } + if _, e = tx.Exec(ctx, "DELETE FROM upload_intents WHERE object_key=$1", key); e != nil { + unavailable(w) + return + } + if tx.Commit(ctx) != nil { + unavailable(w) + return + } + reply(w, 202, a) +} +func validateImage(r io.Reader) bool { _, _, e := image.Decode(r); return e == nil } + +func (s *Server) serveAttachment(w http.ResponseWriter, r *http.Request, uid string, parts []string) { + if r.Method != "GET" || (len(parts) != 4 && len(parts) != 5) || (len(parts) == 5 && parts[4] != "content") { + missing(w) + return + } + id := strings.ToLower(parts[3]) + if !uuidRE.MatchString(id) { + bad(w) + return + } + a, key, e := scanAttachment(s.db.QueryRow(r.Context(), `SELECT `+attachmentColumns+` FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id AND m.user_id=$2 WHERE a.id=$1`, id, uid)) + if errors.Is(e, pgx.ErrNoRows) { + missing(w) + return + } + if e != nil { + unavailable(w) + return + } + if len(parts) == 4 { + reply(w, 200, a) + return + } + q, e := url.ParseQuery(r.URL.RawQuery) + if e != nil || len(q["variant"]) > 1 { + bad(w) + return + } + if a.Status != "ready" { + fail(w, 409, "conflict", "Attachment is not ready") + return + } + variant := q.Get("variant") + if variant != "" && variant != "original" { + found := false + for _, v := range a.Variants { + if v.Name == variant { + a.ContentType = v.ContentType + a.Size = v.Size + key = strings.TrimSuffix(key, "original") + variant + found = true + break + } + } + if !found { + missing(w) + return + } + } + object, e := s.objects.Client.GetObject(r.Context(), s.objects.Bucket, key, minio.GetObjectOptions{}) + if e != nil { + unavailable(w) + return + } + defer object.Close() + info, e := object.Stat() + if e != nil || info.Size != a.Size { + unavailable(w) + return + } + w.Header().Set("Content-Type", a.ContentType) + w.Header().Set("Content-Length", fmt.Sprint(a.Size)) + w.Header().Set("X-Content-Type-Options", "nosniff") + w.Header().Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": a.Filename})) + w.WriteHeader(200) + _, _ = io.Copy(w, object) +} + +// Normalize optional attachments before hashing; JSON key ordering is irrelevant. +func parseMessage(w http.ResponseWriter, r *http.Request) (string, []string, bool) { + var in struct { + Text *string `json:"text"` + Attachments []string `json:"attachment_ids"` + } + // strict decoder accepts an explicitly optional field (suffix '?'). + if !decode(w, r, &in, "text", "attachment_ids?") { + return "", nil, false + } + if in.Text == nil || !utf8.ValidString(*in.Text) || utf8.RuneCountInString(*in.Text) > 4000 || len(in.Attachments) > 10 || (strings.TrimSpace(*in.Text) == "" && len(in.Attachments) == 0) { + bad(w) + return "", nil, false + } + seen := map[string]bool{} + for i, id := range in.Attachments { + id = strings.ToLower(id) + if !uuidRE.MatchString(id) || seen[id] { + bad(w) + return "", nil, false + } + seen[id] = true + in.Attachments[i] = id + } + if in.Attachments == nil { + in.Attachments = []string{} + } + return *in.Text, in.Attachments, true +} diff --git a/internal/messenger/child_linux.go b/internal/messenger/child_linux.go new file mode 100644 index 00000000..dadf05b0 --- /dev/null +++ b/internal/messenger/child_linux.go @@ -0,0 +1,18 @@ +//go:build linux + +package messenger + +import ( + "os/exec" + "syscall" +) + +func configureChild(cmd *exec.Cmd) { + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + cmd.Cancel = func() error { + if cmd.Process != nil { + return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + } + return nil + } +} diff --git a/internal/messenger/child_other.go b/internal/messenger/child_other.go new file mode 100644 index 00000000..05b2534f --- /dev/null +++ b/internal/messenger/child_other.go @@ -0,0 +1,7 @@ +//go:build !linux + +package messenger + +import "os/exec" + +func configureChild(cmd *exec.Cmd) {} diff --git a/internal/messenger/database_test.go b/internal/messenger/database_test.go new file mode 100644 index 00000000..8806e4d6 --- /dev/null +++ b/internal/messenger/database_test.go @@ -0,0 +1,163 @@ +package messenger + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "mtusi/messenger/internal/database" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync" + "testing" + "time" +) + +// Real PostgreSQL + HTTP + the race detector, in the optional Compose tests +// service. Local go test skips this test unless the DB secret path is supplied. +func TestDatabaseConcurrentHTTP(t *testing.T) { + path := os.Getenv("DATABASE_TEST_URL_FILE") + if path == "" { + t.Skip("run make test-race for real PostgreSQL HTTP integration") + } + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + pool, err := database.Open(ctx, strings.TrimSpace(string(raw))) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err = database.Migrate(ctx, pool); err != nil { + t.Fatal(err) + } + api := New(pool, nil, make([]byte, 32), 3600, nil) + ts := httptest.NewServer(api) + defer ts.Close() + call := func(method, path, body, auth string, status int) map[string]any { + t.Helper() + r, err := http.NewRequest(method, ts.URL+path, strings.NewReader(body)) + if err != nil { + t.Error(err) + return nil + } + if body != "" { + r.Header.Set("Content-Type", "application/json") + } + r.Header.Set("Authorization", auth) + if method == "POST" && strings.HasSuffix(path, "/messages") { + r.Header.Set("Idempotency-Key", uuid()) + } + response, err := ts.Client().Do(r) + if err != nil { + t.Error(err) + return nil + } + defer response.Body.Close() + b, err := io.ReadAll(response.Body) + if err != nil { + t.Error(err) + return nil + } + if response.StatusCode != status { + t.Errorf("%s %s: want %d got %d", method, path, status, response.StatusCode) + return nil + } + var v map[string]any + if err = json.Unmarshal(b, &v); err != nil { + t.Error(err) + } + return v + } + username := "go_" + strings.ReplaceAll(uuid(), "-", "")[:20] + user := call("POST", "/api/v1/users", fmt.Sprintf(`{"username":%q,"display_name":"Race","password":"Synthetic-Go-Test-123"}`, username), "", 201) + if user == nil { + t.Fatal("create failed") + } + basic := "Basic " + base64.StdEncoding.EncodeToString([]byte(username+":Synthetic-Go-Test-123")) + session := call("POST", "/api/v1/auth/sessions", "", basic, 201) + if session == nil { + t.Fatal("login failed") + } + bearer := "Bearer " + session["token"].(string) + chat := call("POST", "/api/v1/chats", fmt.Sprintf(`{"title":"Race","member_ids":[%q]}`, user["id"]), bearer, 201) + if chat == nil { + t.Fatal("chat failed") + } + messages := "/api/v1/chats/" + chat["id"].(string) + "/messages" + var wg sync.WaitGroup + for worker := 0; worker < 10; worker++ { + wg.Add(1) + go func(worker int) { + defer wg.Done() + for j := 0; j < 10; j++ { + call("POST", messages, fmt.Sprintf(`{"text":"worker-%d-message-%d"}`, worker, j), bearer, 201) + } + }(worker) + } + wg.Wait() + page := call("GET", messages+"?limit=100", "", bearer, 200) + if page == nil { + t.Fatal("history failed") + } + items := page["items"].([]any) + if len(items) != 100 { + t.Fatalf("expected 100 messages, got %d", len(items)) + } + seen := map[string]bool{} + for _, v := range items { + m := v.(map[string]any) + id := m["id"].(string) + if seen[id] { + t.Fatal("duplicate ID") + } + seen[id] = true + } + // Race a single key through the real server, then verify only one side effect. + var repeatWG sync.WaitGroup + ids := make(chan string, 12) + for i := 0; i < 12; i++ { + repeatWG.Add(1) + go func() { + defer repeatWG.Done() + req, _ := http.NewRequest("POST", ts.URL+messages, strings.NewReader(`{"text":"one key"}`)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", bearer) + req.Header.Set("Idempotency-Key", "same-concurrent-key") + res, err := ts.Client().Do(req) + if err != nil { + t.Error(err) + return + } + defer res.Body.Close() + var m Message + if res.StatusCode != 201 || json.NewDecoder(res.Body).Decode(&m) != nil { + t.Error("repeat failed") + return + } + ids <- m.ID + }() + } + repeatWG.Wait() + close(ids) + var expected string + for id := range ids { + if expected == "" { + expected = id + } + if id != expected { + t.Error("idempotency duplicate") + } + } + var count int + if err := pool.QueryRow(context.Background(), "SELECT count(*) FROM messages WHERE chat_id=$1", chat["id"]).Scan(&count); err != nil || count != 101 { + t.Errorf("want 101 persisted messages, got %d: %v", count, err) + } + t.Log("100 confirmed messages from 10 clients through real HTTP and PostgreSQL; race detector enabled") +} diff --git a/internal/messenger/operator.go b/internal/messenger/operator.go new file mode 100644 index 00000000..afca64bf --- /dev/null +++ b/internal/messenger/operator.go @@ -0,0 +1,137 @@ +package messenger + +import ( + "context" + "errors" + "fmt" + "github.com/jackc/pgx/v5" + "github.com/minio/minio-go/v7" + "strings" + "time" +) + +// Operator-only CLI. It exposes no unauthenticated HTTP mutation endpoint. +func (p *Pipeline) Replay(ctx context.Context, id string, newVersion bool) error { + if !uuidRE.MatchString(id) { + return fmt.Errorf("expected attachment UUID") + } + tx, e := p.DB.Begin(ctx) + if e != nil { + return e + } + defer tx.Rollback(context.Background()) + var status, key, stage string + var version int + e = tx.QueryRow(ctx, `SELECT status,object_key,COALESCE(staging_key,''),pipeline_version FROM attachments WHERE id=$1 FOR UPDATE`, id).Scan(&status, &key, &stage, &version) + if e != nil { + return e + } + if newVersion { + if status != "ready" && status != "queued" && status != "processing" { + return fmt.Errorf("replay failed before version upgrade") + } + if version >= 2 { + return fmt.Errorf("supported pipeline versions: 1 and 2") + } + version++ + if status == "ready" { + stage = key + } + } else { + if status != "failed" { + return fmt.Errorf("only failed can be replayed") + } + } + if _, e = p.Objects.Client.StatObject(ctx, p.Objects.Bucket, stage, minio.StatObjectOptions{}); e != nil { + return fmt.Errorf("source unavailable; restore before replay") + } + var jobID string + if newVersion { + jobID = uuid() + if _, e = tx.Exec(ctx, "UPDATE pipeline_jobs SET finished_at=clock_timestamp() WHERE attachment_id=$1 AND finished_at IS NULL", id); e != nil { + return e + } + _, e = tx.Exec(ctx, `INSERT INTO pipeline_jobs(job_id,attachment_id,pipeline_version) VALUES($1,$2,$3)`, jobID, id, version) + } else { + e = tx.QueryRow(ctx, `UPDATE pipeline_jobs SET attempt=0,lease_token=NULL,lease_until=NULL,published_at=NULL,finished_at=NULL,next_attempt_at=clock_timestamp() WHERE attachment_id=$1 AND pipeline_version=$2 RETURNING job_id::text`, id, version).Scan(&jobID) + } + if e != nil { + return e + } + _, e = tx.Exec(ctx, `UPDATE attachments SET status='queued',error_code=NULL,variants='[]',pipeline_version=$2,staging_key=$3,updated_at=clock_timestamp() WHERE id=$1`, id, version, stage) + if e != nil { + return e + } + event := "operator_replay" + if newVersion { + event = "operator_new_version" + } + if _, e = tx.Exec(ctx, `INSERT INTO pipeline_audit(job_id,event,attempt) VALUES($1,$2,0)`, jobID, event); e != nil { + return e + } + return tx.Commit(ctx) +} + +// Only staging is collected automatically. Live queued/processing work and final +// objects are always protected. Ready stage is redundant; failed source has 7d +// retention. Unattributed aborted uploads/intents have a conservative 24h grace. +func (p *Pipeline) Cleanup(ctx context.Context, apply bool) error { + for object := range p.Objects.Client.ListObjects(ctx, p.Objects.Bucket, minio.ListObjectsOptions{Prefix: "staging/", Recursive: true}) { + if object.Err != nil { + return object.Err + } + if time.Since(object.LastModified) < 24*time.Hour { + continue + } + tx, e := p.DB.Begin(ctx) + if e != nil { + return e + } + if _, e = tx.Exec(ctx, "SELECT pg_advisory_xact_lock(hashtextextended($1,4))", object.Key); e != nil { + tx.Rollback(ctx) + return e + } + var status string + var updated time.Time + e = tx.QueryRow(ctx, `SELECT status,updated_at FROM attachments WHERE staging_key=$1 FOR UPDATE`, object.Key).Scan(&status, &updated) + if e != nil && !errors.Is(e, pgx.ErrNoRows) { + tx.Rollback(ctx) + return e + } + safe := errors.Is(e, pgx.ErrNoRows) || status == "ready" || (status == "failed" && time.Since(updated) > 7*24*time.Hour) + if safe { + p.Log.Info("staging_cleanup", "event", "staging_cleanup", "apply", apply, "status", status) + if apply { + if e = p.Objects.Client.RemoveObject(ctx, p.Objects.Bucket, object.Key, minio.RemoveObjectOptions{}); e != nil { + tx.Rollback(ctx) + return e + } + if _, e = tx.Exec(ctx, `DELETE FROM upload_intents WHERE object_key=$1`, object.Key); e != nil { + tx.Rollback(ctx) + return e + } + } + } + if e = tx.Commit(ctx); e != nil { + return e + } + } + return nil +} +func (p *Pipeline) RunHousekeeping(ctx context.Context) error { + for ctx.Err() == nil { + if e := p.Cleanup(ctx, true); e != nil { + p.Log.Warn("cleanup_retry", "event", "cleanup_retry") + } + if !wait(ctx, time.Hour) { + break + } + } + return nil +} +func ParseOperatorArgs(args []string) (string, bool, error) { + if len(args) != 1 || strings.TrimSpace(args[0]) == "" { + return "", false, fmt.Errorf("expected one attachment UUID") + } + return args[0], false, nil +} diff --git a/internal/messenger/password.go b/internal/messenger/password.go new file mode 100644 index 00000000..925705d8 --- /dev/null +++ b/internal/messenger/password.go @@ -0,0 +1,33 @@ +package messenger + +import ( + "crypto/rand" + "crypto/subtle" + "encoding/base64" + "golang.org/x/crypto/argon2" + "strings" +) + +// Fixed, bounded parameters: 64 MiB, 3 passes, 2 lanes, 16-byte salt, +// 32-byte result. The HTTP layer bounds simultaneous KDF work to 2 calls. +func hashPassword(password string) string { + salt := make([]byte, 16) + if _, err := rand.Read(salt); err != nil { + panic(err) + } + hash := argon2.IDKey([]byte(password), salt, 3, 64*1024, 2, 32) + return "$argon2id$v=19$m=65536,t=3,p=2$" + base64.RawStdEncoding.EncodeToString(salt) + "$" + base64.RawStdEncoding.EncodeToString(hash) +} +func verifyPassword(password, encoded string) bool { + parts := strings.Split(encoded, "$") + if len(parts) != 6 || strings.Join(parts[:4], "$") != "$argon2id$v=19$m=65536,t=3,p=2" { + return false + } + salt, e1 := base64.RawStdEncoding.DecodeString(parts[4]) + want, e2 := base64.RawStdEncoding.DecodeString(parts[5]) + if e1 != nil || e2 != nil || len(salt) != 16 || len(want) != 32 { + return false + } + got := argon2.IDKey([]byte(password), salt, 3, 64*1024, 2, 32) + return subtle.ConstantTimeCompare(got, want) == 1 +} diff --git a/internal/messenger/password_test.go b/internal/messenger/password_test.go new file mode 100644 index 00000000..a24a668c --- /dev/null +++ b/internal/messenger/password_test.go @@ -0,0 +1,20 @@ +package messenger + +import ( + "strings" + "testing" +) + +func TestPasswordFullUnicodeRange(t *testing.T) { + password := strings.Repeat("🙂", 128) // 512 bytes: exceeds bcrypt's 72-byte limit. + hash := hashPassword(password) + if !verifyPassword(password, hash) || verifyPassword(password+"x", hash) { + t.Fatal("password mismatch") + } + if hash == hashPassword(password) { + t.Fatal("salt reused") + } + if verifyPassword(password, "$argon2id$v=19$m=999999999,t=3,p=2$abc$def") { + t.Fatal("unbounded parameters accepted") + } +} diff --git a/internal/messenger/pipeline.go b/internal/messenger/pipeline.go new file mode 100644 index 00000000..8d45ce4e --- /dev/null +++ b/internal/messenger/pipeline.go @@ -0,0 +1,409 @@ +package messenger + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/minio/minio-go/v7" + "github.com/redis/go-redis/v9" + "io" + "log/slog" + "math/rand/v2" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "sync" + "time" +) + +const stream = "attachment.jobs.v1" +const group = "image-workers" +const maxAttempts = 5 +const leaseSeconds = 45 + +type Job struct { + SchemaVersion int `json:"schema_version"` + ID string `json:"job_id"` + Type string `json:"type"` + AttachmentID string `json:"attachment_id"` + Version int `json:"pipeline_version"` + Attempt int `json:"attempt"` + CreatedAt string `json:"created_at"` + Traceparent string `json:"traceparent,omitempty"` + Tracestate string `json:"tracestate,omitempty"` +} +type Pipeline struct { + DB *pgxpool.Pool + Broker *redis.Client + Objects *ObjectStore + Log *slog.Logger +} + +func NewPipeline(db *pgxpool.Pool, broker *redis.Client, o *ObjectStore, log *slog.Logger) *Pipeline { + return &Pipeline{db, broker, o, log} +} +func wait(ctx context.Context, d time.Duration) bool { + t := time.NewTimer(d) + defer t.Stop() + select { + case <-ctx.Done(): + return false + case <-t.C: + return true + } +} +func (p *Pipeline) RunDispatcher(ctx context.Context) error { + for ctx.Err() == nil { + if e := p.dispatch(ctx); e != nil { + p.Log.Warn("outbox_retry", "event", "outbox_retry") + } + if !wait(ctx, 500*time.Millisecond) { + break + } + } + return nil +} +func (p *Pipeline) dispatch(ctx context.Context) error { + // Small per-row transaction + SKIP LOCKED allows multiple dispatchers. A crash + // after XADD before COMMIT gives a duplicate, never a missing durable intent. + for i := 0; i < 32; i++ { + tx, e := p.DB.Begin(ctx) + if e != nil { + return e + } + var j Job + var created time.Time + e = tx.QueryRow(ctx, `SELECT job_id::text,attachment_id::text,pipeline_version,attempt+1,created_at,traceparent,tracestate FROM pipeline_jobs WHERE finished_at IS NULL AND next_attempt_at<=clock_timestamp() AND (lease_until IS NULL OR lease_until 4 { + return fmt.Errorf("WORKER_CONCURRENCY must be 1..4") + } + for ctx.Err() == nil { + e := p.Broker.XGroupCreateMkStream(ctx, stream, group, "0").Err() + if e == nil || strings.Contains(e.Error(), "BUSYGROUP") { + break + } + if !wait(ctx, time.Second) { + return nil + } + } + var wg sync.WaitGroup + for i := 0; i < concurrency; i++ { + wg.Add(1) + go func() { defer wg.Done(); p.consume(ctx, uuid()) }() + } + wg.Wait() + return nil +} +func (p *Pipeline) consume(ctx context.Context, consumer string) { + cursor := "0-0" + for ctx.Err() == nil { + claimed, next, e := p.Broker.XAutoClaim(ctx, &redis.XAutoClaimArgs{Stream: stream, Group: group, Consumer: consumer, MinIdle: time.Minute, Start: cursor, Count: 1}).Result() + cursor = next + if e == nil && len(claimed) > 0 { + p.handle(ctx, claimed[0]) + continue + } + entries, e := p.Broker.XReadGroup(ctx, &redis.XReadGroupArgs{Group: group, Consumer: consumer, Streams: []string{stream, ">"}, Count: 1, Block: time.Second}).Result() + if e != nil { + if e != redis.Nil { + wait(ctx, time.Second) + } + continue + } + for _, entry := range entries { + for _, m := range entry.Messages { + p.handle(ctx, m) + } + } + } +} +func (p *Pipeline) handle(ctx context.Context, m redis.XMessage) { + raw, _ := m.Values["payload"].(string) + var j Job + d := json.NewDecoder(strings.NewReader(raw)) + d.DisallowUnknownFields() + if len(raw) > 4096 || d.Decode(&j) != nil || j.SchemaVersion != 1 || j.Type != "attachment.process.v1" || !uuidRE.MatchString(j.ID) || !uuidRE.MatchString(j.AttachmentID) || j.Version < 1 || j.Attempt < 1 { + p.Log.Warn("invalid_job", "event", "invalid_job") + p.ack(ctx, m.ID) + return + } + // The authoritative attempt and storage locations come from PostgreSQL, never + // from untrusted broker payload fields. A terminal/stale delivery is a no-op. + done, e := p.process(ctx, j) + if e != nil { + p.Log.Warn("job_deferred", "event", "job_deferred", "job_id", j.ID) + } + if done { + p.ack(ctx, m.ID) + } +} +func (p *Pipeline) ack(ctx context.Context, id string) { + // Do not delete an entry unless ACK succeeded; the only consumer group is ours. + if p.Broker.XAck(ctx, stream, group, id).Err() == nil { + p.Broker.XDel(ctx, stream, id) + } +} +func (p *Pipeline) process(parent context.Context, j Job) (bool, error) { + ctx, cancel := context.WithTimeout(parent, 30*time.Second) + defer cancel() + tx, e := p.DB.Begin(ctx) + if e != nil { + return false, e + } + defer tx.Rollback(context.Background()) + var status, key, sha, contentType string + var version, attempt int + var size int64 + var lease *time.Time + e = tx.QueryRow(ctx, `SELECT a.status,a.pipeline_version,a.staging_key,a.sha256,a.content_type,a.size_bytes,j.attempt,j.lease_until FROM pipeline_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.job_id=$1 AND j.attachment_id=$2 AND j.pipeline_version=$3 FOR UPDATE OF j,a`, j.ID, j.AttachmentID, j.Version).Scan(&status, &version, &key, &sha, &contentType, &size, &attempt, &lease) + if errors.Is(e, pgx.ErrNoRows) { + return true, nil + } + if e != nil { + return false, e + } + if version != j.Version || status == "ready" || status == "failed" { + return true, nil + } + var now time.Time + if e = tx.QueryRow(ctx, "SELECT clock_timestamp()").Scan(&now); e != nil { + return false, e + } + if lease != nil && lease.After(now) { + return true, nil + } // live attempt; DB recovery remains durable + if attempt >= maxAttempts { + _, e = tx.Exec(ctx, `UPDATE attachments SET status='failed',error_code='retry_exhausted',updated_at=clock_timestamp() WHERE id=$1;`, j.AttachmentID) + if e == nil { + _, e = tx.Exec(ctx, "UPDATE pipeline_jobs SET finished_at=clock_timestamp(),lease_until=NULL WHERE job_id=$1", j.ID) + } + if e != nil { + return false, e + } + return true, tx.Commit(ctx) + } + token := uuid() + attempt++ + if _, e = tx.Exec(ctx, `UPDATE pipeline_jobs SET attempt=$2,lease_token=$3,lease_until=clock_timestamp()+interval '45 seconds' WHERE job_id=$1`, j.ID, attempt, token); e != nil { + return false, e + } + if _, e = tx.Exec(ctx, `UPDATE attachments SET status='processing',error_code=NULL,updated_at=clock_timestamp() WHERE id=$1`, j.AttachmentID); e != nil { + return false, e + } + if _, e = tx.Exec(ctx, `INSERT INTO pipeline_audit(job_id,event,attempt) VALUES($1,'claimed',$2)`, j.ID, attempt); e != nil { + return false, e + } + if e = tx.Commit(ctx); e != nil { + return false, e + } + p.Log.Info("job_started", "event", "job_started", "job_id", j.ID, "attempt", attempt, "pipeline_version", j.Version) + faultPoint("before_objects") + variants, processErr := p.perform(ctx, j, key, sha, contentType, size) + if processErr == nil { + faultPoint("after_objects") + } + // A short independent commit budget allows recording a timed-out attempt; + // a cancelled/killed process instead recovers via the persisted lease. + finish, stop := context.WithTimeout(context.Background(), 3*time.Second) + defer stop() + tx, e = p.DB.Begin(finish) + if e != nil { + return false, e + } + defer tx.Rollback(context.Background()) + var currentToken *string + var currentVersion int + e = tx.QueryRow(finish, `SELECT j.lease_token::text,a.pipeline_version FROM pipeline_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.job_id=$1 FOR UPDATE OF j,a`, j.ID).Scan(¤tToken, ¤tVersion) + if e != nil { + return false, e + } + if currentToken == nil || *currentToken != token || currentVersion != j.Version { + return true, nil + } + if processErr == nil { + encoded, _ := json.Marshal(variants) + _, e = tx.Exec(finish, `UPDATE attachments SET status='ready',error_code=NULL,variants=$2,object_key=$3,updated_at=clock_timestamp() WHERE id=$1`, j.AttachmentID, encoded, resultPrefix(j)+"original") + if e == nil { + _, e = tx.Exec(finish, `UPDATE pipeline_jobs SET finished_at=clock_timestamp(),lease_until=NULL WHERE job_id=$1`, j.ID) + } + } else { + permanent := errors.Is(processErr, errInvalidImage) + if permanent || attempt >= maxAttempts { + code := "retry_exhausted" + if permanent { + code = "invalid_image" + } + _, e = tx.Exec(finish, `UPDATE attachments SET status='failed',error_code=$2,updated_at=clock_timestamp() WHERE id=$1`, j.AttachmentID, code) + if e == nil { + _, e = tx.Exec(finish, `UPDATE pipeline_jobs SET finished_at=clock_timestamp(),lease_until=NULL WHERE job_id=$1`, j.ID) + } + } else { + _, e = tx.Exec(finish, `UPDATE attachments SET status='queued',updated_at=clock_timestamp() WHERE id=$1`, j.AttachmentID) + delay := time.Duration(1< maxUpload { + return 1 + } + if string(h[4:]) == "eXIf" { + b := make([]byte, n) + if _, e = io.ReadFull(f, b); e != nil { + return 1 + } + return tiffOrientation(b) + } + if string(h[4:]) == "IEND" { + return 1 + } + if _, e = f.Seek(int64(n)+4, io.SeekCurrent); e != nil { + return 1 + } + } +} +func tiffOrientation(b []byte) int { + if len(b) < 8 { + return 1 + } + var order binary.ByteOrder + switch string(b[:2]) { + case "II": + order = binary.LittleEndian + case "MM": + order = binary.BigEndian + default: + return 1 + } + if order.Uint16(b[2:4]) != 42 { + return 1 + } + offset := uint64(order.Uint32(b[4:8])) + if offset+2 > uint64(len(b)) { + return 1 + } + count := int(order.Uint16(b[offset : offset+2])) + for i := 0; i < count; i++ { + pos := offset + 2 + uint64(i)*12 + if pos+12 > uint64(len(b)) { + return 1 + } + entry := b[pos : pos+12] + if order.Uint16(entry[:2]) == 0x112 && order.Uint16(entry[2:4]) == 3 && order.Uint32(entry[4:8]) == 1 { + v := int(order.Uint16(entry[8:10])) + if v >= 1 && v <= 8 { + return v + } + return 1 + } + } + return 1 +} +func orientPNG(img image.Image, orientation int) image.Image { + switch orientation { + case 2: + return imaging.FlipH(img) + case 3: + return imaging.Rotate180(img) + case 4: + return imaging.FlipV(img) + case 5: + return imaging.Transpose(img) + case 6: + return imaging.Rotate270(img) + case 7: + return imaging.Transverse(img) + case 8: + return imaging.Rotate90(img) + } + return img +} diff --git a/internal/messenger/server.go b/internal/messenger/server.go new file mode 100644 index 00000000..79e56516 --- /dev/null +++ b/internal/messenger/server.go @@ -0,0 +1,787 @@ +// Package messenger implements the lab5 contract. PostgreSQL is authoritative. +package messenger + +import ( + "bytes" + "context" + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/redis/go-redis/v9" + "io" + "log/slog" + "mime" + "net/http" + "net/url" + "os" + "regexp" + "strconv" + "strings" + "sync/atomic" + "time" + "unicode/utf8" +) + +type User struct { + ID string `json:"id"` + Username string `json:"username"` + DisplayName string `json:"display_name"` + CreatedAt string `json:"created_at"` +} +type Chat struct { + ID string `json:"id"` + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` + CreatedAt string `json:"created_at"` +} +type Message struct { + AttachmentIDs []string `json:"attachment_ids"` + ID string `json:"id"` + ChatID string `json:"chat_id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` + CreatedAt string `json:"created_at"` +} +type Page[T any] struct { + Items []T `json:"items"` + NextCursor *string `json:"next_cursor"` +} + +func uuid() string { + b := make([]byte, 16) + if _, err := rand.Read(b); err != nil { + panic(err) + } + b[6] = (b[6] & 15) | 64 + b[8] = (b[8] & 63) | 128 + return fmt.Sprintf("%x-%x-%x-%x-%x", b[:4], b[4:6], b[6:8], b[8:10], b[10:]) +} + +var uuidRE = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`) +var usernameRE = regexp.MustCompile(`^[a-z0-9_]{3,32}$`) + +func validText(v string, max int) bool { + return utf8.ValidString(v) && strings.TrimSpace(v) != "" && utf8.RuneCountInString(v) <= max +} +func reply(w http.ResponseWriter, status int, v any) { + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(v) +} +func fail(w http.ResponseWriter, status int, code, msg string) { + reply(w, status, map[string]any{"error": map[string]string{"code": code, "message": msg, "request_id": w.Header().Get("X-Request-Id")}}) +} +func bad(w http.ResponseWriter) { fail(w, 400, "invalid_request", "Invalid request") } +func missing(w http.ResponseWriter) { fail(w, 404, "not_found", "Object not found") } + +// Decode exact object keys (encoding/json structs alone accept case variants). +// Reject null, duplicate keys, trailing documents and invalid UTF-8 as well. +func decode(w http.ResponseWriter, r *http.Request, dst any, fields ...string) bool { + media, _, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil || media != "application/json" { + fail(w, 415, "unsupported_media_type", "Expected application/json") + return false + } + b, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 1<<20)) + if err != nil { + if _, ok := err.(*http.MaxBytesError); ok { + fail(w, 413, "payload_too_large", "Body exceeds 1 MiB") + } else { + bad(w) + } + return false + } + if !utf8.Valid(b) { + bad(w) + return false + } + d := json.NewDecoder(bytes.NewReader(b)) + token, err := d.Token() + if err != nil || token != json.Delim('{') { + bad(w) + return false + } + allowed := map[string]bool{} + for _, f := range fields { + allowed[strings.TrimSuffix(f, "?")] = true + } + seen := map[string]bool{} + for d.More() { + t, e := d.Token() + k, ok := t.(string) + if e != nil || !ok || !allowed[k] || seen[k] { + bad(w) + return false + } + seen[k] = true + var raw json.RawMessage + if d.Decode(&raw) != nil || bytes.Equal(raw, []byte("null")) { + bad(w) + return false + } + } + if _, err = d.Token(); err != nil { + bad(w) + return false + } + if _, err = d.Token(); err != io.EOF { + bad(w) + return false + } + for _, f := range fields { + if !strings.HasSuffix(f, "?") && !seen[f] { + bad(w) + return false + } + } + if json.Unmarshal(b, dst) != nil { + bad(w) + return false + } + return true +} + +type cursor struct { + Scope string `json:"scope"` + Time string `json:"time"` + ID string `json:"id"` +} + +func (s *Server) encodeCursor(c cursor) *string { + b, _ := json.Marshal(c) + mac := hmac.New(sha256.New, s.secret) + mac.Write(b) + v := base64.RawURLEncoding.EncodeToString(b) + "." + base64.RawURLEncoding.EncodeToString(mac.Sum(nil)) + return &v +} +func (s *Server) pagination(w http.ResponseWriter, r *http.Request, scope string) (int, cursor, bool) { + var c cursor + limit := 50 + // ParseQuery explicitly reports malformed escaping, unlike URL.Query. + values, e := url.ParseQuery(r.URL.RawQuery) + if e != nil { + bad(w) + return 0, c, false + } + if v, ok := values["limit"]; ok { + if len(v) != 1 || v[0] == "" || strings.Trim(v[0], "0123456789") != "" { + bad(w) + return 0, c, false + } + limit, e = strconv.Atoi(v[0]) + if e != nil || limit < 1 || limit > 100 { + bad(w) + return 0, c, false + } + } + if v, ok := values["cursor"]; ok { + if len(v) != 1 || len(v[0]) > 2048 { + bad(w) + return 0, c, false + } + parts := strings.Split(v[0], ".") + if len(parts) != 2 { + bad(w) + return 0, c, false + } + b, e1 := base64.RawURLEncoding.DecodeString(parts[0]) + sig, e2 := base64.RawURLEncoding.DecodeString(parts[1]) + mac := hmac.New(sha256.New, s.secret) + mac.Write(b) + if e1 != nil || e2 != nil || !hmac.Equal(sig, mac.Sum(nil)) || json.Unmarshal(b, &c) != nil || c.Scope != scope || !uuidRE.MatchString(c.ID) { + bad(w) + return 0, c, false + } + } + if c.ID != "" { + if _, err := time.Parse(time.RFC3339Nano, c.Time); err != nil { + bad(w) + return 0, c, false + } + } + return limit, c, true +} + +type Server struct { + db *pgxpool.Pool + cache *redis.Client + secret []byte + ttl int + log *slog.Logger + stopping atomic.Bool + kdf chan struct{} + objects *ObjectStore + uploads chan struct{} + instance string + dummyHash string +} + +func New(db *pgxpool.Pool, cache *redis.Client, secret []byte, ttl int, logger *slog.Logger) *Server { + hostname, _ := os.Hostname() + return &Server{instance: hostname, db: db, cache: cache, secret: secret, ttl: ttl, log: logger, kdf: make(chan struct{}, 2), dummyHash: hashPassword("dummy-not-a-real-password")} +} +func (s *Server) Stop() { s.stopping.Store(true) } +func (s *Server) acquireKDF(ctx context.Context, w http.ResponseWriter) bool { + select { + case s.kdf <- struct{}{}: + return true + case <-ctx.Done(): + unavailable(w) + return false + } +} +func unavailable(w http.ResponseWriter) { + fail(w, 503, "unavailable", "Dependency temporarily unavailable") +} +func unauthorized(w http.ResponseWriter, scheme string) { + challenge := scheme + ` realm="messenger"` + if scheme == "Basic" { + challenge += `, charset="UTF-8"` + } + w.Header().Set("WWW-Authenticate", challenge) + fail(w, 401, "unauthorized", "Invalid or missing credentials") +} +func stamp(t time.Time) string { return t.UTC().Format("2006-01-02T15:04:05.000000Z") } + +const userColumns = "id::text, username, display_name, created_at" + +type scanner interface{ Scan(...any) error } + +func scanUser(row scanner) (User, error) { + var u User + var t time.Time + err := row.Scan(&u.ID, &u.Username, &u.DisplayName, &t) + u.CreatedAt = stamp(t) + return u, err +} +func scanMessage(row scanner) (Message, error) { + var m Message + var t time.Time + err := row.Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &t) + m.CreatedAt = stamp(t) + m.AttachmentIDs = []string{} + return m, err +} + +func (s *Server) profile(ctx context.Context, uid string) (User, error) { + // Valkey caches ONLY immutable User JSON. Session validity is always read + // from PostgreSQL first; cache loss never permits anonymous access. + key := "profile:v1:" + uid + if s.cache != nil { + c, cancel := context.WithTimeout(ctx, 80*time.Millisecond) + raw, err := s.cache.Get(c, key).Bytes() + cancel() + var u User + if err == nil && json.Unmarshal(raw, &u) == nil && u.ID == uid { + return u, nil + } + } + u, err := scanUser(s.db.QueryRow(ctx, "SELECT "+userColumns+" FROM users WHERE id=$1", uid)) + if err != nil { + return u, err + } + if s.cache != nil { + raw, _ := json.Marshal(u) + c, cancel := context.WithTimeout(ctx, 80*time.Millisecond) + _ = s.cache.Set(c, key, raw, 300*time.Second).Err() + cancel() + } + return u, nil +} +func (s *Server) authenticate(w http.ResponseWriter, r *http.Request) (User, []byte, bool) { + parts := strings.Fields(r.Header.Get("Authorization")) + if len(parts) != 2 || !strings.EqualFold(parts[0], "Bearer") { + unauthorized(w, "Bearer") + return User{}, nil, false + } + raw, err := base64.RawURLEncoding.DecodeString(parts[1]) + if err != nil || len(raw) != 32 { + unauthorized(w, "Bearer") + return User{}, nil, false + } + digest := sha256.Sum256([]byte(parts[1])) + var uid string + err = s.db.QueryRow(r.Context(), "SELECT user_id::text FROM sessions WHERE token_hash=$1 AND expires_at>clock_timestamp()", digest[:]).Scan(&uid) + if errors.Is(err, pgx.ErrNoRows) { + unauthorized(w, "Bearer") + return User{}, nil, false + } + if err != nil { + unavailable(w) + return User{}, nil, false + } + u, err := s.profile(r.Context(), uid) + if err != nil { + unavailable(w) + return User{}, nil, false + } + return u, digest[:], true +} + +type recorder struct { + http.ResponseWriter + status int +} + +func (w *recorder) WriteHeader(status int) { w.status = status; w.ResponseWriter.WriteHeader(status) } +func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + start := time.Now() + out := &recorder{w, 200} + w = out + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.Header().Set("X-Request-Id", uuid()) + w.Header().Set("X-Instance-Id", s.instance) + budget := 3 * time.Second + if strings.Contains(r.URL.Path, "attachments") { + budget = 30 * time.Second + } + ctx, cancel := context.WithTimeout(r.Context(), budget) + defer cancel() + r = r.WithContext(ctx) + route := "unknown" + defer func() { + if recover() != nil { + fail(w, 500, "internal_error", "Internal server error") + } + if s.log != nil { + s.log.Info("request", "method", r.Method, "route", route, "status", out.status, "duration_us", time.Since(start).Microseconds(), "request_id", w.Header().Get("X-Request-Id")) + } + }() + if r.Method == "GET" && (r.URL.Path == "/health/live" || r.URL.Path == "/health/ready") { + route = r.URL.Path + if r.URL.Path == "/health/ready" && (s.stopping.Load() || s.db.Ping(ctx) != nil || (s.objects != nil && !s.objects.Ready(ctx))) { + unavailable(w) + } else { + reply(w, 200, map[string]string{"status": "ok"}) + } + return + } + if s.stopping.Load() { + unavailable(w) + return + } + if r.Method == "POST" && r.URL.Path == "/api/v1/users" { + route = "/users" + s.createUser(w, r) + return + } + if r.Method == "POST" && r.URL.Path == "/api/v1/auth/sessions" { + route = "/auth/sessions" + s.login(w, r) + return + } + user, digest, ok := s.authenticate(w, r) + if !ok { + return + } + if r.Method == "DELETE" && r.URL.Path == "/api/v1/auth/sessions/current" { + route = "/auth/sessions/current" + tag, err := s.db.Exec(ctx, "DELETE FROM sessions WHERE token_hash=$1", digest) + if err != nil { + unavailable(w) + } else if tag.RowsAffected() == 0 { + unauthorized(w, "Bearer") + } else { + w.WriteHeader(204) + } + return + } + if r.Method == "GET" && r.URL.Path == "/api/v1/users/me" { + route = "/users/me" + reply(w, 200, user) + return + } + if r.URL.Path == "/api/v1/chats" { + route = "/chats" + switch r.Method { + case "POST": + s.createChat(w, r, user.ID) + case "GET": + s.listChats(w, r, user.ID) + default: + missing(w) + } + return + } + parts := strings.Split(strings.TrimPrefix(r.URL.Path, "/"), "/") + if len(parts) >= 4 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "attachments" && s.objects != nil { + route = "/attachments/{id}" + s.serveAttachment(w, r, user.ID, parts) + return + } + if len(parts) < 4 || len(parts) > 5 || parts[0] != "api" || parts[1] != "v1" || parts[2] != "chats" { + missing(w) + return + } + cid := strings.ToLower(parts[3]) + route = "/chats/{id}" + if !uuidRE.MatchString(cid) { + bad(w) + return + } + chat, err := s.getChat(ctx, cid, user.ID) + if errors.Is(err, pgx.ErrNoRows) { + missing(w) + return + } + if err != nil { + unavailable(w) + return + } + if len(parts) == 4 && r.Method == "GET" { + reply(w, 200, chat) + return + } + if len(parts) == 5 && parts[4] == "attachments" && r.Method == "POST" && s.objects != nil { + route = "/chats/{id}/attachments" + s.uploadAttachment(w, r, user.ID, cid) + return + } + if len(parts) == 5 && parts[4] == "messages" { + route = "/chats/{id}/messages" + switch r.Method { + case "POST": + s.createMessage(w, r, user.ID, cid) + case "GET": + s.listMessages(w, r, cid) + default: + missing(w) + } + return + } + missing(w) +} +func (s *Server) createUser(w http.ResponseWriter, r *http.Request) { + var input struct { + Username string `json:"username"` + DisplayName string `json:"display_name"` + Password string `json:"password"` + } + if !decode(w, r, &input, "username", "display_name", "password") { + return + } + if !usernameRE.MatchString(input.Username) || !validText(input.DisplayName, 100) || !validText(input.Password, 128) || utf8.RuneCountInString(input.Password) < 12 { + bad(w) + return + } + if !s.acquireKDF(r.Context(), w) { + return + } + hash := hashPassword(input.Password) + <-s.kdf + u, err := scanUser(s.db.QueryRow(r.Context(), "INSERT INTO users(id,username,display_name,password_hash) VALUES($1,$2,$3,$4) RETURNING "+userColumns, uuid(), input.Username, input.DisplayName, hash)) + var pgerr *pgconn.PgError + if errors.As(err, &pgerr) && pgerr.Code == "23505" { + fail(w, 409, "conflict", "Username already exists") + return + } + if err != nil { + unavailable(w) + return + } + reply(w, 201, u) +} +func (s *Server) login(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 1)) + if err != nil || len(body) != 0 { + bad(w) + return + } + username, password, ok := r.BasicAuth() + if !ok || !usernameRE.MatchString(username) || !validText(password, 128) || utf8.RuneCountInString(password) < 12 { + unauthorized(w, "Basic") + return + } + var u User + var hash string + var t time.Time + err = s.db.QueryRow(r.Context(), "SELECT "+userColumns+", password_hash FROM users WHERE username=$1", username).Scan(&u.ID, &u.Username, &u.DisplayName, &t, &hash) + absent := errors.Is(err, pgx.ErrNoRows) + if err != nil && !absent { + unavailable(w) + return + } + if absent { + hash = s.dummyHash + } + if !s.acquireKDF(r.Context(), w) { + return + } + valid := verifyPassword(password, hash) + <-s.kdf + if !valid || absent { + unauthorized(w, "Basic") + return + } + raw := make([]byte, 32) + if _, err = rand.Read(raw); err != nil { + panic(err) + } + token := base64.RawURLEncoding.EncodeToString(raw) + digest := sha256.Sum256([]byte(token)) + var expires time.Time + err = s.db.QueryRow(r.Context(), "INSERT INTO sessions(token_hash,user_id,expires_at) VALUES($1,$2,clock_timestamp()+make_interval(secs=>$3)) RETURNING expires_at", digest[:], u.ID, s.ttl).Scan(&expires) + if err != nil { + unavailable(w) + return + } + u.CreatedAt = stamp(t) + reply(w, 201, map[string]any{"token": token, "token_type": "Bearer", "expires_at": stamp(expires), "user": u}) +} +func (s *Server) createChat(w http.ResponseWriter, r *http.Request, uid string) { + var input struct { + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` + } + if !decode(w, r, &input, "title", "member_ids") { + return + } + if !validText(input.Title, 100) || len(input.MemberIDs) < 1 || len(input.MemberIDs) > 100 { + bad(w) + return + } + seen := map[string]bool{} + for i, id := range input.MemberIDs { + id = strings.ToLower(id) + if !uuidRE.MatchString(id) || seen[id] { + bad(w) + return + } + seen[id] = true + input.MemberIDs[i] = id + } + if !seen[uid] { + bad(w) + return + } + tx, err := s.db.Begin(r.Context()) + if err != nil { + unavailable(w) + return + } + defer tx.Rollback(context.Background()) + chat := Chat{ID: uuid(), Title: input.Title, MemberIDs: input.MemberIDs} + var t time.Time + err = tx.QueryRow(r.Context(), "INSERT INTO chats(id,title) VALUES($1,$2) RETURNING created_at", chat.ID, chat.Title).Scan(&t) + if err != nil { + unavailable(w) + return + } + for _, id := range input.MemberIDs { + _, err = tx.Exec(r.Context(), "INSERT INTO chat_members(chat_id,user_id) VALUES($1,$2)", chat.ID, id) + var pgerr *pgconn.PgError + if errors.As(err, &pgerr) && pgerr.Code == "23503" { + bad(w) + return + } + if err != nil { + unavailable(w) + return + } + } + if tx.Commit(r.Context()) != nil { + unavailable(w) + return + } + chat.CreatedAt = stamp(t) + reply(w, 201, chat) +} + +const chatColumns = `c.id::text,c.title,c.created_at,ARRAY(SELECT m.user_id::text FROM chat_members m WHERE m.chat_id=c.id ORDER BY m.user_id)` + +func scanChat(row scanner) (Chat, error) { + var c Chat + var t time.Time + err := row.Scan(&c.ID, &c.Title, &t, &c.MemberIDs) + c.CreatedAt = stamp(t) + return c, err +} +func (s *Server) getChat(ctx context.Context, cid, uid string) (Chat, error) { + return scanChat(s.db.QueryRow(ctx, "SELECT "+chatColumns+" FROM chats c JOIN chat_members a ON a.chat_id=c.id AND a.user_id=$2 WHERE c.id=$1", cid, uid)) +} + +var idempotencyRE = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`) + +func (s *Server) createMessage(w http.ResponseWriter, r *http.Request, uid, cid string) { + text, attachments, ok := parseMessage(w, r) + if !ok { + return + } + input := struct { + Text string `json:"text"` + Attachments []string `json:"attachment_ids"` + }{text, attachments} + keys := r.Header.Values("Idempotency-Key") + if len(keys) != 1 || !idempotencyRE.MatchString(keys[0]) { + bad(w) + return + } + body, _ := json.Marshal(input) + digest := sha256.Sum256(body) + ctx := r.Context() + tx, err := s.db.Begin(ctx) + if err != nil { + unavailable(w) + return + } + defer tx.Rollback(context.Background()) + // Serialize only the same scope/key, across processes. Hash collisions only + // serialize unrelated requests; the full unique key still decides identity. + _, err = tx.Exec(ctx, "SELECT pg_advisory_xact_lock(hashtextextended($1,0))", uid+":"+cid+":"+keys[0]) + if err != nil { + unavailable(w) + return + } + var previous, saved []byte + err = tx.QueryRow(ctx, "SELECT body_hash,response FROM message_requests WHERE user_id=$1 AND chat_id=$2 AND key=$3 AND expires_at>clock_timestamp()", uid, cid, keys[0]).Scan(&previous, &saved) + if err == nil { + if !bytes.Equal(previous, digest[:]) { + fail(w, 409, "conflict", "Key already used with another message") + return + } + if tx.Commit(ctx) != nil { + unavailable(w) + return + } + w.WriteHeader(201) + _, _ = w.Write(saved) + return + } + if !errors.Is(err, pgx.ErrNoRows) { + unavailable(w) + return + } + // A row lock prevents cleanup of an attachment while a message links it. + for _, id := range attachments { + var found, status string + err = tx.QueryRow(ctx, "SELECT id::text,status FROM attachments WHERE id=$1 AND chat_id=$2 FOR SHARE", id, cid).Scan(&found, &status) + if errors.Is(err, pgx.ErrNoRows) { + missing(w) + return + } + if err != nil { + unavailable(w) + return + } + if status == "failed" { + fail(w, 409, "conflict", "Attachment processing failed") + return + } + } + var t time.Time + err = tx.QueryRow(ctx, "UPDATE chats SET last_message_at=GREATEST(clock_timestamp(),last_message_at+interval '1 microsecond') WHERE id=$1 RETURNING last_message_at", cid).Scan(&t) + if err != nil { + unavailable(w) + return + } + m, err := scanMessage(tx.QueryRow(ctx, "INSERT INTO messages(id,chat_id,sender_id,text,created_at) VALUES($1,$2,$3,$4,$5) RETURNING id::text,chat_id::text,sender_id::text,text,created_at", uuid(), cid, uid, input.Text, t)) + if err != nil { + unavailable(w) + return + } + m.AttachmentIDs = attachments + for i, id := range attachments { + if _, err = tx.Exec(ctx, "INSERT INTO message_attachments(message_id,attachment_id,position) VALUES($1,$2,$3)", m.ID, id, i); err != nil { + unavailable(w) + return + } + } + response, _ := json.Marshal(m) + _, err = tx.Exec(ctx, `INSERT INTO message_requests(user_id,chat_id,key,body_hash,response,expires_at) VALUES($1,$2,$3,$4,$5,clock_timestamp()+interval '24 hours') ON CONFLICT(user_id,chat_id,key) DO UPDATE SET body_hash=EXCLUDED.body_hash,response=EXCLUDED.response,expires_at=EXCLUDED.expires_at`, uid, cid, keys[0], digest[:], response) + if err != nil || tx.Commit(ctx) != nil { + unavailable(w) + return + } + reply(w, 201, m) +} +func (s *Server) listMessages(w http.ResponseWriter, r *http.Request, cid string) { + scope := "messages:" + cid + limit, c, ok := s.pagination(w, r, scope) + if !ok { + return + } + query := `SELECT id::text,chat_id::text,sender_id::text,text,created_at,ARRAY(SELECT attachment_id::text FROM message_attachments ma WHERE ma.message_id=messages.id ORDER BY position) FROM messages WHERE chat_id=$1` + args := []any{cid} + if c.ID != "" { + query += ` AND (created_at,id)>($2::timestamptz,$3::uuid)` + args = append(args, c.Time, c.ID) + } + query += fmt.Sprintf(" ORDER BY messages.created_at,messages.id LIMIT $%d", len(args)+1) + args = append(args, limit+1) + rows, err := s.db.Query(r.Context(), query, args...) + if err != nil { + unavailable(w) + return + } + defer rows.Close() + items := []Message{} + for rows.Next() { + var m Message + var created time.Time + err := rows.Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &created, &m.AttachmentIDs) + m.CreatedAt = stamp(created) + if err != nil { + unavailable(w) + return + } + items = append(items, m) + } + if rows.Err() != nil { + unavailable(w) + return + } + var next *string + if len(items) > limit { + items = items[:limit] + last := items[limit-1] + next = s.encodeCursor(cursor{scope, last.CreatedAt, last.ID}) + } + reply(w, 200, Page[Message]{items, next}) +} +func (s *Server) listChats(w http.ResponseWriter, r *http.Request, uid string) { + scope := "chats:" + uid + limit, c, ok := s.pagination(w, r, scope) + if !ok { + return + } + query := "SELECT " + chatColumns + " FROM chats c JOIN chat_members a ON a.chat_id=c.id AND a.user_id=$1" + args := []any{uid} + if c.ID != "" { + query += " WHERE (c.created_at,c.id)>($2::timestamptz,$3::uuid)" + args = append(args, c.Time, c.ID) + } + query += fmt.Sprintf(" ORDER BY c.created_at,c.id LIMIT $%d", len(args)+1) + args = append(args, limit+1) + rows, err := s.db.Query(r.Context(), query, args...) + if err != nil { + unavailable(w) + return + } + defer rows.Close() + items := []Chat{} + for rows.Next() { + chat, err := scanChat(rows) + if err != nil { + unavailable(w) + return + } + items = append(items, chat) + } + if rows.Err() != nil { + unavailable(w) + return + } + var next *string + if len(items) > limit { + items = items[:limit] + last := items[limit-1] + next = s.encodeCursor(cursor{scope, last.CreatedAt, last.ID}) + } + reply(w, 200, Page[Chat]{items, next}) +} diff --git a/internal/messenger/transform.go b/internal/messenger/transform.go new file mode 100644 index 00000000..3d9624d4 --- /dev/null +++ b/internal/messenger/transform.go @@ -0,0 +1,104 @@ +package messenger + +import ( + "encoding/json" + "fmt" + "github.com/disintegration/imaging" + "image" + "image/color" + "image/draw" + "image/jpeg" + "image/png" + "math" + "os" + "os/exec" + "path/filepath" +) + +type Variant struct { + Name string `json:"name"` + ContentType string `json:"content_type"` + Size int64 `json:"size_bytes"` + Width int `json:"width"` + Height int `json:"height"` + SHA256 string `json:"sha256"` +} + +func fitDimensions(w, h, max int) (int, int) { + scale := math.Min(1, math.Min(float64(max)/float64(w), float64(max)/float64(h))) + return int(math.Max(1, math.Floor(float64(w)*scale))), int(math.Max(1, math.Floor(float64(h)*scale))) +} + +// Runs in a disposable child process, killed by the parent's 20s deadline. +// The only paths are server-generated temp paths. No client arguments/shell. +func Transform(input, dir string, versions ...int) error { + version := 1 + if len(versions) > 0 { + version = versions[0] + } + if version < 1 || version > 2 { + return fmt.Errorf("unknown pipeline version") + } + quality, webpQuality := 85, "80" + if version == 2 { + quality, webpQuality = 90, "82" + } + f, e := os.Open(input) + if e != nil { + return e + } + c, format, e := image.DecodeConfig(f) + f.Close() + if e != nil || c.Width < 1 || c.Height < 1 || int64(c.Width)*int64(c.Height) > maxPixels { + return ErrInvalidImage + } + img, e := imaging.Open(input, imaging.AutoOrientation(true)) + if e != nil { + return ErrInvalidImage + } + var oriented image.Image = img + if format == "png" { + oriented = orientPNG(img, pngOrientation(input)) + } + bounds := oriented.Bounds() + white := image.NewNRGBA(bounds) + draw.Draw(white, bounds, &image.Uniform{C: color.White}, image.Point{}, draw.Src) + draw.Draw(white, bounds, oriented, bounds.Min, draw.Over) + w, h := fitDimensions(bounds.Dx(), bounds.Dy(), 256) + thumbnail := imaging.Resize(white, w, h, imaging.Lanczos) + crop := imaging.Fill(white, 128, 128, imaging.Center, imaging.Lanczos) + for name, result := range map[string]image.Image{"thumbnail": thumbnail, "crop": crop} { + f, e := os.Create(filepath.Join(dir, name)) + if e != nil { + return e + } + e = jpeg.Encode(f, result, &jpeg.Options{Quality: quality}) + ce := f.Close() + if e != nil { + return e + } + if ce != nil { + return ce + } + } + ow, oh := fitDimensions(bounds.Dx(), bounds.Dy(), 1280) + optimized := imaging.Resize(white, ow, oh, imaging.Lanczos) + p := filepath.Join(dir, "optimized.png") + f, e = os.Create(p) + if e != nil { + return e + } + e = png.Encode(f, optimized) + f.Close() + if e != nil { + return e + } + // Decoding/re-encoding removes EXIF and other source metadata. WebP quality 80. + cmd := exec.Command("cwebp", "-quiet", "-q", webpQuality, "-m", "4", "-metadata", "none", p, "-o", filepath.Join(dir, "optimized")) + if e = cmd.Run(); e != nil { + return fmt.Errorf("webp_encode") + } + result := []Variant{{Name: "thumbnail", ContentType: "image/jpeg", Width: w, Height: h}, {Name: "crop", ContentType: "image/jpeg", Width: 128, Height: 128}, {Name: "optimized", ContentType: "image/webp", Width: ow, Height: oh}} + b, _ := json.Marshal(result) + return os.WriteFile(filepath.Join(dir, "variants.json"), b, 0600) +} diff --git a/internal/messenger/transform_failure_test.go b/internal/messenger/transform_failure_test.go new file mode 100644 index 00000000..a22d8348 --- /dev/null +++ b/internal/messenger/transform_failure_test.go @@ -0,0 +1,23 @@ +package messenger + +import ( + "context" + "errors" + "os/exec" + "testing" +) + +func TestTransformFailureClassification(t *testing.T) { + for _, c := range []struct { + command string + invalid bool + }{{"exit 65", true}, {"exit 2", false}, {"kill -KILL $$", false}} { + e := exec.Command("sh", "-c", c.command).Run() + if errors.Is(classifyTransformFailure(nil, e), ErrInvalidImage) != c.invalid { + t.Fatalf("misclassified encoder termination %q", c.command) + } + } + if !errors.Is(classifyTransformFailure(context.DeadlineExceeded, nil), context.DeadlineExceeded) { + t.Fatal("deadline lost") + } +} diff --git a/internal/messenger/transform_test.go b/internal/messenger/transform_test.go new file mode 100644 index 00000000..0d7799f8 --- /dev/null +++ b/internal/messenger/transform_test.go @@ -0,0 +1,118 @@ +package messenger + +import ( + "bytes" + "encoding/binary" + "hash/crc32" + "image" + "image/color" + "image/jpeg" + "image/png" + "os" + "os/exec" + "path/filepath" + "testing" +) + +func TestFitFloorNoUpscale(t *testing.T) { + for _, v := range [][5]int{{320, 200, 256, 256, 160}, {3, 2, 256, 3, 2}, {301, 200, 256, 256, 170}, {1, 40000, 256, 1, 256}} { + w, h := fitDimensions(v[0], v[1], v[2]) + if w != v[3] || h != v[4] { + t.Fatalf("fit %v got %dx%d", v, w, h) + } + } +} +func TestTransformAlphaAndEXIF(t *testing.T) { + if _, e := exec.LookPath("cwebp"); e != nil { + t.Skip("cwebp provided by Docker test target") + } + dir := t.TempDir() + input := filepath.Join(dir, "input.png") + f, _ := os.Create(input) + png.Encode(f, image.NewNRGBA(image.Rect(0, 0, 320, 200))) + f.Close() + if e := Transform(input, dir); e != nil { + t.Fatal(e) + } + b, _ := os.ReadFile(filepath.Join(dir, "thumbnail")) + img, e := jpeg.Decode(bytes.NewReader(b)) + if e != nil { + t.Fatal(e) + } + r, g, bl, _ := img.At(20, 20).RGBA() + if r < 65000 || g < 65000 || bl < 65000 { + t.Fatal("alpha not composited white") + } + if img.Bounds().Dx() != 256 || img.Bounds().Dy() != 160 { + t.Fatal(img.Bounds()) + } + source := image.NewNRGBA(image.Rect(0, 0, 320, 200)) + source.Set(0, 0, color.White) + var jpg bytes.Buffer + jpeg.Encode(&jpg, source, nil) + // APP1 Exif, little endian TIFF, orientation=6 (90 degrees clockwise). + exif := []byte{'E', 'x', 'i', 'f', 0, 0, 'I', 'I', 42, 0, 8, 0, 0, 0, 1, 0, 0x12, 1, 3, 0, 1, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0} + var oriented bytes.Buffer + oriented.Write(jpg.Bytes()[:2]) + oriented.Write([]byte{255, 225}) + binary.Write(&oriented, binary.BigEndian, uint16(len(exif)+2)) + oriented.Write(exif) + oriented.Write(jpg.Bytes()[2:]) + os.WriteFile(input, oriented.Bytes(), 0600) + if e = Transform(input, dir); e != nil { + t.Fatal(e) + } + b, _ = os.ReadFile(filepath.Join(dir, "thumbnail")) + img, e = jpeg.Decode(bytes.NewReader(b)) + if e != nil { + t.Fatal(e) + } + if img.Bounds().Dx() != 160 || img.Bounds().Dy() != 256 { + t.Fatalf("EXIF ignored: %v", img.Bounds()) + } +} + +func TestPNGEXIFAndMalformedIFD(t *testing.T) { + tiff := []byte{'I', 'I', 42, 0, 8, 0, 0, 0, 1, 0, 0x12, 1, 3, 0, 1, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0} + if tiffOrientation(tiff) != 6 { + t.Fatal("TIFF orientation") + } + for i := 0; i < len(tiff); i++ { + _ = tiffOrientation(tiff[:i]) + } + bad := append([]byte{}, tiff...) + for i := 4; i < 8; i++ { + bad[i] = 255 + } + if tiffOrientation(bad) != 1 { + t.Fatal("bad IFD offset") + } + var raw bytes.Buffer + png.Encode(&raw, image.NewNRGBA(image.Rect(0, 0, 320, 200))) + var chunk bytes.Buffer + binary.Write(&chunk, binary.BigEndian, uint32(len(tiff))) + chunk.WriteString("eXIf") + chunk.Write(tiff) + binary.Write(&chunk, binary.BigEndian, crc32.ChecksumIEEE(chunk.Bytes()[4:])) + data := append(append(append([]byte{}, raw.Bytes()[:33]...), chunk.Bytes()...), raw.Bytes()[33:]...) + dir := t.TempDir() + input := filepath.Join(dir, "input.png") + os.WriteFile(input, data, 0600) + if pngOrientation(input) != 6 { + t.Fatal("PNG eXIf orientation") + } + if _, e := exec.LookPath("cwebp"); e != nil { + return + } + if e := Transform(input, dir); e != nil { + t.Fatal(e) + } + b, _ := os.ReadFile(filepath.Join(dir, "thumbnail")) + img, e := jpeg.Decode(bytes.NewReader(b)) + if e != nil { + t.Fatal(e) + } + if img.Bounds().Dx() != 160 || img.Bounds().Dy() != 256 { + t.Fatal("PNG orientation not applied", img.Bounds()) + } +} diff --git a/scripts/bootstrap.py b/scripts/bootstrap.py new file mode 100644 index 00000000..e7ffab2c --- /dev/null +++ b/scripts/bootstrap.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +"""Generate local-only credentials. Existing credentials are never overwritten.""" +import base64 +import os +from pathlib import Path +import secrets +root=Path(__file__).resolve().parents[1] +directory=root/'secrets' +directory.mkdir(mode=0o700,exist_ok=True) +password_file=directory/'postgres_password' +if not password_file.exists(): + password=secrets.token_hex(24) + password_file.write_text(password+'\n') +else: + password=password_file.read_text().strip() +cache_file=directory/'valkey_config' +if not cache_file.exists(): + cache_password=secrets.token_hex(24) + cache_file.write_text('bind 0.0.0.0\nprotected-mode yes\nport 6379\ndir /data\nsave ""\nappendonly no\nrequirepass '+cache_password+'\n') +else: + cache_password=next(line.split()[1] for line in cache_file.read_text().splitlines() if line.startswith('requirepass ')) +broker_file=directory/'broker_config' +if not broker_file.exists(): + broker_password=secrets.token_hex(24) + broker_file.write_text('bind 0.0.0.0\nprotected-mode yes\nport 6379\ndir /data\nsave ""\nappendonly yes\nappendfsync always\nno-appendfsync-on-rewrite no\nmaxmemory 192mb\nmaxmemory-policy noeviction\nrequirepass '+broker_password+'\n') +else: + broker_password=next(line.split()[1] for line in broker_file.read_text().splitlines() if line.startswith('requirepass ')) +values={'broker_url':f'redis://:{broker_password}@broker:6379/0', 'database_url':f'postgres://messenger:{password}@postgres:5432/messenger?sslmode=disable', + 'cache_url':f'redis://:{cache_password}@valkey:6379/0', + 'cursor_key':base64.urlsafe_b64encode(secrets.token_bytes(32)).decode().rstrip('=')} +values.update({'s3_root_user':'root_'+secrets.token_hex(8), 's3_root_password':secrets.token_hex(24), 's3_access_key_id':'app_'+secrets.token_hex(8), 's3_secret_access_key':secrets.token_hex(24)}) +for name,value in values.items(): + p=directory/name + if not p.exists():p.write_text(value+'\n') +# Compose mounts individual files. Readable by container UID 10001; host +# directory 0700 prevents traversal by other local users. +for p in directory.iterdir():os.chmod(p,0o444) +if not (root/'.env').exists():(root/'.env').write_text('APP_PORT=8085\nSESSION_TTL_SECONDS=3600\n') +print('Local secrets ready (values hidden); .env and secrets/ are ignored by Git.') diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..c4020f27 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose stop lb api worker dispatcher housekeeping +docker compose restart postgres valkey s3 broker +docker compose up -d --wait diff --git a/scripts/rolling.sh b/scripts/rolling.sh new file mode 100755 index 00000000..0d2f4bbc --- /dev/null +++ b/scripts/rolling.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +ids=$(docker compose ps -q api) +count=$(printf '%s\n' "$ids" | wc -l | tr -d ' ') +for id in $ids; do + docker stop --time 6 "$id" >/dev/null + docker rm "$id" >/dev/null + docker compose up -d --no-deps --scale api="$count" --wait api >/dev/null + sleep 2 +done diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..0268b366 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,13 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?required API alias}" +for id in $(docker compose ps -q api); do + alias=$(docker inspect --format '{{.Config.Hostname}}' "$id") + if [ "$alias" = "$TARGET_INSTANCE_ID" ]; then + docker stop --time 6 "$id" >/dev/null + exit 0 + fi +done +printf '%s\n' 'Target API instance is not running' >&2 +exit 1 diff --git a/student-tests/clean_clone.py b/student-tests/clean_clone.py new file mode 100644 index 00000000..00a5a58e --- /dev/null +++ b/student-tests/clean_clone.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +"""Run committed solution from a real local clone, new credentials and volumes.""" +import json,os,socket,subprocess,tempfile,uuid +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +lab=json.loads((ROOT/'lab.json').read_text())['number'] +project='verify-lab'+str(lab)+'-'+uuid.uuid4().hex[:10] +with socket.socket() as sock:sock.bind(('127.0.0.1',0));port=sock.getsockname()[1] +with tempfile.TemporaryDirectory(prefix=project+'-') as directory: + clone=Path(directory)/'repo';subprocess.run(['git','clone','--no-hardlinks',str(ROOT),str(clone)],check=True) + env=dict(os.environ,COMPOSE_PROJECT_NAME=project,APP_PORT=str(port),API_REPLICAS='2',SESSION_TTL_SECONDS='3600',GRAFANA_PORT='0',PROMETHEUS_PORT='0',LOKI_PORT='0',TEMPO_PORT='0') + def run(*args):subprocess.run(args,cwd=clone,env=env,check=True) + def snapshots(): + ids=subprocess.check_output(['docker','compose','ps','-q'],cwd=clone,env=env,text=True).splitlines() + result={} + for id in ids: + item=json.loads(subprocess.check_output(['docker','inspect',id],text=True))[0] + service=item['Config']['Labels']['com.docker.compose.service'] + if service in ['api','postgres','valkey','s3','broker','worker','dispatcher']:result[item['Name']]={'service':service,'started_at':item['State']['StartedAt'],'image_id':item['Image'],'user':item['Config'].get('User',''),'pid1':item['Path']} + return result + try: + run('make','up');run('make','check');run('make','test','BASE_URL=http://localhost:'+str(port)) + before=snapshots();run('make','test-persistence','ACTION_SCRIPT=scripts/restart.sh','BASE_URL=http://localhost:'+str(port));after=snapshots() + for name,item in before.items():assert after[name]['started_at']!=item['started_at'],name+' was not restarted' + for item in after.values(): + if item['service']=='api':assert item['user']=='10001:10001' and item['pid1']=='/usr/local/bin/messenger' + (ROOT/'evidence/persistence-processes.json').write_text(json.dumps({'scope':'disposable real clean clone','before':before,'after':after},indent=2)+'\n') + print('PASS: real clean clone, fresh secrets/volumes, smoke, same-session persistence, actual store/API restart, non-root PID1') + finally: + # Only this unique synthetic project, never the working lab volumes. + run('docker','compose','down','-v','--remove-orphans') diff --git a/student-tests/operations.py b/student-tests/operations.py new file mode 100644 index 00000000..3edc0bfc --- /dev/null +++ b/student-tests/operations.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Stop each real container, then replace them in sequence under writes.""" +import json,os,subprocess,sys,threading,time,uuid +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'tests')) +from client import Client,Failure,check +BASE=os.environ.get('BASE_URL','http://localhost:8085') +def dc(*args):return subprocess.check_output(['docker','compose',*args],cwd=ROOT,text=True,stderr=subprocess.DEVNULL).strip() +def inspect(id):return json.loads(subprocess.check_output(['docker','inspect',id],text=True))[0] +def ready(): + deadline=time.monotonic()+45 + while time.monotonic()=action_done] + check(successes_after,'no successful write after action') + recovery=successes_after[0]-started + return {'successful_writes':len(results),'errors':failed,'successful_rps':len(results)/(time.monotonic()-started+1),'first_recovery_seconds':round(recovery,3),'action_seconds':round(action_done-started,3),'instances':sorted(seen),'confirmed_preserved':True} +def main(): + ready();c=Client(BASE);a,b,e,chat=c.bootstrap();results={'failures':[]} + ids=dc('ps','-q','api').splitlines();check(len(ids)==2,'requires two API containers') + for id in ids: + alias=inspect(id)['Config']['Hostname'] + env=dict(os.environ,TARGET_INSTANCE_ID=alias) + def action():subprocess.run([str(ROOT/'scripts/stop-one.sh')],env=env,check=True,stdout=subprocess.DEVNULL) + try: + result=run_phase(c,a,chat,action);result['stopped_alias']=alias + check(not inspect(id)['State']['Running'],'target is still running') + check(result['first_recovery_seconds']<10,'recovery >10s') + results['failures'].append(result) + finally:ready() + results['rolling']=run_phase(c,a,chat,lambda:subprocess.run([str(ROOT/'scripts/rolling.sh')],check=True,stdout=subprocess.DEVNULL),seconds=2) + check(len(dc('ps','-q','api').splitlines())==2,'rolling lost replicas') + (ROOT/'evidence/operations.json').write_text(json.dumps(results,indent=2)+'\n') + print('PASS: each API stopped and rolling replacement completed; confirmed writes preserved') +if __name__=='__main__':main() diff --git a/student-tests/pipeline.py b/student-tests/pipeline.py new file mode 100755 index 00000000..5d9150a5 --- /dev/null +++ b/student-tests/pipeline.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +"""Black-box fault experiment. Mutates only this synthetic Compose project.""" +import hashlib,json,os,subprocess,sys,time,uuid +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT/'tests')) +from client import Client,check,Failure +BASE=os.environ.get('BASE_URL','http://localhost:8085');C=Client(BASE) +RESULT={'method':'real process/container faults; no forged timestamps','cases':[]} +def dc(*args):return subprocess.check_output(['docker','compose',*args],cwd=ROOT,text=True,stderr=subprocess.DEVNULL).strip() +def sql(query):return dc('exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',query) +def record(name,**kwargs):RESULT['cases'].append(dict(name=name,**kwargs));print('PASS:',name,flush=True) +def meta(a,id):return C.call('GET','/api/v1/attachments/'+id,actor=a,schema='Attachment')['json'] +def wait_status(a,id,status,timeout=130): + start=time.monotonic() + while time.monotonic()-start