From b2bd7501d38e3c78f7a5d3e249b7b5d717a56db6 Mon Sep 17 00:00:00 2001 From: Khakimov Dmitrij Date: Tue, 10 Nov 2026 20:42:03 +0300 Subject: [PATCH 1/2] lab4 done --- .dockerignore | 15 +- Dockerfile | 60 + Dockerfile.storage | 34 + Makefile | 80 +- cmd/main.go | 105 ++ cmd/reconcile/main.go | 57 + compose.example.yaml | 4 +- compose.yaml | 161 +++ contracts/embed.go | 9 + contracts/infrastructure.json | 18 +- contracts/jobs.schema.json | 63 - contracts/openapi.json | 16 +- dto/create_chat.go | 13 + dto/create_message.go | 16 + dto/create_message_json.go | 33 + dto/create_message_json_test.go | 21 + dto/create_user.go | 14 + dto/current_user.go | 10 + dto/error.go | 11 + dto/get_chat.go | 12 + dto/list_chats.go | 18 + dto/list_messages.go | 21 + dto/liveness.go | 7 + dto/readiness.go | 7 + dto/session.go | 8 + dto/timestamp.go | 11 + dto/timestamp_test.go | 15 + evidence/lab4-acceptance.json | 104 ++ evidence/lab4-acceptance.txt | 2 + evidence/lab4-bucket-bootstrap.txt | 8 + evidence/lab4-build.txt | 112 ++ evidence/lab4-deploy.txt | 313 +++++ evidence/lab4-environment.json | 81 ++ evidence/lab4-failover.txt | 4 + evidence/lab4-go-tests.txt | 13 + evidence/lab4-integration.txt | 338 +++++ evidence/lab4-persistence.txt | 3 + evidence/lab4-reconciliation.json | 1226 +++++++++++++++++ evidence/lab4-reconciliation.txt | 7 + evidence/lab4-restart-verification.json | 54 + evidence/lab4-scaling.txt | 6 + evidence/lab4-smoke.txt | 2 + evidence/lab4-status-after-restart.json | 146 ++ evidence/lab4-status-before-restart.json | 146 ++ evidence/lab4-status-final.json | 146 ++ evidence/lab4-status-initial.json | 146 ++ evidence/lab4-update.txt | 3 + evidence/lab4-upload-memory.json | 105 ++ evidence/lab4-upload-memory.txt | 4 + evidence/lab4-validation.txt | 8 + evidence/swarm-grade4.json | 72 + go.mod | 42 + go.sum | 90 ++ infra/README.md | 13 +- infra/haproxy.cfg | 36 + infra/migrate.sh | 8 + infra/minio-entrypoint.sh | 5 + infra/redis.conf | 9 + infra/s3-bootstrap.sh | 11 + infra/s3-policy.json | 21 + infra/schema.md | 25 + internal/app/chat/access.go | 31 + internal/app/chat/attachments.go | 68 + internal/app/chat/create_chat.go | 33 + internal/app/chat/create_message.go | 58 + internal/app/chat/create_user.go | 32 + internal/app/chat/current_user.go | 21 + internal/app/chat/cursor.go | 75 + internal/app/chat/errors.go | 95 ++ internal/app/chat/get_chat.go | 32 + internal/app/chat/list_chats.go | 53 + internal/app/chat/list_messages.go | 70 + internal/app/chat/liveness.go | 13 + internal/app/chat/middleware.go | 43 + internal/app/chat/readiness.go | 25 + internal/app/chat/request.go | 74 + internal/app/chat/service.go | 93 ++ internal/app/chat/service_test.go | 353 +++++ internal/app/chat/session.go | 45 + internal/app/chat/swagger.go | 38 + internal/app/chat/usecase.go | 21 + internal/attachment/image.go | 128 ++ internal/attachment/image_test.go | 139 ++ internal/attachment/json.go | 18 + internal/attachment/json_test.go | 37 + internal/attachment/reconcile.go | 222 +++ internal/attachment/s3.go | 44 + internal/attachment/service.go | 132 ++ internal/config/config.go | 119 ++ internal/pkg/domain/chat.go | 10 + internal/pkg/domain/errors.go | 32 + internal/pkg/domain/message.go | 13 + internal/pkg/domain/pagination.go | 23 + internal/pkg/domain/session.go | 14 + internal/pkg/domain/user.go | 10 + internal/pkg/password/password.go | 65 + internal/pkg/password/password_test.go | 35 + internal/pkg/repository/memory.go | 202 +++ internal/pkg/repository/postgres/chats.go | 78 ++ internal/pkg/repository/postgres/lab4_test.go | 341 +++++ internal/pkg/repository/postgres/messages.go | 96 ++ internal/pkg/repository/postgres/storage.go | 51 + .../pkg/repository/postgres/storage_test.go | 289 ++++ internal/pkg/repository/postgres/users.go | 46 + internal/pkg/session/redis.go | 71 + internal/usecase/chat/repository.go | 28 + internal/usecase/chat/service.go | 182 +++ internal/usecase/chat/session.go | 102 ++ internal/usecase/chat/session_test.go | 19 + internal/usecase/chat/validation.go | 37 + lab.json | 8 +- migrations/001_initial.sql | 54 + migrations/002_idempotency.sql | 10 + migrations/003_attachments.sql | 20 + migrations/004_message_attachments.sql | 13 + scripts/bootstrap.sh | 20 + scripts/compose.sh | 7 + scripts/measure_load.py | 54 + scripts/measure_uploads.py | 85 ++ scripts/restart.sh | 4 + scripts/stop-one.sh | 4 + scripts/swarm.py | 269 ++++ stack.yaml | 164 +++ tests/own/README.md | 20 + tests/own/grade4.py | 120 ++ tests/own/helpers.py | 116 ++ tests/own/lab4.py | 154 +++ tests/own/reconcile.py | 137 ++ tests/own/swarm_grade4.py | 165 +++ 129 files changed, 9319 insertions(+), 109 deletions(-) create mode 100644 Dockerfile create mode 100644 Dockerfile.storage create mode 100644 cmd/main.go create mode 100644 cmd/reconcile/main.go create mode 100644 compose.yaml create mode 100644 contracts/embed.go delete mode 100644 contracts/jobs.schema.json create mode 100644 dto/create_chat.go create mode 100644 dto/create_message.go create mode 100644 dto/create_message_json.go create mode 100644 dto/create_message_json_test.go create mode 100644 dto/create_user.go create mode 100644 dto/current_user.go create mode 100644 dto/error.go create mode 100644 dto/get_chat.go create mode 100644 dto/list_chats.go create mode 100644 dto/list_messages.go create mode 100644 dto/liveness.go create mode 100644 dto/readiness.go create mode 100644 dto/session.go create mode 100644 dto/timestamp.go create mode 100644 dto/timestamp_test.go create mode 100644 evidence/lab4-acceptance.json create mode 100644 evidence/lab4-acceptance.txt create mode 100644 evidence/lab4-bucket-bootstrap.txt create mode 100644 evidence/lab4-build.txt create mode 100644 evidence/lab4-deploy.txt create mode 100644 evidence/lab4-environment.json create mode 100644 evidence/lab4-failover.txt create mode 100644 evidence/lab4-go-tests.txt create mode 100644 evidence/lab4-integration.txt create mode 100644 evidence/lab4-persistence.txt create mode 100644 evidence/lab4-reconciliation.json create mode 100644 evidence/lab4-reconciliation.txt create mode 100644 evidence/lab4-restart-verification.json create mode 100644 evidence/lab4-scaling.txt create mode 100644 evidence/lab4-smoke.txt create mode 100644 evidence/lab4-status-after-restart.json create mode 100644 evidence/lab4-status-before-restart.json create mode 100644 evidence/lab4-status-final.json create mode 100644 evidence/lab4-status-initial.json create mode 100644 evidence/lab4-update.txt create mode 100644 evidence/lab4-upload-memory.json create mode 100644 evidence/lab4-upload-memory.txt create mode 100644 evidence/lab4-validation.txt create mode 100644 evidence/swarm-grade4.json create mode 100644 go.mod create mode 100644 go.sum create mode 100644 infra/haproxy.cfg create mode 100755 infra/migrate.sh create mode 100755 infra/minio-entrypoint.sh create mode 100644 infra/redis.conf create mode 100755 infra/s3-bootstrap.sh create mode 100644 infra/s3-policy.json create mode 100644 infra/schema.md create mode 100644 internal/app/chat/access.go create mode 100644 internal/app/chat/attachments.go create mode 100644 internal/app/chat/create_chat.go create mode 100644 internal/app/chat/create_message.go create mode 100644 internal/app/chat/create_user.go create mode 100644 internal/app/chat/current_user.go create mode 100644 internal/app/chat/cursor.go create mode 100644 internal/app/chat/errors.go create mode 100644 internal/app/chat/get_chat.go create mode 100644 internal/app/chat/list_chats.go create mode 100644 internal/app/chat/list_messages.go create mode 100644 internal/app/chat/liveness.go create mode 100644 internal/app/chat/middleware.go create mode 100644 internal/app/chat/readiness.go create mode 100644 internal/app/chat/request.go create mode 100644 internal/app/chat/service.go create mode 100644 internal/app/chat/service_test.go create mode 100644 internal/app/chat/session.go create mode 100644 internal/app/chat/swagger.go create mode 100644 internal/app/chat/usecase.go create mode 100644 internal/attachment/image.go create mode 100644 internal/attachment/image_test.go create mode 100644 internal/attachment/json.go create mode 100644 internal/attachment/json_test.go create mode 100644 internal/attachment/reconcile.go create mode 100644 internal/attachment/s3.go create mode 100644 internal/attachment/service.go create mode 100644 internal/config/config.go create mode 100644 internal/pkg/domain/chat.go create mode 100644 internal/pkg/domain/errors.go create mode 100644 internal/pkg/domain/message.go create mode 100644 internal/pkg/domain/pagination.go create mode 100644 internal/pkg/domain/session.go create mode 100644 internal/pkg/domain/user.go create mode 100644 internal/pkg/password/password.go create mode 100644 internal/pkg/password/password_test.go create mode 100644 internal/pkg/repository/memory.go create mode 100644 internal/pkg/repository/postgres/chats.go create mode 100644 internal/pkg/repository/postgres/lab4_test.go create mode 100644 internal/pkg/repository/postgres/messages.go create mode 100644 internal/pkg/repository/postgres/storage.go create mode 100644 internal/pkg/repository/postgres/storage_test.go create mode 100644 internal/pkg/repository/postgres/users.go create mode 100644 internal/pkg/session/redis.go create mode 100644 internal/usecase/chat/repository.go create mode 100644 internal/usecase/chat/service.go create mode 100644 internal/usecase/chat/session.go create mode 100644 internal/usecase/chat/session_test.go create mode 100644 internal/usecase/chat/validation.go create mode 100644 migrations/001_initial.sql create mode 100644 migrations/002_idempotency.sql create mode 100644 migrations/003_attachments.sql create mode 100644 migrations/004_message_attachments.sql create mode 100755 scripts/bootstrap.sh create mode 100755 scripts/compose.sh create mode 100644 scripts/measure_load.py create mode 100644 scripts/measure_uploads.py create mode 100755 scripts/restart.sh create mode 100755 scripts/stop-one.sh create mode 100644 scripts/swarm.py create mode 100644 stack.yaml create mode 100644 tests/own/README.md create mode 100644 tests/own/grade4.py create mode 100644 tests/own/helpers.py create mode 100644 tests/own/lab4.py create mode 100644 tests/own/reconcile.py create mode 100644 tests/own/swarm_grade4.py diff --git a/.dockerignore b/.dockerignore index bc16e556..ef48c580 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,8 +1,8 @@ .git .github +.gitignore .env .env.* -!.env.example secrets certs volumes @@ -17,4 +17,17 @@ backups __pycache__ node_modules evidence +examples +scripts +tests +*.md +Dockerfile* +Makefile +requirements-dev.txt +lab.json .DS_Store + +.idea +.vscode +.agents +.codex diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..7cd6ea0a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,60 @@ +FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS goose-builder +RUN GOBIN=/out CGO_ENABLED=0 go install \ + -tags='no_clickhouse no_libsql no_mssql no_mysql no_sqlite3 no_vertica no_ydb no_spanner' \ + github.com/pressly/goose/v3/cmd/goose@v3.27.1 + +FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 AS migrations +COPY --from=goose-builder /out/goose /usr/local/bin/goose +COPY migrations /migrations +COPY infra/migrate.sh /usr/local/bin/migrate +USER 10001:10001 +ENTRYPOINT ["/usr/local/bin/migrate"] +CMD ["up"] + +FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS builder + +WORKDIR /src + +RUN apk add --no-cache gcc musl-dev + +COPY go.mod go.sum ./ +RUN go mod download + +COPY cmd ./cmd +COPY contracts ./contracts +COPY dto ./dto +COPY internal ./internal +COPY migrations ./migrations + +RUN CGO_ENABLED=0 GOOS=linux go build \ + -trimpath \ + -ldflags="-s -w" \ + -o /out/messenger ./cmd +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/reconcile ./cmd/reconcile + +FROM builder AS tests +COPY --from=goose-builder /out/goose /usr/local/bin/goose +ENV GOOSE_MIGRATION_DIR=/src/migrations +CMD ["go", "test", "-p=1", "-race", "-count=1", "-v", "./..."] + +FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 + +RUN addgroup -S messenger \ + && adduser -S -D -H -u 10001 -G messenger messenger + +COPY --from=builder --chown=messenger:messenger /out/messenger /usr/local/bin/messenger + +COPY --from=builder --chown=messenger:messenger /out/reconcile /usr/local/bin/reconcile + +ENV PORT=8080 + +EXPOSE 8080 + +USER messenger:messenger + +HEALTHCHECK --interval=10s --timeout=3s --start-period=3s --retries=3 \ + CMD wget -q -O /dev/null "http://127.0.0.1:${PORT}/health/ready" || exit 1 + +STOPSIGNAL SIGTERM + +ENTRYPOINT ["/usr/local/bin/messenger"] diff --git a/Dockerfile.storage b/Dockerfile.storage new file mode 100644 index 00000000..8b7f7d0a --- /dev/null +++ b/Dockerfile.storage @@ -0,0 +1,34 @@ +FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS minio-builder +ADD --checksum=sha256:8b11129f6a6830768bcdebab569290ad956618f5ff6ef4857b633acb64f95126 https://codeload.github.com/minio/minio/tar.gz/01ce918d8279a20e4706b96a64396146894adee4 /tmp/source.tar.gz +RUN mkdir /src && tar -xzf /tmp/source.tar.gz -C /src --strip-components=1 +WORKDIR /src +RUN CGO_ENABLED=0 go build -trimpath -ldflags='-s -w' -o /out/minio . +RUN echo '8b11129f6a6830768bcdebab569290ad956618f5ff6ef4857b633acb64f95126 /tmp/source.tar.gz' | sha256sum -c - + +FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS mc-builder +ADD --checksum=sha256:4cea8ed209c8f96e59ea1a5cc4901019b54c2be6d6f5f16c6749ea8bdcf4172e https://codeload.github.com/minio/mc/tar.gz/d6541ea280b73a834b64d4097e21f2be77676104 /tmp/source.tar.gz +RUN mkdir /src && tar -xzf /tmp/source.tar.gz -C /src --strip-components=1 +WORKDIR /src +RUN CGO_ENABLED=0 go build -trimpath -ldflags='-s -w' -o /out/mc . +RUN echo '4cea8ed209c8f96e59ea1a5cc4901019b54c2be6d6f5f16c6749ea8bdcf4172e /tmp/source.tar.gz' | sha256sum -c - + +FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 AS minio +LABEL org.opencontainers.image.source="https://github.com/minio/minio" \ + org.opencontainers.image.revision="01ce918d8279a20e4706b96a64396146894adee4" \ + org.opencontainers.image.version="RELEASE.2025-09-07T16-13-09Z" \ + org.opencontainers.image.licenses="AGPL-3.0" +RUN mkdir /data && chown 10001:10001 /data +COPY --from=minio-builder /out/minio /usr/local/bin/minio +COPY infra/minio-entrypoint.sh /usr/local/bin/start-minio +USER 10001:10001 +EXPOSE 9000 +ENTRYPOINT ["/usr/local/bin/start-minio"] + +FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 AS mc +LABEL org.opencontainers.image.source="https://github.com/minio/mc" \ + org.opencontainers.image.revision="d6541ea280b73a834b64d4097e21f2be77676104" \ + org.opencontainers.image.version="RELEASE.2025-08-13T08-35-41Z" \ + org.opencontainers.image.licenses="AGPL-3.0" +COPY --from=mc-builder /out/mc /usr/local/bin/mc +USER 10001:10001 +ENTRYPOINT ["mc"] diff --git a/Makefile b/Makefile index 5d86d171..8c7ec355 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,21 @@ -PYTHON ?= python3 -BASE_URL ?= http://localhost:8080 +PYTHON ?= $(if $(wildcard .venv/bin/python),.venv/bin/python,python3) +GO ?= go +PORT ?= 8080 +BASE_URL ?= http://localhost:$(PORT) +IMAGE ?= mtusi-messenger:lab4 CA_FILE ?= -ACTION_SCRIPT ?= +ACTION_SCRIPT ?= scripts/restart.sh +FAILOVER_SCRIPT ?= scripts/stop-one.sh +RECONCILE_ARGS ?= +export PYTHONDONTWRITEBYTECODE=1 TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help run docker-build docker-run check validate test test-own test-all load test-persistence bootstrap up down migrate test-integration test-grade4 test-lab4 test-reconcile measure-uploads measure-load migrate-status rolling-update test-failover wait bootstrap-s3 reconcile reconcile-apply + +export PORT BASE_URL help: - @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" + @echo "up: build and deploy Swarm stack; down: stop with volumes retained; test-integration: PostgreSQL race/rollback/EXPLAIN; test-grade4: cross-replica idempotency, both failures and rolling update; check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" check: $(PYTHON) scripts/check_contract.py @@ -18,6 +26,11 @@ validate: test: $(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS) +test-own: + $(GO) test -race ./... + +test-all: check validate test-own + load: $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) @@ -25,6 +38,59 @@ test-persistence: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) +bootstrap: + ./scripts/bootstrap.sh + +up: + $(PYTHON) scripts/swarm.py up + +down: + $(PYTHON) scripts/swarm.py down + +migrate: + $(PYTHON) scripts/swarm.py migrate + +migrate-status: + $(PYTHON) scripts/swarm.py migrate-status + +rolling-update: + $(PYTHON) scripts/swarm.py update + +run: + PORT="$(PORT)" $(GO) run ./cmd + +docker-build: + docker build -t "$(IMAGE)" . + +test-integration: + $(PYTHON) scripts/swarm.py test-integration + +test-grade4: + $(PYTHON) tests/own/swarm_grade4.py + +test-lab4: + $(PYTHON) tests/own/lab4.py + +test-reconcile: + $(PYTHON) tests/own/reconcile.py + +measure-uploads: + $(PYTHON) scripts/measure_uploads.py + +bootstrap-s3: + $(PYTHON) scripts/swarm.py bootstrap-s3 + +reconcile: + $(PYTHON) scripts/swarm.py reconcile $(RECONCILE_ARGS) + +reconcile-apply: + $(PYTHON) scripts/swarm.py reconcile --apply $(RECONCILE_ARGS) + +measure-load: + $(PYTHON) scripts/measure_load.py --base-url "$(BASE_URL)" + test-failover: - @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) - $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + $(PYTHON) tests/resilience.py --mode failover --action-script "$(if $(filter command line,$(origin ACTION_SCRIPT)),$(ACTION_SCRIPT),$(FAILOVER_SCRIPT))" --base-url "$(BASE_URL)" $(TLS_ARGS) + +wait: + $(PYTHON) scripts/swarm.py wait diff --git a/cmd/main.go b/cmd/main.go new file mode 100644 index 00000000..6f1f721d --- /dev/null +++ b/cmd/main.go @@ -0,0 +1,105 @@ +package main + +import ( + "context" + "errors" + "fmt" + "log" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/redis/go-redis/v9" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/app/chat" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/attachment" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/config" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/repository/postgres" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/session" + chatusecase "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/usecase/chat" +) + +const shutdownTimeout = 8 * time.Second + +func main() { + cfg, err := config.Load() + if err != nil { + log.Fatal(err) + } + poolConfig, err := pgxpool.ParseConfig("sslmode=disable") + if err != nil { + log.Fatal("invalid PostgreSQL config") + } + poolConfig.ConnConfig.Host = cfg.PostgresHost + poolConfig.ConnConfig.Port = cfg.PostgresPort + poolConfig.ConnConfig.User = cfg.PostgresUser + poolConfig.ConnConfig.Database = cfg.PostgresDatabase + poolConfig.ConnConfig.Password = cfg.PostgresPassword + poolConfig.ConnConfig.ConnectTimeout = cfg.QueryTimeout + poolConfig.ConnConfig.RuntimeParams["timezone"] = "UTC" + poolConfig.ConnConfig.RuntimeParams["statement_timeout"] = fmt.Sprint(cfg.QueryTimeout.Milliseconds()) + poolConfig.MaxConns = cfg.PostgresMaxConns + poolConfig.MinConns = 1 + poolConfig.MaxConnLifetime = time.Hour + poolConfig.MaxConnIdleTime = 5 * time.Minute + pool, err := pgxpool.NewWithConfig(context.Background(), poolConfig) + if err != nil { + log.Fatal("cannot create PostgreSQL pool") + } + defer pool.Close() + redisClient := redis.NewClient(&redis.Options{ + Addr: cfg.RedisAddr, PoolSize: 10, MaxActiveConns: 10, PoolTimeout: cfg.QueryTimeout, + DialTimeout: cfg.QueryTimeout, ReadTimeout: cfg.QueryTimeout, WriteTimeout: cfg.QueryTimeout, + MaxRetries: -1, ContextTimeoutEnabled: true, + }) + defer redisClient.Close() + repo := postgres.New(pool, cfg.QueryTimeout) + sessions := session.NewRedis(redisClient, cfg.QueryTimeout) + useCase := chatusecase.New(repo, sessions, cfg.SessionTTL) + objectStore, err := attachment.NewStore(cfg.S3Endpoint, cfg.S3Bucket, cfg.S3AccessKey, cfg.S3SecretKey, cfg.S3UseSSL) + if err != nil { + log.Fatal("invalid S3 config") + } + handler := chat.NewChat(useCase, cfg.CursorKey).WithAttachments(attachment.New(pool, objectStore)) + port := cfg.Port + e := handler.Router() + e.Server.ReadHeaderTimeout = 5 * time.Second + e.Server.ReadTimeout = 50 * time.Second + e.Server.WriteTimeout = 55 * time.Second + e.Server.IdleTimeout = 60 * time.Second + + serverErrors := make(chan error, 1) + go func() { + log.Printf("HTTP server is listening on :%s", port) + serverErrors <- e.Start(":" + port) + }() + + stop := make(chan os.Signal, 1) + signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM) + select { + case sig := <-stop: + log.Printf("received %s, shutting down", sig) + case err := <-serverErrors: + if !errors.Is(err, http.ErrServerClosed) { + log.Fatalf("HTTP server failed: %v", err) + } + return + } + + handler.Drain() + // Allow two 1-second LB checks to remove this backend before closing the listener. + time.Sleep(3 * time.Second) + ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) + defer cancel() + if err := e.Shutdown(ctx); err != nil { + log.Printf("graceful shutdown failed: %v", err) + if closeErr := e.Close(); closeErr != nil { + log.Printf("forced shutdown failed: %v", closeErr) + } + return + } + log.Printf("shutdown complete") +} diff --git a/cmd/reconcile/main.go b/cmd/reconcile/main.go new file mode 100644 index 00000000..2534aa6c --- /dev/null +++ b/cmd/reconcile/main.go @@ -0,0 +1,57 @@ +package main + +import ( + "context" + "encoding/json" + "flag" + "log" + "os" + "time" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/attachment" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/config" + "github.com/jackc/pgx/v5/pgxpool" +) + +func main() { + apply := flag.Bool("apply", false, "apply safe repairs; default dry-run") + grace := flag.Duration("grace", 2*time.Minute, "minimum object/metadata age (at least 90s)") + timeout := flag.Duration("timeout", 10*time.Minute, "whole scan deadline") + flag.Parse() + if *grace < attachment.MinimumGrace { + log.Fatal("grace must be at least 90s") + } + cfg, err := config.Load() + if err != nil { + log.Fatal(err) + } + pc, err := pgxpool.ParseConfig("sslmode=disable") + if err != nil { + log.Fatal("invalid DB config") + } + pc.ConnConfig.Host = cfg.PostgresHost + pc.ConnConfig.Port = cfg.PostgresPort + pc.ConnConfig.User = cfg.PostgresUser + pc.ConnConfig.Password = cfg.PostgresPassword + pc.ConnConfig.Database = cfg.PostgresDatabase + pc.ConnConfig.ConnectTimeout = 3 * time.Second + pc.ConnConfig.RuntimeParams["statement_timeout"] = "5000" + pc.MaxConns = 2 + ctx, cancel := context.WithTimeout(context.Background(), *timeout) + defer cancel() + pool, err := pgxpool.NewWithConfig(ctx, pc) + if err != nil { + log.Fatal("cannot create DB pool") + } + defer pool.Close() + store, err := attachment.NewStore(cfg.S3Endpoint, cfg.S3Bucket, cfg.S3AccessKey, cfg.S3SecretKey, cfg.S3UseSSL) + if err != nil { + log.Fatal("invalid S3 config") + } + result, err := attachment.New(pool, store).Reconcile(ctx, *apply, *grace) + _ = json.NewEncoder(os.Stdout).Encode(result) + if err != nil { + log.Print(err) + os.Exit(1) + } +} diff --git a/compose.example.yaml b/compose.example.yaml index 792b3f4e..76a785ff 100644 --- a/compose.example.yaml +++ b/compose.example.yaml @@ -1,14 +1,12 @@ # Topology worksheet, deliberately NOT a runnable stack (no application supplied). # Implement compose.yaml or stack.yaml; do not submit this empty file as deployment. # Service names below are logical roles, not imposed container names. -x-lab-number: 5 +x-lab-number: 4 x-required-roles: - api - postgres - load_balancer - s3 - - broker - - worker # TODO: implement real images/builds, network wiring, volumes, healthchecks, # credentials and bootstrap. Details: contracts/infrastructure.json and README.md. services: {} diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..4c221cd3 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,161 @@ +name: mtusi2026lab4 +services: + postgres: + image: postgres:17.6-alpine3.22 + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: + - postgres_password + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: [CMD-SHELL, 'pg_isready -U messenger -d messenger'] + interval: 2s + timeout: 2s + retries: 30 + cpus: 1.0 + mem_limit: 512m + redis: + image: redis:7.4.5-alpine3.21 + command: [redis-server, /usr/local/etc/redis/redis.conf] + volumes: + - redis_data:/data + - ./infra/redis.conf:/usr/local/etc/redis/redis.conf:ro + healthcheck: + test: [CMD, redis-cli, ping] + interval: 2s + timeout: 2s + retries: 30 + cpus: 0.5 + mem_limit: 128m + minio: + image: mtusi-minio:RELEASE.2025-09-07T16-13-09Z + build: + context: . + dockerfile: Dockerfile.storage + target: minio + secrets: [s3_root_access_key, s3_root_secret_key] + volumes: [s3_data:/data] + healthcheck: + test: [CMD, wget, -q, -T, '2', -O, /dev/null, 'http://127.0.0.1:9000/minio/health/live'] + interval: 2s + timeout: 3s + start_period: 15s + retries: 30 + cpus: 1.0 + mem_limit: 512m + stop_grace_period: 20s + s3_bootstrap: + image: mtusi-mc:RELEASE.2025-08-13T08-35-41Z + build: + context: . + dockerfile: Dockerfile.storage + target: mc + entrypoint: [/bin/sh, /bootstrap.sh] + configs: + - source: s3_bootstrap + target: /bootstrap.sh + - source: s3_policy + target: /policy.json + secrets: [s3_root_access_key, s3_root_secret_key, s3_access_key, s3_secret_key] + depends_on: + minio: + condition: service_healthy + cpus: 0.5 + mem_limit: 512m + migrations: + image: mtusi-migrations:lab4 + build: + context: . + target: migrations + environment: + GOOSE_DRIVER: postgres + GOOSE_DBSTRING: host=postgres user=messenger dbname=messenger sslmode=disable connect_timeout=5 + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: + - postgres_password + depends_on: + postgres: + condition: service_healthy + cpus: 0.5 + mem_limit: 128m + api: + build: . + image: mtusi-messenger:lab4 + environment: &api_environment + POSTGRES_HOST: postgres + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + POSTGRES_MAX_CONNS: '10' + REDIS_ADDR: redis:6379 + CURSOR_KEY_FILE: /run/secrets/cursor_key + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + QUERY_TIMEOUT_MS: '2000' + S3_ENDPOINT: minio:9000 + S3_BUCKET: attachments + S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key + S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key + S3_USE_SSL: 'false' + GOMEMLIMIT: 1152MiB + GOGC: '50' + secrets: &api_secrets + - postgres_password + - cursor_key + - s3_access_key + - s3_secret_key + ports: + - '127.0.0.1:${PORT:-8080}:8080' + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + migrations: + condition: service_completed_successfully + s3_bootstrap: + condition: service_completed_successfully + cpus: 1.0 + mem_limit: 1536m + stop_grace_period: 10s + tests: + image: mtusi-messenger-tests:lab4 + profiles: [test] + build: + context: . + target: tests + environment: + <<: *api_environment + INTEGRATION_TESTS: '1' + secrets: *api_secrets + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + cpus: 1.0 + mem_limit: 1g +configs: + s3_bootstrap: + file: ./infra/s3-bootstrap.sh + s3_policy: + file: ./infra/s3-policy.json +secrets: + s3_root_access_key: + file: ./secrets/s3_root_access_key + s3_root_secret_key: + file: ./secrets/s3_root_secret_key + s3_access_key: + file: ./secrets/s3_access_key + s3_secret_key: + file: ./secrets/s3_secret_key + postgres_password: + file: ./secrets/postgres_password + cursor_key: + file: ./secrets/cursor_key +volumes: + postgres_data: + redis_data: + s3_data: diff --git a/contracts/embed.go b/contracts/embed.go new file mode 100644 index 00000000..ef4b85ae --- /dev/null +++ b/contracts/embed.go @@ -0,0 +1,9 @@ +package contracts + +import _ "embed" + +// OpenAPI is embedded so Swagger UI works from a compiled binary and does not +// depend on the process working directory. +// +//go:embed openapi.json +var OpenAPI []byte diff --git a/contracts/infrastructure.json b/contracts/infrastructure.json index a4cfcb7b..dd2edf2d 100644 --- a/contracts/infrastructure.json +++ b/contracts/infrastructure.json @@ -1,5 +1,5 @@ { - "lab": 5, + "lab": 4, "kind": "requirements, not runnable deployment", "required_for_grade_3": [ { @@ -27,22 +27,6 @@ ], "private_bucket": true, "persistent_volume": true - }, - { - "role": "broker", - "choose_one": [ - "RabbitMQ", - "Redis Streams", - "Valkey Streams", - "NATS JetStream", - "documented durable equivalent" - ], - "persistent_volume": true, - "delivery": "at-least-once" - }, - { - "role": "worker", - "separate_process": true } ], "optional_components": [ diff --git a/contracts/jobs.schema.json b/contracts/jobs.schema.json deleted file mode 100644 index 2e9aa484..00000000 --- a/contracts/jobs.schema.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "urn:mtusi:messenger:attachment-job:v1", - "title": "Durable attachment processing job v1", - "type": "object", - "additionalProperties": false, - "required": [ - "schema_version", - "job_id", - "type", - "attachment_id", - "pipeline_version", - "attempt", - "created_at" - ], - "properties": { - "schema_version": { - "const": 1 - }, - "job_id": { - "type": "string", - "format": "uuid" - }, - "type": { - "const": "attachment.process.v1" - }, - "attachment_id": { - "type": "string", - "format": "uuid" - }, - "pipeline_version": { - "type": "integer", - "minimum": 1 - }, - "attempt": { - "type": "integer", - "minimum": 1 - }, - "created_at": { - "type": "string", - "format": "date-time" - }, - "traceparent": { - "type": "string", - "pattern": "^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$" - }, - "tracestate": { - "type": "string", - "maxLength": 512 - } - }, - "examples": [ - { - "schema_version": 1, - "job_id": "11111111-1111-4111-8111-111111111111", - "type": "attachment.process.v1", - "attachment_id": "22222222-2222-4222-8222-222222222222", - "pipeline_version": 1, - "attempt": 1, - "created_at": "2026-09-01T12:00:00Z" - } - ] -} diff --git a/contracts/openapi.json b/contracts/openapi.json index 4face909..138f07c0 100644 --- a/contracts/openapi.json +++ b/contracts/openapi.json @@ -1,8 +1,8 @@ { "openapi": "3.1.0", "info": { - "title": "MTUSI Messenger — лабораторная 5", - "version": "1.5.0", + "title": "MTUSI Messenger — лабораторная 4", + "version": "1.4.0", "description": "Нормативный минимум на 3. Семантика: contracts/README.md. Расширения на 4/5 не должны менять этот API. Наличие кода ошибки в схеме не требует реализовать факультативную причину его выдачи." }, "servers": [ @@ -2151,7 +2151,7 @@ "operationId": "uploadAttachment", "summary": "Принять один PNG/JPEG, до 10 MiB", "responses": { - "202": { + "201": { "description": "Успех", "headers": { "X-Request-Id": { @@ -2936,10 +2936,7 @@ "schema": { "type": "string", "enum": [ - "original", - "thumbnail", - "crop", - "optimized" + "original" ], "default": "original" } @@ -3464,10 +3461,7 @@ "status": { "type": "string", "enum": [ - "queued", - "processing", - "ready", - "failed" + "ready" ] }, "variants": { diff --git a/dto/create_chat.go b/dto/create_chat.go new file mode 100644 index 00000000..25c42a03 --- /dev/null +++ b/dto/create_chat.go @@ -0,0 +1,13 @@ +package dto + +type CreateChatRequest struct { + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` +} + +type CreateChatResponse struct { + ID string `json:"id"` + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` + CreatedAt Timestamp `json:"created_at"` +} diff --git a/dto/create_message.go b/dto/create_message.go new file mode 100644 index 00000000..f89c57c2 --- /dev/null +++ b/dto/create_message.go @@ -0,0 +1,16 @@ +package dto + +type CreateMessageRequest struct { + ChatID string `json:"-"` + Text string `json:"text"` + AttachmentIDs []string `json:"attachment_ids"` +} + +type CreateMessageResponse struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` + AttachmentIDs []string `json:"attachment_ids"` + CreatedAt Timestamp `json:"created_at"` +} diff --git a/dto/create_message_json.go b/dto/create_message_json.go new file mode 100644 index 00000000..b92e99d4 --- /dev/null +++ b/dto/create_message_json.go @@ -0,0 +1,33 @@ +package dto + +import ( + "bytes" + "encoding/json" + "errors" +) + +// Keep absent attachment_ids equivalent to [], while rejecting JSON null and +// wrong types rather than quietly treating them as an empty message body. +func (r *CreateMessageRequest) UnmarshalJSON(data []byte) error { + type payload CreateMessageRequest + var fields map[string]json.RawMessage + if err := json.Unmarshal(data, &fields); err != nil || fields == nil { + return errors.New("Ожидается JSON-объект") + } + text, exists := fields["text"] + if !exists || bytes.Equal(bytes.TrimSpace(text), []byte("null")) { + return errors.New("Требуется строка text") + } + if ids, exists := fields["attachment_ids"]; exists && bytes.Equal(bytes.TrimSpace(ids), []byte("null")) { + return errors.New("attachment_ids должен быть массивом") + } + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + var decoded payload + if err := decoder.Decode(&decoded); err != nil { + return err + } + decoded.ChatID = r.ChatID + *r = CreateMessageRequest(decoded) + return nil +} diff --git a/dto/create_message_json_test.go b/dto/create_message_json_test.go new file mode 100644 index 00000000..0d97cd09 --- /dev/null +++ b/dto/create_message_json_test.go @@ -0,0 +1,21 @@ +package dto + +import ( + "encoding/json" + "testing" +) + +func TestMessageStrictFields(t *testing.T) { + for _, body := range []string{`null`, `[]`, `{}`, `{"text":null}`, `{"text":"x","attachment_ids":null}`, `{"text":"x","attachment_ids":"x"}`, `{"text":"x","extra":1}`} { + var r CreateMessageRequest + if err := json.Unmarshal([]byte(body), &r); err == nil { + t.Fatalf("accepted %s", body) + } + } + for _, body := range []string{`{"text":"x"}`, `{"text":"x","attachment_ids":[]}`, `{"text":"","attachment_ids":["id"]}`} { + var r CreateMessageRequest + if err := json.Unmarshal([]byte(body), &r); err != nil { + t.Fatalf("rejected %s: %v", body, err) + } + } +} diff --git a/dto/create_user.go b/dto/create_user.go new file mode 100644 index 00000000..af41015a --- /dev/null +++ b/dto/create_user.go @@ -0,0 +1,14 @@ +package dto + +type CreateUserRequest struct { + Password string `json:"password"` + Username string `json:"username"` + DisplayName string `json:"display_name"` +} + +type CreateUserResponse struct { + ID string `json:"id"` + Username string `json:"username"` + DisplayName string `json:"display_name"` + CreatedAt Timestamp `json:"created_at"` +} diff --git a/dto/current_user.go b/dto/current_user.go new file mode 100644 index 00000000..b82a032d --- /dev/null +++ b/dto/current_user.go @@ -0,0 +1,10 @@ +package dto + +type CurrentUserRequest struct{} + +type CurrentUserResponse struct { + ID string `json:"id"` + Username string `json:"username"` + DisplayName string `json:"display_name"` + CreatedAt Timestamp `json:"created_at"` +} diff --git a/dto/error.go b/dto/error.go new file mode 100644 index 00000000..c8c17c4d --- /dev/null +++ b/dto/error.go @@ -0,0 +1,11 @@ +package dto + +type ErrorResponse struct { + Error ErrorDetails `json:"error"` +} + +type ErrorDetails struct { + Code string `json:"code"` + Message string `json:"message"` + RequestID string `json:"request_id"` +} diff --git a/dto/get_chat.go b/dto/get_chat.go new file mode 100644 index 00000000..6ed9e1ef --- /dev/null +++ b/dto/get_chat.go @@ -0,0 +1,12 @@ +package dto + +type GetChatRequest struct { + ChatID string `json:"-"` +} + +type GetChatResponse struct { + ID string `json:"id"` + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` + CreatedAt Timestamp `json:"created_at"` +} diff --git a/dto/list_chats.go b/dto/list_chats.go new file mode 100644 index 00000000..43f9eb16 --- /dev/null +++ b/dto/list_chats.go @@ -0,0 +1,18 @@ +package dto + +type ListChatsRequest struct { + Limit int `json:"-"` + Cursor string `json:"-"` +} + +type ListChatsResponse struct { + Items []ListChatsItem `json:"items"` + NextCursor *string `json:"next_cursor"` +} + +type ListChatsItem struct { + ID string `json:"id"` + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` + CreatedAt Timestamp `json:"created_at"` +} diff --git a/dto/list_messages.go b/dto/list_messages.go new file mode 100644 index 00000000..35a38428 --- /dev/null +++ b/dto/list_messages.go @@ -0,0 +1,21 @@ +package dto + +type ListMessagesRequest struct { + ChatID string `json:"-"` + Limit int `json:"-"` + Cursor string `json:"-"` +} + +type ListMessagesResponse struct { + Items []ListMessagesItem `json:"items"` + NextCursor *string `json:"next_cursor"` +} + +type ListMessagesItem struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` + AttachmentIDs []string `json:"attachment_ids"` + CreatedAt Timestamp `json:"created_at"` +} diff --git a/dto/liveness.go b/dto/liveness.go new file mode 100644 index 00000000..830955e9 --- /dev/null +++ b/dto/liveness.go @@ -0,0 +1,7 @@ +package dto + +type LivenessRequest struct{} + +type LivenessResponse struct { + Status string `json:"status"` +} diff --git a/dto/readiness.go b/dto/readiness.go new file mode 100644 index 00000000..21573b78 --- /dev/null +++ b/dto/readiness.go @@ -0,0 +1,7 @@ +package dto + +type ReadinessRequest struct{} + +type ReadinessResponse struct { + Status string `json:"status"` +} diff --git a/dto/session.go b/dto/session.go new file mode 100644 index 00000000..bd2c8807 --- /dev/null +++ b/dto/session.go @@ -0,0 +1,8 @@ +package dto + +type CreateSessionResponse struct { + Token string `json:"token"` + TokenType string `json:"token_type"` + ExpiresAt Timestamp `json:"expires_at"` + User CreateUserResponse `json:"user"` +} diff --git a/dto/timestamp.go b/dto/timestamp.go new file mode 100644 index 00000000..9dab3e29 --- /dev/null +++ b/dto/timestamp.go @@ -0,0 +1,11 @@ +package dto + +import "time" + +// Timestamp preserves PostgreSQL microseconds and is readable by Python 3.9 clients. +// time.Time's variable-length fractional part is not supported by those clients. +type Timestamp time.Time + +func (t Timestamp) MarshalJSON() ([]byte, error) { + return []byte(`"` + time.Time(t).UTC().Format("2006-01-02T15:04:05.000000Z") + `"`), nil +} diff --git a/dto/timestamp_test.go b/dto/timestamp_test.go new file mode 100644 index 00000000..1b48033a --- /dev/null +++ b/dto/timestamp_test.go @@ -0,0 +1,15 @@ +package dto + +import ( + "encoding/json" + "testing" + "time" +) + +func TestTimestampFixedMicroseconds(t *testing.T) { + timestamp := Timestamp(time.Date(2026, 1, 2, 3, 4, 5, 123400000, time.UTC)) + encoded, err := json.Marshal(timestamp) + if err != nil || string(encoded) != `"2026-01-02T03:04:05.123400Z"` { + t.Fatalf("timestamp=%s err=%v", encoded, err) + } +} diff --git a/evidence/lab4-acceptance.json b/evidence/lab4-acceptance.json new file mode 100644 index 00000000..8f453015 --- /dev/null +++ b/evidence/lab4-acceptance.json @@ -0,0 +1,104 @@ +{ + "negative_cases": [ + { + "case": "oversized_bytes", + "status": 413 + }, + { + "case": "damaged_png", + "status": 415 + }, + { + "case": "corrupt_crc", + "status": 415 + }, + { + "case": "mime_spoof", + "status": 415 + }, + { + "case": "not_image", + "status": 415 + }, + { + "case": "unsupported_mime", + "status": 415 + }, + { + "case": "pixel_budget", + "status": 413 + }, + { + "case": "unsafe_slash", + "status": 400 + }, + { + "case": "unsafe_backslash", + "status": 400 + }, + { + "case": "unsafe_control", + "status": 400 + }, + { + "case": "empty_filename", + "status": 400 + }, + { + "case": "foreign_attachment", + "status": 404 + }, + { + "case": "foreign_chat_link", + "status": 404 + }, + { + "case": "unknown_attachment", + "status": 404 + }, + { + "case": "duplicate_ids", + "status": 400 + }, + { + "case": "missing_authentication", + "status": 401 + }, + { + "case": "unknown_variant", + "status": 404 + }, + { + "case": "chunked_oversized", + "status": 413 + } + ], + "failures": { + "client_disconnect": { + "metadata_added": 0, + "spool_files_remaining": 0 + }, + "upload_replica_failure": { + "upload_instance": "alixbvpbjnzvtqvo5hhfpokc6", + "download_instance": "h6fc30dhoabp0jcfl6zrpjdzx", + "recovery_seconds": 0.51, + "sha256": "8502afe2cc5bab33ac834c834874369d29e7896599b5a256d68d80c5395d99c3" + }, + "s3_outage": { + "liveness": 200, + "readiness": 503, + "recovery_seconds": 19.285 + } + }, + "checksums": { + "image/png": "8502afe2cc5bab33ac834c834874369d29e7896599b5a256d68d80c5395d99c3", + "image/jpeg": "41feb40d75a595386c72cfc8a5bc013c2dedc445b73316ebbd3b3eac13491c5d" + }, + "bucket_security": { + "anonymous_get_status": 403, + "create_bucket_denied": true, + "write_outside_prefix_denied": true, + "console_published": false, + "bootstrap_repeats": 2 + } +} diff --git a/evidence/lab4-acceptance.txt b/evidence/lab4-acceptance.txt new file mode 100644 index 00000000..3a0ea764 --- /dev/null +++ b/evidence/lab4-acceptance.txt @@ -0,0 +1,2 @@ +.venv/bin/python tests/own/lab4.py +PASS: PNG/JPEG, bounded uploads, MIME/damage/ACL/link errors, disconnect, private S3, repeat bootstrap, S3 outage. diff --git a/evidence/lab4-bucket-bootstrap.txt b/evidence/lab4-bucket-bootstrap.txt new file mode 100644 index 00000000..533378d4 --- /dev/null +++ b/evidence/lab4-bucket-bootstrap.txt @@ -0,0 +1,8 @@ +image mtusi-mc:RELEASE.2025-08-13T08-35-41Z could not be accessed on a registry to record +its digest. Each node will access mtusi-mc:RELEASE.2025-08-13T08-35-41Z independently, +possibly leading to different nodes running different +versions of the image. + +0pza81k652zzumhh8w5zfuxw8 +mtusi-lab4_s3-bootstrap.0.jjp4j3jx92l1@colima | PASS: private bucket and restricted API account configured. +mtusi-lab4_s3-bootstrap diff --git a/evidence/lab4-build.txt b/evidence/lab4-build.txt new file mode 100644 index 00000000..41301955 --- /dev/null +++ b/evidence/lab4-build.txt @@ -0,0 +1,112 @@ +DEPRECATED: The legacy builder is deprecated and will be removed in a future release. + Install the buildx component to build images with BuildKit: + https://docs.docker.com/go/buildx/ + +Sending build context to Docker daemon 401.9kB +Step 1/35 : FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS goose-builder + ---> 18b460dd1754 +Step 2/35 : RUN GOBIN=/out CGO_ENABLED=0 go install -tags='no_clickhouse no_libsql no_mssql no_mysql no_sqlite3 no_vertica no_ydb no_spanner' github.com/pressly/goose/v3/cmd/goose@v3.27.1 + ---> Using cache + ---> 7012ef857c0c +Step 3/35 : FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 AS migrations + ---> 075c2c1a4068 +Step 4/35 : COPY --from=goose-builder /out/goose /usr/local/bin/goose + ---> Using cache + ---> 6a0ad52336c1 +Step 5/35 : COPY migrations /migrations + ---> Using cache + ---> 19018c7bfac9 +Step 6/35 : COPY infra/migrate.sh /usr/local/bin/migrate + ---> Using cache + ---> 0cc075ba3459 +Step 7/35 : USER 10001:10001 + ---> Using cache + ---> 37de51af32cd +Step 8/35 : ENTRYPOINT ["/usr/local/bin/migrate"] + ---> Using cache + ---> e7539a967679 +Step 9/35 : CMD ["up"] + ---> Using cache + ---> 841d9bb239cd +Step 10/35 : FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS builder + ---> 18b460dd1754 +Step 11/35 : WORKDIR /src + ---> Using cache + ---> 1ba013278a4a +Step 12/35 : RUN apk add --no-cache gcc musl-dev + ---> Using cache + ---> a0ea65f0bded +Step 13/35 : COPY go.mod go.sum ./ + ---> Using cache + ---> e98ccecfb093 +Step 14/35 : RUN go mod download + ---> Using cache + ---> 91d546c71507 +Step 15/35 : COPY cmd ./cmd + ---> Using cache + ---> b4e431a9d5f2 +Step 16/35 : COPY contracts ./contracts + ---> Using cache + ---> 958477d6c98e +Step 17/35 : COPY dto ./dto + ---> Using cache + ---> d115fe2e1af4 +Step 18/35 : COPY internal ./internal + ---> 2f61826e3774 +Step 19/35 : COPY migrations ./migrations + ---> d96dd811bc83 +Step 20/35 : RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/messenger ./cmd + ---> Running in 3d7cd429d659 + ---> Removed intermediate container 3d7cd429d659 + ---> 2ad69db4c288 +Step 21/35 : RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/reconcile ./cmd/reconcile + ---> Running in 7b9a510ed472 + ---> Removed intermediate container 7b9a510ed472 + ---> 4a49dc2719a3 +Step 22/35 : FROM builder AS tests + ---> 4a49dc2719a3 +Step 23/35 : COPY --from=goose-builder /out/goose /usr/local/bin/goose + ---> 5df1413dfc44 +Step 24/35 : ENV GOOSE_MIGRATION_DIR=/src/migrations + ---> Running in c30eee45a5e7 + ---> Removed intermediate container c30eee45a5e7 + ---> 173325b59e7d +Step 25/35 : CMD ["go", "test", "-p=1", "-race", "-count=1", "-v", "./..."] + ---> Running in 384e08125582 + ---> Removed intermediate container 384e08125582 + ---> 187ea6b3bcac +Step 26/35 : FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 + ---> 075c2c1a4068 +Step 27/35 : RUN addgroup -S messenger && adduser -S -D -H -u 10001 -G messenger messenger + ---> Using cache + ---> 33192b561bfc +Step 28/35 : COPY --from=builder --chown=messenger:messenger /out/messenger /usr/local/bin/messenger + ---> 37a18e0c6edf +Step 29/35 : COPY --from=builder --chown=messenger:messenger /out/reconcile /usr/local/bin/reconcile + ---> b0e79d0416ba +Step 30/35 : ENV PORT=8080 + ---> Running in 48da45f81d52 + ---> Removed intermediate container 48da45f81d52 + ---> a7c04e9ff97b +Step 31/35 : EXPOSE 8080 + ---> Running in 671ff92f3fad + ---> Removed intermediate container 671ff92f3fad + ---> e56b07b42f04 +Step 32/35 : USER messenger:messenger + ---> Running in 8003e4ea3b5d + ---> Removed intermediate container 8003e4ea3b5d + ---> 48b4424b156b +Step 33/35 : HEALTHCHECK --interval=10s --timeout=3s --start-period=3s --retries=3 CMD wget -q -O /dev/null "http://127.0.0.1:${PORT}/health/ready" || exit 1 + ---> Running in c05d11001e2b + ---> Removed intermediate container c05d11001e2b + ---> 563fbe4dcd7d +Step 34/35 : STOPSIGNAL SIGTERM + ---> Running in 8fb80a0e0f6c + ---> Removed intermediate container 8fb80a0e0f6c + ---> c5a0a88ba147 +Step 35/35 : ENTRYPOINT ["/usr/local/bin/messenger"] + ---> Running in 9e51b29d5939 + ---> Removed intermediate container 9e51b29d5939 + ---> c28dcd6e4410 +Successfully built c28dcd6e4410 +Successfully tagged mtusi-messenger:lab4 diff --git a/evidence/lab4-deploy.txt b/evidence/lab4-deploy.txt new file mode 100644 index 00000000..6a3e6ce5 --- /dev/null +++ b/evidence/lab4-deploy.txt @@ -0,0 +1,313 @@ +.venv/bin/python scripts/swarm.py up +vphhh79vd5ti8501q6m9lzulj +Local secrets are ready. +DEPRECATED: The legacy builder is deprecated and will be removed in a future release. + Install the buildx component to build images with BuildKit: + https://docs.docker.com/go/buildx/ + +Sending build context to Docker daemon 398.8kB +Step 1/20 : FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS minio-builder + ---> 18b460dd1754 +Step 2/20 : ADD --checksum=sha256:8b11129f6a6830768bcdebab569290ad956618f5ff6ef4857b633acb64f95126 https://codeload.github.com/minio/minio/tar.gz/01ce918d8279a20e4706b96a64396146894adee4 /tmp/source.tar.gz + + + ---> Using cache + ---> 4a2b9310629e +Step 3/20 : RUN mkdir /src && tar -xzf /tmp/source.tar.gz -C /src --strip-components=1 + ---> Using cache + ---> 5166de215d05 +Step 4/20 : WORKDIR /src + ---> Using cache + ---> 1f12c7444ff0 +Step 5/20 : RUN CGO_ENABLED=0 go build -trimpath -ldflags='-s -w' -o /out/minio . + ---> Using cache + ---> 8477e9d7324c +Step 6/20 : RUN echo '8b11129f6a6830768bcdebab569290ad956618f5ff6ef4857b633acb64f95126 /tmp/source.tar.gz' | sha256sum -c - + ---> Using cache + ---> 94162de69c8c +Step 7/20 : FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS mc-builder + ---> 18b460dd1754 +Step 8/20 : ADD --checksum=sha256:4cea8ed209c8f96e59ea1a5cc4901019b54c2be6d6f5f16c6749ea8bdcf4172e https://codeload.github.com/minio/mc/tar.gz/d6541ea280b73a834b64d4097e21f2be77676104 /tmp/source.tar.gz + + + ---> Using cache + ---> 5e135fc72a2c +Step 9/20 : RUN mkdir /src && tar -xzf /tmp/source.tar.gz -C /src --strip-components=1 + ---> Using cache + ---> cbffdcf35dd5 +Step 10/20 : WORKDIR /src + ---> Using cache + ---> 3322378e2ebd +Step 11/20 : RUN CGO_ENABLED=0 go build -trimpath -ldflags='-s -w' -o /out/mc . + ---> Using cache + ---> 0aba858f1d48 +Step 12/20 : RUN echo '4cea8ed209c8f96e59ea1a5cc4901019b54c2be6d6f5f16c6749ea8bdcf4172e /tmp/source.tar.gz' | sha256sum -c - + ---> Using cache + ---> af0a57fdfed1 +Step 13/20 : FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 AS minio + ---> 075c2c1a4068 +Step 14/20 : LABEL org.opencontainers.image.source="https://github.com/minio/minio" org.opencontainers.image.revision="01ce918d8279a20e4706b96a64396146894adee4" org.opencontainers.image.version="RELEASE.2025-09-07T16-13-09Z" org.opencontainers.image.licenses="AGPL-3.0" + ---> Using cache + ---> 57200264604d +Step 15/20 : RUN mkdir /data && chown 10001:10001 /data + ---> Using cache + ---> c7024406f9a9 +Step 16/20 : COPY --from=minio-builder /out/minio /usr/local/bin/minio + ---> Using cache + ---> 0d1965ab7125 +Step 17/20 : COPY infra/minio-entrypoint.sh /usr/local/bin/start-minio + ---> Using cache + ---> e5c21da70a4f +Step 18/20 : USER 10001:10001 + ---> Using cache + ---> 9bc52122d9f2 +Step 19/20 : EXPOSE 9000 + ---> Using cache + ---> 2ca448e77273 +Step 20/20 : ENTRYPOINT ["/usr/local/bin/start-minio"] + ---> Using cache + ---> 3a0797aa5ead +Successfully built 3a0797aa5ead +Successfully tagged mtusi-minio:RELEASE.2025-09-07T16-13-09Z +DEPRECATED: The legacy builder is deprecated and will be removed in a future release. + Install the buildx component to build images with BuildKit: + https://docs.docker.com/go/buildx/ + +Sending build context to Docker daemon 398.8kB +Step 1/25 : FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS minio-builder + ---> 18b460dd1754 +Step 2/25 : ADD --checksum=sha256:8b11129f6a6830768bcdebab569290ad956618f5ff6ef4857b633acb64f95126 https://codeload.github.com/minio/minio/tar.gz/01ce918d8279a20e4706b96a64396146894adee4 /tmp/source.tar.gz + + + ---> Using cache + ---> 4a2b9310629e +Step 3/25 : RUN mkdir /src && tar -xzf /tmp/source.tar.gz -C /src --strip-components=1 + ---> Using cache + ---> 5166de215d05 +Step 4/25 : WORKDIR /src + ---> Using cache + ---> 1f12c7444ff0 +Step 5/25 : RUN CGO_ENABLED=0 go build -trimpath -ldflags='-s -w' -o /out/minio . + ---> Using cache + ---> 8477e9d7324c +Step 6/25 : RUN echo '8b11129f6a6830768bcdebab569290ad956618f5ff6ef4857b633acb64f95126 /tmp/source.tar.gz' | sha256sum -c - + ---> Using cache + ---> 94162de69c8c +Step 7/25 : FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS mc-builder + ---> 18b460dd1754 +Step 8/25 : ADD --checksum=sha256:4cea8ed209c8f96e59ea1a5cc4901019b54c2be6d6f5f16c6749ea8bdcf4172e https://codeload.github.com/minio/mc/tar.gz/d6541ea280b73a834b64d4097e21f2be77676104 /tmp/source.tar.gz + + + ---> Using cache + ---> 5e135fc72a2c +Step 9/25 : RUN mkdir /src && tar -xzf /tmp/source.tar.gz -C /src --strip-components=1 + ---> Using cache + ---> cbffdcf35dd5 +Step 10/25 : WORKDIR /src + ---> Using cache + ---> 3322378e2ebd +Step 11/25 : RUN CGO_ENABLED=0 go build -trimpath -ldflags='-s -w' -o /out/mc . + ---> Using cache + ---> 0aba858f1d48 +Step 12/25 : RUN echo '4cea8ed209c8f96e59ea1a5cc4901019b54c2be6d6f5f16c6749ea8bdcf4172e /tmp/source.tar.gz' | sha256sum -c - + ---> Using cache + ---> af0a57fdfed1 +Step 13/25 : FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 AS minio + ---> 075c2c1a4068 +Step 14/25 : LABEL org.opencontainers.image.source="https://github.com/minio/minio" org.opencontainers.image.revision="01ce918d8279a20e4706b96a64396146894adee4" org.opencontainers.image.version="RELEASE.2025-09-07T16-13-09Z" org.opencontainers.image.licenses="AGPL-3.0" + ---> Using cache + ---> 57200264604d +Step 15/25 : RUN mkdir /data && chown 10001:10001 /data + ---> Using cache + ---> c7024406f9a9 +Step 16/25 : COPY --from=minio-builder /out/minio /usr/local/bin/minio + ---> Using cache + ---> 0d1965ab7125 +Step 17/25 : COPY infra/minio-entrypoint.sh /usr/local/bin/start-minio + ---> Using cache + ---> e5c21da70a4f +Step 18/25 : USER 10001:10001 + ---> Using cache + ---> 9bc52122d9f2 +Step 19/25 : EXPOSE 9000 + ---> Using cache + ---> 2ca448e77273 +Step 20/25 : ENTRYPOINT ["/usr/local/bin/start-minio"] + ---> Using cache + ---> 3a0797aa5ead +Step 21/25 : FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 AS mc + ---> 075c2c1a4068 +Step 22/25 : LABEL org.opencontainers.image.source="https://github.com/minio/mc" org.opencontainers.image.revision="d6541ea280b73a834b64d4097e21f2be77676104" org.opencontainers.image.version="RELEASE.2025-08-13T08-35-41Z" org.opencontainers.image.licenses="AGPL-3.0" + ---> Using cache + ---> 9257dca2de26 +Step 23/25 : COPY --from=mc-builder /out/mc /usr/local/bin/mc + ---> Using cache + ---> 4bea21e69e8d +Step 24/25 : USER 10001:10001 + ---> Using cache + ---> 45d00b9fce3e +Step 25/25 : ENTRYPOINT ["mc"] + ---> Using cache + ---> 0713832dbf4a +Successfully built 0713832dbf4a +Successfully tagged mtusi-mc:RELEASE.2025-08-13T08-35-41Z +DEPRECATED: The legacy builder is deprecated and will be removed in a future release. + Install the buildx component to build images with BuildKit: + https://docs.docker.com/go/buildx/ + +Sending build context to Docker daemon 398.3kB +Step 1/35 : FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS goose-builder + ---> 18b460dd1754 +Step 2/35 : RUN GOBIN=/out CGO_ENABLED=0 go install -tags='no_clickhouse no_libsql no_mssql no_mysql no_sqlite3 no_vertica no_ydb no_spanner' github.com/pressly/goose/v3/cmd/goose@v3.27.1 + ---> Using cache + ---> 7012ef857c0c +Step 3/35 : FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 AS migrations + ---> 075c2c1a4068 +Step 4/35 : COPY --from=goose-builder /out/goose /usr/local/bin/goose + ---> Using cache + ---> 6a0ad52336c1 +Step 5/35 : COPY migrations /migrations + ---> Using cache + ---> 19018c7bfac9 +Step 6/35 : COPY infra/migrate.sh /usr/local/bin/migrate + ---> Using cache + ---> 0cc075ba3459 +Step 7/35 : USER 10001:10001 + ---> Using cache + ---> 37de51af32cd +Step 8/35 : ENTRYPOINT ["/usr/local/bin/migrate"] + ---> Using cache + ---> e7539a967679 +Step 9/35 : CMD ["up"] + ---> Using cache + ---> 841d9bb239cd +Step 10/35 : FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS builder + ---> 18b460dd1754 +Step 11/35 : WORKDIR /src + ---> Using cache + ---> 1ba013278a4a +Step 12/35 : RUN apk add --no-cache gcc musl-dev + ---> Using cache + ---> a0ea65f0bded +Step 13/35 : COPY go.mod go.sum ./ + ---> Using cache + ---> e98ccecfb093 +Step 14/35 : RUN go mod download + ---> Using cache + ---> 91d546c71507 +Step 15/35 : COPY cmd ./cmd + ---> Using cache + ---> b4e431a9d5f2 +Step 16/35 : COPY contracts ./contracts + ---> Using cache + ---> 958477d6c98e +Step 17/35 : COPY dto ./dto + ---> Using cache + ---> d115fe2e1af4 +Step 18/35 : COPY internal ./internal + ---> Using cache + ---> 72552b328db4 +Step 19/35 : COPY migrations ./migrations + ---> Using cache + ---> a68ac4a31860 +Step 20/35 : RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/messenger ./cmd + ---> Using cache + ---> 7303ceff230d +Step 21/35 : RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/reconcile ./cmd/reconcile + ---> Using cache + ---> e6cc7991dac9 +Step 22/35 : FROM builder AS tests + ---> e6cc7991dac9 +Step 23/35 : COPY --from=goose-builder /out/goose /usr/local/bin/goose + ---> Using cache + ---> 5749aa2abe35 +Step 24/35 : ENV GOOSE_MIGRATION_DIR=/src/migrations + ---> Using cache + ---> ca0a6f733ba8 +Step 25/35 : CMD ["go", "test", "-p=1", "-race", "-count=1", "-v", "./..."] + ---> Using cache + ---> c643755b8aac +Step 26/35 : FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 + ---> 075c2c1a4068 +Step 27/35 : RUN addgroup -S messenger && adduser -S -D -H -u 10001 -G messenger messenger + ---> Using cache + ---> 33192b561bfc +Step 28/35 : COPY --from=builder --chown=messenger:messenger /out/messenger /usr/local/bin/messenger + ---> Using cache + ---> 136320eda1d3 +Step 29/35 : COPY --from=builder --chown=messenger:messenger /out/reconcile /usr/local/bin/reconcile + ---> Using cache + ---> 4acf2a8fc638 +Step 30/35 : ENV PORT=8080 + ---> Using cache + ---> 6fbea06a8135 +Step 31/35 : EXPOSE 8080 + ---> Using cache + ---> 333a62acf202 +Step 32/35 : USER messenger:messenger + ---> Using cache + ---> 60a29210a7f2 +Step 33/35 : HEALTHCHECK --interval=10s --timeout=3s --start-period=3s --retries=3 CMD wget -q -O /dev/null "http://127.0.0.1:${PORT}/health/ready" || exit 1 + ---> Using cache + ---> 803ac1f944e5 +Step 34/35 : STOPSIGNAL SIGTERM + ---> Using cache + ---> 40299d0a0f9f +Step 35/35 : ENTRYPOINT ["/usr/local/bin/messenger"] + ---> Using cache + ---> c81ae0702578 +Successfully built c81ae0702578 +Successfully tagged mtusi-messenger:lab4 +DEPRECATED: The legacy builder is deprecated and will be removed in a future release. + Install the buildx component to build images with BuildKit: + https://docs.docker.com/go/buildx/ + +Sending build context to Docker daemon 398.3kB +Step 1/9 : FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS goose-builder + ---> 18b460dd1754 +Step 2/9 : RUN GOBIN=/out CGO_ENABLED=0 go install -tags='no_clickhouse no_libsql no_mssql no_mysql no_sqlite3 no_vertica no_ydb no_spanner' github.com/pressly/goose/v3/cmd/goose@v3.27.1 + ---> Using cache + ---> 7012ef857c0c +Step 3/9 : FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 AS migrations + ---> 075c2c1a4068 +Step 4/9 : COPY --from=goose-builder /out/goose /usr/local/bin/goose + ---> Using cache + ---> 6a0ad52336c1 +Step 5/9 : COPY migrations /migrations + ---> Using cache + ---> 19018c7bfac9 +Step 6/9 : COPY infra/migrate.sh /usr/local/bin/migrate + ---> Using cache + ---> 0cc075ba3459 +Step 7/9 : USER 10001:10001 + ---> Using cache + ---> 37de51af32cd +Step 8/9 : ENTRYPOINT ["/usr/local/bin/migrate"] + ---> Using cache + ---> e7539a967679 +Step 9/9 : CMD ["up"] + ---> Using cache + ---> 841d9bb239cd +Successfully built 841d9bb239cd +Successfully tagged mtusi-migrations:lab4 +kkk3r78zn9bb09xudyehchs4p +mtusi-lab4_migrations.0.mzx2ds4dsbyh@colima | 2026/10/28 19:14:47 OK 001_initial.sql (17.71ms) +mtusi-lab4_migrations.0.mzx2ds4dsbyh@colima | 2026/10/28 19:14:47 OK 002_idempotency.sql (2.54ms) +mtusi-lab4_migrations.0.mzx2ds4dsbyh@colima | 2026/10/28 19:14:47 OK 003_attachments.sql (5.13ms) +mtusi-lab4_migrations.0.mzx2ds4dsbyh@colima | 2026/10/28 19:14:47 OK 004_message_attachments.sql (2.34ms) +mtusi-lab4_migrations.0.mzx2ds4dsbyh@colima | 2026/10/28 19:14:47 goose: successfully migrated database to version: 4 +mtusi-lab4_migrations +Since --detach=false was not specified, tasks will be created in the background. +In a future release, --detach=false will become the default. +Updating service mtusi-lab4_redis (id: zlhenabo1vgqarsazgqa6d897) +Updating service mtusi-lab4_minio (id: ygvoo1fje6vqlxw02ugw7awa8) +Updating service mtusi-lab4_api (id: xiueb8lcewfukmzn8wocqcmjh) +Updating service mtusi-lab4_lb (id: uo65o8allz5azrqbx4zchi811) +Updating service mtusi-lab4_postgres (id: vxi6yu73ah85o0xog5w8r1t5k) +f6qbximmjnxx8ezv05jxgsngz +mtusi-lab4_s3-bootstrap.0.p9vksdlsfz2l@colima | PASS: private bucket and restricted API account configured. +mtusi-lab4_s3-bootstrap +mtusi-lab4_api scaled to 2 +mtusi-lab4_api +PASS: rolling update completed +PASS: Swarm stack ready at http://localhost:8080 diff --git a/evidence/lab4-environment.json b/evidence/lab4-environment.json new file mode 100644 index 00000000..d2856953 --- /dev/null +++ b/evidence/lab4-environment.json @@ -0,0 +1,81 @@ +{ + "docker_engine": { + "cpus": 2, + "memory_bytes": 2053644288, + "os": "Ubuntu 24.04.4 LTS", + "kernel": "6.8.0-117-generic", + "architecture": "aarch64", + "version": "29.5.2" + }, + "host": { + "architecture": "arm64", + "system": "Darwin", + "model": "MacBookPro18,3", + "cpu_cores": 8, + "memory_bytes": 17179869184 + }, + "host_go": "go version go1.27.2 darwin/arm64", + "host_python": "3.9.6", + "images": [ + { + "tag": "mtusi-messenger:lab4", + "image_id": "sha256:c28dcd6e4410ee011a827852286b93296c8a0f20ddb268524b4d92371014403e", + "repo_digests": [ + "mtusi-messenger@sha256:c28dcd6e4410ee011a827852286b93296c8a0f20ddb268524b4d92371014403e" + ], + "source_labels": {} + }, + { + "tag": "mtusi-minio:RELEASE.2025-09-07T16-13-09Z", + "image_id": "sha256:3a0797aa5eadafe23e8a7b205c39511a9280b7500da6ef0e5d604cc74f46c3ed", + "repo_digests": [ + "mtusi-minio@sha256:3a0797aa5eadafe23e8a7b205c39511a9280b7500da6ef0e5d604cc74f46c3ed" + ], + "source_labels": { + "org.opencontainers.image.licenses": "AGPL-3.0", + "org.opencontainers.image.revision": "01ce918d8279a20e4706b96a64396146894adee4", + "org.opencontainers.image.source": "https://github.com/minio/minio", + "org.opencontainers.image.version": "RELEASE.2025-09-07T16-13-09Z" + } + }, + { + "tag": "mtusi-mc:RELEASE.2025-08-13T08-35-41Z", + "image_id": "sha256:0713832dbf4ab9a3b716ab0a4cca6629fec96224bc1425f26ab6ba42c5463a99", + "repo_digests": [ + "mtusi-mc@sha256:0713832dbf4ab9a3b716ab0a4cca6629fec96224bc1425f26ab6ba42c5463a99" + ], + "source_labels": { + "org.opencontainers.image.licenses": "AGPL-3.0", + "org.opencontainers.image.revision": "d6541ea280b73a834b64d4097e21f2be77676104", + "org.opencontainers.image.source": "https://github.com/minio/mc", + "org.opencontainers.image.version": "RELEASE.2025-08-13T08-35-41Z" + } + }, + { + "tag": "postgres:17.6-alpine3.22", + "image_id": "sha256:ef257d85f76e48da1c64832459b59fcaba1a4dac97bf5d7450c77753542eee94", + "repo_digests": [ + "postgres@sha256:ef257d85f76e48da1c64832459b59fcaba1a4dac97bf5d7450c77753542eee94" + ], + "source_labels": {} + }, + { + "tag": "redis:7.4.5-alpine3.21", + "image_id": "sha256:bb186d083732f669da90be8b0f975a37812b15e913465bb14d845db72a4e3e08", + "repo_digests": [ + "redis@sha256:bb186d083732f669da90be8b0f975a37812b15e913465bb14d845db72a4e3e08" + ], + "source_labels": {} + }, + { + "tag": "haproxy:3.2.6-alpine", + "image_id": "sha256:3a819392f5a6af2a9203d1c63ff0feba74f0c940623a558511be06d7d85e361c", + "repo_digests": [ + "haproxy@sha256:3a819392f5a6af2a9203d1c63ff0feba74f0c940623a558511be06d7d85e361c" + ], + "source_labels": {} + } + ], + "go_container_version": "1.26.4", + "test_data": "synthetic; no credentials/tokens saved" +} diff --git a/evidence/lab4-failover.txt b/evidence/lab4-failover.txt new file mode 100644 index 00000000..946745b7 --- /dev/null +++ b/evidence/lab4-failover.txt @@ -0,0 +1,4 @@ +.venv/bin/python tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 112, "failed_attempts": 2, "first_recovery_seconds": 2.061} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/lab4-go-tests.txt b/evidence/lab4-go-tests.txt new file mode 100644 index 00000000..d915cb26 --- /dev/null +++ b/evidence/lab4-go-tests.txt @@ -0,0 +1,13 @@ +? gitflic.ru/akselbant/mtusi2026lab1-fork/cmd [no test files] +? gitflic.ru/akselbant/mtusi2026lab1-fork/cmd/reconcile [no test files] +? gitflic.ru/akselbant/mtusi2026lab1-fork/contracts [no test files] +ok gitflic.ru/akselbant/mtusi2026lab1-fork/dto (cached) +ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/app/chat 13.184s +ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/attachment 2.905s +? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/config [no test files] +? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain [no test files] +ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/password (cached) +? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/repository [no test files] +ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/repository/postgres 2.388s +? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/session [no test files] +ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/usecase/chat (cached) diff --git a/evidence/lab4-integration.txt b/evidence/lab4-integration.txt new file mode 100644 index 00000000..9b82a00d --- /dev/null +++ b/evidence/lab4-integration.txt @@ -0,0 +1,338 @@ +.venv/bin/python scripts/swarm.py test-integration +DEPRECATED: The legacy builder is deprecated and will be removed in a future release. + Install the buildx component to build images with BuildKit: + https://docs.docker.com/go/buildx/ + +Sending build context to Docker daemon 398.3kB +Step 1/25 : FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS goose-builder + ---> 18b460dd1754 +Step 2/25 : RUN GOBIN=/out CGO_ENABLED=0 go install -tags='no_clickhouse no_libsql no_mssql no_mysql no_sqlite3 no_vertica no_ydb no_spanner' github.com/pressly/goose/v3/cmd/goose@v3.27.1 + ---> Using cache + ---> 7012ef857c0c +Step 3/25 : FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 AS migrations + ---> 075c2c1a4068 +Step 4/25 : COPY --from=goose-builder /out/goose /usr/local/bin/goose + ---> Using cache + ---> 6a0ad52336c1 +Step 5/25 : COPY migrations /migrations + ---> Using cache + ---> 19018c7bfac9 +Step 6/25 : COPY infra/migrate.sh /usr/local/bin/migrate + ---> Using cache + ---> 0cc075ba3459 +Step 7/25 : USER 10001:10001 + ---> Using cache + ---> 37de51af32cd +Step 8/25 : ENTRYPOINT ["/usr/local/bin/migrate"] + ---> Using cache + ---> e7539a967679 +Step 9/25 : CMD ["up"] + ---> Using cache + ---> 841d9bb239cd +Step 10/25 : FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS builder + ---> 18b460dd1754 +Step 11/25 : WORKDIR /src + ---> Using cache + ---> 1ba013278a4a +Step 12/25 : RUN apk add --no-cache gcc musl-dev + ---> Using cache + ---> a0ea65f0bded +Step 13/25 : COPY go.mod go.sum ./ + ---> Using cache + ---> e98ccecfb093 +Step 14/25 : RUN go mod download + ---> Using cache + ---> 91d546c71507 +Step 15/25 : COPY cmd ./cmd + ---> Using cache + ---> b4e431a9d5f2 +Step 16/25 : COPY contracts ./contracts + ---> Using cache + ---> 958477d6c98e +Step 17/25 : COPY dto ./dto + ---> Using cache + ---> d115fe2e1af4 +Step 18/25 : COPY internal ./internal + ---> Using cache + ---> 72552b328db4 +Step 19/25 : COPY migrations ./migrations + ---> Using cache + ---> a68ac4a31860 +Step 20/25 : RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/messenger ./cmd + ---> Using cache + ---> 7303ceff230d +Step 21/25 : RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/reconcile ./cmd/reconcile + ---> Using cache + ---> e6cc7991dac9 +Step 22/25 : FROM builder AS tests + ---> e6cc7991dac9 +Step 23/25 : COPY --from=goose-builder /out/goose /usr/local/bin/goose + ---> Using cache + ---> 5749aa2abe35 +Step 24/25 : ENV GOOSE_MIGRATION_DIR=/src/migrations + ---> Using cache + ---> ca0a6f733ba8 +Step 25/25 : CMD ["go", "test", "-p=1", "-race", "-count=1", "-v", "./..."] + ---> Using cache + ---> c643755b8aac +Successfully built c643755b8aac +Successfully tagged mtusi-messenger-tests:lab4 +qs9c5xskdu6nyjiv287eg68b8 +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/cmd [no test files] +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/cmd/reconcile [no test files] +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/contracts [no test files] +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestMessageStrictFields +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestMessageStrictFields (0.00s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestTimestampFixedMicroseconds +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestTimestampFixedMicroseconds (0.00s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | PASS +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | ok gitflic.ru/akselbant/mtusi2026lab1-fork/dto 1.013s +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestRejectsMalformedJSON +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:40 request_id=94533522-1e48-4953-9857-91c107c2aac7 method=POST path=/api/v1/users status=400 response_bytes=140 latency=130.671µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestRejectsMalformedJSON (0.01s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestHidesForeignChat +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:40 request_id=8b136f4b-a1b6-4756-9dda-92871979845c method=POST path=/api/v1/users status=201 response_bytes=141 latency=369.112982ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:41 request_id=df98ca4b-5f1f-4ef1-a8eb-7708ccfe2367 method=POST path=/api/v1/auth/sessions status=201 response_bytes=269 latency=397.145342ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:41 request_id=c535a54a-6ce6-49b0-8ec7-47e70ea04799 method=POST path=/api/v1/users status=201 response_bytes=147 latency=364.216754ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:41 request_id=fea0410a-7349-4fc3-8541-f479066736b7 method=POST path=/api/v1/auth/sessions status=201 response_bytes=275 latency=359.930021ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:41 request_id=b7201791-c957-4ee8-8237-2aab675777ec method=POST path=/api/v1/chats status=201 response_bytes=163 latency=174.381µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:41 request_id=30b379c6-e33c-42bc-963a-54de252a1e44 method=GET path=/api/v1/chats/:chat_id status=404 response_bytes=126 latency=53.002µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestHidesForeignChat (1.50s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestRejectsEmptyMessage +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:42 request_id=92d81694-3fcf-452b-be3d-24e21f938fb8 method=POST path=/api/v1/users status=201 response_bytes=157 latency=411.707889ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:42 request_id=792cc573-4974-4744-86bc-7ce29f78c5a2 method=POST path=/api/v1/auth/sessions status=201 response_bytes=285 latency=357.462544ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:42 request_id=4aec085d-f3fe-4b56-a0f2-b6bb55c32fd5 method=POST path=/api/v1/chats status=201 response_bytes=163 latency=135.922µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:42 request_id=bae49af4-9cb9-414e-8227-7991f680a0ab method=POST path=/api/v1/chats/:chat_id/messages status=400 response_bytes=172 latency=150.047µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestRejectsEmptyMessage (0.77s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestRejectsDuplicateUsername +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:42 request_id=c6f9b126-b2a0-4672-b4de-a59183cffe86 method=POST path=/api/v1/users status=201 response_bytes=149 latency=370.366912ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:43 request_id=c174982c-6a8d-4fc7-8d4b-6cbe68b34d70 method=POST path=/api/v1/auth/sessions status=201 response_bytes=277 latency=365.829696ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:43 request_id=b8771b83-4e05-4335-a6bd-4de115eb5e20 method=POST path=/api/v1/users status=409 response_bytes=170 latency=358.448604ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestRejectsDuplicateUsername (1.10s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestPaginatesMessagesAcrossMultiplePages +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:44 request_id=fb5f658b-7688-4017-9758-bea1d60af1cf method=POST path=/api/v1/users status=201 response_bytes=151 latency=366.583521ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:44 request_id=ddf8c2e9-34d2-4b17-aaa5-66ee679b9a4b method=POST path=/api/v1/auth/sessions status=201 response_bytes=279 latency=393.325397ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:44 request_id=32abde1e-d2e7-46fe-ad10-210533f83bd5 method=POST path=/api/v1/chats status=201 response_bytes=163 latency=136.838µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:44 request_id=227b702f-3d9d-4307-a1a8-30ab4f7aa8f2 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=217.05µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:44 request_id=edc8c38d-6beb-4353-b6c1-d1c8f337282b method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=132.629µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:44 request_id=27b25c6f-8b85-4f74-9a26-bc175b6e490f method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=120.337µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:44 request_id=148f8872-b793-4898-a57f-78f66d57fbb6 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=115.879µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:44 request_id=d2b4fae7-2633-4406-920a-af6bad40e7c9 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=113.254µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:44 request_id=65190391-2026-4f9e-b907-b52379ad8afe method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=141.005µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:44 request_id=094c52b2-7bb7-4ba0-be47-2695d64f4867 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=181.464µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:44 request_id=ebf5592b-9a0e-4ea4-8fa7-b7d2eb14639e method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=725 latency=304.594µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:44 request_id=c0d34ec3-f7c3-4639-8160-d6e32d09ff42 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=725 latency=180.132µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:44 request_id=0e2daae8-eec7-4e5e-a222-2f11927517a0 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=725 latency=168.339µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:44 request_id=b8dc3150-f39e-4d7b-a0af-38ffa5a8936d method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=259 latency=110.796µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestPaginatesMessagesAcrossMultiplePages (0.77s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestConcurrentMessagesAreUnique +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:44 request_id=a35f6c93-191f-4fe2-8640-6390ceab09fc method=POST path=/api/v1/users status=201 response_bytes=153 latency=372.343576ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:45 request_id=4dd49805-f2ef-4166-b188-ebbbc1924d9a method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=371.622652ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:45 request_id=745e9e01-1f34-4e95-81fe-6a3ae7cf9ea4 method=POST path=/api/v1/users status=201 response_bytes=153 latency=394.884355ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:45 request_id=95ac49aa-2560-42da-90c1-902d76341ccd method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=358.911214ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:46 request_id=249bef9d-e9fb-462a-99c3-1946037d1d0c method=POST path=/api/v1/users status=201 response_bytes=153 latency=374.597959ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:46 request_id=ef3349aa-4db5-4ab6-998e-50fa2f3262bb method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=364.93841ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:47 request_id=3007dbff-cdb7-42b9-ac50-77dc7c2ebc4f method=POST path=/api/v1/users status=201 response_bytes=153 latency=396.618954ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:47 request_id=01f269ef-e3c5-4240-9070-37312feb0e6b method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=372.262695ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:47 request_id=ab7c6c66-e836-4704-a516-3c4b51322e74 method=POST path=/api/v1/users status=201 response_bytes=153 latency=358.500208ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:48 request_id=b5f8edd3-d534-401f-8e68-fac601795f3f method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=369.620285ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:48 request_id=1362a848-874b-4b9b-b7fe-735ae74cc76d method=POST path=/api/v1/users status=201 response_bytes=153 latency=401.141119ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:49 request_id=953f6f2b-9fd3-49c5-b660-94a56615c24c method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=376.048273ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:49 request_id=56532a32-ca19-4238-b539-486b339d0bbb method=POST path=/api/v1/users status=201 response_bytes=153 latency=365.0437ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:49 request_id=89c7cf01-2523-4ae6-9a85-dd1a76a61039 method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=362.393272ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:50 request_id=0af98040-fde9-4694-a646-c60f794e097f method=POST path=/api/v1/users status=201 response_bytes=153 latency=360.604883ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:50 request_id=92c16b19-7579-47aa-9fb0-a3595b644fc7 method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=373.615906ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:50 request_id=460293e1-83ce-4a90-9255-cea43854fe3f method=POST path=/api/v1/users status=201 response_bytes=153 latency=366.494027ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=f564332d-7da7-40c0-9842-20cf89412495 method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=375.265646ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=62b8365d-f837-4354-8383-1456f1b72154 method=POST path=/api/v1/users status=201 response_bytes=153 latency=358.727985ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=a2a37a46-5e62-41e6-bfa3-2bce4c2e3ad8 method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=395.19545ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=5267f89c-aaf1-47b4-8c6a-9714ca727915 method=POST path=/api/v1/chats status=201 response_bytes=514 latency=188.799µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=c5b24f54-0511-46a1-82e5-0061ec77cb32 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=208.924µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=f72ec60f-87ae-4bf6-b996-ce192a118119 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=159.047µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=8afe7b58-5dce-42a1-adf2-1145ac360d45 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=148.63µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=7b09b2e8-26b4-42af-bee4-016a8e269a6b method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=160.922µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=81276d1e-5a76-4f9b-a7aa-f9857012dc26 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=260.717µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=8904e3d2-be95-47dc-a2fa-d3eaf50a3578 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=142.714µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=a03ae327-db89-4091-b3af-578a51335de2 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=216.716µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=a498db7b-49fb-4935-bb51-891c908fa4f8 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=139.089µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=2c981e76-6d36-494c-89ba-3336d357095d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=140.255µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=b9b68e40-c117-4bc7-8f41-b16dd197571a method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=150.047µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=7223bf26-18da-419a-a853-3b50f5f04a4b method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=132.047µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=16cc739c-a6a1-4c4c-a0bb-ec393bab1534 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=162.423µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=0aef904a-3e95-41de-b5b4-0bff73ed6722 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=153.297µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=25534c61-60fe-4374-bfda-db1354ea6012 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=145.714µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=53c00161-1317-49e3-84db-eb7c99300efa method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=146.422µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=1adf4594-5a16-4773-bff0-1b097ab75276 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=133.63µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=78902daf-2bbe-4307-a4d3-e6e9042a6291 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=180.09µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=7d53cfa6-047a-48c8-8fef-59024e131a96 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=185.173µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=b2942d4f-3e6a-4c91-8dbf-a32c6af07f43 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=206.424µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=5760c94c-8ecd-4067-832b-11909584e440 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=266.593µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=2b89e564-7bdd-4d5f-9e78-78445d1cbf70 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=153.922µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=f8fe6364-dab9-4122-9364-cc8a4e8e3d75 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=208.757µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=3af65a8b-625c-4fc7-ac62-bfbc353ac7a6 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=151.214µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=755c991e-1d9e-4279-b187-d607b13823ff method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=159.422µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=c36f80e6-63d2-4dc5-a2ee-01e6ab546ef3 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=245.509µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=e03d69f4-1738-49f5-b579-cdf4a632cc45 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=153.131µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=05255c0a-5b54-4b53-b714-08f15260687e method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=180.423µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=70f18e4c-0cdf-40a9-9e45-048f73171b1d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=126.796µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=ed2a3480-d27e-44eb-bc85-bd088d0cd3a3 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=192.631µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=51c5b7ad-f0a6-4127-aca2-1d300b973d21 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=214.091µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=e8718af1-4fbd-4b3b-8189-9c066828773c method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=151.922µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=cd61dc79-f672-45c5-91a5-797ac5bd8680 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=170.214µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=e9226bfb-5572-4513-9a97-0e90e532cec1 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=129.838µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=8286f6ba-c4cc-4eb9-8b10-da07785cf32a method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=164.173µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=d85b2fd6-05bf-4f9b-818a-09ca9ae2df48 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=142.005µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=0ba6f4c9-0162-48d9-b610-bf28b56d39cd method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=144.339µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=296d49e0-2b8b-4493-9835-bee0c4f23261 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=243.426µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=9cf7e296-e25f-4d63-a1db-de1c262e2b8c method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=208.632µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=8fa7d17d-40d0-4dc6-a4bd-b6c514bceb4a method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=133.171µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=9e1b6e9b-7487-446f-b948-175f7d88e714 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=163.506µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=3cf21baa-455a-4dac-b8c2-732150d32af8 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=323.762µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=688c501c-30c9-46bc-ba4d-9d6a9202200d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=572.521µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=65eefba1-b010-405b-b534-8e30d577d755 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=503.852µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=988807ed-875a-4d13-be2c-eb083546e583 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=193.299µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=0ae6f088-d7cb-4d57-a3db-bedb1d222bc7 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=129.088µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=038588a3-1ed6-4c91-bb8c-de5e55113175 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=320.887µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=ff151126-07e7-4100-a2af-e9765d071ca2 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=762.652µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=4cf0b177-670d-41b1-aa74-dafd209611e3 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=202.549µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=f893e111-2ccc-49a7-a34e-38c8248d5f3b method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=145.422µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=ece5e0f5-e795-4d1e-8bfe-2722feed55f7 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=150.047µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=45f9158b-a65f-4527-a741-2a8b3af800ac method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=165.089µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=ddc311c8-9d3f-4977-a159-a4d1daff9bf9 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=166.298µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=a0ab6cc9-4a7d-471e-9b65-932e3089cbc7 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=165.84µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=76e8f478-d9dc-4443-9bb8-310b8e9dd45a method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=266.51µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=96cb53f0-70bd-45aa-b735-6b5003310461 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=353.721µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=f1dda2fb-4016-440f-9fb6-c95a024038e6 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=169.048µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=e553ec3c-0ab2-4007-8363-48f94f36f000 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=141.796µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=78043ec3-1ec7-4c62-970c-950382757345 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=187.298µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=6222f780-9cdf-4d85-b99b-2897e61cbc43 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=808.862µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=73702805-906e-46ed-b38a-0206be2646dc method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=695.567µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=1186e970-9cb0-4db8-a44e-6d823c940ba7 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=410.557µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=98aaedba-db19-4504-a633-8a057f94d54e method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=311.22µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=92a3bfc9-1577-4a57-9bdd-351208fd7d93 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.151249ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=015c2bd0-266c-478a-99c8-516c482aeebe method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=266.801µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=664b2b6c-39ae-46d5-bed9-72caa9acb7f2 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=249.759µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=286c54ea-04d6-4af2-b00e-8ab00720b2ac method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=273.885µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=89ab422c-a181-43b7-8e4c-cbcdfe3c18b0 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.459927ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=9bbdd304-3f76-4f00-85bb-3c964aefb553 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=334.178µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=52dcf62b-df81-4c8f-8b60-c54965aaa140 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.887567ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=68f8ee3e-6eec-4f40-9428-dbeff99fd58a method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.955945ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=efa4cdeb-c02b-4332-a745-9ba14b490f76 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=2.642094ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=73f2e45a-8d59-4540-b193-98f1a4dca76a method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=486.101µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=c74d1c30-bde3-46d2-9b37-566419bf5376 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=507.602µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=65789068-cb57-4684-82d8-e201be139913 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=2.130868ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=fcffe674-7c32-4ee1-b34f-3841ac041030 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=191.966µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=5ea31e02-1c46-4305-ad02-1fa2217b9738 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=4.155481ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=b281c62b-cbeb-4e81-9c96-40757c5a88bf method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=150.547µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=5aceee05-4b98-45ca-974a-1e919fdde991 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=131.963µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=f82374a1-7d40-403d-aa70-5de8f913d2cc method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=167.798µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=5a725d61-e66a-445b-9b40-57ba6183a195 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=207.841µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=8ab66e91-8556-454b-913e-7b2c130dfd8e method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=152.631µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=423f02cb-0169-42fd-a1ed-5b6952cb262d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=132.88µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=7da7d9d5-13e8-4495-9ec8-18ef6966bd41 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=160.839µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=870557ee-8823-4a7a-b78e-a440a1d8a0f7 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=146.588µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=52f91333-d9c9-4365-9453-cdfcb0c3f5a7 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=146.172µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=af8b17fd-7edc-4208-a40d-84ca21e160f3 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=167.798µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=d4c625e7-5ebf-4eea-8d60-45af463edf4c method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=165.798µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=083768b9-8f2a-4681-bcd4-5ae52228e0bb method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=186.59µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=46101c43-7641-481a-869d-3543f25ef54d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=436.099µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=ba6ff758-a13f-4580-8ba2-da63a2351ca8 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=162.255µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=e6379b05-1acd-4b5a-b5a6-99e7bccc5f01 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=134.672µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=d9fb6747-2b9e-41cf-b53c-1a7750fc91f0 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=202.966µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=e5e5ec36-88eb-412f-8c2c-3f82c87c4590 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=171.006µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=8c2d9c31-df32-49d8-9c38-780f4fab1746 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=184.465µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=95fd52ea-b8d8-45df-a68a-ceba9d27b133 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=135.922µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=ae9ee763-096d-480c-b24b-cd349cb18978 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=384.431µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=088337f5-bb46-4911-80fd-fa855dc98200 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=145.463µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=2aa7e674-07c8-4282-8798-fcf641d08133 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=138.797µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=e0339717-77bb-4bd7-a321-34eaf1984fb9 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=163.923µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=74331de8-09a1-4057-8604-bd04d68465f4 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=145.922µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=da1793af-1f78-46eb-8751-52961c9bd4af method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=4332 latency=865.156µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=41cc821a-71d8-48f3-a2ec-618e8d6370b3 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=4332 latency=796.404µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=58db51d8-2b41-4a61-9f48-e28a79b88470 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=4332 latency=728.859µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=6b05dcf0-ee67-43cf-96d4-ef406c94761e method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=4332 latency=616.272µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=8bbd35fc-09c2-4ede-af22-3fb7147d5620 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=4332 latency=611.938µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=10bfa4c2-537d-442a-8e5b-0da397c81cc0 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=3616 latency=446.391µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestConcurrentMessagesAreUnique (7.52s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | PASS +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/app/chat 12.755s +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestSpoolOriginalAndFullValidation +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestSpoolOriginalAndFullValidation (0.00s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestSpoolLimitsBeforeDecode +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestSpoolLimitsBeforeDecode (0.02s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestFilenameAndCancelledUpload +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestFilenameAndCancelledUpload (0.00s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestReconcileRejectsUnsafeGrace +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestReconcileRejectsUnsafeGrace (0.00s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestValid16BitWidePNG +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestValid16BitWidePNG (0.08s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | PASS +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/attachment 1.160s +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/config [no test files] +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain [no test files] +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestUnicodeAndSalt +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestUnicodeAndSalt (2.23s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | PASS +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/password 3.266s +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/repository [no test files] +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestS3WireFixture +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestS3WireFixture (0.00s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestIntegration +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:84: goose up: 2026/10/28 19:17:59 OK 001_initial.sql (24.32ms) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:59 OK 002_idempotency.sql (1.98ms) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:59 OK 003_attachments.sql (4.14ms) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:59 OK 004_message_attachments.sql (1.77ms) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:59 goose: successfully migrated database to version: 4 +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:85: goose up: 2026/10/28 19:17:59 goose: no migrations to run. current version: 4 +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestIntegration/registration_race +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:121: 12 concurrent INSERTs: 1 success, 11 unique conflicts +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestIntegration/rollback +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:139: failed second member INSERT left 0 chats and 0 members +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestIntegration/lab4_attachments_and_reconciliation +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | lab4_test.go:248: ACL, original bytes, cross-chat attachment rejection, ordered/concurrent idempotency and absent=[] passed +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | lab4_test.go:316: dry-run={Apply:false GraceSeconds:90 ObjectsScanned:4 MetadataScanned:2 BytesScanned:94 ListRequestsEstimate:1 StatRequests:3 DBQueries:12 Orphans:1 Missing:1 FreshSkipped:1 LockedSkipped:1 ReferencedMissing:1 Deleted:0 MarkedFailed:0 DurationMS:7 Actions:[{Kind:orphan Key:original/9da5bf1b-bbc2-4c8e-8816-d473c908db4a Applied:false} {Kind:missing Key:original/65f5dff4-bc2f-469b-8b6b-6bc1275f8d6a Applied:false}]} apply={Apply:true GraceSeconds:90 ObjectsScanned:4 MetadataScanned:2 BytesScanned:94 ListRequestsEstimate:1 StatRequests:3 DBQueries:13 Orphans:1 Missing:1 FreshSkipped:1 LockedSkipped:1 ReferencedMissing:1 Deleted:1 MarkedFailed:1 DurationMS:5 Actions:[{Kind:orphan Key:original/9da5bf1b-bbc2-4c8e-8816-d473c908db4a Applied:true} {Kind:missing Key:original/65f5dff4-bc2f-469b-8b6b-6bc1275f8d6a Applied:true}]} repeat={Apply:true GraceSeconds:90 ObjectsScanned:3 MetadataScanned:2 BytesScanned:88 ListRequestsEstimate:1 StatRequests:1 DBQueries:7 Orphans:0 Missing:0 FreshSkipped:1 LockedSkipped:1 ReferencedMissing:0 Deleted:0 MarkedFailed:0 DurationMS:2 Actions:[]}; fresh, locked and referenced objects retained +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestIntegration/keyset_and_explain +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:193: EXPLAIN on 10,000 messages, cursor near the end (9,000): +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:199: Limit (cost=0.29..18.05 rows=51 width=142) (actual time=0.011..0.032 rows=51 loops=1) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:199: Buffers: shared hit=10 +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:199: -> Index Scan using messages_history_idx on messages m (cost=0.29..348.68 rows=1000 width=142) (actual time=0.011..0.028 rows=51 loops=1) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:199: Index Cond: ((chat_id = '3945a437-ac3b-4af6-a505-e156a979dd08'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.9+00'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid))) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:199: Buffers: shared hit=10 +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:199: Planning Time: 0.031 ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:199: Execution Time: 0.061 ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:204: keyset traversal returned 10,000 distinct messages, including equal timestamps +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestIntegration/idempotency_race +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:254: 16 concurrent PostgreSQL requests: one effect, identical replay, changed body conflicts +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestIntegration/goose_legacy_and_down +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:268: goose up: 2026/10/28 19:17:59 OK 001_initial.sql (1.62ms) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:59 OK 002_idempotency.sql (1.13ms) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:59 OK 003_attachments.sql (583.4µs) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:59 OK 004_message_attachments.sql (2.74ms) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:59 goose: successfully migrated database to version: 4 +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:269: goose up: 2026/10/28 19:17:59 goose: no migrations to run. current version: 4 +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:276: adopted legacy schema: all existing messages are unchanged +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:277: goose reset: 2026/10/28 19:17:59 OK 004_message_attachments.sql (5.92ms) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:59 OK 003_attachments.sql (3.96ms) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:59 OK 002_idempotency.sql (2.73ms) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:59 OK 001_initial.sql (9.78ms) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:282: goose up: 2026/10/28 19:17:59 OK 001_initial.sql (10.62ms) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:59 OK 002_idempotency.sql (1.69ms) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:59 OK 003_attachments.sql (3.22ms) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:59 OK 004_message_attachments.sql (1.21ms) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:59 goose: successfully migrated database to version: 4 +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:286: down and up succeeded in isolated schema +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestIntegration (0.64s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestIntegration/registration_race (0.04s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestIntegration/rollback (0.00s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestIntegration/lab4_attachments_and_reconciliation (0.04s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestIntegration/keyset_and_explain (0.08s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestIntegration/idempotency_race (0.01s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestIntegration/goose_legacy_and_down (0.12s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | PASS +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/repository/postgres 1.675s +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/session [no test files] +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestPasswordContract +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestPasswordContract (0.00s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | PASS +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/usecase/chat 1.011s +mtusi-lab4_tests diff --git a/evidence/lab4-persistence.txt b/evidence/lab4-persistence.txt new file mode 100644 index 00000000..4ba6d9c2 --- /dev/null +++ b/evidence/lab4-persistence.txt @@ -0,0 +1,3 @@ +.venv/bin/python tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/lab4-reconciliation.json b/evidence/lab4-reconciliation.json new file mode 100644 index 00000000..0a57b41d --- /dev/null +++ b/evidence/lab4-reconciliation.json @@ -0,0 +1,1226 @@ +{ + "hypothesis": "fixed grace plus per-object transaction lock permits repeatable repair without deleting fresh or linked data", + "grace_seconds": 90, + "faults": { + "s3_put_db_insert_failure": { + "upload_status": 503, + "objects_without_metadata": 1 + }, + "minio_unavailable": { + "exit_nonzero": true, + "summary": { + "apply": true, + "grace_seconds": 90, + "objects_scanned": 0, + "metadata_scanned": 0, + "bytes_scanned": 0, + "list_requests_estimate": 0, + "stat_requests": 0, + "db_queries": 0, + "orphans": 0, + "missing": 0, + "fresh_skipped": 0, + "locked_skipped": 0, + "referenced_missing": 0, + "deleted": 0, + "marked_failed": 0, + "duration_ms": 201, + "actions": [] + } + }, + "postgres_unavailable": { + "exit_nonzero": true, + "summary": { + "apply": true, + "grace_seconds": 90, + "objects_scanned": 1, + "metadata_scanned": 0, + "bytes_scanned": 837, + "list_requests_estimate": 1, + "stat_requests": 0, + "db_queries": 0, + "orphans": 0, + "missing": 0, + "fresh_skipped": 0, + "locked_skipped": 0, + "referenced_missing": 0, + "deleted": 0, + "marked_failed": 0, + "duration_ms": 48, + "actions": [] + } + } + }, + "cost": [ + { + "phase": "fresh", + "synthetic_objects": 100, + "apply": false, + "grace_seconds": 90, + "objects_scanned": 130, + "metadata_scanned": 30, + "bytes_scanned": 31825703, + "list_requests_estimate": 1, + "stat_requests": 7, + "db_queries": 30, + "orphans": 0, + "missing": 0, + "fresh_skipped": 146, + "locked_skipped": 0, + "referenced_missing": 0, + "deleted": 0, + "marked_failed": 0, + "duration_ms": 27, + "actions": [] + }, + { + "phase": "fresh", + "synthetic_objects": 1100, + "apply": false, + "grace_seconds": 90, + "objects_scanned": 1130, + "metadata_scanned": 30, + "bytes_scanned": 31826703, + "list_requests_estimate": 2, + "stat_requests": 7, + "db_queries": 30, + "orphans": 0, + "missing": 0, + "fresh_skipped": 1146, + "locked_skipped": 0, + "referenced_missing": 0, + "deleted": 0, + "marked_failed": 0, + "duration_ms": 64, + "actions": [] + }, + { + "phase": "aged", + "synthetic_objects": 1100, + "apply": false, + "grace_seconds": 90, + "objects_scanned": 1130, + "metadata_scanned": 30, + "bytes_scanned": 31826703, + "list_requests_estimate": 2, + "stat_requests": 1131, + "db_queries": 2323, + "orphans": 1101, + "missing": 1, + "fresh_skipped": 0, + "locked_skipped": 0, + "referenced_missing": 1, + "deleted": 0, + "marked_failed": 0, + "duration_ms": 999, + "actions": [ + { + "kind": "orphan", + "key": "original/337521a8-44be-4941-a1b1-0eec2ada2b3c", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/1", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/10", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/100", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/1000", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/101", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/102", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/103", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/104", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/105", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/106", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/107", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/108", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/109", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/11", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/110", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/111", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/112", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/113", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/114", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/115", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/116", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/117", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/118", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/119", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/12", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/120", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/121", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/122", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/123", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/124", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/125", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/126", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/127", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/128", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/129", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/13", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/130", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/131", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/132", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/133", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/134", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/135", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/136", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/137", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/138", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/139", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/14", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/140", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/141", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/142", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/143", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/144", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/145", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/146", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/147", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/148", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/149", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/15", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/150", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/151", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/152", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/153", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/154", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/155", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/156", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/157", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/158", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/159", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/16", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/160", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/161", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/162", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/163", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/164", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/165", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/166", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/167", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/168", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/169", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/17", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/170", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/171", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/172", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/173", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/174", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/175", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/176", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/177", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/178", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/179", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/18", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/180", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/181", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/182", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/183", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/184", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/185", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/186", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/187", + "applied": false + } + ] + }, + { + "phase": "aged", + "synthetic_objects": 100, + "apply": false, + "grace_seconds": 90, + "objects_scanned": 130, + "metadata_scanned": 30, + "bytes_scanned": 31825703, + "list_requests_estimate": 1, + "stat_requests": 131, + "db_queries": 323, + "orphans": 101, + "missing": 1, + "fresh_skipped": 0, + "locked_skipped": 0, + "referenced_missing": 1, + "deleted": 0, + "marked_failed": 0, + "duration_ms": 129, + "actions": [ + { + "kind": "orphan", + "key": "original/337521a8-44be-4941-a1b1-0eec2ada2b3c", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/1", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/10", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/100", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/11", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/12", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/13", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/14", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/15", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/16", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/17", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/18", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/19", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/2", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/20", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/21", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/22", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/23", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/24", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/25", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/26", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/27", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/28", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/29", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/3", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/30", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/31", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/32", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/33", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/34", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/35", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/36", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/37", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/38", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/39", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/4", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/40", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/41", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/42", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/43", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/44", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/45", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/46", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/47", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/48", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/49", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/5", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/50", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/51", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/52", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/53", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/54", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/55", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/56", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/57", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/58", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/59", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/6", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/60", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/61", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/62", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/63", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/64", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/65", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/66", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/67", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/68", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/69", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/7", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/70", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/71", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/72", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/73", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/74", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/75", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/76", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/77", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/78", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/79", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/8", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/80", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/81", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/82", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/83", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/84", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/85", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/86", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/87", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/88", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/89", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/9", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/90", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/91", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/92", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/93", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/94", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/95", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/96", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/97", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/98", + "applied": false + } + ] + } + ], + "fresh_orphan_preserved": true, + "fresh_metadata_preserved": true, + "locked_candidate_preserved": true, + "dry_run": { + "apply": false, + "grace_seconds": 90, + "objects_scanned": 30, + "metadata_scanned": 30, + "bytes_scanned": 31825603, + "list_requests_estimate": 1, + "stat_requests": 31, + "db_queries": 123, + "orphans": 1, + "missing": 1, + "fresh_skipped": 0, + "locked_skipped": 0, + "referenced_missing": 1, + "deleted": 0, + "marked_failed": 0, + "duration_ms": 54, + "actions": [ + { + "kind": "orphan", + "key": "original/337521a8-44be-4941-a1b1-0eec2ada2b3c", + "applied": false + }, + { + "kind": "missing", + "key": "original/780437df-ef67-4bc1-82e1-209c619b6a3a", + "applied": false + } + ] + }, + "apply": { + "apply": true, + "grace_seconds": 90, + "objects_scanned": 30, + "metadata_scanned": 30, + "bytes_scanned": 31825603, + "list_requests_estimate": 1, + "stat_requests": 31, + "db_queries": 124, + "orphans": 1, + "missing": 1, + "fresh_skipped": 0, + "locked_skipped": 0, + "referenced_missing": 1, + "deleted": 1, + "marked_failed": 1, + "duration_ms": 54, + "actions": [ + { + "kind": "orphan", + "key": "original/337521a8-44be-4941-a1b1-0eec2ada2b3c", + "applied": true + }, + { + "kind": "missing", + "key": "original/780437df-ef67-4bc1-82e1-209c619b6a3a", + "applied": true + } + ] + }, + "repeat_apply": { + "apply": true, + "grace_seconds": 90, + "objects_scanned": 29, + "metadata_scanned": 30, + "bytes_scanned": 31824766, + "list_requests_estimate": 1, + "stat_requests": 29, + "db_queries": 118, + "orphans": 0, + "missing": 0, + "fresh_skipped": 0, + "locked_skipped": 0, + "referenced_missing": 0, + "deleted": 0, + "marked_failed": 0, + "duration_ms": 40, + "actions": [] + }, + "live_linked_preserved": true, + "missing_linked_reference_preserved": true +} diff --git a/evidence/lab4-reconciliation.txt b/evidence/lab4-reconciliation.txt new file mode 100644 index 00000000..7a1e6fb5 --- /dev/null +++ b/evidence/lab4-reconciliation.txt @@ -0,0 +1,7 @@ +.venv/bin/python tests/own/reconcile.py +Fresh scan cost: {"objects_scanned": 130, "metadata_scanned": 30, "list_requests_estimate": 1, "stat_requests": 7, "db_queries": 30, "duration_ms": 27} +Fresh scan cost: {"objects_scanned": 1130, "metadata_scanned": 30, "list_requests_estimate": 2, "stat_requests": 7, "db_queries": 30, "duration_ms": 64} +Waiting92s for actual S3 timestamps: measuring aged-object locking, DB and HEAD cost next. +Aged scan cost: {"objects_scanned": 1130, "metadata_scanned": 30, "list_requests_estimate": 2, "stat_requests": 1131, "db_queries": 2323, "duration_ms": 999} +Aged scan cost: {"objects_scanned": 130, "metadata_scanned": 30, "list_requests_estimate": 1, "stat_requests": 131, "db_queries": 323, "duration_ms": 129} +PASS: real two-store faults, grace, lock, dry-run/apply/repeat, linked data preserved and paginated scan cost. diff --git a/evidence/lab4-restart-verification.json b/evidence/lab4-restart-verification.json new file mode 100644 index 00000000..9702392e --- /dev/null +++ b/evidence/lab4-restart-verification.json @@ -0,0 +1,54 @@ +{ + "api": { + "all_original_tasks_replaced": true, + "mounts_unchanged": true, + "before": [ + "05qp70hqicx1m3uten0t88c9p", + "1diq18xi0jmwpzqzhhq39hjad" + ], + "after": [ + "g2s34c3zuztbxehcxyypsyrxy", + "rz3qizaxp0s2bguzbs57me0op" + ] + }, + "lb": { + "all_original_tasks_replaced": true, + "mounts_unchanged": true, + "before": [ + "psq3iyz4kraq9u3n2mtkyfijf" + ], + "after": [ + "oaiciqwjik1afkel0hdgwvhwy" + ] + }, + "postgres": { + "all_original_tasks_replaced": true, + "mounts_unchanged": true, + "before": [ + "7xbetpe47jvp1uuu2di2uk7vy" + ], + "after": [ + "i3scdtmsi49zwytqye6hc3k15" + ] + }, + "redis": { + "all_original_tasks_replaced": true, + "mounts_unchanged": true, + "before": [ + "x8ejqk47qyz4rgtgygq48wos7" + ], + "after": [ + "tqq4m5ygv0nohjefvbdprapzz" + ] + }, + "minio": { + "all_original_tasks_replaced": true, + "mounts_unchanged": true, + "before": [ + "ww19llqoe6yjvteqx3by0i4r1" + ], + "after": [ + "lfmr13mco26ik0a37xk2ntene" + ] + } +} diff --git a/evidence/lab4-scaling.txt b/evidence/lab4-scaling.txt new file mode 100644 index 00000000..9dad5410 --- /dev/null +++ b/evidence/lab4-scaling.txt @@ -0,0 +1,6 @@ +.venv/bin/python tests/own/swarm_grade4.py +PASS: shared-key race on two replicas; replay/conflict/scope/validation/access/logout +{"action": "stop-one", "target": "rz3qizaxp0s2bguzbs57me0op", "duration_seconds": 15.458, "successful_writes": 1078, "failed_attempts": 7, "successful_rps": 69.739, "first_recovery_seconds": 1.349, "max_success_gap_seconds": 1.055, "instances_before": ["rz3qizaxp0s2bguzbs57me0op", "xcgz2hubz1xrzmrkoyrgwq0tq"], "instances_after": ["j7awgfnedlt9ne1esywrappxx", "xcgz2hubz1xrzmrkoyrgwq0tq"], "confirmed_messages_checked": 1229, "duplicate_bodies": 0} +{"action": "stop-one", "target": "xcgz2hubz1xrzmrkoyrgwq0tq", "duration_seconds": 15.467, "successful_writes": 1091, "failed_attempts": 7, "successful_rps": 70.537, "first_recovery_seconds": 1.384, "max_success_gap_seconds": 1.057, "instances_before": ["j7awgfnedlt9ne1esywrappxx", "xcgz2hubz1xrzmrkoyrgwq0tq"], "instances_after": ["5vy560hk6k0q3eicqodd9dp2z", "j7awgfnedlt9ne1esywrappxx"], "confirmed_messages_checked": 1251, "duplicate_bodies": 0} +{"action": "update", "target": null, "duration_seconds": 44.156, "successful_writes": 3362, "failed_attempts": 0, "successful_rps": 76.139, "first_recovery_seconds": null, "max_success_gap_seconds": 0.152, "instances_before": ["5vy560hk6k0q3eicqodd9dp2z", "j7awgfnedlt9ne1esywrappxx"], "instances_after": ["j201rqxerkxdq0vbcw3armc3m", "uqyjmps3u9fyl976lpj944pc5"], "confirmed_messages_checked": 3519, "duplicate_bodies": 0} +PASS: each original replica stopped and replaced; rolling update under load; confirmations and sessions preserved diff --git a/evidence/lab4-smoke.txt b/evidence/lab4-smoke.txt new file mode 100644 index 00000000..607ff815 --- /dev/null +++ b/evidence/lab4-smoke.txt @@ -0,0 +1,2 @@ +.venv/bin/python tests/smoke.py --base-url "http://localhost:8080" +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/lab4-status-after-restart.json b/evidence/lab4-status-after-restart.json new file mode 100644 index 00000000..be72be0f --- /dev/null +++ b/evidence/lab4-status-after-restart.json @@ -0,0 +1,146 @@ +{ + "stack": "mtusi-lab4", + "docker": "29.5.2", + "services": { + "lb": { + "image": "haproxy:3.2.6-alpine", + "limits": { + "NanoCPUs": 500000000, + "MemoryBytes": 134217728 + }, + "published_ports": [ + { + "Protocol": "tcp", + "TargetPort": 8080, + "PublishedPort": 8080, + "PublishMode": "host" + } + ], + "secret_names": [], + "mounts": [], + "tasks": [ + { + "id": "oaiciqwjik1afkel0hdgwvhwy", + "state": "running", + "container": "ace6b5a9ab2ac3a291bca557e7eb89ba6354ed586625332b99dc3d1cdfaab6b5" + } + ] + }, + "api": { + "image": "mtusi-messenger:lab4", + "limits": { + "NanoCPUs": 1000000000, + "MemoryBytes": 1610612736 + }, + "published_ports": [], + "secret_names": [ + "mtusi-lab4_cursor_key", + "mtusi-lab4_postgres_password", + "mtusi-lab4_s3_access_key", + "mtusi-lab4_s3_secret_key" + ], + "mounts": [], + "tasks": [ + { + "id": "g2s34c3zuztbxehcxyypsyrxy", + "state": "running", + "container": "303dc53972fde0fe71990f644cb1b56658441a4d1cb6410ffc351ccceb0cd3a7" + }, + { + "id": "rz3qizaxp0s2bguzbs57me0op", + "state": "running", + "container": "f69bf37c2237e13f08bf4aa397936d4fed84a17395410f0af469b2cbc3e41699" + } + ] + }, + "postgres": { + "image": "postgres:17.6-alpine3.22", + "limits": { + "NanoCPUs": 1000000000, + "MemoryBytes": 536870912 + }, + "published_ports": [], + "secret_names": [ + "mtusi-lab4_postgres_password" + ], + "mounts": [ + { + "Type": "volume", + "Source": "mtusi-lab4_postgres_data", + "Target": "/var/lib/postgresql/data", + "VolumeOptions": { + "Labels": { + "com.docker.stack.namespace": "mtusi-lab4" + } + } + } + ], + "tasks": [ + { + "id": "i3scdtmsi49zwytqye6hc3k15", + "state": "running", + "container": "1622ddce8784f7dc1092ba808f05583898e08ba57a5c390d227265e4243744dc" + } + ] + }, + "redis": { + "image": "redis:7.4.5-alpine3.21", + "limits": { + "NanoCPUs": 500000000, + "MemoryBytes": 134217728 + }, + "published_ports": [], + "secret_names": [], + "mounts": [ + { + "Type": "volume", + "Source": "mtusi-lab4_redis_data", + "Target": "/data", + "VolumeOptions": { + "Labels": { + "com.docker.stack.namespace": "mtusi-lab4" + } + } + } + ], + "tasks": [ + { + "id": "tqq4m5ygv0nohjefvbdprapzz", + "state": "running", + "container": "dfe7c35a3887061560c608484e33ce1036222d8f683a5729599fb0a4e33065f3" + } + ] + }, + "minio": { + "image": "mtusi-minio:RELEASE.2025-09-07T16-13-09Z", + "limits": { + "NanoCPUs": 1000000000, + "MemoryBytes": 536870912 + }, + "published_ports": [], + "secret_names": [ + "mtusi-lab4_s3_root_access_key", + "mtusi-lab4_s3_root_secret_key" + ], + "mounts": [ + { + "Type": "volume", + "Source": "mtusi-lab4_s3_data", + "Target": "/data", + "VolumeOptions": { + "Labels": { + "com.docker.stack.namespace": "mtusi-lab4" + } + } + } + ], + "tasks": [ + { + "id": "lfmr13mco26ik0a37xk2ntene", + "state": "running", + "container": "8110ff5d0c7a776005653182ecfd927fe59a8b38b02b4645e1bd03fa1c6fdeff" + } + ] + } + } +} diff --git a/evidence/lab4-status-before-restart.json b/evidence/lab4-status-before-restart.json new file mode 100644 index 00000000..e2dd3804 --- /dev/null +++ b/evidence/lab4-status-before-restart.json @@ -0,0 +1,146 @@ +{ + "stack": "mtusi-lab4", + "docker": "29.5.2", + "services": { + "lb": { + "image": "haproxy:3.2.6-alpine", + "limits": { + "NanoCPUs": 500000000, + "MemoryBytes": 134217728 + }, + "published_ports": [ + { + "Protocol": "tcp", + "TargetPort": 8080, + "PublishedPort": 8080, + "PublishMode": "host" + } + ], + "secret_names": [], + "mounts": [], + "tasks": [ + { + "id": "psq3iyz4kraq9u3n2mtkyfijf", + "state": "running", + "container": "c9246107dbeb5aa78d3574865ddb2e9ac2d117c87830ca7be5214bdc8e8768b4" + } + ] + }, + "api": { + "image": "mtusi-messenger:lab4", + "limits": { + "NanoCPUs": 1000000000, + "MemoryBytes": 1610612736 + }, + "published_ports": [], + "secret_names": [ + "mtusi-lab4_cursor_key", + "mtusi-lab4_postgres_password", + "mtusi-lab4_s3_access_key", + "mtusi-lab4_s3_secret_key" + ], + "mounts": [], + "tasks": [ + { + "id": "05qp70hqicx1m3uten0t88c9p", + "state": "running", + "container": "0beaa8438aa0f61c069729d0d3e83220df07e49288451658d3c92f698c1b7fbc" + }, + { + "id": "1diq18xi0jmwpzqzhhq39hjad", + "state": "running", + "container": "bc1d5078efd2145c84e341ae457360f490f32539d7e4f6d5155378c918da69c8" + } + ] + }, + "postgres": { + "image": "postgres:17.6-alpine3.22", + "limits": { + "NanoCPUs": 1000000000, + "MemoryBytes": 536870912 + }, + "published_ports": [], + "secret_names": [ + "mtusi-lab4_postgres_password" + ], + "mounts": [ + { + "Type": "volume", + "Source": "mtusi-lab4_postgres_data", + "Target": "/var/lib/postgresql/data", + "VolumeOptions": { + "Labels": { + "com.docker.stack.namespace": "mtusi-lab4" + } + } + } + ], + "tasks": [ + { + "id": "7xbetpe47jvp1uuu2di2uk7vy", + "state": "running", + "container": "7a24978349b4630091ba38d365fcd74b20c60bd9a27fc15d5cb967e3874e09de" + } + ] + }, + "redis": { + "image": "redis:7.4.5-alpine3.21", + "limits": { + "NanoCPUs": 500000000, + "MemoryBytes": 134217728 + }, + "published_ports": [], + "secret_names": [], + "mounts": [ + { + "Type": "volume", + "Source": "mtusi-lab4_redis_data", + "Target": "/data", + "VolumeOptions": { + "Labels": { + "com.docker.stack.namespace": "mtusi-lab4" + } + } + } + ], + "tasks": [ + { + "id": "x8ejqk47qyz4rgtgygq48wos7", + "state": "running", + "container": "33a57e377891ce0c82807b12757cf0103ef9461502d1a64efdeec506744d9c27" + } + ] + }, + "minio": { + "image": "mtusi-minio:RELEASE.2025-09-07T16-13-09Z", + "limits": { + "NanoCPUs": 1000000000, + "MemoryBytes": 536870912 + }, + "published_ports": [], + "secret_names": [ + "mtusi-lab4_s3_root_access_key", + "mtusi-lab4_s3_root_secret_key" + ], + "mounts": [ + { + "Type": "volume", + "Source": "mtusi-lab4_s3_data", + "Target": "/data", + "VolumeOptions": { + "Labels": { + "com.docker.stack.namespace": "mtusi-lab4" + } + } + } + ], + "tasks": [ + { + "id": "ww19llqoe6yjvteqx3by0i4r1", + "state": "running", + "container": "d71ce1a062fe9ba9060f5266a9f00d89a34583f133a609769bb2510deaa52470" + } + ] + } + } +} diff --git a/evidence/lab4-status-final.json b/evidence/lab4-status-final.json new file mode 100644 index 00000000..73fc9e08 --- /dev/null +++ b/evidence/lab4-status-final.json @@ -0,0 +1,146 @@ +{ + "stack": "mtusi-lab4", + "docker": "29.5.2", + "services": { + "lb": { + "image": "haproxy:3.2.6-alpine", + "limits": { + "NanoCPUs": 500000000, + "MemoryBytes": 134217728 + }, + "published_ports": [ + { + "Protocol": "tcp", + "TargetPort": 8080, + "PublishedPort": 8080, + "PublishMode": "host" + } + ], + "secret_names": [], + "mounts": [], + "tasks": [ + { + "id": "oaiciqwjik1afkel0hdgwvhwy", + "state": "running", + "container": "ace6b5a9ab2ac3a291bca557e7eb89ba6354ed586625332b99dc3d1cdfaab6b5" + } + ] + }, + "api": { + "image": "mtusi-messenger:lab4", + "limits": { + "NanoCPUs": 1000000000, + "MemoryBytes": 1610612736 + }, + "published_ports": [], + "secret_names": [ + "mtusi-lab4_cursor_key", + "mtusi-lab4_postgres_password", + "mtusi-lab4_s3_access_key", + "mtusi-lab4_s3_secret_key" + ], + "mounts": [], + "tasks": [ + { + "id": "uqyjmps3u9fyl976lpj944pc5", + "state": "running", + "container": "2f98c5810a5f8557476af364186cb66e02c46003fece8827e200313d27f2fd74" + }, + { + "id": "j201rqxerkxdq0vbcw3armc3m", + "state": "running", + "container": "68730a58baf57c19b9919e752e34b7ad247ff29d94918f28fb7ede200c1a95f1" + } + ] + }, + "postgres": { + "image": "postgres:17.6-alpine3.22", + "limits": { + "NanoCPUs": 1000000000, + "MemoryBytes": 536870912 + }, + "published_ports": [], + "secret_names": [ + "mtusi-lab4_postgres_password" + ], + "mounts": [ + { + "Type": "volume", + "Source": "mtusi-lab4_postgres_data", + "Target": "/var/lib/postgresql/data", + "VolumeOptions": { + "Labels": { + "com.docker.stack.namespace": "mtusi-lab4" + } + } + } + ], + "tasks": [ + { + "id": "i3scdtmsi49zwytqye6hc3k15", + "state": "running", + "container": "1622ddce8784f7dc1092ba808f05583898e08ba57a5c390d227265e4243744dc" + } + ] + }, + "redis": { + "image": "redis:7.4.5-alpine3.21", + "limits": { + "NanoCPUs": 500000000, + "MemoryBytes": 134217728 + }, + "published_ports": [], + "secret_names": [], + "mounts": [ + { + "Type": "volume", + "Source": "mtusi-lab4_redis_data", + "Target": "/data", + "VolumeOptions": { + "Labels": { + "com.docker.stack.namespace": "mtusi-lab4" + } + } + } + ], + "tasks": [ + { + "id": "tqq4m5ygv0nohjefvbdprapzz", + "state": "running", + "container": "dfe7c35a3887061560c608484e33ce1036222d8f683a5729599fb0a4e33065f3" + } + ] + }, + "minio": { + "image": "mtusi-minio:RELEASE.2025-09-07T16-13-09Z", + "limits": { + "NanoCPUs": 1000000000, + "MemoryBytes": 536870912 + }, + "published_ports": [], + "secret_names": [ + "mtusi-lab4_s3_root_access_key", + "mtusi-lab4_s3_root_secret_key" + ], + "mounts": [ + { + "Type": "volume", + "Source": "mtusi-lab4_s3_data", + "Target": "/data", + "VolumeOptions": { + "Labels": { + "com.docker.stack.namespace": "mtusi-lab4" + } + } + } + ], + "tasks": [ + { + "id": "lfmr13mco26ik0a37xk2ntene", + "state": "running", + "container": "8110ff5d0c7a776005653182ecfd927fe59a8b38b02b4645e1bd03fa1c6fdeff" + } + ] + } + } +} diff --git a/evidence/lab4-status-initial.json b/evidence/lab4-status-initial.json new file mode 100644 index 00000000..80982359 --- /dev/null +++ b/evidence/lab4-status-initial.json @@ -0,0 +1,146 @@ +{ + "stack": "mtusi-lab4", + "docker": "29.5.2", + "services": { + "lb": { + "image": "haproxy:3.2.6-alpine", + "limits": { + "NanoCPUs": 500000000, + "MemoryBytes": 134217728 + }, + "published_ports": [ + { + "Protocol": "tcp", + "TargetPort": 8080, + "PublishedPort": 8080, + "PublishMode": "host" + } + ], + "secret_names": [], + "mounts": [], + "tasks": [ + { + "id": "psq3iyz4kraq9u3n2mtkyfijf", + "state": "running", + "container": "c9246107dbeb5aa78d3574865ddb2e9ac2d117c87830ca7be5214bdc8e8768b4" + } + ] + }, + "api": { + "image": "mtusi-messenger:lab4", + "limits": { + "NanoCPUs": 1000000000, + "MemoryBytes": 1610612736 + }, + "published_ports": [], + "secret_names": [ + "mtusi-lab4_cursor_key", + "mtusi-lab4_postgres_password", + "mtusi-lab4_s3_access_key", + "mtusi-lab4_s3_secret_key" + ], + "mounts": [], + "tasks": [ + { + "id": "0gwv6ha33wg3dao0mqxl9gu0n", + "state": "running", + "container": "f37c59b0556d4bc5b3f5c0432e8c58da4888ca5286635954131ea37c0714ead1" + }, + { + "id": "rbtm0mbmnbzr2oqcz8icho3qr", + "state": "running", + "container": "17d5f64c6fdd70ac3cdc27e655308909d4670ca0a7b743384ac7fe34e172d428" + } + ] + }, + "postgres": { + "image": "postgres:17.6-alpine3.22", + "limits": { + "NanoCPUs": 1000000000, + "MemoryBytes": 536870912 + }, + "published_ports": [], + "secret_names": [ + "mtusi-lab4_postgres_password" + ], + "mounts": [ + { + "Type": "volume", + "Source": "mtusi-lab4_postgres_data", + "Target": "/var/lib/postgresql/data", + "VolumeOptions": { + "Labels": { + "com.docker.stack.namespace": "mtusi-lab4" + } + } + } + ], + "tasks": [ + { + "id": "uy9m8wn36msy9qzttodwobkez", + "state": "running", + "container": "41c7c4bb6a37b443982f168a8e8494847581c2e30d1d6f5bf817284aebaf96b9" + } + ] + }, + "redis": { + "image": "redis:7.4.5-alpine3.21", + "limits": { + "NanoCPUs": 500000000, + "MemoryBytes": 134217728 + }, + "published_ports": [], + "secret_names": [], + "mounts": [ + { + "Type": "volume", + "Source": "mtusi-lab4_redis_data", + "Target": "/data", + "VolumeOptions": { + "Labels": { + "com.docker.stack.namespace": "mtusi-lab4" + } + } + } + ], + "tasks": [ + { + "id": "x8ejqk47qyz4rgtgygq48wos7", + "state": "running", + "container": "33a57e377891ce0c82807b12757cf0103ef9461502d1a64efdeec506744d9c27" + } + ] + }, + "minio": { + "image": "mtusi-minio:RELEASE.2025-09-07T16-13-09Z", + "limits": { + "NanoCPUs": 1000000000, + "MemoryBytes": 536870912 + }, + "published_ports": [], + "secret_names": [ + "mtusi-lab4_s3_root_access_key", + "mtusi-lab4_s3_root_secret_key" + ], + "mounts": [ + { + "Type": "volume", + "Source": "mtusi-lab4_s3_data", + "Target": "/data", + "VolumeOptions": { + "Labels": { + "com.docker.stack.namespace": "mtusi-lab4" + } + } + } + ], + "tasks": [ + { + "id": "jafk96quou17emokdyfk1qi4t", + "state": "running", + "container": "a04bf799191faea4386533680968586c90d67adeef8f37166824aa4723f8ac03" + } + ] + } + } +} diff --git a/evidence/lab4-update.txt b/evidence/lab4-update.txt new file mode 100644 index 00000000..2c2ec68f --- /dev/null +++ b/evidence/lab4-update.txt @@ -0,0 +1,3 @@ +.venv/bin/python scripts/swarm.py update +mtusi-lab4_api +PASS: rolling update completed diff --git a/evidence/lab4-upload-memory.json b/evidence/lab4-upload-memory.json new file mode 100644 index 00000000..5b2026b6 --- /dev/null +++ b/evidence/lab4-upload-memory.json @@ -0,0 +1,105 @@ +{ + "concurrency": 10, + "warmup_uploads": 1, + "memory_method": "cgroup v2 memory.current sampled every >=100ms; values include file cache", + "runs": [ + { + "width": 512, + "height": 512, + "size_bytes": 787252, + "sha256": "1a09c45b4a706ff8009c1adf0e9f10e428d8059d335e074cf4ad456680d4b473", + "requests": 10, + "successful": 10, + "errors": 0, + "duration_seconds": 0.325, + "successful_rps": 30.816, + "latency_ms": { + "0.5": 120.287, + "0.95": 172.127, + "0.99": 172.127 + }, + "baseline_bytes": { + "a43c3f7e6e92": 45240320, + "007b52fbba64": 45174784 + }, + "sampled_peak_bytes": { + "a43c3f7e6e92": 47005696, + "007b52fbba64": 46907392 + }, + "kernel_lifetime_peak_before_bytes": { + "a43c3f7e6e92": 46804992, + "007b52fbba64": 46718976 + }, + "kernel_lifetime_peak_after_bytes": { + "a43c3f7e6e92": 48623616, + "007b52fbba64": 48693248 + }, + "sampled_peak_sum_bytes": 93913088, + "samples": [ + { + "seconds": 0.045, + "bytes": { + "a43c3f7e6e92": 47005696, + "007b52fbba64": 46907392 + } + }, + { + "seconds": 0.325, + "bytes": { + "a43c3f7e6e92": 46116864, + "007b52fbba64": 46080000 + } + } + ] + }, + { + "width": 1024, + "height": 768, + "size_bytes": 2360847, + "sha256": "ad94bc1d209135dea4d3caa0712cee92cfaa97ba3ccdd5797aff6664fcce4a2a", + "requests": 10, + "successful": 10, + "errors": 0, + "duration_seconds": 0.401, + "successful_rps": 24.957, + "latency_ms": { + "0.5": 258.573, + "0.95": 324.694, + "0.99": 324.694 + }, + "baseline_bytes": { + "a43c3f7e6e92": 46067712, + "007b52fbba64": 46063616 + }, + "sampled_peak_bytes": { + "a43c3f7e6e92": 41107456, + "007b52fbba64": 13651968 + }, + "kernel_lifetime_peak_before_bytes": { + "a43c3f7e6e92": 48623616, + "007b52fbba64": 48693248 + }, + "kernel_lifetime_peak_after_bytes": { + "a43c3f7e6e92": 49119232, + "007b52fbba64": 49922048 + }, + "sampled_peak_sum_bytes": 54759424, + "samples": [ + { + "seconds": 0.043, + "bytes": { + "a43c3f7e6e92": 41107456, + "007b52fbba64": 13651968 + } + }, + { + "seconds": 0.401, + "bytes": { + "a43c3f7e6e92": 13234176, + "007b52fbba64": 10956800 + } + } + ] + } + ] +} diff --git a/evidence/lab4-upload-memory.txt b/evidence/lab4-upload-memory.txt new file mode 100644 index 00000000..8077fe94 --- /dev/null +++ b/evidence/lab4-upload-memory.txt @@ -0,0 +1,4 @@ +.venv/bin/python scripts/measure_uploads.py +{"width": 512, "height": 512, "size_bytes": 787252, "sha256": "1a09c45b4a706ff8009c1adf0e9f10e428d8059d335e074cf4ad456680d4b473", "requests": 10, "successful": 10, "errors": 0, "duration_seconds": 0.325, "successful_rps": 30.816, "latency_ms": {"0.5": 120.287, "0.95": 172.127, "0.99": 172.127}, "baseline_bytes": {"a43c3f7e6e92": 45240320, "007b52fbba64": 45174784}, "sampled_peak_bytes": {"a43c3f7e6e92": 47005696, "007b52fbba64": 46907392}, "kernel_lifetime_peak_before_bytes": {"a43c3f7e6e92": 46804992, "007b52fbba64": 46718976}, "kernel_lifetime_peak_after_bytes": {"a43c3f7e6e92": 48623616, "007b52fbba64": 48693248}, "sampled_peak_sum_bytes": 93913088} +{"width": 1024, "height": 768, "size_bytes": 2360847, "sha256": "ad94bc1d209135dea4d3caa0712cee92cfaa97ba3ccdd5797aff6664fcce4a2a", "requests": 10, "successful": 10, "errors": 0, "duration_seconds": 0.401, "successful_rps": 24.957, "latency_ms": {"0.5": 258.573, "0.95": 324.694, "0.99": 324.694}, "baseline_bytes": {"a43c3f7e6e92": 46067712, "007b52fbba64": 46063616}, "sampled_peak_bytes": {"a43c3f7e6e92": 41107456, "007b52fbba64": 13651968}, "kernel_lifetime_peak_before_bytes": {"a43c3f7e6e92": 48623616, "007b52fbba64": 48693248}, "kernel_lifetime_peak_after_bytes": {"a43c3f7e6e92": 49119232, "007b52fbba64": 49922048}, "sampled_peak_sum_bytes": 54759424} +PASS: ten simultaneous uploads at two sizes; all originals verified; memory samples saved. diff --git a/evidence/lab4-validation.txt b/evidence/lab4-validation.txt new file mode 100644 index 00000000..f973c727 --- /dev/null +++ b/evidence/lab4-validation.txt @@ -0,0 +1,8 @@ +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +.venv/bin/python scripts/validate_openapi.py +/Users/dkhakimovig/mtusi2026lab4/.venv/lib/python3.9/site-packages/urllib3/__init__.py:35: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020 + warnings.warn( +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. +PASS: goose3.27.1 migration syntax validated. +PASS: mandatory lab4 contract and public checks unchanged. diff --git a/evidence/swarm-grade4.json b/evidence/swarm-grade4.json new file mode 100644 index 00000000..419dfe10 --- /dev/null +++ b/evidence/swarm-grade4.json @@ -0,0 +1,72 @@ +{ + "idempotency": { + "parallel_requests": 16, + "instances": [ + "rz3qizaxp0s2bguzbs57me0op", + "xcgz2hubz1xrzmrkoyrgwq0tq" + ], + "unique_message_ids": 1 + }, + "failovers": [ + { + "action": "stop-one", + "target": "rz3qizaxp0s2bguzbs57me0op", + "duration_seconds": 15.458, + "successful_writes": 1078, + "failed_attempts": 7, + "successful_rps": 69.739, + "first_recovery_seconds": 1.349, + "max_success_gap_seconds": 1.055, + "instances_before": [ + "rz3qizaxp0s2bguzbs57me0op", + "xcgz2hubz1xrzmrkoyrgwq0tq" + ], + "instances_after": [ + "j7awgfnedlt9ne1esywrappxx", + "xcgz2hubz1xrzmrkoyrgwq0tq" + ], + "confirmed_messages_checked": 1229, + "duplicate_bodies": 0 + }, + { + "action": "stop-one", + "target": "xcgz2hubz1xrzmrkoyrgwq0tq", + "duration_seconds": 15.467, + "successful_writes": 1091, + "failed_attempts": 7, + "successful_rps": 70.537, + "first_recovery_seconds": 1.384, + "max_success_gap_seconds": 1.057, + "instances_before": [ + "j7awgfnedlt9ne1esywrappxx", + "xcgz2hubz1xrzmrkoyrgwq0tq" + ], + "instances_after": [ + "5vy560hk6k0q3eicqodd9dp2z", + "j7awgfnedlt9ne1esywrappxx" + ], + "confirmed_messages_checked": 1251, + "duplicate_bodies": 0 + } + ], + "rolling_update": { + "action": "update", + "target": null, + "duration_seconds": 44.156, + "successful_writes": 3362, + "failed_attempts": 0, + "successful_rps": 76.139, + "first_recovery_seconds": null, + "max_success_gap_seconds": 0.152, + "instances_before": [ + "5vy560hk6k0q3eicqodd9dp2z", + "j7awgfnedlt9ne1esywrappxx" + ], + "instances_after": [ + "j201rqxerkxdq0vbcw3armc3m", + "uqyjmps3u9fyl976lpj944pc5" + ], + "confirmed_messages_checked": 3519, + "duplicate_bodies": 0 + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..7e3203b3 --- /dev/null +++ b/go.mod @@ -0,0 +1,42 @@ +module gitflic.ru/akselbant/mtusi2026lab1-fork + +go 1.26.4 + +require ( + github.com/google/uuid v1.6.0 + github.com/jackc/pgx/v5 v5.7.6 + github.com/labstack/echo/v4 v4.13.4 + github.com/minio/minio-go/v7 v7.0.95 + github.com/redis/go-redis/v9 v9.12.1 + github.com/swaggest/swgui v1.8.9 + golang.org/x/crypto v0.39.0 +) + +require ( + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/labstack/gommon v0.4.2 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/minio/crc64nvme v1.0.2 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/tinylib/msgp v1.3.0 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasttemplate v1.2.2 // indirect + github.com/vearutop/statigz v1.4.0 // indirect + golang.org/x/net v0.41.0 // indirect + golang.org/x/sync v0.15.0 // indirect + golang.org/x/sys v0.33.0 // indirect + golang.org/x/text v0.26.0 // indirect + golang.org/x/time v0.11.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..f7913261 --- /dev/null +++ b/go.sum @@ -0,0 +1,90 @@ +github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= +github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/bool64/dev v0.2.45 h1:3nLKhAS/6Oklk3Mt2lHYSN/Cb4tdAD77KLwzeP+6eYE= +github.com/bool64/dev v0.2.45/go.mod h1:iJbh1y/HkunEPhgebWRNcs8wfGq7sjvJ6W5iabL8ACg= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= +github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/labstack/echo/v4 v4.13.4 h1:oTZZW+T3s9gAu5L8vmzihV7/lkXGZuITzTQkTEhcXEA= +github.com/labstack/echo/v4 v4.13.4/go.mod h1:g63b33BZ5vZzcIUF8AtRH40DrTlXnx4UMC8rBdndmjQ= +github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0= +github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/minio/crc64nvme v1.0.2 h1:6uO1UxGAD+kwqWWp7mBFsi5gAse66C4NXO8cmcVculg= +github.com/minio/crc64nvme v1.0.2/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.95 h1:ywOUPg+PebTMTzn9VDsoFJy32ZuARN9zhB+K3IYEvYU= +github.com/minio/minio-go/v7 v7.0.95/go.mod h1:wOOX3uxS334vImCNRVyIDdXX9OsXDm89ToynKgqUKlo= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/redis/go-redis/v9 v9.12.1 h1:k5iquqv27aBtnTm2tIkROUDp8JBXhXZIVu1InSgvovg= +github.com/redis/go-redis/v9 v9.12.1/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/swaggest/swgui v1.8.9 h1:cxAgIwouPpZPlvX68jY5fpwarzLbkc8/IL6DMj+H460= +github.com/swaggest/swgui v1.8.9/go.mod h1:eTJfgwudbyw9xMwqO26vs82ei2u6//JnUAofx2vGB3M= +github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww= +github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= +github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/vearutop/statigz v1.4.0 h1:RQL0KG3j/uyA/PFpHeZ/L6l2ta920/MxlOAIGEOuwmU= +github.com/vearutop/statigz v1.4.0/go.mod h1:LYTolBLiz9oJISwiVKnOQoIwhO1LWX1A7OECawGS8XE= +golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= +golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= +golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= +golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= +golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= +golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= +golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= +golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/infra/README.md b/infra/README.md index 5a39e862..82e02813 100644 --- a/infra/README.md +++ b/infra/README.md @@ -1,7 +1,14 @@ # Инфраструктура студента -Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли. +Основной стенд использует `stack.yaml`: HAProxy → две API-реплики → PostgreSQL, Redis AOF и приватный MinIO. +Запуск: `make up`. Остановка с сохранением данных: `make down`. +Версии, секреты, ограничения и проверки описаны в [REPORT.md](../REPORT.md). -В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд. +`s3-bootstrap.sh` идемпотентно создаёт bucket и отдельного API-пользователя по `s3-policy.json`. +`minio-entrypoint.sh` читает root keys из Docker secrets. Консоль слушает loopback контейнера. +`Dockerfile.storage` собирает MinIO и mc из исходников с фиксированными commit и SHA-256. +`haproxy.cfg` проверяет readiness реплик. `redis.conf` включает AOF и запрещает eviction. +`migrate.sh` запускает goose без автоматической загрузки файлов окружения. -Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику. +Restart hook перезапускает API, HAProxy, PostgreSQL, Redis и MinIO с сохранением volumes. +Failover hook останавливает API-реплику по `TARGET_INSTANCE_ID`; Swarm создаёт новую task с другим ID. diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..6a3fb1fd --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,36 @@ +global + log stdout format raw local0 + maxconn 1024 + +defaults + log global + mode http + timeout connect 1s + timeout client 55s + timeout server 55s + timeout check 1s + retries 0 + option http-server-close + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold obsolete 1s + hold nx 1s + hold refused 1s + hold timeout 1s + +frontend public + bind :8080 + default_backend api + +backend api + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api + http-check expect status 200 + default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 1 + server-template replica 1-8 tasks.api:8080 check resolvers docker init-addr none diff --git a/infra/migrate.sh b/infra/migrate.sh new file mode 100755 index 00000000..70a11b2f --- /dev/null +++ b/infra/migrate.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -eu +# Keep the password out of argv and disable automatic dotenv loading. +if [ -n "${POSTGRES_PASSWORD_FILE:-}" ]; then + PGPASSWORD=$(cat "$POSTGRES_PASSWORD_FILE") + export PGPASSWORD +fi +exec goose -env /dev/null -dir /migrations -timeout 30s "$@" diff --git a/infra/minio-entrypoint.sh b/infra/minio-entrypoint.sh new file mode 100755 index 00000000..ba249937 --- /dev/null +++ b/infra/minio-entrypoint.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +export MINIO_ROOT_USER="$(cat /run/secrets/s3_root_access_key)" +export MINIO_ROOT_PASSWORD="$(cat /run/secrets/s3_root_secret_key)" +exec minio server /data --address :9000 --console-address 127.0.0.1:9001 diff --git a/infra/redis.conf b/infra/redis.conf new file mode 100644 index 00000000..c9e470c6 --- /dev/null +++ b/infra/redis.conf @@ -0,0 +1,9 @@ +bind 0.0.0.0 +protected-mode no +port 6379 +dir /data +appendonly yes +appendfsync always +save "" +maxmemory 64mb +maxmemory-policy noeviction diff --git a/infra/s3-bootstrap.sh b/infra/s3-bootstrap.sh new file mode 100755 index 00000000..4251f8fd --- /dev/null +++ b/infra/s3-bootstrap.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -eu +export MC_CONFIG_DIR=/tmp/mc +mc alias set admin http://minio:9000 "$(cat /run/secrets/s3_root_access_key)" "$(cat /run/secrets/s3_root_secret_key)" >/dev/null +mc ready admin >/dev/null +mc mb --ignore-existing admin/attachments >/dev/null +mc anonymous set none admin/attachments >/dev/null +mc admin policy create admin attachments-api /policy.json >/dev/null +mc admin user add admin "$(cat /run/secrets/s3_access_key)" "$(cat /run/secrets/s3_secret_key)" >/dev/null +mc admin policy attach admin attachments-api --user "$(cat /run/secrets/s3_access_key)" >/dev/null +printf '%s\n' 'PASS: private bucket and restricted API account configured.' diff --git a/infra/s3-policy.json b/infra/s3-policy.json new file mode 100644 index 00000000..b9fb3b95 --- /dev/null +++ b/infra/s3-policy.json @@ -0,0 +1,21 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetBucketLocation"], + "Resource": ["arn:aws:s3:::attachments"] + }, + { + "Effect": "Allow", + "Action": ["s3:ListBucket"], + "Resource": ["arn:aws:s3:::attachments"], + "Condition": {"StringLike": {"s3:prefix": ["original/*"]}} + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], + "Resource": ["arn:aws:s3:::attachments/original/*"] + } + ] +} diff --git a/infra/schema.md b/infra/schema.md new file mode 100644 index 00000000..9b214069 --- /dev/null +++ b/infra/schema.md @@ -0,0 +1,25 @@ +# Схема и транзакционные границы + +PostgreSQL — источник истины пользователей, чатов, участников и сообщений. +Redis — источник истины сессий: SHA-256 токена → user_id, expires_at; TTL до +абсолютного expires_at, без продления при чтении. Logout удаляет ключ. +Redis использует AOF с appendfsync always и отдельный именованный volume. + +- users: UUID PK, UNIQUE(username), password_hash (Argon2id), display_name, created_at. +- chats: UUID PK, title, created_at; индекс (created_at, id). +- chat_members: PK(chat_id, user_id), FK к chats/users; ordinal сохраняет порядок + участников; UNIQUE(chat_id, ordinal); индекс (user_id, chat_id). +- messages: UUID PK, FK chat_id/sender_id через членство; text, created_at. + Индекс (chat_id, created_at, id) отвечает равенству по чату и keyset-предикату + (created_at, id) > ($timestamp, $id), ORDER BY created_at, id LIMIT $limit. +- goose_db_version: журнал goose CLI; SQL-миграция и запись её версии + коммитятся в одной транзакции. Сервис migrations завершается до старта API. + +Регистрация — один INSERT, гонка разрешается UNIQUE, проигравшие получают 409. +Создание чата — одна транзакция: INSERT чата, затем участников; любой неверный +FK откатывает всю операцию, включая уже вставленных участников. +Отправка сообщения — один INSERT с FK на членство. Чтения идут в PostgreSQL, +кэша истории нет, подтверждённый COMMIT виден следующему запросу. +При конкурентной записи keyset не даёт snapshot: поздний COMMIT с ключом до +уже прочитанного cursor может быть пропущен. created_at задаёт PostgreSQL, +UUID разрешает совпадения времени; часы не используются как уникальный ID. diff --git a/internal/app/chat/access.go b/internal/app/chat/access.go new file mode 100644 index 00000000..36f71a5b --- /dev/null +++ b/internal/app/chat/access.go @@ -0,0 +1,31 @@ +package chat + +import ( + "errors" + + "github.com/labstack/echo/v4" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" +) + +func (h *Chat) authorizedChat(c echo.Context, requestedChat domain.Chat) (domain.Chat, bool, error) { + chat, err := h.useCase.GetChat(c.Request().Context(), authenticatedUser(c).ID, requestedChat) + if err != nil { + return domain.Chat{}, false, h.useCaseError(c, err) + } + return chat, true, nil +} + +func (h *Chat) cursorPosition(raw string, present bool, scope, owner string) (*domain.Position, error) { + if !present { + return nil, nil + } + if raw == "" { + return nil, errors.New("empty cursor") + } + createdAt, id, err := h.cursor.decode(raw, scope, owner) + if err != nil { + return nil, err + } + return &domain.Position{CreatedAt: createdAt, ID: id}, nil +} diff --git a/internal/app/chat/attachments.go b/internal/app/chat/attachments.go new file mode 100644 index 00000000..f326d707 --- /dev/null +++ b/internal/app/chat/attachments.go @@ -0,0 +1,68 @@ +package chat + +import ( + "context" + "mime" + "net/http" + "time" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/attachment" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" + "github.com/labstack/echo/v4" +) + +func (h *Chat) WithAttachments(service *attachment.Service) *Chat { h.attachments = service; return h } +func (h *Chat) uploadAttachment(c echo.Context) error { + chat, ok, err := h.authorizedChat(c, domain.Chat{ID: c.Param("chat_id")}) + if err != nil || !ok { + return err + } + filename, present, err := oneQueryValue(c, "filename") + if err != nil || !present { + return h.invalid(c, "Требуется один filename") + } + if c.Request().ContentLength > attachment.MaxBytes { + return h.writeError(c, 413, "payload_too_large", "Файл превышает 10 MiB") + } + // Both socket reads and the application operation have a fixed deadline. + deadline := time.Now().Add(attachment.UploadTimeout) + _ = http.NewResponseController(c.Response().Writer).SetReadDeadline(deadline) + defer http.NewResponseController(c.Response().Writer).SetReadDeadline(time.Time{}) + c.Request().Body = http.MaxBytesReader(c.Response(), c.Request().Body, attachment.MaxBytes+1) + a, err := h.attachments.Upload(c.Request().Context(), chat.ID, authenticatedUser(c).ID, filename, c.Request().Header.Get("Content-Type"), c.Request().Body) + if err != nil { + return h.useCaseError(c, err) + } + return c.JSON(http.StatusCreated, a) +} +func (h *Chat) getAttachment(c echo.Context) error { + a, err := h.attachments.Get(c.Request().Context(), c.Param("attachment_id"), authenticatedUser(c).ID) + if err != nil { + return h.useCaseError(c, err) + } + return c.JSON(http.StatusOK, a) +} +func (h *Chat) attachmentContent(c echo.Context) error { + a, err := h.attachments.Get(c.Request().Context(), c.Param("attachment_id"), authenticatedUser(c).ID) + if err != nil { + return h.useCaseError(c, err) + } + variant, present, err := oneQueryValue(c, "variant") + if err != nil { + return h.invalid(c, "Некорректный variant") + } + if present && variant != "original" { + return h.writeError(c, 404, "not_found", "Вариант не найден") + } + ctx, cancel := context.WithTimeout(c.Request().Context(), 45*time.Second) + defer cancel() + obj, err := h.attachments.Content(ctx, a) + if err != nil { + return h.useCaseError(c, err) + } + defer obj.Close() + c.Response().Header().Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": a.Filename})) + c.Response().Header().Set("X-Content-Type-Options", "nosniff") + c.Response().Header().Set("Cache-Control", "private, no-store") + return c.Stream(http.StatusOK, a.ContentType, obj) +} diff --git a/internal/app/chat/create_chat.go b/internal/app/chat/create_chat.go new file mode 100644 index 00000000..7e57b2e5 --- /dev/null +++ b/internal/app/chat/create_chat.go @@ -0,0 +1,33 @@ +package chat + +import ( + "net/http" + + "github.com/labstack/echo/v4" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/dto" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" +) + +func (h *Chat) createChat(c echo.Context) error { + var request dto.CreateChatRequest + if err := decodeJSON(c, &request); err != nil { + return h.jsonError(c, err) + } + callerID := authenticatedUser(c).ID + chat, err := h.useCase.CreateChat(c.Request().Context(), callerID, createChatRequestToDomain(request)) + if err != nil { + return h.useCaseError(c, err) + } + return c.JSON(http.StatusCreated, createChatResponseFromDomain(chat)) +} + +func createChatRequestToDomain(request dto.CreateChatRequest) domain.Chat { + return domain.Chat{Title: request.Title, MemberIDs: request.MemberIDs} +} + +func createChatResponseFromDomain(chat domain.Chat) dto.CreateChatResponse { + return dto.CreateChatResponse{ + ID: chat.ID, Title: chat.Title, MemberIDs: chat.MemberIDs, CreatedAt: dto.Timestamp(chat.CreatedAt), + } +} diff --git a/internal/app/chat/create_message.go b/internal/app/chat/create_message.go new file mode 100644 index 00000000..3c0c27b7 --- /dev/null +++ b/internal/app/chat/create_message.go @@ -0,0 +1,58 @@ +package chat + +import ( + "net/http" + "regexp" + + "github.com/labstack/echo/v4" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/dto" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" +) + +var idempotencyKeyPattern = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`) + +func (h *Chat) createMessage(c echo.Context) error { + request := dto.CreateMessageRequest{ChatID: c.Param("chat_id")} + chat, ok, err := h.authorizedChat(c, domain.Chat{ID: request.ChatID}) + if err != nil { + return err + } + if !ok { + return nil + } + if err := decodeJSON(c, &request); err != nil { + return h.jsonError(c, err) + } + request.ChatID = chat.ID + + keys := c.Request().Header.Values("Idempotency-Key") + if len(keys) != 1 || !idempotencyKeyPattern.MatchString(keys[0]) { + return h.invalid(c, "Требуется Idempotency-Key: ASCII [A-Za-z0-9._:-]{1,128}") + } + input := createMessageRequestToDomain(request) + input.IdempotencyKey = keys[0] + message, err := h.useCase.CreateMessage(c.Request().Context(), authenticatedUser(c).ID, input) + if err != nil { + return h.useCaseError(c, err) + } + return c.JSON(http.StatusCreated, createMessageResponseFromDomain(message)) +} + +func createMessageRequestToDomain(request dto.CreateMessageRequest) domain.Message { + return domain.Message{ChatID: request.ChatID, Text: request.Text, AttachmentIDs: request.AttachmentIDs} +} + +func createMessageResponseFromDomain(message domain.Message) dto.CreateMessageResponse { + return dto.CreateMessageResponse{ + ID: message.ID, ChatID: message.ChatID, SenderID: message.SenderID, + Text: message.Text, AttachmentIDs: nonnilIDs(message.AttachmentIDs), CreatedAt: dto.Timestamp(message.CreatedAt), + } +} + +func nonnilIDs(ids []string) []string { + if ids == nil { + return []string{} + } + return ids +} diff --git a/internal/app/chat/create_user.go b/internal/app/chat/create_user.go new file mode 100644 index 00000000..f6078178 --- /dev/null +++ b/internal/app/chat/create_user.go @@ -0,0 +1,32 @@ +package chat + +import ( + "net/http" + + "github.com/labstack/echo/v4" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/dto" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" +) + +func (h *Chat) createUser(c echo.Context) error { + var request dto.CreateUserRequest + if err := decodeJSON(c, &request); err != nil { + return h.jsonError(c, err) + } + user, err := h.useCase.CreateUser(c.Request().Context(), createUserRequestToDomain(request), request.Password) + if err != nil { + return h.useCaseError(c, err) + } + return c.JSON(http.StatusCreated, createUserResponseFromDomain(user)) +} + +func createUserRequestToDomain(request dto.CreateUserRequest) domain.User { + return domain.User{Username: request.Username, DisplayName: request.DisplayName} +} + +func createUserResponseFromDomain(user domain.User) dto.CreateUserResponse { + return dto.CreateUserResponse{ + ID: user.ID, Username: user.Username, DisplayName: user.DisplayName, CreatedAt: dto.Timestamp(user.CreatedAt), + } +} diff --git a/internal/app/chat/current_user.go b/internal/app/chat/current_user.go new file mode 100644 index 00000000..23277dea --- /dev/null +++ b/internal/app/chat/current_user.go @@ -0,0 +1,21 @@ +package chat + +import ( + "net/http" + + "github.com/labstack/echo/v4" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/dto" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" +) + +func (h *Chat) currentUser(c echo.Context) error { + user := authenticatedUser(c) + return c.JSON(http.StatusOK, currentUserResponseFromDomain(user)) +} + +func currentUserResponseFromDomain(user domain.User) dto.CurrentUserResponse { + return dto.CurrentUserResponse{ + ID: user.ID, Username: user.Username, DisplayName: user.DisplayName, CreatedAt: dto.Timestamp(user.CreatedAt), + } +} diff --git a/internal/app/chat/cursor.go b/internal/app/chat/cursor.go new file mode 100644 index 00000000..83836f11 --- /dev/null +++ b/internal/app/chat/cursor.go @@ -0,0 +1,75 @@ +package chat + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "strings" + "time" + + "github.com/google/uuid" +) + +type cursorPayload struct { + Scope string `json:"scope"` + Owner string `json:"owner"` + CreatedAt string `json:"created_at"` + ID string `json:"id"` +} + +type cursorCodec struct { + secret [32]byte +} + +func (c cursorCodec) encode(scope, owner string, createdAt time.Time, id string) string { + payload, err := json.Marshal(cursorPayload{ + Scope: scope, + Owner: owner, + CreatedAt: createdAt.UTC().Format(time.RFC3339Nano), + ID: id, + }) + if err != nil { + panic("cannot encode cursor: " + err.Error()) + } + signature := c.sign(payload) + return base64.RawURLEncoding.EncodeToString(payload) + "." + base64.RawURLEncoding.EncodeToString(signature) +} + +func (c cursorCodec) decode(raw, scope, owner string) (time.Time, string, error) { + parts := strings.Split(raw, ".") + if len(parts) != 2 { + return time.Time{}, "", errors.New("malformed cursor") + } + payload, err := base64.RawURLEncoding.DecodeString(parts[0]) + if err != nil { + return time.Time{}, "", errors.New("malformed cursor") + } + signature, err := base64.RawURLEncoding.DecodeString(parts[1]) + if err != nil || !hmac.Equal(signature, c.sign(payload)) { + return time.Time{}, "", errors.New("invalid cursor signature") + } + var value cursorPayload + if err := json.Unmarshal(payload, &value); err != nil { + return time.Time{}, "", errors.New("malformed cursor payload") + } + if value.Scope != scope || value.Owner != owner { + return time.Time{}, "", errors.New("cursor belongs to another resource") + } + parsedID, err := uuid.Parse(value.ID) + if err != nil || parsedID.String() != value.ID { + return time.Time{}, "", errors.New("malformed cursor id") + } + createdAt, err := time.Parse(time.RFC3339Nano, value.CreatedAt) + if err != nil { + return time.Time{}, "", errors.New("malformed cursor timestamp") + } + return createdAt, value.ID, nil +} + +func (c cursorCodec) sign(payload []byte) []byte { + mac := hmac.New(sha256.New, c.secret[:]) + _, _ = mac.Write(payload) + return mac.Sum(nil) +} diff --git a/internal/app/chat/errors.go b/internal/app/chat/errors.go new file mode 100644 index 00000000..c556830e --- /dev/null +++ b/internal/app/chat/errors.go @@ -0,0 +1,95 @@ +package chat + +import ( + "errors" + "net/http" + + "github.com/google/uuid" + "github.com/labstack/echo/v4" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/dto" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" +) + +func (h *Chat) invalid(c echo.Context, message string) error { + return h.writeError(c, http.StatusBadRequest, "invalid_request", message) +} + +func (h *Chat) jsonError(c echo.Context, err error) error { + if errors.Is(err, errUnsupportedMediaType) { + return h.writeError(c, http.StatusUnsupportedMediaType, "unsupported_media_type", "Content-Type должен быть application/json") + } + var tooLarge *http.MaxBytesError + if errors.As(err, &tooLarge) { + return h.writeError(c, http.StatusRequestEntityTooLarge, "payload_too_large", "Тело запроса слишком большое") + } + return h.invalid(c, err.Error()) +} + +func (h *Chat) useCaseError(c echo.Context, err error) error { + applicationError, ok := domain.AsApplicationError(err) + if !ok { + return err + } + status := http.StatusInternalServerError + switch applicationError.Code { + case "unsupported_media_type": + status = http.StatusUnsupportedMediaType + case "payload_too_large": + status = http.StatusRequestEntityTooLarge + case domain.ErrorUnavailable: + status = http.StatusServiceUnavailable + case domain.ErrorInvalid: + status = http.StatusBadRequest + case domain.ErrorUnauthorized: + status = http.StatusUnauthorized + challenge := `Bearer realm="messenger"` + if c.Path() == "/api/v1/auth/sessions" { + challenge = `Basic realm="messenger", charset="UTF-8"` + } + c.Response().Header().Set("WWW-Authenticate", challenge) + case domain.ErrorNotFound: + status = http.StatusNotFound + case domain.ErrorConflict: + status = http.StatusConflict + } + return h.writeError(c, status, string(applicationError.Code), applicationError.Message) +} + +func (h *Chat) writeError(c echo.Context, status int, code, message string) error { + return c.JSON(status, dto.ErrorResponse{Error: dto.ErrorDetails{ + Code: code, + Message: message, + RequestID: requestID(c), + }}) +} + +func (h *Chat) httpErrorHandler(err error, c echo.Context) { + if c.Response().Committed { + return + } + status := http.StatusInternalServerError + code := "internal_error" + message := "Внутренняя ошибка сервера" + var httpError *echo.HTTPError + if errors.As(err, &httpError) { + status = httpError.Code + switch status { + case http.StatusNotFound: + code, message = "not_found", "Объект не найден" + case http.StatusMethodNotAllowed, http.StatusBadRequest: + code, message = "invalid_request", "Некорректный запрос" + } + } + _ = h.writeError(c, status, code, message) +} + +func requestID(c echo.Context) string { + value, _ := c.Get(requestIDKey).(string) + if value == "" { + value = uuid.NewString() + c.Set(requestIDKey, value) + c.Response().Header().Set("X-Request-Id", value) + } + return value +} diff --git a/internal/app/chat/get_chat.go b/internal/app/chat/get_chat.go new file mode 100644 index 00000000..eaddd9eb --- /dev/null +++ b/internal/app/chat/get_chat.go @@ -0,0 +1,32 @@ +package chat + +import ( + "net/http" + + "github.com/labstack/echo/v4" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/dto" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" +) + +func (h *Chat) getChat(c echo.Context) error { + request := dto.GetChatRequest{ChatID: c.Param("chat_id")} + chat, ok, err := h.authorizedChat(c, getChatRequestToDomain(request)) + if err != nil { + return err + } + if !ok { + return nil + } + return c.JSON(http.StatusOK, getChatResponseFromDomain(chat)) +} + +func getChatRequestToDomain(request dto.GetChatRequest) domain.Chat { + return domain.Chat{ID: request.ChatID} +} + +func getChatResponseFromDomain(chat domain.Chat) dto.GetChatResponse { + return dto.GetChatResponse{ + ID: chat.ID, Title: chat.Title, MemberIDs: chat.MemberIDs, CreatedAt: dto.Timestamp(chat.CreatedAt), + } +} diff --git a/internal/app/chat/list_chats.go b/internal/app/chat/list_chats.go new file mode 100644 index 00000000..1a02d5fa --- /dev/null +++ b/internal/app/chat/list_chats.go @@ -0,0 +1,53 @@ +package chat + +import ( + "net/http" + + "github.com/labstack/echo/v4" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/dto" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" +) + +func (h *Chat) listChats(c echo.Context) error { + limit, err := parseLimit(c) + if err != nil { + return h.invalid(c, err.Error()) + } + rawCursor, cursorPresent, err := oneQueryValue(c, "cursor") + if err != nil { + return h.invalid(c, "cursor должен быть указан один раз") + } + request := dto.ListChatsRequest{Limit: limit, Cursor: rawCursor} + userID := authenticatedUser(c).ID + position, err := h.cursorPosition(request.Cursor, cursorPresent, "chats", userID) + if err != nil { + return h.invalid(c, "Некорректный cursor") + } + + page, err := h.useCase.ListChats(c.Request().Context(), userID, listChatsRequestToDomain(request, position)) + if err != nil { + return h.useCaseError(c, err) + } + var next *string + if page.HasMore { + last := page.Items[len(page.Items)-1] + encoded := h.cursor.encode("chats", userID, last.CreatedAt, last.ID) + next = &encoded + } + return c.JSON(http.StatusOK, listChatsResponseFromDomain(page, next)) +} + +func listChatsRequestToDomain(request dto.ListChatsRequest, position *domain.Position) domain.Pagination { + return domain.Pagination{After: position, Limit: request.Limit} +} + +func listChatsResponseFromDomain(page domain.ChatPage, nextCursor *string) dto.ListChatsResponse { + items := make([]dto.ListChatsItem, 0, len(page.Items)) + for _, chat := range page.Items { + items = append(items, dto.ListChatsItem{ + ID: chat.ID, Title: chat.Title, MemberIDs: chat.MemberIDs, CreatedAt: dto.Timestamp(chat.CreatedAt), + }) + } + return dto.ListChatsResponse{Items: items, NextCursor: nextCursor} +} diff --git a/internal/app/chat/list_messages.go b/internal/app/chat/list_messages.go new file mode 100644 index 00000000..1d7f2520 --- /dev/null +++ b/internal/app/chat/list_messages.go @@ -0,0 +1,70 @@ +package chat + +import ( + "net/http" + + "github.com/labstack/echo/v4" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/dto" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" +) + +func (h *Chat) listMessages(c echo.Context) error { + request := dto.ListMessagesRequest{ChatID: c.Param("chat_id")} + chat, ok, err := h.authorizedChat(c, domain.Chat{ID: request.ChatID}) + if err != nil { + return err + } + if !ok { + return nil + } + limit, err := parseLimit(c) + if err != nil { + return h.invalid(c, err.Error()) + } + rawCursor, cursorPresent, err := oneQueryValue(c, "cursor") + if err != nil { + return h.invalid(c, "cursor должен быть указан один раз") + } + request.ChatID = chat.ID + request.Limit = limit + request.Cursor = rawCursor + position, err := h.cursorPosition(request.Cursor, cursorPresent, "messages", request.ChatID) + if err != nil { + return h.invalid(c, "Некорректный cursor") + } + + page, err := h.useCase.ListMessages( + c.Request().Context(), authenticatedUser(c).ID, + listMessagesChatToDomain(request), listMessagesPaginationToDomain(request, position), + ) + if err != nil { + return h.useCaseError(c, err) + } + var next *string + if page.HasMore { + last := page.Items[len(page.Items)-1] + encoded := h.cursor.encode("messages", request.ChatID, last.CreatedAt, last.ID) + next = &encoded + } + return c.JSON(http.StatusOK, listMessagesResponseFromDomain(page, next)) +} + +func listMessagesChatToDomain(request dto.ListMessagesRequest) domain.Chat { + return domain.Chat{ID: request.ChatID} +} + +func listMessagesPaginationToDomain(request dto.ListMessagesRequest, position *domain.Position) domain.Pagination { + return domain.Pagination{After: position, Limit: request.Limit} +} + +func listMessagesResponseFromDomain(page domain.MessagePage, nextCursor *string) dto.ListMessagesResponse { + items := make([]dto.ListMessagesItem, 0, len(page.Items)) + for _, message := range page.Items { + items = append(items, dto.ListMessagesItem{ + ID: message.ID, ChatID: message.ChatID, SenderID: message.SenderID, + Text: message.Text, AttachmentIDs: nonnilIDs(message.AttachmentIDs), CreatedAt: dto.Timestamp(message.CreatedAt), + }) + } + return dto.ListMessagesResponse{Items: items, NextCursor: nextCursor} +} diff --git a/internal/app/chat/liveness.go b/internal/app/chat/liveness.go new file mode 100644 index 00000000..6b6029f9 --- /dev/null +++ b/internal/app/chat/liveness.go @@ -0,0 +1,13 @@ +package chat + +import ( + "net/http" + + "github.com/labstack/echo/v4" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/dto" +) + +func (h *Chat) liveness(c echo.Context) error { + return c.JSON(http.StatusOK, dto.LivenessResponse{Status: "ok"}) +} diff --git a/internal/app/chat/middleware.go b/internal/app/chat/middleware.go new file mode 100644 index 00000000..353b517c --- /dev/null +++ b/internal/app/chat/middleware.go @@ -0,0 +1,43 @@ +package chat + +import ( + "net/http" + + "github.com/google/uuid" + "github.com/labstack/echo/v4" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" +) + +const ( + requestIDKey = "request_id" + userKey = "current_user" +) + +func (h *Chat) requestIDMiddleware(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + requestID := uuid.NewString() + c.Set(requestIDKey, requestID) + c.Response().Header().Set("X-Request-Id", requestID) + c.Response().Header().Set("X-Instance-Id", h.instanceID) + if h.draining.Load() && c.Request().URL.Path != "/health/live" { + return h.writeError(c, http.StatusServiceUnavailable, "unavailable", "Реплика завершает работу") + } + return next(c) + } +} + +func (h *Chat) authenticate(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + user, err := h.useCase.Authenticate(c.Request().Context(), bearerToken(c)) + if err != nil { + return h.useCaseError(c, err) + } + c.Set(userKey, user) + return next(c) + } +} + +func authenticatedUser(c echo.Context) domain.User { + return c.Get(userKey).(domain.User) +} diff --git a/internal/app/chat/readiness.go b/internal/app/chat/readiness.go new file mode 100644 index 00000000..950ca172 --- /dev/null +++ b/internal/app/chat/readiness.go @@ -0,0 +1,25 @@ +package chat + +import ( + "context" + "net/http" + "time" + + "github.com/labstack/echo/v4" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/dto" +) + +func (h *Chat) readiness(c echo.Context) error { + ctx, cancel := context.WithTimeout(c.Request().Context(), 2*time.Second) + defer cancel() + if err := h.useCase.Ready(ctx); err != nil { + return h.useCaseError(c, err) + } + if h.attachments != nil { + if err := h.attachments.Ready(ctx); err != nil { + return h.useCaseError(c, err) + } + } + return c.JSON(http.StatusOK, dto.ReadinessResponse{Status: "ok"}) +} diff --git a/internal/app/chat/request.go b/internal/app/chat/request.go new file mode 100644 index 00000000..2cb05d4d --- /dev/null +++ b/internal/app/chat/request.go @@ -0,0 +1,74 @@ +package chat + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "mime" + "net/http" + "strconv" + + "github.com/labstack/echo/v4" +) + +const ( + defaultLimit = 50 + maxLimit = 100 + maxJSONBody = 1 << 20 +) + +var errUnsupportedMediaType = errors.New("unsupported media type") + +func decodeJSON(c echo.Context, target any) error { + mediaType, _, err := mime.ParseMediaType(c.Request().Header.Get(echo.HeaderContentType)) + if err != nil || mediaType != echo.MIMEApplicationJSON { + return errUnsupportedMediaType + } + reader := http.MaxBytesReader(c.Response(), c.Request().Body, maxJSONBody) + decoder := json.NewDecoder(reader) + decoder.DisallowUnknownFields() + if err := decoder.Decode(target); err != nil { + var tooLarge *http.MaxBytesError + if errors.As(err, &tooLarge) { + return err + } + return errors.New("Некорректное JSON-тело") + } + var extra any + err = decoder.Decode(&extra) + var tooLarge *http.MaxBytesError + if errors.As(err, &tooLarge) { + return err + } + if !errors.Is(err, io.EOF) { + return errors.New("JSON-тело должно содержать ровно один объект") + } + return nil +} + +func parseLimit(c echo.Context) (int, error) { + raw, present, err := oneQueryValue(c, "limit") + if err != nil { + return 0, errors.New("limit должен быть указан один раз") + } + if !present { + return defaultLimit, nil + } + limit, err := strconv.Atoi(raw) + if err != nil || limit < 1 || limit > maxLimit { + return 0, fmt.Errorf("limit должен быть целым числом от 1 до %d", maxLimit) + } + return limit, nil +} + +func oneQueryValue(c echo.Context, name string) (string, bool, error) { + values, present := c.QueryParams()[name] + if !present { + return "", false, nil + } + if len(values) != 1 { + return "", true, errors.New("query parameter must occur once") + } + return values[0], true, nil +} diff --git a/internal/app/chat/service.go b/internal/app/chat/service.go new file mode 100644 index 00000000..0e4115c0 --- /dev/null +++ b/internal/app/chat/service.go @@ -0,0 +1,93 @@ +package chat + +import ( + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/attachment" + "log" + "os" + "sync/atomic" + + "github.com/google/uuid" + "github.com/labstack/echo/v4" + "github.com/labstack/echo/v4/middleware" + "github.com/swaggest/swgui/v5emb" +) + +// Chat is the HTTP handler and depends only on the application boundary. +type Chat struct { + attachments *attachment.Service + instanceID string + draining atomic.Bool + useCase UseCase + cursor cursorCodec +} + +func NewChat(useCase UseCase, cursorKey [32]byte) *Chat { + instanceID := os.Getenv("INSTANCE_ID") + if instanceID == "" { + instanceID = uuid.NewString() + } + return &Chat{useCase: useCase, cursor: cursorCodec{secret: cursorKey}, instanceID: instanceID} +} + +func (h *Chat) Drain() { h.draining.Store(true) } + +func (h *Chat) Router() *echo.Echo { + e := echo.New() + e.HideBanner = true + e.HidePort = true + e.HTTPErrorHandler = h.httpErrorHandler + e.Use(h.requestIDMiddleware) + e.Use(middleware.RequestLoggerWithConfig(middleware.RequestLoggerConfig{ + HandleError: true, + LogLatency: true, + LogMethod: true, + LogRoutePath: true, + LogURIPath: true, + LogRequestID: true, + LogStatus: true, + LogResponseSize: true, + LogValuesFunc: func(_ echo.Context, values middleware.RequestLoggerValues) error { + path := values.RoutePath + if path == "" { + path = values.URIPath + } + log.Printf( + "request_id=%s method=%s path=%s status=%d response_bytes=%d latency=%s", + values.RequestID, values.Method, path, values.Status, values.ResponseSize, values.Latency, + ) + return nil + }, + })) + e.Use(middleware.Recover()) + + e.GET("/health/live", h.liveness) + e.GET("/health/ready", h.readiness) + swaggerUI := echo.WrapHandler(v5emb.New( + "MTUSI Messenger", + "/swagger/openapi.json", + "/swagger/", + )) + e.GET("/swagger", h.swagger) + e.GET("/swagger/", swaggerUI) + e.GET("/swagger/openapi.json", h.swaggerSpec) + e.GET("/swagger/*", swaggerUI) + e.POST("/api/v1/users", h.createUser) + + e.POST("/api/v1/auth/sessions", h.createSession) + + api := e.Group("/api/v1", h.authenticate) + api.GET("/users/me", h.currentUser) + api.DELETE("/auth/sessions/current", h.deleteSession) + api.POST("/chats", h.createChat) + api.GET("/chats", h.listChats) + api.GET("/chats/:chat_id", h.getChat) + api.POST("/chats/:chat_id/messages", h.createMessage) + api.GET("/chats/:chat_id/messages", h.listMessages) + if h.attachments != nil { + api.POST("/chats/:chat_id/attachments", h.uploadAttachment) + api.GET("/attachments/:attachment_id", h.getAttachment) + api.GET("/attachments/:attachment_id/content", h.attachmentContent) + } + + return e +} diff --git a/internal/app/chat/service_test.go b/internal/app/chat/service_test.go new file mode 100644 index 00000000..bf582c2c --- /dev/null +++ b/internal/app/chat/service_test.go @@ -0,0 +1,353 @@ +package chat_test + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "sync" + "testing" + "time" + + "github.com/google/uuid" + + chathandler "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/app/chat" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/repository" + chatusecase "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/usecase/chat" +) + +type testAPI struct { + server *httptest.Server + tokens sync.Map +} + +type userResponse struct { + ID string `json:"id"` +} + +type chatResponse struct { + ID string `json:"id"` +} + +type messageResponse struct { + ID string `json:"id"` + Text string `json:"text"` +} + +type messagePageResponse struct { + Items []messageResponse `json:"items"` + NextCursor *string `json:"next_cursor"` +} + +type errorResponse struct { + Error struct { + Code string `json:"code"` + } `json:"error"` +} + +func newTestAPI(t *testing.T) *testAPI { + t.Helper() + useCase := chatusecase.New(repository.NewMemory(), &testSessions{values: map[string]domain.Session{}}, time.Hour) + server := httptest.NewServer(chathandler.NewChat(useCase, [32]byte{1}).Router()) + t.Cleanup(server.Close) + return &testAPI{server: server} +} + +func (a *testAPI) request(t *testing.T, client *http.Client, method, path, actorID, body string, wantStatus int, target any) { + t.Helper() + request, err := http.NewRequest(method, a.server.URL+path, bytes.NewBufferString(body)) + if err != nil { + t.Fatalf("create request: %v", err) + } + if body != "" { + request.Header.Set("Content-Type", "application/json") + request.Header.Set("Idempotency-Key", uuid.NewString()) + } + if actorID != "" { + token, _ := a.tokens.Load(actorID) + request.Header.Set("Authorization", "Bearer "+token.(string)) + } + response, err := client.Do(request) + if err != nil { + t.Fatalf("%s %s: %v", method, path, err) + } + defer response.Body.Close() + responseBody, err := io.ReadAll(response.Body) + if err != nil { + t.Fatalf("read response: %v", err) + } + if response.StatusCode != wantStatus { + t.Fatalf("%s %s: status=%d, want=%d, body=%s", method, path, response.StatusCode, wantStatus, responseBody) + } + if response.Header.Get("X-Request-Id") == "" { + t.Fatalf("%s %s: missing X-Request-Id", method, path) + } + if target != nil { + if err := json.Unmarshal(responseBody, target); err != nil { + t.Fatalf("decode response %q: %v", responseBody, err) + } + } +} + +func (a *testAPI) createUser(t *testing.T, username string) userResponse { + t.Helper() + var user userResponse + a.request(t, http.DefaultClient, http.MethodPost, "/api/v1/users", "", + fmt.Sprintf(`{"username":%q,"display_name":%q,"password":"test-password-12345"}`, username, username), http.StatusCreated, &user) + req, err := http.NewRequest(http.MethodPost, a.server.URL+"/api/v1/auth/sessions", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(username+":test-password-12345"))) + response, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + var login struct { + Token string `json:"token"` + } + if response.StatusCode != http.StatusCreated { + t.Fatalf("login status: %d", response.StatusCode) + } + if err = json.NewDecoder(response.Body).Decode(&login); err != nil { + t.Fatal(err) + } + a.tokens.Store(user.ID, login.Token) + return user +} + +func (a *testAPI) requestError(t *testing.T, method, path, actorID, body string, wantStatus int, wantCode string) { + t.Helper() + var response errorResponse + a.request(t, http.DefaultClient, method, path, actorID, body, wantStatus, &response) + if response.Error.Code != wantCode { + t.Fatalf("%s %s: error code=%q, want=%q", method, path, response.Error.Code, wantCode) + } +} + +func (a *testAPI) createChat(t *testing.T, ownerID string, memberIDs []string) chatResponse { + t.Helper() + body, err := json.Marshal(map[string]any{"title": "test chat", "member_ids": memberIDs}) + if err != nil { + t.Fatalf("encode chat: %v", err) + } + var chat chatResponse + a.request(t, http.DefaultClient, http.MethodPost, "/api/v1/chats", ownerID, string(body), http.StatusCreated, &chat) + return chat +} + +func (a *testAPI) createMessage(t *testing.T, client *http.Client, actorID, chatID, text string) messageResponse { + t.Helper() + body, err := json.Marshal(map[string]string{"text": text}) + if err != nil { + t.Fatalf("encode message: %v", err) + } + var message messageResponse + a.request(t, client, http.MethodPost, "/api/v1/chats/"+chatID+"/messages", actorID, string(body), http.StatusCreated, &message) + return message +} + +func TestRejectsMalformedJSON(t *testing.T) { + api := newTestAPI(t) + api.requestError(t, http.MethodPost, "/api/v1/users", "", + `{"username":`, http.StatusBadRequest, "invalid_request") +} + +func TestHidesForeignChat(t *testing.T) { + api := newTestAPI(t) + owner := api.createUser(t, "owner_user") + outsider := api.createUser(t, "outsider_user") + chat := api.createChat(t, owner.ID, []string{owner.ID}) + api.requestError(t, http.MethodGet, "/api/v1/chats/"+chat.ID, outsider.ID, + "", http.StatusNotFound, "not_found") +} + +func TestRejectsEmptyMessage(t *testing.T) { + api := newTestAPI(t) + user := api.createUser(t, "empty_message_user") + chat := api.createChat(t, user.ID, []string{user.ID}) + api.requestError(t, http.MethodPost, "/api/v1/chats/"+chat.ID+"/messages", user.ID, + `{"text":" "}`, http.StatusBadRequest, "invalid_request") +} + +func TestRejectsDuplicateUsername(t *testing.T) { + api := newTestAPI(t) + api.createUser(t, "duplicate_user") + api.requestError(t, http.MethodPost, "/api/v1/users", "", + `{"username":"duplicate_user","display_name":"Duplicate","password":"test-password-12345"}`, http.StatusConflict, "conflict") +} + +func TestPaginatesMessagesAcrossMultiplePages(t *testing.T) { + api := newTestAPI(t) + user := api.createUser(t, "pagination_user") + chat := api.createChat(t, user.ID, []string{user.ID}) + for index := 0; index < 7; index++ { + api.createMessage(t, http.DefaultClient, user.ID, chat.ID, fmt.Sprintf("message-%d", index)) + } + + seen := make(map[string]struct{}) + cursor := "" + pageSizes := make([]int, 0, 4) + for { + path := "/api/v1/chats/" + chat.ID + "/messages?limit=2" + if cursor != "" { + path += "&cursor=" + url.QueryEscape(cursor) + } + var page messagePageResponse + api.request(t, http.DefaultClient, http.MethodGet, path, user.ID, "", http.StatusOK, &page) + pageSizes = append(pageSizes, len(page.Items)) + for _, message := range page.Items { + if _, duplicate := seen[message.ID]; duplicate { + t.Fatalf("message %s appeared on multiple pages", message.ID) + } + seen[message.ID] = struct{}{} + } + if page.NextCursor == nil { + break + } + cursor = *page.NextCursor + } + if len(seen) != 7 { + t.Fatalf("received %d unique messages, want 7", len(seen)) + } + wantPageSizes := []int{2, 2, 2, 1} + if fmt.Sprint(pageSizes) != fmt.Sprint(wantPageSizes) { + t.Fatalf("page sizes=%v, want=%v", pageSizes, wantPageSizes) + } +} + +func TestConcurrentMessagesAreUnique(t *testing.T) { + api := newTestAPI(t) + + users := make([]userResponse, 10) + memberIDs := make([]string, 10) + for index := range users { + users[index] = api.createUser(t, fmt.Sprintf("parallel_user_%02d", index)) + memberIDs[index] = users[index].ID + } + chat := api.createChat(t, users[0].ID, memberIDs) + + start := make(chan struct{}) + errors := make(chan error, 10) + var waitGroup sync.WaitGroup + + for clientIndex := 0; clientIndex < 10; clientIndex++ { + waitGroup.Add(1) + go func(clientIndex int) { + defer waitGroup.Done() + client := &http.Client{Transport: &http.Transport{}, Timeout: 5 * time.Second} + defer client.CloseIdleConnections() + <-start + for messageIndex := 0; messageIndex < 10; messageIndex++ { + body, err := json.Marshal(map[string]string{ + "text": fmt.Sprintf("client-%02d-message-%02d", clientIndex, messageIndex), + }) + if err != nil { + errors <- err + return + } + request, err := http.NewRequest(http.MethodPost, + api.server.URL+"/api/v1/chats/"+chat.ID+"/messages", bytes.NewReader(body)) + if err != nil { + errors <- err + return + } + request.Header.Set("Content-Type", "application/json") + request.Header.Set("Idempotency-Key", uuid.NewString()) + token, _ := api.tokens.Load(users[clientIndex].ID) + request.Header.Set("Authorization", "Bearer "+token.(string)) + response, err := client.Do(request) + if err != nil { + errors <- err + return + } + _, readErr := io.Copy(io.Discard, response.Body) + closeErr := response.Body.Close() + if readErr != nil { + errors <- readErr + return + } + if closeErr != nil { + errors <- closeErr + return + } + if response.StatusCode != http.StatusCreated { + errors <- fmt.Errorf("client %d message %d: status=%d", clientIndex, messageIndex, response.StatusCode) + return + } + } + }(clientIndex) + } + close(start) + waitGroup.Wait() + close(errors) + for err := range errors { + if err != nil { + t.Fatal(err) + } + } + + seenIDs := make(map[string]struct{}, 10*10) + seenTexts := make(map[string]struct{}, 10*10) + cursor := "" + for { + path := "/api/v1/chats/" + chat.ID + "/messages?limit=17" + if cursor != "" { + path += "&cursor=" + url.QueryEscape(cursor) + } + var page messagePageResponse + api.request(t, http.DefaultClient, http.MethodGet, path, users[0].ID, "", http.StatusOK, &page) + for _, message := range page.Items { + if _, duplicate := seenIDs[message.ID]; duplicate { + t.Fatalf("duplicate message ID %s", message.ID) + } + if _, duplicate := seenTexts[message.Text]; duplicate { + t.Fatalf("duplicate message text %q", message.Text) + } + seenIDs[message.ID] = struct{}{} + seenTexts[message.Text] = struct{}{} + } + if page.NextCursor == nil { + break + } + cursor = *page.NextCursor + } + if len(seenIDs) != 10*10 || len(seenTexts) != 10*10 { + t.Fatalf("received %d unique IDs and %d unique texts, want %d of each", + len(seenIDs), len(seenTexts), 10*10) + } +} + +// Session fake is only used by HTTP unit tests; the server always uses Redis. +type testSessions struct { + mu sync.Mutex + values map[string]domain.Session +} + +func (s *testSessions) Ready(context.Context) error { return nil } +func (s *testSessions) Put(_ context.Context, key string, value domain.Session) error { + s.mu.Lock() + defer s.mu.Unlock() + s.values[key] = value + return nil +} +func (s *testSessions) Get(_ context.Context, key string) (domain.Session, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + value, ok := s.values[key] + return value, ok, nil +} +func (s *testSessions) Delete(_ context.Context, key string) (bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + _, ok := s.values[key] + delete(s.values, key) + return ok, nil +} diff --git a/internal/app/chat/session.go b/internal/app/chat/session.go new file mode 100644 index 00000000..29ce755c --- /dev/null +++ b/internal/app/chat/session.go @@ -0,0 +1,45 @@ +package chat + +import ( + "io" + "net/http" + "strings" + + "github.com/labstack/echo/v4" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/dto" +) + +func bearerToken(c echo.Context) string { + parts := strings.Fields(c.Request().Header.Get("Authorization")) + if len(parts) != 2 || !strings.EqualFold(parts[0], "Bearer") { + return "" + } + return parts[1] +} + +func (h *Chat) createSession(c echo.Context) error { + username, password, ok := c.Request().BasicAuth() + if !ok { + c.Response().Header().Set("WWW-Authenticate", `Basic realm="messenger", charset="UTF-8"`) + return h.writeError(c, http.StatusUnauthorized, "unauthorized", "Неверные учётные данные") + } + body, err := io.ReadAll(io.LimitReader(c.Request().Body, 1)) + if err != nil || len(body) != 0 { + return h.invalid(c, "Тело запроса входа должно быть пустым") + } + login, err := h.useCase.CreateSession(c.Request().Context(), username, password) + if err != nil { + return h.useCaseError(c, err) + } + return c.JSON(http.StatusCreated, dto.CreateSessionResponse{ + Token: login.Token, TokenType: "Bearer", ExpiresAt: dto.Timestamp(login.ExpiresAt), User: createUserResponseFromDomain(login.User), + }) +} + +func (h *Chat) deleteSession(c echo.Context) error { + if err := h.useCase.DeleteSession(c.Request().Context(), bearerToken(c)); err != nil { + return h.useCaseError(c, err) + } + return c.NoContent(http.StatusNoContent) +} diff --git a/internal/app/chat/swagger.go b/internal/app/chat/swagger.go new file mode 100644 index 00000000..fc87c65e --- /dev/null +++ b/internal/app/chat/swagger.go @@ -0,0 +1,38 @@ +package chat + +import ( + "encoding/json" + "net/http" + + "github.com/labstack/echo/v4" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/contracts" +) + +var swaggerOpenAPI = makeSwaggerOpenAPI() + +func (h *Chat) swagger(c echo.Context) error { + return c.Redirect(http.StatusTemporaryRedirect, "/swagger/") +} + +func (h *Chat) swaggerSpec(c echo.Context) error { + return c.Blob(http.StatusOK, echo.MIMEApplicationJSON, swaggerOpenAPI) +} + +func makeSwaggerOpenAPI() []byte { + var document map[string]any + if err := json.Unmarshal(contracts.OpenAPI, &document); err != nil { + panic("cannot decode embedded OpenAPI contract: " + err.Error()) + } + // A relative URL keeps Swagger requests on the same scheme, host and port + // as the UI. This also works behind a reverse proxy and on a custom PORT. + document["servers"] = []map[string]string{{ + "url": "/", + "description": "Текущий адрес Swagger UI", + }} + encoded, err := json.Marshal(document) + if err != nil { + panic("cannot encode Swagger OpenAPI contract: " + err.Error()) + } + return encoded +} diff --git a/internal/app/chat/usecase.go b/internal/app/chat/usecase.go new file mode 100644 index 00000000..811a6ef0 --- /dev/null +++ b/internal/app/chat/usecase.go @@ -0,0 +1,21 @@ +package chat + +import ( + "context" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" +) + +// UseCase is the application boundary required by the HTTP handler. +type UseCase interface { + Ready(context.Context) error + CreateSession(context.Context, string, string) (domain.Login, error) + DeleteSession(context.Context, string) error + Authenticate(ctx context.Context, token string) (domain.User, error) + CreateUser(ctx context.Context, user domain.User, password string) (domain.User, error) + CreateChat(ctx context.Context, callerID string, chat domain.Chat) (domain.Chat, error) + GetChat(ctx context.Context, callerID string, chat domain.Chat) (domain.Chat, error) + ListChats(ctx context.Context, callerID string, pagination domain.Pagination) (domain.ChatPage, error) + CreateMessage(ctx context.Context, callerID string, message domain.Message) (domain.Message, error) + ListMessages(ctx context.Context, callerID string, chat domain.Chat, pagination domain.Pagination) (domain.MessagePage, error) +} diff --git a/internal/attachment/image.go b/internal/attachment/image.go new file mode 100644 index 00000000..3e688cfc --- /dev/null +++ b/internal/attachment/image.go @@ -0,0 +1,128 @@ +package attachment + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "image" + _ "image/jpeg" + _ "image/png" + "io" + "mime" + "os" + "strings" + "unicode" + "unicode/utf8" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" +) + +const MaxBytes int64 = 10 * 1024 * 1024 +const MaxPixels int64 = 40_000_000 + +func appError(code domain.ErrorCode, message string) error { + return domain.NewApplicationError(code, message) +} +func unavailable() error { + return appError(domain.ErrorUnavailable, "Хранилище вложений недоступно") +} +func ValidateFilename(name string) error { + if !utf8.ValidString(name) || strings.TrimSpace(name) == "" || utf8.RuneCountInString(name) > 255 { + return appError(domain.ErrorInvalid, "Некорректный filename") + } + for _, r := range name { + if r == '/' || r == '\\' || unicode.IsControl(r) { + return appError(domain.ErrorInvalid, "Недопустимые символы filename") + } + } + return nil +} + +type Original struct { + File *os.File + Size int64 + SHA256, ContentType string + Width, Height int +} + +func (o *Original) Close() { name := o.File.Name(); o.File.Close(); os.Remove(name) } + +// Spool bounds compressed bytes on disk before any decode. DecodeConfig checks the +// pixel budget before a full decode verifies CRCs/truncated scan data. +func Spool(ctx context.Context, body io.Reader, declared string) (*Original, error) { + media, _, err := mime.ParseMediaType(declared) + if err != nil || (media != "image/png" && media != "image/jpeg") { + return nil, appError("unsupported_media_type", "Допустимы PNG и JPEG") + } + f, err := os.CreateTemp("", "messenger-upload-*") + if err != nil { + return nil, unavailable() + } + o := &Original{File: f} + good := false + defer func() { + if !good { + o.Close() + } + }() + hash := sha256.New() + n, err := io.Copy(io.MultiWriter(f, hash), io.LimitReader(&contextReader{ctx: ctx, r: body}, MaxBytes+1)) + if n > MaxBytes { + return nil, appError("payload_too_large", "Файл превышает 10 MiB") + } + if err != nil { + return nil, appError(domain.ErrorInvalid, "Загрузка прервана") + } + o.Size = n + o.SHA256 = hex.EncodeToString(hash.Sum(nil)) + if _, err = f.Seek(0, 0); err != nil { + return nil, unavailable() + } + cfg, format, err := image.DecodeConfig(f) + if err != nil { + return nil, appError("unsupported_media_type", "Повреждённое изображение") + } + actual := "image/" + format + if actual != media { + return nil, appError("unsupported_media_type", "MIME не соответствует содержимому") + } + if cfg.Width <= 0 || cfg.Height <= 0 || int64(cfg.Width) > MaxPixels/int64(cfg.Height) { + return nil, appError("payload_too_large", "Изображение превышает 40 миллионов пикселей") + } + if _, err = f.Seek(0, 0); err != nil { + return nil, unavailable() + } + decoded, format, err := image.Decode(f) + if err != nil || decoded == nil || "image/"+format != actual { + return nil, appError("unsupported_media_type", "Повреждённое изображение") + } + if decoded.Bounds().Dx() != cfg.Width || decoded.Bounds().Dy() != cfg.Height { + return nil, appError("unsupported_media_type", "Некорректные размеры изображения") + } + decoded = nil + if _, err = f.Seek(0, 0); err != nil { + return nil, unavailable() + } + o.ContentType = actual + o.Width = cfg.Width + o.Height = cfg.Height + good = true + return o, nil +} + +type contextReader struct { + ctx context.Context + r io.Reader +} + +func (r *contextReader) Read(p []byte) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + n, err := r.r.Read(p) + if errors.Is(err, context.Canceled) { + return n, err + } + return n, err +} diff --git a/internal/attachment/image_test.go b/internal/attachment/image_test.go new file mode 100644 index 00000000..30700b77 --- /dev/null +++ b/internal/attachment/image_test.go @@ -0,0 +1,139 @@ +package attachment + +import ( + "bytes" + "context" + "encoding/binary" + "hash/crc32" + "image" + "image/color" + "image/jpeg" + "image/png" + "io" + "testing" + "time" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" +) + +func pngFixture(t *testing.T) []byte { + t.Helper() + m := image.NewRGBA(image.Rect(0, 0, 10, 8)) + m.Set(1, 1, color.RGBA{R: 255, A: 255}) + var b bytes.Buffer + if err := png.Encode(&b, m); err != nil { + t.Fatal(err) + } + return b.Bytes() +} +func code(t *testing.T, err error, want domain.ErrorCode) { + t.Helper() + app, ok := domain.AsApplicationError(err) + if !ok || app.Code != want { + t.Fatalf("want %s, got %v", want, err) + } +} +func TestSpoolOriginalAndFullValidation(t *testing.T) { + original := pngFixture(t) + good, err := Spool(context.Background(), bytes.NewReader(original), "image/png") + if err != nil { + t.Fatal(err) + } + defer good.Close() + got, err := io.ReadAll(good.File) + if err != nil || !bytes.Equal(got, original) { + t.Fatal("original changed") + } + if good.Width != 10 || good.Height != 8 || good.Size != int64(len(original)) || len(good.SHA256) != 64 { + t.Fatalf("wrong metadata %+v", good) + } + _, err = Spool(context.Background(), bytes.NewReader(original), "image/jpeg") + code(t, err, "unsupported_media_type") + _, err = Spool(context.Background(), bytes.NewReader(original[:len(original)-12]), "image/png") + code(t, err, "unsupported_media_type") + corrupted := bytes.Clone(original) + corrupted[len(corrupted)-18] ^= 0x80 + _, err = Spool(context.Background(), bytes.NewReader(corrupted), "image/png") + code(t, err, "unsupported_media_type") + var jpegBytes bytes.Buffer + if err = jpeg.Encode(&jpegBytes, image.NewRGBA(image.Rect(0, 0, 12, 9)), nil); err != nil { + t.Fatal(err) + } + jpegOriginal, err := Spool(context.Background(), bytes.NewReader(jpegBytes.Bytes()), "image/jpeg") + if err != nil { + t.Fatal(err) + } + jpegOriginal.Close() + _, err = Spool(context.Background(), bytes.NewReader(jpegBytes.Bytes()[:jpegBytes.Len()/2]), "image/jpeg") + code(t, err, "unsupported_media_type") +} +func TestSpoolLimitsBeforeDecode(t *testing.T) { + // Oversized compressed input is stopped after exactly MaxBytes+1 reads. + source := &countingReader{remaining: MaxBytes + 100_000} + _, err := Spool(context.Background(), source, "image/png") + code(t, err, "payload_too_large") + if source.read != MaxBytes+1 { + t.Fatalf("read %d bytes", source.read) + } + bomb := bytes.Clone(pngFixture(t)) + binary.BigEndian.PutUint32(bomb[16:20], 40001) + binary.BigEndian.PutUint32(bomb[20:24], 1000) + binary.BigEndian.PutUint32(bomb[29:33], crc32.ChecksumIEEE(bomb[12:29])) + _, err = Spool(context.Background(), bytes.NewReader(bomb), "image/png") + code(t, err, "payload_too_large") +} + +type countingReader struct{ remaining, read int64 } + +func (c *countingReader) Read(p []byte) (int, error) { + if c.remaining == 0 { + return 0, io.EOF + } + n := min(int64(len(p)), c.remaining) + clear(p[:n]) + c.remaining -= n + c.read += n + return int(n), nil +} +func TestFilenameAndCancelledUpload(t *testing.T) { + for _, name := range []string{"", "../x.png", "x\\photo.jpg", "a\x00b", "a\nb", "\t"} { + code(t, ValidateFilename(name), domain.ErrorInvalid) + } + for _, name := range []string{"фото.png", "image.jpg", ".."} { + if err := ValidateFilename(name); err != nil { + t.Fatal(err) + } + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := Spool(ctx, bytes.NewReader(pngFixture(t)), "image/png") + code(t, err, domain.ErrorInvalid) +} +func TestReconcileRejectsUnsafeGrace(t *testing.T) { + s := &Service{} + if _, err := s.Reconcile(context.Background(), true, 89*time.Second); err == nil { + t.Fatal("unsafe grace accepted") + } + if MinimumGrace <= UploadTimeout { + t.Fatal("grace does not cover in-flight PUT") + } +} + +func TestValid16BitWidePNG(t *testing.T) { + // Exercise the same RGBA64 + scanline allocation path as an extreme-aspect + // 40M image, at 1/1000 of that pixel count to keep the unit suite small. + img := image.NewNRGBA64(image.Rect(0, 0, 40000, 1)) + img.SetNRGBA64(1, 0, color.NRGBA64{R: 65535, G: 12345, B: 4, A: 32000}) + var encoded bytes.Buffer + if err := png.Encode(&encoded, img); err != nil { + t.Fatal(err) + } + original, err := Spool(context.Background(), bytes.NewReader(encoded.Bytes()), "image/png") + if err != nil { + t.Fatal(err) + } + defer original.Close() + if original.Width != 40000 || original.Height != 1 { + t.Fatal("wide PNG dimensions changed") + } +} diff --git a/internal/attachment/json.go b/internal/attachment/json.go new file mode 100644 index 00000000..cb3958fb --- /dev/null +++ b/internal/attachment/json.go @@ -0,0 +1,18 @@ +package attachment + +import ( + "encoding/json" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/dto" +) + +// MarshalJSON uses the API's fixed microsecond precision instead of time.Time's +// variable fractional precision, which Python 3.9 clients cannot always parse. +func (a Attachment) MarshalJSON() ([]byte, error) { + type fields Attachment + return json.Marshal(struct { + fields + CreatedAt dto.Timestamp `json:"created_at"` + UpdatedAt dto.Timestamp `json:"updated_at"` + }{fields: fields(a), CreatedAt: dto.Timestamp(a.CreatedAt), UpdatedAt: dto.Timestamp(a.UpdatedAt)}) +} diff --git a/internal/attachment/json_test.go b/internal/attachment/json_test.go new file mode 100644 index 00000000..4812d608 --- /dev/null +++ b/internal/attachment/json_test.go @@ -0,0 +1,37 @@ +package attachment + +import ( + "encoding/json" + "testing" + "time" +) + +func TestAttachmentTimestampsUseFixedMicroseconds(t *testing.T) { + value := Attachment{ID: "attachment-id", CreatedAt: time.Date(2026, 1, 2, 3, 4, 5, 123400000, time.UTC), UpdatedAt: time.Date(2026, 1, 2, 6, 4, 5, 0, time.FixedZone("MSK", 3*60*60)), Key: "original/private", Variants: []any{}} + encoded, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + var fields map[string]any + if err = json.Unmarshal(encoded, &fields); err != nil { + t.Fatal(err) + } + if fields["created_at"] != "2026-01-02T03:04:05.123400Z" { + t.Fatalf("created_at=%v", fields["created_at"]) + } + if fields["updated_at"] != "2026-01-02T03:04:05.000000Z" { + t.Fatalf("updated_at=%v", fields["updated_at"]) + } + if fields["id"] != value.ID { + t.Fatal("serialization lost metadata") + } + if _, exposed := fields["Key"]; exposed { + t.Fatal("serialization exposed private object key") + } + if _, exposed := fields["key"]; exposed { + t.Fatal("serialization exposed private object key") + } + if fields["error_code"] != nil { + t.Fatal("ready error_code must be null") + } +} diff --git a/internal/attachment/reconcile.go b/internal/attachment/reconcile.go new file mode 100644 index 00000000..402d7b3e --- /dev/null +++ b/internal/attachment/reconcile.go @@ -0,0 +1,222 @@ +package attachment + +import ( + "context" + "fmt" + "time" + + "github.com/jackc/pgx/v5" + "github.com/minio/minio-go/v7" +) + +type Summary struct { + Apply bool `json:"apply"` + GraceSeconds float64 `json:"grace_seconds"` + ObjectsScanned int `json:"objects_scanned"` + MetadataScanned int `json:"metadata_scanned"` + BytesScanned int64 `json:"bytes_scanned"` + ListRequestsEstimate int `json:"list_requests_estimate"` + StatRequests int `json:"stat_requests"` + DBQueries int `json:"db_queries"` + Orphans int `json:"orphans"` + Missing int `json:"missing"` + FreshSkipped int `json:"fresh_skipped"` + LockedSkipped int `json:"locked_skipped"` + ReferencedMissing int `json:"referenced_missing"` + Deleted int `json:"deleted"` + MarkedFailed int `json:"marked_failed"` + DurationMS int64 `json:"duration_ms"` + Actions []Action `json:"actions"` +} +type Action struct { + Kind string `json:"kind"` + Key string `json:"key"` + Applied bool `json:"applied"` +} + +// Reconcile scans a fixed cutoff and rechecks each candidate while holding the +// same transaction-scoped advisory lock as upload. Metadata, including message +// references, is never deleted. Missing originals become failed tombstones. +func (s *Service) Reconcile(ctx context.Context, apply bool, grace time.Duration) (result Summary, err error) { + started := time.Now() + result = Summary{Apply: apply, GraceSeconds: grace.Seconds(), Actions: []Action{}} + defer func() { result.DurationMS = time.Since(started).Milliseconds() }() + if grace < MinimumGrace { + return result, fmt.Errorf("grace must be at least %s (upload timeout %s)", MinimumGrace, UploadTimeout) + } + cutoff := started.Add(-grace) + for obj := range s.Store.Client.ListObjects(ctx, s.Store.Bucket, minio.ListObjectsOptions{Prefix: "original/", Recursive: true, MaxKeys: 1000}) { + if obj.Err != nil { + return result, fmt.Errorf("S3 listing failed") + } + result.ObjectsScanned++ + result.BytesScanned += obj.Size + if result.ObjectsScanned%1000 == 1 { + result.ListRequestsEstimate++ + } + if !obj.LastModified.Before(cutoff) { + result.FreshSkipped++ + continue + } + err = s.reconcileObject(ctx, obj.Key, cutoff, apply, &result) + if err != nil { + return result, err + } + } + // Bounded keyset pages avoid retaining the entire bucket or a DB cursor while + // making per-object transactions. A later run picks up concurrent new rows. + last := "00000000-0000-0000-0000-000000000000" + for { + result.DBQueries++ + rows, e := s.Pool.Query(ctx, `SELECT id::text,object_key,created_at,status FROM attachments WHERE id>$1 ORDER BY id LIMIT 100`, last) + if e != nil { + return result, fmt.Errorf("metadata scan failed") + } + type metadata struct { + id, key, status string + created time.Time + } + page := []metadata{} + for rows.Next() { + var m metadata + if e = rows.Scan(&m.id, &m.key, &m.created, &m.status); e != nil { + rows.Close() + return result, e + } + page = append(page, m) + } + e = rows.Err() + rows.Close() + if e != nil { + return result, e + } + if len(page) == 0 { + break + } + for _, m := range page { + last = m.id + result.MetadataScanned++ + if m.status != "ready" { + continue + } + if !m.created.Before(cutoff) { + result.FreshSkipped++ + continue + } + if e = s.reconcileMetadata(ctx, m.id, m.key, cutoff, apply, &result); e != nil { + return result, e + } + } + } + if ctx.Err() != nil { + return result, ctx.Err() + } + return result, nil +} +func (s *Service) locked(ctx context.Context, key string, r *Summary) (pgx.Tx, bool, error) { + tx, err := s.Pool.Begin(ctx) + if err != nil { + return nil, false, fmt.Errorf("DB unavailable") + } + var locked bool + r.DBQueries++ + if err = tx.QueryRow(ctx, `SELECT pg_try_advisory_xact_lock(hashtextextended($1,0))`, key).Scan(&locked); err != nil { + tx.Rollback(context.Background()) + return nil, false, fmt.Errorf("lock failed") + } + if !locked { + r.LockedSkipped++ + tx.Rollback(context.Background()) + return nil, false, nil + } + return tx, true, nil +} +func (s *Service) reconcileObject(ctx context.Context, key string, cutoff time.Time, apply bool, r *Summary) error { + tx, ok, err := s.locked(ctx, key, r) + if err != nil || !ok { + return err + } + defer tx.Rollback(context.Background()) + var exists bool + r.DBQueries++ + if err = tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM attachments WHERE object_key=$1)`, key).Scan(&exists); err != nil { + return fmt.Errorf("metadata recheck failed") + } + if exists { + return nil + } + r.StatRequests++ + obj, err := s.Store.Client.StatObject(ctx, s.Store.Bucket, key, minio.StatObjectOptions{}) + if IsMissing(err) { + return nil + } + if err != nil { + return fmt.Errorf("object recheck failed") + } + if !obj.LastModified.Before(cutoff) { + r.FreshSkipped++ + return nil + } + r.Orphans++ + if apply { + if err = s.Store.Client.RemoveObject(ctx, s.Store.Bucket, key, minio.RemoveObjectOptions{}); err != nil { + return fmt.Errorf("orphan deletion failed") + } + r.Deleted++ + } + if len(r.Actions) < 100 { + r.Actions = append(r.Actions, Action{Kind: "orphan", Key: key, Applied: apply}) + } + return nil +} +func (s *Service) reconcileMetadata(ctx context.Context, id, key string, cutoff time.Time, apply bool, r *Summary) error { + tx, ok, err := s.locked(ctx, key, r) + if err != nil || !ok { + return err + } + defer tx.Rollback(context.Background()) + var created time.Time + var status string + r.DBQueries++ + err = tx.QueryRow(ctx, `SELECT created_at,status FROM attachments WHERE id=$1 FOR UPDATE`, id).Scan(&created, &status) + if err == pgx.ErrNoRows { + return nil + } + if err != nil { + return fmt.Errorf("metadata recheck failed") + } + if status != "ready" || !created.Before(cutoff) { + return nil + } + r.StatRequests++ + _, err = s.Store.Client.StatObject(ctx, s.Store.Bucket, key, minio.StatObjectOptions{}) + if err == nil { + return nil + } + if !IsMissing(err) { + return fmt.Errorf("object stat failed (no mutation)") + } + var referenced bool + r.DBQueries++ + if err = tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM messages WHERE $1::uuid=ANY(attachment_ids))`, id).Scan(&referenced); err != nil { + return fmt.Errorf("reference check failed") + } + r.Missing++ + if referenced { + r.ReferencedMissing++ + } + if apply { + r.DBQueries++ + if _, err = tx.Exec(ctx, `UPDATE attachments SET status='failed',error_code='original_missing' WHERE id=$1 AND status='ready'`, id); err != nil { + return fmt.Errorf("mark missing failed") + } + if err = tx.Commit(ctx); err != nil { + return fmt.Errorf("commit missing failed") + } + r.MarkedFailed++ + } + if len(r.Actions) < 100 { + r.Actions = append(r.Actions, Action{Kind: "missing", Key: key, Applied: apply}) + } + return nil +} diff --git a/internal/attachment/s3.go b/internal/attachment/s3.go new file mode 100644 index 00000000..2b1315c9 --- /dev/null +++ b/internal/attachment/s3.go @@ -0,0 +1,44 @@ +package attachment + +import ( + "context" + "crypto/tls" + "net" + "net/http" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +type Store struct { + Client *minio.Client + Bucket string +} + +func NewStore(endpoint, bucket, access, secret string, secure bool) (*Store, error) { + transport := &http.Transport{Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{Timeout: 3 * time.Second, KeepAlive: 30 * time.Second}).DialContext, MaxIdleConns: 16, MaxIdleConnsPerHost: 8, IdleConnTimeout: 30 * time.Second, ResponseHeaderTimeout: 5 * time.Second, TLSHandshakeTimeout: 5 * time.Second, TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12}} + c, err := minio.New(endpoint, &minio.Options{Creds: credentials.NewStaticV4(access, secret, ""), Secure: secure, Region: "us-east-1", Transport: transport, MaxRetries: 1}) + if err != nil { + return nil, err + } + return &Store{Client: c, Bucket: bucket}, nil +} +func (s *Store) Ready(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + for item := range s.Client.ListObjects(ctx, s.Bucket, minio.ListObjectsOptions{Prefix: "original/", MaxKeys: 1, Recursive: true}) { + if item.Err != nil { + return unavailable() + } + return nil + } + if ctx.Err() != nil { + return unavailable() + } + return nil +} +func IsMissing(err error) bool { + code := minio.ToErrorResponse(err).Code + return code == "NoSuchKey" || code == "NoSuchObject" +} diff --git a/internal/attachment/service.go b/internal/attachment/service.go new file mode 100644 index 00000000..3156afbb --- /dev/null +++ b/internal/attachment/service.go @@ -0,0 +1,132 @@ +package attachment + +import ( + "context" + "errors" + "io" + "log" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/minio/minio-go/v7" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" +) + +const UploadTimeout = 45 * time.Second +const MinimumGrace = 90 * time.Second + +type Attachment struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + OwnerID string `json:"owner_id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + Size int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Status string `json:"status"` + Variants []any `json:"variants"` + ErrorCode *string `json:"error_code"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + Key string `json:"-"` +} + +type Service struct { + Pool *pgxpool.Pool + Store *Store + slots chan struct{} +} + +func New(pool *pgxpool.Pool, store *Store) *Service { + return &Service{Pool: pool, Store: store, slots: make(chan struct{}, 1)} +} +func (s *Service) Ready(ctx context.Context) error { return s.Store.Ready(ctx) } +func (s *Service) Upload(ctx context.Context, chatID, userID, name, contentType string, body io.Reader) (Attachment, error) { + if err := ValidateFilename(name); err != nil { + return Attachment{}, err + } + ctx, cancel := context.WithTimeout(ctx, UploadTimeout) + defer cancel() + select { + case s.slots <- struct{}{}: + defer func() { <-s.slots }() + case <-ctx.Done(): + return Attachment{}, unavailable() + } + original, err := Spool(ctx, body, contentType) + if err != nil { + return Attachment{}, err + } + defer original.Close() + a := Attachment{ID: uuid.NewString(), ChatID: chatID, OwnerID: userID, Filename: name, ContentType: original.ContentType, Size: original.Size, SHA256: original.SHA256, Status: "ready", Variants: []any{}} + a.Key = "original/" + a.ID + tx, err := s.Pool.Begin(ctx) + if err != nil { + return Attachment{}, unavailable() + } + defer tx.Rollback(context.Background()) + // Keep the object lock through PUT, metadata insert and COMMIT. A crash releases + // the lock; the grace protects a still-running client PUT at the S3 server. + if _, err = tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, a.Key); err != nil { + return Attachment{}, unavailable() + } + _, err = s.Store.Client.PutObject(ctx, s.Store.Bucket, a.Key, original.File, a.Size, minio.PutObjectOptions{ContentType: a.ContentType, DisableMultipart: true, UserMetadata: map[string]string{"sha256": a.SHA256}}) + if err != nil { + return Attachment{}, unavailable() + } + err = tx.QueryRow(ctx, `INSERT INTO attachments(id,chat_id,uploader_id,filename,object_key,content_type,size_bytes,sha256,width,height) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) RETURNING created_at`, a.ID, a.ChatID, a.OwnerID, a.Filename, a.Key, a.ContentType, a.Size, a.SHA256, original.Width, original.Height).Scan(&a.CreatedAt) + if err != nil { + log.Print("attachment metadata failed; orphan retained for grace-period reconciliation") + return Attachment{}, unavailable() + } + // Never DELETE on an ambiguous COMMIT failure: the DB may have committed. A + // later reconciler checks both stores under the same lock before any cleanup. + if err = tx.Commit(ctx); err != nil { + return Attachment{}, unavailable() + } + a.UpdatedAt = a.CreatedAt + return a, nil +} +func (s *Service) Get(ctx context.Context, id, userID string) (Attachment, error) { + parsed, err := uuid.Parse(id) + if err != nil { + return Attachment{}, appError(domain.ErrorInvalid, "Некорректный attachment_id") + } + ctx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + a := Attachment{Variants: []any{}} + err = s.Pool.QueryRow(ctx, `SELECT a.id::text,a.chat_id::text,a.uploader_id::text,a.filename,a.object_key,a.content_type,a.size_bytes,a.sha256,a.status,a.error_code,a.created_at FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id WHERE a.id=$1 AND m.user_id=$2`, parsed.String(), userID).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.Key, &a.ContentType, &a.Size, &a.SHA256, &a.Status, &a.ErrorCode, &a.CreatedAt) + if errors.Is(err, pgx.ErrNoRows) { + return Attachment{}, appError(domain.ErrorNotFound, "Объект не найден") + } + if err != nil { + return Attachment{}, unavailable() + } + a.UpdatedAt = a.CreatedAt + // Keep the mandatory lab4 ready-only metadata schema. Failed originals are + // represented by 409; internal status remains inspectable in reconciliation. + if a.Status != "ready" { + return Attachment{}, appError(domain.ErrorConflict, "Оригинал вложения недоступен") + } + return a, nil +} +func (s *Service) Content(ctx context.Context, a Attachment) (*minio.Object, error) { + if a.Status != "ready" { + return nil, appError(domain.ErrorConflict, "Вложение не готово") + } + obj, err := s.Store.Client.GetObject(ctx, s.Store.Bucket, a.Key, minio.GetObjectOptions{}) + if err != nil { + return nil, unavailable() + } + if _, err = obj.Stat(); err != nil { + obj.Close() + if IsMissing(err) { + return nil, appError(domain.ErrorConflict, "Оригинал вложения недоступен") + } + return nil, unavailable() + } + return obj, nil +} diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 00000000..ace6c240 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,119 @@ +package config + +import ( + "encoding/hex" + "fmt" + "os" + "strconv" + "strings" + "time" +) + +type Config struct { + S3Endpoint string + S3Bucket string + S3AccessKey string + S3SecretKey string + S3UseSSL bool + Port string + PostgresHost string + PostgresPort uint16 + PostgresUser string + PostgresDatabase string + PostgresPassword string + PostgresMaxConns int32 + RedisAddr string + SessionTTL time.Duration + QueryTimeout time.Duration + CursorKey [32]byte +} + +func value(name, fallback string) string { + if v := os.Getenv(name); v != "" { + return v + } + return fallback +} +func positive(name string, fallback, max int) (int, error) { + n, err := strconv.Atoi(value(name, strconv.Itoa(fallback))) + if err != nil || n < 1 || n > max { + return 0, fmt.Errorf("invalid %s", name) + } + return n, nil +} +func secret(name string) (string, error) { + if file := os.Getenv(name + "_FILE"); file != "" { + data, err := os.ReadFile(file) + if err != nil { + return "", fmt.Errorf("cannot read %s_FILE", name) + } + if s := strings.TrimSpace(string(data)); s != "" { + return s, nil + } + } + if s := os.Getenv(name); s != "" { + return s, nil + } + return "", fmt.Errorf("%s or %s_FILE is required", name, name) +} + +func Load() (Config, error) { + var c Config + port, err := positive("PORT", 8080, 65535) + if err != nil { + return c, err + } + c.Port = strconv.Itoa(port) + pgPort, err := positive("POSTGRES_PORT", 5432, 65535) + if err != nil { + return c, err + } + c.PostgresPort = uint16(pgPort) + max, err := positive("POSTGRES_MAX_CONNS", 10, 100) + if err != nil { + return c, err + } + c.PostgresMaxConns = int32(max) + ttl, err := positive("SESSION_TTL_SECONDS", 3600, 86400*30) + if err != nil { + return c, err + } + c.SessionTTL = time.Duration(ttl) * time.Second + timeout, err := positive("QUERY_TIMEOUT_MS", 2000, 10000) + if err != nil { + return c, err + } + c.QueryTimeout = time.Duration(timeout) * time.Millisecond + c.PostgresHost = value("POSTGRES_HOST", "localhost") + c.PostgresUser = value("POSTGRES_USER", "messenger") + c.PostgresDatabase = value("POSTGRES_DB", "messenger") + c.RedisAddr = value("REDIS_ADDR", "localhost:6379") + c.PostgresPassword, err = secret("POSTGRES_PASSWORD") + if err != nil { + return c, err + } + key, err := secret("CURSOR_KEY") + if err != nil { + return c, err + } + decoded, err := hex.DecodeString(key) + if err != nil || len(decoded) != 32 { + return c, fmt.Errorf("CURSOR_KEY must be 32 random bytes encoded as hex") + } + copy(c.CursorKey[:], decoded) + c.S3Endpoint = value("S3_ENDPOINT", "minio:9000") + c.S3Bucket = value("S3_BUCKET", "attachments") + c.S3UseSSL, err = strconv.ParseBool(value("S3_USE_SSL", "false")) + if err != nil { + return c, fmt.Errorf("invalid S3_USE_SSL") + } + c.S3AccessKey, err = secret("S3_ACCESS_KEY") + if err != nil { + return c, err + } + c.S3SecretKey, err = secret("S3_SECRET_KEY") + if err != nil { + return c, err + } + return c, nil +} diff --git a/internal/pkg/domain/chat.go b/internal/pkg/domain/chat.go new file mode 100644 index 00000000..2de7174e --- /dev/null +++ b/internal/pkg/domain/chat.go @@ -0,0 +1,10 @@ +package domain + +import "time" + +type Chat struct { + ID string + Title string + MemberIDs []string + CreatedAt time.Time +} diff --git a/internal/pkg/domain/errors.go b/internal/pkg/domain/errors.go new file mode 100644 index 00000000..a8447d22 --- /dev/null +++ b/internal/pkg/domain/errors.go @@ -0,0 +1,32 @@ +package domain + +import "errors" + +type ErrorCode string + +const ( + ErrorUnavailable ErrorCode = "unavailable" + ErrorInvalid ErrorCode = "invalid_request" + ErrorUnauthorized ErrorCode = "unauthorized" + ErrorNotFound ErrorCode = "not_found" + ErrorConflict ErrorCode = "conflict" +) + +type ApplicationError struct { + Code ErrorCode + Message string +} + +func (e *ApplicationError) Error() string { + return e.Message +} + +func NewApplicationError(code ErrorCode, message string) error { + return &ApplicationError{Code: code, Message: message} +} + +func AsApplicationError(err error) (*ApplicationError, bool) { + var applicationError *ApplicationError + ok := errors.As(err, &applicationError) + return applicationError, ok +} diff --git a/internal/pkg/domain/message.go b/internal/pkg/domain/message.go new file mode 100644 index 00000000..34879969 --- /dev/null +++ b/internal/pkg/domain/message.go @@ -0,0 +1,13 @@ +package domain + +import "time" + +type Message struct { + IdempotencyKey string + ID string + ChatID string + SenderID string + AttachmentIDs []string + Text string + CreatedAt time.Time +} diff --git a/internal/pkg/domain/pagination.go b/internal/pkg/domain/pagination.go new file mode 100644 index 00000000..d3ab68f4 --- /dev/null +++ b/internal/pkg/domain/pagination.go @@ -0,0 +1,23 @@ +package domain + +import "time" + +type Position struct { + CreatedAt time.Time + ID string +} + +type Pagination struct { + After *Position + Limit int +} + +type ChatPage struct { + Items []Chat + HasMore bool +} + +type MessagePage struct { + Items []Message + HasMore bool +} diff --git a/internal/pkg/domain/session.go b/internal/pkg/domain/session.go new file mode 100644 index 00000000..50029b96 --- /dev/null +++ b/internal/pkg/domain/session.go @@ -0,0 +1,14 @@ +package domain + +import "time" + +type Session struct { + UserID string `json:"user_id"` + ExpiresAt time.Time `json:"expires_at"` +} + +type Login struct { + Token string + ExpiresAt time.Time + User User +} diff --git a/internal/pkg/domain/user.go b/internal/pkg/domain/user.go new file mode 100644 index 00000000..34d52353 --- /dev/null +++ b/internal/pkg/domain/user.go @@ -0,0 +1,10 @@ +package domain + +import "time" + +type User struct { + ID string + Username string + DisplayName string + CreatedAt time.Time +} diff --git a/internal/pkg/password/password.go b/internal/pkg/password/password.go new file mode 100644 index 00000000..1280dca7 --- /dev/null +++ b/internal/pkg/password/password.go @@ -0,0 +1,65 @@ +package password + +import ( + "context" + "crypto/rand" + "crypto/subtle" + "encoding/base64" + "errors" + "strings" + + "golang.org/x/crypto/argon2" +) + +// Bound simultaneous KDF work to 4 * 19 MiB, including concurrent registrations. +var slots = make(chan struct{}, 4) + +const prefix = "$argon2id$v=19$m=19456,t=2,p=1$" + +func derive(ctx context.Context, password string, salt []byte) ([]byte, error) { + select { + case slots <- struct{}{}: + case <-ctx.Done(): + return nil, ctx.Err() + } + defer func() { <-slots }() + if err := ctx.Err(); err != nil { + return nil, err + } + return argon2.IDKey([]byte(password), salt, 2, 19456, 1, 32), nil +} + +func Hash(ctx context.Context, password string) (string, error) { + salt := make([]byte, 16) + if _, err := rand.Read(salt); err != nil { + return "", err + } + key, err := derive(ctx, password, salt) + if err != nil { + return "", err + } + return prefix + base64.RawStdEncoding.EncodeToString(salt) + "$" + base64.RawStdEncoding.EncodeToString(key), nil +} + +func Verify(ctx context.Context, password, encoded string) (bool, error) { + if !strings.HasPrefix(encoded, prefix) { + return false, errors.New("unsupported password hash") + } + parts := strings.Split(strings.TrimPrefix(encoded, prefix), "$") + if len(parts) != 2 { + return false, errors.New("invalid password hash") + } + salt, err := base64.RawStdEncoding.DecodeString(parts[0]) + if err != nil || len(salt) != 16 { + return false, errors.New("invalid salt") + } + expected, err := base64.RawStdEncoding.DecodeString(parts[1]) + if err != nil || len(expected) != 32 { + return false, errors.New("invalid hash") + } + actual, err := derive(ctx, password, salt) + return subtle.ConstantTimeCompare(actual, expected) == 1, err +} + +// A valid fixed encoding for equal-cost verification when username does not exist. +const DummyHash = prefix + "AAAAAAAAAAAAAAAAAAAAAA$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" diff --git a/internal/pkg/password/password_test.go b/internal/pkg/password/password_test.go new file mode 100644 index 00000000..6a0ef969 --- /dev/null +++ b/internal/pkg/password/password_test.go @@ -0,0 +1,35 @@ +package password_test + +import ( + "context" + "strings" + "testing" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/password" +) + +func TestUnicodeAndSalt(t *testing.T) { + ctx := context.Background() + secret := strings.Repeat("🔑", 128) + first, err := password.Hash(ctx, secret) + if err != nil { + t.Fatal(err) + } + second, err := password.Hash(ctx, secret) + if err != nil { + t.Fatal(err) + } + if first == second { + t.Fatal("password hashes reused salt") + } + for _, hash := range []string{first, second} { + ok, err := password.Verify(ctx, secret, hash) + if err != nil || !ok { + t.Fatal("128-code-point UTF-8 password rejected") + } + ok, err = password.Verify(ctx, strings.Repeat("🔑", 127)+"x", hash) + if err != nil || ok { + t.Fatal("password suffix was ignored") + } + } +} diff --git a/internal/pkg/repository/memory.go b/internal/pkg/repository/memory.go new file mode 100644 index 00000000..1c269015 --- /dev/null +++ b/internal/pkg/repository/memory.go @@ -0,0 +1,202 @@ +package repository + +import ( + "context" + "sort" + "sync" + "time" + + "github.com/google/uuid" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" + chatusecase "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/usecase/chat" +) + +// Memory is the lab1 in-memory Repository implementation. +type Memory struct { + mu sync.RWMutex + users map[string]domain.User + hashes map[string]string + userByUsername map[string]string + chats map[string]domain.Chat + messages map[string][]domain.Message +} + +var _ chatusecase.Repository = (*Memory)(nil) + +func NewMemory() *Memory { + return &Memory{ + users: make(map[string]domain.User), + hashes: make(map[string]string), + userByUsername: make(map[string]string), + chats: make(map[string]domain.Chat), + messages: make(map[string][]domain.Message), + } +} + +func (m *Memory) CreateUser(_ context.Context, username, displayName, hash string) (domain.User, error) { + m.mu.Lock() + defer m.mu.Unlock() + + if _, exists := m.userByUsername[username]; exists { + return domain.User{}, chatusecase.ErrUsernameTaken + } + user := domain.User{ + ID: uuid.NewString(), + Username: username, + DisplayName: displayName, + CreatedAt: now(), + } + m.users[user.ID] = user + m.hashes[user.ID] = hash + m.userByUsername[user.Username] = user.ID + return user, nil +} + +func (m *Memory) User(_ context.Context, id string) (domain.User, bool, error) { + m.mu.RLock() + defer m.mu.RUnlock() + user, ok := m.users[id] + return user, ok, nil +} + +func (m *Memory) CreateChat(_ context.Context, title string, memberIDs []string) (domain.Chat, error) { + m.mu.Lock() + defer m.mu.Unlock() + + for _, memberID := range memberIDs { + if _, exists := m.users[memberID]; !exists { + return domain.Chat{}, chatusecase.ErrUserNotFound + } + } + chat := domain.Chat{ + ID: uuid.NewString(), + Title: title, + MemberIDs: append([]string(nil), memberIDs...), + CreatedAt: now(), + } + m.chats[chat.ID] = chat + return cloneChat(chat), nil +} + +func (m *Memory) ChatForUser(_ context.Context, chatID, userID string) (domain.Chat, error) { + m.mu.RLock() + defer m.mu.RUnlock() + chat, exists := m.chats[chatID] + if !exists { + return domain.Chat{}, chatusecase.ErrChatNotFound + } + if !contains(chat.MemberIDs, userID) { + return domain.Chat{}, chatusecase.ErrNotMember + } + return cloneChat(chat), nil +} + +func (m *Memory) ListChats(_ context.Context, userID string, after *domain.Position, limit int) ([]domain.Chat, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + result := make([]domain.Chat, 0) + for _, chat := range m.chats { + if contains(chat.MemberIDs, userID) && isAfter(chat.CreatedAt, chat.ID, after) { + result = append(result, cloneChat(chat)) + } + } + sort.Slice(result, func(i, j int) bool { + return less(result[i].CreatedAt, result[i].ID, result[j].CreatedAt, result[j].ID) + }) + if len(result) > limit { + result = result[:limit] + } + return result, nil +} + +func (m *Memory) CreateMessage(_ context.Context, chatID, senderID, text, key string) (domain.Message, error) { + m.mu.Lock() + defer m.mu.Unlock() + + chat, exists := m.chats[chatID] + if !exists { + return domain.Message{}, chatusecase.ErrChatNotFound + } + if !contains(chat.MemberIDs, senderID) { + return domain.Message{}, chatusecase.ErrNotMember + } + if key != "" { + for _, previous := range m.messages[chatID] { + if previous.SenderID == senderID && previous.IdempotencyKey == key { + if previous.Text != text { + return domain.Message{}, domain.NewApplicationError(domain.ErrorConflict, "Idempotency-Key уже использован с другим телом") + } + return previous, nil + } + } + } + message := domain.Message{ + IdempotencyKey: key, + ID: uuid.NewString(), + ChatID: chatID, + SenderID: senderID, + Text: text, + CreatedAt: now(), + } + m.messages[chatID] = append(m.messages[chatID], message) + return message, nil +} + +func (m *Memory) ListMessages(_ context.Context, chatID string, after *domain.Position, limit int) ([]domain.Message, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + values := m.messages[chatID] + result := make([]domain.Message, 0, min(len(values), limit)) + for _, message := range values { + if isAfter(message.CreatedAt, message.ID, after) { + result = append(result, message) + } + } + sort.Slice(result, func(i, j int) bool { + return less(result[i].CreatedAt, result[i].ID, result[j].CreatedAt, result[j].ID) + }) + if len(result) > limit { + result = result[:limit] + } + return result, nil +} + +func cloneChat(chat domain.Chat) domain.Chat { + chat.MemberIDs = append([]string(nil), chat.MemberIDs...) + return chat +} + +func now() time.Time { + return time.Now().UTC().Truncate(time.Second) +} + +func contains(values []string, target string) bool { + for _, value := range values { + if value == target { + return true + } + } + return false +} + +func isAfter(createdAt time.Time, id string, after *domain.Position) bool { + return after == nil || createdAt.After(after.CreatedAt) || createdAt.Equal(after.CreatedAt) && id > after.ID +} + +func less(leftTime time.Time, leftID string, rightTime time.Time, rightID string) bool { + if leftTime.Equal(rightTime) { + return leftID < rightID + } + return leftTime.Before(rightTime) +} + +func (m *Memory) Ready(context.Context) error { return nil } +func (m *Memory) UserByUsername(_ context.Context, username string) (domain.User, string, bool, error) { + m.mu.RLock() + defer m.mu.RUnlock() + id, ok := m.userByUsername[username] + return m.users[id], m.hashes[id], ok, nil +} diff --git a/internal/pkg/repository/postgres/chats.go b/internal/pkg/repository/postgres/chats.go new file mode 100644 index 00000000..60c9051c --- /dev/null +++ b/internal/pkg/repository/postgres/chats.go @@ -0,0 +1,78 @@ +package postgres + +import ( + "context" + "errors" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" + chatusecase "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/usecase/chat" +) + +func (s *Storage) CreateChat(ctx context.Context, title string, members []string) (domain.Chat, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + tx, err := s.pool.Begin(ctx) + if err != nil { + return domain.Chat{}, dbError(err) + } + defer tx.Rollback(context.WithoutCancel(ctx)) + chat := domain.Chat{ID: uuid.NewString(), Title: title, MemberIDs: append([]string(nil), members...)} + err = tx.QueryRow(ctx, `INSERT INTO chats(id,title) VALUES($1,$2) RETURNING created_at`, chat.ID, title).Scan(&chat.CreatedAt) + if err != nil { + return domain.Chat{}, dbError(err) + } + for i, member := range members { + if _, err = tx.Exec(ctx, `INSERT INTO chat_members(chat_id,user_id,ordinal) VALUES($1,$2,$3)`, chat.ID, member, i); err != nil { + return domain.Chat{}, dbError(err) + } + } + if err = tx.Commit(ctx); err != nil { + return domain.Chat{}, dbError(err) + } + return chat, nil +} + +const chatColumns = `c.id::text,c.title,c.created_at,ARRAY(SELECT m.user_id::text FROM chat_members m WHERE m.chat_id=c.id ORDER BY m.ordinal)` + +func (s *Storage) ChatForUser(ctx context.Context, chatID, userID string) (domain.Chat, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + var chat domain.Chat + err := s.pool.QueryRow(ctx, `SELECT `+chatColumns+` FROM chats c + WHERE c.id=$1 AND EXISTS(SELECT 1 FROM chat_members m WHERE m.chat_id=c.id AND m.user_id=$2)`, chatID, userID). + Scan(&chat.ID, &chat.Title, &chat.CreatedAt, &chat.MemberIDs) + if errors.Is(err, pgx.ErrNoRows) { + return chat, chatusecase.ErrChatNotFound + } + return chat, dbError(err) +} + +func (s *Storage) ListChats(ctx context.Context, userID string, after *domain.Position, limit int) ([]domain.Chat, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + query := `SELECT ` + chatColumns + ` FROM chats c JOIN chat_members member ON member.chat_id=c.id + WHERE member.user_id=$1` + args := []any{userID, limit} + if after != nil { + query += ` AND (c.created_at,c.id)>($3,$4)` + args = append(args, after.CreatedAt, after.ID) + } + query += ` ORDER BY c.created_at,c.id LIMIT $2` + rows, err := s.pool.Query(ctx, query, args...) + if err != nil { + return nil, dbError(err) + } + defer rows.Close() + chats := make([]domain.Chat, 0) + for rows.Next() { + var chat domain.Chat + if err = rows.Scan(&chat.ID, &chat.Title, &chat.CreatedAt, &chat.MemberIDs); err != nil { + return nil, dbError(err) + } + chats = append(chats, chat) + } + return chats, dbError(rows.Err()) +} diff --git a/internal/pkg/repository/postgres/lab4_test.go b/internal/pkg/repository/postgres/lab4_test.go new file mode 100644 index 00000000..53ab8d94 --- /dev/null +++ b/internal/pkg/repository/postgres/lab4_test.go @@ -0,0 +1,341 @@ +package postgres_test + +import ( + "bufio" + "bytes" + "context" + "encoding/xml" + "fmt" + "image" + "image/png" + "io" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync" + "testing" + "time" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/attachment" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/repository/postgres" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +type fakeObject struct { + data []byte + modified time.Time +} +type fakeS3 struct { + mu sync.Mutex + objects map[string]fakeObject + failPUT bool +} + +func (f *fakeS3) ServeHTTP(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + key := strings.TrimPrefix(r.URL.Path, "/attachments/") + if r.Method == "GET" && r.URL.Path == "/attachments/" || r.Method == "GET" && r.URL.Path == "/attachments" { + type item struct { + Key string + LastModified string + ETag string + Size int + } + type listing struct { + XMLName xml.Name `xml:"ListBucketResult"` + Xmlns string `xml:"xmlns,attr"` + Name string + Prefix string + MaxKeys int + IsTruncated bool + Contents []item + } + l := listing{Xmlns: "http://s3.amazonaws.com/doc/2006-03-01/", Name: "attachments", Prefix: r.URL.Query().Get("prefix"), MaxKeys: 1000} + for k, v := range f.objects { + if strings.HasPrefix(k, l.Prefix) { + l.Contents = append(l.Contents, item{Key: k, LastModified: v.modified.UTC().Format("2006-01-02T15:04:05.000Z"), ETag: `"abc"`, Size: len(v.data)}) + } + } + w.Header().Set("Content-Type", "application/xml") + xml.NewEncoder(w).Encode(l) + return + } + if r.Method == "PUT" { + if f.failPUT { + w.WriteHeader(503) + fmt.Fprint(w, `ServiceUnavailable`) + return + } + var data []byte + var err error + if strings.Contains(r.Header.Get("Content-Encoding"), "aws-chunked") || strings.HasPrefix(r.Header.Get("X-Amz-Content-Sha256"), "STREAMING-") { + reader := bufio.NewReader(r.Body) + for { + line, e := reader.ReadString('\n') + if e != nil { + err = e + break + } + size, e := strconv.ParseInt(strings.Split(strings.TrimSpace(line), ";")[0], 16, 64) + if e != nil { + err = e + break + } + if size == 0 { + break + } + chunk := make([]byte, size) + if _, e = io.ReadFull(reader, chunk); e != nil { + err = e + break + } + data = append(data, chunk...) + if _, e = reader.Discard(2); e != nil { + err = e + break + } + } + } else { + data, err = io.ReadAll(r.Body) + } + if err != nil { + w.WriteHeader(400) + return + } + f.objects[key] = fakeObject{data: data, modified: time.Now().UTC()} + w.Header().Set("ETag", `"900150983cd24fb0d6963f7d28e17f72"`) + w.WriteHeader(200) + return + } + obj, ok := f.objects[key] + if !ok { + w.WriteHeader(404) + if r.Method != "HEAD" { + fmt.Fprint(w, `NoSuchKey`) + } + return + } + if r.Method == "DELETE" { + delete(f.objects, key) + w.WriteHeader(204) + return + } + w.Header().Set("Last-Modified", obj.modified.UTC().Format(http.TimeFormat)) + w.Header().Set("ETag", `"abc"`) + w.Header().Set("Content-Length", fmt.Sprint(len(obj.data))) + w.Header().Set("Content-Type", "image/png") + if r.Method == "GET" { + w.Write(obj.data) + } +} +func lab4Backend(t *testing.T, pool *pgxpool.Pool, repo *postgres.Storage, user domain.User, chat domain.Chat) { + ctx := context.Background() + fake := &fakeS3{objects: map[string]fakeObject{}} + server := httptest.NewServer(fake) + defer server.Close() + client, err := minio.New(strings.TrimPrefix(server.URL, "http://"), &minio.Options{Creds: credentials.NewStaticV4("test-key", "test-secret", ""), Region: "us-east-1", MaxRetries: 1}) + if err != nil { + t.Fatal(err) + } + service := attachment.New(pool, &attachment.Store{Client: client, Bucket: "attachments"}) + var imageBytes bytes.Buffer + if err = png.Encode(&imageBytes, image.NewRGBA(image.Rect(0, 0, 10, 8))); err != nil { + t.Fatal(err) + } + upload := func(name string) attachment.Attachment { + t.Helper() + a, e := service.Upload(ctx, chat.ID, user.ID, name, "image/png", bytes.NewReader(imageBytes.Bytes())) + if e != nil { + t.Fatal(e) + } + return a + } + first := upload("first.png") + second := upload("second.png") + requireCode := func(err error, want domain.ErrorCode) { + t.Helper() + e, ok := domain.AsApplicationError(err) + if !ok || e.Code != want { + t.Fatalf("expected %s, got %v", want, err) + } + } + outside, err := repo.CreateUser(ctx, "attachment_outsider", "Outside", "hash") + if err != nil { + t.Fatal(err) + } + _, err = service.Get(ctx, first.ID, outside.ID) + requireCode(err, domain.ErrorNotFound) + obj, err := service.Content(ctx, first) + if err != nil { + t.Fatal(err) + } + data, err := io.ReadAll(obj) + obj.Close() + if err != nil || !bytes.Equal(data, imageBytes.Bytes()) { + t.Fatalf("original mismatch: %v", err) + } + otherChat, err := repo.CreateChat(ctx, "Other", []string{user.ID}) + if err != nil { + t.Fatal(err) + } + _, err = repo.CreateMessageWithAttachments(ctx, otherChat.ID, user.ID, "", "cross-chat", []string{first.ID}) + requireCode(err, domain.ErrorNotFound) + _, err = repo.CreateMessageWithAttachments(ctx, chat.ID, user.ID, "", "unknown", []string{uuid.NewString()}) + requireCode(err, domain.ErrorNotFound) + sent, err := repo.CreateMessageWithAttachments(ctx, chat.ID, user.ID, "", "ordered", []string{first.ID, second.ID}) + if err != nil { + t.Fatal(err) + } + replay, err := repo.CreateMessageWithAttachments(ctx, chat.ID, user.ID, "", "ordered", []string{first.ID, second.ID}) + if err != nil || sent.ID != replay.ID { + t.Fatalf("replay failed: %v", err) + } + _, err = repo.CreateMessageWithAttachments(ctx, chat.ID, user.ID, "", "ordered", []string{second.ID, first.ID}) + requireCode(err, domain.ErrorConflict) + textOnly, err := repo.CreateMessageWithAttachments(ctx, chat.ID, user.ID, "text", "empty-equivalent", nil) + if err != nil { + t.Fatal(err) + } + replay, err = repo.CreateMessageWithAttachments(ctx, chat.ID, user.ID, "text", "empty-equivalent", []string{}) + if err != nil || textOnly.ID != replay.ID { + t.Fatalf("absent/empty differs: %v", err) + } + + const callers = 12 + outcomes := make(chan domain.Message, callers) + failures := make(chan error, callers) + var concurrent sync.WaitGroup + start := make(chan struct{}) + for i := 0; i < callers; i++ { + concurrent.Add(1) + go func() { + defer concurrent.Done() + <-start + m, e := repo.CreateMessageWithAttachments(ctx, chat.ID, user.ID, "", "concurrent-attachments", []string{first.ID, second.ID}) + if e != nil { + failures <- e + } else { + outcomes <- m + } + }() + } + close(start) + concurrent.Wait() + close(outcomes) + close(failures) + for e := range failures { + t.Fatal(e) + } + identity := "" + for m := range outcomes { + if identity == "" { + identity = m.ID + } + if m.ID != identity { + t.Fatal("duplicate concurrent attachment message") + } + } + var effects int + if err = pool.QueryRow(ctx, `SELECT count(*) FROM messages WHERE chat_id=$1 AND idempotency_key='concurrent-attachments'`, chat.ID).Scan(&effects); err != nil || effects != 1 { + t.Fatalf("concurrent effects=%d %v", effects, err) + } + t.Log("ACL, original bytes, cross-chat attachment rejection, ordered/concurrent idempotency and absent=[] passed") + + old := time.Now().Add(-3 * time.Minute) + orphan := "original/" + uuid.NewString() + fresh := "original/" + uuid.NewString() + locked := "original/" + uuid.NewString() + fake.mu.Lock() + delete(fake.objects, first.Key) + v := fake.objects[second.Key] + v.modified = old + fake.objects[second.Key] = v + fake.objects[orphan] = fakeObject{data: []byte("orphan"), modified: old} + fake.objects[fresh] = fakeObject{data: []byte("fresh"), modified: time.Now()} + fake.objects[locked] = fakeObject{data: []byte("locked"), modified: old} + fake.mu.Unlock() + if _, err = pool.Exec(ctx, `UPDATE attachments SET created_at=$1 WHERE id=ANY($2::uuid[])`, old, []string{first.ID, second.ID}); err != nil { + t.Fatal(err) + } + hold, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer hold.Rollback(ctx) + if _, err = hold.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, locked); err != nil { + t.Fatal(err) + } + dry, err := service.Reconcile(ctx, false, 90*time.Second) + if err != nil { + t.Fatal(err) + } + if dry.Orphans != 1 || dry.Missing != 1 || dry.ReferencedMissing != 1 || dry.LockedSkipped != 1 || dry.FreshSkipped < 1 || dry.Deleted != 0 || dry.MarkedFailed != 0 { + t.Fatalf("bad dry-run %+v", dry) + } + var status string + if err = pool.QueryRow(ctx, `SELECT status FROM attachments WHERE id=$1`, first.ID).Scan(&status); err != nil || status != "ready" { + t.Fatalf("dry-run mutated metadata %s %v", status, err) + } + applied, err := service.Reconcile(ctx, true, 90*time.Second) + if err != nil { + t.Fatal(err) + } + if applied.Deleted != 1 || applied.MarkedFailed != 1 { + t.Fatalf("bad apply %+v", applied) + } + fake.mu.Lock() + _, hasOrphan := fake.objects[orphan] + _, hasFresh := fake.objects[fresh] + _, hasLinked := fake.objects[second.Key] + _, hasLocked := fake.objects[locked] + fake.mu.Unlock() + if hasOrphan || !hasFresh || !hasLinked || !hasLocked { + t.Fatal("unsafe object cleanup") + } + _, err = service.Get(ctx, first.ID, user.ID) + requireCode(err, domain.ErrorConflict) + _, err = repo.CreateMessageWithAttachments(ctx, chat.ID, user.ID, "", "failed-attachment", []string{first.ID}) + requireCode(err, domain.ErrorConflict) + replay, err = repo.CreateMessageWithAttachments(ctx, chat.ID, user.ID, "", "ordered", []string{first.ID, second.ID}) + if err != nil || replay.ID != sent.ID { + t.Fatalf("historical replay changed after missing: %v", err) + } + repeated, err := service.Reconcile(ctx, true, 90*time.Second) + if err != nil { + t.Fatal(err) + } + if repeated.Deleted != 0 || repeated.MarkedFailed != 0 || repeated.Orphans != 0 || repeated.Missing != 0 { + t.Fatalf("not repeatable %+v", repeated) + } + t.Logf("dry-run=%+v apply=%+v repeat=%+v; fresh, locked and referenced objects retained", dry, applied, repeated) +} + +func TestS3WireFixture(t *testing.T) { + fake := &fakeS3{objects: map[string]fakeObject{}} + server := httptest.NewServer(fake) + defer server.Close() + client, err := minio.New(strings.TrimPrefix(server.URL, "http://"), &minio.Options{Creds: credentials.NewStaticV4("test-key", "test-secret", ""), Region: "us-east-1", MaxRetries: 1}) + if err != nil { + t.Fatal(err) + } + payload := []byte("exact object bytes") + _, err = client.PutObject(context.Background(), "attachments", "original/test", bytes.NewReader(payload), int64(len(payload)), minio.PutObjectOptions{ContentType: "image/png", DisableMultipart: true}) + if err != nil { + t.Fatal(err) + } + obj, err := client.GetObject(context.Background(), "attachments", "original/test", minio.GetObjectOptions{}) + if err != nil { + t.Fatal(err) + } + defer obj.Close() + got, err := io.ReadAll(obj) + if err != nil || !bytes.Equal(got, payload) { + t.Fatalf("wire fixture did not store original: %q %v", got, err) + } +} diff --git a/internal/pkg/repository/postgres/messages.go b/internal/pkg/repository/postgres/messages.go new file mode 100644 index 00000000..3453e78a --- /dev/null +++ b/internal/pkg/repository/postgres/messages.go @@ -0,0 +1,96 @@ +package postgres + +import ( + "context" + "errors" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "slices" +) + +func (s *Storage) CreateMessage(ctx context.Context, chatID, senderID, text, key string) (domain.Message, error) { + return s.CreateMessageWithAttachments(ctx, chatID, senderID, text, key, nil) +} + +func (s *Storage) CreateMessageWithAttachments(ctx context.Context, chatID, senderID, text, key string, ids []string) (domain.Message, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + if ids == nil { + ids = []string{} + } + tx, err := s.pool.Begin(ctx) + if err != nil { + return domain.Message{}, dbError(err) + } + defer tx.Rollback(context.Background()) + // Serialize this idempotency scope before validating attachments. A replay remains a + // replay even when reconciliation has since marked an original as missing. + if key != "" { + _, err = tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, senderID+":"+chatID+":"+key) + if err != nil { + return domain.Message{}, dbError(err) + } + old := domain.Message{ChatID: chatID, SenderID: senderID} + err = tx.QueryRow(ctx, `SELECT id::text,text,attachment_ids::text[],created_at FROM messages WHERE sender_id=$1 AND chat_id=$2 AND idempotency_key=$3`, senderID, chatID, key).Scan(&old.ID, &old.Text, &old.AttachmentIDs, &old.CreatedAt) + if err == nil { + if old.Text != text || !slices.Equal(old.AttachmentIDs, ids) { + return domain.Message{}, domain.NewApplicationError(domain.ErrorConflict, "Idempotency-Key уже использован с другим телом") + } + return old, nil + } + if !errors.Is(err, pgx.ErrNoRows) { + return domain.Message{}, dbError(err) + } + } + // Lock each metadata row against reconciler updates until the link is committed. + for _, id := range ids { + var status string + err = tx.QueryRow(ctx, `SELECT status FROM attachments WHERE id=$1 AND chat_id=$2 FOR SHARE`, id, chatID).Scan(&status) + if errors.Is(err, pgx.ErrNoRows) { + return domain.Message{}, domain.NewApplicationError(domain.ErrorNotFound, "Объект не найден") + } + if err != nil { + return domain.Message{}, dbError(err) + } + if status != "ready" { + return domain.Message{}, domain.NewApplicationError(domain.ErrorConflict, "Вложение не готово") + } + } + m := domain.Message{ID: uuid.NewString(), ChatID: chatID, SenderID: senderID, Text: text, AttachmentIDs: ids} + var storedKey *string + if key != "" { + storedKey = &key + } + err = tx.QueryRow(ctx, `INSERT INTO messages(id,chat_id,sender_id,text,idempotency_key,attachment_ids) VALUES($1,$2,$3,$4,$5,$6) RETURNING created_at`, m.ID, chatID, senderID, text, storedKey, ids).Scan(&m.CreatedAt) + if err != nil { + return domain.Message{}, dbError(err) + } + return m, dbError(tx.Commit(ctx)) +} + +func (s *Storage) ListMessages(ctx context.Context, chatID string, after *domain.Position, limit int) ([]domain.Message, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + query := `SELECT m.id::text,m.chat_id::text,m.sender_id::text,m.text,m.attachment_ids::text[],m.created_at FROM messages m WHERE m.chat_id=$1` + args := []any{chatID, limit} + if after != nil { + query += ` AND (m.created_at,m.id)>($3,$4)` + args = append(args, after.CreatedAt, after.ID) + } + query += ` ORDER BY m.created_at,m.id LIMIT $2` + rows, err := s.pool.Query(ctx, query, args...) + if err != nil { + return nil, dbError(err) + } + defer rows.Close() + result := []domain.Message{} + for rows.Next() { + var m domain.Message + if err = rows.Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.AttachmentIDs, &m.CreatedAt); err != nil { + return nil, dbError(err) + } + result = append(result, m) + } + return result, dbError(rows.Err()) +} diff --git a/internal/pkg/repository/postgres/storage.go b/internal/pkg/repository/postgres/storage.go new file mode 100644 index 00000000..22a93b14 --- /dev/null +++ b/internal/pkg/repository/postgres/storage.go @@ -0,0 +1,51 @@ +package postgres + +import ( + "context" + "errors" + "log" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" + chatusecase "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/usecase/chat" +) + +type Storage struct { + pool *pgxpool.Pool + timeout time.Duration +} + +var _ chatusecase.Repository = (*Storage)(nil) + +func New(pool *pgxpool.Pool, timeout time.Duration) *Storage { + return &Storage{pool: pool, timeout: timeout} +} + +func (s *Storage) Ready(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + return dbError(s.pool.Ping(ctx)) +} + +func dbError(err error) error { + if err == nil { + return nil + } + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) { + switch pgErr.Code { + case "23505": + return chatusecase.ErrUsernameTaken + case "23503": + return chatusecase.ErrUserNotFound + } + log.Printf("postgres operation failed: sqlstate=%s", pgErr.Code) + } else if !errors.Is(err, pgx.ErrNoRows) { + log.Print("postgres operation failed") + } + return domain.NewApplicationError(domain.ErrorUnavailable, "Хранилище недоступно") +} diff --git a/internal/pkg/repository/postgres/storage_test.go b/internal/pkg/repository/postgres/storage_test.go new file mode 100644 index 00000000..838f1ad0 --- /dev/null +++ b/internal/pkg/repository/postgres/storage_test.go @@ -0,0 +1,289 @@ +package postgres_test + +import ( + "context" + "errors" + "net" + "net/url" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/config" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/repository/postgres" + chatusecase "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/usecase/chat" +) + +func TestIntegration(t *testing.T) { + if os.Getenv("INTEGRATION_TESTS") != "1" { + t.Skip("run make test-integration against the Compose stack") + } + ctx := context.Background() + cfg, err := config.Load() + if err != nil { + t.Fatal(err) + } + pc, err := pgxpool.ParseConfig("sslmode=disable") + if err != nil { + t.Fatal(err) + } + pc.ConnConfig.Host = cfg.PostgresHost + pc.ConnConfig.Port = cfg.PostgresPort + pc.ConnConfig.User = cfg.PostgresUser + pc.ConnConfig.Database = cfg.PostgresDatabase + pc.ConnConfig.Password = cfg.PostgresPassword + pc.ConnConfig.RuntimeParams["timezone"] = "UTC" + pc.MaxConns = 4 + admin, err := pgxpool.NewWithConfig(ctx, pc.Copy()) + if err != nil { + t.Fatal("connect failed") + } + defer admin.Close() + // Identifier is generated by the test, never from an API request. + schema := "test_" + uuid.New().String()[:8] + if _, err = admin.Exec(ctx, `CREATE SCHEMA `+schema); err != nil { + t.Fatal(err) + } + defer admin.Exec(ctx, `DROP SCHEMA `+schema+` CASCADE`) + pc.ConnConfig.RuntimeParams["search_path"] = schema + pool, err := pgxpool.NewWithConfig(ctx, pc) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + runGoose := func(t *testing.T, command string) { + t.Helper() + migrationDir := os.Getenv("GOOSE_MIGRATION_DIR") + if migrationDir == "" { + migrationDir, err = filepath.Abs("../../../../migrations") + if err != nil { + t.Fatal(err) + } + } + connection := url.URL{Scheme: "postgres", User: url.User(cfg.PostgresUser), Host: net.JoinHostPort(cfg.PostgresHost, strconv.Itoa(int(cfg.PostgresPort))), Path: cfg.PostgresDatabase} + connection.RawQuery = url.Values{"sslmode": {"disable"}, "search_path": {schema}, "connect_timeout": {"5"}}.Encode() + commandCtx, cancel := context.WithTimeout(ctx, 45*time.Second) + defer cancel() + commandExec := exec.CommandContext(commandCtx, "goose", "-env", "/dev/null", "-dir", migrationDir, "-timeout", "30s", command) + commandExec.Env = append(os.Environ(), "GOOSE_DRIVER=postgres", "GOOSE_DBSTRING="+connection.String(), "PGPASSWORD="+cfg.PostgresPassword) + output, err := commandExec.CombinedOutput() + if err != nil { + t.Fatalf("goose %s: %v: %s", command, err, strings.ReplaceAll(string(output), cfg.PostgresPassword, "[redacted]")) + } + t.Logf("goose %s: %s", command, output) + } + runGoose(t, "up") + runGoose(t, "up") + var versions int + if err = pool.QueryRow(ctx, `SELECT count(*) FROM goose_db_version WHERE version_id=1 AND is_applied`).Scan(&versions); err != nil || versions != 1 { + t.Fatalf("migration versions=%d error=%v", versions, err) + } + repo := postgres.New(pool, 2*time.Second) + t.Run("registration_race", func(t *testing.T) { + const count = 12 + results := make(chan error, count) + var wg sync.WaitGroup + start := make(chan struct{}) + for i := 0; i < count; i++ { + wg.Add(1) + go func() { + defer wg.Done() + <-start + _, err := repo.CreateUser(ctx, "same_username", "Same", "test-hash") + results <- err + }() + } + close(start) + wg.Wait() + close(results) + successes, conflicts := 0, 0 + for err := range results { + if err == nil { + successes++ + } else if errors.Is(err, chatusecase.ErrUsernameTaken) { + conflicts++ + } else { + t.Fatal(err) + } + } + if successes != 1 || conflicts != count-1 { + t.Fatalf("successes=%d conflicts=%d", successes, conflicts) + } + t.Logf("12 concurrent INSERTs: %d success, %d unique conflicts", successes, conflicts) + }) + user, err := repo.CreateUser(ctx, "history_owner", "Owner", "test-hash") + if err != nil { + t.Fatal(err) + } + t.Run("rollback", func(t *testing.T) { + _, err := repo.CreateChat(ctx, "must rollback", []string{user.ID, uuid.NewString()}) + if !errors.Is(err, chatusecase.ErrUserNotFound) { + t.Fatalf("unexpected error %v", err) + } + var chats, members int + if err = pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM chats),(SELECT count(*) FROM chat_members)`).Scan(&chats, &members); err != nil { + t.Fatal(err) + } + if chats != 0 || members != 0 { + t.Fatalf("partial transaction: chats=%d members=%d", chats, members) + } + t.Log("failed second member INSERT left 0 chats and 0 members") + }) + chat, err := repo.CreateChat(ctx, "History", []string{user.ID}) + if err != nil { + t.Fatal(err) + } + t.Run("lab4_attachments_and_reconciliation", func(t *testing.T) { + isolated, e := repo.CreateChat(ctx, "Lab4", []string{user.ID}) + if e != nil { + t.Fatal(e) + } + lab4Backend(t, pool, repo, user, isolated) + }) + _, err = pool.Exec(ctx, `INSERT INTO messages(id,chat_id,sender_id,text,created_at) + SELECT md5(i::text)::uuid,$1,$2,'synthetic message '||i, + timestamptz '2026-01-01 00:00:00+00' + (i / 10) * interval '1 millisecond' + FROM generate_series(1,10000) i`, chat.ID, user.ID) + if err != nil { + t.Fatal(err) + } + if _, err = pool.Exec(ctx, `ANALYZE messages`); err != nil { + t.Fatal(err) + } + t.Run("keyset_and_explain", func(t *testing.T) { + seen := make(map[string]bool) + var after *domain.Position + for { + page, err := repo.ListMessages(ctx, chat.ID, after, 97) + if err != nil { + t.Fatal(err) + } + if len(page) == 0 { + break + } + for _, message := range page { + if seen[message.ID] { + t.Fatal("duplicate on page boundary") + } + seen[message.ID] = true + } + last := page[len(page)-1] + after = &domain.Position{CreatedAt: last.CreatedAt, ID: last.ID} + } + if len(seen) != 10000 { + t.Fatalf("history count=%d", len(seen)) + } + rows, err := pool.Query(ctx, `EXPLAIN (ANALYZE, BUFFERS) + SELECT m.id::text,m.chat_id::text,m.sender_id::text,m.text,m.created_at FROM messages m + WHERE m.chat_id=$1 AND (m.created_at,m.id)>($2,$3) ORDER BY m.created_at,m.id LIMIT $4`, + chat.ID, time.Date(2026, 1, 1, 0, 0, 0, 900000000, time.UTC), "00000000-0000-0000-0000-000000000000", 51) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + t.Log("EXPLAIN on 10,000 messages, cursor near the end (9,000):") + for rows.Next() { + var line string + if err = rows.Scan(&line); err != nil { + t.Fatal(err) + } + t.Log(line) + } + if err = rows.Err(); err != nil { + t.Fatal(err) + } + t.Log("keyset traversal returned 10,000 distinct messages, including equal timestamps") + }) + t.Run("idempotency_race", func(t *testing.T) { + isolated, err := repo.CreateChat(ctx, "Idempotency", []string{user.ID}) + if err != nil { + t.Fatal(err) + } + const count = 16 + results := make(chan domain.Message, count) + failures := make(chan error, count) + start := make(chan struct{}) + var wg sync.WaitGroup + for i := 0; i < count; i++ { + wg.Add(1) + go func() { + defer wg.Done() + <-start + message, err := repo.CreateMessage(ctx, isolated.ID, user.ID, "exactly once", "shared-key") + if err != nil { + failures <- err + return + } + results <- message + }() + } + close(start) + wg.Wait() + close(results) + close(failures) + for err := range failures { + t.Fatal(err) + } + var first domain.Message + for message := range results { + if first.ID == "" { + first = message + } + if message.ID != first.ID || !message.CreatedAt.Equal(first.CreatedAt) || message.Text != first.Text { + t.Fatal("concurrent replay changed result") + } + } + var rows int + if err := pool.QueryRow(ctx, `SELECT count(*) FROM messages WHERE chat_id=$1`, isolated.ID).Scan(&rows); err != nil || rows != 1 { + t.Fatalf("expected one durable effect, got %d: %v", rows, err) + } + _, err = repo.CreateMessage(ctx, isolated.ID, user.ID, "different", "shared-key") + applicationError, ok := domain.AsApplicationError(err) + if !ok || applicationError.Code != domain.ErrorConflict { + t.Fatalf("expected conflict, got %v", err) + } + t.Log("16 concurrent PostgreSQL requests: one effect, identical replay, changed body conflicts") + }) + t.Run("goose_legacy_and_down", func(t *testing.T) { + // This is the disposable test schema, never the application's public schema. + const digestSQL = `SELECT md5(string_agg(ROW(id,chat_id,sender_id,text,created_at)::text,',' ORDER BY id)) FROM messages` + var before, after string + if err = pool.QueryRow(ctx, digestSQL).Scan(&before); err != nil { + t.Fatal(err) + } + if _, err = pool.Exec(ctx, `DROP TABLE goose_db_version; + CREATE TABLE schema_migrations(version text PRIMARY KEY,applied_at timestamptz NOT NULL DEFAULT now()); + INSERT INTO schema_migrations(version) VALUES('001_initial.sql')`); err != nil { + t.Fatal(err) + } + runGoose(t, "up") + runGoose(t, "up") + if err = pool.QueryRow(ctx, digestSQL).Scan(&after); err != nil { + t.Fatal(err) + } + if before != after { + t.Fatal("legacy migration changed message data") + } + t.Log("adopted legacy schema: all existing messages are unchanged") + runGoose(t, "reset") + var tables int + if err = pool.QueryRow(ctx, `SELECT count(*) FROM information_schema.tables WHERE table_schema=$1 AND table_name IN ('users','chats','chat_members','messages','schema_migrations')`, schema).Scan(&tables); err != nil || tables != 0 { + t.Fatalf("down left %d application tables: %v", tables, err) + } + runGoose(t, "up") + if err = pool.QueryRow(ctx, `SELECT count(*) FROM messages`).Scan(&tables); err != nil || tables != 0 { + t.Fatalf("up after down failed: %v", err) + } + t.Log("down and up succeeded in isolated schema") + }) + +} diff --git a/internal/pkg/repository/postgres/users.go b/internal/pkg/repository/postgres/users.go new file mode 100644 index 00000000..ad3f8c15 --- /dev/null +++ b/internal/pkg/repository/postgres/users.go @@ -0,0 +1,46 @@ +package postgres + +import ( + "context" + "errors" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" +) + +func (s *Storage) CreateUser(ctx context.Context, username, displayName, hash string) (domain.User, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + var user domain.User + err := s.pool.QueryRow(ctx, `INSERT INTO users(id,username,display_name,password_hash) VALUES($1,$2,$3,$4) + RETURNING id::text,username,display_name,created_at`, uuid.NewString(), username, displayName, hash). + Scan(&user.ID, &user.Username, &user.DisplayName, &user.CreatedAt) + return user, dbError(err) +} + +func (s *Storage) User(ctx context.Context, id string) (domain.User, bool, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + var user domain.User + err := s.pool.QueryRow(ctx, `SELECT id::text,username,display_name,created_at FROM users WHERE id=$1`, id). + Scan(&user.ID, &user.Username, &user.DisplayName, &user.CreatedAt) + if errors.Is(err, pgx.ErrNoRows) { + return user, false, nil + } + return user, err == nil, dbError(err) +} + +func (s *Storage) UserByUsername(ctx context.Context, username string) (domain.User, string, bool, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + var user domain.User + var hash string + err := s.pool.QueryRow(ctx, `SELECT id::text,username,display_name,created_at,password_hash FROM users WHERE username=$1`, username). + Scan(&user.ID, &user.Username, &user.DisplayName, &user.CreatedAt, &hash) + if errors.Is(err, pgx.ErrNoRows) { + return user, "", false, nil + } + return user, hash, err == nil, dbError(err) +} diff --git a/internal/pkg/session/redis.go b/internal/pkg/session/redis.go new file mode 100644 index 00000000..13721b4b --- /dev/null +++ b/internal/pkg/session/redis.go @@ -0,0 +1,71 @@ +package session + +import ( + "context" + "encoding/json" + "errors" + "log" + "time" + + "github.com/redis/go-redis/v9" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" +) + +type Redis struct { + client *redis.Client + timeout time.Duration +} + +func NewRedis(client *redis.Client, timeout time.Duration) *Redis { + return &Redis{client: client, timeout: timeout} +} + +func unavailable(err error) error { + if err == nil { + return nil + } + log.Print("redis operation failed") + return domain.NewApplicationError(domain.ErrorUnavailable, "Хранилище сессий недоступно") +} + +func (s *Redis) Ready(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + return unavailable(s.client.Ping(ctx).Err()) +} + +func (s *Redis) Put(ctx context.Context, key string, value domain.Session) error { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + data, err := json.Marshal(value) + if err != nil { + return err + } + // Absolute expiry prevents TTL extension by network delay or restart. + return unavailable(s.client.Do(ctx, "SET", "session:"+key, data, "PXAT", value.ExpiresAt.UnixMilli()).Err()) +} + +func (s *Redis) Get(ctx context.Context, key string) (domain.Session, bool, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + var value domain.Session + data, err := s.client.Get(ctx, "session:"+key).Bytes() + if errors.Is(err, redis.Nil) { + return value, false, nil + } + if err != nil { + return value, false, unavailable(err) + } + if err = json.Unmarshal(data, &value); err != nil { + return value, false, unavailable(err) + } + return value, time.Now().Before(value.ExpiresAt), nil +} + +func (s *Redis) Delete(ctx context.Context, key string) (bool, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + count, err := s.client.Del(ctx, "session:"+key).Result() + return count > 0, unavailable(err) +} diff --git a/internal/usecase/chat/repository.go b/internal/usecase/chat/repository.go new file mode 100644 index 00000000..004d1f67 --- /dev/null +++ b/internal/usecase/chat/repository.go @@ -0,0 +1,28 @@ +package chat + +import ( + "context" + "errors" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" +) + +var ( + ErrUsernameTaken = errors.New("username is already taken") + ErrUserNotFound = errors.New("user not found") + ErrChatNotFound = errors.New("chat not found") + ErrNotMember = errors.New("user is not a chat member") +) + +// Repository is the persistence port required by the chat use case. +type Repository interface { + Ready(ctx context.Context) error + UserByUsername(ctx context.Context, username string) (domain.User, string, bool, error) + CreateUser(ctx context.Context, username, displayName, passwordHash string) (domain.User, error) + User(ctx context.Context, id string) (domain.User, bool, error) + CreateChat(ctx context.Context, title string, memberIDs []string) (domain.Chat, error) + ChatForUser(ctx context.Context, chatID, userID string) (domain.Chat, error) + ListChats(ctx context.Context, userID string, after *domain.Position, limit int) ([]domain.Chat, error) + CreateMessage(ctx context.Context, chatID, senderID, text, key string) (domain.Message, error) + ListMessages(ctx context.Context, chatID string, after *domain.Position, limit int) ([]domain.Message, error) +} diff --git a/internal/usecase/chat/service.go b/internal/usecase/chat/service.go new file mode 100644 index 00000000..0ca9425e --- /dev/null +++ b/internal/usecase/chat/service.go @@ -0,0 +1,182 @@ +package chat + +import ( + "context" + "errors" + "strings" + "time" + "unicode/utf8" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/password" +) + +type UseCase struct { + repository Repository + sessions Sessions + sessionTTL time.Duration +} + +func New(repository Repository, sessions Sessions, ttl time.Duration) *UseCase { + return &UseCase{repository: repository, sessions: sessions, sessionTTL: ttl} +} + +func (u *UseCase) CreateUser(ctx context.Context, user domain.User, secret string) (domain.User, error) { + if !validUsername(user.Username) { + return domain.User{}, invalid("username должен соответствовать [a-z0-9_]{3,32}") + } + if !validRequiredText(user.DisplayName, 100) { + return domain.User{}, invalid("display_name должен содержать от 1 до 100 символов") + } + if !validPassword(secret) { + return domain.User{}, invalid("password должен содержать от 12 до 128 символов") + } + hash, err := password.Hash(ctx, secret) + if err != nil { + return domain.User{}, err + } + created, err := u.repository.CreateUser(ctx, user.Username, user.DisplayName, hash) + if errors.Is(err, ErrUsernameTaken) { + return domain.User{}, domain.NewApplicationError(domain.ErrorConflict, "Пользователь с таким username уже существует") + } + return created, err +} + +func (u *UseCase) CreateChat(ctx context.Context, callerID string, chat domain.Chat) (domain.Chat, error) { + if !validRequiredText(chat.Title, 100) { + return domain.Chat{}, invalid("title должен содержать от 1 до 100 символов") + } + if len(chat.MemberIDs) < 1 || len(chat.MemberIDs) > 100 { + return domain.Chat{}, invalid("member_ids должен содержать от 1 до 100 участников") + } + + normalizedIDs := make([]string, len(chat.MemberIDs)) + seen := make(map[string]struct{}, len(chat.MemberIDs)) + callerIncluded := false + for index, memberID := range chat.MemberIDs { + normalized, valid := normalizeUUID(memberID) + if !valid { + return domain.Chat{}, invalid("member_ids содержит некорректный UUID") + } + if _, duplicate := seen[normalized]; duplicate { + return domain.Chat{}, invalid("member_ids содержит повторяющиеся UUID") + } + normalizedIDs[index] = normalized + seen[normalized] = struct{}{} + callerIncluded = callerIncluded || normalized == callerID + } + if !callerIncluded { + return domain.Chat{}, invalid("текущий пользователь должен входить в member_ids") + } + + created, err := u.repository.CreateChat(ctx, chat.Title, normalizedIDs) + if errors.Is(err, ErrUserNotFound) { + return domain.Chat{}, invalid("member_ids содержит неизвестного пользователя") + } + return created, err +} + +func (u *UseCase) GetChat(ctx context.Context, callerID string, chat domain.Chat) (domain.Chat, error) { + chatID, valid := normalizeUUID(chat.ID) + if !valid { + return domain.Chat{}, invalid("Некорректный chat_id") + } + return u.chatForUser(ctx, chatID, callerID) +} + +func (u *UseCase) ListChats(ctx context.Context, callerID string, pagination domain.Pagination) (domain.ChatPage, error) { + if pagination.Limit < 1 || pagination.Limit > 100 { + return domain.ChatPage{}, invalid("limit должен быть целым числом от 1 до 100") + } + items, err := u.repository.ListChats(ctx, callerID, pagination.After, pagination.Limit+1) + if err != nil { + return domain.ChatPage{}, err + } + page := domain.ChatPage{Items: items, HasMore: len(items) > pagination.Limit} + if page.HasMore { + page.Items = page.Items[:pagination.Limit] + } + return page, nil +} + +func (u *UseCase) CreateMessage(ctx context.Context, callerID string, message domain.Message) (domain.Message, error) { + chatID, valid := normalizeUUID(message.ChatID) + if !valid { + return domain.Message{}, invalid("Некорректный chat_id") + } + if _, err := u.chatForUser(ctx, chatID, callerID); err != nil { + return domain.Message{}, err + } + if !utf8.ValidString(message.Text) || utf8.RuneCountInString(message.Text) > 4000 || (strings.TrimSpace(message.Text) == "" && len(message.AttachmentIDs) == 0) { + return domain.Message{}, invalid("text должен содержать от 1 до 4000 символов") + } + if len(message.AttachmentIDs) > 10 { + return domain.Message{}, invalid("Не более 10 вложений") + } + seen := map[string]bool{} + for i, id := range message.AttachmentIDs { + normalized, ok := normalizeUUID(id) + if !ok || seen[normalized] { + return domain.Message{}, invalid("Некорректные или повторные attachment_ids") + } + seen[normalized] = true + message.AttachmentIDs[i] = normalized + } + var created domain.Message + var err error + if repository, ok := u.repository.(interface { + CreateMessageWithAttachments(context.Context, string, string, string, string, []string) (domain.Message, error) + }); ok { + created, err = repository.CreateMessageWithAttachments(ctx, chatID, callerID, message.Text, message.IdempotencyKey, message.AttachmentIDs) + } else if len(message.AttachmentIDs) > 0 { + return domain.Message{}, notFound() + } else { + created, err = u.repository.CreateMessage(ctx, chatID, callerID, message.Text, message.IdempotencyKey) + } + if errors.Is(err, ErrChatNotFound) || errors.Is(err, ErrNotMember) { + return domain.Message{}, notFound() + } + return created, err +} + +func (u *UseCase) ListMessages(ctx context.Context, callerID string, chat domain.Chat, pagination domain.Pagination) (domain.MessagePage, error) { + chatID, valid := normalizeUUID(chat.ID) + if !valid { + return domain.MessagePage{}, invalid("Некорректный chat_id") + } + if _, err := u.chatForUser(ctx, chatID, callerID); err != nil { + return domain.MessagePage{}, err + } + if pagination.Limit < 1 || pagination.Limit > 100 { + return domain.MessagePage{}, invalid("limit должен быть целым числом от 1 до 100") + } + items, err := u.repository.ListMessages(ctx, chatID, pagination.After, pagination.Limit+1) + if err != nil { + return domain.MessagePage{}, err + } + page := domain.MessagePage{Items: items, HasMore: len(items) > pagination.Limit} + if page.HasMore { + page.Items = page.Items[:pagination.Limit] + } + return page, nil +} + +func (u *UseCase) chatForUser(ctx context.Context, chatID, callerID string) (domain.Chat, error) { + chat, err := u.repository.ChatForUser(ctx, chatID, callerID) + if errors.Is(err, ErrChatNotFound) || errors.Is(err, ErrNotMember) { + return domain.Chat{}, notFound() + } + return chat, err +} + +func invalid(message string) error { + return domain.NewApplicationError(domain.ErrorInvalid, message) +} + +func unauthorized() error { + return domain.NewApplicationError(domain.ErrorUnauthorized, "Требуется действительная сессия") +} + +func notFound() error { + return domain.NewApplicationError(domain.ErrorNotFound, "Объект не найден") +} diff --git a/internal/usecase/chat/session.go b/internal/usecase/chat/session.go new file mode 100644 index 00000000..07d16684 --- /dev/null +++ b/internal/usecase/chat/session.go @@ -0,0 +1,102 @@ +package chat + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "strings" + "time" + "unicode/utf8" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/password" +) + +type Sessions interface { + Put(context.Context, string, domain.Session) error + Get(context.Context, string) (domain.Session, bool, error) + Delete(context.Context, string) (bool, error) + Ready(context.Context) error +} + +func validPassword(value string) bool { + n := utf8.RuneCountInString(value) + return utf8.ValidString(value) && n >= 12 && n <= 128 && strings.TrimSpace(value) != "" +} + +func tokenKey(token string) string { + sum := sha256.Sum256([]byte(token)) + return hex.EncodeToString(sum[:]) +} + +func (u *UseCase) Authenticate(ctx context.Context, token string) (domain.User, error) { + decoded, err := base64.RawURLEncoding.DecodeString(token) + if err != nil || len(decoded) != 32 { + return domain.User{}, unauthorized() + } + session, exists, err := u.sessions.Get(ctx, tokenKey(token)) + if err != nil { + return domain.User{}, err + } + if !exists || !time.Now().Before(session.ExpiresAt) { + return domain.User{}, unauthorized() + } + user, exists, err := u.repository.User(ctx, session.UserID) + if err != nil { + return domain.User{}, err + } + if !exists { + return domain.User{}, unauthorized() + } + return user, nil +} + +func (u *UseCase) CreateSession(ctx context.Context, username, secret string) (domain.Login, error) { + if !validUsername(username) || !validPassword(secret) { + return domain.Login{}, unauthorized() + } + user, hash, exists, err := u.repository.UserByUsername(ctx, username) + if err != nil { + return domain.Login{}, err + } + if !exists { + hash = password.DummyHash + } + valid, err := password.Verify(ctx, secret, hash) + if err != nil { + return domain.Login{}, err + } + if !valid || !exists { + return domain.Login{}, unauthorized() + } + var raw [32]byte + if _, err = rand.Read(raw[:]); err != nil { + return domain.Login{}, err + } + token := base64.RawURLEncoding.EncodeToString(raw[:]) + expires := time.Now().UTC().Add(u.sessionTTL) + if err = u.sessions.Put(ctx, tokenKey(token), domain.Session{UserID: user.ID, ExpiresAt: expires}); err != nil { + return domain.Login{}, err + } + return domain.Login{Token: token, ExpiresAt: expires, User: user}, nil +} + +func (u *UseCase) DeleteSession(ctx context.Context, token string) error { + deleted, err := u.sessions.Delete(ctx, tokenKey(token)) + if err != nil { + return err + } + if !deleted { + return unauthorized() + } + return nil +} + +func (u *UseCase) Ready(ctx context.Context) error { + if err := u.repository.Ready(ctx); err != nil { + return err + } + return u.sessions.Ready(ctx) +} diff --git a/internal/usecase/chat/session_test.go b/internal/usecase/chat/session_test.go new file mode 100644 index 00000000..5096e1ba --- /dev/null +++ b/internal/usecase/chat/session_test.go @@ -0,0 +1,19 @@ +package chat + +import ( + "strings" + "testing" +) + +func TestPasswordContract(t *testing.T) { + for _, value := range []string{"", "too-short", strings.Repeat(" ", 12), strings.Repeat("x", 129), strings.Repeat("x", 12) + string([]byte{0xff})} { + if validPassword(value) { + t.Fatal("invalid password accepted") + } + } + for _, value := range []string{strings.Repeat("🔑", 128), "valid:password", " password with spaces "} { + if !validPassword(value) { + t.Fatal("valid Unicode password rejected") + } + } +} diff --git a/internal/usecase/chat/validation.go b/internal/usecase/chat/validation.go new file mode 100644 index 00000000..5fe43df1 --- /dev/null +++ b/internal/usecase/chat/validation.go @@ -0,0 +1,37 @@ +package chat + +import ( + "strings" + "unicode/utf8" + + "github.com/google/uuid" +) + +func normalizeUUID(value string) (string, bool) { + parsed, err := uuid.Parse(value) + if err != nil { + return "", false + } + normalized := parsed.String() + if !strings.EqualFold(value, normalized) { + return "", false + } + return normalized, true +} + +func validUsername(value string) bool { + if len(value) < 3 || len(value) > 32 { + return false + } + for _, char := range value { + if char != '_' && (char < 'a' || char > 'z') && (char < '0' || char > '9') { + return false + } + } + return true +} + +func validRequiredText(value string, maxRunes int) bool { + length := utf8.RuneCountInString(value) + return utf8.ValidString(value) && length >= 1 && length <= maxRunes && strings.TrimSpace(value) != "" +} diff --git a/lab.json b/lab.json index db0c9c35..cc86272e 100644 --- a/lab.json +++ b/lab.json @@ -1,8 +1,8 @@ { - "number": 5, - "title": "Очереди и асинхронная обработка изображений", - "contract_version": "1.5.0", + "number": 4, + "title": "S3-хранилище и вложения", + "contract_version": "1.4.0", "default_base_url": "http://localhost:8080", "grading": "3; 4 includes 3; 5 includes 4 plus one completed research track", - "scaffold_has_application": false + "scaffold_has_application": true } diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..2433b274 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,54 @@ +-- +goose Up +-- +goose StatementBegin +DO $$ +BEGIN + -- Adopt the schema created by the previous runner without changing its data. + IF to_regclass('schema_migrations') IS NOT NULL THEN + IF NOT EXISTS (SELECT 1 FROM schema_migrations WHERE version = '001_initial.sql') + OR to_regclass('users') IS NULL OR to_regclass('chats') IS NULL + OR to_regclass('chat_members') IS NULL OR to_regclass('messages') IS NULL THEN + RAISE EXCEPTION 'incomplete legacy schema: cannot adopt migration 1'; + END IF; + RETURN; + END IF; + CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() + ); + CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() + ); + CREATE INDEX chats_history_idx ON chats (created_at, id); + CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id), + user_id uuid NOT NULL REFERENCES users(id), + ordinal integer NOT NULL CHECK (ordinal >= 0), + PRIMARY KEY (chat_id, user_id), + UNIQUE (chat_id, ordinal) + ); + CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id); + CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id) + ); + CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id); +END; +$$; +-- +goose StatementEnd + +-- +goose Down +DROP TABLE messages; +DROP TABLE chat_members; +DROP TABLE chats; +DROP TABLE users; +-- The old journal must not claim that the dropped schema is still applied. +DROP TABLE IF EXISTS schema_migrations; diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..fb02f93c --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,10 @@ +-- +goose Up +-- Additive: old lab2 writers can still insert NULL keys during migration. +ALTER TABLE messages ADD COLUMN IF NOT EXISTS idempotency_key text + CHECK (idempotency_key ~ '^[A-Za-z0-9._:-]{1,128}$'); +CREATE UNIQUE INDEX IF NOT EXISTS messages_idempotency_idx + ON messages(sender_id, chat_id, idempotency_key); + +-- +goose Down +DROP INDEX messages_idempotency_idx; +ALTER TABLE messages DROP COLUMN idempotency_key; diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 00000000..a8b812ad --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,20 @@ +-- +goose Up +CREATE TABLE IF NOT EXISTS attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + uploader_id uuid NOT NULL REFERENCES users(id), + filename text NOT NULL, + object_key text NOT NULL UNIQUE, + content_type text NOT NULL CHECK (content_type IN ('image/png','image/jpeg')), + size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + width integer NOT NULL, + height integer NOT NULL, + status text NOT NULL DEFAULT 'ready' CHECK (status IN ('ready','failed')), + error_code text, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CHECK ((status='ready' AND error_code IS NULL) OR (status='failed' AND error_code IS NOT NULL)) +); + +-- +goose Down +DROP TABLE IF EXISTS attachments; diff --git a/migrations/004_message_attachments.sql b/migrations/004_message_attachments.sql new file mode 100644 index 00000000..1fd41a98 --- /dev/null +++ b/migrations/004_message_attachments.sql @@ -0,0 +1,13 @@ +-- +goose Up +ALTER TABLE messages + DROP CONSTRAINT IF EXISTS messages_text_check, + ADD CONSTRAINT messages_text_check CHECK (char_length(text) BETWEEN 0 AND 4000), + ADD COLUMN IF NOT EXISTS attachment_ids uuid[] NOT NULL DEFAULT '{}'; + +-- +goose Down +-- Attachment-only messages cannot exist in lab3; retain an explicit placeholder when downgrading. +UPDATE messages SET text='[attachment]' WHERE text=''; +ALTER TABLE messages + DROP COLUMN IF EXISTS attachment_ids, + DROP CONSTRAINT IF EXISTS messages_text_check, + ADD CONSTRAINT messages_text_check CHECK (char_length(text) BETWEEN 1 AND 4000); diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh new file mode 100755 index 00000000..3a183673 --- /dev/null +++ b/scripts/bootstrap.sh @@ -0,0 +1,20 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +umask 077 +mkdir -p secrets +chmod 700 secrets +for name in postgres_password cursor_key s3_root_secret_key s3_secret_key; do + if [ ! -s "secrets/$name" ]; then + openssl rand -hex 32 > "secrets/$name" + fi +done +for name in s3_root_access_key s3_access_key; do + if [ ! -s "secrets/$name" ]; then + openssl rand -hex 8 > "secrets/$name" + fi +done +# Compose mounts files directly on Linux; the unprivileged API must read them. +# The host directory remains private (0700). +chmod 444 secrets/postgres_password secrets/cursor_key secrets/s3_root_access_key secrets/s3_root_secret_key secrets/s3_access_key secrets/s3_secret_key +printf '%s\n' 'Local secrets are ready.' diff --git a/scripts/compose.sh b/scripts/compose.sh new file mode 100755 index 00000000..078e4f0d --- /dev/null +++ b/scripts/compose.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +if docker compose version >/dev/null 2>&1; then + exec docker compose --env-file /dev/null -f compose.yaml "$@" +fi +exec docker-compose --env-file /dev/null -f compose.yaml "$@" diff --git a/scripts/measure_load.py b/scripts/measure_load.py new file mode 100644 index 00000000..82481f07 --- /dev/null +++ b/scripts/measure_load.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Reproducible local benchmark with resource samples; no credentials in output.""" +import argparse +import json +import os +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +STACK = os.environ.get('STACK_NAME', 'mtusi-lab4') + +def containers(service=None): + label = 'com.docker.swarm.service.name='+STACK+'_'+service if service else 'com.docker.stack.namespace='+STACK + return subprocess.check_output(['docker','ps','--filter','label='+label,'-q'],text=True).split() + + +def counts(): + sql = 'SELECT json_build_object(\'users\',(SELECT count(*) FROM users),\'chats\',(SELECT count(*) FROM chats),\'messages\',(SELECT count(*) FROM messages))' + return json.loads(subprocess.check_output(['docker', 'exec', containers('postgres')[0], 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', sql], text=True)) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--base-url', default=os.environ.get('BASE_URL', 'http://localhost:8080')) + parser.add_argument('--duration', type=int, default=30) + parser.add_argument('--concurrency', type=int, default=4) + args = parser.parse_args() + command = [sys.executable, str(ROOT / 'tests/load.py'), '--base-url', args.base_url, '--concurrency', str(args.concurrency)] + subprocess.run(command + ['--duration', '3'], check=True, stdout=subprocess.DEVNULL) + before = counts() + ids = containers() + samples = [] + process = subprocess.Popen(command + ['--duration', str(args.duration), '--output', str(ROOT / 'evidence/swarm-load.json')], stdout=subprocess.DEVNULL) + try: + while process.poll() is None: + stats = subprocess.check_output(['docker', 'stats', '--no-stream', '--format', '{{json .}}', *ids], text=True) + samples.append({'elapsed_sample': len(samples), 'containers': [json.loads(line) for line in stats.splitlines()]}) + time.sleep(1) + finally: + if process.poll() is None: + process.terminate() + code = process.wait() + if code: + raise SystemExit(code) + report = {'warmup_seconds': 3, 'dataset_before': before, 'dataset_after': counts(), 'samples': samples} + (ROOT / 'evidence/swarm-resources.json').write_text(json.dumps(report, indent=2) + '\n') + print((ROOT / 'evidence/swarm-load.json').read_text()) + print('Resource samples and dataset counts: evidence/swarm-resources.json') + + +if __name__ == '__main__': + main() diff --git a/scripts/measure_uploads.py b/scripts/measure_uploads.py new file mode 100644 index 00000000..7a0de357 --- /dev/null +++ b/scripts/measure_uploads.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Ten simultaneous proxy uploads, checksums, latency and cgroup memory samples.""" +from concurrent.futures import ThreadPoolExecutor +import hashlib +import json +import math +from pathlib import Path +import sys +import threading +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT/'tests/own')) +from helpers import Client, check, containers, docker, png, save, upload + + +def memory(ids): + result = {} + for container in ids: + raw = docker('exec',container,'cat','/sys/fs/cgroup/memory.current').stdout + result[container] = int(raw) + return result + + +def percentile(values, p): + return sorted(values)[max(0, math.ceil(len(values)*p)-1)] + + +def kernel_peak(ids): + return {container:int(docker('exec',container,'cat','/sys/fs/cgroup/memory.peak').stdout) + for container in ids} + + +def main(): + c = Client(timeout=55) + alice,bob,_,chat = c.bootstrap() + upload(c,alice,chat,png()) + ids = containers('api') + results = {'concurrency':10, 'warmup_uploads':1, 'memory_method':'cgroup v2 memory.current sampled every >=100ms; values include file cache', 'runs':[]} + for width,height in ((512,512),(1024,768)): + content = png(width,height,noisy=True) + sha = hashlib.sha256(content).hexdigest() + baseline = memory(ids) + baseline_kernel_peak = kernel_peak(ids) + samples = [] + barrier = threading.Barrier(11) + def worker(number): + local = Client(timeout=55) + barrier.wait() + return upload(local,alice,chat,content,filename='concurrent-'+str(number)+'.png') + started = time.monotonic() + with ThreadPoolExecutor(max_workers=10) as pool: + futures = [pool.submit(worker,n) for n in range(10)] + barrier.wait() + while not all(f.done() for f in futures): + samples.append({'seconds':round(time.monotonic()-started,3), 'bytes':memory(ids)}) + time.sleep(.1) + responses = [f.result() for f in futures] + elapsed = time.monotonic()-started + samples.append({'seconds':round(elapsed,3), 'bytes':memory(ids)}) + for response in responses: + att = response['json'] + check(att['sha256']==sha,'concurrent upload checksum mismatch') + downloaded = c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob) + check(hashlib.sha256(downloaded['bytes']).hexdigest()==sha,'concurrent downloaded bytes changed') + latency = [r['elapsed']*1000 for r in responses] + peak = {container:max(sample['bytes'][container] for sample in samples) for container in ids} + run = {'width':width, 'height':height, 'size_bytes':len(content), 'sha256':sha, + 'requests':10, 'successful':len(responses), 'errors':0, 'duration_seconds':round(elapsed,3), + 'successful_rps':round(len(responses)/elapsed,3), + 'latency_ms':{str(p):round(percentile(latency,p),3) for p in (.5,.95,.99)}, + 'baseline_bytes':baseline, 'sampled_peak_bytes':peak, + 'kernel_lifetime_peak_before_bytes':baseline_kernel_peak, + 'kernel_lifetime_peak_after_bytes':kernel_peak(ids), + 'sampled_peak_sum_bytes':max(sum(sample['bytes'].values()) for sample in samples), + 'samples':samples} + check(all(value<1536*1024*1024 for value in peak.values()),'API exceeded memory limit') + results['runs'].append(run) + print(json.dumps({k:v for k,v in run.items() if k!='samples'}),flush=True) + save('lab4-upload-memory.json',results) + print('PASS: ten simultaneous uploads at two sizes; all originals verified; memory samples saved.') + + +if __name__=='__main__': + main() diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..211f35b4 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec python3 scripts/swarm.py restart diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..8a93f7f0 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec python3 scripts/swarm.py stop-one diff --git a/scripts/swarm.py b/scripts/swarm.py new file mode 100644 index 00000000..a40d801f --- /dev/null +++ b/scripts/swarm.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python3 +"""Single-node Swarm lab lifecycle; never deletes volumes or prints credentials.""" +import json +import os +from pathlib import Path +import subprocess +import sys +import time +import urllib.request + +ROOT = Path(__file__).resolve().parents[1] +os.chdir(ROOT) +STACK = os.environ.get('STACK_NAME', 'mtusi-lab4') +API = os.environ.get('API_IMAGE', 'mtusi-messenger:lab4') +MIGRATIONS = os.environ.get('MIGRATIONS_IMAGE', 'mtusi-migrations:lab4') +TESTS = os.environ.get('TESTS_IMAGE', 'mtusi-messenger-tests:lab4') +S3 = os.environ.get('S3_IMAGE', 'mtusi-minio:RELEASE.2025-09-07T16-13-09Z') +MC = os.environ.get('S3_CLIENT_IMAGE', 'mtusi-mc:RELEASE.2025-08-13T08-35-41Z') +PORT = os.environ.get('PORT', '8080') +SECRETS = ('postgres_password', 'cursor_key', 's3_root_access_key', + 's3_root_secret_key', 's3_access_key', 's3_secret_key') + + +def docker(*args, capture=False, **kwargs): + return subprocess.run(['docker', *args], check=True, text=True, + stdout=subprocess.PIPE if capture else None, **kwargs).stdout + + +def inspect(kind, name): + return json.loads(docker(kind, 'inspect', name, capture=True))[0] + + +def tasks(service): + ids = docker('service', 'ps', '--filter', 'desired-state=running', '-q', STACK+'_'+service, capture=True).split() + return [json.loads(docker('inspect', task, capture=True))[0] for task in ids] + + +def wait_service(service, count=1, timeout=120): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + active = tasks(service) + if len(active) == count and all(t['Status']['State'] == 'running' for t in active): + # This lab pins storage to this manager and runs on one Docker engine. + healthy = True + for task in active: + container = task['Status'].get('ContainerStatus', {}).get('ContainerID') + state = json.loads(docker('inspect', container, capture=True))[0]['State'] + healthy &= state.get('Health', {}).get('Status', 'healthy') == 'healthy' + if healthy: + return + time.sleep(1) + raise RuntimeError('service did not become healthy: '+service) + + +def wait_api(): + wait_service('api', int(os.environ.get('API_REPLICAS', '2'))) + deadline = time.monotonic()+45 + while time.monotonic() < deadline: + try: + with urllib.request.urlopen('http://localhost:'+PORT+'/health/ready', timeout=3) as response: + if response.status == 200: + return + except OSError: + pass + time.sleep(.5) + raise RuntimeError('LB readiness timeout') + + +def job(kind, image, command, integration=False): + name = STACK+'_'+kind + # Fixed service name is a deployment lock: concurrent runners fail to create it. + args = ['service', 'create', '--detach', '--no-resolve-image', '--name', name, '--mode', 'replicated-job', + '--replicas', '1', '--restart-condition', 'none', '--network', STACK+'_data', + '--constraint', 'node.labels.mtusi_lab4_storage==true', + '--limit-cpu', '1', '--limit-memory', '1g' if integration else '128m', + '--secret', 'source='+STACK+'_postgres_password,target=postgres_password', + '--env', 'POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password'] + if integration: + for value in ['INTEGRATION_TESTS=1', 'POSTGRES_HOST=postgres', 'POSTGRES_USER=messenger', + 'POSTGRES_DB=messenger', 'REDIS_ADDR=redis:6379', 'CURSOR_KEY_FILE=/run/secrets/cursor_key', + 'S3_ENDPOINT=minio:9000', 'S3_BUCKET=attachments', + 'S3_ACCESS_KEY_FILE=/run/secrets/s3_access_key', 'S3_SECRET_KEY_FILE=/run/secrets/s3_secret_key']: + args += ['--env', value] + args += ['--secret', 'source='+STACK+'_cursor_key,target=cursor_key'] + for secret in ('s3_access_key', 's3_secret_key'): + args += ['--secret', 'source='+STACK+'_'+secret+',target='+secret] + else: + args += ['--env', 'GOOSE_DRIVER=postgres', '--env', + 'GOOSE_DBSTRING=host=postgres user=messenger dbname=messenger sslmode=disable connect_timeout=5'] + docker(*args, image, *command) + wait_job(name, kind, 300 if integration else 90) + + +def wait_job(name, kind, timeout=90): + try: + deadline = time.monotonic()+timeout + while time.monotonic() < deadline: + ids = docker('service', 'ps', '-q', name, capture=True).split() + states = [json.loads(docker('inspect', i, capture=True))[0]['Status']['State'] for i in ids] + if 'complete' in states: + docker('service', 'logs', name) + return + if any(s in ('failed', 'rejected') for s in states): + docker('service', 'logs', name) + raise RuntimeError(kind+' failed') + time.sleep(1) + raise RuntimeError(kind+' timed out') + finally: + docker('service', 'rm', name) + + +def bootstrap_s3(): + name = STACK+'_s3-bootstrap' + args = ['service', 'create', '--detach', '--no-resolve-image', '--name', name, '--mode', 'replicated-job', + '--replicas', '1', '--restart-condition', 'none', '--network', STACK+'_data', + '--constraint', 'node.labels.mtusi_lab4_storage==true', + '--limit-cpu', '0.5', '--limit-memory', '512m', '--entrypoint', '/bin/sh', + '--config', 'source='+STACK+'_s3_bootstrap,target=/bootstrap.sh', + '--config', 'source='+STACK+'_s3_policy,target=/policy.json'] + for secret in SECRETS[2:]: + args += ['--secret', 'source='+STACK+'_'+secret+',target='+secret] + docker(*args, MC, '/bootstrap.sh') + wait_job(name, 'S3 bootstrap') + + +def reconcile(): + active = tasks('api') + if not active: + raise RuntimeError('no API task available for reconciliation') + container = active[0]['Status']['ContainerStatus']['ContainerID'] + docker('exec', container, '/usr/local/bin/reconcile', *sys.argv[2:]) + + +def build(): + docker('build', '-f', 'Dockerfile.storage', '--target', 'minio', '-t', S3, '.') + docker('build', '-f', 'Dockerfile.storage', '--target', 'mc', '-t', MC, '.') + docker('build', '-t', API, '.') + docker('build', '--target', 'migrations', '-t', MIGRATIONS, '.') + + +def up(): + swarm = json.loads(docker('info', '--format', '{{json .Swarm}}', capture=True)) + if swarm['LocalNodeState'] == 'inactive': + docker('swarm', 'init', capture=True) # join token stays out of logs + swarm = json.loads(docker('info', '--format', '{{json .Swarm}}', capture=True)) + if not swarm['ControlAvailable']: + raise RuntimeError('run on the Swarm manager') + if len(docker('node', 'ls', '-q', capture=True).split()) != 1: + raise RuntimeError('automatic lifecycle supports one Swarm node; see REPORT.md for image distribution') + docker('node', 'update', '--label-add', 'mtusi_lab4_storage=true', swarm['NodeID']) + subprocess.run(['./scripts/bootstrap.sh'], check=True) + for secret in SECRETS: + known = docker('secret', 'ls', '--format', '{{.Name}}', capture=True).split() + if STACK+'_'+secret not in known: + docker('secret', 'create', STACK+'_'+secret, 'secrets/'+secret) + build() + existing = docker('service', 'ls', '--format', '{{.Name}}', capture=True).split() + # Upgrade the shared schema before replacing any running API image. + fresh = STACK+'_api' not in existing + if not fresh: + job('migrations', MIGRATIONS, ['up']) + # Initial deployment keeps API at zero until the one-shot migration succeeds. + env = dict(os.environ) + env['STACK_NAME'] = STACK + if fresh: + env['API_REPLICAS'] = '0' + docker('stack', 'deploy', '--resolve-image', 'never', '-c', 'stack.yaml', STACK, env=env) + wait_service('postgres') + wait_service('redis') + wait_service('minio') + bootstrap_s3() + if fresh: + job('migrations', MIGRATIONS, ['up']) + docker('service', 'scale', '--detach', STACK+'_api='+os.environ.get('API_REPLICAS', '2')) + if fresh: + wait_api() + else: + update() + print('PASS: Swarm stack ready at http://localhost:'+PORT) + + +def restart(): + # Force replaces every task while retaining the same Swarm secrets and volumes. + services = ('postgres', 'redis', 'minio', 'api', 'lb') + previous = {name: {t['ID'] for t in tasks(name)} for name in services} + for service in services: + docker('service', 'update', '--detach', '--force', STACK+'_'+service) + deadline = time.monotonic()+75 + while any(previous[name].intersection(t['ID'] for t in tasks(name)) for name in services): + if time.monotonic() > deadline: + raise RuntimeError('restart did not replace every original task') + time.sleep(1) + wait_service('lb') + wait_service('postgres') + wait_service('redis') + wait_service('minio') + wait_api() + + +def stop_one(): + target = os.environ['TARGET_INSTANCE_ID'] + matches = [t for t in tasks('api') if t['ID'] == target] + if len(matches) != 1: + raise RuntimeError('TARGET_INSTANCE_ID is not a running API task in this stack') + task = matches[0] + container = task['Status']['ContainerStatus']['ContainerID'] + print('Stopping task '+target+' container '+container, flush=True) + docker('kill', container) # Swarm replaces it with a different task/instance ID. + + +def update(): + previous = {t['ID'] for t in tasks('api')} + docker('service', 'update', '--detach', '--force', '--no-resolve-image', '--image', API, STACK+'_api') + deadline = time.monotonic()+180 + while time.monotonic() < deadline: + state = inspect('service', STACK+'_api').get('UpdateStatus', {}).get('State') + if state == 'completed' and previous.isdisjoint(t['ID'] for t in tasks('api')): + wait_api() + print('PASS: rolling update completed') + return + if state in ('paused', 'rollback_started', 'rollback_paused', 'rollback_completed'): + raise RuntimeError('rolling update: '+state) + time.sleep(1) + raise RuntimeError('rolling update timed out') + + +def status(): + result = {'stack': STACK, 'docker': docker('version','--format','{{.Server.Version}}',capture=True).strip(), 'services': {}} + for name in ('lb','api','postgres','redis','minio'): + service = inspect('service', STACK+'_'+name) + spec = service['Spec'] + template = spec['TaskTemplate'] + result['services'][name] = { + 'image': template['ContainerSpec']['Image'], + 'limits': template.get('Resources',{}).get('Limits',{}), + 'published_ports': spec.get('EndpointSpec',{}).get('Ports',[]), + 'secret_names': [x['SecretName'] for x in template['ContainerSpec'].get('Secrets',[])], + 'mounts': template['ContainerSpec'].get('Mounts',[]), + 'tasks': [{'id':t['ID'],'state':t['Status']['State'], + 'container':t['Status'].get('ContainerStatus',{}).get('ContainerID')} + for t in tasks(name)]} + print(json.dumps(result,indent=2)) + +def main(): + action = sys.argv[1] + if action == 'status': status() + elif action == 'up': up() + elif action == 'build': build() + elif action == 'bootstrap-s3': bootstrap_s3() + elif action == 'reconcile': reconcile() + elif action == 'wait': wait_api() + elif action == 'restart': restart() + elif action == 'stop-one': stop_one() + elif action == 'update': update() + elif action == 'migrate': job('migrations', MIGRATIONS, ['up']) + elif action == 'migrate-status': job('migrations', MIGRATIONS, ['status']) + elif action == 'test-integration': + docker('build', '--target', 'tests', '-t', TESTS, '.') + job('tests', TESTS, [], integration=True) + elif action == 'down': docker('stack', 'rm', STACK) + else: raise RuntimeError('unknown action: '+action) + + +if __name__ == '__main__': + try: + main() + except (RuntimeError, subprocess.CalledProcessError) as error: + print('FAIL: '+str(error), file=sys.stderr) + sys.exit(1) diff --git a/stack.yaml b/stack.yaml new file mode 100644 index 00000000..da599861 --- /dev/null +++ b/stack.yaml @@ -0,0 +1,164 @@ +version: '3.8' +services: + postgres: + image: postgres:17.6-alpine3.22 + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + volumes: [postgres_data:/var/lib/postgresql/data] + networks: [data] + healthcheck: + test: [CMD-SHELL, 'pg_isready -U messenger -d messenger'] + interval: 2s + timeout: 2s + retries: 15 + stop_grace_period: 20s + deploy: + replicas: 1 + placement: + constraints: [node.labels.mtusi_lab4_storage == true] + resources: + limits: {cpus: '1.0', memory: 512M} + update_config: {parallelism: 1, order: stop-first} + redis: + image: redis:7.4.5-alpine3.21 + command: [redis-server, /usr/local/etc/redis/redis.conf] + configs: + - source: redis + target: /usr/local/etc/redis/redis.conf + volumes: [redis_data:/data] + networks: [data] + healthcheck: + test: [CMD, redis-cli, ping] + interval: 2s + timeout: 1s + retries: 15 + stop_grace_period: 20s + deploy: + replicas: 1 + placement: + constraints: [node.labels.mtusi_lab4_storage == true] + resources: + limits: {cpus: '0.5', memory: 128M} + update_config: {parallelism: 1, order: stop-first} + minio: + image: ${S3_IMAGE:-mtusi-minio:RELEASE.2025-09-07T16-13-09Z} + secrets: [s3_root_access_key, s3_root_secret_key] + volumes: [s3_data:/data] + networks: [data] + healthcheck: + test: [CMD, wget, -q, -T, '2', -O, /dev/null, 'http://127.0.0.1:9000/minio/health/live'] + interval: 2s + timeout: 3s + start_period: 15s + retries: 15 + stop_grace_period: 20s + deploy: + replicas: 1 + placement: + constraints: [node.labels.mtusi_lab4_storage == true] + resources: + limits: {cpus: '1.0', memory: 512M} + update_config: {parallelism: 1, order: stop-first} + api: + image: ${API_IMAGE:-mtusi-messenger:lab4} + environment: + INSTANCE_ID: '{{.Task.ID}}' + POSTGRES_HOST: postgres + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + CURSOR_KEY_FILE: /run/secrets/cursor_key + POSTGRES_MAX_CONNS: '10' + REDIS_ADDR: redis:6379 + SESSION_TTL_SECONDS: '3600' + QUERY_TIMEOUT_MS: '1000' + S3_ENDPOINT: minio:9000 + S3_BUCKET: attachments + S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key + S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key + S3_USE_SSL: 'false' + GOMEMLIMIT: 1152MiB + GOGC: '50' + secrets: [postgres_password, cursor_key, s3_access_key, s3_secret_key] + networks: [backend, data] + stop_grace_period: 15s + healthcheck: + test: [CMD, wget, -q, -T, '2', -O, /dev/null, 'http://127.0.0.1:8080/health/ready'] + interval: 2s + timeout: 3s + start_period: 10s + retries: 3 + deploy: + replicas: ${API_REPLICAS:-2} + endpoint_mode: dnsrr + resources: + limits: {cpus: '1.0', memory: 1536M} + restart_policy: {condition: any, delay: 1s} + update_config: + parallelism: 1 + order: start-first + delay: 5s + monitor: 15s + failure_action: rollback + rollback_config: {parallelism: 1, order: start-first} + lb: + image: haproxy:3.2.6-alpine + configs: + - source: haproxy + target: /usr/local/etc/haproxy/haproxy.cfg + networks: [backend] + ports: + - target: 8080 + published: ${PORT:-8080} + protocol: tcp + mode: host + healthcheck: + test: [CMD, haproxy, -c, -f, /usr/local/etc/haproxy/haproxy.cfg] + interval: 5s + timeout: 2s + retries: 3 + deploy: + replicas: 1 + resources: + limits: {cpus: '0.5', memory: 128M} + update_config: {parallelism: 1, order: stop-first} +networks: + backend: + driver: overlay + data: + driver: overlay +volumes: + postgres_data: + redis_data: + s3_data: +configs: + haproxy: + file: ./infra/haproxy.cfg + redis: + file: ./infra/redis.conf + s3_bootstrap: + file: ./infra/s3-bootstrap.sh + s3_policy: + file: ./infra/s3-policy.json +secrets: + postgres_password: + external: true + name: ${STACK_NAME:-mtusi-lab4}_postgres_password + cursor_key: + external: true + name: ${STACK_NAME:-mtusi-lab4}_cursor_key + s3_root_access_key: + external: true + name: ${STACK_NAME:-mtusi-lab4}_s3_root_access_key + s3_root_secret_key: + external: true + name: ${STACK_NAME:-mtusi-lab4}_s3_root_secret_key + s3_access_key: + external: true + name: ${STACK_NAME:-mtusi-lab4}_s3_access_key + s3_secret_key: + external: true + name: ${STACK_NAME:-mtusi-lab4}_s3_secret_key diff --git a/tests/own/README.md b/tests/own/README.md new file mode 100644 index 00000000..b446401a --- /dev/null +++ b/tests/own/README.md @@ -0,0 +1,20 @@ +# Собственные проверки lab4 + +`make test-lab4` проверяет реальные PNG/JPEG, лимиты, MIME, повреждения, права, приватность bucket и отказы. +Тест останавливает именно API-реплику, которая приняла файл, и проверяет скачивание через другую. +`make measure-uploads` запускает десять клиентов на двух размерах PNG и сохраняет замеры cgroup memory. +`make test-reconcile` внедряет ошибки S3/БД и проверяет grace, блокировки, dry-run, повторное исправление и стоимость обхода. +Эти сценарии используют реальный MinIO. Запускайте их последовательно на учебном Swarm-стенде. + +Результаты находятся в `evidence/lab4-*`. Подробная методика приведена в [REPORT.md](../../REPORT.md). + +Публичные tests/*.py и обязательный контракт сохранены. + +- `make test-own`: Go tests с race detector. Без INTEGRATION_TESTS=1 PostgreSQL-сценарий пропускается. +- `make test-integration`: отдельный Swarm job, Go 1.26.4 и goose. Временная схема, миграции с нуля/повторно/legacy/reset/up, регистрационная гонка, rollback, 10 000 сообщений с одинаковыми timestamps, EXPLAIN и 16 конкурентных запросов одного ключа. +- `make test-grade4`: `swarm_grade4.py`, ровно две API tasks. Проверяет общий ключ через LB, одинаковый результат, 409, границы ключа, область пользователя/чата, права и logout. По очереди останавливает обе исходные tasks с восстановлением, затем выполняет rolling update под нагрузкой. Проверяет историю, отсутствие дублей и повтор исходного ключа после воздействия. Сохраняет `evidence/swarm-grade4.json`. +- `make test-persistence`: перезапуск всех API, PostgreSQL, Redis, MinIO и LB; volumes/secrets сохраняются. +- `make test-failover ACTION_SCRIPT=scripts/stop-one.sh`: публичный сценарий отказа; `make wait` дожидается восстановления двух реплик. +- `make measure-load`: 3s прогрева + 30s, concurrency 4, метрики и Docker stats в `evidence/swarm-*`. + +Сценарии отказов, persistence и rolling update выполняйте **последовательно**, на локальном учебном стенде. Все данные синтетические; пароль и токены в результаты не записываются. Старый `grade4.py` относится к Compose-стенду lab2 и не является entrypoint текущей работы. diff --git a/tests/own/grade4.py b/tests/own/grade4.py new file mode 100644 index 00000000..60f89b87 --- /dev/null +++ b/tests/own/grade4.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Additional lab2 acceptance checks. Never print credentials or session tokens.""" +import base64 +from concurrent.futures import ThreadPoolExecutor +import os +from pathlib import Path +import subprocess +import sys +import time +import uuid + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from client import Client, Failure, check, run + +ROOT = Path(__file__).resolve().parents[2] + + +def compose(*args, ttl=None): + env = dict(os.environ) + if ttl is not None: + env['SESSION_TTL_SECONDS'] = str(ttl) + result = subprocess.run([str(ROOT / 'scripts/compose.sh'), *args], env=env, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=180) + check(result.returncode == 0, 'compose operation failed: ' + args[0]) + + +def login(c, username, password, expected=201): + basic = base64.b64encode((username + ':' + password).encode()).decode() + return c.call('POST', '/api/v1/auth/sessions', expected, + headers={'Authorization': 'Basic ' + basic}, schema='Session' if expected == 201 else None) + + +def main(): + c = Client(timeout=5) + alice, bob, eve, chat = c.bootstrap() + credentials = alice['registration'] + login(c, credentials['username'], 'incorrect-password', 401) + login(c, 'unknown_' + uuid.uuid4().hex[:12], 'incorrect-password', 401) + c.call('GET', '/api/v1/users/me', 401, headers={'X-User-Id': alice['id']}) + basic = base64.b64encode((credentials['username'] + ':' + credentials['password']).encode()).decode() + c.call('GET', '/api/v1/users/me', 401, headers={'Authorization': 'Basic ' + basic}) + response = c.call('GET', '/api/v1/users/me', actor=alice, headers={'X-User-Id': bob['id']}, schema='User') + check(response['json']['id'] == alice['id'], 'X-User-Id changed session owner') + c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=alice) + c.call('GET', '/api/v1/users/me', 401, actor=alice) + c.call('DELETE', '/api/v1/auth/sessions/current', 401, actor=alice) + print('PASS: wrong password, unknown user, Bearer-only access, X-User-Id ignored, logout') + + # Maximum UTF-8 password length and a colon are supported without truncation. + username = 'unicode_' + uuid.uuid4().hex[:12] + password = '🔑' * 127 + ':' + c.call('POST', '/api/v1/users', 201, body={'username': username, 'display_name': 'Unicode', 'password': password}, schema='User') + login(c, username, password) + login(c, username, '🔑' * 127 + 'x', 401) + print('PASS: 128-code-point Unicode password including colon; suffix verified') + + username = 'race_' + uuid.uuid4().hex[:12] + def register(_): + return Client().call('POST', '/api/v1/users', [201, 409], body={ + 'username': username, 'display_name': 'Concurrent', 'password': 'synthetic-password-123' + })['status'] + with ThreadPoolExecutor(max_workers=8) as pool: + statuses = list(pool.map(register, range(8))) + check(statuses.count(201) == 1 and statuses.count(409) == 7, 'registration race broken') + print('PASS: HTTP registration race: one 201, seven 409') + + # Revoke before restart: absence must also survive AOF recovery. + for service in ('redis', 'postgres'): + compose('stop', service) + try: + c.call('GET', '/health/live', schema='Health') + start = time.monotonic() + c.call('GET', '/health/ready', 503) + check(time.monotonic() - start < 4, 'readiness error exceeded timeout budget') + c.call('GET', '/api/v1/users/me', 503, actor=bob) + c.call('GET', '/api/v1/users/me', 401, headers={'X-User-Id': bob['id']}) + login(c, bob['registration']['username'], bob['registration']['password'], 503) + c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 503, actor=bob, body={'text': 'must fail'}) + finally: + compose('up', '-d', '--wait', service) + c.call('GET', '/health/ready', schema='Health') + c.call('GET', '/api/v1/users/me', actor=bob, schema='User') + c.call('GET', '/api/v1/users/me', 401, actor=alice) + print('PASS: ' + service + ' outage bounded; live=200, ready=503, protected=503, anonymous=401; session recovered, logout retained') + + # The same signed cursor remains valid across an API process restart. + c.message(bob, chat, 'cursor before restart 1') + c.message(bob, chat, 'cursor before restart 2') + page = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=1', actor=bob, schema='MessagePage')['json'] + check(page['next_cursor'] is not None, 'missing cursor') + compose('restart', 'api') + deadline = time.monotonic() + 30 + while True: + try: + c.call('GET', '/health/ready', schema='Health') + break + except Failure: + check(time.monotonic() < deadline, 'API did not recover') + time.sleep(.2) + from urllib.parse import quote + c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?cursor=' + quote(page['next_cursor']), actor=bob, schema='MessagePage') + print('PASS: cursor and existing session survive API restart') + + try: + compose('up', '-d', '--wait', '--force-recreate', 'api', ttl=2) + short = c.user('expiry') + c.call('GET', '/api/v1/users/me', actor=short, schema='User') + time.sleep(2.2) + c.call('GET', '/api/v1/users/me', 401, actor=short) + compose('restart', 'redis') + compose('up', '-d', '--wait', 'redis') + c.call('GET', '/api/v1/users/me', 401, actor=short) + print('PASS: SESSION_TTL_SECONDS=2 expires; Redis restart does not revive expired session') + finally: + compose('up', '-d', '--wait', '--force-recreate', 'api', ttl=3600) + print('PASS: all grade-4 HTTP checks; normal TTL restored to 3600') + + +if __name__ == '__main__': + run(main) diff --git a/tests/own/helpers.py b/tests/own/helpers.py new file mode 100644 index 00000000..ca4dcfb3 --- /dev/null +++ b/tests/own/helpers.py @@ -0,0 +1,116 @@ +"""Local lab4 experiment helpers. Never print credentials or session tokens.""" +import json +import os +from pathlib import Path +import random +import struct +import subprocess +import sys +import time +import uuid +import zlib + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, check + +STACK = os.environ.get('STACK_NAME', 'mtusi-lab4') +MC = os.environ.get('S3_CLIENT_IMAGE', 'mtusi-mc:RELEASE.2025-08-13T08-35-41Z') + + +def docker(*args, data=None, check_result=True): + return subprocess.run(['docker', *args], input=data, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, check=check_result) + + +def containers(service): + ids = docker('ps', '--filter', 'label=com.docker.swarm.service.name='+STACK+'_'+service, + '--format', '{{.ID}}').stdout.decode().split() + check(bool(ids), 'no container for '+service) + return ids + + +def sql(statement): + result = docker('exec', '-i', containers('postgres')[0], 'psql', '-U', 'messenger', + '-d', 'messenger', '-At', '-v', 'ON_ERROR_STOP=1', data=statement.encode()) + return result.stdout.decode().strip() + + +def reconcile(apply=False, grace='90s'): + args = ['exec', containers('api')[0], '/usr/local/bin/reconcile', '--grace='+grace] + if apply: + args.append('--apply') + return json.loads(docker(*args).stdout) + + +def upload(c, actor, chat, content, filename='sample.png', mime='image/png', expected=201): + from urllib.parse import quote + return c.call('POST', '/api/v1/chats/'+chat['id']+'/attachments?filename='+quote(filename, safe=''), + expected, actor=actor, raw=content, headers={'Content-Type': mime}, + schema='Attachment' if expected == 201 else None) + + +def png(width=320, height=200, noisy=False): + def chunk(kind, body): + return struct.pack('>I', len(body))+kind+body+struct.pack('>I', zlib.crc32(kind+body)) + rng = random.Random(419) + lines = b''.join(b'\0'+(rng.randbytes(width*3) if noisy else b'\x18\x60\xc0'*width) + for _ in range(height)) + return (b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)) + +chunk(b'IDAT', zlib.compress(lines))+chunk(b'IEND', b'')) + + +def oversized_dimensions(): + body = struct.pack('>IIBBBBB', 10000, 4001, 8, 2, 0, 0, 0) + return b'\x89PNG\r\n\x1a\n'+struct.pack('>I', len(body))+b'IHDR'+body+struct.pack('>I', zlib.crc32(b'IHDR'+body)) + + +def save(name, result): + (ROOT / 'evidence').mkdir(exist_ok=True) + (ROOT / 'evidence' / name).write_text(json.dumps(result, indent=2, ensure_ascii=False)+'\n') + + +def wait_ready(timeout=90): + deadline = time.monotonic()+timeout + while time.monotonic() < deadline: + try: + Client(timeout=2).call('GET', '/health/ready', schema='Health') + return + except AssertionError: + time.sleep(.5) + raise AssertionError('readiness did not recover') + + +class Probe: + """Short-lived mc service; root secrets remain inside Docker secrets.""" + def __enter__(self): + self.name = STACK+'_probe_'+uuid.uuid4().hex[:8] + args = ['service', 'create', '--detach', '--no-resolve-image', '--name', self.name, '--network', STACK+'_data', + '--restart-condition', 'none', '--limit-memory', '512m', '--limit-cpu', '0.5', + '--entrypoint', '/bin/sh'] + for secret in ('s3_root_access_key', 's3_root_secret_key', 's3_access_key', 's3_secret_key'): + args += ['--secret', 'source='+STACK+'_'+secret+',target='+secret] + docker(*args, MC, '-c', 'sleep 3600') + deadline = time.monotonic()+45 + while time.monotonic() < deadline: + ids = docker('ps', '--filter', 'label=com.docker.swarm.service.name='+self.name, '-q').stdout.decode().split() + if ids: + self.container = ids[0] + return self + time.sleep(.3) + docker('service', 'rm', self.name, check_result=False) + raise AssertionError('probe did not start') + + def mc(self, *args, data=None, admin=False, expected=0): + access = 's3_root_access_key' if admin else 's3_access_key' + secret = 's3_root_secret_key' if admin else 's3_secret_key' + script = ('set -eu; export MC_CONFIG_DIR=/tmp/mc; ' + 'mc alias set local http://minio:9000 "$(cat /run/secrets/'+access+')" ' + '"$(cat /run/secrets/'+secret+')" >/dev/null; exec mc "$@"') + result = docker('exec', '-i', self.container, '/bin/sh', '-c', script, 'sh', *args, + data=data, check_result=False) + check((result.returncode == 0) == (expected == 0), 'unexpected mc outcome: '+args[0]) + return result + + def __exit__(self, *args): + docker('service', 'rm', self.name, check_result=False) diff --git a/tests/own/lab4.py b/tests/own/lab4.py new file mode 100644 index 00000000..8622723b --- /dev/null +++ b/tests/own/lab4.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""Real S3/API negative cases and failures; only synthetic lab data.""" +import hashlib +import http.client +import json +import os +from pathlib import Path +import subprocess +import time +import uuid +from urllib.parse import urlsplit + +from helpers import Client, Probe, ROOT, STACK, check, containers, docker, png, oversized_dimensions, save, sql, upload, wait_ready + + +def main(): + c = Client(timeout=55) + alice, bob, eve, chat = c.bootstrap() + data = png() + result = {'negative_cases': [], 'failures': {}, 'checksums': {}} + for name, body, mime, filename, status in ( + ('oversized_bytes', b'x'*(10*1024*1024+1), 'image/png', 'large.png', 413), + ('damaged_png', data[:50], 'image/png', 'broken.png', 415), + ('corrupt_crc', data[:-16]+b'\xff'*16, 'image/png', 'crc.png', 415), + ('mime_spoof', data, 'image/jpeg', 'fake.jpg', 415), + ('not_image', b'plain text', 'image/png', 'fake.png', 415), + ('unsupported_mime', data, 'application/octet-stream', 'file.png', 415), + ('pixel_budget', oversized_dimensions(), 'image/png', 'large-dimensions.png', 413), + ('unsafe_slash', data, 'image/png', '../file.png', 400), + ('unsafe_backslash', data, 'image/png', 'x\\y.png', 400), + ('unsafe_control', data, 'image/png', 'x\x00.png', 400), + ('empty_filename', data, 'image/png', '', 400), + ): + upload(c, alice, chat, body, filename, mime, status) + result['negative_cases'].append({'case': name, 'status': status}) + for content, filename, mime in ((data, 'изображение.png', 'image/png'), + ((ROOT/'tests/own/fixtures/sample.jpg').read_bytes(), 'photo.jpg', 'image/jpeg')): + att = upload(c, alice, chat, content, filename, mime)['json'] + downloaded = c.call('GET', '/api/v1/attachments/'+att['id']+'/content', actor=bob) + check(downloaded['bytes'] == content, 'original bytes changed') + check(att['sha256'] == hashlib.sha256(content).hexdigest(), 'incorrect SHA256') + check(downloaded['headers'].get_content_type() == mime, 'incorrect content MIME') + result['checksums'][mime] = att['sha256'] + att = upload(c, alice, chat, data)['json'] + for suffix in ('', '/content'): + c.call('GET', '/api/v1/attachments/'+att['id']+suffix, 404, actor=eve) + c.call('GET', '/api/v1/attachments/'+str(uuid.uuid4())+suffix, 404, actor=alice) + c.call('GET', '/api/v1/attachments/'+att['id']+suffix, 401) + upload(c, eve, chat, data, expected=404) + c.call('GET', '/api/v1/attachments/'+att['id']+'/content?variant=thumbnail', 404, actor=bob) + other = c.call('POST', '/api/v1/chats', 201, actor=alice, + body={'title':'other chat', 'member_ids':[alice['id'], bob['id']]}, schema='Chat')['json'] + path = '/api/v1/chats/'+other['id']+'/messages' + for ids, status in (([att['id']], 404), ([str(uuid.uuid4())], 404), ([att['id'],att['id']], 400)): + c.call('POST', path, status, actor=alice, body={'text':'', 'attachment_ids':ids}, + headers={'Idempotency-Key':uuid.uuid4().hex}) + result['negative_cases'] += [{'case': x, 'status': s} for x,s in ( + ('foreign_attachment',404), ('foreign_chat_link',404), ('unknown_attachment',404), + ('duplicate_ids',400), ('missing_authentication',401), ('unknown_variant',404))] + + # The proxy must stop oversized chunked bodies even without Content-Length. + address = urlsplit(c.base) + conn = http.client.HTTPConnection(address.hostname, address.port, timeout=55) + conn.request('POST', '/api/v1/chats/'+chat['id']+'/attachments?filename=chunked.png', + body=iter([b'x'*65536]*161), encode_chunked=True, + headers={'Authorization':'Bearer '+alice['token'], 'Content-Type':'image/png'}) + response = conn.getresponse() + check(response.status == 413, 'chunked upload bypassed byte limit') + response.read(); conn.close() + result['negative_cases'].append({'case':'chunked_oversized', 'status':413}) + + # Disconnect halfway through the declared body. No object/metadata can be published. + before = int(sql('SELECT count(*) FROM attachments;')) + conn = http.client.HTTPConnection(address.hostname, address.port, timeout=5) + conn.putrequest('POST', '/api/v1/chats/'+chat['id']+'/attachments?filename=aborted.png') + conn.putheader('Authorization', 'Bearer '+alice['token']) + conn.putheader('Content-Type', 'image/png') + conn.putheader('Content-Length', str(len(data)+10000)) + conn.endheaders(); conn.send(data[:40]); conn.close() + time.sleep(1) + check(int(sql('SELECT count(*) FROM attachments;')) == before, 'aborted upload published metadata') + leftovers = [docker('exec', container, 'find', '/tmp', '-maxdepth', '1', '-name', 'messenger-upload-*').stdout.decode() + for container in containers('api')] + check(not any(leftovers), 'aborted upload left spool files') + result['failures']['client_disconnect'] = {'metadata_added':0, 'spool_files_remaining':0} + + with Probe() as probe: + key = sql("SELECT object_key FROM attachments WHERE id='"+att['id']+"';") + raw = docker('exec', containers('minio')[0], 'wget', '-S', '-O', '/dev/null', + 'http://127.0.0.1:9000/attachments/'+key, check_result=False) + check(b'403' in raw.stderr, 'unauthenticated direct S3 GET was not forbidden') + probe.mc('stat', 'local/attachments/'+key) + probe.mc('mb', 'local/forbidden-'+uuid.uuid4().hex[:8], expected=1) + probe.mc('pipe', 'local/attachments/outside-prefix', data=b'x', expected=1) + result['bucket_security'] = {'anonymous_get_status':403, 'create_bucket_denied':True, + 'write_outside_prefix_denied':True, 'console_published':False} + service = json.loads(docker('service', 'inspect', STACK+'_minio').stdout)[0] + check(not service['Spec'].get('EndpointSpec', {}).get('Ports'), 'MinIO publishes an external port') + for _ in range(2): + subprocess.run(['python3', str(ROOT/'scripts/swarm.py'), 'bootstrap-s3'], check=True, + stdout=subprocess.DEVNULL) + check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==data, + 'repeated bucket bootstrap changed data') + result['bucket_security']['bootstrap_repeats'] = 2 + + # Stop the replica that accepted THIS upload, then read through another one. + accepted = upload(c,alice,chat,data) + source = accepted['headers']['X-Instance-Id'] + env = dict(os.environ, TARGET_INSTANCE_ID=source) + started = time.monotonic() + subprocess.run(['python3',str(ROOT/'scripts/swarm.py'),'stop-one'],env=env,check=True, + stdout=subprocess.DEVNULL) + deadline = time.monotonic()+10 + while True: + try: + downloaded = Client(timeout=2).call('GET','/api/v1/attachments/'+accepted['json']['id']+'/content',actor=bob) + check(downloaded['headers']['X-Instance-Id']!=source,'stopped upload replica still serves bytes') + check(downloaded['bytes']==data,'cross-replica original changed') + break + except AssertionError: + check(time.monotonic()0 + + # Keep one old linked object and remove a second original behind PostgreSQL. + live = upload(c,alice,chat,data)['json'] + c.message(alice,chat,'keep live original',attachments=[live['id']]) + missing = upload(c,alice,chat,data)['json'] + message = c.message(alice,chat,'keep missing reference',attachments=[missing['id']])['json'] + missing_key = sql("SELECT object_key FROM attachments WHERE id='"+missing['id']+"';") + probe.mc('rm','local/attachments/'+missing_key,admin=True) + current = reconcile(apply=True) + check(sql("SELECT status FROM attachments WHERE id='"+missing['id']+"';")=='ready','fresh metadata changed before grace') + result['fresh_metadata_preserved'] = current['fresh_skipped']>0 + + # Measure a real scan at two bucket sizes, crossing an S3 page boundary. + cost_prefix = 'original/cost-'+experiment+'/' + try: + for group,count,total in (('small',100,100),('extra',1000,1100)): + script = ('mkdir -p /tmp/cost/"$1"; i=1; while [ "$i" -le "$2" ]; do ' + 'printf x > /tmp/cost/"$1"/"$i"; i=$((i+1)); done') + docker('exec',probe.container,'/bin/sh','-c',script,'sh',group,str(count)) + probe.mc('mirror','--quiet','/tmp/cost/'+group,'local/attachments/'+cost_prefix+group,admin=True) + summary = reconcile() + check(summary['objects_scanned']>=total,'scan skipped S3 objects') + result['cost'].append({'phase':'fresh', 'synthetic_objects':total, **summary}) + print('Fresh scan cost:',json.dumps({k:summary[k] for k in ('objects_scanned','metadata_scanned','list_requests_estimate','stat_requests','db_queries','duration_ms')}),flush=True) + print('Waiting92s for actual S3 timestamps: measuring aged-object locking, DB and HEAD cost next.',flush=True) + time.sleep(92) + for total in (1100,100): + summary = reconcile() + check(summary['orphans']>=total,'aged scan did not inspect all orphan fixtures') + result['cost'].append({'phase':'aged', 'synthetic_objects':total, **summary}) + print('Aged scan cost:',json.dumps({k:summary[k] for k in ('objects_scanned','metadata_scanned','list_requests_estimate','stat_requests','db_queries','duration_ms')}),flush=True) + if total==1100: + probe.mc('rm','--recursive','--force','local/attachments/'+cost_prefix+'extra/',admin=True) + finally: + probe.mc('rm','--recursive','--force','local/attachments/'+cost_prefix,admin=True) + + # S3/DB unavailability must stop reconciliation without destructive guesses. + for service in ('minio','postgres'): + try: + docker('service','scale','--detach',STACK+'_'+service+'=0') + deadline=time.monotonic()+30 + while time.monotonic()0: + print('Waiting for actual S3 object age to exceed90s grace:',round(remaining,1),'seconds',flush=True) + time.sleep(remaining) + sql("UPDATE attachments SET created_at=clock_timestamp()-interval '10 minutes' WHERE id IN ('"+live['id']+"','"+missing['id']+"');") + + # A second process holds the upload lock: even an old orphan is protected. + lock_script = "BEGIN; SELECT pg_advisory_xact_lock(hashtextextended('"+orphan+"',0));\n" + lock = subprocess.Popen(['docker','exec','-i',containers('postgres')[0],'psql','-U','messenger','-d','messenger','-At','-v','ON_ERROR_STOP=1'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + lock.stdin.write(lock_script.encode()); lock.stdin.flush() + try: + deadline = time.monotonic()+10 + while sql("SELECT pg_try_advisory_xact_lock(hashtextextended('"+orphan+"',0));") != 'f': + check(time.monotonic()=1 and orphan in objects(probe),'upload lock did not protect orphan') + finally: + lock.stdin.write(b'ROLLBACK;\n'); lock.stdin.close() + lock.wait(timeout=15) + result['locked_candidate_preserved'] = True + # The apply above may mark the aged missing row; restore only its internal state + # to compare the following dry-run with an unchanged, well-defined fixture. + sql("UPDATE attachments SET status='ready',error_code=NULL WHERE id='"+missing['id']+"';") + dry = reconcile() + check(dry['orphans']>=1 and dry['missing']>=1,'dry-run did not detect injected inconsistencies') + check(orphan in objects(probe),'dry-run deleted an object') + check(sql("SELECT status FROM attachments WHERE id='"+missing['id']+"';")=='ready','dry-run modified metadata') + applied = reconcile(apply=True) + check(orphan not in objects(probe),'apply did not delete old orphan') + check(sql("SELECT status FROM attachments WHERE id='"+missing['id']+"';")=='failed','missing original not marked failed') + check(sql("SELECT count(*) FROM messages WHERE id='"+message['id']+"' AND '"+missing['id']+"'::uuid=ANY(attachment_ids);")=='1','repair deleted linked metadata or reference') + c.call('GET','/api/v1/attachments/'+missing['id'],409,actor=alice) + check(c.call('GET','/api/v1/attachments/'+live['id']+'/content',actor=bob)['bytes']==data,'repair deleted a live linked original') + repeated = reconcile(apply=True) + check(repeated['deleted']==0 and repeated['marked_failed']==0,'repeat apply was not idempotent') + result.update({'dry_run':dry, 'apply':applied, 'repeat_apply':repeated, + 'live_linked_preserved':True, 'missing_linked_reference_preserved':True}) + save('lab4-reconciliation.json',result) + print('PASS: real two-store faults, grace, lock, dry-run/apply/repeat, linked data preserved and paginated scan cost.') + + +if __name__=='__main__': + main() diff --git a/tests/own/swarm_grade4.py b/tests/own/swarm_grade4.py new file mode 100644 index 00000000..fd37a5bd --- /dev/null +++ b/tests/own/swarm_grade4.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +"""Grade-4 acceptance through the public LB; writes only synthetic test data.""" +from concurrent.futures import ThreadPoolExecutor +import json +import os +from pathlib import Path +import subprocess +import sys +import threading +import time +import uuid + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT/'tests')) +from client import Client, Failure, check, run +from resilience import history + +STACK = os.environ.get('STACK_NAME', 'mtusi-lab4') + + +def docker_json(*args): + return json.loads(subprocess.check_output(['docker', *args], text=True)) + + +def task_ids(): + ids = subprocess.check_output(['docker','service','ps','--filter','desired-state=running','-q',STACK+'_api'], text=True).split() + return {i for i in ids if docker_json('inspect',i)[0]['Status']['State']=='running'} + + +def wait_ready(): + subprocess.run([sys.executable, str(ROOT/'scripts/swarm.py'), 'wait'], check=True, stdout=subprocess.DEVNULL) + time.sleep(3) # DNS and HAProxy rise interval + + +def idempotency(c, alice, bob, eve, chat): + key = uuid.uuid4().hex + barrier = threading.Barrier(16) + def send(_): + local = Client(timeout=5) + barrier.wait() + return local.message(alice, chat, 'one concurrent effect', key=key) + with ThreadPoolExecutor(max_workers=16) as pool: + results = list(pool.map(send, range(16))) + expected = results[0]['json'] + check(all(r['json']==expected and r['status']==201 for r in results), 'concurrent replay changed result') + instances = {r['headers']['X-Instance-Id'] for r in results} + check(len(instances)>=2, 'same-key race did not reach two replicas') + path = '/api/v1/chats/'+chat['id']+'/messages' + c.call('POST', path, 409, actor=alice, body={'text':'changed'}, headers={'Idempotency-Key':key}) + c.call('POST', path, 404, actor=eve, body={'text':expected['text']}, headers={'Idempotency-Key':key}) + other = c.message(bob, chat, expected['text'], key=key)['json'] + check(other['id']!=expected['id'], 'key leaked across users') + second_chat = c.call('POST','/api/v1/chats',201,actor=alice,body={'title':'Key scope','member_ids':[alice['id'],bob['id']]},schema='Chat')['json'] + separate = c.message(alice,second_chat,expected['text'],key=key)['json'] + check(separate['id']!=expected['id'], 'key leaked across chats') + for bad in ['', 'x'*129, 'bad key', 'bad,key', 'bad/key']: + c.call('POST', path, 400, actor=alice, body={'text':'invalid key'}, headers={'Idempotency-Key':bad}) + c.message(alice,chat,'128 ASCII characters',key='x'*128) + retry = uuid.uuid4().hex + c.call('POST',path,400,actor=alice,body={'text':''},headers={'Idempotency-Key':retry}) + c.message(alice,chat,'valid after invalid',key=retry) + check(sum(m['id']==expected['id'] for m in history(c,bob,chat))==1, 'race created duplicate') + # Revocation must be rechecked even for an already committed key. + c.call('DELETE','/api/v1/auth/sessions/current',204,actor=alice) + c.call('POST',path,401,actor=alice,body={'text':expected['text']},headers={'Idempotency-Key':key}) + print('PASS: shared-key race on two replicas; replay/conflict/scope/validation/access/logout', flush=True) + return {'parallel_requests':16, 'instances':sorted(instances), 'unique_message_ids':1} + + +def exercise(action, target=None): + c = Client(timeout=2) + alice,bob,eve,chat = c.bootstrap() + original_key = uuid.uuid4().hex + original = c.message(alice,chat,'confirmed before '+action,key=original_key)['json'] + before = task_ids() + stop = threading.Event() + events = [] + confirmations = [] + lock = threading.Lock() + def worker(number): + local = Client(timeout=2) + key = uuid.uuid4().hex + body = 'load-'+key + while not stop.is_set(): + started = time.monotonic() + try: + result = local.message(alice,chat,body,key=key) + with lock: + confirmations.append(result['json']) + local.call('GET','/api/v1/users/me',actor=bob,schema='User') + event = (started,time.monotonic(),result['json'],result['headers']['X-Instance-Id'],None) + key = uuid.uuid4().hex + body = 'load-'+key + except Failure as exc: + event = (started,time.monotonic(),None,None,str(exc)) + with lock: + events.append(event) + time.sleep(.03) + threads = [threading.Thread(target=worker,args=(i,)) for i in range(4)] + for thread in threads: thread.start() + time.sleep(2) + env = dict(os.environ) + if target: env['TARGET_INSTANCE_ID'] = target + started = time.monotonic() + proc = subprocess.Popen([sys.executable,str(ROOT/'scripts/swarm.py'),action],env=env,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + try: + code = proc.wait(timeout=180) + check(code==0, action+' command failed') + completed = time.monotonic() + time.sleep(15 if target else 5) + finally: + stop.set() + for thread in threads: thread.join(timeout=10) + if proc.poll() is None: + proc.kill(); proc.wait() + ended = time.monotonic() + wait_ready() + window = [e for e in events if e[0]>=started] + successes = [e for e in window if e[2] is not None] + failures = [e for e in window if e[2] is None] + check(bool(successes), 'no writes during '+action) + recovery = min(e[1] for e in successes if e[0]>=completed)-started if target else None + if target: + check(recovery<=10, 'failure recovery exceeded 10 seconds') + check(target not in [e[3] for e in successes[-20:]], 'stopped instance returned') + check(target not in task_ids(), 'old task still running') + else: + check(before.isdisjoint(task_ids()), 'rolling update did not replace every replica') + replay = c.message(alice,chat,original['text'],key=original_key)['json'] + check(replay==original, 'committed idempotency result lost after '+action) + all_messages = history(c,bob,chat) + by_id = {m['id']:m for m in all_messages} + check(by_id.get(original['id'])==original, 'pre-action confirmation lost') + check(all(by_id.get(m['id'])==m for m in confirmations), 'confirmation lost/changed') + texts = [m['text'] for m in all_messages] + check(len(texts)==len(set(texts)), 'uncertain retry created duplicate') + times = [started]+sorted(e[1] for e in successes)+[ended] + max_gap = max(b-a for a,b in zip(times,times[1:])) + check(max_gap<=10, 'writes stalled for more than 10 seconds') + report = {'action':action,'target':target,'duration_seconds':round(ended-started,3), + 'successful_writes':len(successes),'failed_attempts':len(failures), + 'successful_rps':round(len(successes)/(ended-started),3), + 'first_recovery_seconds':round(recovery,3) if recovery is not None else None, + 'max_success_gap_seconds':round(max_gap,3), + 'instances_before':sorted(before),'instances_after':sorted(task_ids()), + 'confirmed_messages_checked':len(confirmations)+1, + 'duplicate_bodies':0} + print(json.dumps(report),flush=True) + return report + + +def main(): + wait_ready() + c = Client() + results = {'idempotency':idempotency(c,*c.bootstrap())} + originals = sorted(task_ids()) + check(len(originals)==2,'grade4 scenario expects exactly two API replicas') + results['failovers'] = [exercise('stop-one',target) for target in originals] + results['rolling_update'] = exercise('update') + (ROOT/'evidence/swarm-grade4.json').write_text(json.dumps(results,indent=2)+'\n') + print('PASS: each original replica stopped and replaced; rolling update under load; confirmations and sessions preserved') + + +if __name__=='__main__': + run(main) -- 2.54.0 From cbe9d8748cd02ffc39b896c0cf6870c4130d73d4 Mon Sep 17 00:00:00 2001 From: Khakimov Dmitrij Date: Tue, 10 Nov 2026 22:13:40 +0300 Subject: [PATCH 2/2] lab5 done --- .dockerignore | 5 + Dockerfile | 9 +- Makefile | 33 +- PORTING.md | 19 + README.md | 17 +- REPORT.md | 279 +- RESEARCH.md | 185 ++ cmd/main.go | 10 +- cmd/worker/main.go | 115 + compose.yaml | 33 +- contracts/infrastructure.json | 18 +- contracts/jobs.schema.json | 63 + contracts/openapi.json | 16 +- evidence/lab5-deploy.txt | 176 ++ evidence/lab5-failover.txt | 4 + evidence/lab5-faults-initial.txt | 12 + evidence/lab5-faults-main.txt | 17 + evidence/lab5-faults-tail.txt | 2 + evidence/lab5-faults.json | 86 + evidence/lab5-go-tests.txt | 14 + evidence/lab5-integration-initial.txt | 355 +++ evidence/lab5-integration.txt | 389 +++ evidence/lab5-persistence.txt | 3 + evidence/lab5-research.csv | 7 + evidence/lab5-research.json | 2099 +++++++++++++ evidence/lab5-research.png | Bin 0 -> 163470 bytes evidence/lab5-research.svg | 2752 +++++++++++++++++ evidence/lab5-smoke.txt | 2 + evidence/lab5-validation.txt | 19 + infra/README.md | 32 +- infra/redis.conf | 2 +- infra/s3-policy.json | 4 +- internal/app/chat/attachments.go | 13 +- internal/app/chat/errors.go | 3 + internal/attachment/async.go | 156 + internal/attachment/reconcile.go | 7 +- internal/attachment/render.go | 267 ++ internal/attachment/render_test.go | 81 + internal/attachment/service.go | 50 +- internal/attachment/worker.go | 618 ++++ .../attachment/worker_integration_test.go | 99 + internal/pkg/repository/postgres/lab4_test.go | 7 +- internal/pkg/repository/postgres/messages.go | 2 +- .../pkg/repository/postgres/storage_test.go | 7 +- lab.json | 8 +- migrations/005_async_attachments.sql | 18 + migrations/006_attachment_jobs.sql | 18 + migrations/007_attachment_outbox.sql | 10 + migrations/008_attachment_replays.sql | 12 + migrations/009_backfill_pipeline.sql | 12 + migrations/010_storage_cleanup.sql | 5 + requirements-research.txt | 2 + scripts/benchmark_pipeline.py | 332 ++ scripts/measure_load.py | 2 +- scripts/measure_uploads.py | 4 +- scripts/plot_pipeline.py | 51 + scripts/swarm.py | 58 +- stack.yaml | 49 +- tests/own/README.md | 34 +- tests/own/helpers.py | 29 +- tests/own/lab4.py | 154 - tests/own/lab5.py | 327 ++ tests/own/swarm_grade4.py | 2 +- 63 files changed, 8907 insertions(+), 307 deletions(-) create mode 100644 PORTING.md create mode 100644 RESEARCH.md create mode 100644 cmd/worker/main.go create mode 100644 contracts/jobs.schema.json create mode 100644 evidence/lab5-deploy.txt create mode 100644 evidence/lab5-failover.txt create mode 100644 evidence/lab5-faults-initial.txt create mode 100644 evidence/lab5-faults-main.txt create mode 100644 evidence/lab5-faults-tail.txt create mode 100644 evidence/lab5-faults.json create mode 100644 evidence/lab5-go-tests.txt create mode 100644 evidence/lab5-integration-initial.txt create mode 100644 evidence/lab5-integration.txt create mode 100644 evidence/lab5-persistence.txt create mode 100644 evidence/lab5-research.csv create mode 100644 evidence/lab5-research.json create mode 100644 evidence/lab5-research.png create mode 100644 evidence/lab5-research.svg create mode 100644 evidence/lab5-smoke.txt create mode 100644 evidence/lab5-validation.txt create mode 100644 internal/attachment/async.go create mode 100644 internal/attachment/render.go create mode 100644 internal/attachment/render_test.go create mode 100644 internal/attachment/worker.go create mode 100644 internal/attachment/worker_integration_test.go create mode 100644 migrations/005_async_attachments.sql create mode 100644 migrations/006_attachment_jobs.sql create mode 100644 migrations/007_attachment_outbox.sql create mode 100644 migrations/008_attachment_replays.sql create mode 100644 migrations/009_backfill_pipeline.sql create mode 100644 migrations/010_storage_cleanup.sql create mode 100644 requirements-research.txt create mode 100644 scripts/benchmark_pipeline.py create mode 100644 scripts/plot_pipeline.py delete mode 100644 tests/own/lab4.py create mode 100644 tests/own/lab5.py diff --git a/.dockerignore b/.dockerignore index ef48c580..4fc15f64 100644 --- a/.dockerignore +++ b/.dockerignore @@ -31,3 +31,8 @@ lab.json .vscode .agents .codex + +**/.env +**/.env.* +**/.idea +**/.vscode diff --git a/Dockerfile b/Dockerfile index 7cd6ea0a..69970d6c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS goose-builder -RUN GOBIN=/out CGO_ENABLED=0 go install \ +RUN GOMAXPROCS=2 GOFLAGS=-p=1 GOBIN=/out CGO_ENABLED=0 go install \ -tags='no_clickhouse no_libsql no_mssql no_mysql no_sqlite3 no_vertica no_ydb no_spanner' \ github.com/pressly/goose/v3/cmd/goose@v3.27.1 @@ -25,6 +25,7 @@ COPY contracts ./contracts COPY dto ./dto COPY internal ./internal COPY migrations ./migrations +ENV GOFLAGS=-p=1 GOMAXPROCS=2 RUN CGO_ENABLED=0 GOOS=linux go build \ -trimpath \ @@ -32,6 +33,8 @@ RUN CGO_ENABLED=0 GOOS=linux go build \ -o /out/messenger ./cmd RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/reconcile ./cmd/reconcile +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/worker ./cmd/worker + FROM builder AS tests COPY --from=goose-builder /out/goose /usr/local/bin/goose ENV GOOSE_MIGRATION_DIR=/src/migrations @@ -39,6 +42,8 @@ CMD ["go", "test", "-p=1", "-race", "-count=1", "-v", "./..."] FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 +RUN apk add --no-cache libwebp-tools=1.6.0-r0 + RUN addgroup -S messenger \ && adduser -S -D -H -u 10001 -G messenger messenger @@ -46,6 +51,8 @@ COPY --from=builder --chown=messenger:messenger /out/messenger /usr/local/bin/me COPY --from=builder --chown=messenger:messenger /out/reconcile /usr/local/bin/reconcile +COPY --from=builder --chown=messenger:messenger /out/worker /usr/local/bin/worker + ENV PORT=8080 EXPOSE 8080 diff --git a/Makefile b/Makefile index 8c7ec355..6f2aa27c 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ PYTHON ?= $(if $(wildcard .venv/bin/python),.venv/bin/python,python3) GO ?= go -PORT ?= 8080 +PORT ?= 18085 BASE_URL ?= http://localhost:$(PORT) -IMAGE ?= mtusi-messenger:lab4 +IMAGE ?= mtusi-messenger:lab5 CA_FILE ?= ACTION_SCRIPT ?= scripts/restart.sh FAILOVER_SCRIPT ?= scripts/stop-one.sh @@ -15,7 +15,7 @@ TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) export PORT BASE_URL help: - @echo "up: build and deploy Swarm stack; down: stop with volumes retained; test-integration: PostgreSQL race/rollback/EXPLAIN; test-grade4: cross-replica idempotency, both failures and rolling update; check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" + @echo "up: build and deploy isolated lab5 stack; test-lab5: pipeline faults; benchmark: queue research; down: keep volumes; test-integration: PostgreSQL race/rollback/EXPLAIN; test-grade4: cross-replica idempotency, both failures and rolling update; check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" check: $(PYTHON) scripts/check_contract.py @@ -69,10 +69,10 @@ test-grade4: $(PYTHON) tests/own/swarm_grade4.py test-lab4: - $(PYTHON) tests/own/lab4.py + $(PYTHON) tests/own/lab5.py test-reconcile: - $(PYTHON) tests/own/reconcile.py + $(PYTHON) tests/own/lab5.py measure-uploads: $(PYTHON) scripts/measure_uploads.py @@ -94,3 +94,26 @@ test-failover: wait: $(PYTHON) scripts/swarm.py wait + +.PHONY: test-lab5 benchmark plot-research worker-stop worker-start +test-lab5: + $(PYTHON) tests/own/lab5.py + +benchmark: + $(PYTHON) scripts/benchmark_pipeline.py --base-url "$(BASE_URL)" + +plot-research: + $(PYTHON) scripts/plot_pipeline.py + +worker-stop: + $(PYTHON) scripts/swarm.py worker-stop + +worker-start: + $(PYTHON) scripts/swarm.py worker-start + +.PHONY: failed-jobs replay +failed-jobs: + $(PYTHON) scripts/swarm.py failed-jobs + +replay: + $(PYTHON) scripts/swarm.py replay --replay "$(JOB_ID)" --operator "$(OPERATOR)" --reason "$(REASON)" diff --git a/PORTING.md b/PORTING.md new file mode 100644 index 00000000..447b5208 --- /dev/null +++ b/PORTING.md @@ -0,0 +1,19 @@ +# Переход с lab4 на lab5 + +Исходный проект `mtusi2026lab4` сохранён без изменений. +Go module path сохранён для существующих импортов. +Нормативные контракты и публичные проверки lab5 восстановлены из исходного skeleton. + +Загрузка теперь возвращает `202 queued` после staging PUT и PostgreSQL COMMIT. +Старая строка ready без вариантов проходит backfill 009 и получает все три варианта. +При обновлении `make up` останавливает только API своего стека перед backfill. +Это исключает запись старым процессом после миграции. +Volumes и Docker secrets сохраняются. + +Стек, образы и секреты используют имена lab5. Внешний порт по умолчанию — 18085. +Стенд lab4 на 8080 не изменяется. +Redis хранит сессии и durable Streams в одном AOF volume. +Worker работает отдельным сервисом; API не выполняет resize. + +Новая схема, команды и проверки описаны в [REPORT.md](REPORT.md). +Старые `evidence/lab4-*` относятся к исходному проекту и не доказывают выполнение lab5. diff --git a/README.md b/README.md index 009d6997..d178c9ef 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,13 @@ Отделить приём файла от тяжёлой обработки, корректно переживать повторную доставку и остановку worker. -Это **skeleton**, а не готовый сервер. Здесь есть задание, контракт и публичные проверки. Реализацию приложения, Dockerfile и требуемую инфраструктуру пишет студент. Язык и framework свободные; UI не обязателен. +Реализация использует Redis Streams, PostgreSQL outbox и отдельный worker. +API сохраняет оригинал в приватный MinIO staging и возвращает `202 queued`. +Worker создаёт JPEG thumbnail, JPEG crop и WebP optimized. +Запуск: `make up`; адрес: http://localhost:18085; Swagger: `/swagger/`. +Проверки: `make test`, `make test-integration`, `make test-lab5`. +Инструкции и доказательства находятся в [REPORT.md](REPORT.md). +Исследование перегрузки описано в [RESEARCH.md](RESEARCH.md). [Карта курса](COURSE.md) · [Процесс fork/PR](CONTRIBUTING.md) · [OpenAPI](contracts/openapi.json) · [Семантика API](contracts/README.md) · [Проверки](tests/README.md) · [Отчёт](REPORT.md) · [Примеры JSON](contracts/examples.json) · [Памятка преподавателю](MAINTAINERS.md) @@ -52,16 +58,17 @@ make check # После реализации запустите свой сервер/стенд по REPORT.md. -# По умолчанию тест использует http://localhost:8080. +# По умолчанию Makefile использует http://localhost:18085. make test ``` -`.env.example`, `compose.example.yaml` и `contracts/infrastructure.json` описывают настройки и роли компонентов. **compose.example.yaml — список ролей с пустым services, не запускаемый стенд.** Реализуйте свой `compose.yaml`/`stack.yaml`, закрепите версии образов, заполните локальный `.env` и опишите bootstrap/миграции. Все зависимости поднимаются из этого репозитория; ссылка «у меня БД уже установлена» не заменяет воспроизводимость. +`compose.yaml` и `stack.yaml` запускают готовый стенд с закреплёнными версиями образов. Роли компонентов описаны в `contracts/infrastructure.json`. `make bootstrap` создаёт секреты в исключённом из Git каталоге `secrets/`; Swarm использует Docker secrets. Миграции выполняются перед запуском API. Все зависимости поднимаются из этого репозитория. Порядок запуска описан в [REPORT.md](REPORT.md). ```sh # Compose-вариант после реализации: -docker compose --env-file .env config --quiet -docker compose --env-file .env up -d --build +./scripts/compose.sh config --quiet +make bootstrap +./scripts/compose.sh up -d --build --wait # Дождитесь /health/ready; затем make test. # Для Swarm укажите эквивалентные build/push/deploy-команды в REPORT.md. ``` diff --git a/REPORT.md b/REPORT.md index e98f75e9..49fa5eff 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,268 @@ -# Отчёт — лабораторная 5 +# Лабораторная 5: очередь изображений -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +Целевая оценка: **5**, исследование — **трек 1, очередь под перегрузкой**. +Redis уже хранит сессии. Redis Streams добавляет очередь без второго брокера. +Исследование сравнивает три настройки concurrency/prefetch. Методика и результаты: [RESEARCH.md](RESEARCH.md). +Публичные контракты и проверки lab5 сохранены. -## Запуск из чистого clone +## Запуск -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Установите Docker с поддержкой Swarm и Python 3.9+. +Локальные Go-проверки требуют Go 1.26.4 или новее. +Запустите стенд: -## Архитектура и решения +```sh +make up +make wait +make test +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +API: http://localhost:18085. Swagger: http://localhost:18085/swagger/. +Стек `mtusi-lab5` не использует volumes, секреты и порт исходного стенда lab4. +Скрипт поддерживает один узел Swarm. Порты хранилищ не опубликованы. -## Доказательства критериев +Bootstrap создаёт случайные секреты в локальном каталоге `secrets/` и Docker secrets. +Каталог исключён из Git и Docker context. Скрипты не загружают `.env`. +Версии закреплены в Dockerfile/stack.yaml: -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Компонент | Версия | +| --- | --- | +| Go / Alpine | 1.26.4 / 3.23, базовые digest в Dockerfile | +| PostgreSQL | 17.6-alpine3.22 | +| Redis | 7.4.5-alpine3.21 | +| HAProxy | 3.2.6-alpine | +| MinIO | RELEASE.2025-09-07T16-13-09Z, фиксированный commit и SHA-256 | +| mc | RELEASE.2025-08-13T08-35-41Z, фиксированный commit и SHA-256 | +| libwebp-tools | 1.6.0-r0 | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Go compiler использует `-p=1` и `GOMAXPROCS=2` при сборке контейнеров. +Это ограничивает память сборки на Docker VM с 1,91GiB RAM. +Первый параллельный build получил OOM; последовательный build устранил эту ошибку. +`SKIP_STORAGE_BUILD=1 make up` повторно использует уже собранные образы MinIO/mc. +Обычный `make up` собирает все образы из репозитория. -## Эксперименты и отказы +Compose запускает те же роли с одной API-репликой: -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +```sh +make bootstrap +./scripts/compose.sh up -d --build --wait +make test +./scripts/compose.sh down +``` -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Остановите один стенд перед запуском другого на том же порту. +`make down` сохраняет volumes и Docker secrets. +При обновлении `make up` останавливает API перед миграциями 005–010. +Миграция 009 переводит старые ready-оригиналы lab4 в очередь без повторной загрузки. +Старый object key временно становится staging key. +Worker удаляет этот staging только после публикации готовых вариантов. +Эта пауза исключает запись старой API-версией после backfill. +Откат миграций удаляет jobs/outbox; сохранять принятые задачи при downgrade нельзя. -## Ограничения и следующие шаги +## Путь принятой загрузки -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +1. API проверяет сессию, членство в чате, имя, bytes, MIME и пиксели. +2. API сохраняет проверенный оригинал в приватный `staging/`. +3. PostgreSQL одной транзакцией сохраняет attachment, job и outbox. +4. API возвращает `202` со снимком `queued` после COMMIT. +5. Publisher читает outbox через `FOR UPDATE SKIP LOCKED` и делает XADD. +6. Worker получает сообщение через XREADGROUP и арендует job в PostgreSQL. +7. Renderer создаёт варианты в отдельном процессе с deadline. +8. Worker сохраняет оригинал и три варианта по постоянным versioned keys. +9. PostgreSQL атомарно публикует ready и метаданные всех вариантов. +10. Worker подтверждает запись через XACK и удаляет подтверждённую запись XDEL. + +API не выполняет resize. Проверка входного изображения в API сохраняет требования lab4. +Она включает полный decode для обнаружения повреждений, но не создаёт варианты. +Каждая API-реплика проверяет не более одного upload одновременно. +Локальный spool ограничен 10MiB и удаляется после staging PUT или ошибки. + +API publisher работает независимо от worker. Worker также может опубликовать ожидающий outbox. +Row locks исключают конкурентную публикацию одной outbox-строки. +Разрыв между XADD и PostgreSQL COMMIT может создать дубль; потеря принятой задачи не возникает. +Подтверждение XADD означает durable запись Redis с `appendfsync always`. +XACK означает устойчивый результат обработки либо устойчивую запись следующего состояния. +Это at-least-once доставка с идемпотентным эффектом, а не exactly-once доставка. + +Redis хранит stream `attachments:jobs:v1`, группа — `image-workers-v1`. +Одно поле stream `job` содержит JSON по `contracts/jobs.schema.json`. +Поля включают UUID job/attachment, version, attempt, type, schema_version и created_at. +Сообщение не содержит bytes, credentials, filename, URL или S3 key. +Worker берёт доверенные storage metadata из PostgreSQL. + +## Состояния, retry и защита результата + +PostgreSQL — источник истины состояний queued/processing/ready/failed. +Queued и processing можно прикрепить к сообщению своего чата. +Content до ready и после failed возвращает 409. Метаданные доступны во всех состояниях. +Failed нельзя прикрепить заново. Ранее подтверждённые сообщения и их idempotency replay сохраняются. + +Claim увеличивает attempts и fence, задаёт lease_until. +Фиксация результата требует текущий fence, pipeline_version и ещё действующий lease. +Она также требует ровно одну изменённую строку attachment нужной версии и состояния. +Старый процесс не публикует ready после аренды новой попыткой. +Объекты используют ключ `original//v/`. +Имена эффектов: original, thumbnail, crop, optimized. +Повтор той же версии пишет те же байты в те же четыре ключа. +Новая версия получает другой prefix. Стандартный pipeline_version равен 1. + +XAUTOCLAIM возвращает pending-доставки умерших consumers после lease. +Истёкший processing возвращается в queued через устойчивую транзакцию и новый outbox. +Attempts учитывают управляемые попытки обработки; простой дубль ready не увеличивает attempts. +Перед ack дубля ready проверяются метаданные и наличие четырёх S3-объектов нужного размера. +Дубль не изменяет updated_at, варианты или готовый результат. + +Временные ошибки хранилищ, timeout и worker crash используют ограниченный retry. +Постоянные ошибки изображения и исчезнувший staging дают failed сразу. +Backoff после попытки n: `2^min(n,5)` секунд плюс случайные 0–499ms. +По умолчанию разрешено пять попыток. Финальный public error_code безопасен и не содержит внутренних данных. +Реестр failed хранится в attachment_jobs; последняя причина сохраняется в last_error. + +```sh +make failed-jobs +make replay JOB_ID= OPERATOR= REASON='Причина повторной обработки' +``` + +Replay принимает только failed с доступным staging и свободным admission slot. +Он сохраняет job_id и versioned effect identity, сбрасывает счётчик попыток и увеличивает fence. +Транзакция записывает operator, reason, предыдущую ошибку и время в attachment_replays. +Replay ready или ещё активной работы отклоняется. После истечения retention replay отклоняется. + +## Изображения и пределы ресурсов + +Thumbnail — JPEG quality85, вписан в 256×256 без увеличения, размеры округляются вниз. +Crop — центрированный квадрат JPEG128×128 quality85, малые исходники увеличиваются. +Optimized — WebP quality80, method4, вписан в 1280×1280 без увеличения. +JPEG/PNG EXIF orientation1–8 применяется до изменения размеров. +Alpha компонуется на белый. Варианты не сохраняют EXIF; cwebp получает `-metadata none`. +Уменьшение каждого маленького файла не обещается. Оригинал сохраняется побайтно. + +API ограничивает bytes10MiB и 40 миллионов пикселей до полного decode. +Worker повторно проверяет размер, SHA-256, DecodeConfig и pixel budget. +Resize/decode/cwebp работают в дочернем renderer; CommandContext ограничивает весь процесс. +Renderer создаёт отдельную process group. Deadline убивает renderer и потомков, включая cwebp. +Общий pixel semaphore ограничивает одновременно распакованные исходники 40Mpx на worker service. +Orientation использует представление исходника, без второй копии полного изображения. +Renderer получает GOMEMLIMIT384MiB; worker service ограничен cgroup1536MiB и1CPU. +GOMEMLIMIT является мягким Go-лимитом; жёсткий предел задаёт cgroup всей службы. + +| Настройка | По умолчанию | Назначение | +| --- | --- | --- | +| WORKER_CONCURRENCY | 2 | Одновременные обработки, максимум8 | +| WORKER_PREFETCH | 4 | Ёмкость локальной очереди, максимум32 | +| WORKER_TIMEOUT_SECONDS | 25 | Вся попытка: admission пикселей, download, renderer и PUT | +| WORKER_LEASE_SECONDS | 40 | Lease; минимум timeout+5s | +| WORKER_PIXEL_BUDGET | 40000000 | Общий pixel budget, максимум80000000 | +| WORKER_MAX_ATTEMPTS | 5 | Число попыток новой job, задаётся API при upload | +| MAX_OUTSTANDING_JOBS | 200 | Максимум queued+processing на весь стенд | + +Prefetch ограничивает ожидающую локальную очередь. Активные процессы добавляют concurrency к числу полученных сообщений. +Настройки bounded; число принятых незавершённых jobs ограничено отдельно. +Admission использует общий PostgreSQL advisory lock и count queued/processing. +Этот lock также защищает replay. Превышение возвращает 429 с Retry-After. +Проверка и вставка выполняются в одной транзакции, поэтому две API-реплики не превышают лимит. +Lock удерживается во время staging PUT. Это ограничивает throughput upload и измеряется в исследовании. + +## Retention и согласованность S3 + +Очистка выполняется каждые пять секунд, не более100 metadata rows за проход. +Ready staging удаляется после устойчивой публикации всех вариантов. +Failed staging и частичные конечные объекты сохраняются семь дней для расследования и replay. +После семи дней cleanup удаляет bytes и ставит storage_cleaned_at, но сохраняет metadata/job/audit. +Replay и cleanup блокируют одну metadata row; очистка не удаляет текущую работу. +Старый stage без metadata имеет grace90s и общий с upload advisory lock. +После PUT без COMMIT API возвращает ошибку; такой stage удаляется после grace. +Готовые конечные вложения сохраняются бессрочно, включая ещё не привязанные к сообщению. +Их staging очищается одинаково. Автоматическое удаление готовых конечных вложений не реализовано. + +Reconciler lab4 сохранён: `make reconcile`, `make reconcile-apply`. +Он защищает текущий versioned prefix и staging_key, включая legacy staging в original-prefix. +Отсутствующий ready-оригинал атомарно переводит attachment и job в failed. +Reconciler не удаляет ссылки из подтверждённых сообщений. +Reconciler контролирует original; внешняя утрата отдельного варианта требует ручного восстановления. +Проверка дубля пишет безопасный retry-log, если готовый эффект недоступен. +Автоматический ремонт внешних удалений вариантов не реализован. + +## Проверки и исследование + +Подготовьте зависимости проверок: + +```sh +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements-dev.txt -r requirements-research.txt +make check validate +make test-own +make test-integration +make test +make test-lab5 +make test-persistence +make test-failover +make benchmark +make plot-research +``` + +Go-тесты проверяют геометрию, EXIF1–8, alpha, повреждения, MIME, concurrency и транзакции. +Интеграционный тест проверяет stale/expired fence и защиту более новой pipeline_version. +Fault suite внедряет ошибку после staging, разрыв COMMIT/publish и два broker restart. +Он останавливает worker после S3 effects и после result COMMIT перед ack. +Проверяются повторная доставка, lease recovery, bounded retry, permanent failure и audit replay. +Отдельный renderer_timeout блокирует настоящий cwebp на FIFO и проверяет уничтожение потомков. +Очистка проверяется для ready, unconfirmed и expired failed staging. +Все пользователи и изображения синтетические. Токены и секреты в evidence не сохраняются. +Запускайте fault, persistence и benchmark последовательно на учебном стенде. + +Исследование трека1 использует реальные свежие измерения, а не результаты исходного проекта. +Пары concurrency/prefetch: (1,1), (2,4), (4,8); размеры PNG512² и1536². +Отдельный опыт проверяет предел admission при остановленном worker и последующее восстановление. +Raw данные, графики, ограничения методики и объяснение для защиты находятся в [RESEARCH.md](RESEARCH.md). +Исследование завершено: все 120 измеряемых загрузок стали ready, шесть сочетаний настроек и размеров проверены. +Пара concurrency=2, prefetch=4 дала лучший throughput для обоих размеров: 4,013 и 0,851 изображения/с. +При остановленном worker лимит 8 принял 8 из 16 загрузок; остальные получили 429. +После запуска все 8 заданий завершились, очередь опустела, новая загрузка прошла. +Текстовое сообщение сохранилось при полной очереди. Исходные настройки восстановлены. +Итог `PASS` и наблюдения: `evidence/lab5-research.json`; таблица: `evidence/lab5-research.csv`. +Исторические evidence/lab4-* не считаются доказательством lab5. + +`make test` прошёл на новом API lab5. Журнал: `evidence/lab5-smoke.txt`. +Host race-тесты и контейнерные интеграционные тесты прошли. +Интеграционный журнал: `evidence/lab5-integration.txt`; исходная ошибка fixture сохранена отдельно. +Прежний fixture удалял migration journal из уже обновлённой схемы. +Теперь он сначала возвращает физическую схему lab4, затем проверяет её импорт. + +Все обязательные fault cases проверены на реальных PostgreSQL, Redis Streams и MinIO. +Первый полный вызов подтвердил основные durability/crash/retry сценарии и выявил гонку тестового stop helper. +Desired replicas=0 ещё не означает остановку старого consumer-контейнера. +Helper и `make worker-stop` теперь ожидают фактическое отсутствие работающих контейнеров. +Поздние независимые cases повторены через `--tail-only` с новыми пользователями и изображениями. +Повтор завершился exit0; прежние успешные cases и время продолжения сохранены в JSON. +Исходный полный вызов завершился exit1 и не объявляется успешным полным прогоном. +Журналы: `evidence/lab5-faults-main.txt`, `evidence/lab5-faults-tail.txt`. +Все case-level результаты находятся в `evidence/lab5-faults.json`. + +Отдельный initial fault обнаружил старый worker при обновлении образа с прежним tag. +`make up` теперь принудительно обновляет worker вместе с API. +После исправления фактические JPEG/PNG EXIF проверки прошли. +Статические проверки и команды воспроизведения собраны в `evidence/lab5-validation.txt`. +`make test-persistence` прошёл после перезапуска компонентов собственного стенда. +Подтверждённые данные и существующая сессия сохранились: `evidence/lab5-persistence.txt`. +`make test-failover` прошёл при остановке одной из двух API-реплик. +Зафиксированы 123 успешные записи, одна неудачная попытка и восстановление за 0,696s. +Подтверждённые данные и сессия сохранились: `evidence/lab5-failover.txt`. + +## Соответствие критериям + +| Уровень | Критерий | Реализация и доказательство | +| --- | --- | --- | +| 3 | Возможности lab4 и переход upload на 202 | `PORTING.md`, публичный smoke, persistence/failover | +| 3 | Устойчивый приём до ответа | MinIO staging + PostgreSQL attachment/job/outbox; cases `stage_before_commit`, `commit_publish_and_restarts` | +| 3 | Отдельный broker/worker, durable at-least-once | Redis Streams/AOF; два broker restart, worker stop/start | +| 3 | Четыре состояния, оригинал и thumbnail | Контракт API; smoke, `small_alpha_exif`, `bounded_retry_replay` | +| 3 | Ограниченные повторы и единый результат | `max_attempts`, versioned keys; cases `crash_after_objects`, `commit_before_ack_and_duplicate` | +| 4 | Crop/WebP, качество и ограничения | Renderer, pixel budget, process deadline; EXIF/alpha/geometry tests и `renderer_process_group_timeout` | +| 4 | Восстановление разрыва COMMIT/publish | Transactional outbox; case `commit_publish_and_restarts` | +| 4 | Backoff/jitter, permanent failure, failed registry/replay | Cases `bounded_retry_replay`, `permanent_stage_error`; audit replay | +| 4 | Crash после S3, lease/fence, bounded concurrency/prefetch | Crash cases, `TestFenceIntegration`, конфигурация worker | +| 4 | Безопасная очистка staging | Case `staging_cleanup`; ready/failed/unconfirmed retention | +| 5 | Трек 1: настройки, размеры, backpressure и восстановление | Методика, raw данные, графики и ограничения в `RESEARCH.md` | + +Источники case-level результатов: `evidence/lab5-faults.json` и журналы split-run выше. diff --git a/RESEARCH.md b/RESEARCH.md new file mode 100644 index 00000000..742a681b --- /dev/null +++ b/RESEARCH.md @@ -0,0 +1,185 @@ +# Исследование: очередь под перегрузкой + +Выбран трек 1 лабораторной 5. Исследование сравнивает скорость, задержку и память при разных настройках обработки. +Отдельный сценарий проверяет ограничение очереди и восстановление после перегрузки. + +## Гипотеза + +Рост числа параллельных обработок уменьшает ожидание, пока хватает процессора. +После насыщения процессора дополнительные обработки увеличивают память и конкурируют за вычисления. +Ограничение очереди сохраняет конечное число принятых заданий, когда worker остановлен. + +## Что означают настройки + +- `WORKER_CONCURRENCY` ограничивает число параллельных обработок внутри одного worker. +- `WORKER_PREFETCH` ограничивает буфер заданий перед обработкой. +- `WORKER_PIXEL_BUDGET` ограничивает суммарное число пикселей у активных обработок. +- `MAX_OUTSTANDING_JOBS` ограничивает число заданий со статусом `queued` или `processing` в PostgreSQL. + +Redis Streams хранит сообщения до подтверждения. PostgreSQL хранит состояние задания и намерение публикации. +Сообщение Redis содержит идентификаторы. Байты изображения остаются в S3. +Полная очередь приводит к HTTP `429`. Текстовые сообщения продолжают работать. + +## Методика + +Стенд включает один worker, две API-реплики, PostgreSQL, Redis и MinIO. +Ограничения CPU и памяти worker сохраняются между запусками. +Исходные изображения — PNG с псевдослучайными RGB-пикселями. Генератор использует фиксированное начальное значение. +SHA-256 каждого изображения записывается в результат. + +Скрипт сравнивает шесть запусков: + +| Размер PNG | Concurrency | Prefetch | Число изображений | +| --- | ---: | ---: | ---: | +| 512×512 | 1 | 1 | 20 | +| 512×512 | 2 | 4 | 20 | +| 512×512 | 4 | 8 | 20 | +| 1536×1536 | 1 | 1 | 20 | +| 1536×1536 | 2 | 4 | 20 | +| 1536×1536 | 4 | 8 | 20 | + +Перед каждым запуском worker завершает все задания. Одно изображение прогревает pipeline после изменения настройки. +Измерение начинается после прогрева. Восемь HTTP-потоков отправляют двадцать изображений. +Скрипт опрашивает метаданные до `ready` для всех принятых вложений. +После измерения скрипт проверяет оригинал и контрольные суммы трёх вариантов у одного вложения. + +## Метрики + +| Метрика | Определение | +| --- | --- | +| Throughput | Число готовых изображений / время всего пакета, включая upload и опрос | +| p95 до ready | 95-й процентиль времени от начала POST до первого ответа `ready` | +| p95 после 202 | 95-й процентиль времени от ответа `202` до первого ответа `ready` | +| Queue lag | Возраст старейшего задания `queued` из текущего чата; `processing` исключается | +| RSS | Сумма `VmRSS` worker, процессов обработки и `cwebp` | +| Размер очереди | Число заданий `queued` и `processing` в PostgreSQL | + +Процентиль рассчитывается методом ближайшего ранга. Для двадцати изображений p95 соответствует девятнадцатому значению после сортировки. +Скрипт снимает RSS и queue lag между опросами API. Пауза между циклами составляет минимум 250 мс. +Фактические интервалы содержатся в JSON. Команды Docker и запросы API увеличивают этот интервал. + +## Результаты измерений + +Дата измерений: 10 ноября 2026 года. Архитектура — arm64, Docker Engine — 29.5.2. +Docker VM располагает двумя CPU и 1,91 GiB RAM. +Worker ограничен одним CPU и 1536 MiB памяти. Эти ограничения одинаковы во всех шести запусках. +Размеры входных файлов составили 787252 и 7081647 байт. +Все 120 измеряемых загрузок вернули `202` и завершились статусом `ready`. + +| PNG | Concurrency / prefetch | Throughput, изобр./с | p95 до ready, с | Queue lag max, с | RSS max, МиБ | +| --- | ---: | ---: | ---: | ---: | ---: | +| 512×512 | 1/1 | 3.316 | 4.76 | 4.08 | 32.2 | +| 512×512 | 2/4 | 4.013 | 4.33 | 3.17 | 54.2 | +| 512×512 | 4/8 | 3.903 | 4.45 | 2.67 | 90.3 | +| 1536×1536 | 1/1 | 0.774 | 22.79 | 21.48 | 74.2 | +| 1536×1536 | 2/4 | 0.851 | 21.47 | 18.20 | 134.3 | +| 1536×1536 | 4/8 | 0.847 | 20.74 | 16.60 | 224.2 | + +![Результаты шести измерений](evidence/lab5-research.svg) + +PNG-версия графика: [lab5-research.png](evidence/lab5-research.png). +Точные значения, дополнительные метрики и исходные наблюдения доступны в JSON и CSV. + +Для этого стенда рекомендована пара **concurrency=2, prefetch=4**. Она совпадает с настройкой по умолчанию. +Она дала максимальный измеренный throughput для обоих размеров. +Относительно 1/1 throughput вырос примерно на 21% для маленьких изображений и на 10% для крупных. +Пиковый RSS при этом вырос с 32,2 до 54,2 МиБ и с 74,2 до 134,3 МиБ. + +Переход от 2/4 к 4/8 не увеличил throughput. +Для крупных изображений p95 улучшился примерно на 3,4%, а RSS вырос примерно на 67%. +Для маленьких изображений выросли и RSS, и p95. +Выбор 1/1 уменьшает расход памяти ценой большей задержки и меньшей скорости. + +Результат согласуется с гипотезой об ограниченной пользе параллелизма при одном CPU. +Он не доказывает причину насыщения: upload, S3 и PostgreSQL также входят в измерение. +Меньший queue lag при 4/8 не означает пропорционального ускорения pipeline. +Метрика исключает задания `processing`, число которых может расти вместе с concurrency. +Разницу в несколько процентов следует проверить повторными сериями перед изменением рабочих настроек. + +## Перегрузка и восстановление + +Скрипт останавливает worker и устанавливает лимит восемь заданий на обеих API-репликах. +Затем шестнадцать запросов конкурируют за этот общий лимит. +Проверка требует восемь ответов `202`, восемь ответов `429` и восемь заданий в PostgreSQL. +Заголовки `X-Instance-Id` подтверждают участие обеих API-реплик. + +После запуска worker все восемь принятых заданий завершаются. +Скрипт проверяет пустую очередь и новый успешный upload. +Скрипт восстанавливает исходные настройки после эксперимента, в том числе при ошибке проверки. + +Фактический результат: + +| Проверка | Результат | +| --- | --- | +| Остановленный worker, лимит 8, предложено 16 | 8 ответов `202`, 8 ответов `429` | +| Участие API | Подтверждены две разные реплики | +| Незавершённые задания при остановленном worker | Ровно 8 | +| Текстовое сообщение при полной очереди | Сохранено успешно | +| Принятые задания после запуска worker | Все 8 стали `ready` | +| Незавершённые задания после восстановления | 0 | +| Следующая загрузка после освобождения очереди | `202`, затем `ready` | + +Счётчик завершения проверяется после ожидания готовности worker. +Часть обработки проходит во время этого ожидания, поэтому отдельное поле drain не измеряет полное время восстановления. +Опыт подтверждает общий лимит между репликами и сохранность принятой работы после `202`. +Эксперимент завершился с `PASS`. Исходные настройки worker и API восстановлены и проверены отдельно. + +## Повторение + +Запустите стенд: + +```sh +make up +make test +``` + +Остановите другие нагрузочные проверки. Запустите исследование: + +```sh +make benchmark +``` + +Установите зависимости для графика: + +```sh +.venv/bin/python -m pip install -r requirements-research.txt +``` + +Создайте график: + +```sh +make plot-research +``` + +Скрипт сохраняет исходные наблюдения в `evidence/lab5-research.json`. +Таблица находится в `evidence/lab5-research.csv`. Графики сохраняются в SVG и PNG рядом с таблицей. +JSON содержит версию Docker, число CPU, память, образ worker и ограничения контейнера. + +## Ограничения + +Измерения относятся к одному локальному стенду. Пакет из двадцати изображений не определяет производственный SLA. +В каждом запуске есть прогрев, но независимых повторений для доверительного интервала нет. +Concurrency и prefetch меняются вместе. Результат сравнивает пары настроек и не изолирует влияние каждого параметра. + +HTTP upload, проверка изображения, S3 и PostgreSQL также влияют на throughput. +Глобальная блокировка при приёме задания сериализует участок записи staging и регистрации задания. +Увеличение concurrency не устраняет это ограничение API. +Admission ограничивает незавершённую работу. Concurrency и бюджет пикселей ограничивают память обработки. +Этот лимит не ограничивает общий размер готовых вложений на диске. + +Опрос добавляет задержку к наблюдаемому времени до `ready`. +Выборка RSS может пропустить короткий пик. Общие страницы памяти могут учитываться несколько раз. +Небольшая часть RSS относится к самой команде наблюдения внутри контейнера. +Значение RSS не равно использованию памяти cgroup с файловым кешем. + +## Объяснение для защиты + +1. API возвращает `202` после сохранения файла и задания. Обработка изображения продолжается отдельно. +2. Redis хранит очередь доставки. PostgreSQL хранит состояние и восстанавливает публикацию через outbox. +3. Concurrency задаёт число одновременных обработок. Prefetch задаёт запас заданий перед обработкой. +4. Задержка растёт, когда задания поступают быстрее обработки. Бесконечная очередь увеличивает ожидание и расход ресурсов. +5. Лимит очереди возвращает `429` для новой работы. Работа после уже выданного `202` сохраняется. +6. После запуска worker очередь завершается. Повторный upload проходит без изменения принятого ранее задания. +7. Подтверждение Redis происходит после устойчивого результата. Повтор доставки безопасен благодаря состоянию задания и ключам вариантов. + +Это доставка at least once. Повтор доставки возможен, поэтому результат защищён от повторного применения. diff --git a/cmd/main.go b/cmd/main.go index 6f1f721d..f3be9581 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -63,7 +63,15 @@ func main() { if err != nil { log.Fatal("invalid S3 config") } - handler := chat.NewChat(useCase, cfg.CursorKey).WithAttachments(attachment.New(pool, objectStore)) + options, err := attachment.LoadOptions() + if err != nil { + log.Fatal(err) + } + attachments := attachment.NewAsync(pool, objectStore, options) + background, stopPublisher := context.WithCancel(context.Background()) + defer stopPublisher() + go attachments.RunPublisher(background, redisClient) + handler := chat.NewChat(useCase, cfg.CursorKey).WithAttachments(attachments) port := cfg.Port e := handler.Router() e.Server.ReadHeaderTimeout = 5 * time.Second diff --git a/cmd/worker/main.go b/cmd/worker/main.go new file mode 100644 index 00000000..2d5de868 --- /dev/null +++ b/cmd/worker/main.go @@ -0,0 +1,115 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/attachment" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/config" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/redis/go-redis/v9" + "log" + "os" + "os/signal" + "syscall" + "time" +) + +func main() { + if len(os.Args) > 1 && os.Args[1] == "render" { + if len(os.Args) != 5 { + os.Exit(2) + } + if err := attachment.Render(context.Background(), os.Args[2], os.Args[3], os.Args[4]); err != nil { + if errors.Is(err, attachment.ErrInvalidImage) { + os.Exit(2) + } + os.Exit(1) + } + return + } + failed := flag.Bool("failed", false, "list failed job registry") + replay := flag.String("replay", "", "failed job UUID to replay") + operator := flag.String("operator", "", "operator identity") + reason := flag.String("reason", "", "replay reason") + flag.Parse() + cfg, err := config.Load() + if err != nil { + log.Fatal(err) + } + options, err := attachment.LoadOptions() + if err != nil { + log.Fatal(err) + } + pc, err := pgxpool.ParseConfig("sslmode=disable") + if err != nil { + log.Fatal("invalid DB config") + } + pc.ConnConfig.Host = cfg.PostgresHost + pc.ConnConfig.Port = cfg.PostgresPort + pc.ConnConfig.User = cfg.PostgresUser + pc.ConnConfig.Password = cfg.PostgresPassword + pc.ConnConfig.Database = cfg.PostgresDatabase + pc.ConnConfig.ConnectTimeout = 3 * time.Second + pc.ConnConfig.RuntimeParams["timezone"] = "UTC" + pc.ConnConfig.RuntimeParams["statement_timeout"] = "5000" + pc.MaxConns = int32(options.Concurrency + 4) + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + pool, err := pgxpool.NewWithConfig(ctx, pc) + if err != nil { + log.Fatal("DB configuration failed") + } + defer pool.Close() + store, err := attachment.NewStore(cfg.S3Endpoint, cfg.S3Bucket, cfg.S3AccessKey, cfg.S3SecretKey, cfg.S3UseSSL) + if err != nil { + log.Fatal("S3 configuration failed") + } + service := attachment.NewAsync(pool, store, options) + if *failed { + rows, e := pool.Query(ctx, `SELECT id::text,attachment_id::text,attempts,max_attempts,coalesce(last_error,''),updated_at FROM attachment_jobs WHERE state='failed' ORDER BY updated_at DESC LIMIT 100`) + if e != nil { + log.Fatal("cannot query failed jobs") + } + defer rows.Close() + records := []map[string]interface{}{} + for rows.Next() { + var id, att, code string + var attempt, max int + var updated time.Time + if e = rows.Scan(&id, &att, &attempt, &max, &code, &updated); e != nil { + log.Fatal("invalid failed job") + } + records = append(records, map[string]interface{}{"job_id": id, "attachment_id": att, "attempts": attempt, "max_attempts": max, "error_code": code, "updated_at": updated.UTC()}) + } + if rows.Err() != nil { + log.Fatal("failed jobs query interrupted") + } + json.NewEncoder(os.Stdout).Encode(records) + return + } + if *replay != "" { + operation, stop := context.WithTimeout(ctx, 10*time.Second) + defer stop() + if err = service.Replay(operation, *replay, *operator, *reason); err != nil { + log.Fatal(err) + } + log.Print("Replay committed with audit record") + return + } + r := redis.NewClient(&redis.Options{Addr: cfg.RedisAddr, DialTimeout: 2 * time.Second, ReadTimeout: 2 * time.Second, WriteTimeout: 2 * time.Second, MaxRetries: -1, ContextTimeoutEnabled: true}) + defer r.Close() + log.Printf("Worker started: concurrency=%d prefetch=%d timeout=%s lease=%s pixel_budget=%d", options.Concurrency, options.Prefetch, options.Timeout, options.Lease, options.PixelBudget) + for ctx.Err() == nil { + err = service.RunWorker(ctx, r) + if err != nil { + log.Print("worker broker connection will retry") + } + select { + case <-ctx.Done(): + return + case <-time.After(time.Second): + } + } +} diff --git a/compose.yaml b/compose.yaml index 4c221cd3..7d43834e 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,4 +1,4 @@ -name: mtusi2026lab4 +name: mtusi2026lab5 services: postgres: image: postgres:17.6-alpine3.22 @@ -29,7 +29,7 @@ services: timeout: 2s retries: 30 cpus: 0.5 - mem_limit: 128m + mem_limit: 384m minio: image: mtusi-minio:RELEASE.2025-09-07T16-13-09Z build: @@ -66,7 +66,7 @@ services: cpus: 0.5 mem_limit: 512m migrations: - image: mtusi-migrations:lab4 + image: mtusi-migrations:lab5 build: context: . target: migrations @@ -83,7 +83,7 @@ services: mem_limit: 128m api: build: . - image: mtusi-messenger:lab4 + image: mtusi-messenger:lab5 environment: &api_environment POSTGRES_HOST: postgres POSTGRES_USER: messenger @@ -107,7 +107,7 @@ services: - s3_access_key - s3_secret_key ports: - - '127.0.0.1:${PORT:-8080}:8080' + - '127.0.0.1:${PORT:-18085}:8080' depends_on: postgres: condition: service_healthy @@ -120,8 +120,29 @@ services: cpus: 1.0 mem_limit: 1536m stop_grace_period: 10s + worker: + image: mtusi-messenger:lab5 + build: . + entrypoint: [/usr/local/bin/worker] + environment: + <<: *api_environment + WORKER_CONCURRENCY: '2' + WORKER_PREFETCH: '4' + WORKER_MAX_ATTEMPTS: '5' + WORKER_TIMEOUT_SECONDS: '25' + WORKER_LEASE_SECONDS: '40' + WORKER_PIXEL_BUDGET: '40000000' + secrets: *api_secrets + healthcheck: {disable: true} + depends_on: + migrations: {condition: service_completed_successfully} + s3_bootstrap: {condition: service_completed_successfully} + redis: {condition: service_healthy} + cpus: 1.0 + mem_limit: 1536m + stop_grace_period: 10s tests: - image: mtusi-messenger-tests:lab4 + image: mtusi-messenger-tests:lab5 profiles: [test] build: context: . diff --git a/contracts/infrastructure.json b/contracts/infrastructure.json index dd2edf2d..a4cfcb7b 100644 --- a/contracts/infrastructure.json +++ b/contracts/infrastructure.json @@ -1,5 +1,5 @@ { - "lab": 4, + "lab": 5, "kind": "requirements, not runnable deployment", "required_for_grade_3": [ { @@ -27,6 +27,22 @@ ], "private_bucket": true, "persistent_volume": true + }, + { + "role": "broker", + "choose_one": [ + "RabbitMQ", + "Redis Streams", + "Valkey Streams", + "NATS JetStream", + "documented durable equivalent" + ], + "persistent_volume": true, + "delivery": "at-least-once" + }, + { + "role": "worker", + "separate_process": true } ], "optional_components": [ diff --git a/contracts/jobs.schema.json b/contracts/jobs.schema.json new file mode 100644 index 00000000..2e9aa484 --- /dev/null +++ b/contracts/jobs.schema.json @@ -0,0 +1,63 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:mtusi:messenger:attachment-job:v1", + "title": "Durable attachment processing job v1", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "job_id", + "type", + "attachment_id", + "pipeline_version", + "attempt", + "created_at" + ], + "properties": { + "schema_version": { + "const": 1 + }, + "job_id": { + "type": "string", + "format": "uuid" + }, + "type": { + "const": "attachment.process.v1" + }, + "attachment_id": { + "type": "string", + "format": "uuid" + }, + "pipeline_version": { + "type": "integer", + "minimum": 1 + }, + "attempt": { + "type": "integer", + "minimum": 1 + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "traceparent": { + "type": "string", + "pattern": "^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$" + }, + "tracestate": { + "type": "string", + "maxLength": 512 + } + }, + "examples": [ + { + "schema_version": 1, + "job_id": "11111111-1111-4111-8111-111111111111", + "type": "attachment.process.v1", + "attachment_id": "22222222-2222-4222-8222-222222222222", + "pipeline_version": 1, + "attempt": 1, + "created_at": "2026-09-01T12:00:00Z" + } + ] +} diff --git a/contracts/openapi.json b/contracts/openapi.json index 138f07c0..4face909 100644 --- a/contracts/openapi.json +++ b/contracts/openapi.json @@ -1,8 +1,8 @@ { "openapi": "3.1.0", "info": { - "title": "MTUSI Messenger — лабораторная 4", - "version": "1.4.0", + "title": "MTUSI Messenger — лабораторная 5", + "version": "1.5.0", "description": "Нормативный минимум на 3. Семантика: contracts/README.md. Расширения на 4/5 не должны менять этот API. Наличие кода ошибки в схеме не требует реализовать факультативную причину его выдачи." }, "servers": [ @@ -2151,7 +2151,7 @@ "operationId": "uploadAttachment", "summary": "Принять один PNG/JPEG, до 10 MiB", "responses": { - "201": { + "202": { "description": "Успех", "headers": { "X-Request-Id": { @@ -2936,7 +2936,10 @@ "schema": { "type": "string", "enum": [ - "original" + "original", + "thumbnail", + "crop", + "optimized" ], "default": "original" } @@ -3461,7 +3464,10 @@ "status": { "type": "string", "enum": [ - "ready" + "queued", + "processing", + "ready", + "failed" ] }, "variants": { diff --git a/evidence/lab5-deploy.txt b/evidence/lab5-deploy.txt new file mode 100644 index 00000000..e3d056c2 --- /dev/null +++ b/evidence/lab5-deploy.txt @@ -0,0 +1,176 @@ +.venv/bin/python scripts/swarm.py up +vphhh79vd5ti8501q6m9lzulj +Local secrets are ready. +DEPRECATED: The legacy builder is deprecated and will be removed in a future release. + Install the buildx component to build images with BuildKit: + https://docs.docker.com/go/buildx/ + +Sending build context to Docker daemon 461.3kB +Step 1/39 : FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS goose-builder + ---> 18b460dd1754 +Step 2/39 : RUN GOMAXPROCS=2 GOFLAGS=-p=1 GOBIN=/out CGO_ENABLED=0 go install -tags='no_clickhouse no_libsql no_mssql no_mysql no_sqlite3 no_vertica no_ydb no_spanner' github.com/pressly/goose/v3/cmd/goose@v3.27.1 + ---> Using cache + ---> 15f4b2e3468a +Step 3/39 : FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 AS migrations + ---> 075c2c1a4068 +Step 4/39 : COPY --from=goose-builder /out/goose /usr/local/bin/goose + ---> Using cache + ---> 6a0ad52336c1 +Step 5/39 : COPY migrations /migrations + ---> Using cache + ---> 01a2e64cbb7d +Step 6/39 : COPY infra/migrate.sh /usr/local/bin/migrate + ---> Using cache + ---> 7382e24476ff +Step 7/39 : USER 10001:10001 + ---> Using cache + ---> 6c54ddff6a36 +Step 8/39 : ENTRYPOINT ["/usr/local/bin/migrate"] + ---> Using cache + ---> 7e8134892cf2 +Step 9/39 : CMD ["up"] + ---> Using cache + ---> adb6e8d9b58a +Step 10/39 : FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS builder + ---> 18b460dd1754 +Step 11/39 : WORKDIR /src + ---> Using cache + ---> 1ba013278a4a +Step 12/39 : RUN apk add --no-cache gcc musl-dev + ---> Using cache + ---> a0ea65f0bded +Step 13/39 : COPY go.mod go.sum ./ + ---> Using cache + ---> e98ccecfb093 +Step 14/39 : RUN go mod download + ---> Using cache + ---> 91d546c71507 +Step 15/39 : COPY cmd ./cmd + ---> Using cache + ---> 7d74a0366782 +Step 16/39 : COPY contracts ./contracts + ---> Using cache + ---> 89e9eb49cf2e +Step 17/39 : COPY dto ./dto + ---> Using cache + ---> b3608ba9d910 +Step 18/39 : COPY internal ./internal + ---> Using cache + ---> 420381f245d1 +Step 19/39 : COPY migrations ./migrations + ---> Using cache + ---> 244708889b45 +Step 20/39 : ENV GOFLAGS=-p=1 GOMAXPROCS=2 + ---> Using cache + ---> a86cf22e4c7d +Step 21/39 : RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/messenger ./cmd + ---> Using cache + ---> e1ea9cbf840a +Step 22/39 : RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/reconcile ./cmd/reconcile + ---> Using cache + ---> 4669d728553f +Step 23/39 : RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/worker ./cmd/worker + ---> Using cache + ---> fe51ba375215 +Step 24/39 : FROM builder AS tests + ---> fe51ba375215 +Step 25/39 : COPY --from=goose-builder /out/goose /usr/local/bin/goose + ---> Using cache + ---> 3ef73620b0a6 +Step 26/39 : ENV GOOSE_MIGRATION_DIR=/src/migrations + ---> Using cache + ---> 43905e46829a +Step 27/39 : CMD ["go", "test", "-p=1", "-race", "-count=1", "-v", "./..."] + ---> Using cache + ---> 037b4f43fd3c +Step 28/39 : FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 + ---> 075c2c1a4068 +Step 29/39 : RUN apk add --no-cache libwebp-tools=1.6.0-r0 + ---> Using cache + ---> 3a741cc26d52 +Step 30/39 : RUN addgroup -S messenger && adduser -S -D -H -u 10001 -G messenger messenger + ---> Using cache + ---> 444b794708dc +Step 31/39 : COPY --from=builder --chown=messenger:messenger /out/messenger /usr/local/bin/messenger + ---> 32fc6da7172f +Step 32/39 : COPY --from=builder --chown=messenger:messenger /out/reconcile /usr/local/bin/reconcile + ---> 49b1f874a5e0 +Step 33/39 : COPY --from=builder --chown=messenger:messenger /out/worker /usr/local/bin/worker + ---> f7ab6619c5c8 +Step 34/39 : ENV PORT=8080 + ---> Running in 0f8f2d74139c + ---> Removed intermediate container 0f8f2d74139c + ---> ab0764165606 +Step 35/39 : EXPOSE 8080 + ---> Running in 082ca714270a + ---> Removed intermediate container 082ca714270a + ---> 9a16ff1522d8 +Step 36/39 : USER messenger:messenger + ---> Running in 2fa42c5d86db + ---> Removed intermediate container 2fa42c5d86db + ---> 50c7d9a72042 +Step 37/39 : HEALTHCHECK --interval=10s --timeout=3s --start-period=3s --retries=3 CMD wget -q -O /dev/null "http://127.0.0.1:${PORT}/health/ready" || exit 1 + ---> Running in 8b938c89cf6d + ---> Removed intermediate container 8b938c89cf6d + ---> 74607fcd70f9 +Step 38/39 : STOPSIGNAL SIGTERM + ---> Running in e747d914802f + ---> Removed intermediate container e747d914802f + ---> 95748bec26d8 +Step 39/39 : ENTRYPOINT ["/usr/local/bin/messenger"] + ---> Running in b502e1c9cff0 + ---> Removed intermediate container b502e1c9cff0 + ---> 66e3c9bc004e +Successfully built 66e3c9bc004e +Successfully tagged mtusi-messenger:lab5 +DEPRECATED: The legacy builder is deprecated and will be removed in a future release. + Install the buildx component to build images with BuildKit: + https://docs.docker.com/go/buildx/ + +Sending build context to Docker daemon 461.3kB +Step 1/9 : FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS goose-builder + ---> 18b460dd1754 +Step 2/9 : RUN GOMAXPROCS=2 GOFLAGS=-p=1 GOBIN=/out CGO_ENABLED=0 go install -tags='no_clickhouse no_libsql no_mssql no_mysql no_sqlite3 no_vertica no_ydb no_spanner' github.com/pressly/goose/v3/cmd/goose@v3.27.1 + ---> Using cache + ---> 15f4b2e3468a +Step 3/9 : FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 AS migrations + ---> 075c2c1a4068 +Step 4/9 : COPY --from=goose-builder /out/goose /usr/local/bin/goose + ---> Using cache + ---> 6a0ad52336c1 +Step 5/9 : COPY migrations /migrations + ---> Using cache + ---> 01a2e64cbb7d +Step 6/9 : COPY infra/migrate.sh /usr/local/bin/migrate + ---> Using cache + ---> 7382e24476ff +Step 7/9 : USER 10001:10001 + ---> Using cache + ---> 6c54ddff6a36 +Step 8/9 : ENTRYPOINT ["/usr/local/bin/migrate"] + ---> Using cache + ---> 7e8134892cf2 +Step 9/9 : CMD ["up"] + ---> Using cache + ---> adb6e8d9b58a +Successfully built adb6e8d9b58a +Successfully tagged mtusi-migrations:lab5 +mtusi-lab5_api scaled to 0 +9mm30n5f5blu6kdfvhvartwr0 +mtusi-lab5_migrations.0.1hb6hxpeyw5j@colima | 2026/11/10 18:25:49 goose: no migrations to run. current version: 10 +mtusi-lab5_migrations +Since --detach=false was not specified, tasks will be created in the background. +In a future release, --detach=false will become the default. +Updating service mtusi-lab5_lb (id: tnlj2tzpx91wj066wqi3y5k44) +Updating service mtusi-lab5_postgres (id: f4xn3ammg3isejqsm9cqng6yo) +Updating service mtusi-lab5_redis (id: kz94wxnzhqwettqixbya643dl) +Updating service mtusi-lab5_minio (id: oaf1v50i58o60qnbk5flk0k59) +Updating service mtusi-lab5_api (id: oey97yu6vv15p4dn10pptf3b8) +Updating service mtusi-lab5_worker (id: lctq7ul45bu47a6vjbvt6xgeo) +hs7eigkxh4jah526bbx4nlt0t +mtusi-lab5_s3-bootstrap.0.veoz00w0hdfq@colima | PASS: private bucket and restricted API account configured. +mtusi-lab5_s3-bootstrap +mtusi-lab5_api scaled to 2 +mtusi-lab5_api +PASS: rolling update completed +PASS: Swarm stack ready at http://localhost:18085 diff --git a/evidence/lab5-failover.txt b/evidence/lab5-failover.txt new file mode 100644 index 00000000..b3f3d790 --- /dev/null +++ b/evidence/lab5-failover.txt @@ -0,0 +1,4 @@ +.venv/bin/python tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:18085" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 123, "failed_attempts": 1, "first_recovery_seconds": 0.696} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/lab5-faults-initial.txt b/evidence/lab5-faults-initial.txt new file mode 100644 index 00000000..757ddf02 --- /dev/null +++ b/evidence/lab5-faults-initial.txt @@ -0,0 +1,12 @@ +.venv/bin/python tests/own/lab5.py +Traceback (most recent call last): + File "/Users/dkhakimovig/mtusi2026lab5/tests/own/lab5.py", line 302, in + main() + File "/Users/dkhakimovig/mtusi2026lab5/tests/own/lab5.py", line 156, in main + variant_check(c,bob,item,body,shape) + File "/Users/dkhakimovig/mtusi2026lab5/tests/own/lab5.py", line 101, in variant_check + check((variants['thumbnail']['width'],variants['thumbnail']['height'])==expected_thumb,'thumbnail geometry') + File "/Users/dkhakimovig/mtusi2026lab5/tests/client.py", line 24, in check + raise Failure(message) +client.Failure: thumbnail geometry +make: *** [test-lab5] Error 1 diff --git a/evidence/lab5-faults-main.txt b/evidence/lab5-faults-main.txt new file mode 100644 index 00000000..34926db2 --- /dev/null +++ b/evidence/lab5-faults-main.txt @@ -0,0 +1,17 @@ +.venv/bin/python tests/own/lab5.py +PASS: input/ACL, three actual formats, geometry, EXIF, alpha +PASS: COMMIT/publish gap, queued link, API/broker restarts, worker recovery +PASS: crash after objects, expired lease/fencing, same four keys +PASS: COMMIT before ack, duplicate no-op and ready replay rejection +PASS: bounded retry/backoff, failed registry, safe audited replay +Traceback (most recent call last): + File "/Users/dkhakimovig/mtusi2026lab5/tests/own/lab5.py", line 310, in + main() + File "/Users/dkhakimovig/mtusi2026lab5/tests/own/lab5.py", line 277, in main + failed=wait_state(item,'failed') + File "/Users/dkhakimovig/mtusi2026lab5/tests/own/lab5.py", line 100, in wait_state + return until(lambda: job(att), lambda j: j['state']==state, timeout) + File "/Users/dkhakimovig/mtusi2026lab5/tests/own/lab5.py", line 39, in until + raise AssertionError('condition deadline exceeded; last='+str(last)) +AssertionError: condition deadline exceeded; last={'id': '4fc5ab5e-3e10-4e78-a6ea-9599d6f941be', 'attachment_id': '7122d895-5b38-480f-8238-8df26958810f', 'pipeline_version': 1, 'state': 'ready', 'attempts': 1, 'max_attempts': 2, 'available_at': '2026-11-10T18:47:28.488754+00:00', 'lease_until': None, 'fence': 1, 'last_error': None, 'created_at': '2026-11-10T18:47:28.33972+00:00', 'updated_at': '2026-11-10T18:47:28.488935+00:00'} +make: *** [test-lab5] Error 1 diff --git a/evidence/lab5-faults-tail.txt b/evidence/lab5-faults-tail.txt new file mode 100644 index 00000000..474922c1 --- /dev/null +++ b/evidence/lab5-faults-tail.txt @@ -0,0 +1,2 @@ +PASS: input/ACL, three actual formats, geometry, EXIF, alpha +PASS: all lab5 fault, retry, replay, durability, rendering and cleanup checks diff --git a/evidence/lab5-faults.json b/evidence/lab5-faults.json new file mode 100644 index 00000000..2fba181a --- /dev/null +++ b/evidence/lab5-faults.json @@ -0,0 +1,86 @@ +{ + "stack": "mtusi-lab5", + "cases": { + "input_and_acl": [ + { + "name": "oversized", + "status": 413 + }, + { + "name": "damaged", + "status": 415 + }, + { + "name": "pixel_budget", + "status": 413 + }, + { + "name": "mime_spoof", + "status": 415 + }, + { + "name": "filename", + "status": 400 + }, + { + "name": "unsupported", + "status": 415 + } + ], + "small_alpha_exif": "PASS", + "private_s3": { + "anonymous_get": 403, + "outside_prefix_denied": true + }, + "stage_before_commit": { + "upload_status": 503, + "metadata": 0, + "fresh_orphan_preserved": true + }, + "commit_publish_and_restarts": { + "job_id": "51f7ee76-3566-4837-a91d-74d5a0afec7b", + "api_restart": true, + "broker_restarts": 2, + "link_queued": true + }, + "crash_after_objects": { + "attempts": 2, + "objects": 4, + "fence": 2 + }, + "commit_before_ack_and_duplicate": { + "ready_unchanged": true, + "ready_replay_rejected": true + }, + "bounded_retry_replay": { + "max_attempts": 2, + "retry_available_at": "2026-11-10T18:47:14.297456+00:00", + "first_backoff_seconds": 2.492, + "failed_attempts": 2, + "audits": 1 + }, + "permanent_stage_error": { + "attempts": 1, + "error_code": "invalid_image" + }, + "renderer_process_group_timeout": { + "deadline_seconds": 2, + "elapsed_seconds": 2.27, + "cwebp_observed": true, + "descendants_after_failure": 0 + }, + "staging_cleanup": { + "ready_stages_remaining": 0, + "linked_original_preserved": true, + "failed_retention_days": 7, + "expired_failed_stage_removed": true, + "expired_replay_rejected": true, + "unconfirmed_orphan_removed": true + } + }, + "started_at": 1794336125.445686, + "resumed_at": [ + 1794336751.851368 + ], + "finished_at": 1794336891.683626 +} diff --git a/evidence/lab5-go-tests.txt b/evidence/lab5-go-tests.txt new file mode 100644 index 00000000..b0e96060 --- /dev/null +++ b/evidence/lab5-go-tests.txt @@ -0,0 +1,14 @@ +? gitflic.ru/akselbant/mtusi2026lab1-fork/cmd [no test files] +? gitflic.ru/akselbant/mtusi2026lab1-fork/cmd/reconcile [no test files] +? gitflic.ru/akselbant/mtusi2026lab1-fork/cmd/worker [no test files] +? gitflic.ru/akselbant/mtusi2026lab1-fork/contracts [no test files] +ok gitflic.ru/akselbant/mtusi2026lab1-fork/dto (cached) +ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/app/chat 14.228s +ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/attachment 2.227s +? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/config [no test files] +? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain [no test files] +ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/password (cached) +? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/repository [no test files] +ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/repository/postgres 1.666s +? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/session [no test files] +ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/usecase/chat (cached) diff --git a/evidence/lab5-integration-initial.txt b/evidence/lab5-integration-initial.txt new file mode 100644 index 00000000..0b97ef45 --- /dev/null +++ b/evidence/lab5-integration-initial.txt @@ -0,0 +1,355 @@ +.venv/bin/python scripts/swarm.py test-integration +DEPRECATED: The legacy builder is deprecated and will be removed in a future release. + Install the buildx component to build images with BuildKit: + https://docs.docker.com/go/buildx/ + +Sending build context to Docker daemon 460.3kB +Step 1/27 : FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS goose-builder + ---> 18b460dd1754 +Step 2/27 : RUN GOMAXPROCS=2 GOFLAGS=-p=1 GOBIN=/out CGO_ENABLED=0 go install -tags='no_clickhouse no_libsql no_mssql no_mysql no_sqlite3 no_vertica no_ydb no_spanner' github.com/pressly/goose/v3/cmd/goose@v3.27.1 + ---> Using cache + ---> 15f4b2e3468a +Step 3/27 : FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 AS migrations + ---> 075c2c1a4068 +Step 4/27 : COPY --from=goose-builder /out/goose /usr/local/bin/goose + ---> Using cache + ---> 6a0ad52336c1 +Step 5/27 : COPY migrations /migrations + ---> Using cache + ---> 01a2e64cbb7d +Step 6/27 : COPY infra/migrate.sh /usr/local/bin/migrate + ---> Using cache + ---> 7382e24476ff +Step 7/27 : USER 10001:10001 + ---> Using cache + ---> 6c54ddff6a36 +Step 8/27 : ENTRYPOINT ["/usr/local/bin/migrate"] + ---> Using cache + ---> 7e8134892cf2 +Step 9/27 : CMD ["up"] + ---> Using cache + ---> adb6e8d9b58a +Step 10/27 : FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS builder + ---> 18b460dd1754 +Step 11/27 : WORKDIR /src + ---> Using cache + ---> 1ba013278a4a +Step 12/27 : RUN apk add --no-cache gcc musl-dev + ---> Using cache + ---> a0ea65f0bded +Step 13/27 : COPY go.mod go.sum ./ + ---> Using cache + ---> e98ccecfb093 +Step 14/27 : RUN go mod download + ---> Using cache + ---> 91d546c71507 +Step 15/27 : COPY cmd ./cmd + ---> Using cache + ---> 7d74a0366782 +Step 16/27 : COPY contracts ./contracts + ---> Using cache + ---> 89e9eb49cf2e +Step 17/27 : COPY dto ./dto + ---> Using cache + ---> b3608ba9d910 +Step 18/27 : COPY internal ./internal + ---> 44f1a9efcc66 +Step 19/27 : COPY migrations ./migrations + ---> acefcd8a83af +Step 20/27 : ENV GOFLAGS=-p=1 GOMAXPROCS=2 + ---> Running in 6f0922eb42d3 + ---> Removed intermediate container 6f0922eb42d3 + ---> 2cda01161c62 +Step 21/27 : RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/messenger ./cmd + ---> Running in 26952912ceb7 + ---> Removed intermediate container 26952912ceb7 + ---> 889afe1963f8 +Step 22/27 : RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/reconcile ./cmd/reconcile + ---> Running in 04665a280741 + ---> Removed intermediate container 04665a280741 + ---> 43cd9030aa04 +Step 23/27 : RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/worker ./cmd/worker + ---> Running in 5b0325120af9 + ---> Removed intermediate container 5b0325120af9 + ---> 66906c7f16a8 +Step 24/27 : FROM builder AS tests + ---> 66906c7f16a8 +Step 25/27 : COPY --from=goose-builder /out/goose /usr/local/bin/goose + ---> 7e67bb289391 +Step 26/27 : ENV GOOSE_MIGRATION_DIR=/src/migrations + ---> Running in 1b5b33f07921 + ---> Removed intermediate container 1b5b33f07921 + ---> 3aba3008d340 +Step 27/27 : CMD ["go", "test", "-p=1", "-race", "-count=1", "-v", "./..."] + ---> Running in bb4863e04cfd + ---> Removed intermediate container bb4863e04cfd + ---> 471b66a42df0 +Successfully built 471b66a42df0 +Successfully tagged mtusi-messenger-tests:lab5 +e5yt4nofpygdupx0vhbijo6dj +mtusi-lab5_tests.0.wqzm1lukwaby@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/cmd [no test files] +mtusi-lab5_tests.0.wqzm1lukwaby@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/cmd/reconcile [no test files] +mtusi-lab5_tests.0.wqzm1lukwaby@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/cmd/worker [no test files] +mtusi-lab5_tests.0.wqzm1lukwaby@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/contracts [no test files] +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestMessageStrictFields +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestMessageStrictFields (0.00s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestTimestampFixedMicroseconds +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestTimestampFixedMicroseconds (0.00s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | PASS +mtusi-lab5_tests.0.wqzm1lukwaby@colima | ok gitflic.ru/akselbant/mtusi2026lab1-fork/dto 1.013s +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestRejectsMalformedJSON +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:44 request_id=78adaf84-7aaa-42f4-a084-a2db8cffcf98 method=POST path=/api/v1/users status=400 response_bytes=140 latency=131.98µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestRejectsMalformedJSON (0.00s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestHidesForeignChat +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:45 request_id=1bf6ff0c-9371-4b01-9941-77e94d023b10 method=POST path=/api/v1/users status=201 response_bytes=141 latency=365.251444ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:45 request_id=f50eb79b-cca7-4993-8191-8734738e9c40 method=POST path=/api/v1/auth/sessions status=201 response_bytes=269 latency=398.873152ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:45 request_id=5a967846-e8a8-449a-a09d-c8d365025b46 method=POST path=/api/v1/users status=201 response_bytes=147 latency=381.399429ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:46 request_id=714a25ea-10d1-4b85-8c24-ec4d371bb7b6 method=POST path=/api/v1/auth/sessions status=201 response_bytes=275 latency=422.991476ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:46 request_id=2345f91f-0c23-45d3-a491-8dfd5556d174 method=POST path=/api/v1/chats status=201 response_bytes=163 latency=188.281µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:46 request_id=0336f4ad-d624-4ecf-9cdf-51301294d0e8 method=GET path=/api/v1/chats/:chat_id status=404 response_bytes=126 latency=284.797µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestHidesForeignChat (1.57s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestRejectsEmptyMessage +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:46 request_id=44c2744c-4cd0-4c7e-992d-a4dfb0b061b6 method=POST path=/api/v1/users status=201 response_bytes=157 latency=368.07845ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:47 request_id=af848f3f-b52b-4af3-85f8-bd965300d1ee method=POST path=/api/v1/auth/sessions status=201 response_bytes=285 latency=366.256457ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:47 request_id=23df23c7-578f-4c76-a10b-104f2426b9cd method=POST path=/api/v1/chats status=201 response_bytes=163 latency=133.647µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:47 request_id=b90bf297-9272-4bfb-a83f-47b326cb5ce1 method=POST path=/api/v1/chats/:chat_id/messages status=400 response_bytes=172 latency=152.233µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestRejectsEmptyMessage (0.74s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestRejectsDuplicateUsername +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:47 request_id=178aa11a-597f-493f-b227-c667f089540b method=POST path=/api/v1/users status=201 response_bytes=149 latency=351.691662ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:47 request_id=2f3ba9cf-b14c-4ae9-adfd-ec9f3028a50e method=POST path=/api/v1/auth/sessions status=201 response_bytes=277 latency=367.085423ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:48 request_id=0a727f2e-dc6b-4c93-8639-de50d77626b5 method=POST path=/api/v1/users status=409 response_bytes=170 latency=359.315715ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestRejectsDuplicateUsername (1.08s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestPaginatesMessagesAcrossMultiplePages +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:48 request_id=b3142e33-707a-46f5-b178-35509037b4cc method=POST path=/api/v1/users status=201 response_bytes=151 latency=409.355414ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:48 request_id=448e921b-0b3a-434c-8961-b6aa51530d88 method=POST path=/api/v1/auth/sessions status=201 response_bytes=279 latency=387.814711ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:48 request_id=8cf5775f-be80-476a-8f65-1fae6ba4d114 method=POST path=/api/v1/chats status=201 response_bytes=163 latency=162.61µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:48 request_id=9e3a4eaf-e3f0-4c1c-88a7-651b43fde65c method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=266.503µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:48 request_id=57d23cd6-2482-4656-8621-670b43ef8d3d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=186.906µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:48 request_id=92183f25-6ab2-4c74-a970-9cf52a20d989 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=167.945µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:48 request_id=8da17cdd-9817-47dd-89e5-156bf0009a30 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=167.528µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:48 request_id=364216f6-407d-4f49-97ac-1d79491067b4 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=176.779µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:48 request_id=7abdd979-4157-4d85-bdb7-41b31fdb5612 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=130.438µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:48 request_id=30e71f25-5fcc-47fa-9349-eaa52423b428 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=134.814µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:48 request_id=17a01d04-e615-4d86-8016-fbf560e6f1e6 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=725 latency=380.521µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:48 request_id=f9ebd8b6-f40a-49b9-8c46-03524ae60c7e method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=725 latency=183.197µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:48 request_id=ec02dec4-7e4c-4fdd-b05e-0115f5485111 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=725 latency=174.862µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:48 request_id=90cae6ae-fe4e-4e2a-8765-2ad81c9abaf1 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=259 latency=114.519µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestPaginatesMessagesAcrossMultiplePages (0.81s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestConcurrentMessagesAreUnique +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:49 request_id=0430c49b-26f0-4036-b772-e6267ecd0b14 method=POST path=/api/v1/users status=201 response_bytes=153 latency=370.09502ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:49 request_id=7f5a51e7-7002-4917-84b0-dc9725be54a9 method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=360.387497ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:50 request_id=a349814a-0888-445e-9726-a33a6712c43c method=POST path=/api/v1/users status=201 response_bytes=153 latency=360.66339ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:50 request_id=fb3e86ae-a739-4e7d-bc78-559fadbbf787 method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=394.640836ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:50 request_id=0fc70956-e867-4377-96e5-329dda30cd31 method=POST path=/api/v1/users status=201 response_bytes=153 latency=367.006565ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:51 request_id=0f9813dc-7c0d-4163-9793-882e84468d33 method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=353.933555ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:51 request_id=4440bfa8-9d31-4297-9343-80ded2259f6a method=POST path=/api/v1/users status=201 response_bytes=153 latency=375.799206ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:51 request_id=56c0bfa7-4938-47a5-ac31-543cddca96e9 method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=366.48411ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:52 request_id=36983337-938e-4c9f-868a-7cc2f1525f5e method=POST path=/api/v1/users status=201 response_bytes=153 latency=373.712012ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:52 request_id=a2853b02-5099-4cbd-aa6c-cbfb617a289d method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=505.163831ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:53 request_id=2170619b-d50a-41a7-a77f-d38fa3fe45a2 method=POST path=/api/v1/users status=201 response_bytes=153 latency=379.44649ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:53 request_id=7fccfc33-4433-459d-929d-c59c71d7d620 method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=440.462082ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:54 request_id=f235d12f-6bf2-4483-93e5-5515c87d8b68 method=POST path=/api/v1/users status=201 response_bytes=153 latency=467.557845ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:54 request_id=c039f7d0-879a-4c87-aa6f-b26a7b0be534 method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=380.753972ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:54 request_id=e084cc17-3e34-4b4d-a505-8ee4ee3c2ed9 method=POST path=/api/v1/users status=201 response_bytes=153 latency=430.362118ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:55 request_id=77f5d7ed-f8a4-44bb-a7c6-35105b72861c method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=361.400539ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:55 request_id=2cee44b0-3bde-4274-a345-2a450e37d592 method=POST path=/api/v1/users status=201 response_bytes=153 latency=387.358501ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:56 request_id=8351a9bb-2d81-485e-9461-d3e1e180b1c7 method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=478.984823ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:56 request_id=4784bfab-a8bd-47ce-b569-4b7e519d4f38 method=POST path=/api/v1/users status=201 response_bytes=153 latency=694.465485ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=0876d022-4dd6-45ff-9a5d-74b7b4dece6e method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=507.870998ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=1af0e642-3a02-4cd6-88eb-295353c88b7c method=POST path=/api/v1/chats status=201 response_bytes=514 latency=303.26µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=72b9ccc9-d30f-4da3-9662-05f15722602f method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=651.443µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=81eb5c5b-f9a3-4403-9b8a-5ed360f4a78d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=359.06µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=95416f6f-e84b-4532-bf7e-4c70850733ca method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=594.975µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=c52d75be-211a-414e-bf1d-ce287a437a5a method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=253.001µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=3acdd29e-915f-4252-9928-1bb1561a6f8a method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=266.628µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=401c57a7-dda1-45a8-854e-1f90fec29b4d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.086266ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=b1a2a6f0-d442-425a-9185-b06af53e0287 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=495.209µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=e28e56e9-5b4b-4634-83a9-8ffa261246d8 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=3.567434ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=1be19fc0-187a-4240-b396-51f254a28846 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.795385ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=83f80360-8ebb-40c8-b3f9-83b596a60dba method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=201.617µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=f56ff57c-7170-41f5-b81f-ace5f259000d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=2.246253ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=51e5b495-8329-4087-afb7-be0ec16db915 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.059136ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=cdc680e8-fda4-45ab-9cc2-976891d745cb method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=3.058181ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=4f378cb9-5335-4a3f-a204-c04c512bc1ac method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=270.587µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=d15ea22e-b4d5-44ba-ad3f-a0d1cbded11d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=3.963708ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=ff677057-1037-4998-a7ee-d3ed7516cc16 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.902029ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=2edd5096-e6f7-41ef-a265-8ec8beeae9f5 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.739251ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=75d1f157-564b-46f0-80f0-a88d7da83642 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=2.940161ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=61cdfd52-4d67-4af2-a3cc-b7384ef86be9 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=191.199µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=fc5faa10-3c15-4e66-9f27-4de48fefa69a method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=161.194µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=6dec8a6b-f059-4615-9f28-7c0e265bdced method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=168.403µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=0ba37f3e-5012-4648-9cb9-fe6a9e391286 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=263.753µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=13417c22-fafc-481a-abd5-456111490e41 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=229.539µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=19f8ac4f-fc2a-4a64-9096-3a6cb8c0dc12 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=443.95µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=f1ca84e3-d6f6-4a64-bd98-e2aa24f96793 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=171.696µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=b6ab587d-f519-469f-88e8-9aa15221cfb1 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=160.111µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=12ec636a-0fbf-4c26-9834-59363cc0e5d9 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=346.683µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=94742954-5139-412c-940f-19dde72f8d50 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=217.537µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=2e6d7919-71d0-4411-9f86-a5312c7ea148 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=237.707µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=2acd4a86-79d3-47ac-9370-490faba63dd8 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=222.621µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=9fd113a4-8b1f-487d-83a0-98f21d3106a7 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=356.06µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=51e37132-149a-4c3b-98c9-cfade19a3ecc method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=299.883µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=056f9612-6681-49cb-be83-a5188a4ad784 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=198.45µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=f0933a5e-5e7d-4f67-9d21-b3cdc26b5ee7 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=367.978µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=fd5381a8-23d5-4e49-a2b3-a7446b8dc94a method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=368.145µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=18cb05e3-b296-47b9-b011-996dd1069d13 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=680.406µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=8b636c5e-df70-4e1c-907c-8a516890d49e method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.473581ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=5b7fa9d4-2545-4b7a-bd14-e2a0391d895d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=219.078µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=7e5258f4-6322-4d5d-857d-7724852f18a5 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=179.405µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=700838c9-da94-4e08-889e-e3346b73d285 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=197.325µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=cc55fc35-3e25-4d9c-b4ce-1aa9f7a296ed method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=193.741µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=d21bae1c-8414-4fef-ae5e-59ecc5c8e481 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=171.612µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=9ae11a53-4372-4cf8-9459-bb9055e5bb9f method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=268.212µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=4ae0bd79-fb34-41fa-aaff-47459dead558 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=177.614µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=c49603b9-6709-4f40-826b-0cb39c439bb5 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=170.237µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=ef856d83-34ef-46ee-b047-82ceaa8c3813 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=195.325µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=948df662-0eea-4e28-ac14-15d972735bfe method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=277.755µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=6e05a773-85ea-4308-873b-f413fc23ab67 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=210.077µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=741af3a0-5dbf-48ad-a053-d941cd97ee49 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=180.989µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=81450fe0-2922-4349-9f40-d3eb9199456d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=153.193µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=47bffcec-6f7f-4bba-8fc5-321e6d954324 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=167.028µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=4fe65afb-595d-4227-b676-ff0745afdc77 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=22.151061ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=a30669a7-089e-497a-b5ac-d7e3a3330b59 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=221.871µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=7f746257-b6b6-464c-a964-f4b666230e3b method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=278.297µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=cb03cc55-07b0-4c0f-8ff6-73ca6916e076 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=252.209µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=92d14786-6c5f-4cdd-89cb-fb75ef8a2375 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=225.58µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=3c195e16-268b-494b-ad32-9135c3ea6b9e method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=388.899µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=e1f9a9e4-3ae3-477c-8fd6-c8d068d8f68a method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=636.358µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=e230085e-920a-4cda-ac31-5a08e846406d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=502.085µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=b26a0e99-c1cf-4895-8a7c-b87f9010e4bf method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=962.245µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=eab81116-9035-4834-8d33-f179a4fc5757 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.862814ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=d8226a69-ff35-4e40-99dd-8c0c42cd15ba method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=358.143µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=a1d94b08-0423-4a8e-985e-8a62e07c0071 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=2.488835ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=ce158939-7a9b-4829-9e30-a9570247b067 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=309.719µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=7694a9dd-aad5-4b96-9d52-bf022c2d6101 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=2.671325ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=a75cbf2d-24f5-44ff-8b3e-2efc8858daeb method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=3.563808ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=cccd74dd-6055-4538-b589-6439bb2e2249 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=199.409µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=682cf78c-492a-4e2e-8d0f-600709d40332 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=268.004µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=b0fe5b42-55a9-4080-b152-17958d1f9f72 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=180.447µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=01ea3680-f345-41b6-9cf5-eaf5a42f98ce method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=201.368µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=41a3b666-77e0-45cd-903a-59c37d094a19 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=209.66µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=52fb0977-6696-48ed-914f-56bb16207ec5 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=314.261µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=a306642c-bb87-41a5-9c51-8d3dea67923b method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=266.67µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=47909112-0b12-4627-a977-eabc090eec23 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=277.297µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=628fa77e-5549-4bca-8ffb-671089e4a4e4 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=227.83µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=115e720e-d906-4299-a557-478100f07ffa method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=561.136µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=fdffc6bf-74ff-4e02-a849-d86f8469edf9 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=180.823µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=70c555f2-9f81-45c3-9cb1-85b4e80106f7 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=188.031µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=897b8deb-775f-4a0d-ac02-b8eca6e6a4ca method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=168.445µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=5b02ee7d-ae2b-45fa-8e5d-e796648368a4 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=278.255µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=59aff7ee-4d4d-487f-89dd-a9b9e1df8705 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=228.664µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=6d858392-c3f1-403d-82d7-3790e81b96da method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=232.164µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=5f077336-b854-49e5-8642-22dddecd1680 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=287.299µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=44d32f91-3883-4a4f-9722-4718d1f29ed6 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=363.228µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=b043468b-dcd8-4085-b21b-72427260aa7e method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=265.67µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=c4c49e7c-e296-4263-beb6-232f11a79069 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=178.071µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=22a15da2-c0e5-42ca-b117-18ecea6f4be5 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=178.155µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=9c6590f9-f07f-4bdd-a3fa-ccbfa2b34080 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=296.383µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=06413c5e-5e2f-4a5a-b3d8-72e79e095b1d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=213.16µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=c4fdcea5-2cb3-4852-a3e6-8f2cca4150c6 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=235.748µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=1e5ec364-bd31-459a-9e20-4df15f9201e5 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=162.861µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=97042c78-9f32-4690-bed4-3fdd12e6f90c method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=169.237µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=1294c60c-07eb-49e2-9688-8273b867a7b0 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=410.319µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=6b1d6ba4-31d5-4eca-a5cf-f4ab203fc38f method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=150.275µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=c9717b03-a4c6-437b-80b3-319a1b010200 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=293.008µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=e8d813c8-6743-4004-a193-4f9517cc29f0 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=237.081µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=7163a415-e1ce-4c92-88e9-87d3e7b5ebc4 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=993.375µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=e52dfded-ef1e-4815-9507-8b4c57b32403 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=368.062µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=8c03578f-5787-409f-b5ab-e61013240c28 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=330.056µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=ea8a1f84-ab01-4d6b-b4b2-1edd2745ea4a method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=362.228µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=7bd598d0-dfbe-4226-9d16-8815ace71d18 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=4332 latency=1.28755ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=a36de45d-8bf9-4ec2-ae7a-ab0d5badfd91 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=4332 latency=970.455µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=79982f51-85fd-4e19-b8df-6c6865ed1929 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=4332 latency=1.137775ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=6045807f-6a92-440d-b54e-dc7b4eee2e76 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=4332 latency=879.523µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=c4eadc24-2b1e-4318-81cc-90c604831194 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=4332 latency=669.904µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=0eb733fc-c439-41b9-b979-c46d54fd5a5b method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=3616 latency=583.306µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestConcurrentMessagesAreUnique (8.46s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | PASS +mtusi-lab5_tests.0.wqzm1lukwaby@colima | ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/app/chat 13.752s +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestSpoolOriginalAndFullValidation +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestSpoolOriginalAndFullValidation (0.00s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestSpoolLimitsBeforeDecode +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestSpoolLimitsBeforeDecode (0.02s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestFilenameAndCancelledUpload +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestFilenameAndCancelledUpload (0.00s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestReconcileRejectsUnsafeGrace +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestReconcileRejectsUnsafeGrace (0.00s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestValid16BitWidePNG +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestValid16BitWidePNG (0.11s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestAttachmentTimestampsUseFixedMicroseconds +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestAttachmentTimestampsUseFixedMicroseconds (0.00s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestFitGeometry +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestFitGeometry (0.00s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestOrientationsAndAlpha +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestOrientationsAndAlpha (0.00s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestEXIFOrientation +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestEXIFOrientation (0.00s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestFenceIntegration +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestFenceIntegration (0.07s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | PASS +mtusi-lab5_tests.0.wqzm1lukwaby@colima | ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/attachment 1.223s +mtusi-lab5_tests.0.wqzm1lukwaby@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/config [no test files] +mtusi-lab5_tests.0.wqzm1lukwaby@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain [no test files] +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestUnicodeAndSalt +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestUnicodeAndSalt (2.38s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | PASS +mtusi-lab5_tests.0.wqzm1lukwaby@colima | ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/password 3.400s +mtusi-lab5_tests.0.wqzm1lukwaby@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/repository [no test files] +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestS3WireFixture +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestS3WireFixture (0.01s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestIntegration +mtusi-lab5_tests.0.wqzm1lukwaby@colima | storage_test.go:84: goose up: 2026/11/10 18:20:05 OK 001_initial.sql (21.11ms) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:20:05 OK 002_idempotency.sql (1.75ms) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:20:05 OK 003_attachments.sql (5.64ms) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:20:05 OK 004_message_attachments.sql (1.97ms) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:20:05 OK 005_async_attachments.sql (5.92ms) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:20:05 OK 006_attachment_jobs.sql (4ms) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:20:05 OK 007_attachment_outbox.sql (2.56ms) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:20:05 OK 008_attachment_replays.sql (3.23ms) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:20:05 OK 009_backfill_pipeline.sql (2.32ms) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:20:05 OK 010_storage_cleanup.sql (817.31µs) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:20:05 goose: successfully migrated database to version: 10 +mtusi-lab5_tests.0.wqzm1lukwaby@colima | storage_test.go:85: goose up: 2026/11/10 18:20:05 goose: no migrations to run. current version: 10 +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestIntegration/registration_race +mtusi-lab5_tests.0.wqzm1lukwaby@colima | storage_test.go:121: 12 concurrent INSERTs: 1 success, 11 unique conflicts +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestIntegration/rollback +mtusi-lab5_tests.0.wqzm1lukwaby@colima | storage_test.go:139: failed second member INSERT left 0 chats and 0 members +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestIntegration/lab4_attachments_and_reconciliation +mtusi-lab5_tests.0.wqzm1lukwaby@colima | lab4_test.go:248: ACL, original bytes, cross-chat attachment rejection, ordered/concurrent idempotency and absent=[] passed +mtusi-lab5_tests.0.wqzm1lukwaby@colima | lab4_test.go:321: dry-run={Apply:false GraceSeconds:90 ObjectsScanned:5 MetadataScanned:2 BytesScanned:106 ListRequestsEstimate:1 StatRequests:3 DBQueries:14 Orphans:1 Missing:1 FreshSkipped:1 LockedSkipped:1 ReferencedMissing:1 Deleted:0 MarkedFailed:0 DurationMS:15 Actions:[{Kind:orphan Key:original/7a8e9f88-b271-4b9e-9431-b9aa433f1489 Applied:false} {Kind:missing Key:original/76f08381-064d-45b3-afa7-3e8bcb6a5dc3 Applied:false}]} apply={Apply:true GraceSeconds:90 ObjectsScanned:5 MetadataScanned:2 BytesScanned:106 ListRequestsEstimate:1 StatRequests:3 DBQueries:15 Orphans:1 Missing:1 FreshSkipped:1 LockedSkipped:1 ReferencedMissing:1 Deleted:1 MarkedFailed:1 DurationMS:7 Actions:[{Kind:orphan Key:original/7a8e9f88-b271-4b9e-9431-b9aa433f1489 Applied:true} {Kind:missing Key:original/76f08381-064d-45b3-afa7-3e8bcb6a5dc3 Applied:true}]} repeat={Apply:true GraceSeconds:90 ObjectsScanned:4 MetadataScanned:2 BytesScanned:100 ListRequestsEstimate:1 StatRequests:1 DBQueries:9 Orphans:0 Missing:0 FreshSkipped:1 LockedSkipped:1 ReferencedMissing:0 Deleted:0 MarkedFailed:0 DurationMS:3 Actions:[]}; fresh, locked and referenced objects retained +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestIntegration/keyset_and_explain +mtusi-lab5_tests.0.wqzm1lukwaby@colima | storage_test.go:193: EXPLAIN on 10,000 messages, cursor near the end (9,000): +mtusi-lab5_tests.0.wqzm1lukwaby@colima | storage_test.go:199: Limit (cost=0.29..17.99 rows=51 width=142) (actual time=0.013..0.032 rows=51 loops=1) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | storage_test.go:199: Buffers: shared hit=10 +mtusi-lab5_tests.0.wqzm1lukwaby@colima | storage_test.go:199: -> Index Scan using messages_history_idx on messages m (cost=0.29..347.47 rows=1000 width=142) (actual time=0.012..0.027 rows=51 loops=1) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | storage_test.go:199: Index Cond: ((chat_id = 'f74d97f7-bb21-47ea-9177-5f171f6cead5'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.9+00'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid))) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | storage_test.go:199: Buffers: shared hit=10 +mtusi-lab5_tests.0.wqzm1lukwaby@colima | storage_test.go:199: Planning Time: 0.034 ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | storage_test.go:199: Execution Time: 0.054 ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | storage_test.go:204: keyset traversal returned 10,000 distinct messages, including equal timestamps +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestIntegration/idempotency_race +mtusi-lab5_tests.0.wqzm1lukwaby@colima | storage_test.go:254: 16 concurrent PostgreSQL requests: one effect, identical replay, changed body conflicts +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestIntegration/goose_legacy_and_down +mtusi-lab5_tests.0.wqzm1lukwaby@colima | storage_test.go:268: goose up: exit status 1: 2026/11/10 18:20:06 OK 001_initial.sql (2.39ms) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:20:06 OK 002_idempotency.sql (1.26ms) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:20:06 OK 003_attachments.sql (2.48ms) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:20:06 OK 004_message_attachments.sql (2.81ms) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:20:06 goose run: ERROR 005_async_attachments.sql: failed to run SQL migration: failed to execute SQL query "ALTER TABLE attachments\n DROP CONSTRAINT attachments_status_check,\n ADD CONSTRAINT attachments_status_check CHECK (status IN ('queued','processing','ready','failed')),\n DROP CONSTRAINT attachments_check,\n ADD CONSTRAINT attachments_check CHECK ((status='failed' AND error_code IS NOT NULL) OR (status<>'failed' AND error_code IS NULL)),\n ADD COLUMN variants jsonb NOT NULL DEFAULT '[]',\n ADD COLUMN updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),\n ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1,\n ADD COLUMN staging_key text;": ERROR: column "variants" of relation "attachments" already exists (SQLSTATE 42701) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- FAIL: TestIntegration (0.64s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestIntegration/registration_race (0.03s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestIntegration/rollback (0.00s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestIntegration/lab4_attachments_and_reconciliation (0.07s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestIntegration/keyset_and_explain (0.09s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestIntegration/idempotency_race (0.02s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- FAIL: TestIntegration/goose_legacy_and_down (0.05s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | FAIL +mtusi-lab5_tests.0.wqzm1lukwaby@colima | FAIL gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/repository/postgres 0.667s +mtusi-lab5_tests.0.wqzm1lukwaby@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/session [no test files] +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestPasswordContract +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestPasswordContract (0.00s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | PASS +mtusi-lab5_tests.0.wqzm1lukwaby@colima | ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/usecase/chat 1.013s +mtusi-lab5_tests.0.wqzm1lukwaby@colima | FAIL +mtusi-lab5_tests +FAIL: tests failed +make: *** [test-integration] Error 1 diff --git a/evidence/lab5-integration.txt b/evidence/lab5-integration.txt new file mode 100644 index 00000000..0069f7b7 --- /dev/null +++ b/evidence/lab5-integration.txt @@ -0,0 +1,389 @@ +.venv/bin/python scripts/swarm.py test-integration +DEPRECATED: The legacy builder is deprecated and will be removed in a future release. + Install the buildx component to build images with BuildKit: + https://docs.docker.com/go/buildx/ + +Sending build context to Docker daemon 461.3kB +Step 1/27 : FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS goose-builder + ---> 18b460dd1754 +Step 2/27 : RUN GOMAXPROCS=2 GOFLAGS=-p=1 GOBIN=/out CGO_ENABLED=0 go install -tags='no_clickhouse no_libsql no_mssql no_mysql no_sqlite3 no_vertica no_ydb no_spanner' github.com/pressly/goose/v3/cmd/goose@v3.27.1 + ---> Using cache + ---> 15f4b2e3468a +Step 3/27 : FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 AS migrations + ---> 075c2c1a4068 +Step 4/27 : COPY --from=goose-builder /out/goose /usr/local/bin/goose + ---> Using cache + ---> 6a0ad52336c1 +Step 5/27 : COPY migrations /migrations + ---> Using cache + ---> 01a2e64cbb7d +Step 6/27 : COPY infra/migrate.sh /usr/local/bin/migrate + ---> Using cache + ---> 7382e24476ff +Step 7/27 : USER 10001:10001 + ---> Using cache + ---> 6c54ddff6a36 +Step 8/27 : ENTRYPOINT ["/usr/local/bin/migrate"] + ---> Using cache + ---> 7e8134892cf2 +Step 9/27 : CMD ["up"] + ---> Using cache + ---> adb6e8d9b58a +Step 10/27 : FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS builder + ---> 18b460dd1754 +Step 11/27 : WORKDIR /src + ---> Using cache + ---> 1ba013278a4a +Step 12/27 : RUN apk add --no-cache gcc musl-dev + ---> Using cache + ---> a0ea65f0bded +Step 13/27 : COPY go.mod go.sum ./ + ---> Using cache + ---> e98ccecfb093 +Step 14/27 : RUN go mod download + ---> Using cache + ---> 91d546c71507 +Step 15/27 : COPY cmd ./cmd + ---> Using cache + ---> 7d74a0366782 +Step 16/27 : COPY contracts ./contracts + ---> Using cache + ---> 89e9eb49cf2e +Step 17/27 : COPY dto ./dto + ---> Using cache + ---> b3608ba9d910 +Step 18/27 : COPY internal ./internal + ---> 420381f245d1 +Step 19/27 : COPY migrations ./migrations + ---> 244708889b45 +Step 20/27 : ENV GOFLAGS=-p=1 GOMAXPROCS=2 + ---> Running in bf624bf36a2e + ---> Removed intermediate container bf624bf36a2e + ---> a86cf22e4c7d +Step 21/27 : RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/messenger ./cmd + ---> Running in f0e750b5bd34 + ---> Removed intermediate container f0e750b5bd34 + ---> e1ea9cbf840a +Step 22/27 : RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/reconcile ./cmd/reconcile + ---> Running in d70ebca4dc47 + ---> Removed intermediate container d70ebca4dc47 + ---> 4669d728553f +Step 23/27 : RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/worker ./cmd/worker + ---> Running in e5f260b0016c + ---> Removed intermediate container e5f260b0016c + ---> fe51ba375215 +Step 24/27 : FROM builder AS tests + ---> fe51ba375215 +Step 25/27 : COPY --from=goose-builder /out/goose /usr/local/bin/goose + ---> 3ef73620b0a6 +Step 26/27 : ENV GOOSE_MIGRATION_DIR=/src/migrations + ---> Running in 4b59132a45e1 + ---> Removed intermediate container 4b59132a45e1 + ---> 43905e46829a +Step 27/27 : CMD ["go", "test", "-p=1", "-race", "-count=1", "-v", "./..."] + ---> Running in 9d6cbda904bc + ---> Removed intermediate container 9d6cbda904bc + ---> 037b4f43fd3c +Successfully built 037b4f43fd3c +Successfully tagged mtusi-messenger-tests:lab5 +4sbaveec2mbbm7q290z3mc4bh +mtusi-lab5_tests.0.ei8mkr461zio@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/cmd [no test files] +mtusi-lab5_tests.0.ei8mkr461zio@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/cmd/reconcile [no test files] +mtusi-lab5_tests.0.ei8mkr461zio@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/cmd/worker [no test files] +mtusi-lab5_tests.0.ei8mkr461zio@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/contracts [no test files] +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestMessageStrictFields +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestMessageStrictFields (0.00s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestTimestampFixedMicroseconds +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestTimestampFixedMicroseconds (0.00s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | PASS +mtusi-lab5_tests.0.ei8mkr461zio@colima | ok gitflic.ru/akselbant/mtusi2026lab1-fork/dto 1.016s +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestRejectsMalformedJSON +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:34 request_id=a120ebf4-66d8-4d9e-b9ef-404d1f0b8ad1 method=POST path=/api/v1/users status=400 response_bytes=140 latency=186.094µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestRejectsMalformedJSON (0.01s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestHidesForeignChat +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:35 request_id=525ea333-7fbc-4427-8ae2-082330c0b56f method=POST path=/api/v1/users status=201 response_bytes=141 latency=429.523996ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:35 request_id=3a55c465-0b19-4fe6-884c-12b97845202c method=POST path=/api/v1/auth/sessions status=201 response_bytes=269 latency=400.777506ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:36 request_id=91bc227a-7b92-459b-a9c7-c4a640a1ca3c method=POST path=/api/v1/users status=201 response_bytes=147 latency=384.621416ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:36 request_id=da2274f8-4ab2-4c54-981b-4b6375c89c48 method=POST path=/api/v1/auth/sessions status=201 response_bytes=275 latency=418.414288ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:36 request_id=d5be3cdc-dab8-48ae-8e97-027a7839b8d0 method=POST path=/api/v1/chats status=201 response_bytes=163 latency=196.223µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:36 request_id=c68e4735-48e5-4446-acbc-b756b3f00c63 method=GET path=/api/v1/chats/:chat_id status=404 response_bytes=126 latency=55.016µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestHidesForeignChat (1.64s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestRejectsEmptyMessage +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:36 request_id=f144edb2-0a71-4953-9712-670b4307d049 method=POST path=/api/v1/users status=201 response_bytes=157 latency=406.815108ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:37 request_id=a7cab490-48fc-49d4-8104-8eb04cb0e41f method=POST path=/api/v1/auth/sessions status=201 response_bytes=285 latency=405.008764ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:37 request_id=41a94031-c08c-4c45-9b9b-8ad54a797f9b method=POST path=/api/v1/chats status=201 response_bytes=163 latency=106.863µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:37 request_id=f2a1b6f2-94d2-4f39-9bac-ebae19dab481 method=POST path=/api/v1/chats/:chat_id/messages status=400 response_bytes=172 latency=151.21µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestRejectsEmptyMessage (0.82s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestRejectsDuplicateUsername +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:37 request_id=a4e26cce-1333-49cf-ac98-37aba8e40cf9 method=POST path=/api/v1/users status=201 response_bytes=149 latency=384.550494ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:38 request_id=bf6b6a52-a044-4104-b8df-71842f061941 method=POST path=/api/v1/auth/sessions status=201 response_bytes=277 latency=370.705527ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:38 request_id=53201beb-d49c-4fc0-8f28-815f7579b8f5 method=POST path=/api/v1/users status=409 response_bytes=170 latency=370.869332ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestRejectsDuplicateUsername (1.13s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestPaginatesMessagesAcrossMultiplePages +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:38 request_id=ac404967-cf50-4886-ab88-75fba0d66fbe method=POST path=/api/v1/users status=201 response_bytes=151 latency=399.448651ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:39 request_id=f4f96a97-08ed-49bb-8074-506429860102 method=POST path=/api/v1/auth/sessions status=201 response_bytes=279 latency=471.634839ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:39 request_id=eadaf592-9287-4360-86c8-4387b3829097 method=POST path=/api/v1/chats status=201 response_bytes=163 latency=126.203µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:39 request_id=2ac4a3fa-0fb8-40f0-b3a1-7af36163d37a method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=197.723µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:39 request_id=52535278-cbce-435e-bd1e-37a433e56dfa method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=179.885µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:39 request_id=5b11d2a6-386b-47be-aff7-317eb4fa2707 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=160.38µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:39 request_id=861d72de-e7b9-4bbc-a5e5-b366f4387472 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=140.582µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:39 request_id=0bddc2b2-b304-498d-9e09-1470cc7d31a7 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=119.618µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:39 request_id=2dc5031e-2f06-46e2-8ac7-0761ddad5e4d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=153.17µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:39 request_id=0d0e5d73-d66f-4083-a5d0-686c6dec4c3c method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=159.254µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:39 request_id=ac89c25b-9da9-42e3-9cd1-0c1e1f264ee1 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=725 latency=390.862µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:39 request_id=de3853f3-74ae-4083-bcc8-598bb45ab9b9 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=725 latency=242.07µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:39 request_id=43b8dc81-f545-4a36-a052-6dea583fdff4 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=725 latency=175.425µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:39 request_id=305b99d8-fcb8-4fed-a648-24c5f4e817dc method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=259 latency=121.993µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestPaginatesMessagesAcrossMultiplePages (0.88s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestConcurrentMessagesAreUnique +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:39 request_id=f62e7a14-b7ae-4c24-bc64-cc47ac50dc93 method=POST path=/api/v1/users status=201 response_bytes=153 latency=376.003337ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:40 request_id=9922e1a1-b272-4cb4-807e-9c872404a471 method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=371.739811ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:40 request_id=413c97b5-8149-4c0d-92d5-4d9120882e9a method=POST path=/api/v1/users status=201 response_bytes=153 latency=371.796731ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:40 request_id=417099fe-cbba-430c-ba5f-694776a15347 method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=401.933506ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:41 request_id=2cd09de2-85fe-4ad0-bc87-8187a366a970 method=POST path=/api/v1/users status=201 response_bytes=153 latency=395.859968ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:41 request_id=409307c0-72d4-44c4-9aa5-33b3c5752aa8 method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=396.505304ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:41 request_id=ebec8d8c-e6a1-4c8c-8122-e0130516d2fc method=POST path=/api/v1/users status=201 response_bytes=153 latency=373.820076ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:42 request_id=385f8376-dbcc-4c0f-bd5b-22e6cbf989a4 method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=408.30886ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:42 request_id=2e8f957b-af2d-4f82-890c-17ee5ddd9bdc method=POST path=/api/v1/users status=201 response_bytes=153 latency=401.205396ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:43 request_id=41a56fb0-e2cb-4312-a57c-6457c00f79fe method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=381.081401ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:43 request_id=121e271b-7634-48e5-8395-90b992a18b75 method=POST path=/api/v1/users status=201 response_bytes=153 latency=400.063202ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:43 request_id=cf890726-06a9-4a26-b148-48ad9d16c1f4 method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=377.397779ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:44 request_id=1d7ed2cb-025a-487e-9995-06a61e6ea59f method=POST path=/api/v1/users status=201 response_bytes=153 latency=380.557862ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:44 request_id=fb4f8e2f-725e-4559-beb8-d828632a0f1e method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=395.192982ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:45 request_id=357d86d1-d0be-479a-83df-b91f0da90c9f method=POST path=/api/v1/users status=201 response_bytes=153 latency=371.774229ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:45 request_id=38b565f2-da72-489b-8149-9e55aa927f0d method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=442.382653ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:45 request_id=b7b29434-a765-45ab-a3df-bcd2b31e5642 method=POST path=/api/v1/users status=201 response_bytes=153 latency=372.279093ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=731b7f36-a09c-46b0-beb5-69cec9ba6abe method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=370.335795ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=42c59da2-0dcb-4be0-a7f4-0de2f6a388a6 method=POST path=/api/v1/users status=201 response_bytes=153 latency=375.164369ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=b59fa1d6-c07f-4b0c-a80f-df0c91db11da method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=405.93703ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=092cd05e-deb7-4378-b811-25c9f7cd0bad method=POST path=/api/v1/chats status=201 response_bytes=514 latency=186.533µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=8ca6b6d5-4ae2-4464-8b7c-abdb8835cadd method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=208.303µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=9904aa19-424a-4c52-bcba-ec103dae6fab method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=158.803µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=439132c6-6030-45ae-bb6b-032782ad78d8 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=162.073µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=62bee418-056f-4d68-ad0b-d1494dae9149 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=169.233µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=6f1c1df5-2e1a-4a1f-a1e2-07c9b356c41b method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=168.571µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=e7097d00-0825-45a0-9703-43848b1b86f2 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=187.857µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=94a4778a-b603-4947-99e6-7262cb63a02c method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=178.462µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=61f7f04e-7749-4007-80fa-5348d23cc79b method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=158.596µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=339f3ebd-f56b-46c4-afb4-d838b6d19e22 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=202.095µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=664575f1-de2b-4035-8290-02621be7def0 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=567.213µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=c6933c53-6a84-4462-bd1b-a27c6104dce5 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=276.674µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=0d005f58-b73e-4c0c-80ef-90333f841d6d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.219435ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=571fead6-abcc-4205-a7fd-7d48ce65449a method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=875.838µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=ebb9eb40-526f-4139-a951-eb89822d28d7 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.808419ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=8328795d-6c01-4702-831b-9679f1c66839 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=153.215µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=b3030fcc-c444-49d7-a1c2-6132b1baa8da method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=142.662µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=2c933d51-f44b-4fac-a675-72c85774c512 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=484.438µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=c41778c0-56b2-49c1-9a82-1159463cffc6 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=2.414535ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=38ff6efb-454d-4772-8ba8-20683532d351 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.087575ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=4610cbff-d3c7-4083-a7b1-01621d4129cb method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.651312ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=8c2687e3-8b73-4720-bfa5-7596c6fdd5ed method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=3.211241ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=86372bfa-614a-4f92-bf1e-df5b302505bc method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=3.222457ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=a7cd6676-8436-4544-966b-bbc0c0241148 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=766.079µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=70bdf943-11df-447a-bfe1-3a5c22172a0d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=153.299µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=41665126-c28d-4623-943b-12c375e7cd4e method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=187.898µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=f53ba09b-53a2-4071-b898-f828b948e171 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=153.423µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=673d03d2-6c91-435f-8b8c-8aff5b58c314 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=142.455µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=fea6b8ec-d431-492d-b61b-9403f165f818 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=143.863µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=74604d89-3eac-4ee0-8ff1-a70255fe6908 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=167.908µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=42cf5708-e2e9-45c7-addc-375633b507eb method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=157.106µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=d7d86eee-ce6b-4c36-8b29-559b33612678 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=143.241µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=694a46d1-d858-4046-96b4-832eaa0f9983 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=147.09µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=c8afd69d-af62-4dd5-a275-b9446299c738 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=135.378µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=4cf77251-b87b-445b-b1b4-14096daa4bc4 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=135.543µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=2fb59757-2e18-4d3d-855a-2e258a19414f method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=144.856µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=f1d91a2c-c587-49ea-9868-7c70f1abffeb method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=148.415µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=1d2a1995-c266-4a4b-9544-b54480cebd2c method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=139.31µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=3912cb0a-9060-4302-84a5-5d7c786d71c4 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=137.861µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=93908403-558e-450b-866a-58f5a321c88d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=286.317µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=5ee59046-acf0-48c8-bb7d-5f21abe094d8 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=198.163µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=23c591a6-d56b-4018-b130-861d8e0589c8 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=135.999µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=31bbf0d9-a4af-4848-aefa-a771dcd55c40 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=309.453µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=1198591d-c43b-4c9b-bc76-4fad9f4bc40c method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=150.153µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=17e0a733-80ae-4f83-ae88-2fab97a5f044 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=156.734µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=7f40a24a-d447-45d6-beae-8b920e9bdbc7 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=134.592µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=bb29dbe8-0483-400a-9b60-02cd83c15485 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=505.546µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=e0da57fa-9e99-4f0c-8516-abdd4af85e46 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=725.602µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=428d1db9-9475-44e5-9eb6-c2c0b92bf418 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=491.226µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=eea31235-f85e-43df-bc23-9f6f0e7e7adb method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.944666ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=57a62d96-2f93-408e-9eb2-ce3e13c70a29 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.149243ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=1f5dd0c0-80f3-443d-a7ae-62c5aed88439 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.051817ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=2de77a11-98c7-45f5-91da-31b23e2d91c5 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=468.836µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=ac732004-5639-4c64-a815-de0e49ed3321 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=2.150691ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=54228bc8-0df7-4a23-b2d1-03ae892bd7f2 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.314626ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=d0d42de0-7fb6-46ae-b3bf-ca0d5950b03f method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.556204ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=68605d50-22f1-4dcb-88b3-a956b64e5cb3 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=366.981µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=d1b73e5c-4917-4a78-9c82-8add850b5079 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=282.924µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=dddebfc6-6f64-479f-80c3-87030922367c method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=295.381µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=4f3b527c-4137-4cb2-9473-49d56a420bbb method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=147.049µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=b9b06cbc-f7ac-44d6-9811-0fa2543cca70 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=153.34µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=e732f927-fe6d-40b9-a9ad-7782274e3650 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=345.377µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=0f10e634-118c-4cef-990c-1a7abb919ab5 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=360.07µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=4be5aad6-3a1d-494e-b0de-29e3a2972552 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=183.76µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=eddafdc2-c9b9-43d1-88f4-10f1c56071d2 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=134.964µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=b6c0615f-dce5-4631-aa5f-7dc2bf061681 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=290.705µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=a6c82fb3-0419-4bfb-badb-e4f45ca50d92 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=347.984µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=41704ab7-1120-4aba-84e8-628dc70275b1 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=589.562µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=76201778-5847-487a-b229-629354621449 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.083023ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=c0d50461-50d5-45a9-aaad-5af34d30a440 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=120.52µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=c1d344b9-dba4-4392-9e54-1efa06170360 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=169.605µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=f64ae733-f80b-4c1c-a953-e9ff5d10113b method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=128.011µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=6e66ae23-1cdf-4620-a8eb-0c12e6f01495 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=128.425µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=fb60631d-2a25-4a80-bddf-6667c0e12e2c method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=144.111µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=17c0b29d-7dd4-4b61-ade5-695194f19a68 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=124.824µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=6aa57338-428c-45a8-9d77-bd8c524f58d0 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=151.685µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=c886be6f-ee67-44a8-abe6-84e489cf975c method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=196.921µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=de16609f-4c14-412f-a4f1-f28d527599db method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=145.891µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=6876ce9a-90c3-4431-a056-7508ae43b034 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=118.988µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=2849e735-0d5e-4ab1-9932-d8be2bcdf65c method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=126.603µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=a883d328-3055-45f2-b971-ce43be573940 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=125.445µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=9f5f45ed-6b88-458d-9525-9326030919e1 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=126.356µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=0f74d6b0-8931-464a-aa5a-f26ad12daec4 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=119.94µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=fbdc3ebd-e88b-4864-8131-8ad4b89eb410 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=143.158µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=a7ba86ee-181b-4476-a7ab-a734c4ee266c method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=134.84µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=5e51912a-012a-4c56-9313-ceb9d44c4cc5 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=261.485µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=a3130f83-d27c-4ec2-87d3-59027067ea6a method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=124.245µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=b9e5b9b3-bbd6-4568-8753-d4351dc96839 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=147.545µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=4565e89e-caa9-4e4d-a078-a7ba9cbc479d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=140.593µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=3df97f91-bacf-4194-a2df-e91b8a94d2a8 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=176.723µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=27f66f60-b97d-4d3e-9503-1af72c291662 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=127.225µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:47 request_id=64e38cb8-6eb5-4a3a-a01d-090dccca1035 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=152.636µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:47 request_id=ae0c3b4f-fa7e-4f02-abf9-c4e5ca5f4236 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=199.983µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:47 request_id=58d8552c-cda1-47d2-8881-d1a43b03ad3e method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=129.335µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:47 request_id=1dab28fb-6c71-4aa4-b8d5-da8e86da9e8a method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=172.743µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:47 request_id=ce0e5337-dc4f-4b11-8afc-1edf7653bf7f method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=347.195µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:47 request_id=28900f9b-33f1-4266-8950-5b7ed26d3867 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=143.56µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:47 request_id=cfcd75a4-9fc8-4bd7-82ff-929b69ce829a method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=131.59µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:47 request_id=9fd19029-673f-4935-9668-84a2d0e683d5 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=124.839µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:47 request_id=4ef3f1ae-50c1-413e-8c4e-1dafa5fb2d1c method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=137.803µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:47 request_id=cbcb3daa-1979-4392-bc47-6f5b20ea3070 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=289.151µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:47 request_id=a5ea6313-9465-4d20-b068-2553d3bee370 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=4332 latency=776.079µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:47 request_id=48187e18-3ecd-4fbd-a062-c6de8a354daf method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=4332 latency=1.41448ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:47 request_id=1fc7fb98-1c6d-4376-832c-530e21630c58 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=4332 latency=694.317µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:47 request_id=abfeb221-d940-4411-a679-64678c036f6e method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=4332 latency=606.963µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:47 request_id=b0e89015-1603-402a-939c-eb16b757c1a6 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=4332 latency=536.633µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:47 request_id=01fed6c0-16a3-4df6-9b1b-bb20b74d7fe2 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=3616 latency=481.751µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestConcurrentMessagesAreUnique (7.82s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | PASS +mtusi-lab5_tests.0.ei8mkr461zio@colima | ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/app/chat 13.393s +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestSpoolOriginalAndFullValidation +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestSpoolOriginalAndFullValidation (0.00s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestSpoolLimitsBeforeDecode +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestSpoolLimitsBeforeDecode (0.03s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestFilenameAndCancelledUpload +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestFilenameAndCancelledUpload (0.01s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestReconcileRejectsUnsafeGrace +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestReconcileRejectsUnsafeGrace (0.00s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestValid16BitWidePNG +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestValid16BitWidePNG (0.10s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestAttachmentTimestampsUseFixedMicroseconds +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestAttachmentTimestampsUseFixedMicroseconds (0.00s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestFitGeometry +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestFitGeometry (0.00s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestOrientationsAndAlpha +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestOrientationsAndAlpha (0.00s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestEXIFOrientation +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestEXIFOrientation (0.00s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestFenceIntegration +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestFenceIntegration (0.07s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | PASS +mtusi-lab5_tests.0.ei8mkr461zio@colima | ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/attachment 1.244s +mtusi-lab5_tests.0.ei8mkr461zio@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/config [no test files] +mtusi-lab5_tests.0.ei8mkr461zio@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain [no test files] +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestUnicodeAndSalt +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestUnicodeAndSalt (2.70s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | PASS +mtusi-lab5_tests.0.ei8mkr461zio@colima | ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/password 3.754s +mtusi-lab5_tests.0.ei8mkr461zio@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/repository [no test files] +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestS3WireFixture +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestS3WireFixture (0.00s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestIntegration +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:85: goose up: 2026/11/10 18:24:55 OK 001_initial.sql (16.88ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:55 OK 002_idempotency.sql (1.65ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:55 OK 003_attachments.sql (3.72ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:55 OK 004_message_attachments.sql (1.35ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:55 OK 005_async_attachments.sql (4.89ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:55 OK 006_attachment_jobs.sql (2.81ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:55 OK 007_attachment_outbox.sql (2.14ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:55 OK 008_attachment_replays.sql (3.36ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:55 OK 009_backfill_pipeline.sql (1.54ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:55 OK 010_storage_cleanup.sql (431.27µs) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:55 goose: successfully migrated database to version: 10 +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:86: goose up: 2026/11/10 18:24:55 goose: no migrations to run. current version: 10 +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestIntegration/registration_race +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:122: 12 concurrent INSERTs: 1 success, 11 unique conflicts +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestIntegration/rollback +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:140: failed second member INSERT left 0 chats and 0 members +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestIntegration/lab4_attachments_and_reconciliation +mtusi-lab5_tests.0.ei8mkr461zio@colima | lab4_test.go:248: ACL, original bytes, cross-chat attachment rejection, ordered/concurrent idempotency and absent=[] passed +mtusi-lab5_tests.0.ei8mkr461zio@colima | lab4_test.go:321: dry-run={Apply:false GraceSeconds:90 ObjectsScanned:5 MetadataScanned:2 BytesScanned:106 ListRequestsEstimate:1 StatRequests:3 DBQueries:14 Orphans:1 Missing:1 FreshSkipped:1 LockedSkipped:1 ReferencedMissing:1 Deleted:0 MarkedFailed:0 DurationMS:9 Actions:[{Kind:orphan Key:original/93a9cd5a-2728-4884-9071-a77f1e762d68 Applied:false} {Kind:missing Key:original/fc633700-4e23-4d86-a40c-6242cfad8d27 Applied:false}]} apply={Apply:true GraceSeconds:90 ObjectsScanned:5 MetadataScanned:2 BytesScanned:106 ListRequestsEstimate:1 StatRequests:3 DBQueries:15 Orphans:1 Missing:1 FreshSkipped:1 LockedSkipped:1 ReferencedMissing:1 Deleted:1 MarkedFailed:1 DurationMS:7 Actions:[{Kind:orphan Key:original/93a9cd5a-2728-4884-9071-a77f1e762d68 Applied:true} {Kind:missing Key:original/fc633700-4e23-4d86-a40c-6242cfad8d27 Applied:true}]} repeat={Apply:true GraceSeconds:90 ObjectsScanned:4 MetadataScanned:2 BytesScanned:100 ListRequestsEstimate:1 StatRequests:1 DBQueries:9 Orphans:0 Missing:0 FreshSkipped:1 LockedSkipped:1 ReferencedMissing:0 Deleted:0 MarkedFailed:0 DurationMS:2 Actions:[]}; fresh, locked and referenced objects retained +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestIntegration/keyset_and_explain +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:194: EXPLAIN on 10,000 messages, cursor near the end (9,000): +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:200: Limit (cost=0.29..17.99 rows=51 width=142) (actual time=0.023..0.041 rows=51 loops=1) +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:200: Buffers: shared hit=10 +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:200: -> Index Scan using messages_history_idx on messages m (cost=0.29..347.47 rows=1000 width=142) (actual time=0.022..0.036 rows=51 loops=1) +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:200: Index Cond: ((chat_id = '4f6f6bd1-9d3d-4b0a-b567-b15a8a04c64f'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.9+00'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid))) +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:200: Buffers: shared hit=10 +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:200: Planning Time: 0.027 ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:200: Execution Time: 0.068 ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:205: keyset traversal returned 10,000 distinct messages, including equal timestamps +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestIntegration/idempotency_race +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:255: 16 concurrent PostgreSQL requests: one effect, identical replay, changed body conflicts +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestIntegration/goose_legacy_and_down +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:265: goose down-to: 2026/11/10 18:24:56 OK 010_storage_cleanup.sql (2.55ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 009_backfill_pipeline.sql (467.83µs) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 008_attachment_replays.sql (3.28ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 007_attachment_outbox.sql (3.2ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 006_attachment_jobs.sql (3.22ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 005_async_attachments.sql (2.97ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 goose: no migrations to run. current version: 4 +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:271: goose up: 2026/11/10 18:24:56 OK 001_initial.sql (1.76ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 002_idempotency.sql (1.26ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 003_attachments.sql (478.98µs) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 004_message_attachments.sql (2.98ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 005_async_attachments.sql (6.7ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 006_attachment_jobs.sql (2.98ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 007_attachment_outbox.sql (1.94ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 008_attachment_replays.sql (2.86ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 009_backfill_pipeline.sql (2.12ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 010_storage_cleanup.sql (485.17µs) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 goose: successfully migrated database to version: 10 +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:272: goose up: 2026/11/10 18:24:56 goose: no migrations to run. current version: 10 +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:279: adopted legacy schema: all existing messages are unchanged +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:280: goose reset: 2026/11/10 18:24:56 OK 010_storage_cleanup.sql (2.13ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 009_backfill_pipeline.sql (705.12µs) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 008_attachment_replays.sql (2.89ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 007_attachment_outbox.sql (2.62ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 006_attachment_jobs.sql (3.52ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 005_async_attachments.sql (3.49ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 004_message_attachments.sql (4.03ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 003_attachments.sql (3.47ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 002_idempotency.sql (961.4µs) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 001_initial.sql (8.51ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:285: goose up: 2026/11/10 18:24:56 OK 001_initial.sql (10.59ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 002_idempotency.sql (1.5ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 003_attachments.sql (3.42ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 004_message_attachments.sql (1.18ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 005_async_attachments.sql (3.95ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 006_attachment_jobs.sql (2.66ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 007_attachment_outbox.sql (1.8ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 008_attachment_replays.sql (2.38ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 009_backfill_pipeline.sql (1.43ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 010_storage_cleanup.sql (484.05µs) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 goose: successfully migrated database to version: 10 +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:289: down and up succeeded in isolated schema +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestIntegration (0.72s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestIntegration/registration_race (0.03s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestIntegration/rollback (0.00s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestIntegration/lab4_attachments_and_reconciliation (0.06s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestIntegration/keyset_and_explain (0.08s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestIntegration/idempotency_race (0.01s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestIntegration/goose_legacy_and_down (0.20s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | PASS +mtusi-lab5_tests.0.ei8mkr461zio@colima | ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/repository/postgres 1.752s +mtusi-lab5_tests.0.ei8mkr461zio@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/session [no test files] +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestPasswordContract +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestPasswordContract (0.00s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | PASS +mtusi-lab5_tests.0.ei8mkr461zio@colima | ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/usecase/chat 1.018s +mtusi-lab5_tests diff --git a/evidence/lab5-persistence.txt b/evidence/lab5-persistence.txt new file mode 100644 index 00000000..c7cc5f34 --- /dev/null +++ b/evidence/lab5-persistence.txt @@ -0,0 +1,3 @@ +.venv/bin/python tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:18085" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/lab5-research.csv b/evidence/lab5-research.csv new file mode 100644 index 00000000..c1c4fb8a --- /dev/null +++ b/evidence/lab5-research.csv @@ -0,0 +1,7 @@ +width,height,concurrency,prefetch,offered,accepted,completed,throughput_rps,ready_p95_ms,after_accept_p95_ms,upload_p95_ms,queue_lag_peak_seconds,rss_peak_bytes +512,512,1,1,20,20,20,3.316,4761.802,4554.64,328.115,4.078925,33816576 +512,512,2,4,20,20,20,4.013,4327.625,4169.426,173.305,3.172338,56823808 +512,512,4,8,20,20,20,3.903,4453.48,4281.149,191.95,2.665913,94711808 +1536,1536,1,1,20,20,20,0.774,22792.626,21836.999,1073.912,21.476154,77754368 +1536,1536,2,4,20,20,20,0.851,21471.061,20489.723,1147.664,18.202527,140812288 +1536,1536,4,8,20,20,20,0.847,20744.497,19885.435,1134.032,16.602282,235130880 diff --git a/evidence/lab5-research.json b/evidence/lab5-research.json new file mode 100644 index 00000000..61be85ed --- /dev/null +++ b/evidence/lab5-research.json @@ -0,0 +1,2099 @@ +{ + "track": 1, + "started_at": "2026-11-10T19:03:13.613546+00:00", + "stack": "mtusi-lab5", + "worker_image": "mtusi-messenger:lab5", + "worker_image_id": "sha256:66e3c9bc004e8a21f70f7f537d56d64b847ea099719a3b756cd8da9e3bfad5f7", + "environment": { + "docker_server": "29.5.2", + "docker_cpus": 2, + "docker_memory_bytes": 2053644288, + "architecture": "arm64", + "python": "3.9.6" + }, + "worker_limits": { + "NanoCPUs": 1000000000, + "MemoryBytes": 1610612736 + }, + "initial_settings": { + "worker": { + "WORKER_CONCURRENCY": "2", + "WORKER_PREFETCH": "4" + }, + "api": { + "MAX_OUTSTANDING_JOBS": null + } + }, + "requests_per_case": 20, + "upload_concurrency": 8, + "sampling_interval_min_seconds": 0.25, + "method": { + "throughput": "completed / burst makespan including client upload and polling", + "ready_latency": "client POST start to first observed ready; nearest-rank p95", + "queue_lag": "age of oldest queued PostgreSQL job in this chat; excludes processing", + "rss": "sum VmRSS for worker, renderers and cwebp; shared pages may count repeatedly", + "limitations": "single host; warmup per case; sampled peaks and polling latency; finite burst, no production SLA" + }, + "cases": [ + { + "concurrency": 1, + "prefetch": 1, + "width": 512, + "height": 512, + "size_bytes": 787252, + "input_sha256": "205c1114728681c54a23968d77501d0948b5abea6dfa6522bee2f91eeac3dfe2", + "offered": 20, + "accepted": 20, + "rejected": 0, + "completed": 20, + "makespan_seconds": 6.031, + "throughput_rps": 3.316, + "ready_p50_ms": 3005.016, + "ready_p95_ms": 4761.802, + "after_accept_p95_ms": 4554.64, + "upload_p95_ms": 328.115, + "queue_lag_peak_seconds": 4.078925, + "outstanding_peak": 19, + "rss_baseline_bytes": 15818752, + "rss_peak_bytes": 33816576, + "samples": [ + { + "seconds": 0.583, + "outstanding": 2, + "queued": 1, + "processing": 1, + "queue_lag_seconds": 0.01802, + "rss_bytes": 29978624 + }, + { + "seconds": 1.136, + "outstanding": 19, + "queued": 18, + "processing": 1, + "queue_lag_seconds": 0.573877, + "rss_bytes": 33816576 + }, + { + "seconds": 1.71, + "outstanding": 17, + "queued": 16, + "processing": 1, + "queue_lag_seconds": 1.10088, + "rss_bytes": 30461952 + }, + { + "seconds": 2.245, + "outstanding": 14, + "queued": 13, + "processing": 1, + "queue_lag_seconds": 1.551341, + "rss_bytes": 30990336 + }, + { + "seconds": 2.733, + "outstanding": 12, + "queued": 11, + "processing": 1, + "queue_lag_seconds": 1.994326, + "rss_bytes": 31334400 + }, + { + "seconds": 3.207, + "outstanding": 10, + "queued": 9, + "processing": 1, + "queue_lag_seconds": 2.390667, + "rss_bytes": 31141888 + }, + { + "seconds": 3.691, + "outstanding": 8, + "queued": 7, + "processing": 1, + "queue_lag_seconds": 2.818544, + "rss_bytes": 30248960 + }, + { + "seconds": 4.178, + "outstanding": 6, + "queued": 5, + "processing": 1, + "queue_lag_seconds": 3.272491, + "rss_bytes": 31404032 + }, + { + "seconds": 4.708, + "outstanding": 4, + "queued": 3, + "processing": 1, + "queue_lag_seconds": 3.762719, + "rss_bytes": 30318592 + }, + { + "seconds": 5.157, + "outstanding": 3, + "queued": 2, + "processing": 1, + "queue_lag_seconds": 4.078925, + "rss_bytes": 28913664 + }, + { + "seconds": 5.598, + "outstanding": 1, + "queued": 0, + "processing": 1, + "queue_lag_seconds": 0, + "rss_bytes": 15974400 + }, + { + "seconds": 6.03, + "outstanding": 0, + "queued": 0, + "processing": 0, + "queue_lag_seconds": 0, + "rss_bytes": 15974400 + } + ], + "observations": [ + { + "attachment_id": "646b2385-3ab6-445d-8b78-d1a8814c4e80", + "ready_ms": 662.87, + "upload_ms": 189.316 + }, + { + "attachment_id": "dd9a1cba-9a24-4de8-be88-5ab566c354f4", + "ready_ms": 1202.492, + "upload_ms": 229.308 + }, + { + "attachment_id": "f118b697-73b8-4c02-b5cb-a32c2c26c4c7", + "ready_ms": 1211.582, + "upload_ms": 252.032 + }, + { + "attachment_id": "0b3b8787-4622-4244-9f62-2f017ac7b36f", + "ready_ms": 1782.331, + "upload_ms": 269.074 + }, + { + "attachment_id": "5158f4c7-9a48-439f-a55d-d102a6b62b72", + "ready_ms": 1782.638, + "upload_ms": 288.937 + }, + { + "attachment_id": "9c04af07-d559-47a6-8141-07fa3600f1ae", + "ready_ms": 2283.593, + "upload_ms": 301.658 + }, + { + "attachment_id": "1c5eb5a6-4c88-46ae-bbe2-5775aa37a0dd", + "ready_ms": 2291.712, + "upload_ms": 328.115 + }, + { + "attachment_id": "75f0c388-755d-42f5-963d-cf4a5d2e3a3e", + "ready_ms": 2742.883, + "upload_ms": 341.367 + }, + { + "attachment_id": "4e5b2b2f-a535-4d4d-ad66-b044d3591611", + "ready_ms": 2602.965, + "upload_ms": 218.028 + }, + { + "attachment_id": "2906e543-876e-4573-9589-b34ccd299a16", + "ready_ms": 3005.016, + "upload_ms": 232.298 + }, + { + "attachment_id": "9d93b13c-aad0-488b-964b-69eee71b6c7e", + "ready_ms": 3043.318, + "upload_ms": 205.818 + }, + { + "attachment_id": "b2b3b8fe-087b-48fe-8e1a-ac63a2af9995", + "ready_ms": 3427.563, + "upload_ms": 255.924 + }, + { + "attachment_id": "ebed9087-a7e7-4747-a007-bde8811e6835", + "ready_ms": 3459.608, + "upload_ms": 254.825 + }, + { + "attachment_id": "bf03b424-088c-4524-b987-cd3412d8f7a2", + "ready_ms": 3862.413, + "upload_ms": 233.23 + }, + { + "attachment_id": "4ea92ffb-c786-4cf6-aea9-928187c709d7", + "ready_ms": 3903.646, + "upload_ms": 258.777 + }, + { + "attachment_id": "c63b7647-09ad-4c51-9288-d96b96053ea8", + "ready_ms": 4367.295, + "upload_ms": 230.246 + }, + { + "attachment_id": "72468d5c-21c3-474e-9c9c-975580cdb287", + "ready_ms": 4347.311, + "upload_ms": 215.974 + }, + { + "attachment_id": "697018a9-f526-4d61-8d76-4ad2cd59ad70", + "ready_ms": 4761.802, + "upload_ms": 207.163 + }, + { + "attachment_id": "2bb19513-f2c9-4b58-a06d-f338afb9598c", + "ready_ms": 4715.44, + "upload_ms": 167.403 + }, + { + "attachment_id": "b9005407-6f57-4553-8521-1acc7b0fbafe", + "ready_ms": 5106.537, + "upload_ms": 142.814 + } + ] + }, + { + "concurrency": 2, + "prefetch": 4, + "width": 512, + "height": 512, + "size_bytes": 787252, + "input_sha256": "205c1114728681c54a23968d77501d0948b5abea6dfa6522bee2f91eeac3dfe2", + "offered": 20, + "accepted": 20, + "rejected": 0, + "completed": 20, + "makespan_seconds": 4.983, + "throughput_rps": 4.013, + "ready_p50_ms": 2356.701, + "ready_p95_ms": 4327.625, + "after_accept_p95_ms": 4169.426, + "upload_p95_ms": 173.305, + "queue_lag_peak_seconds": 3.172338, + "outstanding_peak": 18, + "rss_baseline_bytes": 15888384, + "rss_peak_bytes": 56823808, + "samples": [ + { + "seconds": 0.485, + "outstanding": 6, + "queued": 4, + "processing": 2, + "queue_lag_seconds": 0.069336, + "rss_bytes": 44666880 + }, + { + "seconds": 1.083, + "outstanding": 18, + "queued": 16, + "processing": 2, + "queue_lag_seconds": 0.611984, + "rss_bytes": 46854144 + }, + { + "seconds": 1.67, + "outstanding": 16, + "queued": 14, + "processing": 2, + "queue_lag_seconds": 1.192728, + "rss_bytes": 56823808 + }, + { + "seconds": 2.252, + "outstanding": 12, + "queued": 10, + "processing": 2, + "queue_lag_seconds": 1.705331, + "rss_bytes": 46981120 + }, + { + "seconds": 2.843, + "outstanding": 10, + "queued": 8, + "processing": 2, + "queue_lag_seconds": 2.197292, + "rss_bytes": 56717312 + }, + { + "seconds": 3.499, + "outstanding": 7, + "queued": 6, + "processing": 1, + "queue_lag_seconds": 2.749598, + "rss_bytes": 44724224 + }, + { + "seconds": 4.037, + "outstanding": 4, + "queued": 2, + "processing": 2, + "queue_lag_seconds": 3.172338, + "rss_bytes": 49577984 + }, + { + "seconds": 4.537, + "outstanding": 2, + "queued": 0, + "processing": 2, + "queue_lag_seconds": 0, + "rss_bytes": 38957056 + }, + { + "seconds": 4.983, + "outstanding": 0, + "queued": 0, + "processing": 0, + "queue_lag_seconds": 0, + "rss_bytes": 16973824 + } + ], + "observations": [ + { + "attachment_id": "ce826562-f2d7-4646-a8cc-666fdc0e3445", + "ready_ms": 648.575, + "upload_ms": 93.272 + }, + { + "attachment_id": "14901dea-5a21-4972-b727-483944df4289", + "ready_ms": 1200.2, + "upload_ms": 75.948 + }, + { + "attachment_id": "f47aaab9-212f-4e51-96b4-a5f6560f8705", + "ready_ms": 1203.505, + "upload_ms": 110.701 + }, + { + "attachment_id": "099ea578-7850-4f8d-8405-5ded1065fda3", + "ready_ms": 1193.826, + "upload_ms": 122.626 + }, + { + "attachment_id": "408f9f25-d253-457b-ab48-c721d7009ef2", + "ready_ms": 1770.196, + "upload_ms": 158.926 + }, + { + "attachment_id": "e20381b7-ba3a-4c55-9101-0f04ac9d1146", + "ready_ms": 1807.439, + "upload_ms": 156.373 + }, + { + "attachment_id": "ef819d0c-67b9-4807-8aad-135d313de3a2", + "ready_ms": 2323.718, + "upload_ms": 172.673 + }, + { + "attachment_id": "1e74c982-9b68-4daf-bc86-b245cf8b3f55", + "ready_ms": 2243.281, + "upload_ms": 173.305 + }, + { + "attachment_id": "d1ce4c95-59b1-4f54-92f8-be6d4492b320", + "ready_ms": 2302.181, + "upload_ms": 166.344 + }, + { + "attachment_id": "4d512530-df1e-4cb7-854a-acd1b637eeae", + "ready_ms": 2356.701, + "upload_ms": 183.129 + }, + { + "attachment_id": "ab8f6f91-fd57-4358-91e7-4bd066c8cb63", + "ready_ms": 2899.235, + "upload_ms": 168.626 + }, + { + "attachment_id": "b775bd82-d85b-43b2-99e8-5c83564b3372", + "ready_ms": 2871.317, + "upload_ms": 170.58 + }, + { + "attachment_id": "49e64a58-dd48-4f80-a751-945ba319225a", + "ready_ms": 3432.304, + "upload_ms": 165.885 + }, + { + "attachment_id": "04a04ce5-153c-46be-9a61-db23f60cc080", + "ready_ms": 3456.342, + "upload_ms": 168.861 + }, + { + "attachment_id": "138ec4c7-c9ca-4588-aafb-156b0d09668c", + "ready_ms": 3868.115, + "upload_ms": 159.786 + }, + { + "attachment_id": "5bd994da-1dd8-4203-b31a-11038114d25c", + "ready_ms": 3935.046, + "upload_ms": 162.486 + }, + { + "attachment_id": "304e9528-c937-49c4-b218-2503fa608921", + "ready_ms": 3923.622, + "upload_ms": 160.39 + }, + { + "attachment_id": "ea458347-e9ac-4e16-bdc1-8bd337b7a51f", + "ready_ms": 3903.069, + "upload_ms": 153.094 + }, + { + "attachment_id": "9ba18ea9-13c3-433d-a446-a16b0e13dd19", + "ready_ms": 4344.972, + "upload_ms": 161.284 + }, + { + "attachment_id": "e96a7be8-9b5c-4711-9c52-45da94063afe", + "ready_ms": 4327.625, + "upload_ms": 158.199 + } + ] + }, + { + "concurrency": 4, + "prefetch": 8, + "width": 512, + "height": 512, + "size_bytes": 787252, + "input_sha256": "205c1114728681c54a23968d77501d0948b5abea6dfa6522bee2f91eeac3dfe2", + "offered": 20, + "accepted": 20, + "rejected": 0, + "completed": 20, + "makespan_seconds": 5.124, + "throughput_rps": 3.903, + "ready_p50_ms": 2900.885, + "ready_p95_ms": 4453.48, + "after_accept_p95_ms": 4281.149, + "upload_p95_ms": 191.95, + "queue_lag_peak_seconds": 2.665913, + "outstanding_peak": 20, + "rss_baseline_bytes": 15736832, + "rss_peak_bytes": 94711808, + "samples": [ + { + "seconds": 0.556, + "outstanding": 5, + "queued": 5, + "processing": 0, + "queue_lag_seconds": 0.095001, + "rss_bytes": 65802240 + }, + { + "seconds": 1.102, + "outstanding": 20, + "queued": 16, + "processing": 4, + "queue_lag_seconds": 0.658051, + "rss_bytes": 94711808 + }, + { + "seconds": 1.807, + "outstanding": 16, + "queued": 12, + "processing": 4, + "queue_lag_seconds": 1.258819, + "rss_bytes": 82518016 + }, + { + "seconds": 2.391, + "outstanding": 13, + "queued": 9, + "processing": 4, + "queue_lag_seconds": 1.72038, + "rss_bytes": 73089024 + }, + { + "seconds": 2.982, + "outstanding": 11, + "queued": 7, + "processing": 4, + "queue_lag_seconds": 2.266387, + "rss_bytes": 66252800 + }, + { + "seconds": 3.593, + "outstanding": 7, + "queued": 3, + "processing": 4, + "queue_lag_seconds": 2.665913, + "rss_bytes": 78663680 + }, + { + "seconds": 4.172, + "outstanding": 4, + "queued": 0, + "processing": 4, + "queue_lag_seconds": 0, + "rss_bytes": 59797504 + }, + { + "seconds": 4.669, + "outstanding": 2, + "queued": 0, + "processing": 2, + "queue_lag_seconds": 0, + "rss_bytes": 16719872 + }, + { + "seconds": 5.123, + "outstanding": 0, + "queued": 0, + "processing": 0, + "queue_lag_seconds": 0, + "rss_bytes": 16781312 + } + ], + "observations": [ + { + "attachment_id": "6fb4bcdb-d3a6-422e-aadf-5114682fbf6c", + "ready_ms": 1168.149, + "upload_ms": 118.86 + }, + { + "attachment_id": "38bd01ea-2541-4e0a-9273-a9056a7546e4", + "ready_ms": 1214.942, + "upload_ms": 114.237 + }, + { + "attachment_id": "9f617f89-4585-47be-a0e5-f55ce010b21b", + "ready_ms": 1243.659, + "upload_ms": 78.523 + }, + { + "attachment_id": "3248421b-96f1-423b-8c6e-ab29cbab2829", + "ready_ms": 1263.479, + "upload_ms": 102.431 + }, + { + "attachment_id": "ae7fb249-3399-4e9f-8d51-afc46c98e7f7", + "ready_ms": 2434.688, + "upload_ms": 155.371 + }, + { + "attachment_id": "84e866c0-c4a1-4996-898b-e19f90bf28f9", + "ready_ms": 2449.042, + "upload_ms": 163.852 + }, + { + "attachment_id": "0a3ebb63-4cd5-44d7-aa17-edf8972ea240", + "ready_ms": 2386.365, + "upload_ms": 166.507 + }, + { + "attachment_id": "655decd7-e199-45b0-9084-9242cba0fb63", + "ready_ms": 2489.014, + "upload_ms": 133.543 + }, + { + "attachment_id": "8ff95066-f752-49c2-832a-30f6f5aa7fc9", + "ready_ms": 2484.853, + "upload_ms": 178.776 + }, + { + "attachment_id": "5ee41346-dc3b-45ab-ab40-fab1b40a4b66", + "ready_ms": 2900.885, + "upload_ms": 191.95 + }, + { + "attachment_id": "689cfe15-7c9d-4917-a09c-67f7009e8f07", + "ready_ms": 2953.14, + "upload_ms": 146.877 + }, + { + "attachment_id": "f72ee233-7d79-4d9f-9dd8-2a1914974fd6", + "ready_ms": 2976.284, + "upload_ms": 143.164 + }, + { + "attachment_id": "60de5aea-2dc6-4e9e-9834-cde5d7da064a", + "ready_ms": 2954.715, + "upload_ms": 183.028 + }, + { + "attachment_id": "8c2bb984-9c2c-4c1f-9da7-d0ff68f4a6a2", + "ready_ms": 4023.765, + "upload_ms": 187.25 + }, + { + "attachment_id": "0dd7d082-238b-4865-b1cb-02424d3d783e", + "ready_ms": 4064.043, + "upload_ms": 182.425 + }, + { + "attachment_id": "82a36d66-a13f-415b-b54d-e900af9997c2", + "ready_ms": 4057.21, + "upload_ms": 183.918 + }, + { + "attachment_id": "8f59adc2-b335-42b5-844e-93c1fa2526ed", + "ready_ms": 4031.183, + "upload_ms": 193.777 + }, + { + "attachment_id": "cd599cbe-b96b-4d28-8120-0ab9a8e90242", + "ready_ms": 4453.48, + "upload_ms": 172.33 + }, + { + "attachment_id": "d8ca0870-e258-4e7e-a244-ffa672e85bdb", + "ready_ms": 4426.811, + "upload_ms": 148.3 + }, + { + "attachment_id": "35abe4b6-19fd-437c-befc-1dc4d4d41cf8", + "ready_ms": 4497.526, + "upload_ms": 186.67 + } + ] + }, + { + "concurrency": 1, + "prefetch": 1, + "width": 1536, + "height": 1536, + "size_bytes": 7081647, + "input_sha256": "015b73f610b2145516211f8039594f43e64dfdaf0326b2874b294727f114de02", + "offered": 20, + "accepted": 20, + "rejected": 0, + "completed": 20, + "makespan_seconds": 25.854, + "throughput_rps": 0.774, + "ready_p50_ms": 12939.763, + "ready_p95_ms": 22792.626, + "after_accept_p95_ms": 21836.999, + "upload_p95_ms": 1073.912, + "queue_lag_peak_seconds": 21.476154, + "outstanding_peak": 20, + "rss_baseline_bytes": 15982592, + "rss_peak_bytes": 77754368, + "samples": [ + { + "seconds": 1.02, + "outstanding": 0, + "queued": 0, + "processing": 0, + "queue_lag_seconds": 0, + "rss_bytes": 33550336 + }, + { + "seconds": 1.8, + "outstanding": 10, + "queued": 9, + "processing": 1, + "queue_lag_seconds": 0.688855, + "rss_bytes": 53231616 + }, + { + "seconds": 2.378, + "outstanding": 15, + "queued": 14, + "processing": 1, + "queue_lag_seconds": 1.426091, + "rss_bytes": 53948416 + }, + { + "seconds": 2.879, + "outstanding": 20, + "queued": 19, + "processing": 1, + "queue_lag_seconds": 1.947211, + "rss_bytes": 16060416 + }, + { + "seconds": 3.423, + "outstanding": 19, + "queued": 18, + "processing": 1, + "queue_lag_seconds": 2.417889, + "rss_bytes": 53731328 + }, + { + "seconds": 4.011, + "outstanding": 19, + "queued": 18, + "processing": 1, + "queue_lag_seconds": 2.991099, + "rss_bytes": 74772480 + }, + { + "seconds": 4.552, + "outstanding": 18, + "queued": 17, + "processing": 1, + "queue_lag_seconds": 3.494125, + "rss_bytes": 53624832 + }, + { + "seconds": 5.073, + "outstanding": 18, + "queued": 17, + "processing": 1, + "queue_lag_seconds": 4.010811, + "rss_bytes": 74854400 + }, + { + "seconds": 5.596, + "outstanding": 17, + "queued": 16, + "processing": 1, + "queue_lag_seconds": 4.539016, + "rss_bytes": 53174272 + }, + { + "seconds": 6.251, + "outstanding": 17, + "queued": 16, + "processing": 1, + "queue_lag_seconds": 5.130687, + "rss_bytes": 64868352 + }, + { + "seconds": 6.763, + "outstanding": 16, + "queued": 15, + "processing": 1, + "queue_lag_seconds": 5.607154, + "rss_bytes": 48951296 + }, + { + "seconds": 7.335, + "outstanding": 16, + "queued": 15, + "processing": 1, + "queue_lag_seconds": 6.118839, + "rss_bytes": 60424192 + }, + { + "seconds": 7.84, + "outstanding": 15, + "queued": 14, + "processing": 1, + "queue_lag_seconds": 6.573697, + "rss_bytes": 50601984 + }, + { + "seconds": 8.348, + "outstanding": 15, + "queued": 14, + "processing": 1, + "queue_lag_seconds": 7.107352, + "rss_bytes": 53428224 + }, + { + "seconds": 8.827, + "outstanding": 15, + "queued": 14, + "processing": 1, + "queue_lag_seconds": 7.583777, + "rss_bytes": 16121856 + }, + { + "seconds": 9.337, + "outstanding": 14, + "queued": 13, + "processing": 1, + "queue_lag_seconds": 7.987629, + "rss_bytes": 52092928 + }, + { + "seconds": 9.826, + "outstanding": 14, + "queued": 13, + "processing": 1, + "queue_lag_seconds": 8.50605, + "rss_bytes": 64462848 + }, + { + "seconds": 10.355, + "outstanding": 13, + "queued": 12, + "processing": 1, + "queue_lag_seconds": 8.956849, + "rss_bytes": 50950144 + }, + { + "seconds": 10.835, + "outstanding": 13, + "queued": 12, + "processing": 1, + "queue_lag_seconds": 9.439881, + "rss_bytes": 57647104 + }, + { + "seconds": 11.36, + "outstanding": 12, + "queued": 11, + "processing": 1, + "queue_lag_seconds": 9.732071, + "rss_bytes": 48742400 + }, + { + "seconds": 11.899, + "outstanding": 12, + "queued": 11, + "processing": 1, + "queue_lag_seconds": 10.240891, + "rss_bytes": 53321728 + }, + { + "seconds": 12.396, + "outstanding": 12, + "queued": 11, + "processing": 1, + "queue_lag_seconds": 10.797879, + "rss_bytes": 16224256 + }, + { + "seconds": 12.877, + "outstanding": 11, + "queued": 10, + "processing": 1, + "queue_lag_seconds": 11.191622, + "rss_bytes": 53665792 + }, + { + "seconds": 13.351, + "outstanding": 11, + "queued": 10, + "processing": 1, + "queue_lag_seconds": 11.66977, + "rss_bytes": 77754368 + }, + { + "seconds": 13.824, + "outstanding": 10, + "queued": 9, + "processing": 1, + "queue_lag_seconds": 11.950794, + "rss_bytes": 52498432 + }, + { + "seconds": 14.358, + "outstanding": 10, + "queued": 9, + "processing": 1, + "queue_lag_seconds": 12.498662, + "rss_bytes": 57901056 + }, + { + "seconds": 14.837, + "outstanding": 9, + "queued": 8, + "processing": 1, + "queue_lag_seconds": 12.758517, + "rss_bytes": 47919104 + }, + { + "seconds": 15.313, + "outstanding": 9, + "queued": 8, + "processing": 1, + "queue_lag_seconds": 13.243371, + "rss_bytes": 53309440 + }, + { + "seconds": 15.775, + "outstanding": 9, + "queued": 8, + "processing": 1, + "queue_lag_seconds": 13.706933, + "rss_bytes": 62353408 + }, + { + "seconds": 16.325, + "outstanding": 8, + "queued": 7, + "processing": 1, + "queue_lag_seconds": 14.200687, + "rss_bytes": 53628928 + }, + { + "seconds": 16.786, + "outstanding": 8, + "queued": 7, + "processing": 1, + "queue_lag_seconds": 14.666856, + "rss_bytes": 70582272 + }, + { + "seconds": 17.278, + "outstanding": 7, + "queued": 6, + "processing": 1, + "queue_lag_seconds": 15.03883, + "rss_bytes": 49065984 + }, + { + "seconds": 17.734, + "outstanding": 7, + "queued": 6, + "processing": 1, + "queue_lag_seconds": 15.50651, + "rss_bytes": 52953088 + }, + { + "seconds": 18.344, + "outstanding": 7, + "queued": 6, + "processing": 1, + "queue_lag_seconds": 16.077438, + "rss_bytes": 16084992 + }, + { + "seconds": 18.792, + "outstanding": 6, + "queued": 5, + "processing": 1, + "queue_lag_seconds": 16.479718, + "rss_bytes": 53477376 + }, + { + "seconds": 19.245, + "outstanding": 6, + "queued": 5, + "processing": 1, + "queue_lag_seconds": 16.930317, + "rss_bytes": 76738560 + }, + { + "seconds": 19.695, + "outstanding": 5, + "queued": 4, + "processing": 1, + "queue_lag_seconds": 17.31214, + "rss_bytes": 48459776 + }, + { + "seconds": 20.195, + "outstanding": 5, + "queued": 4, + "processing": 1, + "queue_lag_seconds": 17.799775, + "rss_bytes": 60489728 + }, + { + "seconds": 20.648, + "outstanding": 5, + "queued": 4, + "processing": 1, + "queue_lag_seconds": 18.259385, + "rss_bytes": 15990784 + }, + { + "seconds": 21.143, + "outstanding": 4, + "queued": 3, + "processing": 1, + "queue_lag_seconds": 18.717576, + "rss_bytes": 53784576 + }, + { + "seconds": 21.606, + "outstanding": 4, + "queued": 3, + "processing": 1, + "queue_lag_seconds": 19.175829, + "rss_bytes": 73236480 + }, + { + "seconds": 22.058, + "outstanding": 3, + "queued": 2, + "processing": 1, + "queue_lag_seconds": 19.563274, + "rss_bytes": 52822016 + }, + { + "seconds": 22.516, + "outstanding": 3, + "queued": 2, + "processing": 1, + "queue_lag_seconds": 20.017596, + "rss_bytes": 66035712 + }, + { + "seconds": 23.125, + "outstanding": 2, + "queued": 2, + "processing": 0, + "queue_lag_seconds": 20.531987, + "rss_bytes": 47538176 + }, + { + "seconds": 23.606, + "outstanding": 2, + "queued": 1, + "processing": 1, + "queue_lag_seconds": 21.033856, + "rss_bytes": 53334016 + }, + { + "seconds": 24.05, + "outstanding": 2, + "queued": 1, + "processing": 1, + "queue_lag_seconds": 21.476154, + "rss_bytes": 66752512 + }, + { + "seconds": 24.52, + "outstanding": 1, + "queued": 0, + "processing": 1, + "queue_lag_seconds": 0, + "rss_bytes": 51838976 + }, + { + "seconds": 24.956, + "outstanding": 1, + "queued": 0, + "processing": 1, + "queue_lag_seconds": 0, + "rss_bytes": 63279104 + }, + { + "seconds": 25.408, + "outstanding": 1, + "queued": 0, + "processing": 1, + "queue_lag_seconds": 0, + "rss_bytes": 16261120 + }, + { + "seconds": 25.854, + "outstanding": 0, + "queued": 0, + "processing": 0, + "queue_lag_seconds": 0, + "rss_bytes": 16261120 + } + ], + "observations": [ + { + "attachment_id": "72ff9bf3-1dff-4644-a4d6-ae4bd7a617a8", + "ready_ms": 2978.062, + "upload_ms": 489.72 + }, + { + "attachment_id": "abe6a64b-f1e5-40d5-8ceb-3a7da3999640", + "ready_ms": 4113.182, + "upload_ms": 582.949 + }, + { + "attachment_id": "1dadde6d-8831-4035-91d1-00dd21897af4", + "ready_ms": 5167.355, + "upload_ms": 651.946 + }, + { + "attachment_id": "408e49cd-a346-42b2-8c8c-9ef1613ff231", + "ready_ms": 6855.691, + "upload_ms": 771.882 + }, + { + "attachment_id": "8277e968-cb19-42b4-9d1d-99d4221e1545", + "ready_ms": 7948.368, + "upload_ms": 855.385 + }, + { + "attachment_id": "e26f1421-f2a4-4427-a07e-af265f8f0b65", + "ready_ms": 8890.393, + "upload_ms": 944.604 + }, + { + "attachment_id": "e19e3655-8a95-4379-b94f-224ec7b3855e", + "ready_ms": 9851.15, + "upload_ms": 1016.483 + }, + { + "attachment_id": "6ef71b51-44a7-4423-b9e3-89541b97e970", + "ready_ms": 11385.961, + "upload_ms": 1105.276 + }, + { + "attachment_id": "110f5859-5437-4fc5-90c9-81791efeab6b", + "ready_ms": 11913.871, + "upload_ms": 658.419 + }, + { + "attachment_id": "c8a362ec-ea03-4cd2-8eba-3438a6debef2", + "ready_ms": 12939.763, + "upload_ms": 850.829 + }, + { + "attachment_id": "64ded537-4b0a-4745-b4dc-3900ca79c812", + "ready_ms": 14150.284, + "upload_ms": 670.543 + }, + { + "attachment_id": "bf549960-5414-4d26-96f3-dd6431f6a987", + "ready_ms": 15205.165, + "upload_ms": 1006.479 + }, + { + "attachment_id": "cb4c3dbb-75b0-458e-873a-69fec2a480e4", + "ready_ms": 15905.145, + "upload_ms": 891.157 + }, + { + "attachment_id": "14436df8-d3da-4b50-8e68-7b724da9be51", + "ready_ms": 17584.614, + "upload_ms": 1045.919 + }, + { + "attachment_id": "0ed68cb8-e3e6-45c2-9773-bf934db75454", + "ready_ms": 18110.868, + "upload_ms": 791.025 + }, + { + "attachment_id": "30515ce8-61dc-4211-ad60-f09c2da702dd", + "ready_ms": 19659.633, + "upload_ms": 1046.097 + }, + { + "attachment_id": "cca7e14a-127d-4deb-8e52-7dc87a7ad36b", + "ready_ms": 20262.643, + "upload_ms": 750.774 + }, + { + "attachment_id": "7a185ba2-5492-45d6-bfcf-ddd7fdaf04d1", + "ready_ms": 22053.501, + "upload_ms": 1073.912 + }, + { + "attachment_id": "978ae03a-8e42-4bdf-9beb-9ae487f82651", + "ready_ms": 22792.626, + "upload_ms": 955.627 + }, + { + "attachment_id": "c428751a-9273-415a-86d7-2f7f69d0664b", + "ready_ms": 23836.824, + "upload_ms": 721.881 + } + ] + }, + { + "concurrency": 2, + "prefetch": 4, + "width": 1536, + "height": 1536, + "size_bytes": 7081647, + "input_sha256": "015b73f610b2145516211f8039594f43e64dfdaf0326b2874b294727f114de02", + "offered": 20, + "accepted": 20, + "rejected": 0, + "completed": 20, + "makespan_seconds": 23.508, + "throughput_rps": 0.851, + "ready_p50_ms": 12230.053, + "ready_p95_ms": 21471.061, + "after_accept_p95_ms": 20489.723, + "upload_p95_ms": 1147.664, + "queue_lag_peak_seconds": 18.202527, + "outstanding_peak": 20, + "rss_baseline_bytes": 15872000, + "rss_peak_bytes": 140812288, + "samples": [ + { + "seconds": 0.804, + "outstanding": 0, + "queued": 0, + "processing": 0, + "queue_lag_seconds": 0, + "rss_bytes": 15970304 + }, + { + "seconds": 1.609, + "outstanding": 7, + "queued": 5, + "processing": 2, + "queue_lag_seconds": 0.522618, + "rss_bytes": 85676032 + }, + { + "seconds": 2.335, + "outstanding": 13, + "queued": 11, + "processing": 2, + "queue_lag_seconds": 1.388412, + "rss_bytes": 91406336 + }, + { + "seconds": 2.964, + "outstanding": 20, + "queued": 18, + "processing": 2, + "queue_lag_seconds": 2.023378, + "rss_bytes": 102551552 + }, + { + "seconds": 3.531, + "outstanding": 20, + "queued": 18, + "processing": 2, + "queue_lag_seconds": 2.574746, + "rss_bytes": 62296064 + }, + { + "seconds": 4.141, + "outstanding": 18, + "queued": 16, + "processing": 2, + "queue_lag_seconds": 2.974395, + "rss_bytes": 86925312 + }, + { + "seconds": 4.741, + "outstanding": 18, + "queued": 16, + "processing": 2, + "queue_lag_seconds": 3.553895, + "rss_bytes": 90320896 + }, + { + "seconds": 5.322, + "outstanding": 18, + "queued": 16, + "processing": 2, + "queue_lag_seconds": 4.137822, + "rss_bytes": 117186560 + }, + { + "seconds": 6.059, + "outstanding": 18, + "queued": 16, + "processing": 2, + "queue_lag_seconds": 4.877053, + "rss_bytes": 24940544 + }, + { + "seconds": 6.63, + "outstanding": 16, + "queued": 14, + "processing": 2, + "queue_lag_seconds": 5.248695, + "rss_bytes": 91217920 + }, + { + "seconds": 7.249, + "outstanding": 16, + "queued": 14, + "processing": 2, + "queue_lag_seconds": 5.841495, + "rss_bytes": 91856896 + }, + { + "seconds": 7.835, + "outstanding": 16, + "queued": 14, + "processing": 2, + "queue_lag_seconds": 6.43863, + "rss_bytes": 140812288 + }, + { + "seconds": 8.424, + "outstanding": 14, + "queued": 12, + "processing": 2, + "queue_lag_seconds": 6.719826, + "rss_bytes": 79773696 + }, + { + "seconds": 9.003, + "outstanding": 14, + "queued": 12, + "processing": 2, + "queue_lag_seconds": 7.268924, + "rss_bytes": 93224960 + }, + { + "seconds": 9.723, + "outstanding": 14, + "queued": 12, + "processing": 2, + "queue_lag_seconds": 7.961311, + "rss_bytes": 109162496 + }, + { + "seconds": 10.343, + "outstanding": 14, + "queued": 12, + "processing": 2, + "queue_lag_seconds": 8.675691, + "rss_bytes": 66420736 + }, + { + "seconds": 10.913, + "outstanding": 12, + "queued": 10, + "processing": 2, + "queue_lag_seconds": 8.881136, + "rss_bytes": 89284608 + }, + { + "seconds": 11.473, + "outstanding": 12, + "queued": 10, + "processing": 2, + "queue_lag_seconds": 9.410813, + "rss_bytes": 92332032 + }, + { + "seconds": 12.095, + "outstanding": 12, + "queued": 10, + "processing": 2, + "queue_lag_seconds": 10.044173, + "rss_bytes": 120737792 + }, + { + "seconds": 12.639, + "outstanding": 12, + "queued": 10, + "processing": 2, + "queue_lag_seconds": 10.643923, + "rss_bytes": 47562752 + }, + { + "seconds": 13.156, + "outstanding": 10, + "queued": 8, + "processing": 2, + "queue_lag_seconds": 10.865062, + "rss_bytes": 90583040 + }, + { + "seconds": 13.667, + "outstanding": 10, + "queued": 8, + "processing": 2, + "queue_lag_seconds": 11.419681, + "rss_bytes": 93134848 + }, + { + "seconds": 14.213, + "outstanding": 10, + "queued": 8, + "processing": 2, + "queue_lag_seconds": 11.932928, + "rss_bytes": 117977088 + }, + { + "seconds": 14.717, + "outstanding": 10, + "queued": 8, + "processing": 2, + "queue_lag_seconds": 12.476196, + "rss_bytes": 16855040 + }, + { + "seconds": 15.259, + "outstanding": 8, + "queued": 6, + "processing": 2, + "queue_lag_seconds": 12.747498, + "rss_bytes": 89051136 + }, + { + "seconds": 15.805, + "outstanding": 8, + "queued": 6, + "processing": 2, + "queue_lag_seconds": 13.312622, + "rss_bytes": 92135424 + }, + { + "seconds": 16.369, + "outstanding": 8, + "queued": 6, + "processing": 2, + "queue_lag_seconds": 13.827427, + "rss_bytes": 119685120 + }, + { + "seconds": 16.916, + "outstanding": 8, + "queued": 6, + "processing": 2, + "queue_lag_seconds": 14.41196, + "rss_bytes": 61480960 + }, + { + "seconds": 17.474, + "outstanding": 6, + "queued": 4, + "processing": 2, + "queue_lag_seconds": 14.838173, + "rss_bytes": 91725824 + }, + { + "seconds": 17.982, + "outstanding": 6, + "queued": 4, + "processing": 2, + "queue_lag_seconds": 15.313376, + "rss_bytes": 93421568 + }, + { + "seconds": 18.543, + "outstanding": 6, + "queued": 4, + "processing": 2, + "queue_lag_seconds": 15.910134, + "rss_bytes": 125739008 + }, + { + "seconds": 19.008, + "outstanding": 6, + "queued": 4, + "processing": 2, + "queue_lag_seconds": 16.389491, + "rss_bytes": 16818176 + }, + { + "seconds": 19.538, + "outstanding": 4, + "queued": 2, + "processing": 2, + "queue_lag_seconds": 16.706092, + "rss_bytes": 89124864 + }, + { + "seconds": 20.023, + "outstanding": 4, + "queued": 2, + "processing": 2, + "queue_lag_seconds": 17.209332, + "rss_bytes": 91959296 + }, + { + "seconds": 20.522, + "outstanding": 4, + "queued": 2, + "processing": 2, + "queue_lag_seconds": 17.719409, + "rss_bytes": 108560384 + }, + { + "seconds": 21.036, + "outstanding": 4, + "queued": 2, + "processing": 2, + "queue_lag_seconds": 18.202527, + "rss_bytes": 69586944 + }, + { + "seconds": 21.526, + "outstanding": 2, + "queued": 0, + "processing": 2, + "queue_lag_seconds": 0, + "rss_bytes": 86437888 + }, + { + "seconds": 22.05, + "outstanding": 2, + "queued": 0, + "processing": 2, + "queue_lag_seconds": 0, + "rss_bytes": 92631040 + }, + { + "seconds": 22.568, + "outstanding": 2, + "queued": 0, + "processing": 2, + "queue_lag_seconds": 0, + "rss_bytes": 112336896 + }, + { + "seconds": 23.064, + "outstanding": 2, + "queued": 0, + "processing": 2, + "queue_lag_seconds": 0, + "rss_bytes": 78319616 + }, + { + "seconds": 23.507, + "outstanding": 0, + "queued": 0, + "processing": 0, + "queue_lag_seconds": 0, + "rss_bytes": 16228352 + } + ], + "observations": [ + { + "attachment_id": "99688333-40bf-4d41-a4fe-323fad0904e9", + "ready_ms": 3560.499, + "upload_ms": 408.011 + }, + { + "attachment_id": "f147a922-d1fc-42cc-a66b-142d571f82df", + "ready_ms": 3554.025, + "upload_ms": 457.303 + }, + { + "attachment_id": "7658c2fe-3bb2-4ca4-8412-d0eb20a2b8ef", + "ready_ms": 6052.072, + "upload_ms": 679.643 + }, + { + "attachment_id": "89f86138-8fcd-4105-9892-c9dfe9dbecdc", + "ready_ms": 6066.401, + "upload_ms": 583.962 + }, + { + "attachment_id": "ceb8fe8c-a841-495e-bd34-5aad13dd6d75", + "ready_ms": 8403.356, + "upload_ms": 771.179 + }, + { + "attachment_id": "bf6a320d-9cfd-4029-a1d5-6ffd3c558aed", + "ready_ms": 8440.405, + "upload_ms": 887.69 + }, + { + "attachment_id": "3ac7646a-b9ca-4e27-8aa3-763ac97e994b", + "ready_ms": 10349.01, + "upload_ms": 981.57 + }, + { + "attachment_id": "00c411bc-e7e7-4ac4-90d9-2fa2d2c98a1c", + "ready_ms": 10339.33, + "upload_ms": 1076.033 + }, + { + "attachment_id": "71e1a5b8-497c-4c4e-958e-28e0ae2ca882", + "ready_ms": 12230.053, + "upload_ms": 833.319 + }, + { + "attachment_id": "821c31eb-38ce-4a1c-9955-e9fcc2859ff8", + "ready_ms": 12176.461, + "upload_ms": 1020.951 + }, + { + "attachment_id": "ae840ffa-f8c9-4fbc-b7bc-bd59ba5e9e72", + "ready_ms": 14133.905, + "upload_ms": 1012.104 + }, + { + "attachment_id": "5ffb85c0-0441-46c8-9f39-21b3138ce662", + "ready_ms": 14053.062, + "upload_ms": 1005.261 + }, + { + "attachment_id": "70dcb558-6257-4a9d-83bb-1494a55a9ef2", + "ready_ms": 16140.026, + "upload_ms": 1082.357 + }, + { + "attachment_id": "4e0240ab-c8c8-494d-8bcc-1c43cd447738", + "ready_ms": 15936.966, + "upload_ms": 985.338 + }, + { + "attachment_id": "8d0b3749-7ace-412c-9b19-e8a28eabb84f", + "ready_ms": 18123.235, + "upload_ms": 1204.862 + }, + { + "attachment_id": "d1236a0d-6869-4930-a6a2-e90ede460170", + "ready_ms": 17782.071, + "upload_ms": 926.795 + }, + { + "attachment_id": "acfc46ce-c102-4b10-b483-c820ce2dc416", + "ready_ms": 19926.281, + "upload_ms": 1147.664 + }, + { + "attachment_id": "7b22deab-2e71-4a84-92bb-135d3b5f2fbf", + "ready_ms": 19347.044, + "upload_ms": 652.552 + }, + { + "attachment_id": "c35b9a9b-1241-4a6d-895a-99773d78f318", + "ready_ms": 21471.061, + "upload_ms": 981.339 + }, + { + "attachment_id": "a83f78fc-4201-4da9-8519-1bdcf403aa21", + "ready_ms": 21587.644, + "upload_ms": 936.535 + } + ] + }, + { + "concurrency": 4, + "prefetch": 8, + "width": 1536, + "height": 1536, + "size_bytes": 7081647, + "input_sha256": "015b73f610b2145516211f8039594f43e64dfdaf0326b2874b294727f114de02", + "offered": 20, + "accepted": 20, + "rejected": 0, + "completed": 20, + "makespan_seconds": 23.602, + "throughput_rps": 0.847, + "ready_p50_ms": 13459.561, + "ready_p95_ms": 20744.497, + "after_accept_p95_ms": 19885.435, + "upload_p95_ms": 1134.032, + "queue_lag_peak_seconds": 16.602282, + "outstanding_peak": 20, + "rss_baseline_bytes": 16134144, + "rss_peak_bytes": 235130880, + "samples": [ + { + "seconds": 0.813, + "outstanding": 0, + "queued": 0, + "processing": 0, + "queue_lag_seconds": 0, + "rss_bytes": 45719552 + }, + { + "seconds": 1.493, + "outstanding": 6, + "queued": 2, + "processing": 4, + "queue_lag_seconds": 0.174778, + "rss_bytes": 143278080 + }, + { + "seconds": 2.202, + "outstanding": 13, + "queued": 9, + "processing": 4, + "queue_lag_seconds": 0.968807, + "rss_bytes": 155238400 + }, + { + "seconds": 2.933, + "outstanding": 19, + "queued": 15, + "processing": 4, + "queue_lag_seconds": 1.613169, + "rss_bytes": 165273600 + }, + { + "seconds": 3.638, + "outstanding": 20, + "queued": 16, + "processing": 4, + "queue_lag_seconds": 2.323971, + "rss_bytes": 168144896 + }, + { + "seconds": 4.287, + "outstanding": 20, + "queued": 16, + "processing": 4, + "queue_lag_seconds": 3.036557, + "rss_bytes": 182329344 + }, + { + "seconds": 4.921, + "outstanding": 20, + "queued": 16, + "processing": 4, + "queue_lag_seconds": 3.659427, + "rss_bytes": 209608704 + }, + { + "seconds": 5.538, + "outstanding": 20, + "queued": 16, + "processing": 4, + "queue_lag_seconds": 4.347039, + "rss_bytes": 208084992 + }, + { + "seconds": 6.12, + "outstanding": 17, + "queued": 14, + "processing": 3, + "queue_lag_seconds": 4.708041, + "rss_bytes": 146980864 + }, + { + "seconds": 6.721, + "outstanding": 16, + "queued": 12, + "processing": 4, + "queue_lag_seconds": 5.034121, + "rss_bytes": 155770880 + }, + { + "seconds": 7.302, + "outstanding": 16, + "queued": 12, + "processing": 4, + "queue_lag_seconds": 5.617518, + "rss_bytes": 165101568 + }, + { + "seconds": 7.836, + "outstanding": 16, + "queued": 12, + "processing": 4, + "queue_lag_seconds": 6.172094, + "rss_bytes": 169320448 + }, + { + "seconds": 8.419, + "outstanding": 16, + "queued": 12, + "processing": 4, + "queue_lag_seconds": 6.723611, + "rss_bytes": 184619008 + }, + { + "seconds": 9.004, + "outstanding": 16, + "queued": 12, + "processing": 4, + "queue_lag_seconds": 7.310578, + "rss_bytes": 225910784 + }, + { + "seconds": 9.585, + "outstanding": 16, + "queued": 12, + "processing": 4, + "queue_lag_seconds": 7.892007, + "rss_bytes": 0 + }, + { + "seconds": 10.185, + "outstanding": 14, + "queued": 10, + "processing": 4, + "queue_lag_seconds": 8.34158, + "rss_bytes": 144318464 + }, + { + "seconds": 10.798, + "outstanding": 12, + "queued": 8, + "processing": 4, + "queue_lag_seconds": 8.711244, + "rss_bytes": 154238976 + }, + { + "seconds": 11.505, + "outstanding": 12, + "queued": 8, + "processing": 4, + "queue_lag_seconds": 9.414926, + "rss_bytes": 165646336 + }, + { + "seconds": 12.073, + "outstanding": 12, + "queued": 8, + "processing": 4, + "queue_lag_seconds": 9.9939, + "rss_bytes": 178364416 + }, + { + "seconds": 12.684, + "outstanding": 12, + "queued": 8, + "processing": 4, + "queue_lag_seconds": 10.584393, + "rss_bytes": 196341760 + }, + { + "seconds": 13.288, + "outstanding": 12, + "queued": 8, + "processing": 4, + "queue_lag_seconds": 11.205643, + "rss_bytes": 200409088 + }, + { + "seconds": 13.831, + "outstanding": 11, + "queued": 7, + "processing": 4, + "queue_lag_seconds": 11.641966, + "rss_bytes": 150261760 + }, + { + "seconds": 14.407, + "outstanding": 10, + "queued": 6, + "processing": 4, + "queue_lag_seconds": 12.033763, + "rss_bytes": 193028096 + }, + { + "seconds": 15.067, + "outstanding": 9, + "queued": 5, + "processing": 4, + "queue_lag_seconds": 12.521832, + "rss_bytes": 176140288 + }, + { + "seconds": 15.668, + "outstanding": 8, + "queued": 4, + "processing": 4, + "queue_lag_seconds": 13.11874, + "rss_bytes": 158498816 + }, + { + "seconds": 16.187, + "outstanding": 8, + "queued": 4, + "processing": 4, + "queue_lag_seconds": 13.66311, + "rss_bytes": 170463232 + }, + { + "seconds": 16.776, + "outstanding": 8, + "queued": 4, + "processing": 4, + "queue_lag_seconds": 14.235394, + "rss_bytes": 197406720 + }, + { + "seconds": 17.333, + "outstanding": 8, + "queued": 4, + "processing": 4, + "queue_lag_seconds": 14.829522, + "rss_bytes": 196517888 + }, + { + "seconds": 17.899, + "outstanding": 7, + "queued": 3, + "processing": 4, + "queue_lag_seconds": 15.288305, + "rss_bytes": 209166336 + }, + { + "seconds": 18.463, + "outstanding": 7, + "queued": 3, + "processing": 4, + "queue_lag_seconds": 15.737111, + "rss_bytes": 130351104 + }, + { + "seconds": 18.99, + "outstanding": 5, + "queued": 1, + "processing": 4, + "queue_lag_seconds": 16.061496, + "rss_bytes": 180559872 + }, + { + "seconds": 19.488, + "outstanding": 5, + "queued": 1, + "processing": 4, + "queue_lag_seconds": 16.602282, + "rss_bytes": 184385536 + }, + { + "seconds": 20.008, + "outstanding": 4, + "queued": 0, + "processing": 4, + "queue_lag_seconds": 0, + "rss_bytes": 159518720 + }, + { + "seconds": 20.47, + "outstanding": 4, + "queued": 0, + "processing": 4, + "queue_lag_seconds": 0, + "rss_bytes": 175730688 + }, + { + "seconds": 20.953, + "outstanding": 4, + "queued": 0, + "processing": 4, + "queue_lag_seconds": 0, + "rss_bytes": 184320000 + }, + { + "seconds": 21.55, + "outstanding": 4, + "queued": 0, + "processing": 4, + "queue_lag_seconds": 0, + "rss_bytes": 235130880 + }, + { + "seconds": 22.051, + "outstanding": 4, + "queued": 0, + "processing": 4, + "queue_lag_seconds": 0, + "rss_bytes": 111206400 + }, + { + "seconds": 22.708, + "outstanding": 1, + "queued": 0, + "processing": 1, + "queue_lag_seconds": 0, + "rss_bytes": 59723776 + }, + { + "seconds": 23.178, + "outstanding": 1, + "queued": 0, + "processing": 1, + "queue_lag_seconds": 0, + "rss_bytes": 17543168 + }, + { + "seconds": 23.601, + "outstanding": 0, + "queued": 0, + "processing": 0, + "queue_lag_seconds": 0, + "rss_bytes": 17477632 + } + ], + "observations": [ + { + "attachment_id": "09e001f8-6730-41fd-bd5c-0105e0a519a1", + "ready_ms": 5635.479, + "upload_ms": 379.774 + }, + { + "attachment_id": "fa2633da-3cfe-4d74-b345-d01ecee4c603", + "ready_ms": 6197.281, + "upload_ms": 811.621 + }, + { + "attachment_id": "06ab46de-9fe5-4294-8667-3697727ec660", + "ready_ms": 6192.099, + "upload_ms": 643.883 + }, + { + "attachment_id": "8b125e8a-db3f-48fc-a2a6-70fc90f0da7d", + "ready_ms": 6211.204, + "upload_ms": 465.159 + }, + { + "attachment_id": "7b8a17f9-71cc-452d-b05b-b04a109e0f86", + "ready_ms": 9635.43, + "upload_ms": 1295.364 + }, + { + "attachment_id": "3a216daa-9184-4143-aa24-c2fb26fe65b7", + "ready_ms": 9654.948, + "upload_ms": 898.299 + }, + { + "attachment_id": "3428eb34-cd42-4c1c-9bab-5733561fd44a", + "ready_ms": 10266.06, + "upload_ms": 1051.242 + }, + { + "attachment_id": "ff49cac4-62c3-4005-b114-71f116d51d29", + "ready_ms": 10847.423, + "upload_ms": 1134.032 + }, + { + "attachment_id": "9fae56a4-ba64-4947-809e-34c10051205c", + "ready_ms": 12577.222, + "upload_ms": 751.709 + }, + { + "attachment_id": "c627cf28-b4e7-42e9-bdf9-83d6ef9f780b", + "ready_ms": 13459.561, + "upload_ms": 949.39 + }, + { + "attachment_id": "cd56a914-37d4-4b51-bb19-60f6a3378ced", + "ready_ms": 14769.857, + "upload_ms": 1113.794 + }, + { + "attachment_id": "9ff00757-950e-4fad-83ab-f1fecbc97cfa", + "ready_ms": 15101.025, + "upload_ms": 1013.618 + }, + { + "attachment_id": "928fadf0-75c4-45ce-9718-91d7101cf7c5", + "ready_ms": 16911.15, + "upload_ms": 739.552 + }, + { + "attachment_id": "da2cbbd5-3afa-42d0-9a37-2b7fbe6629dc", + "ready_ms": 17620.17, + "upload_ms": 1059.124 + }, + { + "attachment_id": "973b6abd-880b-49d3-accd-101d9cc87c3e", + "ready_ms": 17220.489, + "upload_ms": 772.069 + }, + { + "attachment_id": "9ff48419-a077-4b62-a0dc-f779ccd7f87b", + "ready_ms": 18925.466, + "upload_ms": 1036.736 + }, + { + "attachment_id": "45f54fd6-1c3c-44e8-9a69-f34f28e31be5", + "ready_ms": 20580.767, + "upload_ms": 858.08 + }, + { + "attachment_id": "36296a7a-7fda-4b12-b026-c6fc433d9753", + "ready_ms": 20744.497, + "upload_ms": 933.986 + }, + { + "attachment_id": "ceefd3ec-372a-4264-a35a-7944b2717699", + "ready_ms": 20668.569, + "upload_ms": 783.134 + }, + { + "attachment_id": "0389dd0c-e1f0-4c01-a5ad-08c3e93af110", + "ready_ms": 21584.932, + "upload_ms": 901.456 + } + ] + } + ], + "overload": { + "limit": 8, + "offered": 16, + "accepted": 8, + "rejected_429": 8, + "observed_api_instances": 2, + "outstanding_worker_stopped": 8, + "ready_after_resume": 8, + "observed_drain_after_worker_healthy_seconds": 0.062, + "outstanding_after_recovery": 0, + "followup_upload": 202, + "text_messages_during_overload": "PASS" + }, + "result": "PASS", + "restored_initial_settings": true, + "finished_at": "2026-11-10T19:08:13.363296+00:00" +} diff --git a/evidence/lab5-research.png b/evidence/lab5-research.png new file mode 100644 index 0000000000000000000000000000000000000000..aa5eaaabceafc43dc98616429ec0830a4428fe10 GIT binary patch literal 163470 zcmbTdbyU-D_&S#L8b&Ax2p9~cyGusr1Qc{MjE;dg zI;3;kbIhQ=Nb$YbKc|~^&}LJQ9;-CCj2%{ z%Jr`|zTONmp}($i?eb0EO^Hh{_erNSk5=zFD0_Z(K;im!);{-atgY>)Chnx|j9flP zdFCv5<^TEkaAVNUm-7F9M?nGQpnUVc-$_&7mIjORe?D@q{r~?JR_lpX_gCM&mHJLo zzjX$;FMU#^XCmFd9;O?w4_$=5?M)TGP-`#UDeUam|G2znd*S({7cBOuz~}P=RkumY z(J~W02n15s>$|_k#45kq=V!87ErzY$32qRUl19~UG^~c4j6Lalrakl-+*^^NyiW9g z6hW5REN*80TXH`S1_uXAT<1G##;uYKHfNgG`(mGw)%k+VNUt zpF^@^O+|LBR)Yy(n>Bl)zmmmPo(*-6n>c3 zDz++rTWKhMmO^7%x6}|O6MmKE{Ds;nH)ZjSy7{$|%3ilvix|<4Pi*LX_=VLQiYoh( z6clharZ*HpaaHp%;uC7(>(B?%b(X(wu!wt>^rVW9Ht#Q{BPN2*oL!rAMprvcHw@2W zNBEr4gc>%VKObR&ywFHh{Air0zJ|su>e55C6q_hU0ng0~lxMtl7{C(m_KI(gtqd1W z@0`6V@@*#Iz(iSHy?pr>c{zfv-nSy~7VORU_(WQqe*B*=L(*?jDOdgTXMfvrSNgucg|lRpeid% z15WmJL}1_~&4kggg!By!)^(Fh%7@cBw=)#C&QWD22|F^MB8aC)no`*%?JUks1D?15 zFZ-dq=*xE1HUn**Gd|Xh`xEviXx|EMUf!w;9RB^t87SJHhFR!qY3_IqNB$c~bj-ElDzVB@E*$L?r zhp*6&)+7)``rcTYv5=hHp3|zvxm-Pfk zB3gH{`<1{qXQq>cY+olPCXS$B%Bz#L9-iRbRSjeRuvGZ8UDKeWz3RSp6DE zIOe}4?lLQ#z+g0Uytisnuh;V#WhQhgJn3MOU>4}pO_1XLAZhvYnx2^MPL`wMWZB4` zlQN!w1LfqNnzPmP!yMxNpo_`!4&TNbPes|a5`@8F~+yD?B^xF=9dIyR)~CaEB4o-&ifi`U`oTD9h9JNqva~X0s(fH+ z)dACfo6R*{{Ec<+EJx=kzM8L~;0y&4_C^cl-@^i{uQr)C-Kf^iQ>id-539FGi-Au_ z3DF!(xyMd&aQ{BQA!>Qb>e}|4i+z4#Feq?->*4O5Y+mER%Jv^Mal2-hEv+s`)KIeGv0% zY1nKzIUg%<+_N^_SZm@r4l9$|#GLYbuWEH_uqjvbSF4h%qOf_}xngSN1b#Sr%WthB z{KRp%X{pg`8m`wk7sm2htagWku4!;|#yO?;dbL9nPQ>vnc$C81yvH9$nUNoV`efGpP8qm<8w8%Lz;=zl3`nfl|dFO6(o_hpj#yh*aKACiYed4~* z{o&+gj=L%a#R|ERalP)7IDkUiB+8oiCkonY9VdlFPQeCabsYmweP06|NSOVb$qvrS zm1l(AAx-o_IhsiAu{k5L<}D~FNJ;zbuk@?q-R0I%A?Hm0{c-D7b2YK$oDc0K#`S_< z!9vKVc#P5t-|$H7gAeFX>REnAB- zuEhy`$$A_GyDqhfaA@8e3C0DU@Z}w~tP~Yf+S=M)&$kEVFxdh3@!8Rz&%t@B$?_>3 z-QC>;;b^p&>XjzWuV5I)XIOQyi9|fP-igOq$rN@!Ml28J1{3xt1%6kV_;39P)^FPB z6%Y1YpO9lsh!M3;5Ae(=DB%1+%?)<3EqNE`TX^QItDwtw7W-($OF#Br_i9R&B%<`4=D%&1H4&HLX(FH=yqn(=K=h2NOA+X|PL;Z> zu?3osRW0t6KYgvKvy=SA>jfENTE{j8MFxL1 z?F$Nuwm52S<=3%-4Q-koamOP&oM#pI?1v+D_#8W9*j73L7Ukco z%;!y89?Ewfq)pFzIX~#PnC9p7_DO7;=6(`-*Sgke+BgqtYMRz{USkB8^YXL9uTS?~ z5ebIp)ljmS+bKk!K?{81Xd!~6)?fn>20d>B2cVW$iKfGLBE9@Oh z;^#5m8lLHb&P8@6_NS4T3ScPI^SGC@)&3(W;#@d~vAvaQf?-s#rAfh+Pz`q9rOa0y zLGj@zx@rPvs4R|$(bPM$Yj+ZKXJgOwkxIdWIG;@THmDbkb9;n|h4VDbe~#Z@!JqE* z)qcFI5dZ6o#`&BDg|1g~ZEyLzTGO8!reaaXi5FKas?5-VWz`KMA8T0n?0d2o`ntYr z#Nz>!Fo{PRF7vd@BsyRM=}TGmbjWZi>{PN^%dE=gqhFch%Pxa1*+dRs_0nnNbRj&e z`w2{u^ND^U^_)$>jK`2_E9t2!I77AK!nVnv^Y=}8umv8@Ir&Att>J<7_zLmElx(Cl zbl;(j<64WBo3=4Lj-c3j?({&6vN)A@zMZA@n5;)#6OzKb)9BdGPY+y)kuo$j9KoFb z@0qvDix$1aa}z|lRk_3oOk2O$UqS7-OPLlGBieOD>uO^nVG}VK&Jp;ZziJ{q*k-P) zCg|H{0ei^Fn^mc{W3{-zNk_Bc=jiL08`!{-qoJCqTxG^u_UdNu#0Q`{COD!A`<`u= z9=>5qh#55`PUy+*=1yWrdWRcInvvXPGh7fmzXE}xEhcKK zX;<`7Zg`~wzrfYf+6LSiwiCM$)Q}Unh*@9g4@%YY_(myZuCyJLY$}Uf_WM&AS_z>q z$lNt~J})*44~J2=QQr~Kpg`*?e$e|{3g(BM%Tagtl8bRF1W7xy!8#!|Sd+cUN|VfD zhp&{Wy2M_bIC`guW$8fC;G5Wm{w<>}%}!O-VjLcH>Ud zXl-<*p`;i|!}DveYTL^{N7k@=cBVq56KX>56&BHA*4P$N*Wce~{lpfr2TM~tSom0R zd5fWy1ZTD(deP81%TcXUF10hR+&I&nb90RFx1F{Tjd=|nrlFwFQd3pnJVT+_ai#6C zMMs>iXpwGZ=Cjkk?XBj9)pmHp-uS*9BC>9l`%md`O*63YA(H>Fo#dj(^BPq#X|c=f7f z16K<3DS32D3U)=hYM;fvm6})V(kdMQ-DE=*#k{?)?^WWIiK-o(;K7xjItH^W=1_!lKJk zm)j_#NiOPUz3tEm#>oeuns&AMKfgYk)C>b;eZ8RU$_oKyhj2E}v3uIi!$mqUlu#9k z>$3V}JDb;3#jpLp9W2i0o_NQdx#1)k&uef)1T@L%?aK*V+2(2zd)Qeop0iI_*Yjrt z_xJJJiB0Z{y)>t*W+ZB>X}PWfzk_M7Py=4W&It&TMsLQ*xfvfLH}ebuc5blu;ep=f zf1W>Yhu7ifYQ499eje(hFA!(b#JQ1jGqVN#jLj=&s>`#5FSIftO9TcXdOMlr5 zhe*M?2}qH|I#S$wMPa>1GyXe0ALe(|i@m0ppLzd%uo%{rARw@l+zhtScL{-_jM+ZS z(B{WL1O~CGsldj-D!+JUzB4v93BK6Vh14@$j#oIsIix0hcf0ywX{S$L=SNeD|Mo8! zij6SL@@%vD!1R`YfPlypIQ}=?i~)rUaQPC&-%!vk+L(@t0#vK_>v;4)px+sag4KUy zOuppI3zNeQ&~#eO86I7fgYY-aMmYVJZM?=I>SpPZA`^CH{7(RKVh5)f;Ss8t+i}{i zD=b}b3wh9jVbpy(&3jBW(A z(3^SmrNrc97EfmQT}}K|!VY4>HeZg{mcdp8iQJiyA|B449nQRS=EY{1;Soz9kwpRD zwQrl&J|OrGR;SPA(f_=aWeOi^nMN)+hpF6M4E!ImgRGCX%`#)|QX6`1Oo`TQ2J8*% zUC-YFz~jn!x(iqSyY?;|ff$zQG-P9ZG0-E4NEDrZq$V4rL5sY9SphOl$TgPXA+Wjqjj)@ zKjANR&R>LAN@>|zAxqzTJ}WPxFbCTNUkhg&YI`a?c)GgKjN&G^J$V8c5%|7QWa|(?a7}xn?tG0tcF&mR{Jsk#UkcX;AUCl#)?v2 zc-H(9;BsuNo+QEnt*G(Qjq;BpnHYUPbgA7rx|K%z?b9a%J6W$v5dL-r^f8q4M6D*! z@EsEmh7~b>>gd)jR^=?hJ|%_5XzsAb`MVSpzohYz&>44tWXgEsl~yB^4#-gVwp6y3 z6gT7^cOzYCNxBlEFek3*bi1on%b0jWfgR~MVPD?d7PAWWv1{(y?_4dZia1TxxRMfH zKX6?p^~Kw{f1%7Ous{HZ??IfoY4cEFrc6ymS=+&;%QY@Hn#*`zz!IRQDot+Y11YW@ zOy>EIA&}9lVoMpbByMfSLWe((J6Y-c%iz$VNe!5dIJF)|H@6GwiOs%|z#U*Tj@Q&Q z73D!LXw^?{Ltkt(AIw}PB0xb&;F4R+-`1~)>-7U2%K_k=jibW+v=t?F`K%je+(!?% z&6bers&9bKFht%4HH8Nf0?oUhcr0!Tu$e(?gjkV|Lz1j3V{1OM!52dAzta)c$Q`*; z|L2pk@zE2XxwniZ4O0L;lsOpHf&Gxytc48ZKFAX+M1fUCGD*^dO>u=030M9uN$H+T zsZ%790yF^8-@YZ>zp4TXu2d=|hh1@89)A_HJz6@v@e)PKTd7+Vfctdu z)z-O1&rP}aNt9XU7pyGwYLfU#_(YsbKGcss@|~o&Rsl1!S!o3y~l5<&5RLEI_g7&5I%5e6eQ2hIbq1J(z-a{ z6zPNA3|o3l0ul}XPTFzAHmlc^>-Em;WgU*ETUs0s(f~N^1%0Zti0jtQdM&OYnSBNE zkabTvk43$K={K4QX`{gEDm%uqq*Glcv$vj=3{3(e!U39}qJ$#sK!fx$Yb$P(@$u9% zxh%;`5-aG3T2dcY)D@r0U4lR9M$;5JLlC&sUs-p$=4V4`ulj{*oOXulJ3Zn#ENuy9 z_xW?Z@pz@!1R+XG(}{u2n;rEF~%IvZ98>5aP!-T==T$oFS3by zzMnwEwT!Nw)qt?kUDxQZcrqcmxf%Ww;lPi+0)HYtv5%*kZz8sM%tHe_z%`pR%0KIf zBn7a!(>0gcoXDM?LNGGpzN@97CjkX}`Z}LrCmI~NVu+|3Devg#^!kxE3yLE4^tBgZ z4CHE5cOIVgEhEtVj^|F=3Cgslq`f zoYn1}_`cr3%Kn`{a(L- zsysvrgBhse(hvm73*$Y_TRBCBe^wTn1QT`-fldU#vZVX-3$VLOFL1cu1`rV;)rOaz zBL>#@{SAE&HrU`_WXtTOO5I__T-Qtt@tlHGk@UQ~f*)UB@_zf9A=4D6y7?>#}Z_u;Hz|BCc%Np@ahb9EUJ=`|na-HtN;}niObM=VGoh#LQi{ zBE=uq*F+=toN~SKBkEn#x`F)r|1=+j-WgOiC+(+PtYBTN>RZ-d49I78k-f|6U2=Ne%hdDJ$WS_h$aF=;7|2qO1IlCIS+2c_Pdc}^w@bL?Fd{ob_9MK% z_t$W`WT7(ok@*X(tKH0v=F*%7n^mpJq8QcH(ehYtW2Gv~aY?LqoBhul>tE(!cB}d~ zls{xO^g--ans{|h?}|AGw!wl<)@vr6_>)y1s3t-2ou!!WC}ygT7dyYHvV{ab)?~`E zhtdTQ4K%lUqXG|37oSw{I@8c?cd`*hxHQ`1GA*kcHLP(a^r`(wUHDR)PVQqKO#1VzDARI=O8zhc-_Q0f zl-=)4EmaZ_>Wq0zsPIJMiF4!cNS-Mf#$#@Y0zoMSb{|WXAt;mGaCx{>kPn0b``R!! z?G(Zx(De7kXo!JkHPeigLmfQ-RgIE$;d>J~Z`gNC z!hfYplH~zUwSruL>w=4uYA#< zn{1IX62|VYrH_u9A(|%^v}rs}J4q{F@$p6zK>4%&Rd8oj5bN4dgvIWEZ0bal&Y#Foo_5cHm@RF0DY?)R5E1e0DAY=H@U}FKEOSgmT9dg zouEnXjPs>NwKYXKpL7&B0;*10K*5+I6RuYrjzE?3RtYH4k*#ujA7MJ0B{@}<>H|@Z zBV#^ZSmfI0^wt}y|Q6z}EtyhmuBs;e(k02TU zYDr1>+L(X?Ml7S{n0{Bz&yb3i<)!7jARtn(S3qo~pPB)Js`0Xrqr0h9F7s{9AZDj& zU&sZEaa{`u;qQ&A54kC>+h0bIP7f0l;Qr-a7K5Z_9aohC>WHy;dpjddT{tXcbW6ja z#ggXQgVmmOs8l=@TGIK~Qe0qXN1(Pye`C15^x;XcKMMbq?rBM}SjnH^(ipldQgAxrWUQ#Dy6)-@zfULq{RNf*G z(!1rb_>e?Hy_IwYh;tJQv|*rd&BRe*0FyDIJAr7{B~nw`r8N_SxGS9qlj)cX9d{-w zp3m<(Ju5oZSGCN1C#c3FCIy7WlC90XaypolA6|a-4ykDc@_bxb8s*6G6wVuw+p1{@P%ssK6CSX$?gT7_m{G{C$RXMpJ3c(Q9I(y4iG8 zxHA;e1{F^yfv51cU@e+oNFww2nsx+BIgWu!DhtUJDL=nMDQilC4ty6>kt?i18stUF-%5?S3>GqH5CiKd1O9AALP3$mFa`GAl)H(_3{cFI&y~ep!ZV3` zIvC^fb*nI@yBCB6K`vCN_9_D)Z@ZsTBe@VSIV$9q@853C%6L2cH^#z${Rt!%)$Y&QOQr%y_OwF5`8 zyHm4>iQM~dfujsKE3Sz0(NTwcEA0c9PnT=I;Wjma!XIjoKz&awQSiq~k>$9qh6MMq zUlQ+R_!!oV6O}8`(h>i9xj9(!?tL^}GM$?$Q3FGoA01 z@<^X8bqiGYzonw@i6V(TjK`PCTg?TAXAkVPkrg85KZR9|yk;%W7WS@CNOP0zUFnP5 zc=zSpg!81)=r6g-3GqzwnLe`eti79En#I07x;DLbQ1NkacE4SV%jG5CS59rlkjWFAC$%{ZgJf9wpp_G^n zI=eC@$?Rw}Ao$YK*8X04Br)wl*M4T$bs%*6emBokPv5{?^=+kwu^{s$gCqdEwD`VN zIH0U5>=EHFhsYxDLCUD(pSSG8$ zNFUPsC%qT2Lp3eR7pKut)Um3 z1-suen*Mj?JkQUfPgWI08}c`|`lE+|=kr5C??GkE+lqK-sp?JyhLHAPXy3Kx>1bV$ z4rfvx5Fl@jJ$eP?i5lSSD|EjjOD^q!2{F}x>(uMKl}%018Ls~r>@4N#ya#J;jR@igM%@eo=-`2~Ib*0}TKA%UUF=T=Z@M zzbX9gJ!7jlrw>hlMr`DTD1C{~cY6bxvBzJlcqha6>=M48=RDk{hw#}osmG6HZQ;ThU^;FIypW}^Zxyy-4obI zhsLeni&(^ z=)}ScWK`*8sGG|DS=r~O+G?V(&yHGMFHag!+^-F48@AS|vOQ!}A>$b96&ofFx1YB+ z#IyN~_Ri+aC+dZ4#2YzPd{z_Va^MJWFwaZ&QDB3|0iG5Z&nXthb+K$K9?exoBJHG; z&OH&M{|{==EE`ZBLN#I4bpi}2W2+d(vEeNzKq_#SBGXIzfgD05$J5xycZ|;?Ysi{U z-K)rBwo=uqQt#ZfbiWNzuT$!TUq$oPPHT27I(gUa5-_J}58C%L0}Po$j(d1<2`PcG z>f-BFukw;({2)^@spX=;GPX@}w$jWFr2k%@VE_EBNVCX3pzSTP%H*99oxkcU` zIgT~s9jSCjEQB|h)8XniU~8~DIAmyL0#U#{(fonRmsw^@TZuP3d&RKKfH5AvsRGTr zlOP@-%+$+sq1>G<)ekT%JEc{IGZaA|JZ_gIFE(7$J0FKqJIz(-6I{FV@$zdIQg-t@Tpk`Ur^<_03@=O)lIT_D}?Y&i%gIWxlOuPJ5O;4<731t8fZ zP4Q;KQdZROBq`M$z_K^k8|E@ulz>87jmP8bS+pDG@-EwN-Z8m<2TAqcrq_xUl6`r3 zxn|mP(g1{Erd6*|wbSSHd)fK;dF;qT*a?KQK5_U$UDEOUkb4bhGH=O8-Q){7b8T<# zHO+vT^rhJxzbpP#$WDdT&>z&}ew@uV4{AZ>2-FOZu zgs)+Ri&@GW%nK+&k00+G>=ejf`l!^fT-hK>1cG(le5IhRQ^WFu+A?Hjp+(X0Gp(<( z0TI5>J7wcuma0tQGJUVH zWV64eB*4eMi|lg7eTX@X|5`!5q-215b<((ox6(~YhF0IZdMkoH&{2bQYc zFZnLH=T1~ZRVu?S3i!1>zZ_Vd&A^>eXB7H*a<6&Oj!4>)eRNWe9q)1K@MdZJ@<#>s zW#%axBkWOR%ysVkjR7KS(c{?B#Y$$4<;=j~;rtxRQA7EOg=&-I(6!^9t2%F`&nT7~ z{vwA&l#*BchQDC4-Jb%VyyXA-*tM@<+{jf-_lDFGwKS$-jdo-v^|l#q?kh7kW$ZC;Qj{LS9x$my|e6THuQV8Yr(wH;5Ee{W>9=x++5@ zognr>_XUU%x7fk?4>by_2+U^}U;8TaYN`$>OX8kuW747LY1(44Pn%W|n-YB4k&4)c zG%>hc21q$wmqLK}Pvd$Idfl!$e6UbEfA-M;S;SUhf1w+gX14DxotCw7?6$H3KJ4z|Eco#L8!Evm1kEq$5+UfkEV>3*w@dkDVAm3nP(ybjl~mvwxZh_L;xrO zDKxuC&wB&_S@POCIIe_pCxZmNdjcR?DQOA2iQDhUg-J%dlO} zRaL$sBvl>Q1$C&zm4kb%u%SY#Qb9dvnD+QXJDRC76Y){8lkO*d9#4q)Y|Sipei`nEiPq^ zxaYaLbKLDYf>yuO67{*Ma$%W{iluGHGsSy_CA`%OBikCwWE(11MfaoOp*^7(t-;3# zyKQfz@sgKM02apmS;pWF!!LhrWiOcEs=FAwEkd&+17#4GSfoM;EA`n8yoXuR@DX9N zk2}f^&|;zkC&+7D2$Q36^MjyU)m6BP;yqYr0rZQ~!(!eflpXw3b0JL79)4OHsfAoo zRx)@#gMxCHOqe36vL%_}DbcxT8Mfp&$DH??Ft#QAqChsJXg2R%J_Oqkay;WmbG&w< zfwllG+LMfu9QIlPU4FIZ`TAzTs zCi0mvZ%YlU-Ef6F!sY1iHU_g8W~^u*o%*vmRBBvs!Pe50a-tO`rIS#91EVLu5H!&_6x#kg=HD8GuhGLAyy(a>jEb0OuR-4`+u;^cpXc{Sa znZ%#!%xSMae6 zD$VxQy#2i1_OLtfbw`(W|NSoLj~qEeNTKM)-QxrB?HJ(1FIvippRS6`QY1X*=t_?So&Ka@yPn%qTKV=P7cJ>RjJdzcIm?Z zu1v%4+O2cot4~u*)+y#d;;y2VI_(E2>7fxwN${CAjYURAmH;|Y17ykxa-a-Yd#;IZ z|2>%EBldCuli>e+4gL>``=C2S_rKp#WE}s`lmE|8v@V_j72yAT+{m;v4S=v3 z1hVrUf~SW@b7|*|fg5!o?@?e}5tb;>CrJ!4Qg7Q}9>I9NbI)dsH6j&JLc+*~T%@4xF0(4r)xiriYIY zHl6G1>oq%%=3}KATw~b8M?g?Zz^L9WH+iNbn#Bl&d!{!@05L65awK_@*X&mm#chzn z?S7c<{~GuvWkA_-zK#1AWg#Cza_nT11~T=EOu46NT5KYtqSm*&1Zo>M8oDThr4tW4 zK5GR3yuj}L4TyxE;(!s+34`0pp(eDH8CGK?$ZD@2NFlBR=woXmdlVW9I^9UEv)SrkZq!qUv3@|joygf1273Cs_b;ZUQ=VAI zNPv){W@i8}(i$i-@o$X`Y7ePx2uRX6ezOnnQd0dQ$04O0{V6=lOOF0syc&o23? zef;H%zGq}%H&?Dl3eqyHoq(4KDd)0q$xrok8$Roimg(>V;Dv^a?9m{_9rFF$^0^KU zPDadNw0QMCsSiP%rwk3G{!>U#RYUt-%)&g zrB%unRObbNSvdQM`o_wG1t2LgdNBzfFQ1{n>Hv08dBS!T4}ALyz}S9_lo~BE&!_k- zB=Q^KPS*L<@v?;0)^=K{DPU)su3`&^9<<+NV5rxc0|uqmqg4TVopXOfLdVC)J%R8C z{PNU=v~Y|Nh!8;5m-Ev-hpzm){I@Md;B)neu8|`QWJb!OSs=YsBbW;pC4 zA3pPf>Vh2MLD88{d73zotoh$176swkH9&hK#f+`2^1%6l!<0T3;Srobil_#tp6jq4 zyrATow=+1bvpay@bxj+CqK3@A(grfa7f2)^QOaD+4-+L*fWHXXcIP1aE902^15bIp823ytblvn>JvL6fK<4T?-iN9G70umsqIjn(|DCLZU*@! zV55FKw$PeLr4e%Kkw5BtW4*CKs<_wH-fStG0N^`}-86ET z8%d~n*+T*qihXDIBy_&vV++{Pb@&noeb_iR%RXQZt^~OJ4LdfQG*D^bdNb+-gdNi= z0l?#_-?{j*D``M1n3=|odgkpFtgiuAum(WBduxJvFIxw)6&fWmLZwOSFiM|QRhlcn zq?-$i8Yb(eAd#6eDLjl$tT_wgFOJn>6ToK7f_na5j9s;onIXmdcZ8D^^y?iE1SmzU z>k;QQR5k8K03&=9n2YQ2wF}xq`2p4dFuP2CI1Z1hBG`%+T>k+i$F-Wvd|rHr(r^$1 zm}E$+Xo?(S@FPi^ZKb^&mT z1P4r(1PDFr0vOtO zFE#^G!&Ko?3tudLCv^b%P)wu@C@aE$9{EnQ$hRW^I|&BuFD?;6vZ$T*yj)_^I1=Cq zDtRzjSz7xF^oU@*OmMH&{fkmd z9=`gYDD^(bKD?VZ{`%_qD`0y?iKPPR#blNoycSRUzX@mfvl$Ca99OWn#|oB9gufm{ zQ||I_wgjA`Kzq|xfugUV33;-I)XcM zf%V#uAQ}t=K#rR7?13>6wY)<kJ{_#E!Qbt3IED$GFZ%~%+s%amAh~YN5Q<-;?^G=5V zErg^aKtDc94wZ|aKEJ^BXMe{9!{Lqw_aF@@^_|`!u+BGK(}Xx?{wARno$B%N(uCu^ zyZj10>3Mw2Y^x1ko9tWqC%}D#F6pyeCT+%~DP-gLoDadwN*JsBNf=^63Ut}Jd~Nj6 z%wbhb@gx8jH{d3Iv0!9qZ2fWtzZY99n2H&C*ZBibq;uqHm4cx{+`Fr6VzN{0pkwPC zq%s(6Rw(}QN-&cgfLb1cClKtfvK@N2`sm4%CobUL6j)XGHB^u$?tW%BOEU?G;NuPORR^%8oD6IlWE|-L5^XW^WUt2PUH0s$ zMCrvvXR2@^7G7o*R5$8%=#IZkx-d}XJDHSLY{pl*sBwSC63k~gG4W`B%T`fz#M#M} zuVo9g9JbNq!}DSJ_<3B7%z)=3oBmt>>?q;f_#;f1wsL@~9?Z6RZxe@8jZ&Db9od%k zvhiojom8!XIS*HB$0GGwfKZ+sqES)9h+>p(bqTVxB%oumwoQ}6Im>|_7G%@d@2gRh z7C#J{racI#x+F?t{y>GZY6kQu4L z4@r!$IXknzf)YrEA`2v{C`&UeBIeSB9*9xD!3r0SR<~5m2&r?c7QT0bAvla_ zWn0A6Fb3qeD4aCMSWi5JCTVx(45^9p{i&U(Y;q(!+rSV z4)Q5|q#&gFY8Ah103-J*S6Yb_B2j~F5(x_g@hBtsfM>~kgje^FtMW+(%B#ACJifpi zTt-ZwmDts&ojy5HysB^#A~;=*;FKu=8FY}~=rn*$*t7JbpbzSLL#zB{G#BFID|^Ld zyNphdF%e?v{ec;K`M9uTrv|RfuKWk+#ycL*whoj_9re*>h6#p5u{?Xzs_ycpZ6(P1 z$KwvQsv@z6if<-dVlGio*i|IOXi$P-wE4bWGVE@%;yW^8Q0U^m0ps&-FUBL_R@bax zu=@BBu;_kZ{-L1Tg*<*|pISB-NAm|Jzq>Z>{`pAD6Zzq)VD8>ErgjCKsL^8=TmAbTQ$5Ebq_$W(ODPJ z3J|lP*geR|Az?R1TG~ZW_KL^0-?l%xzhKxf_+f%^M zW9z0?o((76Lp9#dY&Y##_7;vn%KUPQ8-}a3#6${q;sKJ?-3MA=>OS)<=PQ3`)56_b|Fe8f({CpSoMjqvVv@JNEXLyy-#6weM zkuoB~qHAx|jAIMpzJe_b22R|4k!V9w1+Ulo1=BE_qkwggmf`!1%qCKj_aHj6WRZLm22{o$o2cYY2mBujUYJ&1+Qf<5Imq?1--zv*2(NA}!6WZ8QA z$DQ~&Iv+KS>lLgAp9ri#9>6;NL)V(1lPN#a&+7ymkHb@h)FVI5`@VzYS-%A+PQy2E zVP)`NUSEn_$<)fBj(n4q@y4st^m(DT^SR}Y)Pv@RR^~`2C7<@C^NDEQS3a@NwiOe7 zE_nePLz$3!ww;QtDS4GBn1fd?F7$uR`+P`t`@+S&chEph7`ADS{#+WEgnH1vWc3B| zp3)hFvLg-&p%-YcUL^rgA$MhDX#ik^FZ-YMC3gE|U?Hxce%$;Vflf3qVrZk=5F^Mu?32Q(K}F(QZ^X8I1q#hblt8!T$XG*t*5# z&BR^sCp}!T^%|9vh3JT`T#!cRA2PcXacJ~hH_i@r0m{ZrV>M!Uh(DNAiRBqM1^xxJ z0J;gxsG47gO+j4)3LNXq_m^t9XK+8g3p8W(42@}o+PKWLPQj?hyTXdkn8#t{;ZLe? z%}4hKw+v{kPH}9`L0`FB#0Sp20J57);Q?Hdq3q0C1$<=lt1OIQ_-`EX^>xQX*>pOn zteg_h!_|LiaXM>s8Zan$4CWGWFC)t0J~&kTxLNkcij#Y2A>s39I`}C*oJ%UED923) z@NwI*10BP@%}Aaym)xWm%l(<5Jk?y>UU-3OLniL_LX?@HRri+OD3NSv(Wa)>c(#)W3UX zxZ3;CBR3iWZ@qi9T<{Cor|~P(vI9jrQl~m;Y^E8;u`HsQhS>-c7cPE=Qi}0~#_QqD z+cmi~)Olg9)y@TRusXqb?s0E`v_v|%3{8(fai*TC>< z30ou0UnafkTJg0< z2mZgmN0b>m3vqK4-uOLN;E}Hsiva^2nhW07?8LQxZ7X(Kv!UCO&1LVCPB-)cyEZDD z`l?6m13cYWktQWYKiE4R?h&3S6J*hGN=5?~`#)P?DHJHI^f|=1CKfc1ddk!@T_CJo z2e3S1Wfqqde>o{*#RX)98v_m6G;wBuE2}&=_MXpAuVGcofXc}K`|rw-9HF*=7BZcI zZAuV+C4k?f$t7|yeB;Kcemq7k4wTLu%qwu6bRW7PP5!gOwmrvtLGC#vQ|R7 z`fv~Dz(n-iF*P;gt5WKU-GE$*s9M%xQVJS(m+D6|i#2sp0d0Ydkt7i#5WDSh z)&w$r0}qCCJP92>t!MTM<1eo0DF#Ru_SGNlEH-|NLYLo0;AB^D#r_&2Xtt%pOv#zo zw$ElEfv1PB(4iB~+0$_hh-(k08v1GaHG=0bfBO~z8(KMrY2n^&X`yY!&aZySvIq&u z&GZf^mb;vCl~v@???+-AJ_NlEx9%Np$o|~{SGfQK`K&M|K#$ymIroj!*jc4pe=nTs!4hW0`e03s#(;eN) zU@OEy*^~(ZhaK4f)^Hyfu`M?D*#;=s(bJa~Nj_ApvJ~$Gf=o3BSpnAS20PceJ)D-7 zNM?tVfJL%;9&SSj@VnxMw=zznWv0KC@#D`1Wugq80foB)$TJp`MSOO5>g?@Bv-*tE5IU$fNP6QsX9|BovsR* zQvdd;@`rzh%p_-0pI;LB>N3~%MKYUZ3(s{6nVh^DpGz4q=*wTS$ZGP<@Jt(q|71JvWqug@$HUr} z0>%6@7N&ChFR2jKCMsVZ-L5m!6_Q?Z6qu;Vf+hP_&74j~G2X&f=yp=7=isE(b7l>6 zv~MO;iUS8Q^61HNz)*C6qcf&J`_aozI;sKe-*4)jZdBUn;}xESxAZYBX01|=jLcw| zr&{f)n!uyE0T*f&?rSWC18Th353^*Um~8khoL8t=6DPj%7Qr}Ot(CGNHpyQVy9uD< zle~4#>(3iN6^Q)HvB+?auwyNR^q+XR9H+(2}0hE~CV5AWRJ zH{Y-7g4ov*83@x{2vsPZ-Ya;w4V&<|t2J&weQ_l(DMSUsJ0R1_Q{AG+q`?*+KL(k5C=_9=B50y(!_;CBZo2ww+0XM})ocgaEDB(yfF`de==2 zI=6R@&?G`1RaccigNw$c7ZM&aCQK?A53%i^ciWw^iSWlxQz8ci@AQ=}J_tc&RH^dJ zX0ViTE98gAgi`ZzP0F3fRLw@mN2AJro9yFtc13hQT3GUSR_X7*bQTeiF6f|U$Yl9z zT4*=TTD}&zI^bKfR3SWaRB{EmrOl-oyP%U852E+cljy2)n{=&Z4XSPK7Y5SQ$46%o zcC%YRK&qE6W(*qGdOki%eJid{j9WdIn^yGCwS&KuwA3AvfSZ@C<<>eNO>#FiUXEb~ zE94S`h$RavQ(mX7J=-Ut2ytCq1K!|%Sx1GZN7D>*QoRaYpGqU<{dLmm?!6F9w*Zu4 zB36z;ux$KeoBoVh zRYoL-A1LA`@^77Hm-C99Z2V|RJthd-s_lTMMJo2ZLyV=o7Dpv9k=!`vge(6&`Zm3! zcj^CO?#;ug48OMFO_3C7kPsM`E6;+-S96KkjJB?AyS0;krR|fO7@#P5VWs+6DdVZT+okf}09TZ@ zm#2H{XsON{JQ#|{r~Nk1`;)DkAax~6I$yqo^EFy5Ix1#K>b{K5`s(@CQ;ws#vyvQp z7>zC;%hPq^IgfC+w{MpEz4@R_T!A}dp+0W@^;RfhGGAS((qf~EhnW~)K z1h9XmT*}u_nc~jQwohK^j_?U9;UsPu-wbCAF|Uuf#%e&|yqHp}01rKhbE!gU_`~hc zueo^d%S_{p z4uAHiymj_UmPL%w-Yc})OIfUfYC+$zNp|!vaG;U3`E-FJx6zkb4x{Z_a(P(%*Emav z=x4x3Mv!h-F#e%ZqFQy;kEmFcaZ&Q~o?>m1TeHj>3G!O}UGfbr@?wuuqlUN}6n6YY z_J4f|btL)xI>vXHi%xa5XZ`zFmWF70zhas~nkLS^_LVE%sykGk)j*2EtMaZ)TY17P)Eal`D;7rr~p0B%NC_R5?J;l#>uttga;(u)cFr(L!oY=r( zR_YPzI&}G1;dKMyyNquqXa%JJ6cX5BJ$iTwsy_0^o7dHLj!3RN8|=R)Q0@s%ZV z{M>^`!(zH4j^)g~=l`fs;90r7@0X&KGwiD6d34X85qr$nJQ!C}ialE>lP(x@#&6*2 zT^t0tr7Xei8@9W44Ho{IikUQzd#a3F&C{9%y+1v=AF(*|1|<0F%t#_5e&4%_ooSYY z2Eh3fUZ53BUDr~RJa?wxK?QoJ3AlzG^MbzRPk>{~^*bG9o#tfzb;fmy4g1OpqF2k4 z%c^HJYK1!x-O>`2s19OIQ4%@Qs!%D9ZdVID?2@HnUEDUF3;!skps%kV<8w0C(}P7D z06zTMX?3l6ulAa*%(##){2967`?r)nDPbZdfq{**-zxNS7W3S1DKE&xxcQXX>OG7R zhx0UXWT3Aib_=DFs6X$e+)HLMSY(8eOnvtvAXs0{t<>oSwcff2Pru~*ohFagr_axu z*V+IW05I%IG<2x}Ure;ruULum$h{PXlgagwI%)7wuLfQ7^VC5ezM zNeRL|D zin{c$0k1e6L5udT+WDC}%ifiW1{P;b2H#NU(SN0~4^k>;rAB@#gT?}>&i;H$DV{us z#8H+3$eN8aoQX7Zq>lyZGZbP@)n+g3+vt7gyGN7X9=&D${D;B*mPU-i%R4S63K&V| zA@JT9y>*7&2imlG;t0isS97-PSNOD?33mdV(^*aIsckBfD)(s(zY6wKV0=+rwAmjf zfCD<0!1ZyfSLK1)AGaKC@+g6Oj{zUPVmhAIU1-oS5vq@2=r={N>BI5L@%qVh;=prm zucaPN4vo01Ux|z%9u8xxxOs>{4sD#ZIV3O4MR+@g(qzOJiN2*GZaa zy|3!(Cxdpld&Bm@Q%Z(O#uG~TBzq~o@{sPq2T4~xu}nq z3n_g&{&0#yW2w(tuS0!R$gRlK*hs{1wH17soI?_Zym-;YvKQzp)?Lw zlRga!3C|*fItSx1${p6Fo%B(en0CA<0Rga-+8Y1ZN4E4_Hpz=rJk>^F%k#k3Lwvv1 zYGi{r8J`j->FqhVnpOB-v!c~BdD2lZ4Zu$x#x_f!=PGEhUT&zKlILO+4k3Mn# z#s1(=Xg$cakE#xRxA1?KlYxtep3+gq7YGqTF{o>$By5$MBPz@GbHjKP-wDKUuU3ydJ0-?kGYS+Y6B*iKtzi z`PKXLHpU=TY0k3nhF_TP7R3wm1|F$U=;Pabkap#64K|g$oJDaEiQSkHjR8X{HwEG7 z%-F=2(0OPewnkAW9Y+x^|7bV~L__zeD#wDjJJniifLYh<_pQX)mG-*Mk9VcUeb?@v zp8HXv-L()DJ8)U&3TIvByBv!@^M62$Q2fcEn!^o<1RqqZ6~(?sbzQD@%!{+e&mN`O zd~?+~V?hNS^OQ4wb_2eRYWjNoVY2AEIoy;{dTVyv>nr0@7cmPm>F*G~60Bv`!~|li zv*SDaB^iUexFh!NfE<1ALXK>m-D3_Wy5|FhDS{uM)oIkH`1@Owsp?OIo@-XUSrJY1 zX4BA=w1%)~9y37jh}u0fCPPUb(zA7aYc^HSW6)wG+b+whjx$xI=gf0D_RTO3?ern6GR$VI>Yv>53TScgp5z88~dQ~JQlOBGf4DCmXYG1tW&0omTC zqeJG2df6*VNdyop9@{I#6fE)Uy5TZp#om=QoS+pneZ8S(^zEut0LD%l8yYSL+I%IfjsR=L&5 z)}fE7^Wb-;vt0crWv{AM9RVHP{Qc2lCOa-dio5j|9=y=WA-NiPHAS+-rAeNr zz77?LEs4JRjZao>k2KzHVu39O*LjAew?m91{Ghn)%Zbu?ihO+TRzZl&G?S~d51jY~ zF8M#yre$1j=??|Zo8Or_J*KN%B%pML>5C!`p^l8b{VizK8y(h=UK1dWv;&Y%{F~486WaNu0REC2PY|**U6XAu z0*{U4xT}d4>#e%!j3s8(21`zlytuewdzT&$U_2~WL0?ugD=EV6zI8(1Y%-fXnbod! zT+c0knx&`OKgj*8>AMx33Lj;xHv&3m|Jo6-4#U(RvY0<-9uEQTIl38XlxY$`uHwAzdK4T216)Ma6Fg5<-R9)RjE`&bw@Tx?Vv zbhoR*@~w(abG~@se@r4WJ9jD#|Hx`+;Y$Aa2?n9vAtP;{m-rE@#Qr}|7{%psRz%fU z(Y}G&Zsg?HPD7O`QcQ#z)<-Ao>W_0Y$NRR_nOO>7KF8g4ZSS$q``Ys-D1aq+r!M#j zu>t39-mcaS4W7&Khh}~Axk%oY`YsHndrtuF>*7CTx@#Az;%_MOYt@HKOYt*m-pyg2nMmB1Ve);m`Lxe01OvhvOZnMcx<=#e9bEzLj znAeWGe=fvk0)|)Leg$+21#c=c*Ox|)fFEFqbZu!uts^!v@-Qz%ooC|^s#e~ORyn7J zRU{=bIUq-`ppcMBY$6B~4}bO9UQ%pHCCpN!5_Hslw8xyD9dGbe=<#f()&QC_B4%23 z2K6oFBY-#T>O2bP-Y9>C2+8owS3gWR zK%>Z=i4p1XwQt{pdpHF4pH3efr@w!fx$b6_7&d)Qt;_cFehrmvVhCn*I{XFVJemXr zuF*_Bw9V}Hn1cy}>H%up6Jn;39g- zb*JN1eeMoj1Q#IJLawRXnAqNVZJzfk!P-t-UN@&ZRey9_d*rusW6HCx$7$DS$RMs} z_dE$4sfz*_b&io+`Gme2_CI@Y=Q zsh#qLGinvH(XJ6Ul?a*hF z<#Rbq=D;Jzb?D?4BZeL^wJ1o-`Ei*}v9Lczuqm=D$tg$sv;Mv(zyq!rLN87~{c|a0 zRH&9p>9R;DKK6@fGB;&M4em%^YrMP;r?bq$WnD`}%ni$-?CY`piT!myRCdqLIpd|# zZf^&|O|IZ4mCj~o$WAG(XW(u}6{mz;ih8B$_6#oy78Wm+qT>XGBhyoC$S}zqhi*=( zsE6R*3o8y^j_gS0P6qcSfyK(mt3R$n?aIT5FGc(*$4WU=(A#P2w3zI<%1m#$n0vzp zusLO-uhLlR4vUpi_Fm4ziA9d3xAS72u#d?9)}+s7NeuJH`>1B5g&h^w&t4J0tZ<}? zBnS>nTWVR3ah_Ihz2seiU*Pn*vmGGMVx`3ur!-PizUojq&d#OXQ=lM%nek{HhAwIk z;dM6dQI}wD?u#%-JZZmR|BjW{FM0BX%1j$ajG^U)Zh@P>GmbN3$md*bowXT<``5P9 zj!WzYJxMirj-ycHRc}p1l-Nrvc#H16>{HlKIuWM^cf54B;6OR!(gwRaT}7A>KB(7m zxQ$=sM@@##2q0}`DR|=^x9kuN%;WiU$y3>ahg~U?$6q?Q6eM>p zR46n&9kwr%vS{d=aSco39(aDI-#K8|>7Z|)>G}Lr|C$UW4@pj{lJ~^g>gv6ByjAN|+LP}D8^8+XE z5Rj;=%pm&#DBf1ZK#~&K)ZOhyWiNvI-=*KkEjmk<#Y$_1L z+Q8}Z_fb)Ym7i$bv9VY>Y+P;TK+uT`cKaK? z`!tW1pfAd*L(cJFx8M^Z1yzxX1oAK)Lgn#$nrjB!Zx#xdS|BC&ms+YU^kjy+&VFem zxekxL1;Wxqy7EsC+EYl98G4^D;y8Hefpacg-zz#K#ea7TVaPKD^BB~R%7rsBtnUU= ziho30jkVptHkZi^xg*p8#r`frv8{k;I0OrT5i?O7!eYi0DlyrnfqI8Fb}g}V|$}m~TiN@f^6|>=* z*E2dsN_T(O<%I}eGu!%H-CB^NPGIB_qpKh4iOWTRGn^bkuqBUyx(7G9Sq!OBO!1_j-y0*Yu5cS@20k`Q2 z;27eO#OvD3TW4-yIiKpN8ETt9?AKe8k@WGiujQ-IJy)e(qa!XO?aS9PqD9&>LSe_2 zX@%+^I-4g-FVbr>e5C8?m{roy%kVgiSWXxRV)snNO#Ikv9X6~EkT!Xwq@*^rF*I#p z9%>7j5RO*Jta9Jk**U+ma*&){UPonpzON1Fye4pVl*u3hj%@n#=hrsfMLj;_-$<2s zZ?mnx|Ds2B3ZCv4eS;!N0fEev0&!v->z$=b7`N_ghmXSM@MC#p#!--#_5q4~RhsjN z3~!1~LrLi+0vhpN8O%_m-v-NR8(_VFEx5QBK(%@BI3bwgHE>irExL>H>rJN*z@N{% z4LYs>~>KVx~pMH!Xg6)vxYvb+ijY_Aw2Y zh`M=a2=xt!*vt4wImc(AU^%z336{|s1BuDtOUu0I@YUO&`lup_euG}=Z=b1V1NzEl!4qYO;dc(^1waBU-@fF5Vhc%c23uFUsL#F zh_OTqbqIE=R6m4h8&&ZrNXzlYTjry06!$Nc-zcGfO2!yO&TjXp*WL?!HVxtd-s4t@?lnsuyYa^?(29I3{TiP{ZK2os~Hu1jYQ8HJZ6at19iV6M_-7>?hYz zK@*5-=G8TICmC~dFP@8eslJjpVS>f6)J+&wPum>7ggwf?Rl@2`8#Q!M|2J{e*WU^O zFG1&%=AcOMF=))`eoFjWAlqDBim%)-WkcWXbcwJ}AzV>Jtg@i=!>zf@5quQa63~Y$ zz^eKB#VOr7PuHIwkoh9l^hfG+DE@VeF^E?L!cU_jbg>0vhEg4RON>&V^=un>v}bUn zFR#e566xrnzcZ@w)Fo6gnUr%4U~> z^3_R=^ZYtGnv;xXYV;ARVqyC4iWHQmDtH$i_BM7BgQw-3SiXjeYm*#;9L3j1WhVbP_Ea}|%Cj%I3)i_FTp z+s6J9)i1G3VM9%u1+gF(Zvz2 zykE?u;gSmD*$`T|HfS@}-QBu{p-=Skt!u?~hrBaY#WmV=1a#ld6P$l~lwI`L1&j8N z1^6nsS(GxD^h40`4t1yBvjyCw9O}2hSP?L0W9QzPk;~uES69b+$@BmRLD2jB0p$O* z$17R#5GqSYecHY#SSb3)p#Z54t#8D}t5%WmMs?Rj${4PRm9bUDxDn!1xFiZryDgph z00`D~ZW8X2NLJZ&GSySEzl9eSf?4Zgc-8M^f%|_H8v48}yBvzCXW^ zTo5C@wCgPuM9z<;klZ_6yLMvA`~LXr3{?fD8|pLmus`~Th69YPBjW8C;-qYUPguAIS`Ep5wOC`cRR_Qg~s8C9W-{|pE}!f|Tbms{PROTEOZcgu`D=##c; zcCSmA04&5aJ$VwcOt3)NgFayD^2y-nF$6>#f{KC0b#$!SwWPFEn2Up>7+frd4(Rb$ z>MkP|RAwEqYR0^vP_y{@oWM(!QPA?VNXB@@y5g(cWs2ep3)r5waxIOfSEEAw(pp(ymJDo7Fw-C)V;8xXhaO zV&1a7JoKEzI}nNlvAnK02xkirqxSTXY>LS;K%0p86Hkh$x&BR^X5n z;g3Q=w`0P5_v8U0P|8MiKm>;yCGeTi8hyuP z7DjoWAWBa+pv2V7L}_s&if{^A!4G48I}o;QfjZ3ldk^d&jA7YIy(D2K!w5A0PYF-& zuif|vso4h=)Q`*|E?wy_3sMfQK1w%nQIWEfB!|4*ZM3uBsg=fc`;0~w#EDk6f8^i2 zj^!AkLO549!MAV|#R)qcYD2ux-ls1xf&Jz3F<$QsH zNhnRw^83|uWxwf)49kYHl8Z5?IGxdrjf>r!5@q_|s?P-kPwQ(mJaLuPma9_uz)mD9 zfuo~Rg~a8~&BJ6|R7{V6-@F*9a^JC2YI{*=1SA{yS_4&8Q$z|1I#o>|a3X^xc*(MG zsi5`8izKRv{mPQ93jq&q#M3N&PE_$TWYBOfp) zSp&lMAplu2H^f`6C?sfPQZexFJny2aZamLAnP1=+59naJMQAx zhGH3O%b@R4w2}L!id)3tamNj|>gC_3xP4M6+;~9SA3CkthBf2SXmOo#=6g4bQ6;?) zXR5V#I;&mbm0?;~!>z>Kx6EhNICU`eOpsY>*IXQx|`1pq}HqPWPoOpPjfX_wQXTm5;!cS3O6dUg15 zNR9LFQ9+`{4jthvT--alLflD0f0Qi?BL$y3_^_I?kc95c1eE9P428Z|Y+A*7Eg^~af zvmuI(U*a*-!Uc023JKF`m~YtI(SwpI%EszNf;agEWPYS<87QceGZrQoY&}ymvwQ4k z{A?hy@6AiKC}&6&&@)s_3PnIJAAy$W5*StBfS)~v$YzltDs^7&h;VQT%#DikxbJHd zC9&NT*b}r@E82gz3fbQ)k@~YMj-y%2UFDAaOoN#!Le@j z9^c5aVr3vU^MeHy>rOcy?*QsBN| zqpME$SV5h>KPJD|#BDM0likjPpZr+CEbaQa(X)_#WWP9->M9c&9ZlrNk{;Lk+ALeu z!cy9yJCAyq&AXP)G+k%qzGA+8X1AYVXP5^zu9<=q)iw91gM|53q|t6>uhmeJvBPFj zOhj@C^J&N0TYbveb4N8On0n}`TSS;9)m1ei6!Bc^+8+-3pgMeq?y5&IhTXH!`XPw} zEc84e46cW1o}$oJjrd0zY^u{Y)kN0@{^aJd5It>C4=uS@a)|oO;%Ixh<=Dq4?ZbY< zR3W3nr*UicH)53YHF3{JGT>L<_!cig+wfzkO#@!dHM^MuF>972JdG%E?_N57R(PTPhR+|s@{o`Y` zwlfI1dHd3+EKcaCVgzzy1I|2YW=)2zYbJO0@xy#!>+XTp?=3SV`fllbZQWjqQ+H&h z!cCJUO^A}zKjzu@r|HcxpCvcU)wPf%e(o8`GUq6OHZHfj3vt_$R=|`U>!k<>$r7wW z&N;wBr1O*MTUCA(xDYiS>cIDw)q^_3o2u=aWk#lW2>8)Un$}gNs_? zg!-7eEE!wSL%nom>e_p9zw3h3Kys1)70c`R}{&~kQjh2MutlN9!xhjJ69}<#h>#(2gp&-l-$;g4iHYrZW z84HdZ{!>v5S4?7di!Bsywu}{P6%fWWIaP@o3wY?Ywdq*BP^jWh3J;va*u9V7a9Jm1n_1UK0{R;F4iVhD<^w8Qb1TQoZRzcglZra~H>!tnR6qhs zcX_d!qX6~oXpNgZVl-hKnPAjkggH2ezXd*BXltln^(8dE$s`uO1?_&x`TU z$b?X#d%ehSW~8gFhCmUIKlseE*2kVNwhnR0Z=ZNuHOlr`*UtLbiz!BXJN0V9{2Q9% zWb2D>%qdF#cpO5hhC|L|BVEy9gQRdfTHba~*A;V2t5GO5Cgzbm$2iEe$p@m2F}8du zGU1EsA%;+^q96tPl<5W{y9b4f-bpj3JfSi(lXpQy;0YH|9G&_C$^+OSpi@ zUtE6Mpec1TNq}oWS$mJOWLikRf`LRc&P}w$+r z&{?>BpE4T?%`w3PxJ5?vUOeE%EtHz&^cF=yr!?V0)~jUrP$QE1xP+)a_RGxYj;mIo zqjUiCc<@meM#{ZLN|T|s{*mGLm#ibAj2L43D&(%lh)8H<-GV0%z4Tmc=kyisoAtfGK?c9+RHVDtYC~BvpyWYj`w9^vy2U-HM<>p-KN;MsUj1N63-` zy^qYVtQ^y*nP*+Kiq>c-)tzneo5042K!qa64ygp^b8Bf-+54cKbAiG11_m{0HreeE zaAUQ0GXzM_h}9>IU3qjD&G6gcx@dWOBRb}BYyF)nC6qMPzt4$M(>+x*el`zxl=qfH z6&Jm@FUDtds?R7}XQ8FhFKO$9O;4K7%?z+TVCsYQO}gJLVB1tpM%2vTd2e#%*Cop| zOe9u_f*Q@OYc0Cmg%9`omj@-&`i@WM&$EVR%k&`GBv4|WJi@(vVaDEhf8kldUsIo(#5nb#t{GiKH+ZtKRs z;PF<=%)6vrw~FWro7q= zr{s%okl1F|qW=xYdBz})cr%GS7->YbjY)~6Pw&@lmU3ztn};mo3vDj*_CI?K+< z-608eT2{k+;K|kOXvSM-O;}4VsU<{BF#CxIpo|zAn=eMtA)7R~M0EWEg!YAd{Jyt# zcG-mSpNqa0=vqMgB0Gk#7-3E*Vc&pvj>v<3>nEP$5YLx2yInI&Q*!jdz*=wP{F6@> zYe&xXwKAt`5V1)8O&p}hgng$y=~oEw&Q-hhhEc3Z+tSuOEaoQJUo!eDRD?#VU77Da zNF_NBjdR;ZSdX+fJ8EX`c1G&D=s%9^o4vm)yPdM>`>~|VQN@~-Zd2o~`?ZXn8%dt> zh3chzA4BqD6Rx4|vK@HcpmWk#413PzQ<5H5<8@hnPH+s@*a$1P>-z`~YW*SVb4USs z@xbr{PW2o2O(62T*_`GTZmzFE_gP+}pn?)K@I;1cO4zS)0-Cu2D+q{`c;X zw-*eUcucm&W0iRQDg_*}LT<|!5_1F}hk($ZD!uf&A$iazw(h;p8!f1Jqyln(y?|)t zCKtQ(oL0-+keNi7@Q9;>CF#Rcq%V&C4GewOiZ@;=#- zvM-Z+0}g(5vxydncc*M36h)94EfRTx)2_Iycuuy&b*JLLMyg+x8;e}djq9#^{VPK_ zM=Pc~_a*U*;=_z}1MGF$Q?=c>)a2LY9>#a)atvq&{(b(?WoHZU?v2t>I6ldGY+hNlhI(@SOvTMQao>Ky!kgv4~X>>($SXUTlXf@K6EC!i*#J{S~W0-%# z$yHaq>R+wmspd;Pn72ehLq0E8W&ZC;oCLB5e1Fz^b5ag)Uv)O#H(Tn|a)ThLa)%6l zSS%kBYvld;A#$Lk<=Hs|xU>^D^I)I%zxA|?GvjV~OPD3BoV!RCqu%_9#@}z~MQaQ8 z>N(EWKL=@Evq{VQ6@^VqQf1W!q!`Bunqy-24r%=NBsnj9z}GM{0~>wM69%ZFCR<_c zP||`?`V_0xEuq@_o@FHCydm8~ObJ{fJtRfCd<17PzPzf}-FMhb5ZuUBDox){@F$viFUDE)ltcg2?u zwz@{7!IowN_v+OBwh-U*^VdZ05vt7NO( zB-5dPi;P;5#?oTc*o z`Mt#ol3cNM22DCD-O3!Mh0?Y%YWDE_a-}00&M@AwAtK7 zt^CwgBM#Ob%gM{HBhZsaa8vIAAyHOeKlRz6@DlY%Gk6xvPMh;qls#E>=hc-G8uN6= zA6vgLm}{b9{ifq0sEU!7G((0wbEimG!|aNXzqd$y+ux6;Ib&S*XeC z@0@k9V_#koL49=h-fs3z*~9!}hdtp*)r@^Z`LL$krl=SxSBp$aMh&F^6FqqOX={3_ zHCE)1y(1wuvyOf zGzUAHQ;>Mi_4dY!e3VD>tG};#aII>mD;HjHOdEQ@r|xroES$g1(&m#N>Y^WAiPJ7= z+!bpNex}WyMQU~Q8Q)m9)VIeMMzTwFv-Xz1T)Qc+>rL(5hDpSMztw-VO6c3pg9+>+vA=$h z;YjV)(HJv$r=;=;NIBqPyoe;D=Z!_}l@}8&N7@$C+}m;VS~#YIB;#3Zd3_NaJ0+Ji zd0i&kr^~C8*q2v-m}Y2oQhLC1?d95D(sDKC?^{Y^dG)Li4}?%!p%yk6ISfpJ(v*}- z)`|*-TegkURF~5} z$Qn^(tU^LPB!?xy9*Xx}h1=5bg!Q2#l%?Qx*Wu*s3?4JY&%sVDy5o&N({7;=c)8^# zCB54n&(+|gd4$+Oz0Zh&zz@q71Y64aa%`yY`^6b0v*z@f{rhrZTH(wcTa>&!T-B>P z5;19}erTSX;~p~%FsdW~8p|DL`Sfi-V1fguxe;WLF1W%a0ePYw%nsDQIU*Vc1d+YX zune{hMLgw$?@v8>aZ8oeRznC|Ff!!b|V*b4sq>W3E z5u>sw#rH?Sb!-t@%;nJP9f#>M&KuRlK0xk$jhKHeBc4)U<-*Qu{U{;lUSA#?0r`S4 zjQ=eLaJL_bWHs2qCjfAKx!Q5@4vp{2^iDGK;0o6KJ!uzz%%k z&Mamvi9Ak~;JrSy4q~EcONN}Ysnr(0OJNRj)SDq_e{)eY^=@7wIoGhgd~IR0s)UuC z=INa@ie^?(*RMH-{ODaWUM-b#8xMuL=e5G!6m=W?3-;fAR?L@eTKt>AptTTtKyYA@ zTHH2&>lg%-F_Yy#$Y=ww3T)gsbN#%a=`$}$FrZZ|xQKA}V1!4+e<9X_9lD2*hvwPs zpme6P<7y6wEw(#tpofRYG66=i41)cV9l}O$0j8|_!wcr=e{*3!ncy?$BN5x#Lyy7h z@m37#+q09*0aIjs?^M01^gX7d3Jh*yXxKOr&JPN;dgl}B8`4z%MSx+`)f?cExP-?e zmNE!&2pI~V<4xzE1BvZccl*0gqDG`LI_z@6BAXjAw=laea zBFkmSLq&P@dxU(bJs5BlM9l(I)D}!Umjix+Y9~4#%>Md;=5qLfg!0g@vL_Hfr`MYl zNZ@RUyvmhdbY*YvP}~nQ?Zo@^ZBN|n%>JgqTPFZTl%7YsAu5E40%|q0H4w3I{@#OW zsDYipGHD8H-A~vD=m1W16=g~3tQXfwjCsT zjS;f4Vao&}zyPncdA24n_+;OF1U*lq)@k3)eI%-(vN#CDU9qeg5ikdH=;KRfJ=B9; z1!RJnTN8yCi_H$XOiugk=IT^F_@RlyymM=Cfxda@G0eFSbLLcf4VlbHVJ|YjLfVi-K(3qUVcK2|Eor>{od*sur+W+$}`rlP~`P(?jZV z$4oju;4=?M_yU8*cH*7?#iw2{+Hx_k41N^iCSPq1 zd^^a=KX>HW&txpGi00F4$$9=ez_QwStG=J2IdewsY1jTs!9!6ev#eY1H0AJZAvSqx zMM}Ow!AOAwOZ+hHQE(U9ZP@bi2=tPdfT>stKH!&yrL*3p&On>%{@}xxJM!|x>jRS_ z7~+5u>=V9QswZ{q`o9q9E2mog3<3F`lR8KaWVE`#Z{}LYMACPdNWQm#$lDR4tXq_Y zsIUKNt*QwhkY>;Vhsl~OK>NSF4Lzg&wX`HGCp%|(;!FWI1=(j!GpTl6JFfobabH7q z&sf{>`nw0c5tX(!!o~$8E;1!d7^dM;rV+gtukT7WC zjgd%NCf$6J>3CC(-#wQjzkq(N_tOKukyGM!y?Id!qJxjM+5px2o5XK!pRiGu>e+I$ znkNq6q^;JgVZ*+@dr*oJuSG5bj!x? z@2_2`lx+*bV&$sP@IgtOkSDZ>yUu@4SkNv8%8`-~1>t9$fJqSIPqs^t59gGAlbE=& zx6+pzUGw>Gwi0)ct-wiBb^JXjG)F)_VTPDo+>pksdeEqQu7KxQ1sG#}(ldqA(B?HZ z6$D-JdJxo?|A6~bKG2IAH+s##g^GQh)sC2-Eb`u7UnW4#Qn|qk1bTWYpIWb_SnGnj znhsQIg_SIZ4WA9Dxv6vZ4$PV#G~lLoly zj+@Bm{)#O)cKJcVqWo*b>uyeX#LhAKxkv!OJT zx0_DP&IW>LMVa*gslfn0=aE!wLNO^BNDQ56&aL$p8jNGe3%~}5abW{IyrAM4LG(T4_}KD1ey79(HuAe&4; z!_NauT$Vs2;W(g-@X~=@Uzu^4i{o!<#Kg!!6Js$_p1;7N-*`Q0>#dl50*@B%17c|m zG~99!K@B3hMqtM}5!F9O2>H4%x*$jHCrBl)BG$Q4(L+F#M0k2d;Got)@$@_Hp{u{* zpeC=X766C&H~zsjch00^LXLeECx{!T06Eh@Oev3;LHtWJ6m z81qX3PY{b*3Pm1d4SOj#qHkzwba@K9Y0Cj-xIl~~T`_I8Td#a^x2L+wFo&Qjdyv_^ zcj%f(XXqdwEBsN!JVMbNc&3tmwNJArRW2Sf!wRpjg$7asek!}0cmEivoCK^1UXw4K zXdmB^091VacWqSq?h(#g4sgzwm-8>Lnq>@>FfPSLLIIhbeT-bx{!RSfV_vx#Qr>gN ziMJ#`GqPNWJ9fcqQD8%Nw>ej8!#u%*ZTiq-v;EF29GWSK&-wq`62r)eg*k}$qbSZN3lXGbHKtz<)}lB zs_THJeGFFC%Vmg*;qDj+uWtIVUA~;Doq6D`K2Jyd_*tBGUQWo&KT2$Xdqdg9th<)&Uxj2}lYwk#pdmr*td%(=f`OeXwz+b%37;eAcSSt|hHRka*FZhDYK zD~2;lAe8q~4SkPh#0K1OKd{M zK1Ezt126fX`hX{?Y@Z#&5%v*uaxfGnP4;g02F@HlJ7r@c|GsQVM~XP52}hwNXkWkSn59s^UlOTa015N5SofaUVKFYx6tL5Mr|*f=<{ z;({`O_Y93?QdkGBD_3S$S8SkdDL_%kloI4?G|S6-C#$_F=H393*0*EUHzv!D#+38& z&_y04o8NzK#)sT*mM6#r%aP&`nDMqC{&62TD(>C@Gl%nF90eu_AsO8=JB$1(iZess zkFjIn@%(a<{U^>;0oLMA^UOz;oA^+DlZ}4;LBq*I-I%7 ze?zjtJ*zgcX&zcL=%I zgV(9YsEeF7{Ww>3P1RQsju=duR_D40AXo8wX`j7dV-k zW#?re=qD150mpb2TYII~eNWgtJ#@z{qfW>pf%%#Sy;EObbpGXW_3n# zF=m?J!_PLaFSgIDLXQWs(RV@|=R+&*a0 z!W;SZB2614t&LoIm;4;DvRPh9x)F%4g)%)6;dlMuwSgm4j2;A(Ub~=`DMQ9hiC;bl zhv-nKb98!M!yBMb9GTispYLs^ZjXSGf?y#K#176JB&(&lQwtf0-n*aRbdmk}F7nAJ z)l@iahe4lb1EqzC1PAhe5rR2oA1uI3#0ILBnAc#% zOt`V4*9o~*D`M@A@P~NB#40i_^JE`Dy(naTN^rP5sAkG-hHw+szh^q#Z@g2_*RGoJ z0oVcyDIVjq5aw$63YkIS096mJnuu>)PyV&?jCH54BhDf!{Ps5x8Ow9532PcNwcwqAP@84BN4!Go~oB{aWmJoFvHRdyn$Z{V;4EkXxQfnT} z*eQpZPc=-&f8>RcPxaG2#kAO!`gD66&s7*X=ZogK0GV^AU=wFVrBxrNHLQs1{Y%Ji zCXvp1A|7g`;J`5iP>3$GGNv{3g49>=u@<)O$p}RX?lrXoG^8y5!|cy*O-nF`uLvUD z2mm7%2)Q1ymB*EoAc|B>fd2)b&141;qPl&#abyVma8>kPYg#MmBafK0-fzg+YZjnJ zygmGi_17erfS2S6A@Yg`5esMQ~dNztS zqp0W$C3yVN+<62d**wTI`mP3-oyxy`@Jxw-VSAYbXg$F0qEHJ;@S3ldUy}S0^#(Fw zVHnuZV`XSi$F5gq|Izny|V9;<$n{mLPPn0k%Ds6X_1YgHTHJ zzO=M-LiQda2OWY*eGe+t@X;aTJP-}gVa&BP5n z?`)xrJitr!MH{zj&^LfvCU`Ae9ACeD`E83wl~$SLI9Ld3j=fg5c63CxSCXi5pfcYJ zs^Y1C&*U+xt?WcH3^k`AQsZhF85#Y&X^ss3-aX5yIap%e8T+<9t@Yp%P-&F^0`xp8 zc^u&4MR0;~XuP?rS8Oc%GbIzIB_H7E$fUdhcZIiCx`09c{|P%U{6+$E)Ls~P5D&wl zWCS)h1z>L720{}1u^RUZaEbp!dV=x%+f13sNhFA%l#l$~iSr76Ukah%1@PvDAxAe- zRszy%4m%DGUnGA=tS~9}5?{YI0hVMrILl91^JtekXobQ|6zeCgh3}@2c{Ttox$ftN zH$!^N#mVWkzaSBM4N-pu|J~!C>_92`V2}_#P~n~PFO0<>r^0RkhCK{$sST)}LaeWY zX-OWu`A0W!B90b#6CF*{uKx%?^d+!{anpJdXImcA#*f79_9=as@|vVljmvlC==)Fd)3Qv8Ed;uR{BuyGLuesbnZP z(u6{|^p1+04EX`qgb5%~MRPw6R#zHFFJLV5aUhz{zz!L)wqSNMN_YLMm%a6PeVv#6@@UmM%n%s{ zb7f?}L_tmn9n2C9D3E!&ZlFMG+0i1r&XPRx;Og`lFR)1!^)S$S4ov|^UsokBSjo#a)l~;4 zFq$3;rGB;NU1bw6zuE%qP7FSNLlR?zBwmvN)%aTxi!bk#|1H$y$*TH$0ok>g0mkGh zQT9BZu)5XVZ>uZ2ZZ(~E`Ep~qY<>-bFG-?1$l-1_KCrJp5I3A7+^&SO|F94YZ$xN5 z+1<|zG7%%2qQ)DvaJ19Q^=bCLISoHUq3;nt<~+tsPP&D3Hz1)uUJFF_4*(%p!@Srf zUD0uvrHY>JQ3Ln|nPc>n(evZ|E8h-p48xQSqo?Pc?zB&BLh=yUiOdFeyRsso*Nf!H z(mBXT9gregzh3x&7FKX`E8ji{`$@k9^XRfyo}ptrVB`swD?A083j00WOOfXmVuig8 zmbi;x+?4%l?$iHb>%HT#e&6t6Qc@vl5>XkcP-cjz4~pzkWF=%|udEPlQC6ARD^Xe5 zX`9(IqhYU-J)h(9`FwxR@AbT%{)^jvzwh_^x~}s)kK;Ixv%K84tZV{-tiK{+=$)YE z|J9gPG_!5-T&kM!_D}nomv=omti{-1L%xr?BJzpDOsR|y=4~(NkJg#)()}EgsXzb- ziqk)6-+g?rv#{n@lXznmCS4Ke#n{LwxV%-P+sAbj&pQ3}pjl@@PL6p~+S%^yOA9l3 z02{xG;}D9qeW6z@K6_kLRP^5-B64fJ&Ls@L3ORJ49mca;99yxf@-Oe#C8}q@EQfuL z@bXqWN{?eE3I`eC1Y2gS`f_(vo@qjNlU~O~|A)qo!nG(j%Bj6P1EZYDXpX~L{|sO6 zsyuSGS+k716~q05Z%BVKeEyG+eyewyc6E+FiH}%@$K)!&E6rQZehsUp;HVFo3b)7QGe=q?|5FDk9|a8qJ%S*|BdWb$e92j~S0FAwm&D zp0Bz+B+WoYiG9`i(EuZ>p4&6(PRGnk1p`I>yePKf;OfD9`$usLz{#fzY^ynHfjM9% z+ZMecebp9kGO%l@`FW;{CsV)(G0?i~{wgo}SOG z!gKG3QgmY6XF6aj5sT)-JHH)AE-u0(x5R$~)$Z6FEIOXN~GFET4}r1Lg6!*Yr+r`(cu?lmTXV+Rm_GD=TMGHz!HMwlw}A0@ET6V8FsNA+$n z6N+GU9f%#9LL*h%x}7?zeSZn~SxD0Wdpo8GzZHa#Trw^Q^%@z2^cn-p-n4Amv`O~_ z(?<3RFs-4rJ?!M8SN}Z<=9CL6Bjd*iPw5s7NC-)JE=u^qiD%J#x&^IZxpiJ;X_A5k2|jq5k_X@R*NHO{5rUP}If{I>K>;K2{S^ z&Uc@)D}<2}Va9Ue;^KTgtg%8zN!628BVM^~oQz2$R5L&Y=m+-RVfitC>YR(JF|(!| zCR2Xv5Ak?IMwz}f03>-0oM;{igi^CTRFyR~%B{!AW|G!5t4cLLgroF@#h26!rgxGR z`lnVeEwE_wJ95%4kFP9$UDsEXsadW(wRs>9$Ynlbly%Xc+e-`GD;YIs(nAJ3S68Uz z6dBt)Zim;S1SeHS!2@ z>JZScqGF0G)GNYNwQm++3o}$^iDlxQ1#~6rdX67Fk!JDF)J$~6zZtQrg#gV7Y-#5$ z_84j}$C`^wCnHm8-9~a#DI6O}e{;sx5ZQqXQoj>F#c!6HjX{{zCqu(D|DIdDU7hML zn0)D&g=}`-fJgp@k{x_Ls#wy46_0o9*rD0N(X0&>5#CI7cN0)<-LL8B4xQDOle=BX zpi|=R5_VY>Q&k8c2_&rk24)ccA81ql_WAuG_wMEFpMxTUz;%b%R^v-b1oRI+WqbiQ zeG2v*K5)P$#P~RNkJW6L;ML~0!T(;B+6zuOSO@*$`;9HL-LVxOI`uj9=POmo7nAO6 z#6h)__S}B8fcf|*e|~;`!ldcJm{vinWlw&u`e2POnSBJHu5?9~McKOy4QBN@`&f_J z7zit|l9$9_r!`lf4-XSCR`{I*!=lP{l-6&)Q&)cd>YSN6ygF)1vKyh@{F*DdnK67R z%@p6Ubq%z%v{n;fl*3GRPhn$LApHa9vW`}m;Me0>{|5v>wRB=9tvl}Y%}nAum61Jw zF&pV1HJX8BKQWsn%Qlrz2c4MLUHcQ<~Sc$^2a%KguKQ9=}h?+Sj$@d3rBvFmb6~^rKNU# zyK+t^q=edO|0@VKGE`s%|Iqd&h{oRTZWu@EFxBHoC@kQ9(fn@~F0Gq|K6y7nL{5*i zHKVA1Vwo~5`yaNlgPKNGpUluBQ{aAzhhKQM6`i6^iC4(rL!_O;N40Wx{97?S68~?t z9-{I+efVFnhTZXhbOw{H2Q&EwUTecJ=?vMIe1<1-nHFLb#pJHy(ZY`jmFqrX3u(kR ztRBBPtW4-A_^-^>Jai!=n-q9mU|f0}A72hO){e=RS6RaA7``MWdbKwiXIm3FP2TnR;!N|-m%>e&ALq-AB5F&uOQngdN$ zRaKK?#m*Hx$*tgOjp6N}U9lcb?^qnoMhr=biL#cXyM`Rcb4bky13($EFa}deFukb9 z=DSyxuYz4javW`MLN&9=0t`-J9|mx$h0&f=W3WXyaFqFg*mUvW|DMqX20f5t8IH*G zq=qWqIgGT{VIGzWu{wA2Bw0Fce!(&qN;DG^1_;9r!}Bq&7Ba~*isF5{g)*q{u@ zHKaUrP@2Ww??7BOCbW_pua4JCAPB~cgg?+dktFb6-4{-A89owpuyg(tz&3u<7uB>O zvTuKu!J9r9+mS`GBjY`3oP&hnBL_dp{UD)=910)N_Wlnoddz%hNLh~{F@@F==@2c>dnJ%|c#NXGzpxLk4sFI{v)KB3PnVejvnMfjM;VcuZ z$c2?Lff~0m?Z6Qd-3)AwFrQB>|MDeeVvu}2uYz=KK^nK|5i>G0mE@p9AD@XQSbux- z$`X|b@x7R7NA93ItS$*{l#Y^yu0*ZtHvFZuWn+nl8(~rkyHJ~7{y_U~upzmyWMdwN zc}Iv?h9~}?8$Mr(bx5l1#@+5kul=ld8 zK4as1+(y)T8^|e2;i#2EO>>^wm?l2@s=oU`zHiHvA$9LOcJV=EcqsmtN|>@iC0g8Fz2 za!acp{1;FRARasZa5cRknFM&v3BXGSwpXVcIEFlOECTOePeNR$@DEt$-_%+KIlL-T zo1;5n>9z{Uj^L1R_Mp)NDwR8&QAnM&_QC!8*FYYiuFGp9q`da+A;D;}FqV1^S1b4- ze^U4=EBhB}7U2>5WNzLql79AtTZM=_AR&uu;;Z{6BL%^dz&LzGzfU^b&LGHjkdZd? zOxLi=ho9gkzv;u=2lF%MXMg;F=jE_SLOzvd|L+Lg^QX&X)@>kTW+cA*ZcWGL z(1b~8i>C*0V9VMVnQ2@)F~fd!Ms(Y~n2f_D?;zFk2s|ubDo2UuQ$Kel{mENSJfA7P zyk9bS@9Gi56DD&2KYem&H?%x0Z#E@h+MLon$2XjlBbYRqhXFmgRl{UXl(5Pw(-~7h zAfPd|PBNHO0`AQ}ndLFb5{&cVk%jYHT32)SinPavg0J|CeUiigDtUzexI>}`F&Nmv zgB~Yr78lp1)u&qxe*fD~;B@v6Wi1roAO~;!8O$Qubgk?r1Oqp)DeSfUS6{lj(R8ca zx-}c9SEJ^TI**!hcaxEg4Iy{I5QeynVg$+aR~R~qEh65uelPheiW@@#H$cq`i_?*V zdVY!gOttf0)`r5o3WMm3=H}+6Vcgp|nLvIiU6JybF@EUtcg%=-jq!W!FwDDxdz~ZA z&S2LH*rg3%kXp33xW zLw#lj;v;RaoMZ=H;dBfoe?IMYg5RJl-A32(K{0XxF;??9yH+c}O?+?O=E9wG^?73| zS8kh0gzVv^)SgQMTCa`n4SzlY*GXX5#<+3z0f=(-8E<}UB+vYyP|3y2TT{j44OR6> z0c^I$m!N7$f4%^Q)L~+hYz%*WCWwm!_`<4pB-z6jALCjibBNlEzNeGbRr1m~jVK zmk}UKes?;b;yWxAdH3&sj|6ImyAtK85EfpQCFPXTdt~(Pf9a2xFHvOrd|g38!W^~A zj;1`8B4mRI&(1W=m%WhTuq1!r`apSgbq(#>|6tK;y9K#oZ)dEb@+z>v-Oj~?Wy0s9 zB$TIBjx{{@G3m{lW3#2sK9tg(*7)qn89SfUgRt#0tnlK7BP$tj%U0DVZ1^_t z7{%qo$f$ESJw--uL4+10IJ4TkrOZVT;&x}=cDd=~RNiu9nLQGyu|Ftj+pq&Vv!ON9 z9FlOLO!2yM3>1F|Ge1_=n?QlCTd@kt**L1AtZgpSG!}3ACnUYm@j+gPHFdYC6F4Aa+_Ddp%W*%;SVqt zZH*&vY|WO`RSHUdXN#A#d3k1>#aki8qnL)AN-9~P*gI(HU3-l%uxo<^QsT}P1oy~F z6U3#{PD6fr=eBz*tds6cUeZ}8Y&o$9Yejw2>KkR?je@}+EZm_n{@qhn)=$*N!e>Es zO1qP}^!oD=6zV?H_M_v=OKkYFR38gz_MFD&_%riSpW458knP0YBZyli7b2J^gGNUi z{Q{T-liHK2pZV+R=;&B>BkVEo^U-=YsZB+t_IX~X@#6lqSL znbQ>CiAa^q6?kCxss6%AN48&5m_gU`x59}dbN(w_%wLf`zm5qj>I%W1iO~q5zt$@M zD@Wpmz~aR)n_iM~lsSrF10@wA_Mm7K12RV-`4M7$?G#;reSHy`?akE3!j{O?NbOY6 zPadizEYd@m1I*tlgZKy+u{1{zbc~W8`h)zWgC{xC5LuNGHXFfF_O*^aCCMrJV=j2AySJg%kJ6B%*9bWZI{;J+kenF zV;`kI;JzPfKJ|OdSBb1JqA25&wmAP#XJH;ICPFwhs>xg0{rOWR`1Q@ic-#yZ1jfia zHZJIP^`GCeum0Bg%a>z|F?n|yEOIN(P2H77ZFy{-T(Sqv#0O%0u0fESVJGYOmwAY@ za{32<@iW>6jZ3CWsYl~-W7}`gkcIrGE$I^sX-AulpXm0bNqsogUx36e3zH9nH+T)cvq9I( zYUy)*qMW;PGVS#z$QaVaNj?_xzD1v%$y9$AQIUpU26`J&lA#f6)~A(}yX1sajSebOnL3WzW{ zBtYz^pWrp%s+vK@vLQzY)~(-uINclz;z{M6V1T3N=4NoY33hWIp>8UCz<=KM5m=zA z`}gmkr1vB<>q^aKm|tYOy^-41n4BY3h{K>Ym`7s-(0wOI$C;fE4?4#`s3lo2QXtLL zxm8eWksN#FRK5a6Ul<5CaScOPt*)A(56QA*uJ1D2d9{;rWip;xb^okgAJ6*FWXX)A z_(hlX?Dz`jSf|EgO1S^&)onBv*p`#fC>B2d;5ulbgNu^lUm=27_M9H0DoTC5sb@Ae zs91Jd{V~t$U|vOr?K}ClQL1hGNGWhm_4W@68JqYxulv7KX(?qG)^6E(DQVy9CnHLi z+WXvU9*8Dc)+bqpJFb-Wmj@}1eW~V1?)Uim7#e^|BhXj0e>(u4$skb`~K=flk(b-uiOis_h5%;y$TdV4J~P$iX? z9x{}k6_!b1+cnA-%O6)Snx&l6B&z26ByvbBW$K2YhUU599mDHo_{C4YZ2~x7eMeoM z)6`Su(B-F*k%vjSZzi249MD8gFY{%xZu>)cFE!`8Py9NQ*U^}+rGeyDDl6UC^!P&o zNBM=M7ZB~$kbJzq%SUY{U zqS49qxe{3UNY+A&Urv*~YF1bHUVg3D37;RFzu9<6yj@x`c_>}pqhV;|bhVABdE=M1 z5i%GTh@)I{#vCt^pJLNNcQh(GSJ8r6uf`#>s~1-z4ly73x#RZ&6_>aU9XMu3!oxQ z#r)fr&t%syEZ?OpK1F-?fM*v$5sbPV2*-YgemoTRZMU&_r)#B`~vt;8u=$$5Bg& zw;tLruC8uokQ&du)cfJy)(ITP5p@|mSYwh(&tN7zAK~yeL}v>3gc6-!+Hngp?MilIL0VPj?)OsPS-TMnMI z%CenG=K1M_t3W!E?)ZeN|Kajd(-zP-(_&hBFs3dp-n@W;`&{A z`jz3wvo70&?S7r!sxx;p^mN_O_Ax%~K+Z!#ryIJP$8=DC)RgSjwDzm2EujY}FT7ni zoBs~Idjli?*u1kv=_U&y3tW*Tb(y5=q|Nl$w8$)qpd76d*#DpPN|WTy{;4Ck} z4yDn!zA$#QjA=h5 z^_|TJz2mYe3rmg8g}m3+kNfA|SvEJYgGqJ8Q@i!3#4XQd#!G87JkAZ{*&U?S?h^-H z`vl6+r|M$jhUXS;SoEaqS0B&s-=k22E1>wY!%*5#T4KWDrfTwJm(6=Rle96YW6mlW zya;=q=T`8KtnA$1ZDeiz-R*$W&twi6@?+#)uHQ7}m4;H}rojD+SEl?Dq6^MG;VS`+ zeTbyn^wZu(x?aJ!>K@{xmzCbq?p0#=@qW_Thtzjd24IR;2m{{d^w-)*zk-c7bLI{Jw-z^@+ zr&9c0PwrRYpW&<&PFX>>*yzLf;ToBUSI~_ed5ejdtjoTdUpLaqfQRY#v|+iTDKHg)mP|v{&89Jdg zqd+Q-er_~S@{#84^jw^^y}HgHIo;Ewu`X=@L*#rr3`noCCQxigF&o`{=wYpWQR0pQ zlyTdS5B;6QAs4=|2o2=fLzEuhT$Sga<^8%F0SyYd280T zwEfOxO|lo>rCd2hyKDLB9Rdy~oRWVul%i_=4MmPgLV|x+5xzrjpd#NmN|X!w905$Y=4hs+jdruG|FfGvyBFC z0M47`)?M*{sd17~btvzpU#~vzQGBK6BQ@$4OF4f2f-WbNi=qJ%A|jgREg8Io%!N?; z6hNQxCb7ZO4Mc0}$JivDbN$bcq-z2Gw~H|y&G@YsrO7#>q%3mJ#l=&wU-{s*lY#1X zsSbVk;;nVVn;tv?mblQCZ=W0~>83f+S8XB3zpsd}qib)pAHD&tg|`$dgw$>fXm2*q zU)+(s(@^?XW53jA1;4TOoJyTxp<4Asg~T_!Srt$C#~bX)x%ThBk%ONZeM1hO7ifXu zWeB!&5?&O)@5TJ<|Gk2O;--*{X^5CDuXA8XJ$JC;YJ6G!?MZf-Hy#U4SMSv|=Ka4v zrSa3DPa3$jj=7Nv7bBFieuO(xYNq|Ulz@PGMKI6Lrs;+0Q9;WCBVEsN+Tx3COzyr( zv1TU!>nj+|kbg!AI3v{4&hfMrI%i(!fa>TUr|~YO%vBoo%k8@(Bdg&=+R4~y2TN8T z`4e`x)2kf~6es0$bRz$eRij-M=D+q?yLa{19HuK}P;H(;s9WQ4ko<_DewWiKk?=bL zwM_*i!s_3gkX^b!T~W`=Opn~M|r zX@@{uc^PomO3XEicz<>EoxeZ)_tW26AJj}gKsMIjZeTM$|H#WLLfw9RZ0nXSd(pNM z41!PrF5SIQB1|>3k;GH_`aYReto8S9rQd@eh378d1QX0!C2o4yI954L3QEDt@75xl zw~kR+eWKTH{r7DRZqu`xn{2^izEdAduDpl)e}9tYxpL#)y?c8}rUIH_+g_Q20_e96 z7c$!ipkl-1Ao($coiyV4$L}_4=Y`0B-)i0zkx8M9(<3P!k(Fgzc4(r`Wg2K87fUg# zibL!d_>tF;0!Tk2gepa=TrC$3+PuyAm=E{WL|wuViUXJw;TWSrBL1Gp`x5-SXQCeU zb_9f5EIA(t1J&{iFyi|M%Y2UgK>Q}aZF~V=zn=g?+YV_~Vn>q50pw+iyOd{7VhK|`!{3@?U(Y7sb`@wUk zm7huR?iRHYdV*|7AI!Ou3s^bMM$-vyz_eB^A;|ZC5}qkyyQJkuM^KHtx)Ij?=40-5V`cd7_ zd-UkhJ73Ac;dGaByKT()mi5%7wX+VS8XZ{k-c4%peDkQ-_oiuP>7nUoGp4%)C=E?8 zEXSVU1^uOhTXjUt8&5;@&4uGadFw8-6IOKe&(A+Msc0#}5urs-3+BvR;l;q+awPRv zcU_!<;&5#u1ek%+t%%L{nT$}EO}KVLFsKLwsQRwj1=;)C9#c|LKW)3p>}z2&3&B<* zQ~stE|C2MH3D5II+)#SvE|o{Xu_I^Q7_i^kijH+*t%sHSVZXCqA7JZqY)r}Xj+M=8 zS$a3Md0{Z*WYrf0&}gm$4v8Kxp1-dRNqFdZGN>s5bTMqbW_~YZ^5#28uN9@1XRi#_ z#`-XoF`Iu)^)FFk-BwV5&JHE(fDx(WFC}0puCDU-E9}&he#dHYAxob#Q{AegKY#q_ zQoIx`>1Kp8OUzZ7Tq(<_==aowvvLll-7}F#mJU1;(;N&+NJu!$b_K$W*JuRMLeu32 zdcf&!JeuiGJ7R1A|MW@a&_3`tpcBu9}%!OTR=2?^gzjoBA z0FB&hT3cBloKWy*5&LoUK%}rms1CTdHmRBNAZ|rq#!8-atn0|YOeVO5jCACwtdU(% zGEHUOo>EU3P3Z$PGh|mVu5VYX-LWF+JQm4v{CcfeCn5HI8OZzb z8R9SSi^K43)n0lI`s{cX4|xlh+D0X@9IF>1EtBV*FIu;VKDhB)$X;m&DQ%ZyirKRB zi>9yF>M@8{s44mrbML-s$Pp^Qd~)cQxRZI+bE8EdjSRy-Tw2#zghn{W7>y z3vFXLH0nKzd0inTOhuP17l8VphxS{LM)EF^^nT&U-6hz%GfrdQ%;Y5csv}yi@Mjt@ zOAc9x9d#AT`Sr_a>k;#}37o$HY;MjUd0kn5B9m1j0cns)+s1bS!-z2T=~{w<6Y5kK z=+tu#`|;J^hb^R!LN3Ens=|6cb-%dq0fGW4 zVO%QpEd{C;9DWijx*0lRsh9FHW_?26pv&utmL699{XM(HWaZP*DO21h3<*fQI}}@D zH=ccY$~8Y3umzaQV+@(Al-;TO%a5%RK~&E6(w{j0Z+!YCMJ|~npo`sG-S0k%{#-PU z8#AN^n`NmAkRc&CoRUo*-y`8v6}-1;C+X;4#lgDedBBArwf12{ zII-Lq-@9f;NV?_mdvS4B1Q(MJ+4VCexi4xHV*N|3pcisk!c45_pW|d*OaB+Qeh&-A zYKZ2{?Z?kf=B=&N&`%E;d_H3Wov>qN#el@h?iY8nWnx3!Z0R<_)Mcz;p|)oL6_Ut) z9^nIxk6(%(QXF>%16^}_N<8Qg%fQdZ7*(I|m#p$-p+a8SvW@yNLI2urlI)}7aizH3 z0-O#WqXSK*JBUE>Si1WIengm?T}ZIXF0Gf7#+Y zt*!8&%epYUTp}0S#;@eYNrL?0OOl-Kqd4vAAUu|dUo201+%Gso2VV8pJN6e0G{+O=M$x+EItgKP{XKtmpy}ay`Ujk*m#DNtggpDNC4n|MDqmS{nz@0l-frK4#x3!wJ|`dW`@Hvy*5Kttm>Pd-%*$#P zGWv&;bul^Mg3n1_?o+C}A|1RE>C+n3szLz=X=n@Okj&b9&APLSBqyDfKlaJ%{;}U{ zen>A&47*qC)Vsb{aRKHHt;uY+AvI9IiN^&EToCfmhz{qG*2}J1rYoCXloEH7N+clR zgIAm}QH@2>V&(>RYduCd1^J?s|^n7G~7JEau` zlH3^WdH?mTsv8BKFY9FV2t|4vA|yMo2;q-oU8CnEtdNc+iGKjeSe-jB@dt0KVo%!% zoqv3dXCeJY3ELmO_Km6aYwu{jdTILGc-<}W7$tc>^9{24Zs0_WA-}ANQKc$vOz){Z zC>URQ^+Ni^%Zwsne_*&&fwqpzYXnD+Ud~}Y<}#5mNc9Iwjfno!R5^sqmAp;gZqNGW zD(|rU=)8Mcqjw_xa+aK*`F>YdrM3L5oG)PWi-0LuOf$8atDtLYGre}mNBt$DiDrmC z8ncgnakFiVWUrl3aQ)QY>epBMDy3DF@>F!Sxg^p>^SOZo1tWfRA0BVdQN+FNpL!*n zpTZ_G1?7T7gk`~W^BaI6RTJ-SZR)(ikX7NTw=`=9MFRUq^U3vDH6^L4WuA$b{;xb# z?;&&3+asRWY?hC-Q0is$8VZ><^e9~wF%zb#1Y^a|=KkBegJS>G(fWAKamTme!_oy% z++xLeXBrPR#CvO-+(_02?qGP6W3)JiQr{H=xXf>eT2R1MD!_QYMCfUA8uJbhDPfx9 zHx}P)7zwUnCSF-e+c#VLKaOrJ^>Jj6Y5J|_Bgd;Uc+X#_sV+N~L-pB0WQ=XlGNoLH z%tcS>+OkMy!NIGtZf%91?3v}w^tbVTG`jl13x(0nqcPSD>yuz{~%ft8A= zObU}D??NhD2`G=J>QN=3dUmRVrj&vi>pzl;M;<&)E_s_e{H@5HIzL(YfNoWn-p{IcYW#Z=N%y^!l zfmZQ+tjHFs#-l9xa(%A>cYV{VyeQI~N zU;P33e-JcFXOKpit&#SuAI#D;RIj{ACd2BDCWl>GsvX#^`n4lXDYBig31Fn7!vu;@ z;XaEJjYp5|%S?l?GY$vGibkYdm1{US6iS{R;JF|au>wSPC+a>lx%ksA5W^W$*QTCn_dr#onm`4UD)#zw2 zRgInC;Y?PZb%VK~IKEwqpVhc$zixo4aPm!5j`D(;>FomW`!KQv2VxNGZ7@~8)`vns z{?JR)knJsjO4_&aTiNO~DPoBMQ^1}59_DLskC%Fa_c&g6U zorttZ-NKrl;FC__dt#|*xL@zYO4lcfcAx#b%+h~atxcb1iiEP8plWt103?KpweUnJ zC@x2`A4alnrka^}WpTSfH=2r)Mv|L%;r{0d=9G%?fia!gGf<3eU)B_QJ0t%imDLI_{;L#4$Z*6hRz)(6o0ai1%StK&9#Y;%I4+5U3 zsIw;-kC#n<%i8ZqTN`vUmjr<}@!}x_0nLlhvK1E>;wDllj{QxLH77vSt0k@sd?cz8 zpE7lfO06ZoV-;bkdQ_h^aVqp+RiZW=@nfhvb}s1z8dK(h*ZYN{!H?rdy>jmS$%iT_L(J&Xg+P* zfqFL)Ifve8=6N%sVPp}PAg_3NCo(L0=La1(dbDT`hbhn#z0TTrSE3N@SbG@)>8gQd ze^!YKLWbT`Q=&&8e7|_9G5}|F`_+I|fbHbHPiLEDe;#V_Om?o58VlEPhaPUt(ke>E zo-kMZ!IT~0ToFo8;jHvTMR~tnm-z;(g&H5^p-alBIbAEs`3F8C0U2x3WC|C}&mTXk zL@)*WF)GA`o;q@w(gOsgFyoFD8oU6*g#3ZH2S$j~duHlpU!ZbRPf^`_BASQ1(l^tK zi$>Jy+W|-?g_QcvBLz-?34l6~J{Er4v`WaJR0=xK!yw);6KiY>BNq4NIl0WxrL8x-*e&!sTb{w2$mLZZC)cjG##_0?g5 z^5~Q`W%M006|e*^W7nW0I=9exGBmVTjqJd>Z(Yyx#fjs>p4cI#v(BzHA~3(2Ym zQG5VY%S=Dd>bF=t-J$F--ldDa;V6?%5Fkps?*Qx3MN^Lm52}$f|GMd%JRmcRkBb;a z;d=_L0f#}{J#KBi*zY=8ot#G+zOTifIcY((Z8+`bKph*S$D47>V}awHI2Tvbtg&xm zTXzr1t`F%#Z`cH~k@X|)e>CZmYHC4q0mwL?AO z^cFGCsi-k!(&}NQ>E3gjfMc4Kt z;AP=~_p#i9fGkG@-fw^AhAdC-L@3-XPos!VwCk-%^3Gq*$j~b;Z~z|=ytH{cgj9Fj zVTe)>pIS4NST2W~{|?Yu>jglVQP_E5t2&s;G#*Q>csaTTK}m7(_78#()INbIQxmM! z&n6dy0(CUU>;+C3`hc(C%Q>~W>&cL+_jz2IDRFJLI;p>DsVaXS>!oURKCFlUcPw*+ zSq$?ML|K=Hpn%@?fFcbAJc#kgdnL;Du*B=wEnA@Rhmo zleadWF*J+|6Ha`BrZfc{KgRF`;WBlQ_q=)nOzZ^Uf}~^q)WxJi*-3X2rW4{D){#sU zCEMPP&vkEGH5OyNqb2Y#b2cm=E&!@JSN%5+NR_<{(3W4o@q2tJMx;9W%2J5+ScJ1z&9Z^atU%%sES>r$m9hxVk z^rvR~@%D%kgL&p%G%k_WF2k;ugLR@=HG3L8@FIo;7W&7<&;QZ%+u3$RmxE`y)ba7F zeZms=g|15{oRwt##9gLLbr%JSI6p*`kJ0;NG_*@xT-9^rYUr|5^FrfNWyqncjQhnk z%n?Tf(C59?_wdD+(vbfif=od%tJLbbN72>laAP^{A)1Rzc~V@u$pNR_JlG^%&z-zQ zT_^sd$yL%J!9>$F*y2bh4&Ae(8i-DRSqMqXk(drAZN3`6BV#ZJ9rQ{5Qvq#9BG42w z|36TTmD7oSGYHfbXV#*s1|uE@dOLmi2D6v>Z>y{G;==>l7uwDDA&~j)Qf6(m@EV(! zCxrH_+S8Ay$Bw+gI(hs)09%O#%4&P8{?A8kfBX~B0l8uMwFZEvpAm_dT<-5x)K@rB z8i`$d8u`)tS5WrYk6y6lX7w+37GpWJ(Zs!O)M0@g#oyDu+>P50KJC7UzUy)u&rZ3# z;=XVTyI+g2Of>S(EDf4+b~GpXqLw~(=VvT}kp2BHEjvWyQJu453TU-W2lTMw7+1gN zGn7Ui)8S{I9@(o=>h{8NC%@jv?;I?L2A4$yw`48PzA7ep;1t#KIKGd&OU;PIKr!!+ zROT~;E>U^J=la_;RE-HPD&Wz@2nqL6!cZE~{lTtU=etybaxA-8Yz>s%$Cku>eurGp zdkjFU5*&CU0d9dm2bJZxG7dE=M2QzzwD1Kv94mHaYJ%P&k-3x?w znp?7nOpIsC8Hu0vi;#57B~M{A7XVivjIqnn+_?{XOt9tqbojYc^^blPl1?K}JN9hT z-XhE9hViY&zrH-C6#J}di+Z*K=hoSPEv$QAjAw!aeVaavpmE<)IJ9x5aC!`v>cw{J zGVQLBqj&bv)AF5vcbII`c#3_=)uj$ ze|rh;KrTxo_Q8wt6w59hk=t4<11)P~YxZy)K*Xc5ZMQxt?_+XnEYCUTbek(?zl!E*!W7yW!yGjwqplSxyBbBOK zn7cMy%8D*B=T(vB!8$!g@r|T!U-aRR;zY@M;SC7^oy;#&BzXw_YgjltdtD1&DO$~X zTf-06OYLaSIyuv_CHaD`W5^^Jj>)8#5=(xE(EvJmzHUYt-3rwU!c%uIDKQgnOeYT= z2+92~|96}hm2z#HD)#*KWlKMs&E(&hXG+Bb{TEIgN?jlcf$v(wx3mOs|8&QI+_uY_ z;j8q_7gNFckukPUW#6&glm>Sy*^v-MaB4mxd8UO`&sv(nFmcB;NCkPSN88g&Ae*u} zWH#{UJc!y}WSyZaOXHq>-LCY^38r2tJC!m9Xb;nAzE+?LCpmtPD%FWY$IhNT>p*>r z`S4%?IoWTP0XrhFxXIe&5-IU|O@0_?Csjp?H3-G2s5GUW?}c*Yti2obKT7#*9$mVl zt*=hu2&;O%?A-(LwWv%M*PY4Z^m~q*nuHDZf(a1XZ#OFSNPWdCW;|?ifK5#3zy-~S zjr#Ro#a5jKH4kod(X$-uX~KQk%pcCuICEdKobZw^n@Kxf-!YhxA`zY2kE%fW%Uzx<>iq1IrEiY$K7~f!v<<%aT@l=fvDadha=~LUp5A-@@3Q9=Oc~-mIJ4Kk_oAfwg`C5tBcSY{ZCv6O#}7Y9A5Kl8``m;{ zc0Bb75__3)Qu?8pHsMiaANG*WfW&Dli#faLx;s~EA~zJoyhO6pmtR}L>Z+#zaSc1M zj}Q0KW+Rlu>8dlXg}Vm>dUhN!mB9N)xxCe2uQ zDi;mo5pU1Q=k4>D@5%%m>d#ULhoZjC;u+B{eN36ttkC(K;TXE=j3$x3znPPpDcdio zbqmh3#*9lSF`9RyX~2s`o_1j82X9cdEbHagPIe*P^EzUSLtg>%~=N+_s#NUuMS*3XQ(!kYBg89DCN(B)X&Jek-Z zx3t-TE2td;g$;7A6@6Y1`_{EjnQ02BZ-w#325Dq}@DF{Mee>jb0P;jqP(`Dg<2GaE zg64j~k%idIJ0(`)if831hk+lwP_h}uB5g_ZoKyV?+cmgd0tdHHCRDOZPyJp+TzU8;Ljr^M(vR#gMPj_{hromg%{ zmu!?Vqafqhm3`};O-@ypPW_Z~_)H|p|O8-18Wc_3Wbe&XFuA=RcW zSs9!ciJ|X?b?t6(pf*gj^($-z0|4%MH{8o)Txgm}A}DH7YVMRmXI{WN`#n$v*pU8pM8@VQ5l2s=Yl$ZU5g-sXF)O;lxr7H23PYnWu=L;z)(c86+M_+B?xUEPXnwqS5A$KSxZ*>HORg*Rc(yluoZ8G8 zeQ{XCvi;W+QbtMJu+!5@eek9nk{Pf$b~d!>U|HNs&FPp3AyS=`y%lb4EpW8B?>IIO zFZ!kBnlbBZhYmDKt1bu(oW4j$ofF_A;?dG$gRD7F)LQS=dyByNsoJ{40_DotgwMiz z_Kj^&LM~>%z@${Iom{NwF!{z`9MOS7;3Uf~_X=M%a>jipI)kCoG?F!Og@RfZ(Ew&c z@yN%$?SqG$%H!^2BafB#nY-WSqg$(iSq$kowImYc%t`GHUMxiNnlpx`|F}-!EFl9EM9P zuZuh*{1qSV`Uy$GL&W9rZEDLE$sqkYGbZ1^aG;B~lHXNIN z!+(3$toy}#G7qUV#=8mPPSw!=t>INpsgY*a)_Iqf{*$X<*1YyQrF&|0kA%Mr!i^9@ zy_3k)cix%%t57)V%7Ab zq~lTw3xaqztQW03g_6%uEh#K7FIZ=#d+TZrRB<>?zkO2CxHYw2&q%RidA(lUQpH;9eZL^E z;6c$4;G9_YaF|BYQe<45$`tt51T1mF4Wy~E;XfGO;m8q_K^keQDZ~;f4Uq?Ee9K&x zT87}{7nn%lC5H4V=19|q3GfY1)t(MTyN#Wx%X%AEo`lA?3WVA!5$4m{@mqwCxS$=O z%Bh{I2(_@*#%v+4dV{o!uMD}w1~L*JhHzw?Fk-&PQKd{;YcuIl@l%Ra8j3?(g0yR7 zcb8lmEBdS4@KtB`dw`=MeZm^rW5$cKeO8C5n8apZ9+9MiUjT2hWVv@(YjKI`4 zoc)r%JZnDF?%fvDQb|c=fHAtQ7_*3c&qd#CpZfFU5uiv!#m6ExkCQU-n6oc^OI!TB zq}odZnY@ntW$zE2ehul1w(r9)KdnnKz+$_B-0+Iw_9Z$dFl>LFFMc1o_R&jidqzDQ zKcl&A%%}a_%8|c!tzQPS6n38~=Wz0&hkpgOt|#_R)%n8|Y7eOhNnW$xk!2PQY2rV3 zN2Q0f%|QL$6zvwya+aSSHg@*}=^Ht|la8dUW50nA)EKw4yw_#94l}&#tn2A_9bvEj z8=JMS_#xFDTBbk~PYwgz9LC9AEc8*O81rn7%YOZw+c>g?OUzxv9A7H!Y(c+`F2 zo7`3Yf66RTQfAF$+8(>TZE`uTWB7(lI%@Em?(>7X$bG{NOJPWseDN8&DCCiIu_EKB z>Dae#9}6l~2e(p`VP{^E5}Oq~l0Ax`#5YSQ$?YN~oNIp0d7`7tA74Nsk6%6E?tT%G z=DBg=?DPkZ$ryd^Ky+9+6@qORk)|?PEr6K}?>B&cM609}HcZ#2TnCJC(DF=^8c+TB z%)HLF!xxWylp;;;S5W*~jdwUoc(6`R9Z-#tzTd%LFtG14ST@P%6$D6D&@awbnVGku z?+hQE%-J0FwK#wlV|b2Y&bj={tdh<3ZV#aWJk)QInt6vCMrxuR18%h!?!vG4CBLC` zHz*4>TLbTyyHxf+?8jS}e8or4$+I*B0~TZd@Noyjp!g*t6fbg*hfEv%+4NfZX$Qx3 zqRNtsX&r=OYOEwcFbx*QWGy*}gw^!?ZQL4%lFwvq4?t=oT)K%v+*oF5SmhIbYq_8< z;P3V5PEqp`v36Z!_zI%;z26%0j?5mG@1Z}Q;zI!Wbu{T0i-!6_2i!pV$bGpES zjH7YTtt>9n8kxs80&5qxmEy?T>%Hp;-H*4~cd0lNe!SHQ=bUVb8q8Keku_!W44to@ zoiBJV_F~tr5`eGl`;|3HlzkXRWqK~mcU&1af4u#ttjO7|XYQ|9>z=cE9L_3>5yY|S zrVLMlDNia%pO|S&NVFA_N%4F`V?v6guUVn=^$AH5%$|cz+I**1-u|tVXUpfMObWQL zzQX)W9O~k^UyJ{Hns0A7^}X%y$`qu@nNDlACQ9GL(~tLYEdaUmVo0LYZ*$$rH!6RZ zrJIM*Q=3Di^z`?Q`|>+yey;Jdi8)gMne!L8)*7RLOJYmpqSY-1Q_UrovUD=L6v{K{ z)1Z>x$@z2AI8G*ThkqYwei5-Dni=Y&F$WF0H{TwT@I_Z}Q3-vq{3?qP$xXzq8 zQ{!pDvoA5;Ax9@zwsK2;Zq)#Y4j66dpb30rw$S&%%E+Y1H&~aYa>s1b}bfyMG$~(#=VCRr`qNiS4GFMivF{aHja49KGh@83eOZ2M zJoyubX+fIr`hIY;dRwwfa$~E?0nf1)=7Mdp3*1~>_oMJdpeX+LlT7}V`}S zA0eq!CT0rv{QRn35oq$ifA6Beg227?be(Fa#@Bm|!-V;m!!Ae!-#0K1rq!apbY z-%8CE;qSb-37GjB@RnfEs`5&36G@*L0TN_v-MY1UvmEQSb?okg{W>ScGoE?k z<6GAsWjEmWXSq*_a*#MI_NG105ZK%U4UTBJj58U+mfNJ|L##9=$2u>ecHRe|BM^A# z7q}4a2V5}GpNeZ=>&Tx;#6hTvIf-b)Y&}ECq))^*?#+3TpO^FI28wXtW+N|z>QA26=BC$mVs(G|JYOpyLWaX7hj-E=< z;1-TG<>amE(AVKwa2Yih561Oyu6CH)p+l!&D`*g0P7sm2W^9J%R#O9)7Hco#bhOrfoaQ&QP+??ijTn;|X&Z9alw3Tp$|g zFz6%0rbqmhBM5n*^tJ(9=Qi&I<>UlEJasR(d7oFv!ur-wlo7&O0QjsKY&tuB#Z1TL z1^R?9RzQZybifs+`CuRoU4-GvW2Mlv>-6#Y!0fb(8=RQdm4Nw~Lkg+76qT}_5mLuF;W*Bn8 z^-ob0)jQNuZL_0&*MIgJg`Z6r2<_sQGEP4pw7}WI1|DFBGc1tT3XlM@lPj9}so^oH zYFSe8pbRP*(>()+Fl$dh{IWy)w-16&a4Y;3PS@yD9}MF=K@HmXTb`%YF|8I7y5=+U z)h1hggiEHg+$QW_p8d6(X<|Qou-;4Le%yR(1E0Rp-m{z4&s;imTDO|n7ySIu#!>Q3 z?ApByvU~PC`>cI}xb~4Cs_1X%yVO&Wsf>tOKU{NwS}HcskxfW3U22YKJmE$Wf^1T~ z>z=~cQ^FQ?alMoHVPmfCQZ6Fz&8rTMtNFJ6E}@+`d404_#tM;IcNV53RC7Zj1EjpL zE1gB!?LcuAp@dNiwPek7E^!<)UNx(UQoZi)di9SOFt5ZB)TCDF^)q{y>43&kA+Nsu z{z2y&!5*qI;M*>aPrT#^l$U-abvVTYKHkw(3#1f0^iH4uI-S`nCN8dpdhiZB@v(Oi zBpa9g$srfM>^0jFR0z0XyOm&9iKSNny#BpiC&_+OQ2DwchldU|-hl58=7&h+klKQD z(eKr%6DI;sK08_Z&({~WcS#AmD(l$rY%r0zgZNJsXU7F8ELj%8ybVK!P>M3)=3Z*Q zA1|-!w?UV{T(BY8DOOZLAb&Y6*anQI9#rm~2xhP2dQ{W)+vYTJ+-yo+(dRm8rKd*d zuSAc}T)mZ*bq_r~eTqTpda88m@hI|$h_>4>8~j-JYX7k*QE_TT`{_p1d9PjxUF(jf zR-~{Pj7omQ?Tvz$T7=Oj%rCbC{RxCq)j_JAcdIA&kny~Y_^P{OS+< zt?_^=Ir6t%XV!a%Vxa3F)aE2*uf=+~P=bgNi(HG0hAUy%Lxlb^f&F=+^!fxQg=<-~_XT8zXS8yHql5@sxdEZ8bMLZ0xZ8v?Q zx)giqJ@ApUG!nbCY6a^-6&vcFUx+yHb)pF%!Y=sNt(lSDvGq_58R zK5J@pJV$n4O2!m{euj7vinA1?lF94!V;wwX!QtBhr>huxIxxV#k9V+1I;gE-lOD_= zfPeG;9eply$YC@G#q4^I*&cOie05RrmOa+O5P)1IU@Yzef!Mmu1Mm$IA&QvQtC8_x zOZsxe!wN&%lW06BjJsT>hxh1Y+3gz*3fYhdKu#O(cHNK4RT@l}5&&hah)ILyXCj$r z><53^(>@HnuB?G@aIh`+987DCuNGj|JEK_U=CPgdCjitXWX^-WCA|4I?E_H5C;G!( z@*?T#Zj)^yg3sxD5q%KX$+JR}a3mrUh@XS6$ce~4A)MD7815KlBD&%^S|jvm`N|DS zuUbKnH9$-x1nLZtG3Fq!K9T3<3{I)8S0xMiY~eM;X=IIbX(pw=6GE_H@jrI#`Kdib z6iQf{DIW7K!plYxjuE`>DQe~saz8&ES{pG$K3|YDHPSAKM0HR8pc z^cQ?2*_2a1^sf8n2&tMV)AD1eBo^=6+H0PN-eyC>$lp5IWS56zfryMvp~}pXn)eT! z)SembBB-X9nk!IXwK-aI-cev8F1axuBqV;ESYg<|Kvt74%=_5MyZn_IIeJKdfjP7s zw3PkIc|ev=LFL}0Hk6HQFB6*T3a=WB8Xsyr4DH35_1+h*ekCJbcIT&>2Tg&*|105^ zni2S1Dj58Po=rr|uY~nw@g_>+x_}qC)DXSIJ66$8<)QrFLeVTIvox?xsOLM9x5(Z^ zj^f>cJ&~wu{lW29hK9W2aM%a{mI!fMo>+G3SYR-<`GG>A)~%Vqpdg~#Kk-uUEsMAq zImXLX`Mh)ph1g6%Ur}CjDbNbhcEamB=k1&UW7S}!W+Gj`tSdwyQA(jYmM3mLK4lox z_Na|KpX<$Y`K|J+H_S^=6$KAk3Hy)%7}`d!dm*(njI0TRheTp~s<9ux_M2nWbq_`V zI4r3ts5ik1V5-NI+(?Rz66T;GVoE@K$tBA0_VVSH!@l(4exg$ zn204$p}AjACpIV1WlCb^Oe5UP=JUA7i1;u&s5F9OtT-_RMfhBPcRAL~~$&c*l5Br3OLWx@J`0>8e zV(MyY!N4|Dq5L9&@Pfp9|M$Je$#p*Dz@|!=H6+p#4kaoaFm8&qpK~humJk3gZR3Dl z?Ryfe2fM$U7iuuw!b23^9Ow6Eyj6ec!u13L)H`rd zdUuj_>ya*`REXi&3F4D_SBK6w965X#>O|+v9=!`RB|O^roOglJ&P(54`3j=iKnz1^<{Bb%8nrR!J7En)CUIA zMi~T#UDlnC@1Gzw4ZuX9;JlQhxckfg){fiMQ>F}3*-sQ%ho5QI?_lL=cr~05NWd{P zKwN8sm9oc0=lmP1hp$n(yd2v+1t0d9$5gc>`9_NO{`^YzlSD)TJW)bX^2p)C%81}T zj-fWmy>DN&_|h>FH$ophH$EZ1iO^8-p|hsd@F6Y(_}z*5lCZ)Z3b1%Atl*D5l#Mz6 ziEvD@4xgZx-;b`PwR#JHRIIYk Tj&Tg}K^S1r*rW87A4<+wghzLpn`t5nF5py*4 zsO#EG**@3|mv^?vFaNWMs4|{KCL#zTS}=h(J z!Ey&`L}j@bJ1@Q$Chd>WEm9fL=lZXytIrpC;k+m*oGQP1LFe1s0OnMF@b1%hx4&q?WylrA49;*5_g?SB${igjfY$u6OYYogDp=vAxI(@1vyf%Kv%-MuQFX zz2Y{VH)TdPk$-075)nU><7AytDhK3*+tz$?CwU+gZ>Ot25LtynPgHA6T}9%XbwUb@4O z3p0y&1T@=GB-Q;JmLfA`O_`?##okNrE(1kQ5DVS-n~#XbgjZWg8)A}P8rvA2V)NE~G9b!dN2f*~0vCG&EYF>LKHD3Mw z0b>shM(6=gk$4x)Py%2x?eh8b4$KC@9_s2&eVeyrJ`Wx7azVr$1}Mm-RVn)V0QFDg zhwM!rI(Xe2CLtFT(&a>|N`hw?LLqNodbW>HhJL?Z-ql_sbF?}e)H4Ij^f9EH<(z&8 zzb9v>E#LG0S-3Uz+`jn{o+GN|Ty!A0?*Wiz1A0IzL;DVD#7UWBr%pP5R*x4$h>nh< z>dX0C7ES#Y8YOR;kwo}`+8|gVO7uTNh5AQX_VwOKVP6BuE(NKvwOa8dupcyoO$ zSOGzp@G4mFy09DznS5d=bxs)#>Lg%X3Y>p8;nS3Zxf}>i63uKd`7Hb8PMgkSisKXO zJy8q_#%YVst%t(7V4S|Yj2ni_jz^>P&pecq!h5}r`A0=v{=Rwlb!Bq|=!6{(ivZ(W z%>au9VSOD*G87Mq+OJ1Q+D~EM$>hCm-8$)2E`fo8ZZ?TlNB&G4lf*L!Xfks9itldY zh27fNmS&Tdad0NeVAu2qG@yPO@_@0%R{;C_Q!oC#`|2Tb!Vd%4`Qdj2^`zkYA>#Rg zjYLIW{wg1zgwf7A5{Wgs>xBZZ8JeiWA#ZFmRvni}1%%yj1t(!un`JxG6p%(J1%(Qi7Sh z2{(>xU7x9e-W`3l=2}T{Tmbb_mJrcx{ z2ybZAFAKR=RAG@MeW}QFz$m&l20kwyylfM!AYhMEM(qErx9L1?HWB|#i1)8#XZT>l ziyT=v-c;Mvd8_Ug#J(&CtO_b3ff6M3-Fs-4u9tj1m%0o2eE>S87wAnE$kKVT0LVV6 zZM#MU08+G4)%MVaioPPoE8yE1GtzRhnnwn4|Mi&^oYdB2xe%DBv>MAWT;Xkeqp1{l z??^D-?iJ}Cxuym(wVF0{R?vNC_I4Nr_EH9ey$TzYYFaH~1ze2AJ2&EG0$t}YWUhSwYe2Ai*hlD0MCprx-I(Gx?qm^A4bTH9s zFuiJpw9-RDnEGddNCw=~N8|0J94)+J(gj-%8kqULX&nJgPox23ZM->?desLw?|GT|sj=(z+k388P1m;hV$%iQycHnCWDHs8W%A4KC z9(dW#zmw101z4TVvNP=&Ls%jOO=on?Wbo#HRxH^w zp*_Wucywq6hE4We(;MZzjNs?Ow172 z55zT+pn4yt0VgMC7vhlB1zOaH`*jk4H*wVw2V?O7%Nj!Aj{}Bjr1ie22;XvvoFMkmNQ z30@oh41+D)l)1?PUJ$cIiV2s2EM?tO-rh9MARHao|FDdL%T-p8JJ|_vSZvV)Medz` z)6pK4hQ6Cr=ESYOda+2@DUo~>3vwXBfpQ2vx`62lnQs+Iuj%}XxHcZZ zV&xWrFD9mS#DA@WTT~Q94{D$~H6(ZD@ljT$GYQ06HRASR)z)66W0u5L0=NbzkvI~O zEIyI0_;OE!#DTi#UDWD#DP zGQ%Y+VktZWoXQZH%AKvtDV>RLf;i?Hd zpFK5!OSNhXMU~kZ-pL<_h$rVSD$5#5L>uu$p0#wQ5=&2y+AQ@_@0(t1`-@}uhufFK z1p+rLwr7=`A@#zo2iZ>~++TRSu0(jb*N^v(uTI+)#GN`@oxbKi5kN_Mqih(A5h`xk zuq2%5Xkh1X&nE7zys-$19P=mOK#~8Eogrb%kCQFVjrJ3XQ9c@hZm$AP0k$Wei@yrvRa^sn*=`kW`Z2AwRMx2@`->U2 zkNxwsX<}{x9EV#h#u2$n8n4c}1vk4FtaiFIp*%bd!_opV3t5`oiOquaK8uro-|@NS zzvgr>)P3dUAzdpagCfX-yL>_)cmC(MwKYK!P7BxUE@ef7GrNT1c5URl8fh-6EU=nQ zx}`%j>u8V)`07Wkvn$!_Iv?;bUkl)oP8(7{fTwPg3S9x5NQ_Bw$~&lb)TgE+D8yzU zTc&*Cv#Tc8ijW1@rhX}-T#LueWC+s<1aDc*^!6kgZ+_*Sjh2ZIn)dW}h)+E&q5jI= zN0Rb`KJT_LD9c(l1UlQ&O_n{dZj})vk&GW6^z1}cjh&z%aS{BPPK5ln!`a{&62uLX z;bWw04So9h*iF&Q&LCT(&S}+_FaVzaGM`&xN&bErsly_CeHzB=13Lp2%|q1}qefj~ z`ZxCkO%FfgnbGI4ooFF<=a=F+;kX;bWypKikZnkajMDB_%aKdEG3zK_*0#ToaDaHG z6dnT?X~inx+XPb_{q>i&=Ag-oJX!}xrnX`Mislf10$WDYBnS%x-p1MW)ewR1Z z0!MwfOYOpF4S}n(GEk0-=@IaVB&7xUh&-bz_tgEGrE>8gPcWE})%WTOVY+uT!gNd6 z6}WkIdsRb%*#pD8e$UT{UzjfQJ3N(GGV8uMnd3xem*?WqeVRK?^H_#WWK`9^ znzwVkb%QCfr}tU|$Ii;?;*sPB2NW$W!Wh!m)=Wni+XgcpP?X8Z?_RsU@t|r-;*=Ce zMjo$25DYm{?F&RLQpL^9?Up^Xc?8`nL6pWH5Wr^~wMREk8!L}xnGGR}@pQ4{t>8p%hd7@C6h1WzTfu?9r1E6D zwA&ft?eee}lR6^R!ZY1>pZnzY8TV=HQYI$8eaF|WXN^#Wkq`F=?SXtwE*>HIN3jCh z#b+}zwI6>G?av>&s56yXAK$WA=#kadyx>dD%E&`-M8f4%Wf8A(-WMf4U6JGsC9Hei z-l5sywtG0PW4AXvHEFK?BH@Je@hMyw`wC%67kx zS%cvcrEf<>_|dVNCb8Vkc;VxHJkEUwX-fg% zd--=+9WWRg?ilM!mi(wE(a>H#JD2pwhbW=KWA8#GryN~-#nz^56@&Lq0jp~@dFWpo zocQR=v3ffrpJtV9{$}>5*~0iQ0=JcOMCRT^UE_51497Mm$0O?bpo;wo7WCt&qw^tE z5m^*C{>Zy7Syva4`HqMEFuD^>UGW#{t6UrUlzOeKrAv481O;|pmj5Be)6f|o5^NWi z^ExoHk6SRhw8Snj%pwE&pu7IPH|czCvfoE6`7+8^Be+eop``HR=I^0(B3l1fF8yv5e|FSls`H@j$PO>H7QPEJlf$Pj%qyIh-&vfb1uOPjt|e&uc63Gi97vcFVF*f zfzI*mtFP@+R=T=ZjfP9^N$Fc^eIHc>FMRYU$ze`2H9yv>qx~uE*a5GS%|93Q|zptPRWy@c#Y_@5hAkz8t_y) z@$A}FHo<$6lvaz16pUNOLD4D)SiyaU2y-TmyM4W~qLUPQglQ%Z1_cEfB3|v*?x~8g zOm~NidnXps)umgGE&kND{8@N8jCuL5=$t#gWg#4my?Gh&J)Q9bYHC%bs<~?$+9J6G z1IFcZ6Im(VW7C#?8mC;M#KX5oEC0L@HY+!t3vg2Hraj8A|als_oQM@b-?D+I{##M&<0n zS`Wb!>GuVz=qQ6XM!sL31FFyk*`ZiD!7E@;a1;?T50=aBbcA(UXtzKVIHaP-G)Ic) zh|qZ72D01j+4ihQ=Yi}JB7ZC@UTpCAhMH^`(ZO)Z9J2TC{cTK=Ds7f-K6yLd@!l>Q zX+Ev>fq`9a`YA;NHu__KH)|&`1+qL{m~WqS9++)!{;oN=*zQr;tmFG)V2S6gTULCZ z`*%3^4W5n|Rx1luR?28j2+C25-pTFE+-5kWz1&Po68;)y4y-}v+ULc~=c)!j7csQa z(Q?tf^KQTPPIi6=m%8o&-6^MXwe?l%G2*j@Z!K-&g*iWRAMe(l3f3+!lV;``<=|L0 z-}z9N@6*ibLZm#$II$KQ83=vTXYuizt*X0AoYdue_a zGwu0{%{(97&-jhB&;77VT9Z21-J&z{ZbwRI)wQwdhr(JONs`oUCUMVOFPaPCKU}TC zTTiI`z58bHGu%KLvE)FbQ-Kxxt3xks^J_NT=d-zVt;?GSEy5Ip%0mGNQP zWksTu3PHksA^oosl-6ge9*J+|Q5V{!o>HbGv@6;?u_IpDB;Gx#MmlU*_ua($v%^yw z?lC{u=JSm{+nQf}bef;OaPPNRr+pAJ3K=ZK7(Na4x}jXm-S;*{@si z=KKX0&>D@BdoPFhvW=}UrTo~RcRQ6IbeshK$5UG;u21pQrNo{stgT#HbK`!*ZvE-b z_~Hov`(r@*U+rhav)S87yO8os`J{AzqzZ*vFoEV_v6;~8MnoITgL4w>j%_$D0FLVV@!BH zt8QkPF!g6=*zB=-&2%!8bG}w(`&yd^f>GIVS}wU?wWqCX8}>-jXIc%fT0wKwV{t@h zPl|p{$#qk{71jz}o1Qivk6T#AVb~w|i`FZu(9t?n8|F7-Ks+M7bpaD)HDA)m2k%7% zNnzJ?d+Nf98Y{*pG!Gw5Z@en*9KmzEPxS4RIWA{@<^c!Kuuj9sM!4IBZ63aMiVL~e z3qe=WK-!JBRUVA4VvPv1lK5oIDDVDFsJEf}ws(}++^5ZJ=saf?<%fozg-vlP_>OGw zk=e8U-lDu#PsKzPUlcuWLbA4l+L!*3f>LK5n>C=}_#)Y%kCDQ2N0d7c;WU5gg z8*_Wa87&=d8WeDUPA@?{I4}pjUtW&Ar%}A*%j|uj@RTrJuF7k}V1HZO#BAk*>V@lm zl5jEN5q&;DFhY0hLSMrmpek9dN11LCkuHkoBqHS6wd=suOOloBYC|O1a-WEbQdSc2 z9Nc+YMB~`R$t7QL8vE({#F=L-owdYD%O|EX0>k*q3loF~#@d^gP!=BVImHS1$MbuA zdt`n7i)I1-bIoTm;xp9C?D_oq7Nl(qtzw?~MYoA=UfGbDUFB(c+e+NmAm{Fat|`cwFUG}CrjnTMLR4IJ(H zPA5gZM#@0^DeFgCvrm(re}reS0r@wYS0DF_S3=0d@32k{=-`jjV>KO#s$mMT&7JZ0 zwC+6(3R?^bJ&U4*cHr^E`)K6)fDV!i)(b$4+xtK_Sk^_{8Fs!rGyy+}zwgI{xf5 zURZ|u)A(lk5pOzNPq<-V(BIkRW82e}e67)PE=Y2h{3FiE#Wqjl4+nzeE!K-9DL>fY z6aQLymGM$}p|U8}@ zhir8PiK>Tf-GK@=(=2CuR#YKGXRT$HR^CM)`VMSxlQmmPRC4en;9LjA|BAw^qTM z(&YByCHpcv`NH3WL+v|V=ax|fO4zn>!++Amh{okmM(vP&zMlNr!oBNWE-;NgR!Erq zVZrmTlDK?aUg8-=MH9XExG*mDn(uY=KyYM~nb>+mr+>xgij5Sy8zWt?WAxYXY~)LA z7*ah>d$wG{@rlk^39&@IvRkwr(}Gy**M0hWY{d3|U{-rjF(vjoJavkz1Zzxqe2T-{ z1GdX4g{4p5jHk@?ii-#N@0`(p6sKK%P)@fVwcc>LNa_YM8qG0^6leVQh9vgDY(2dy zYGI~O?NgB##mkF}-c>`V3>%o zUar-b1<@faZ5ei1oc?d@pE4!)yUHQSk&;HGACNnX97z5*{g?XN%XNQ9!qh+ACD-$> z*JbbD*16Q*^*H}a0gNg_{`1B!biV<8>aPVy{`dh?e-r&5zUqspoD*tKFgATs_v|m# zG1T`!Z3`r;9O3}@q3%#EuIm7Dw)~kJyJC2AjU1y?rQ&>j913xF!m^TQ6=i; zxAAM-6ICZ5M<^!;M<>)AfxxZWf4PD)VYF$-Kc8VddO3pqB#2xnz4zH86QC^#m6IlE+3HE!nFI_HE?>2Y|Hp5{`1f(eKc1p5B+NY< zn-r}iNT`d4678D=f8cIL%ga2WN zwIuNeavA`2pNE#u43;);}k#a?Yd&=|Uu}3PqfQHTtzM*~(^{p9~L)a5fy@*s$&~_@>CVKe?tge#K z@{UBj`O#DoDfP|%wRmF%hN}oXB=|jn^lw*nu8fgs=QkUX)g}ERqe%gDtnbx-+*`{p zyX2mr$apCUJ`4Q$^Hb*{CrO5p`pX#A1-E+6z5n9YjdzvvqS9R189^-xP`|O!xSZ-8 zq$Zz?hAm`f*t%kfDF1d3%Elu%4Riu<9-^IT6H$&7{6yk8WO5MI90B^S2@$E^_!RK= zdvhkRSP$2j$^Uzq=`mF@rFBKrS-)xRF< zzqt$jTQ~5gIg>YW0f7mW3J640y#-9KNG$=njMy#4OmkH(9?hZtuibqUrj>w0WvvkRDnn>0K(jl`2%%NImt=| zD@23YFJNX-ElAKcqauI*PE4{OKu+V9I$}hEEPC7<_%!HPvS}=iP zmU~uARxKhO>8^?q%ypfv3@D~DTIiAwP+9OcS21(Jj0l8>(!DOQMU-KMUtt5NyamdlvsDE@jMDO_mlP0*ohtEu-6weM_&gdv7b@F*1E2Vn66t}%L4Ql9FO+NTi z_s?l_t6Vl{&vUQrg6oV^OQn7EuqHN#7`8bK4;z%$g`a=gCh@&43`rV3#dhP}x?W8> zN%4)wf9Ol4TRtIg)QFbo(kAgv@epwtWO{OudULKay<*45yp9<>XZC)`AADr?Ea#a+ zeNju;TYl&;P_7g^?n~jUYZJ?9U&8+K=XP4Jg7S^}?eEhg<<2~iy|UqMr32x5ah`f3Eot~$pt!Qu5@s2K})>A#-&1;;0LOiiH-UM{W@9GjQWm}jQR z_G3ilFH;6tsf;;U_D&lUnv24bEoQJm6fS+teL=+PHnz&eB+bXZ^u6E@1oFroKsSxl z9B@OQLS+QRyTMQK(>GBG@vRftH9pL(uHS=Aer=p?j}{mj`WG+8!_AJ=a7$3P$dRsx_RloWc(Z_Z_^<;ZvrS)g@6mvZ;|f0-N?|!j zQ2oE|*Qq+>-IyM1b`O^gUNi89>T6<3;y#fr`mF?KLEgr0nW&*mN?L|UN@ zOGz^#PjiDw1L6An6_f5cL_U537D#rBbUjuFI*BEZCp!)0%`~AVfom$1=J$By8|Y6; zz^uDawe4;PV~$7mpxe)vV7V1(W0>%g&Vj+O8}0F1Zgm*j%oU=ld)rgB*KbLL99;%s z!(hCHq7fld3(X3@;EEbHfqvo&VeJZ;Q73enf#6D(q4o4m5HZFZotMeJ`BZQ$e>aY- zGXp6|7tr$V6-CfhbX{4oeQ$HG(jwmc<@t#Id~A$w{d6bLJAbI{RiO`e|D^;)2E-t$ zU>cGF^w#~CIww7b^KM`U?{aLl51TnJ2)ZUTfW<7+P(BcVipbp>9q{RHG2yRk$t^!a zerN}pGcFUl#%KzeNJY*C9jWmOVd+(Cum*DnpfY>}I;<13xP|J@SL#SZ8z`m3WVRCF z=geSg6=A%Gf;V8;pZ(=CoMP0SuK~a-gWW_YT!5y&aW3)t@dK6rQmhmFX!V>N!90nY zvr}43_Q*Hv$MgOGms(NPqpnreJoHSB1f z`JTc(3P!&~j%~)PDc_V8hu3ZuQ6jb^3pcy>XtckZn;vl#fUcx{C{N;A!Pc~A!gO9y z3jaLni`nY>vOtK|oeKR0Ypt;?YhFr$Ym;ALU~q7!12ld!AA+Bng7C|Hx3iK^3(M+y zMUfeM`bG;X@4k5v-q|&eHQ1O;EdP827WP1X$-U;rA>!VsX||d2Rr2hOU!YdQqVyTDNOPh-m|Ucr!n2-nJdC-&Bl@4J-sWVeM-xg<7AhYnwltJk#qclL&g9OGP zMyHU;v-H+3@p`#WuYf{ST3XvNjb^I9`z-W+8WSce7euI>?( z^Gy?LukHeYys>vX-4SwT!sN_6@BcY7!(3~=JSGQ{ksrxJqIZ|@9atShj9gLeZQy5OS?Xok+q(loa@y-{B0-PW zGE!(xVT?5`Dgv&n?bv5WvxxlXV+w7I>|Qdz+L3pAeG(Q{xJZV!@#_)jrFbhu8c`8k zF>6O*Ph5Iv4_}A6-cN{vN--yhZVTkPAd!J=`vD`Btxj2Qe2+zfPn7v_)_N($VOHl} zn`0MxK}I#7%`gpew#>Wn7u3riCm~3}TIV<+d1jdcv8-ls3YMLM5XZ|BKV2j?X9E>P%c{%uQ;lxxp0 zgP_;uwsyMzvnrM^ZbGgnlYDVuTE=*hZJzZPF$YSRdSPV)MzKo(RRJ$$JT#?oqt9Z1(-@PErONINl+Cf@k@lICP5t`{dAdw3P( z0?r`gHI#>zaL4JE{IpBX3ZaDIfwHk+?x?(Ft=02+f} z@xS{<<@yq`n9)&xSR8sMLL0YYhuc|WvEA_uXlnY_aUY&V`@5G+Z6gS~ND!^(&icLm zT?M;BIyWME1dv8VsIe9u;k?WtN>@Nq(eYWy14NKfDTZulC%nEESgxheCGzZ8kcujQ z%VI`{DTtlZzOQ`xyuXBC9M%+H&2k(s8@!#EQJgYNrMK zg8dBO8ZXA!w|W@r2!8ieHlsZq;p>vDh}r|Y3%*?;fOFLo)FL?0-fW4K@t1d)sOCus zLX7CehzKOp1442GtjyiQJ-fHEtl1{222TiCBAD{(g*S9drzI;%$lj7}< zKui_Ti<;gU=$+xAb-Ka^Y7Pu{7tGT!SlEWI-vWOSu~?NQo^%agiCTON0bO_(!kV8V zdkMT#bBN)l9?_gUybGo!BL{4p1^ZeuQ&F-|1>@5Yvn%1wjpy*IAoiA-@Wt27ava(I za{TXH!jwRU$BEjAkY@oMZ<428X^f1gY!x9(qQdj%xPH{*!a+=29KB3BAu54pF@R^U z+6&OS6B2S4&S(q)b?0$P1by^rO!8)(?R)jSY=gBgp?+c|+Es7ZI!rcntW6QNS!k+6 zbqGa9#MwHoa`Y&XxXTkK7-G^=^x;DpIwf%tK@SQOXF*o=*PnymUc_9Tna1nUbc@)# zZ!~RN+rL?n=H2ng8;vGjnAKXWHkoKAKr;c8J(g`XFw>yqy2B`!&-Z!H3`EDJM4E=_ zyI=0z#?#t62qzOzZnl($RQmSvmG_a0b5T27|C(h1((o~u8S*66Y4N&%LcyFLSE>4J z`+E1uSCGJ;0^41RFijed>aH5tk9N<_S;aXcd2=VC&Ye>9g}=PCX6$aA7)RFmM8kjy z=elVF6e0*cD+DLt{Ml_DxsDK!?x%k}59P}+$UUuN+Q&W5c{|jmx<_4xjd;D@*|M*2 za70CSyl!N-d;Ynqr`gl@+G0X)R#ye(@lNC(2Vyp^RZWrDFI5RDtYs*SIr|Q#+gJB+ zKTpn=oMkb48U=#Xy`D2{o(Nz&5xPyjS&OH-6tVo4Px1^u*+@TKigYJDXma5hc7S+? z&8E$xAGE>185CWuA@KXcqACa?negU;b|`49iEobSRCAttw(UJAgAS+!@4OI-yl~*= z-MjWlyI}-$5gC@n$Q(wgFE&)kjL*04t77~`FT!&0p5|aJl3&~G^R7bZqnWCXQCE|g zIyC}8X)MqaBdr)!_b8VabI%a|d`NVK7ZfWBOQPk+j~_S6Mq(`vEBe8M-7qwYO1?44 z0yCBai#m`}759IBUK>@8jk&@lHyN39WJmwum}R-?Q^+EoKrB$p%Hq!1 z?VoQ;I}Z1nB@HOv-P-j8@R3%1Lt&F{d&VxfZI6Q28+RZfZv^2WE*We!Z25y$+vhNl zZM0zm?F(wVWJa}NuDT%dC=F}UWnJr5)l~sVtcqu-z)M3_$aX~;;rh86iFA8sm^-IT z&Spz1TINoMb|9}4Ezimw=sDW%2_xR)3Fk@4kp$8Dva9y#A)_zyqPX2aW!2jYb19Cl zA_Z-6bt>6DLR#X8a#lqqG5zQ+ewDNvI_8V0Ty4L0Xx_RBih^P!WEEMMJd%WY&9tG< zWj-J#(js>5j7;SI0tu|Zw(}6Pu!3lsN59^N<3Jr-;q0Na*KU$NboKzOZHOlAu>Ygx zzQcZK$E=k)jPjl@n8x8Ij~Sdi+8Wti>Fu<24!#IUTGC4M9w*{jw}?*Ak! z%l4eMLZ!+S>dVTWFRg3sS08x6o`k)_7t$mN0N*ECJOOdCuG%bMHyryma#GO?mf>0M z3#40T>Rno9Yin!Y3JD~j&KCQ3rrtkisa53)v3y=w5J7^!=MWkAZ-B9s%O}0VU+Ny&*yul&XFfKo4E(e+C}IVHv>lcU~U!OW>P|ml8Khc-6HLY>%o9P_MlL*XOS7B-rb4M1?AB&gNT@f z0d_FwZo8tV!|jNz)J6tobFx?6+&Jg5X+nfF>6AzmSpa!z zT9R^C0CA`o!>+HHNP0(ooJ7KJDX@@`%dj1MAclT@w7 zm+DG}=GI`)T#;i4cmF+RFluTxhrD$`Z=Nq$m&roLpJ7@HO?4R9V?{K`- zw53^7FHS}TsMSD@U`TlQ&fTXH_mUl?zznXX3lNPXFCVvYrnex+&q)U9={F0Dcmd7{@d z&dX?2{dLMWrnNyPw#wXPgsgAg0%Uz7|7jp4-3dww<5DbE-P-kRmVNV-$xbBYf65+5 z75%e@I|L5Y1vuRA^>eZ(<^twhP0lWoT!uy-9b!^)TQO!T?BS)7X+X#r7M9c9GqtiM zpHV2PNb@~bfk4sV2!gMVCZ|__x1;HMW8G0PL`3ey(cGxTx^Ch^9S;`!BXaW}BJD<+Zu3^VvA z8m`WP!9==ONa*Nt%kvr7W}2Qim?P+k$g&*xDIrmNuVa9FgUlh^#UE_?1%sSeY`AD? zye0zJ5$b+Il}q!6F7`oA)nY4Foeblu?S9RfN#hPhhDTl-U!RgIQ88U@cJiD{?EbXG z8!~6&l~Yb^FFHN#PW+0G$R;F6$-FMU<~#A)<@T9)L%2m4x#$~&(u$+5HKuI!gxlvJ zh#ihodZlHqLXUODJaYt7=4O`>wHxEI_0<1%>C2+|C3VAs(CiG>?p~JB48!tG$x%x*^R{yTlgP`}SiG`jDv1|8Va2GTYtxx^0DLF=O?YkD`aZfwKZO^1nkuhL~IY2rp+k|unB$NFYu z(1@_x1qfmH_ZNuLB$usoT=GctHhL9h{d`B7HxniB{@ECp{-<`H6}+%0#1`uA(2m}0 zPesruntv_aMJIeq%FG^InFX9dYAKZ;CXgG^JDEnHkV6_Qv`G7bvjDaZ0K?F$pO6^c zD}KQ#NUUjx)0-69E}e{tsBWE|-RXY*#gU2oaTZH;p=h;@Zkv0*`TyopHK~{SiR}kr zu_E>=0XW}9pf`Atlz>AO-P~IuG^DW)VXg2ZLHkc}GQ0M_sb+gjdI!Npsxfnz@QgD0 z)}CZFoJC^ubK=)X_K37;)9a`7Pee@I5$aU<**6r}0Z&L!^m+~|d-#Jh%TkDHXaQAI z+w6sN*O;Vw5=93q`Q3-n-_tBKO`)7LLOGIji4r?;_S$DSNZ-4^!?wF(v~NBi#g+uE zIf&2=kMKj8pHX@J`)rdLr4{=~58c-~BqLo6oiJ*QeDPr-Z_S!j8+RGF7?_l9GxL7D zuupbT=@sm#df1dwz6Yub1Yp9o@%yoepUAT7<3JtO3On9Y)CEJ()AB9Ng{&y%txmp> zaB2%8%A&EBkZ(?FoFhd-QR0e(Dze!71yT-c*!sgA=DXrk|8*Xq2wQS|4IE|&pwa>6 z(&r1eD^=Ch?!+R)B8^RY%%Q5bLA!<00Dpgf{Y4aE4#0J(lQfZa>(-yohHlmuyGBB0 zJD6M~#?(OCsYm2+@{#tyTy0Qf;D0aPKi^dq)9&01r?V)de2l7l<}b`wZrG`}Jyif& zVuJ`CY8|9cOY z%eguu{7LIRLyVqAkNI3@-<*L* zjw6b9=P}=bP;SUXW#prY80+zbyT$f(k9|L}z#W$P{gd2Xi64G!+Qa*3fY2;}@(6Qo zw^w*9Fg?&vm1u^X**D4+6=6fHwPTIOG0G6LlTP+RAJY-p>?Xk$=cyMbIG&b2f{meA z4V(rwZyi#c>oaMBWzOe`JwF^C?9qtVCU)O^iGH zZ2z2Yeoolsm~ICQE0hrCRM?(LP-7H`>p{HxAAL330K`~O7(>|G#_A7AeK=r~JVzSKn z?1jhr2zw$Jt!_D6cx3KH@0?uw=2@13Ia0U_W)?OIpHm(}kx?k#bi(}wr+ivt?)ej8 zQ80~1Ug`CT++Zb!aOWam$FGRbus$|#Mp`HfyDpC<|V6(tC6#H7_=qXmTRw5_`A%wT9&QXtWu|FUm! z_}~9EjKA*)e*T-3V4sx}^z0S?y>l7E1eGOD(S}`F32I=Zlp7coMG7Zh522PE2y!VO za0%>JG=Tik1o6$M^1Gcj2UvV%V`x$Y2_PxKF8H{u=l@x`e*ej06+uoYg=(iR%C;^5 z=|cW6$^zSqw{ci@iT`$wBa*ovI}}ZNKvBN#dRB zMmHg1r8Cp=TvZ^`+a?xTbrm%i@= zOnD?>%P8wqTb1e2_8Iu*GIw;#^1hM#EeIEKGqF11-jxQI zsx~Pf=y;w=Jr#0%y5W|R(dhLI0(Hz3Awc{Gi5%BRKgvMPu{?%dAjp^ zB**cIcjN=q*M-n+KGW6KDTqpi1bOmDe7lr3iN6^wG#oPCY;pVMrpz~4N_|<2JZlDS z20!6h@BHwU=)XSdrOVnfc(0>qc5iXu>fkMhyEi`Fxjy2W@UOtC+k=Zb2A^pH8Z~a_ zw5xT7q;PaJ->euCOmVnX@>WJ6Fvw<@g?gBmuL#z;VDG}pKpG}j-nj3Ar!q42V%})dQ-{J-BBR#Am6D4493$uVj9qdLpl`Tc!N!%c zuseX$^Y#Ij1t~%D4YJ^Y**y`?>~twb&JqmCxkXkV&kV{(Mc;T5h=Ysm90CqxBI;Sw zhUU-b;(2%MkRxPWqG;+&DHjML-4?;>*|7VRT7UhE-K1{=DFVT>DrNpBo|OM`#S9;} z&pMsgHgPnE{_GClZ;|bPk$x#|Nxg{EvCe5et@eC<;ed;hhmQ`HozHELmeF~0#%A++ zzDY~LL(dPIKD62T&~V;^dP6RB;M5!hVqn)ZHQZ_@K+3VoQtD;zKV&6G#?luNdM~(w zF)wVxg^hg=!f_@Ac}$NvrWJ8$A+&L2`F^UGN6#GTYN_x@1{`P=(s0KXd^Ek?kJsMf zcUL#)D&{El$*X!nL;v68DA5VpN=?z?ZWmmV>>IPsc1~PLj5*yxA21}N{pQTrrbR(5 z6Ye#uf-7d-si!Poe`VR?`zVa=6nHA2T<+%v621_MyTr1y6E>aWZx$g>>je2E3>&0j z-vVjT>x`{mJ>?aU&YEOh8DeSzzbyf4i&P*}#I30l4qc&$u*xAE_;sJ%jZ_N&)E9_; z*YNrEE4`%3(kDu$W3I!@Lcw-}9DD&%js&@~%4~?J`zEhnbuKchc0Q5Wedg9}ZHJ-1 zZ+CSrcrsmozNotOzGUR2t(5B-g8|3Q0-;Oe6;!e-gz8lD^tJM->z-^$b>Kq~f%ekB z0W4ICSicL9vzX7m=<2YhyJBr7MvHy?h3I;_r*{6wYri%acbW!XBcvRF^U46=YA!$a zc?SoTdxRCq1^BL_z5|sDS&8*b3a8XJ%v>n?+fCl()~QXVZHw}fuLnDiT%}0S1=zn@ z4h?ml|K!wl7+(x{^zofhiR+)_cyXVp@1VLtfoK44gHOvokbaV51n&uT43}0g^Hc87L);JVyk}bLDA6q(2Ls*R}0G z|CMm(>3IaPyO8H(o}SCz3Ky45DIT69Fs>eUXvbO_*`u6s4`k8ENdUtBz)|mYUdBpK zjFnOO{43~SYuawJ7fKWRT~W7;Lu%6PG#h#8Q5xwY2wyM1WIpU&uR09^CUt#sYGn=1 zFHQtqn^C#hviEFUgH6Ehk1@tM85>}Qxo(}^j581r0ZXuQS0 z@wn)&Lss9}TFQ!S9xeR?%CFEZQ7Jz8kk^O<0eyO zt5Mr2keU9vbJ%rcZnnFo>2H;g_Da)&x9?@;)oMD%NV@e6dNt+ z@9dYN3(lElIS|p4Kbz15=61fc>Dp$%fbVQwTwOWY*=0XE1s~3vUT@;Nb${9S9@qvv zx;$Jbp*N}YwajCDV1{O?dK=Bw+@(KhB}BKC8ZglDb#$^hjJh<{cpS8nWVo66d_JX< zZ~4#Vjf?xG4to9SdL92EwaMLwzTj?0X%F*xasOXoHD|x^rgas_4~?^Im%XBbf8La| z#%ZpxX8_e0ydLrI_{%(txGQG1f!TBw)uV4HdWDOcZw0^da`kw!PIkRjHjp~2U?hLf z3dYeW2Ku~317NP+g^SO)K+T~p{ zaIj16-jl#E)@e;o!)+35t{So{etXbJ58R^?ckPak97oWz6TWkY=eVjkzdi82MHC_{ zZ4TSni2xdpKbv`WRvSE{;E&1xm=r>JeYHZc9~cAyR_q`m4SD^tXn3ZA~>@` zH||J%PWlbrV^40SEBPcV{V%%SI;zUO`xm8KVj&>PrUWTLLZus&l9FzfZb6Wi4lzI$ zAR&#g76M9#2!f=5N{BQ_Nh96f`MBTr{LZ=e+%?AjXAecz6W=-Kr)D&M$v>*8cHEx_ z73J=}cPahNN>dZ0WV$Qu-1>8x@GLe}>MHt+bi?XiC3JJ}(Rm$@bIUD)XF96_qY+DR z3$m%nkZmz^WvWx0{QUi%08J^jrdRYoKtmCud#~D>TLNpNUkcki#&V$D-N2ZHl`g25 zt55<)P7t;lAPKC2lOt6wv4Hwwc5`)&x#|a1wrILz6ZhQc2ms?!>47omfB;3}Spu%_b(fGjZaF3T9e%IRCw1N)cjAKA&+XS<+t_or`r_6!*kTfK@$c235kEuPkylm<+Oe7ki!bW5=BHF6 z(!^ooZZ`H)f_G|CxDmUQlvyD9dyYfpmRVy^1KFeoKcqrL@q5g7$AEA(Ex`FxG|kcs zC=c|#GOvwPyFCXvgZg+7z%hDX3uPxEgi;5@^VZ8z;o(ytpn;V>IEb5OeoJkN7s5y! zz6GiH-#%I3l%+x*SnD;5{@(saqpT-X&zKWpNRF8+9Rm9MZsclex9$WQXPWMkWZALs8cB{HQ_y0*Mjp> zKUM?Nl?I9&M-j&oeoTJet$AR!ZpkFYS&b?epaxW5I;a7wIIyCe($BR;WAiNYn#`YL zPv;+RaRDSWC})hQIAFT&Y=rf)naH;I2Auo5tC=F#uNwqXkdv2-;kb;~Rg#GzYjJkU z@KWD^mDVX@3My!PvV&t;N0KqE;W5a`PKHS_O4J3Em62qGZzl`ppof=Z!}GF zfg_g{rNz+9%~g+M@p@hQ>0i0IX1!!SFGE4Og?`W#k`F%&KEv{mNN}b)f4o4iLiHu6 zqpu+0w?7vUzhdh2!?`XxBw^Qa*W52Vrv|oyrViTi7NUdInfq1uZF^)=P)tneYcnWn zwIWR?RUE{ZF;JLY+1~{X-Qn24&}&2cw$|2s#?bTikN$jUJz^_owJDha+oM^!-e}z};_V z`e3g)oGAnz2Ww;0(Ku}w&ee)YQ~jCXQWF z%|NqYzvl^O3i884dQFcMbPHzzrC&~CZ>bsc;+id8wq#$KH!lFpAclBSy!@z*u6tKP|GW+FP<@ncBg-@Czx zlvqT$>{vAc>61c`HGFj_@%l@@qdp*g#4oeH_xYWp{G5STuVZQ96jF`_0XXui2NlYX z#z(TiI2Fj8H*_RR$+f?=tnJjv)C{Ml89%$QFbdAqcIfe*!+DYoqPV4tqt9TcjE!Q! zcif_zc!s-amS4CSzRupt`bOO(O1<{1 zP^eH~rHTylQK<9PYIL9#7SeO?lCi0R{uHAPHDqzg)4|CP5)L#TnZup;VCDIdGEVq% ztKz`@Ys&>6Ym$w5i>Y_jjFd8;g5x?S@(k~1pyUR7<=2rWY46^S?X2`(B8YcV3qDHNU;>etUka(q;7f&joa8CG-@);1B^AnuR5?Ih}%| zyhNctkNKaA&pF+%-NyU{#X)T~|67(72e(7AN4k9z! zXWzI@KDA9G!Z(lOC`J7K1G&BIGxG!8DaniMHzOj2H!9l7n`Sa`ubiHF|B%vWM|NtF zB_#3+e(ME1iy%h-S5OPWVng&<@I*2RWUk<91QP7u6MQ>!bLv==wRcl~{ogNWZSjmr zT+nq)%*)ioB()#xS_ppVmmyj{dQ<6*5kT6vy)d*4N&s}-1kknJw+|2tre7(VXiy3PL^{*?ab!1qogpoD zX$u4mMLY1wbwrFe5FJZ1y&Ta_6SaT#gw$W0;MbuDRJ`2XulepoY&r(NHwefK`jSk> zE00h=LzZv8_#*P&k3>a7!m6wnf9g=jWmwd(o)2jac&U%wUuR#Az~Wk@x5)jyFxKQL zSGehXki{hr#{oj614nZ+5P$c9;-dhaAwQ~0^(6zW{(Hk&*iD4~S`4bM>#L)XH=dty z8&8rHFDx|F{b<)kk^C#1gFj(vdAolA+Im^g*(i)NgFrvq1*>UsGqe%hMOn~g@^TE| zD;>!2L(+Z2JWy|6LFv8Y^;xEKhzUHQ&W-$>#fpFS8vkIFAANvM1H@KE?Q@tax+OhDV4XY*R1P{h%0RNR-#Im@z0i&ocyLJDGI!-tvbr`?+L4xtVaVL>`Q|y z(zk&!E5yzLWg7*OIx7$!)C8T>k*#rbzfco(Fhzv@Iq=AA{OJQz|VdJ zib~JQ5S*_1=3K=J{G4>rs6*AkOoAN9cU7-^R`EMs#Eue{KShA)U619rsq>I}JNt@HT-rg4eShWw?K_>O zM0_gzmZ*=@PiakBvl@Kn2p)eYcq~mpVbf`8@oaZx?>zEspk07sQjI{tN@N|bQ_huN zlZ~=u^p{@_*3Vb92j97KC;IW@(8k6_tX1yHDrBl_^D9AeXcS02Y8IDFp#*X` zFbeQ14F!^oNjL?K2P??MJ`AZl!O?QNxq7FnX_J-RaCU|Y9iRkd{$cRvqZcCyYM`TF z(enJcj=%8GK6cv66Svioi!icG<&UjJygd9~vj%y7_Hf7;f#2DY+z3+6-$SA&{GXWY z)+y-X6I`~SP!!hMfFPU;;a(aw_Oo2MhRJm3{Q%<4&sjk-TDKXU6JXkDSID z^UVj@V(%MQ|CUlT#v`{`abiOr-U2}oS-`>*-SHBmcR8u|yf>!guMMq0rEiB8ba3Sg zh8~@i1L86zAio?%VLgF19X;4BpJTh?{#Ldzvwa?`@pHxbg~sB+=f{H$SRBn)J5KNv z#9yNMZiCt>7Szl|7E9onaC{cgT@g2N2%y7*I>kyLMbu$$x=~Nd#`d-zaTWZzb|~f< z)6C>bk+);)gi6dRYq`A~qfM|gmN^wimF z{k;3#{HkA}kGU$j5~tqUyYM%`gvidGya_)NdoLB32JC??Gcz;xvG%iv90X`Jc?@d+ zG}RQN`-g{EU;DCgAiSgD)<@yO_6Hi8c*z8Xq^GpOUm0NzI`H~jjcf889`^?W6SWac zRBux3*9hJ>H0ZMyCh*VT`UFc6`@PR3A1WYCd_S*F4gsuN3Ejf7R2 zxIbTb{F5-7en0J-+;BTncL{H9R^5|RQA(OZ z4i@pJ`$okk+E}98zFH$eOXI8^lB}iMn{7in{=dA<1Xy7vlZH%#l=&H09AU6^IlN*I zU1aKoVuo$9a^n+qUVN}gF5TJGA?l)w2eMJ@D=>ORyQ=^N8nERvon?`DKmv%r>TXT< zOx$$|-QUPUyTR`R7U!sSlT@OKU718u?P`c?{HRsl&U9ulWyodos;)v;eA(>G#%ATD z6E$*oTRHamU=b{qT|-si(|g!1yVUoBUj2LaHWVv%^^LuZ=;-vE4W)VVk|Vuk6FW^g zwyf5Ax$&Ujf@&!5F12QIFEd~T5tdBx9i&oNZ;?O~^$czaCip&x<}g)>ZV?rq;=hXl z7Nfs)^$nX_1qT|PavpgjvF)J7E~1Q_n*m*?AnbtsL5{EBf#s7g`9}>a{2qRF?{hv@ z?X+^t6DBI>oCh*0tmvBh=i0EyV_C~Y6y6&GFOYF);~rFke9dBfgXaV#E251VS^`;} zX>z%2!ERYlc2|9E+Py_3X7DwAMJhmF((50O}C4@&EZI0gzy~a7$Ay^6G zeguPu@>tjzp9RC4V-KS+;z5y_$?DY3k5_DtSExrWLYZq$@3-`FOu0iRdC*dvWB#(cP zql&%Dwpj~zxy_zmxu*+vAb`^T02lhTFz;t&UUELrD_vXAQpvY~MvLM(V|61bC8g$; z_!E`{m&<2JiUUt2AFdS|#oIahcY59^cvF$&OT0xI{eXiqN0CFYdtLIEf~6P#Up8?5DwGXW&XXb@k^*`FS55grmSlj2VVItrvE*?W(9Dk z!H*aNz?%AahJ+U{Sko5`h^G$eYIfiAzU&Y`i=$>hwBg)m{!oA`)6VC3J;7ldQqd%hIZ?TOL$34o(jB9H_l!H= zOoA=fp8R_e1ZLm0`?Ov`Zb`hUn#KTfB%H1B`hHx88xc&uj_Y-Vmof2dMA4%lcmvYm zKSHmGaRrYu4M_8^)mlIwNKt*o;{8k8h-@ia$F%LdfYS;*g_DE1`&A5_Wwf=M9Bw== zm|eZTM+Tx=2ukGcZ)DJiLeH_dSJ=D^8xX(*u` zK?UQkWX`P;z@(9}B0js*E&T}8AN)bC>}4TQPz&By4XnH#PkxCrCRS3(F&NguM;sh} zZq$bBfRXKihS>a(Q`qY{A_2(eQv;uYRKKNK~m0a;W1*8q*S`Bhl>K)cQvrrkAc3gae*vs&#w6e76TrmB77|Y$Ai>l=mb2F7K|G`#u zMlF&W8l?lw-Ll*VXS1_c@YR>$)%*9;!5EQ=_x+(VEy<%{!3$3M$0lw<%~-bU^euHg%K~2H&GJ{9mvu>a3(-oi?!Y}_CE$?6i@v+ z%il|F^L5;*sqOcOZ};|N?k=Q}8fX)U|J?aou4~@Rd4393E@iuFrCudPMD?wsZSC#3 z|LG<|!@`6BOha~GRsLiC-bV)0TH~y!$aAcG4 z-*pBkj|%#WmmuUzea>ACTDwrNg;4Ht5dUay4N6xkxj0|){h6LEM8m=j`CO+kUP>kC4J=Kz5eEyU?u0uy#rX=&+UE$pEtL)=VD4`$V7ohE9&1uP4T zCwope6DBaVdT92v{CI+vCZ0Sxh5&lMsvz>e!|?+ZC&}~N{bM@*H`grWxmP83+_fnc z+MlzO(t9RGmi>DF{yj3Y079zUAdVJY+$2)PYlgh9N?;zmHb&j3M)6E|!2Yrio%aSz z3PoXa@OWMXX~$A6>_nj`R&E4>AT1*^x-5=jfFFTs_uQQHLwE%P&M{5M*c~}_JbJ%S z<%rC1_%OpEPvOh@@N}B4A47`pv{{FN_UYWSK;4O(*3Sr1{J%`LrLW%Eclu-Vsg28P zVdM9vTtRB<*Loe*0fG2->YzG2aZw#u%XgI}`D-s4z?F~gxMh{I>B`XPB#d)4CiJ~i zQ4lhQw2_+{8B&a((G)hjB`1f2hmU_t#sKz{U9C=_)@l|seX#O_2Bc=H-lr^TAt@4u3*O1TW#J-7!Fn^3`fl}PJZy) z!4B`t_1ss}-QhAEQFZDA;b5VfoSU1QUXj5W*yh>I0}+%6dyJN>6s=^_tNzMZ>6(J^0%a4t~c?taO<`Wc?$ zu*bIl&Ut0QO#Mb4Eb;H^g?^utaHIOpRBO9=pOlYG#Py?B=2bag0OT;myQsor7LdSz9{}o0Ra5x3573OP24i z1A<5m13?Lq+OMV(V#eyda|0#jrh!^E@s7B%#KLiC11rl;~{`D{?aa?lzGt>zD+KDuJw7- zCn)iElF1>T{VRFQfz2XT=gs{eY!a)iG9Erof`*Pla$(-I?g_q&2CM@AI=jI)E17Zdv zpp-b2^gVpHMxFuwD^E3+<+Mo@R{T6?~*~sK1s)>Upn^mu!22^ghLS4i|zRHPu?~pgZTXD7x8#R zq(o2a6H|Wvr>eKHhdV!M5QvFpx~1tc%Y)fjaup%_b;PyddT!&;HEa1n)^8^{%%$Lm zM5SFZV#G|Rw~w?Qc{5lA@K|}6Tj@Nq8r5TU=A%xiqZ2_ot)h5$h)ap&r3N+2*|o{W z(i@s8oLPG7q$&T{Ix?p@BfK>orCM4$uwgUM3t)fVz((fjI)TBVGt{jBQ)4TxgVN^T)M`d(Ep9*-c{t`NU&Od;`< zLqynSYTq*vNuudKSG}NQkwa-h8+9XBTt)ScBSKrsX19ZyL9v1j}tb!q|1aK@$ z9iDXyuQ2QRwFmTc$Ml*7{`F_X3U+ugs{e>;@Ju`@8;UsIFvbC+2Q_}rmFDL*$kg-5WFn7qMhfbYsKxx zl(+e{WmlX99<i_jIpa zn$D_S<(x%QD#;kE{rw5U(Uo~GT)<-G_&u|mV0&jr|4$W!dS$O!mfN0z_BSLr7)K*T zxb@Bv=hh!Yt?{yGubKRc^}cw1s1_AJ^vVTT`+NyQ1$qC7Fog`uUBe7@1+Rng1o{ss zWcy8h4d5_joQetxUm@m0tP2L0f%VdJvO(|FBY=~hjr@5pXL1NUnmy zO{2F~xJU|;mnm;F@tO0paYJwxm8HHfkghq0kiwF$uCM=h8g*$rZh-;hsA72GoQISJ{cPM+E)cSM4|NU$M_D0+IJ*R*XpbZ5Lvb zUSP#Ayf2a-_nm6ZuZxqDo<)=eUZx5}nB6|u~hqvUlFmw1Q%!NUiy=?COS?5tlxlRDdY|Ya z%Uxd(#^B+SQnrsqP}%f1xc_A2muWS3Abfj+1t@dkhqtS8gcG!RU-pxeh$u9hlM-{j zS?$XwN0wE`+NuI(DE?w-@ML3hmoPxU`C9!1K9!r&(66V3how^?=>-@4qSsxcqsLMn z$z%mMxqMT$xJ7e*jcCoh`sCW$B!On}AicTObRKko?&n?}aM8iw-0=Yb+Bftm?=BVZ z2OZ2Y=#O7?p|i2IgglcC@bP4WcxBO|A(avFC?A6*b!iuT<@!4bMRC-hUXUp|eJF*o z>~=P$;??%<^>NLml^V*o$$$8N#B$cVH15A~8vnV!S!?3%HUD8!Vy2xlTFvbQ4F_sz zqPsglt4n*m85|r5S>_5N@ZMapEa5%O8!xOFPGo!q=A@qOxUXVa<+J`A{FYuxL9!9u zQfNUvg>Mj+q;}#`ZtHJ%yvH?3UaP{+cS=c1qjQNwHop=jy?s*x;kad@nD}Modf(^A z_siGcED1w+3eMyrLG49S7a9)_H7wcF1;(!SU!zrxZR15vzaLd9C)X2bzp2u3H@eV~ zB06J91$D`$4p#7D#ly?eWdcnIBS4H`F#Qpl8NJGBpm8G%EqvjGyb61~6F%vY#$7m_rQ{~v}e{V?|Uoz3x^b}_Hz_UA7-0|X86ps57^&D=B~{yuRxbnMM@kK z&7Bq>d#HE`2RO-3K3mI4ZsZk=q-RxA8*iae$QvN6RigRD+{_gFK#lB-O;GlojA;v( zm|KqJHkHv^BOcbieSyB8IeLWA%JUZv>%OR7gDJpJugcjN2vZ`&Du*3O$kNoTQdgI| z>AUdi;ReV_v7^AEXn3BQ{=eZc^k@Mc_0s6)nY96n3hx*x``h`{$mq0FRc2oNvv&H; z;H!CCrLGwS%gXjEg3tu-Nn#AsK#RP?5#O{G09C8Zlxjw{yi5b5g(G|%8teYNE+%v9 zF+SVhZi?iiG;DP$R3Qp36_Fn9SZZ3TkKFMHmupLj~V*RcJTk z%j2#mdE5tZ)SWM+ufyfPk%)qWBIT&EJN0<{gj-B-p#Pw$F=fRZ4)P!U=K&#Go3G{mOHB z*DXv3<>@1cWc2CE0-kfcy&xMDkeXQ6HduG)a$s@ck*so8u+B(&BIU;EdP3}+P4CpI;MOwHSv-^X z!~?73K4qJ)-pq`x*8mY&>FbB9jCE2Umr*s8HJ3~W5W^&)Yk>;o?0;r>Z3)+9SY%;( z`4A_fNIKnS|E;vc$F7l6V9esj&1&3n!RFiy5k-S5^5TmEbiSx?Rqh}8>I`Ykp-&>i zbmi?!Jifl{zJ(cIFazmsKV%{myrDQfdIOMb~eb9k~JJs8QuXcz?L zb2!ng$-|K!5&6yqA9w|N;NF`2tAZvFbEM))>-^0+XLEb@#Wp_osaQ-0XX_1ILn_A* z0BdikevFnYh zOLjoOzzqC{jVRQULz>mV%-33(CG#v@tFWMb3e1(C`|g21n`iKquDJ34!crT3S@!PE z&`ixlpbk9PlUYiAxVk8MH~qDglosQ*?l|4Rvs38)U${Lb!R*6<37zY$!GdX4K=%%7 zYH+ywX6pF&NjKeET?(v^FDlk^>Wf-cwdpjCY#0HUpt~bew$bs3tDPAa7Z3nZ`sGi%PL&+ch8ZM;eDSjfE+`~o z>{CkS#Wy!*EB;&ML|e$9#(TJ+ z2K1c^_!Tr?C|hXCe9=}=#K=?(HaIoKB#z`HU=?P*YSaG-VEp3;k88RaG#INW7%AXW z`*i=D)euFoqoA6Tdgkl@p5;HjJr<@zx`!@{*{8R)g|U@$;9pr1Tdfc>=FKu^XNNy5)AN^DJTubMkKhFM*>}v)NO^u!b&e{j*OJX4Df_ar zBc9A@$F?_J8OytexS4XR%eLC8aK(_%>^YBjhnUKyR77z#KeW?Hcr}pLew%EHcB)XC z19TBeO0HeNYiV1UI?5D`CsR|6iDe8V>B)4Z)r1=%+*f$jN@nT5+}GUn<(8=IJJUBys258izJEh{z1p{|>gHiTrL{ zG_<6bnMs%|<4^18M)A!b5e0bK_*BEI*HQTlx23t6nvRa}(W6HYC(KK`_bfV{5Rs!W zaUY|~L32RxtIam7j#G>Uh z6}PTpnhiX9guFP+8JL!j#;+;@-=d`q8|NrU;a@a62YbL#|hYaXs zq`}0z+G6Xv`xi^K=3h^j`+Yc)6I)AeHe5>R<>&6R1$U2z<&4=648KGQmr;d*u9Z>6 z^|wok%gyp^%j&Inp4U2y0uKB4V;u`U+;<+NjjQ3_TJ9p8a=D3blPc-B9^Kc9^l%-_ z0r(-h&mCjePFG`1^OJ@5WA?-@?D}9#=T9BW~T|V254}xT114M(>{p|PO_6cr!d|v^35J<-1 zEkQn)<6n|6nGw9XfImhl(&%b+C#%(8?v>16(MVhe@83Vj#FK@io&B`x*P59o7P-LN zC359aC!Uo>6Pss=J?IEM=y363?OAN%4Y2?D!b)ozwlbL}N;N%Gm%baX z+~xG2oHv{ZWj)X z8jVRL=-K!<;=8X}&9B{j5ME^_>lh?WiaL`R^*Z!Z^T>?MX-a+Le}Cy+L_-V&gO9_% z!xdGdO%E z5z3eDhZyq@X>ye(qK&QxRfwWG{`|8MN%s1`@9OO1%_JhJz>x5BH5BS$N|p{TsRnWO z+u~tLpilFJ)eAfCAoG1=^XD`-@&by-{2vbx(GdQ~6QVA$&WKT^>py82;cN44lH*}z z@mJEEjy%!66onIX(eLrb8gc@kynkcP|8S3YgB90U!Ir_^usmz*rassbod%%pEJM z4~#ae3qcIH4}bh?(P1R_!;eGzea2!tincMRA5z=g*ivJW;B}8c*#mE`_io zH!>=ob_cYrh}iX##>-$WnF9hy;&3)tr?A($fyJfF?Rb>Fw;aof8KtuwJ=!rU{Mz$vM51n zuEUnd1H|?@K`R_cGD}$TNVuVAoGQ8#8wZOqUo8Ldz5Qr?H&-ZGuq|-(F&BiPND zvUe|4tbR9DOqQz{)vP~siQnV?TRuJxtH$Y7yPCIbf?NQWYmC^*iZX913_=j4QROX|j|cYN!hlu?y8_s@pFVrm2JX}i zh@qGP{OH$kITs+P;gDdG=l2XYXJvuB2D|h-#V$i9Bv&s?%_cXzs_Yd(JnAQkMu+H% zi9{v6ku+J2U(<2j2YI;!I4%`&So#^psB6oZs4g#-%V}!4MSNu8qKUtd;hr`~T2s-! z>GF6G_mKbLN{r}j`JC`bOla$63KTylF^NOZjjIGPy73)h#OyiSD)|Wn5irB|EScYj zCmAz_&XVVS0aiJ}DEGuYUV|6fqy7CN`sC&Z%%=T!^-OMzpr~Kp3@rp#EIV8_s1w?W zkJ~t|G6`EplM&>f@6)@`|K!D659l!W7Yi~6f?m=xGGcgfKoH=jb3s!*=n^cH`2^xw zd#_lXgEVUw&>n(FJr^PqmZ}J1fN#H3);_ma2DyP^frnQu9-Vjr%|7f}`ic|L2==Pg zF+$}?|8KX!`qlZ|zum^HyMVF0y;JEQz^GE+tMTz8xt~idZfAy(Ef|X>3=9m(wnfy? zmBn_(h=_;Su3-c28GGpml|5Ec4^_%diSDK6XdC;M^nee=LKFj6X~0bcPHoXkiCVyj z36a{&NAM3#+zN-^J9tyCDk_i?10#9gd9>;lq4lOv{fP$b1t@GbN>S%de9EJmW_U0XdzL51 zFeP<*7Y>J15GT58qxlR424@i1kR`vTC|wIKzhwLt z{)Kk@dOO^h-}6JoUnvpROm`d_M)aJSFlz;TH?vo%6B2!0(~zF0?`D$E8C7ikEV81U z6?%OV(vw+&)XS^w_|+}JFYANdSmO%LPBDZ#x3;uAW)+fJ9gZ%c=U7@*zt!{i>uCOh z5+MeLht%hnR(EDE*3#6v^W=`&Z3SzG<^ohzv?K(04FNH6F1T}jz>}{HdFn+O>H^+L zlhug$2jYEo_aHm&kGfaLCIB*L_;onZ2?8nzjY-KY6%B0Vpf3inJV(hWK%@$0sky8*N@^=_#t!>pJv1@GLTU z!CAUfsa~YdR!*9~g?-;BSa-J23cU3k3nELunH_ZQrsAW;H+kkN)UXj z^7PFFITsm<+%Yy^J|{aLqZ3z19J^`cXZVk;cig$4X1KZ#I0px?b>{aFVM*H%o1d?Mp7~TeB#eZFL_0gG;`)PmgJ4%Ty?LhgQR~U|k-JxND_rG!XF&wBbR3!f zsK#^QS*kwQXNb}L4dFzESsm$c@sTY_n6+d<3xT}vVLbOXk8>cr`PW2)B*a6M$|MKI zy;TcUFt(mWHX{@(@R_QI+PktVN^Ky1tQ(G6-0dBc(0DQ6G60`&ZE-27g|9w zy7zfvy9v$e9`IoGFn7I=5Oe`tv6!_|lS-`P5%1(4dd3}=wG6rdOB!AvBO~J|3l}F6 zQGxOUYpk7^vZAJ}gpNc;#Vc|h3?o7x#1B5>BgPjA$%h4;>~>@%SIq13CTdh;A9H2+ zun#7>Au>C+*Pn8q%U~x4)JnblyWWI#f_CSeiA+bD09kp=dTcI{SuVYsxA&EDRFL`yVh{bo71}-xzTxRIN zJnP)G=d-)^6y`;{f=DqOXgODaRe6mPOA&v*?G^51tU5*peefK0NQO~3hE8jsy&{NN z0q>6Q8XS?>^@ij6xW~<4UM@O^W!Kmxvi~mt_cDGr@7iv{2}me;3g@}#TGb3yY7fYjzq z=>}1kNaL+(bQ8JmOhC%qGjOZu5t{0@rfojFRl1b;ysvE=M4+O_0_v=so*T1UcA0_n zu&Gfm8?ATIHay%F(gjP|IL`Ws*}L!!cC!~7+d8`_RHdgu&|m?@o~k=vVh&1sZM2#SjwAi)(%+uc zG`0MlY?>v6-=E->Fsn3!7zfqLqsUAxzi#!pJhMekw@oBsV1fd?`$fFL$Y>)F zY9s>)1>noKz+g>e;m09}g(z~e$<0yyGyh^vNe$D_gyVJq< z+C$TAFIV{IR@@ewZf#-pPxNqJiQI=j;C9tvXjY%H6JWi1iLEd^)I&z)Ls0E63%qYb z7XCf6Ulc#cwoPYg2HntA{sCKIz(G|+b{o^p!S3P2pcpa;P?{=(?aSFk{Hy2wihL2U zkjCaqu(n_BvfHd~)dD(9$(Oc>*cn9+ap4v_$q#pfwI}jA_1|trnvCSxqV;su_^2@H z=p>XM)uYg+KNNWg8L?`&^Jly7+k{mbJ`>^xDoNcu2hrDzSojTJ|h49STjc>&SNdH zc$Qs7Tn1xWUD7$jim`gpK6mb`7&>m|qQk2n*^0z;Oemd^C-@I)vxZGjzf|S889Yis zhiu$*j)}>+#>F)wL+TQ4HpZa&`->F6z6iBu`2$mqVvV`GA)wN{14`YEy}c;DV+{`0 zin#?ghNqvu$nh4=v6IaNf8Ae*-!wDrVvLkHNI_=zYP$cwNg3UPuRxl`u`Mj~HG!Vw zP(QCPRDsK|zEfZGb)<+S?cz6C2z2l=r8Iyh=z3|6G18Z5?k=8JmznPUMB35iBFbN8DN-U72+}K#Q#wsZK!H+g;%y?dBF}p$o4w zdt~OjEj^heNzIXc-cUk@AdiOnBytj~b+Zj)Cuuyk7hgMjV!x<72@@fX*&q%0(Kk@o zfVr=JdvIRQB<)h=I(yEO_@pR@o)0go9_D~1`;y&j)k5yQ&z^A?>;+lx;buFpzkX8W z43pozo!vHl*EaskJF&H1y1$*U=Zzk9e;*R1B%ZsAOG-Q!5D^~cv8Dpfz`lG1g%i8G z%X^A{Qu2;{Nm8W6Aw#ldZww7lj~86+W|6Dmk+Y5iVGN@m2!G~umtHWVvIl={e>b#% zvlh5_z!Rs>hfi9&Ey}U^S`(SrFyE8$WoyrOrF1sAuo*h1)O8J7mR)2Q? z3C#VEmVUN^F+0Dup@JSQzB)@Z%VFNS3l{zFZhWJJc3);}YH@yywKl`Lh18m%c*&tq6T=F<#GT^w z4xdwmna=jH_ZurTZwFlxe&`O`k0ozV%jimHOngylZUA+wh%sm7JCJ|tz)WGzU7F}Y znQc}Gc3q{GTD}Z*n5wB+@K+8ANe=L$$pAb$-IQCVkxT5=Rhb>!KK_NMR~v^CTC@tqE8*#%{@2_RG#$a9r>=domu4zW0y&DFvc9XElg}Ze zxl%w$GndOp5EI$-m0RU1K`aF-24-AnBd%KkK7iz@oY|YK^HL|o&4KauQgwsx*6Ssi zmL&#JHm+ZFrxe{w1ZE@m4!m2J>_^29NM#O0ub3x2@T=xL5EYm)eh?HC1Qy1t0nCt3 z0HyLQG)jsvU=RABq@|MEt%E>NBsmy(@x~K52+VxZumKK1;x|5;>UZ2V_*@u7Sq05^+Bb)S^uBL`jcl z-9pV&Sb?Yf{+dTofw7fcgw};^Av5o;j-46C`FM-kgRTBSL!UGRm?8n-Wb;VwPLQoa z-K|dLIjE=C!ae(FcsUPbIwxBIfk7cFP^9+2*#8!SW_LWFJbM;dT6zTx6Ek|F5b2@u zbS2)(K*F@Mk&l#}^8vcH6wt5hklIoD{RBc&U#v>f=u}<&lgSJSD_|sny~lMPE8l+x zW@)4X)iWjTDJv?Y91Ax~`UQ1q?meukd4LSn}fU1EL9C>BjVyePrxl z=LkSgsa7SoKLET<3TVBI`44{})$P}4b*V)A#aKOJ^%@@eG;UbISC?gyF>2!ROj+Ru zCOyGZaekh8i7h+*w%@^A@|_}pAFygtx2<>Ybdis!Ckwdjn_+V}(-YkFKkW6987oN4 z#EYZwx_kv^zDcw(1gpDq$AMaWa!s=`Q6@B8rJjdfm2#Vu_3{q2}--<|viY z_}+(>XRp14agDDE*EPOKrviPvF4tSmA931py!Z1^ooB^s1(we}G++~DeKh0OO(?KN z^p}YtLRC_^Y_we%d8py@*D`$6e%{!ym zwM*oYj}DLA-1gWhtQD2tWi<@yO!W7-hOF|sJaXf({d3yo-w@+cBwCs>Ty8P4*fChS z`Pz{3C5X_Rs;j6Emq+$zkQUZM`cjzEjBku4HHS$TH3Eg}{4vukv)iIwm_462_$qxP zN$%szbdqG+F&;ZwGEy3w-C;;88;D5oa8`OIO@zK@i=j?nRSTjwzH~_WH$;qlp9s%a z4L3;MxLQ_e_+>w0ph-dfUmB#(jW$Zg2J(do^ylU&f6Zb=9Q3E18&039viLUa9gBRC zy7z5aZT$pB`4lPY1OuAB1Bdw6oT86<>`}@jwyN08ALIFiZcjh&#*k*MqX9GfY?Lpe&L_@s<{na^Gpn?s| zs;(Aok$=c_$BL$Z@ID+WvqX84-EF^t+?|r|6Awz@t-v>VeswY;GMdR${^j8IXUb=QQB%nRMOpO=1tiBtXnLx5$y3u$d#s|VVPJ!(Z$%>UR=rq+|9#Sz#JH0%9crgb`Rb@WvgXtL{%@)bzJvxC-{19 zh_DH<9J@yjnSvpgVO`e+j=;!B=?>oEVE%b8*t`*k z&GLT{Y0`LweJSm%N-5nod?Y-7b@IT$q*%jW8;c#eXLRz|SxFi>Z!g)> zK|YCwmoBogBDGa>_akuDf0;#Y_0F0oeDaW;d%>d%hMN%qcE>lIBlTcC;m9dmaW{NTsMF{`zO{cmcYcBM{{~aMO|*l8u4u6*pH3E#xO?A$^}x{%EgRy9C(fC_Fm6jaX*(pWW|d zHMW~uNNQ+rTld~5P;G|m(Em0dZvL9_xS=_^94!3rurSHo#*8~#=ck9Y^+v>Z1Og7c zo9TQIuHrWs7jY5Dx`dIhyXr{q{P_Re^{?gGA}WZE9WU2nz6Y6oDo69Nh(%T%+Muyr zW}Tj8!Ul~im*_ZI>8{=1I{!H6!klk&%;k}QTqQ>_vqQQX8X>c!#fuE{L+z4zUf0~J zCgBVs=GLqU!EQPN(8ILYHO-s{z%1jK9Oc`%F>X-Ss^edw^}%c#qi)tNBBj*X2Px%> z_;NBn#Q=3zwF|jYckT}N0F5s0OY{eAEdxfnOYAIfDFsl=EIWVA?0ig#wZ>yKy&Mbs zIBsp0vaQn%lW}TY1#+i!dlyt6NK)s*NJj>^VkPA#3-1d5aslot@65^q=_ZGak3$s3 zv?_y8`uL?NGZRcr1P-2miA8H*C~RvKzQ(VshVOlPbVdfVy5o7ZnFWo$lr;3V5Y2ej zy_p@{y9G+f6|l0vD8e?g@<5XOrM$eY9BifYQ*9YMemFS3WAu@BV?Ur@Ffe4@yeKHHbTe_T}N_b%Gk5)OlXT8O;0$$c%^e>E73F}f3hIG!Jz`#HQBR0+cR1@YK zu8Rvky>;;)sJ%2dS5MDlRu>W)x?5X&+3(-K^F3ojgjZIN75^?>YOvagZH1XyXKb`h*;Kh19w{iEL0(wmG>HM_KE#Wls9MN9DY1gbz9bX zH8<;I=ZnrpMHJ z*(IgabW!YT5#=?GVv^@IsNGpoz z>@wn22waEk34~;qL6?BA1S&8$VdMI$w+IwaV_sOheC>WFxIt;%z_38&H_S9;-M(JA zv1y>M?^SrX@XbPwrY(#Yes|W6DZv-RIf3q@4-NN%R3e8PRpAo~HzexxySFPMwCIB1 z=w`&*clA=i4YQInIhjN?fS0QkzblK2@yFh?9I3_>53dN?s=3jS$+9RT1tt}Epk$RFL@H|_uL z<+#q2?)Y~e-FvLv@fj;?L#ke!ShyOqVCaB>EN1wdx2!IfZMjia&83XFkS z;U`DGKYW&iy)FtVN4>g#F%Lb89yvc$zJ8yVRuN8BJo2r@dHucSR&Rv62dO8AW*MmW zU!)0JM(zE3ec?s5-eHcQ2GzUhF&svNN{i_Nn3(njYV|EZmD+<9i+EFM9BVdR3iqr% zeo`12dYqRo(b>Awh>tw*zIoK$-?x^!%fdwnBMLGRCBnonpueJD6Te#NkFUx)dg-mQ z8~;|jEHogl{GOBpaYcBkhiQq5hlebjlu}r}!mc|MA-Md)!d1O5Gy%6ay{JJ1+i>Au zp2}NwRYj(`RQ`AD`AIh~1WMi~K7WuT(~!!Vj6kx^QZ8DBc3CFT{-xTE(FbJXXw#c$ zJrq`X{FVw1RC6vHWG)X=oLz#BR~=y!MqYXtj}Za}6kuA$D#F)z6fUDlxQ#qyeEN&e zrW3D>59*Pfb6Zq!?3KiSx)j0}`?pI`^j;jTIglV5fJ>+c`GSL{5QXP^0Ef_p#hCTb zn`9-+^?Q>PeOg_f@~=)rEQ);Sl)P8>HqhKx#B-dZ2W4hSBE7tZZ>M53^9VTC0sKwt z3(ul(Z5T*5|(>T~_4CZELHb5c?03q0CI^lZH3owANuU3Lg*v>+G zbd;tvKsia@-*WKnmRtw$P1eYJC_4SXTW_S*sCXnR3mb_j3b`${BdQ z$5k@fBC;!wDdPicPWtqJtS&3cP_~GgaSt4#1#>Iy=1x4es%RjcbG>|9MMP>`l&>2r z3&YV2Wy2Ebpd?9FR+c}-5JC;5dqB19>FY~7 zUydG{6?B4{2A}!^f3{Zj%Sx(U_h?a+PM^F%xF|VC^EB_Bc;9k9gnN6SuI=6`>LlN- zN3U_Gw?Ff0b}6G=?3n91!NWm>8|U*zRaI3sOyxN;EM^Uc-{&U`H0p+d==Wva+T|teUT^JVCL) zgn=cF`qv-lDbz=>AoJ}%2)e)M$G?16V6VR`6GhH3P|F9W;4S{H*v$^wJZ8+s>&u@( z0LEW)H#M~EQd3#jn*b#S>J{G_4mZmzIj%tCag0@WQN8R#ZG@MmTFWNB6YplD>f)ktN_%EpRh3lp!z)} z>he>$b?DqI73?cJ+Zo~G@$1(vSBXo)C2#oTPxrGC$$4wA_~EB($naY_a2%83UtNm} z7lAvJH#e^)=F=l~!jmowbbo&Fc~m#Z;YqnRr@h%Qoef0nCT&&B<1daSmrxdQ#JVS^ z*iEOITz=J`Hip=CkcBYFDkdk?tZy_GzJ&Ip0{f}Se+FtOZmeRc0fIcUaYrqIZN%~8 z=|`;QWxBj8TqxxyWG~^$nyTdm{bAd<$zhovS+xb88Or^l-2uj_k$0KH97ohb4aw9Z zqQsDG5J<~4Jll2UDnG5|T-W3tu8S06EnS#(`~lAoUoS2D&rNs87o^s^1eKz&e(Dko z6s;$|N3FLNb{k)VUs|#|VwDL()@*#9wSo$I%^PD2Kf(H<^;mf5x`|!95af#B{JRiKR0gU=O42BL;YP5;dM+H=3;8rdnY;v zd)HD!q)o2T~FR z_|IDXT&C*yZGY8=p&_qx=WZdXE^fR1^|cN8M+F92^3&p;KD z-FFX;?QJjhSqBNjYs_gIi(u$i3CFVTfq{54%hO}-jMe2i-qgz6698*JQMk`PTFA=hAeZ0Zq6p6@+Y0Bk8>pqRoPk` zLA~A_K?F00I`;~E6FzsG?UF7Tg*77~6%fn0%n98q>tqge;Og4-8-*`do0nww--Z6q z*LOgC8?|g6o9Q$c+bLkM0qO-@frVbQNrwp;)z!_IF^GW6Vrps6j(fV%f%X!LqUScM zpHG-PKN&;(8`&)o#-?VnRxAFq%6k*#R#hfz=-Ra^IKBVA@(oYZ<=IH) z&F`YXT(8zqIqg9~0SQG&h@=o7IWLXMHuc07G(@Grr%xD&D>E&aCqd}v3v&(Ef~>gT z>CVBY9uFLfTS|SrKGm4*E`+_~F*0bHf+NxLo|FgJftEgt8-v%(z*7!ko9U-cUQ#=OJNrojhKgY7Q5}GKkq4=zLc3+ z-^f*8FV?kf*a2kT;5NwbgR0$;sc{)5W*cCo4MzwHxf$n{_-=96f>;y-Z6n2Gfk+x1 zr6Dc&s1)C4wBWR%$E6q?ry1m(cFrGA>t)+9e_>*LTvtq?%j3&(4%d;+X9wSWT1mf@ zQw;*z_t@Du(kjf54Xg3SASzR7#uF}3P1J*1?ms>1Hur&`%VCn1+j6)<_2&Wf*bKK9 z8;npuFZgmGo$taRcck6~x8$Qlyp^g91rMqO3tGZWZ+bg_w^~N#-H-j<5gYu&CX-eR z@i*A?vokPJk(E~X_y$}u)8&E@Z>MJz4t&dm@R^z{Ucv9ubUvZsu#v{6DLs5CEnCZ!lF8{QT+yEUx>)rBGsKoO z$SXV#nrOOQ0M1h4AjHhD8zLfP)@=a1v1Z4KIHP(or`_tF(UG%XW@lsaeux}mQvCBs z^+j}3l!KE=R$uQtyv+9^2p}9j7t*%3L30&8{PeNsFpdq=k~3n02NKe+nZOeL0<>sNYAdc#pe&WurV&=bck_*C0f6SWy$#WY-U-`m~l zdmx{C0Sat5M%UJ1v4|}a^O^mj_plWj#b7dTDxMc;yCz>KX6D+KKGR?^ih5Eme|=f92_=h z6y{I)_;?j`zu&TKQju|W$4iW>)p{+cdyuH}9m?e_E>Cugfu1Oyqg;q?=A<{OCbqc_>grhkCxCMEd`tf3<^Uc;(Y@G z0@mz2EljRjocl^n1}osiK+LTShmg3|jk{;C$U;tweDi9( zHNb>z;&l3ZdXAhqWF`dRni%=_X8iyA1?eo46B?2)H@Wwup!SXJ`z{2c+^&RguA$?a z4b(j(?ga8_CfJF*L|5Jv@EN|{TU~oK9$&`aTjv~$P9f7NO&ZzMev8t@Z_)ISj>NgP zefOW!&;@83;AQK){5>VM za&NqSqI%HZ1+{dhO+P-|n#&fT&p?P3JEI=!|>GNqymm;_g-23q6b3p zlcg~Af?|Bnn~2yiV&~SIfxT>V=HNLC%jS0e)#W!x1k{hwj&)eOFldv2>uqM~fq^{I zlb8+TB(UAvJ?&i96lhPWCdt(-fvdUu%UIPjP?ChRBz$DgjwU#)GkE0OVLe^TacA>0 z5UtWU$0{%Gsu{Vr?44t-bH-FnY1z+ zRFdSUps_xNA@HDL%NR`FQaw9BTDO8l*`WXAaLdCC9$vVVFfXLKB2#@dPX1i}lbusI zu7CC432Riz3NOkYZSagHCqziTL~)fXvM>fs^~Cc}G^8>SBqW_Zifb4G0h5Q!n=}X- z*i)|UH3PJ99HRi+emVFcM@U$fZs(nz(3~4Thk?K#e+z={eiql6J%!;}W?Yo|lfl&0 zhB!g=1RtNGA-WrDv;SElyFT#OW$7o3+!IAXhHV|Vj1+7wjD#80^ZTWyhciH2LhGdE zzDWE6ITM2d*mVczN=b%gwIF@|6d$z3k5O$$g)qrW`{zF&d z^;{D~>hDXyn~)Pb@o3Y7gkt8Z0#Lq}<0>*hLUPT(PTX)$sluFyN9_vExa5CK}~=wGAQ&2wxPw*_-Ur*u8Iy+_mXPrlu~f@2P~7f>{)J_ zNtCAYFVMto`U@#5GQ{G->0=Qhs%G3`S*njv{A69;w~g3S$R5|;W2T`jT$v}?s`~m6 zQGqAEA~*^)1xG;#<3|r1#dU94`Z^vo&FGw>xV&9m9(w+C*sUzg;YPr7;{D+vJfdX7 zkX+Xm8FL2)d{tL`h^I7OHIl9t*bV66Iu-UaQkWOr)(-9u8;cZ6Sj(e9)qU6nTc!BZIYV6Gl zbUJqwV~r)QIgI=bKCuoZP&C*}4s_~PR(!BZ2*Octl(>dDy8#c|z0Sz2A=4QpXMT?~ zzMU!->(cMvjVK7q;T^mzvtJotQhvB=d81bV>?GdrHeQsDI&%x_0$MFBFXGA|eeyG~ zN^~fXo}ABznjf+=UJ@6Amx+*=$ru0Ua0@nIU6J(m~k=(3-HT<^c2o*@>&hayYheVKmY z&qK}U<&xWQY@?)Rl7ONj&)by$Ssa~5apbV@lr9kM>@Tou!evf&8cL0%Ii-hZ?xtM2 zVn+t1&CZi-hqK-1hPMBHK39#fpM z;V+^if3d94YGEcj4;HThjI}c7K>m7%!qcb^!8VJi@|g)uyu+1z#iv}`OBfax1uMqJ zX?i`vtG7le#>oj_SD?@zCPMS{>%kxHd!N+69JyZr&da%xb;P|P= z1yo$Je`jTCYWCRM4|=8$1`?@Yj0)r@0^34ulXv3>CgaW45tZlxTGdjj(Jo1l$(V@} zvG1De&ntJP!b5})5?wsyW&MBD4`<+%!(AFi-j={N>QB$9OqDo8WbJ%AEYt%y$7l<^ zsFUPh?N_s2y72SQfQ4VPf_S++0;@1`c$>XK<{Vm*_xE(5{fB=SJ+{nXQJpmY?(J2n zIa2@xA!{z&4O#P><&@@m&EMwPM(f`%a>IrVgLj@kR~m#$aahq&`eY}!+5WG;2Oj>` z#&`VUBCl_{Gb0^c3hJV2p1;@qW!&yx&?OR8Vb91&;)z6z$uq*h7<0XRKc4gg7N6*| zd6p6eZU%gvPF1eH67ZcMD{twC&D zT)N^Ob@d(;n!$7Pzkjo>S#x}e42eQ)0(wyY8md zXe{>qd;)Eh$%`%&Pln2}=O(NnRQdAvF3v;EQP>GzF(5Sd&6Kxn(;-WpJx?c_FG?AB zc{z-zO-8w3-nF9mA~$RA^?k1AW~t@;5-3kK%8d9YS3>GV6q0bOjp6BT(SpMcrYSL_ zp6m?p0Wj$3r1Tu_u;et1K9B1G!79&72_~KFGi7$8UJh~42%~$+=BXJjTz`u<`6{;@ z`<*>k&ntW|abkr`ip~MDGY?>>#)2#0gPiT8JrC$!R0j6={N% zW(_n3;;(;v|6V=VLBcgO4<4n-^P|pNKp*^c#@&sh$#&VtQF@R3p7kpt3{2SG#%-e| zHN+%Gozp1Np&*r`mU#9t`4SYd_P;SWL&HuFq=@&|%uKw6grFb;loA_1-nR_H9n%d`Xx>+wDox60 z(RHloHcAQoM``YUFr#)dU5wAI&IS`Np6eif=WwGfS5sEGIpeLBadiR@}SRXAWiX z5v#$8Z}nlgKMzhyT5qBN)W!FK60GTTjNWV@_-5(j5Mi$nXh9KP(>Q`lEaOW~_QpfD z2ySv-BGxyi3qvVGj)!ut+cvjcBj9dS->3z{j%&1M>Zy5B-FkFh2@$iQ@+RaD;15}{ z?v-|yoMlx}Q5gshyp5C$+|x3c_-6ZXkqbKi9g^XfJE^C+hKJcEK$*X;EZuI)R?~C~u<-t~hs^fcT7w(V z)EZ04p4oy?hb!;fA9LbHR~8l)h|t@3j2TCgP4G)Pt1*l2w%M{w);AXei!ak83exT_ zqBhzo30SY@fYcI#+Y;J|nG8nts`#IF?rJcQN%-)am*MfN&^u4H+DQgX3Sr;3$DTg> zPUqc2H`)jHeH8$~ z9gIVcH)V1VaOxP*|5SWMjgtM96@>9xurZm4rBKBD-z!pR_G14JGEltfmxQ-*R!tMaY7l^pAS#_wbN5Rl|ef$kM+yB&60@Xv{ivz z-{&-CkAK7OKEe?M2J-357S2Cc$2N#-X$V{d|E(in|6-UeqU+F-pZ4TUO2y2+=#fHBG#axbZsD=S0Ab`RW({4mW>BCa6IF+zXFy5CANMdp#R~ zgdI|cDIju{IJ)UH^R9cnLqnl7^vo&1DZnw60SzH$uv~_z^04=Jb@zijOF;lK(zSp8 zD0$nlVCI$v_I2w*!A?B}L#$85ZbI-I$hCb)PfwqrDbaEd^6L~3Ix;_Bu8OW&Hv2=I zpRlqUe0a{pW3joiPoJbcSV~>w$NI7hJp1p#%5CqJGxm~!vs#o9F_S?5kVPgs6hE>X zSH)^!7HhQT7p8aV-S1XqSa_kY%)^#kHd0~fH-CXR7r{F!cn)^xhO%q8xGpJpO{Loq z3Z5S%UP1W0rvLlxljTjheWI%6CFkhLZ56`Z11u?6 zU1W-=QgMZ5R;Jz<0|7^}WFCN49&C7TTKZuG+}~dOkeN{{N;u$T=?nt^8Z`?GZsHte z(}PjWR=c;{P5P4nj7*D8aB+{I+pC0SSJPg8N6f!koyb<)g&DdevxVCy1S>d#F0!mIN6Y9k(klJ)91^ABOhs#$ zW8TCwhjRrZIB9SUq?hiedAwHIAw6XjU!viyIKv-ljhIOp%?lX?foYuQ$?~Y~y%*y2 zjEoF15qJRFeJ@c9c(QYA)*6@5{4Rs zhXNSp1G+uU^Mrt&y2YL8<#t~e8l76}xp*k4}QtI$Xg==K@ZS~M>-^52cwVu>5!flr?(S4nwx?9LF^=3a03-gROB)MPmJ2S49)v=U~)yK0kZZh@$ z;okk`nexL$GEc7gdV+b$^ZBWRBwe@A>3Sr)>vd%jMw{;l-kKiidd>^D!6YY!vg-0R z|E551>u}D%P1672Uf2_{9E~quzKHqyX2?ceTT9~ll!>{7F#2U(DOLd3MmD|3?=rGE zO59E_czvup=LMYrk-Rps8zq_07#;gh`m@pfBib~(nP}4r2Qk53-c^A?%n?E(@H#O< z9|REEcs0)g;8&vY_C(3*mboK$!Y9`1EPkW0f>29xL4EWWC9QIEv%Mdq{p`qcJ0>fe zwGyTiHbb?ZZaZm@9IVTZqD}8u3Kq<98HVwCyZc}y+#X)V1}UY^Ar4U~fnH6U;>)H7 z)T)?h&3e_&w>Kk%(S1VsV>B%xg22>;j}r9UpxE*q(N$%HArQt}Sz!M`)OP_D=33|4 zWvYA4K)}H?%Ae?70OXaU0co1<^=rVFgeCL$*zp zS@Mr8vf+1lJWBjf>*sHG>b4=g4G;z+4kY&{L%t-V_Rv8QcTzbfS^O^a;B&DLn@@Up z)XS>MxI1IaUw+f@^iv#o^EBmHz!@%ih$8%iph=Ru-s!H{5#w{6SuGJj*N(eT6KOPwM04xdIx{m}`taeo zpC5;;6rmL72h?#F6u53xK+2~?E|uq7T4s~NdN-JEOtCDHac)m%`H|)xUa)e1sQg_^ zl_$9Du8eG=oc$cma5fZ~1KxQ7#Y)DUHA_;#Yng8qtV)JQZk~Rfm%3Jyh)Tm6%ndt* zZt{;^+~h4)i0LFxQMmCcUof1*!w}Hc*eIZQ< z!^%wcyM}%HSbl_&AbX}D7MV)^~SbzoKyQ;_C4R&9`gtxY|UP^eG=| z6?;`#d9NflTVSYW2W7>@XSno~2dlSUK%r$)C}r`;Iq^o0UGCh4ZSl9nqJ7vss-JGx zQdQsZ@O!HJ?ak~i)yHO!hOw+eXTzx=p0hJ&6Lmu8VMC1+f#jRaM}9KD0V9j!{N2K; z>Ah9Im9?rxtMHgJS}saDK0w++W_})g?0-S}WO!ktjl#?{zm%ZZhQ$S&;Ah3k1Xj_0 z?9QnXQvJzo<7?Wp;;oKhsGmA$%27q82l{E*?h%Ez@}bS5ZJ$Evt`XX6+_q!mbyD>y z2;mE!5k&7SSu%n_!FSLZZo%Y;XrY+JZ6Y))L1#>l+{?A${{6AgVxrXG@}wkzX^v1X z{>(;I9IR!H`8HZvNkV*lW$*M2YzHyAemgviw)k&&bIU6nKh}@M9?}B$a4e#kAikEI z=WiSo-K28ga2rMg&pp0emC?O_SV4M7HQf+{wjuut_wV3Y z^=n`7ajR6}ZF*4D(JwhSu4L*Q52`2V5X+_I(*2+wYP5(g$ToYpCr2RQlUuz4<9`ML zly(_zC=sJf>k{kEY(_pIDHl1M5xk}OW_!f^F0Uz3ARYJ2*HjyQEh;)E>3s8;H@#QQ ztLq;rLboV)1{Uu0*3+8d+b~voI8@`M(E*Z6x3KH?o16Z(wCBNjKMX*%i}po3XgQi+ zG}r2U2=M)j1H+R7t?0}buol>V;i7)@r-ybryXW+4)?zAYw73ToeT9dKw!M_W`TD?p zwdOeQB?Uh7=gl~Fm+TGTy+Osey6xLJAKH;gc}&rKse7RF*3B`O2ASsI`o?`;ycr2| zXkDB5g>-urI$WxeVZXdf0@b z$<*eD`$F}Y=z+!MD#pH>02LyB;YrkvjL{#HlNERt?{Pn4KqS<(Q=dts@@W1w%(7sN z{YJFK1YgQM&h^y8GaPX(D}H6J&9mL;A~rI8%NQKciM-3$7bli2>9cZ%Jv6C$cFOUI z9ku4E0>>NQi+-=-Iqd^68oT)7_Z^Wy(Pdz=TaM zRgUdrezR89ebTq+=slfHfD&PY)`IS6_^Q{94t92*++Gn4A#Wl!C{aY}gdm-R0D z>i8DxnB=Fqn>`=LOIRuzb=Yj4F~?+<1YM>xYi96A%>)9vDPi8g zA%$1!OGQrEA{x_QG+Q6=kAKU8noY8}dEvg%2v?az)j~bSuHw~q1?-uckuO@5fNfcT znFE_HXR+8cp~JpmcpoV16|qvHD17BQAeZ|mPs!=}!1|%lshFJv4(=Eu(JL#870>y} zlE`T{USK=H(>k49H5WWlS<3MX2sHVy z5AU;yIbFxr5MwD)BD!gHclCl+t@l9VhRHSRK-HG#7%TnirZrlAH?n&iK&mcp+RkjO zU6?XE7ms4N?XaHdlC|LDO}!qwTW|V8V>9@yI4D;2xrW}DK7S=|K97wUQ%u^c+&a#v zwI>c$ zzlX4UGS$m>FiocJzHmxQH-5SBt($e#MX&x_s&wNTs%P}+r{9Es4EtC&%L7pAiNcv- z)7@=`iBGT5`H?P}h#xi58AB#~%pqCwd_yekc0^84(xgQna$lWp({I@M!*6je+5E@1 z$ZZU>3A<)2GS@jh3ke_2BFplF|=0p83HEy&nIfOkigjW({WgNA(A%@S7_dxje0n(JpgL2C#!Jc4h#SSa2b5Y zTM>C2xm8P~BX#~W;dEh8YV(-mx#5h$)Td((*QQ7W(LdSJ9EW01MlsE&4(ian2k9-% zd~av-+juEPbVudrRlc`6-ai8JboF_qcpqVo^Am;1vZqviI`VY@h#NNrvh!6pJ2RR2 z!Q5Zu#PxA~ViA9@`Xbc$t>ar0 z&h(k5oXfsS-0LtYVcd#}ee=({uI(IIw5R9I%&kqQXHlBw^m$lS$Fo2Jc-lVb!i9`2 zdB}LP`QU~=V9y0l%ufcbYj9-c__CZEK^wd7Knf1Wy>F7=-*<`0vg@mH^bx!wZGBLy zib)tpn5#rXk^cGhOh;2Y_-#%CJu^{Bd}@bM+|ziB#U=uw4n%s9nJ(BIWnL|wT`q6u2F(x==3WSNeV1K-Q2M%#$E^2!a(oMu%gvjle@XUIl+*z+Xb+_K-OMTn3>Uzu8bDFhAJhT|h z+T7a0SU7cyk&`+iHv7|7MkK~?G8+%j{h~LUe&Q;g=b_+auf+3sJ;Td(dU&JCr)B#o z;@Ryw97AW|TE4l@r|1ciL^G3W}oaLpgFu#%lU^Q8w#)W;N^# zoEb-wzhTKdITYszTS7lKXTXwv^w2 zGT9C~LQ?jJ8WcbK?{BmgdcVSaNH>F1zQjyZShQ=0$+i?*nH=*xm%_Ssm_P8(8L_p7 z2Dy-)Ds^8Z00boxL|!#m^8z^6y7CwBA5yHHx=2odDt!>G440B+V3YsLw7bz$b&%Zcm zm)%z6vD+!vs?g5g|93nYwy5Ruve`C5KVbMN)>Ztdq2Vfs;3#AlRzSGbwk(KU=pmcM-Os`z!E+evqVm7GPe}eyfZ^Pq z*X?DG&+g7*kJ26gEIw5921irNp_^jD#GBaSiWWl2o>xu~c9IPT(h2{%TxH#CE0f0UI>z0uH%K<<3 z&#OxO=pFxKi^rSJ199KhIpH)vN!wK7;6@)fRK4$YUAqC%Y4lyEz9sfUWcP-E4qSn6 za{X{_WFM2P(scUgEccc3#e0md{LpW>Hwre^mTY#?LpA&#~?+ zRn7KhRY=3Ao~LNH?;}`B&yGLhaNPN_%W_c}XsKDyv4}Qi(%)a7@7z`{4V!0b=_LB| zmu)_RmAXQXo*{_rSjwyS$!&pxkGJ&2JkC6{xO;S_2vhy5&gNT9ADi2bqIZulIP%X} z8-G7{6Kb?V&QCI|sMIt<9{k-LITwy+-1Wb>*LHL>`7i!qu99EyR;Sg(@>7NpGmixg zlx;IbSos&un5S925~3PQh%dfJe(HL*w@C_Z&JvA}PFj{I{A0_E6Zz7(fw{17ZO+4w zP6waf!l~xPu%s&4jLhGnp3#)AKI}m)2iM*4&?+~I4>BG_jpAX?fwb+sp&=n#ORjMX zAgWd2!_-wE!vS701tKa;^b(`^R=oQPrfx}>P|wuZQofYp0&Z3w4X09jtoj2|d;~?p zWT??eLPDaq{{EqWl{`GxK)Q0@#m7g5z#!d6^$Bv3Tyr|C14=?tO2ibf=5Bd;dE=bq zU&n3oNZWg-I|XPbZ}d`fO>^hIIFm_Pi~fmKF2lT5i!|N6Yr6wD7EOMAW6V^_q7QuG z-p6V~`4aV}a{1)FKAx^#0Kv*1N|u#SM7#pfdj)+3i0dW_h!F^dMBpgM?hwK-f;Dt# z8IQcI+Z58vG0xp%eVY4i$CdJDaddRQJa1bo`t?ps24}^|xd{UQe|5SUo+#4bl0^+eW&`uHHS*4`P>*}BE5~&w>G2EHV_g@0^wNHG>@;z=nnh|kGB=&?6jW% zt2VeX|u!?Dfbl};7K}pzCri997F5_XDl(LCQKBL%heN@Xp<}u56~ie98w=@zzVp! z5>kh#uV>gl?@LP%+j(X<1ep68#n@b-d|m)%WrP*a-&GIQLJBNTcRUL&!S1)b5%vr| zcOrb-AF*H)Y|HR1_0)Cb^|V^ScCO{}RW~Xd-VdrTYSvSeBLG^$+rzbk{^Q_g7z&+- z!`zmKVm7^W;n^iX{6TJupp0MicDTt1^6dUL^5TX-v$mr6k)F>Ts~a2cYHslEc*a8G zljA!#;Ic_GO5@Q^v7G7W`A(Qv>jBjGPd;fIL3!kf!7o9j6L&&M?ypC7()&g zE=fA{s#0bz`s;|k8o-a)vqTGTkfoh|DnzSiGQpMS*!O8ek>zP31xQlv+9=dXdJW=M*QMbe<})=gO8kg){=0Cwe>VAuOuBY{|spl zZn8TL`^vP|iK^#=9ECdesCHP61Kp0?%|oqGsDl>j=ZP13p{5%(aXN!mYCvf zzW_uR4dcbq8;+r6J2m^m9?p@XM#>)v+{SL4p|4)JZM}?h!W-&a1@h4>y!vwY>X~I$ z=T)auF$p;~rUzuZ^j59#@bp}(7+aJTb3KBQh+d)tUxQv5dbac2-t}1;Yqo6Y!F)sC z$OT7kM55Z><2k$2%*TgqH}e*Fx20I6?2!>$BWfWgHGLho!HB7X(c#vY8d(CxH{=xZ zuhq?jJCP3|?4>*T>g&h@S?fp^+Wxd&En<(B^JE zJ8HqPQ!{J7TTeHpQ1`!9k`$cV(+h8>6rx4y5Tmt?>tZO$oB)TVW z=z@6}pJ>z9Y>!;bTmRr2?hEJY!9|<2I}pxH~nyE3OD(W6#1%r{sn35+Nr;z~Y@haCaXB4!$KlH3*; z;FLMJnWf%@(Hn@+e8b=fdHn3XU8=6@Ux4}TbMv<52d73 zb_3C{NA6%%f>OrE$agwHTd+f^YE|((B+m&k)Qwc!w?qhk@I#A-l5^W5QYH zA21rHZ7-ol7}gt(r)wNKie_r4rIQ&Q&d^RozsXiWqkj9esgLdM1DsV%K{(itfg~le zBtgpVT6LB>_*&UtWBNxZNJL0NbRAVxXqPn!m5g$bu~#T24lIk%(QU;|00VlfnyAez zZTCUzI)(t>;9^fq(p7_#AL6tO4tD|3OJrjdvl0{iPnSGcsdM?RKj+(Gc`g?_FLRV1 zhlq&$b^lSQ&P1Y!f(K}Ew=i7K%7BYCj-VkIuq^dv@p3x*9=hl3PI}-RpwdbBy>MfzQ+BhV6hJ1W|e$5X9IF3Nqr}05X-1K2DOH@d~)wj z08M3QtW(=H+D(8j;U!}rsw4J^j`2k5=zg%>l{9$a$NK1(c8={n`9$f++pRb zj&8MXthwc3uvxq38oM0aR4K%Pghq!(n`uj$Lb^glB{=D8&3!awg;!G(gZim(j~Ng<}^vI!4 zx)vpWO83#oXADQWOzDs@@!UwMP?9$y|A^lhiNmL~$oQsk`%*tY-|&HzEu|A5(Fg zrW-9iyz4|qk-Ee2TPX!AmW#wtYEQK@QMUw+7EkQhzP<5X=N|JPwR^PQ31M&U}ha5FDJ_4mR(i1{_t{PzoJsZUUl6x)t z&hy@hh<_BGMP>lmzRQr;7ov8W&gGX5n$Z$7@wNy5UK}U7rbTd=M9rK~6iJe_VFsvl zCpYCI7h@1crTP!mdV4Ftpfoy&&4|b867yGX6)hT}wLf_LfS4?wTmnu*T!xc+ z-{+l7Ly`jAQF(QSvgwcXqMM`5E_!MRkIhJZ@e&bJ$Ck)+R)V)krjMhoA~j&|&bPRUY(5 zb<$>k*r*-j4)>#00NtlP5fq=_mB#hdr{<2Kl)KR?$=MQZ@lPI$X6>u}jmqvbA!bbC zXwott)QUccpU~uG`5pG(#JxWwY4^uy=G{IK*VaD48DSmv>g7fyE};zxJS?1>_0=w% zo^swILgZn%5}-q8qB79rVlT4)zyR*H@$-*3uio9aRFeLpdyVTf3FZT8XCy4#CBQWN z8qon0a@JP@zEAI0l==B=$Ed^YIaCCow!u51hJ3#E4_710W*?OGwwC=UD$5-xn@^n< zwvG0n>mT2=by)V@F>&1y`?6hjwqx6{vgI5@--A7=lGupUVjVJJc{oH&ete=}Hvj;itTTgfLWXrj}xm}~e#Avh2t`OB&;Swa0G;J1;l$11!oBmH+o)W5+8##N^ z3ez5NN!fP)3CvsFO>~JJG0hl?4vaYB6tNOJx1Wgt`hg6Wtu`9MbK6|Hm4E&EWjb#@ z;Sc&_-Q&u|G_fLvR7p7-LWm#-aPgpL24P`+pW>RpN6<|gAt zt}Tr24GjjiCr(1x0+o;vhmU^8*KgnMsVXsr_s*X*S25l*5D@?P>{ya{9axol1MZ|G zAnpu#;tG`Hx=lkj8X%(KXsw?9fyQYGE2~LUpycHg5TK4X=dK5(CCdI`&DH1k?usnC z+5y&q!BhYi@{vD)$gS?`LfmGZTYgb)v#-ny&CzF8{JG6~H@eDeoRrmIc9nKlR#wgr z>?gNp{N}p{40~&1S*ORkOqSa4%m>|!{?q3&q$=DkIQ-+!ZI*vRiT}}PR8KEh`V>1%t}5Vrx!&Er466xwG0PbD!YhoV>h1xo1T zXOhsQS9O>kQW53|aFs4B7}|F-_+n6qL%`uC*U!=t% zEDDFiMcBY-6eO0%U`3Kjz+g3AeH}&9`i|$zhm)4q4KMLFT5l>jw!)jPhC4wC!;jw51ba@d6~$d+^H3id-Q$W^dJec&`c9H0FW{v744|8^ zZ&;M?{`6c~^-5nG{CkW_fS9G-s0#l?fynq>ANlJlt?_|2#|4=0_CTd`=5iW1xHnyf z(=-liDT;0%ziG2DAOG~+Ggep;z-OoN)t#iIBruv-u?f!;zHa@8hNtJ;E{@ursW_&5 z`-GZrSp6d7a55{mIlv$rfI*!AgGgIPU{K$`%W;(YR(RlhjBweppF?-bN<-Rsg(!8P z=S5;dUUTVnu0Nb*%eSjmu@Rc4mxqUG$F@Bm!@|Pq3w&G|vDRuB;AX`l`i=mjIc-HR z1%TsYlBi^9J3{D)Hpg}_2E8Uyn;4+oMq+VIjOn8xj07_st&KlV!|z(}|BmykZL7g7 zFq_`nqlcR_={BwUr`&}(77(?6%3Xe$9-Ae<$Q0e;QB6rcx}cQGfl zEi*+6^!^~#nb*_48mwue{`+zF4F65n!@Gj0=n_*|JR8A={;AITK%PfDF>90~lZU6> z*Cf(T-pt?0hv=e&{St<7CcRnoS9ZgHd-2g(UaJGfX0(;8ALf=lTduC=DUrKm$)`O3 zlvTUZLu^Le-JSaHm2g^gyeiznY4L!MIV43mJtX}H|NhkF&$P!zyxzXqXdPUqEg7lZ zx;PEPgzM8jZU%zAz3pBs-bpdP74kas^@ct?nOAk$6&Ig1%fW7r+b|r_b}kX(9XHG9KVdJ z)WX^D*mlRTmrN@KJ-jriy3Y-4<{wg8VKx_0)w4f$q%FH-)8DI#=1Ks8Xhw z%5Q8}K5O^k2WH_$;*Yz}uN}MpVi0QoJ|H_^yPzw%9v{CMBqiy47%>}ytV^cG(mPR* zP1>N?6kKBrvZCwor)`ms({U+tlCi z`8MXJhUd&5+FpG+)#}&)_d|O$7W?7`evTSG_w|t*%6`~eIyV}LzEl_e($_}cfq?Z`RxwyncF-RV+gIZM!Hf`VAU1QnrAOARYjaR92BO@aVxA4KD6WgXU z?(OCEX*?kX71}vSHyuuUlg~1){;1tG^wJ%q+@BChm%YK3aVo*sxCPBIaY8=KB}CPz zsTWZ7RYF}ciBA$iEeUOEXBPS`T(?T9q)s>Sul#nYz4&nN@aBz-)KvZ^O%;3qq-M-<9b#sBMA6zV|>llXS&@ zCNKWJ_Zy)yGL_I=bfw`?6~8(H4qXhELR?TEzSoVLH)phu$aH$#dAcw=WNUOwQDQA$ zNvJf{^t)OnDk|@|!YE5^?JJ-dzHY0aX0!I2Mt`)`=hR zYn{3sDt;Xb=yu0rEOamZmDj(2{y^cm`7E`fh?-9MqfQ2@&9%b9?qMyx5AL=kGs`&v?VSX{C<*k?Vm*TwAUD?# zL6_fx!ch`c*Y_P_)5Q>2yUlELB>L%;0E7-+;CaA%zpjBD5$gONCiV%AGqm8PZB$) znoKO?+3zInEREr9V!uazZ}%9nR$%utKibFzy_%pg9aXY!xIn&5XTPE zX>oh_J1WgJLyG$E6#D%+fELYE9s~LO5c*E9_{|fZ~TlK69n810Sd=)Q^X zh4)jYWV1(qO?|=p@}IU>l1UW4tB@s8+{^xMaxrmxzuSK!crU0-V{e^Aqfom2D8a)fcd8 z?Av$BfqU&iGHi8)21N_k{U%9C{p*pDHnp9i>{a&GP_|@dMWO7y=lA*6{rOzq-}U=lzy7-K>$>mG z`@GNV^&F4qP~)q_i{ry~W&B#M3MV@7w)mQIHzp=Ye9YR-n>31T6q8uUW&z z&)@w}-(N55#lAyTc%=lj<9kO6gAASS74& zoUp?28Lg8Q;RTpf@P3973V(IZYzJAi#x@h5Ga6eG^7YI0c67U#j=>+N;NW4`F))}} z(nBuqw4FmbXHjTqs5!`T|KP>*zKQ&$M=Z&(E1eNRLC0L^cl`#v=&^pbv?s-Fg8t+c zcToGMhED?1=DtTF=b{_-)ooc>E!?oPYaYz_+xldTrVr>YTJt#pI{gRCy9JP~^f`0Y znazW_g>?qN{kIl-`IQK2azVD*{ZA)(qQUM7gUu<0RfEO60hy7`Fz=3oF7RhGT`!Tl zw#43&%&KrIMhBIpnV!ycqR&u=tl&@!-TGLgaa~$gg^sCfTqB}`d2ePH5>&ttb+G8k zN)Ecz%c8DLM7~~K3}{5Kth;y|j`%@piPp@#AqyktXslPgKlgg*YWV0GI`(v9p4do< ze#Pw9(IZ{58X(5O7Tx}~SiL9_=bb*SSPrH|l|G|`_1Hz-O($5Rpl^{en(03G<1V%O zAp}Vzm_$I@9W^RYD6gF>YBdLHw`I&Aj3>BZEgx?<#;Lmzp-WILEv8 zO{8sbFo=7V?9Ds*M@jhZ#5eonN_xwVyZmY6<&C=4tsRS->gjzoXlJbSOtc7!QBSb< zX&;B`u60E8xUYmDCOc#Zz>x$OJ4&xG`DA_KJj^gG{^{GJn65Lpa9cNg#&#GO)!Je< znxm9e^%5rrEoGDnW@ddqQwYb`%b`pn%jC_$opx5@tVm6Bsc)j&x#3F&MjLD?DPZXlE*f=C2-2SV0{t?uSbWpISISP}-(}V&dr_u)PVS`X(7lao10qe%PIvc}O}#q=kj7 z#DW*{GI5>t?!JyCC$csmb;) z-nra|)JJxg>x*OVqIxtOaZMQ_vG}nL2IC=NMtywajfjPDIo!kL9ukPcdpuIGz;G>x z^88s#zv#|@${Z7aw|qG2Wv!yC6-7j8KQq=ITD3kOWH7Zdx9c<16b(gE|ee>#)P3PwfC}sN%UVX~W zYK^&t$+42Xk+n`W%TGi){;S6XLe%6#Rso4loB(rNLhPurufMhG?%xz|6+|{bBR~7z zjrcl>lAVKUd;Fpj>36P^vU53fTAY09_}Q!Q?|qrG*T_|0o`Lspize8>6EAKM># z0;xPhD*}%8444<^zl42MhaBOB>9%kg_9AtJI%6Jnu%5_w^&VMVq1=BnoK?8JEZ<5- zlHAY-Tp8zbn`t$>$QejcD^2@$((<~eX2{;dMh>0ZFX^YR!^@=yLy0u~f%DG*2(sqv z_kxn}_5EE?aanb^X(tq{oOA^zNZ#C~6#oOys4GK&^HvVItz;Dl)WBEcm`~Hc2xNJQ zyTnU`lsCtE>Od(W7$WP-Gpi_RSE*;r<7Zvo5J2$|%P@m7>4rlr-*-+ea9kYq5SB{5;9IKp}jY=~4#` zDdpZbGPEG3(tXsof32ff=S!!y;UNDaG#RB{_M)UJtt_^l%2;X>*Y9$ca%L9IQuAN3 zV&rPDulcb#wx8s8`BU8b^fQ$0+~x^5IIdk22+{NP%b&bfDtz}#SN`_)_8&J*5aS8% z&y`RnZ^ZWSPR*!d-_7JyOTJm$mcQ)TkeAi{7t`uD<1KYqYxV7Nl)|d4^SiQKY<{!R z{IBV(IDG80-g;)^>|HSaOQMpQ2~}_do!C(1qm4BuRhu`<$oO#i2Eh-JBJ4zUrtHCp z=EVFjCeEJ^B}9JA2t=GbWN}{9iD*4!>v9u7Y6q%l+vF=fe7y26>E8!&CoK!stCxR2)>gU~E zy}Ns3-!U)hlj}-6YiOC{2nW>08JIKr-gG*0^|>?dy5crPQc=zZDcAY6vfmG8IzNcxJzDqWl($ zk1@?R%6>90_XM^T?b1bHPRi}5Qis@#O`j%&+@uqt4*uRc&A>{vIp&`%m`&iKt{yS} zkgNRdtvgfe$fPWNA7d*W?%?26sHmveR(X7%Zpm<(61Cw=1bR!8)$tdx9+gXoQ&3NTYa1UFmEzu z?I~kR3luq`f-LGjUk4BMRTlUr8$$GdrjG8G^bbQ%)y9FIZ_OCxFJ(xKW$S53iap=6 z7-(Smo^PjiV9->perRUX1iNOlSOoR)$S*~9yKJTXmj#^t+@RCvKbxPLMk;MwiYHBg zSbU zg1w?H?WZ<2Hms-dxgU_FEXql5|Hjl_Ry1Q3-vG<4dnw+&rq4)1-8VZHn^leNy({`T>)b z1omAhxt@tlE1Obu<+E|=zAYgD-lHhzkCa&Hu7oOs1qDK}k%lyn`u=_z0&G>cK=qzQUb#(>p4nLFuHBwtFJ5o< zU}N({D)s+wle4Gw|7>zl?s->w8gQ{6zb7=&4NvC{uXc!iBWj|>Cf)cRz4$#L)?lXT z+*GJka%uLYgrI)bEfVVlhW+<)jk){A($cmjp4$EaoeU4wwnlwc!}PIPAKx4(yeTfr zw@`ozhU#WX(PP%Y4(M2-j5>RD;*6XzrMr(GKTar0#VvL7#e!~?pYylCX|vZa-kDNl z<jmbtXk3Jsk+Af783T5;^;rn!v(AwsZfFM^^Z35y`j+qo%yoCcoQojL#2RY z@LjSl?Bb(Xx93D=9{qh*$`w$|8I_}S%2rxUX?HeMfr~$Tf3GCy*)0~*k!75gkGN|L z6uFZ|x%Wmh(#i@$gr|SF$)y&&>UFLfjMB)>whl2)ubnPst5&(z8=MncvrfB_E50tC z$90wd`w;!SS*^XlxAtyz*KW7^sJD0Drm|ac5W(xHMAntvb4Y76P})=aKTzX?>|nv^ zYZi9IaBkJkkxI`eq_T*Vx23o>NFb<4?eqIlmumG0% z_p`2n>JL@h(ym}f3sPyjqW;0pb-xd?v$G=;6Af(`q>lmtQOuiT5x442>u*Lh#S$}p z+3r}pm~gerEgD0nINM!%yO>R$46dkvv@P|PP005dOa7d_Js*scQWPGK$05}osxISj z{pZN&3GK}hz4~*v);eYW6seRI$Nnzo;NZyFkT?KbZ;+ev0^dZia~xPrM#>jB|)$UV7U$HBqQ9s!$=aIIB??~3pRx4!rLf*9n_@|0O$;~Ml? zi7}2y|6t_StNq)qBplxhb7qc_4zEOC{U5lw(bD*lQTQspUof&#T6X;gVds`OcSLHu zE1PPnWm6Wf@U4=ywn$k;K4SdEbcyH8g~ODlT0=Q*9BbD@Dx{J_2|I5mYQvaV**^kk z<2u#(2RGD=jFhRMI-^oB*r1)h@7KQ3yKk`L$Rw*va0x-VyU;JY&~f|gxU0pP;{6>G zyY`-d(z39ye0f;w=hTND1mY-MI6vNgaTv?9DW`U$Ls?Zz%2QR|6{~7Y`}LW)&xCRo z++f6{2@K#_E&(&dq3nI^c0Vulbx$rjUO!v_QI3&IyP%+Ah0Q0`s7GKZDG=VOZqpr# z`2_Z{3Rs(arA2>74!m~zTWt5XmNQDy3#i*#TRF=kXMuJW2cz4bJ;Nw{sPw;)ZsvcB z+PF3fbE!%*1{gP+cuvwr;d{qth+3d_o(SGw7i0i~u}UA%AG88EYmaFJ2q=*jsE_&l zhkmd+7>${yL2jaPKU=b_fyb|SfI3WV%-_jX& z>o;Ct5LcPH_3JkRd>;!q*vccl#xe6S~HB zz>xKn$mNIO3j&YXADRH3Nq$kNZNmwBeo9e_tF(|*G@bMcR?mhn>kU*7@%WgS*y}wF zQ43D>k=Q5$uX=U$m;TrGfgWJ~DVVe@SP%Ha?cOpp4mr@Mu##r~6-@K}vrx0D(rsE$ zRreJ4gC3M)`XWMJYg2S1uO)P56>DZ-)cCo^Pp2wp5x}c*@lD*j9uF2~=C?-FOU2(L zT;lJSE|Q)HS@cG~A_v`HJ)6TXfc|UY*1k7%0jhY1x=GPgb6>1iwxriAT-?&czkaKl zN9l=}jINdK9I~mt1;*x=99_O%uu?GI6W86Ow+ljm5`k}%z?5mN4Udjh*oPQy*@B;G zQO9x&R)c3~`>y2pJuK0T_m`ihhda77Vs`8Xm%s|b0o_$c<>ji4yL#v;3!@9Z!N zWxAIUYnN}H0^uQdIb=erPEzUQWbYRC%=iyGi~fhz2@XSG^=lZUw#z*ge#$@^6ZMD= zJ#8zqA4rrUbDsnyxwNA5<^!7aAIRjwd7`oAS9kNrK{N@JuC@pJ5LpprT)VsFeqCN+ z$VO~;uX$C_w>tVvabtBMFd~Vv?jd^YSKc>~%1Y+u)(z&B%GA=joZ41HWeyo_Q=pc2 z*m*je^9dwu2{AsntouDY-)M1HTsF=Z*v~EyrAhdODG8`&=d!O0cEto#?I0vAXHe)Q zYA43982)Wfw1T`^nw2X}2{WNOyV(N+9Ss`)9f_=h(c&>2`ZSSvcSjHe8+XF z%{&>KD^-{Rrna_zzmp!&4m;2Qe1 zQ(v}$P!&#_)of*UE=ei6K=5vMh$EK1in-DjdGM?3X9t#Ro2tU)gxYD@ubMhnE8hLi zlq~;jtwIYwq88D4{|M!AP+j>F4N_X$usui`|u_;@1>{S!@ z5jSmCc{jQ2jp&~JQldqmZ(`ZMgNiS4gT8e73#mskYZ_%PvjKFmcAGn)DyPwtZHvK`@St+pTgGVR`K{KM6( zEdJ28ESQNzCYxS+9{~%Uq+T@0KR&TRh9JvuYi+ymArq665%ffRA*`{Rd-tX}D;|_7 zIOeSGJ_&yAdfri=-J|7x#M0IO{<@gh*cP0)&C0=wF`v_jANy6&pT*)#*JN*RSTpJ- z_ZfJVDy}qpG*1%Ug+y#rN8>vN1%|ulE#%VzoWualuz>kok^7>F4OmE(KP61%kT27Z z3^I@waEPl^G-`kw{OpY(Gz+NS3^uoGWe zy_Ag1d#3T7qFf7;gyOAG`rGA2oTUVzzQ98U}#5tWbsCRp0VLKrqIsNFE| zgsWI98hb~YoFvoY67|4R0t;`2Q;glNkUdY=IA6qqnH*iG83IY-=xct5O%`#p7$@~I zi>Mz6ZwtIuq^`Od2_>Mer?;0^6;ymYD?kIlfT--|asv&kHS#;7h*Cg4+kBFz<_aY3 zaC39>CIAYr06@IF@h)IY=M`6xXWhRGu5pglp;74Uv`}A>!8aECl2XBi;AO7|v?tBuOp>9lFThBo{n@bVcUP)ZHbQQ$EkBAAQ+;^fBbi_6ePV z>G&(C5M4yT*c+Vtu7XwSsm(MBx%tzCzlosf6stRjkPI>^nH~>7t$CKK%QILZTuE+_ zf`)Wbm>#}@yw0LZ%yUx{R>AdcE<&uTwCUJ;Cm+xh#~vPR)+2GX6*{Ar2RH48mckNF&ByhM!UZS5zbx{C+_#>h;;j~?K}JUS;<9P4B5jf zH8kJPC}wzLBqWPGquSN`{y%BANTuCiiHRa?#Hj%p2^vAyZ#E1vS7l{2p((b0T_UmH zaWC*+ELeNCiJuu%LoEfAhy!WjeKeJL%>YbK*59FttHVxXtQWFRBf%u8twuVb+|fV( zC9nI4M1r&WxsY(OxBwL@TmA1%O$v&Q@a!J1Vpm#QS|UGvlGgmm1ACl?WR8tV9eqyA zxt&^7Xt0N5P(d;<1XwiH(>eM7d1FZwM2O{zBv_2kI^MiOEXPtR8oN;cSFga;=JXip zCn&5GEiE(9tgK!uj_{Fi-lc=+C47DufvaC(iTyyAa@Rc;Mk)`^6NsC}U!CRUXdkvn#0lx_7-7NMouaFy52w?2cT(U@(y|g$1Xy zBH*r*Qc?={(koCO3)I%>sd84E9<4ptCnWh$7v$~t9!)Il!x|a)XRi(^zn{S9Fon+u zos@(0irON$7$*$TU^3t8&BPDS!^>3#N=8ztg~*W|%JjmmpZodd%CF5m z{d`oHTZz(2?X4|JqP3w4O@~*(2aYG*G$&Hzp~!-uWJp7!BEGJrU+-5NBEPl*v4FX` zx%tmVbo3eh66**gBC$FmJia}!NXBc?9aq2Ii{V|KA)pj zn4-{_v0P?;XR&WR-8bq6DMj|pkhmCAv~JCNqrP;!nC~s|NSaCiC{=iiqX~0zLDwaz z=#e|K^aN)O4Z zt<@;wht5ZMc)1;eS|Z9&?b_^SM(fN-w&i1ZR!Z}U%K5FZapqR)K+K;Sd1sBV$cKPR zxcsrr)S(95_&J{0dCg|2|N9B1rmS6@g??p{eu1*WMY{bo;ZFRpK*VT01W|V`H;2&% z9iXV_?~n;i&mD!+t)$3EjqO#1Sj;hls6YeJl8OfL(Fe?*rSXQXveU*otXecdgG!yo zED}meN)`@^RW*mJ+zHi=ug{!`iLPFZ5I1bn&Mb{cX%3Ol)G5&+{nXj1wmJNu*2{9^ z>hPm6Ai1Fegi9QTbU>`d45ay2h|5#uJV>?EK_O!)R{H|Se0+^SZkuW7A6(K0a0&5$ za7jseu=Okf6C>kMTntdX5o>&hlh$_U!~!O&?dIlW$e>_o={ot`NKT!r?PZ={X>P0H zocx7q8}zL)b3sfYVI)yH6mM80IE;J}T4yhiN+4$lrDK)!;0Xv-WfrPtxwiqu;7q=d z`+fqUnK&}=nJXlR)7(F~k2T+P{x`m>5%{}FL3(cb`f%i#o0CTWTRG*fJP)<#K2SEy zRui8W#f_JZv)z6g$4Q%$iH|2)j@`N8g$j7gEiSeo{gJt-?GX+6a0Xh$d=1+h^{YZNH)G%lq z(i)At1km4C05yeWMbD7H3if?a_3aAZ`yl$zBe!WMoGV0Xo_C7Om@#WoruxNM7}Om7 z;vQjsY*hQoF5~RV*Af3 zjMC%415$K&0otRuliU@Oq0gUNz^;xi@X~V+yKw5iD-a7JXenUrT{bj&vwRN(PHmIS z@-O2zsQdKf*dkz4L$b*wrz=;E=Mm~Zl@)3OY+*D1A#|&ma2I;I4Byp_W4hrdI^bX~ zRO_3^@aS@{S(lxh?8{B_J;r4r&BaMnU8WG7b$_#@UWSW0xO~*FdT+D-Fs86>N3lbE za~J{wM9E25;>5pO12$g;!xToc|Ke;SXqAteJNutC1Ha~tt6U=x2U+f#rB9*^hRF4N z6=WaLzRHF9I6oQ^eHNmby%~^`klei1cdq%Uu*ss4`@&MRekV5XXB3BBOJtxweFtAm z{|KWHla?3SaQC=DM`D9!=m;C|PG9>JA-!YtIkzXV7aU-TgZ*e-ZH8Ge4aq48�{+ zU_>WUzcKK=Eojpv3#~L8AUm5IoyW*{Q!mn<3z^$8F&}AULPD$PruWm+mzle4g3}A{ zRHree-TikX8ZWgq@%s|$#>J2szmDXte9Wzk%L(#(Jl}~v@L2rNhjCVn@*t(Sdk^9l z36NSr*dpE4-JLur0c(ZUlA4+yn66@yo*i9NRL}|DJ6zMNW&Zh4%Kp-f$a~bWXvotL zNj?YcmCTcJOyf6?IJM!;Tvub)y<6kRMqa11jD9QQs`LobwvG98Qcpx&YD7a{&57gKf z3O@Y*+5~185l((%9F(3(*LL1oY|5fXTU5i~x(RO=#!$tVG?X7~ye{fLnkJl>3rDsQ6ZZ#id(40(>_- zU_#uNE+P-*g9T^;+m^Z9{qF8(JPb2`UZz$vam+!A#K@p(KfqP~#ZT%-SW(ZTe$dhV zTJ?uK@}9cD+*azK1W!3fu>(xly;c@NF+rv47-^#WK})ciBnH`R4+=_7q~Hcd)BsT+ zOkoAe7e}i5(Y=Uwnh0pcWaZ?Vp_^o?dsEf>!r+v4RALU;K?v5p1`?hPpmxw*J^mht zJZZq;P_1`)4G1Ft;8jGY(KO5kssO-kZcv?m5xOE*=F42Pz#Nf(Qa1Et(x@=fMOrIq zyjw>_Q29+kT&c@X&64#Sea&dsqI1Gl`hLIqW~r~zi&MfFNtdqRJI;|k;bHfOmVdE{ zlk^4=pPVRa<&Jvs;)SB_#PnV825q>EKYwpYU6S{Mw6wxHxGDEZ3s}mAIXLP`P(-<4 z>KiYcCU`gNu;6^Lbz3?M1|hI{KfoV^pYQDWl@Dwoh^pepjng=ZykDG~<}*NNtZ+!e zcl;TdpnG*zqsyWBLvMBW1ca>K(>V~b=9L~lv`e_E{wvKjr;)WH2>WY!p-oA=&07wn+k>Tg4+07Ah`bc9rye9JtxXIWTc~I#AHZmnIN0GLD^$I#_+Vb!Ps>B z9+Pysyz`qiN8{1RGxjlPEOk~ARo-wHi7HLd0q#u6Cg_n8R>9im0w~R)iAR1pKQ1Am zz9euWo}`S;G8`JWv(!R zy@UO5g*r^jS^W>TGlPSJ6=2KV+$g z+yBt5)0#S1yIFpzUfl?vua#9mK9`boby@uv@?%kszHEgpnB8d7UZ9(q&WpZWkSNPpi2Cylts(>af>U*H?^%h~;dcB1 zTzxOF&VQq@6zRL5bME}FbB3ymGp4X23}je0`y`Qzy%YO=l~w~mEDc`C~!PTxN`$`6p@jU zDc4hA$zh#Qm%Xstj8-Kuo{}UAiv52)rDP~r{(+<-q5`m+@tmkI;QeACs>PXU&D4oI zW8a+-Cci*KF!<=YLpfkznar`hnJRWayCcu^@V!~a{-ZKJ)hIrBE%#V^%Lgs}YBnt2 zO6c&zy)p{3mzxy!$>?MpOFWuU?IN0Fz9ldR!hbWsr!x_|&8@Bc7Pmo4DID_rc@%Uy z-HV+U{$0Z@=b-D1j33vW`x{bI0J;4gvHOUUm1LDqR>M8?jCqJ`Lr9MkCw0u&a9w5g ze|XzYbLlSDo~6sIMqzWA2`k!bJq6#Yfa5Ah1a_C-K3#gfWYzlR(973Uw5+vrk%6_5 zQFuT5;Gvh4dJ$L7L;%%L39%s$2@Q<|aYtfNYD$sycP0dtX`2Mqb2Cev5w*ur2NIC_ z0Q@y^b&r6DEnoQq54x=~b7Up9IGq(Sb75nESG~7x=-x&~)01zq2!Zx29eWpKwL=UO z*Y3=n)77FFELQ{qPE~%o^E@hbQOgTr^d+Km4mYh)RJrG3a&L_W5I|zX$Ho8e3$|)8~=OG)+C^Obej15`W1*?4RDM9wD{HAuWl$dbu`thELvHzJDRb()w z{bHJCFBs9C?dD>Ut$u+Jkyg~;H0Nj-X>dRQj8!`BC>Cu&#!?}pESt5apaPxlOrl0w ztsx60iilP8qkC}gm*g{N&*D&>=W4hAB>t&^N`TM2i7I?_6#U>F;#48AhL1A9bD=yQ zeHDX&MXk~OkHoC%TX5^(-tFCce)`wbwSGQhkts!IH4Er7hmI;)E&T2cB_0|0|Aa-N zBiM446V#Nl!8SHtQ*AERY<4(b6W0*4PEnFaHPV51*q_80!LN?@an#pk4e3PfGvN(yy-}<;joEa-4`UQFcCI~T6&$+9z1|6OQB_2-M zO0Vu^k}Jbwphbv{jHD(#c`^#Yq_TZ8loOaBw^rnvs|=MjS_vS2=_~$+oBoKR>H_S& zM6=yH}X}oI1>^QGJ>-N87n}UVK|^P3n8=}`vT84 zFQp(zduu&0{aCnRaN{#qy1UbyKOD0Ztdq~&fXapdiJuSY+a*%Shoe{%H?0)aL%eMt z-wuAPP=1z_mGMVzo|d)q{*(dFjbk@4#oh>wf8#arNpe@JH75+xm3^ z!446VXveB))&$sPF_n)NqQ&gTwxX0k+!^HY{=7V+Hl!QW@l zM1+e*M_85fUDxgd{Oh6Ad&8IcrjG`NU-CnVo!tpR3ZrNJ%F3^b_02p)mdw9J7Yc&Ig6EVz3DG2rPvmNi9|2H+^ z#pXm4ys{H`%X3Y*Gj^jN1dw{C%jM+A!#yDiGiB<|D$#c?t3)9rZ?&$DtrOjaR&pdZ zVf4M(Q^S{pjS5r3H1M(R?|5QlH6scA7KQkutW2lmkhRT*>| zQ@{Cy&tyH;8X4$S`R`t(zVxR0B>d_MyimXjf32TaTXbLKh#38OX5hLK{>I^!!(DZz zk+09z646~nlooOXs3obLOVYbUgK=cW^}-|nb5m^3IvHNb`a4heMNxrd^d!nu3II$G z@f*nBiz2cjkB?)FEK@??QbO~optHjiw1UdzVWKmKOzlLCFIbn0E-A#05WP0AO*zj} zYiOwWc*9oPw5nV=dg_^4`6Y&c-H0pXR1&mDkSOYA=y~{lS{W61t+GokC2JJdAS;VU z+mN9uWL!P|%gS7n(t?aXuJIy=TFvUy%t8V3EU1GyC|g{H-h+F>SeY5zQles+1kXmL z$ABWcvD-u%A0^Wy{6^R~5+geWR6a?|3_JUv{HlWP4-{Dxff@wHArW(PbBb2nGi_hX ze|qH#jmrwgizD-`58cwz()RD=9j15zsvYLbxC1k9l;r&u&TFw0-rZQf`)>NuYTl$( zN}qo`D>6^Du(%Vh6B0>9yHGYxxU@?23(?8}Q|J}o1Tobnl-j&{B__H?b14^2-hc5A z_{;y)kA+->u0VG32J@9*D*UK1w3$yNB9`YB~h{Gd&d?KG!f#a zBCy{qnqDy?eXo+MYYDQ`_7we%D+U#vbDA*B;cXkY{4(+RG~l6Gbdk) zExbb7VFY_Xz)|qJ!wo^_FoDRW0mtfU2=(6`8lA{Ubj%r#fF!q7SCSOu-29;weuh}h zsj~5MoAYM+mmn$wXzsG=5kh{_ZD^T*Xp2mmVgu2*Y`2;oFo8{7egh~L`1p}Z0*b8! zhmzqbkdrcg6W>fC-9zkNRl^u8{`-9{f}U6m(L-3QO?|gq$@=>ArF9-sGbV_8m%&al zg(s&j8T=bWz!Q}Ry`diPx$?PBSgrN(jIESAOa+UTjlEby zhOs+V1q0oLSq@f`{38??G_uU#TNMpk1eafUQ~l4hA%_>y<>sC=gblBgWyp#I3q_ua zv171a+9WxQm+Ca(G1S{ zzm9yQbrR6ecSSEkp%EZM?w8~bffvYx5| zJvw%}6FeexK>pxRB9O5FXL@XQh3vV-c%5q$37~nn^F1a(3tiWR-WX+fUe2ffFH51y zH?AbX0^lQZIs(5r_iF1V%)%Zpr3Q=gjlff&yfB4DDK%eON!fo z)c%D;8*1-{l->pLHUK%V!r#(KoThn+WotaH@|W2x*K^Kfx2AJ|W~SSfal| zv@5gF>odWGGzzM3Eo30-M9F)H3b|i3pMFyh)ZEhSo5Kh;L-l#!i5_xj3=bzSUMDX} zq5CRVHNS-#>n4@X;=i@y8t;%53^eZ|MIEQI}w*toqn)aCTIkZRGM&WSpx z7Q6TXZPR~MMOykQE!l_qU{7!4xsLwy$m;k-tIB5sP zMbz&xZRuYMILuvV8-Unu0ZM7mKrMeRV-Ic(Hd7re%}cCDoqdDNv01XTge4Lc%tGS)RRGUC2Xf79b#$VD)W^0pZ) z$&R~Ccm5dx0qUcq?lD{H&ghdPQLKPfwuef&+l-<+(8u!sT`lWp0EnOn%7c~=1^Cm; zURf=Yc-Wdled&?|bqK74%Z63)*+Sz-7kMko(6Leu0j_D7Sx1sY z8U#KN7$ChQh;DTX>UM0W2TOD5MP0Lo=hR>tb`3!&UohMG{R+I(JR!Q61H1Xi?}hh`+Yq#^3dl~r!Y~mkY3gQ~9?LS?aEcXR7t#>3L|Y9;)K-f6l;x&A-^7 z=XE@mg-B53{DreKP*O-3$$JfkXSpz256g}tcc2jqJcC;tB9PoHf3bYcHJ9z zOfXT^8~<2RB4D_utNkKAIvNeKj>zQX`Nv0CI#A6xu5m}S@A9|LNE$;MPjq3Z9}s1yo_F zzJC3R3x3@}1o~sIgfgLwetkR`XcV-cKdL_@I2akFG&D8ab3QCVRU8!p{d5;tJso9% zKyHAN9x}8>%5xy&6+^GS0AvKS<+(!lM(gdF~ zc8$1ZT4#jN_glb`&1m`|J3Bi9A;cdBfNkFR3}jyTv-bg}DHh<$$!F1WW*cUjSW%L_jhR8ta@rP9xTMz|;s}`8OI`IBBbgE?mQx85^ECB1&Y{GQ;KLkO|8x+MlSKwh*iTNp# z3p0v6rxlyY$Y9UMUy2@>-X3U?Y$f{`-D6Z9R_v!-0k3oaU&s)srz8w8Wh3bY?Z5ff z(l1s5J$<#zr@eL5NS%qH{#5)A@$_hg!Sw{Rs!=d;)4ITHE7JA&nYzlk(*#&hi7ac1;GPCHC9?gb5G*why-aOn}U4bl_7@SrMmLUjXd@{v* zA>zjmy}~fwR}jV|m%hjNovW{D8~XeThz{wxxjSWbfyOvC1uc)P@~6b8tX!frZ9l&{ zu)6MwrUz>qA<~b~#w!5^w8dc;q~>iV02cgR^GLM507;{@@ih>AUR)IJAZmj9`7zp( zi~YU8$-dpb714zxIfNDt)r+pRk2M1`xc_|}k|Dd%5P+icSvc?r^C`1+O8#l(V;;YD z4OmAi`B8XvBlF~!aJnw^x@VTYCFP!H1ayLtf==KS(7+VT&GAQs>~Pr7WwCR&q8qNO zIS3|LMQ%U_;P=*f&4#6tfCd2B$J_*dOX(oY&hTeO&KeP&gK!$0e>IoKFp%fr(nUBQ z`6u`%pZD(G^zvt1mlYLKdIw`^+8{RJmr+!3#;KA>r5!{M0g7z3E39P&fU#^=5aR)J zw-%s0x!Wqnd4kUYi!J)~`6Z96!A;~bS_x1)f z;ONgxe8I6>IO8kcFDIK2T5cqcvpU>>dlA#l+x(H5lKYGkj!Hk4JkP148k*!E3UPf= zaJ}O-UvI(1h=motNorLS;P%AI2o0r>W*Pqvn|t@cMflH3yYQ&J(y2jju;IZ5sp;MY zZ|c7)>(c-7@59$SRSPnds3l6ka3doPV%PXXJ`6*jI$4zz^jq}OH#wl@c#hx_XoDd9 z%&yfDp!{O5^ho%a`mV=Rf{~_ow;_*+$UyX;J)lr2n}z_fW0Xi!<_USkoiga0o7>w3 zsz%5pI@Zi0G)6d~GO~hWw0%~V& z1}Cr>C~F04;9OU+S}aX40AhLnrdz26+Ibu_*0x=lDH3Zasx82594sUq{$>we>@1>w z_wL871OFG~CnIzy^aM#z_xYqMz<-&)bs1bBZ`>`zcZB-P)q&@sw&)G^xG+XT}$VfmiIBs2~Ve9@3$2sHoaGF8wfv( zXg|Sc(HhuB$6zNLewLbncA*cl(w=TSKWQeIJ87Qnj(`$vpltOVYOuRomKKpt+4axdTZMdtUX zmr;9G#CyN;_KGG(mt|Aeef)2yc-TewXn8!t989xU)=e6z;?DH+`thD_^sh6n71QSS z^VAPpoSh+X7#6SvaS)nhb#=8}xv2%Hv4ikj2He`JwB9z>p5Gzt;u_LoHGM(UJPCm< zPE$oh&Zjmy-0g9im?be(N-({VN8Zb&fo(a<-q*gzmOMl{>6;*yYlE&$lMkl+#G|lL}v7K$o!Z!^TW(&^tU{ z_D@GLCrFXYHFv)@HYU2*neDXbIh8viQajXAy~O%j+KGTFO_-hUcCWTX2K4p%17UE3 z;AW)uSm}EfAa>>Nbj#iu0B>@lJJ0dfAtWaswC-CoEdy!2mDcH6oE!VJ%3?pW$oILw z_Ds|$DtZMf(N4H!#nh^;8E|o5bH{LNLziWF*2cn@1lIQ%Iv({KOYR2i6F_B;0(*iM zFoK@wWNe*z0$bhU`_3RPU5KfZrG{ndTK1+nyc&|Phw{*Ng>N2kDxnN^2^O&JF>k0B zdkBz5aAN0sqhGI0J4tjvyn8r1=yoFIj_A2fwD z2I#rL($?XEW*OsTM7X2?(c#z|)mpaQPO<718Md-x0rgEh91;5$wu^+hOgA$AHrKg| z^N6KSB1m)p*S@PlMh58HimiABjLM55Bko=I@d^^s2V0d!S04{dJEN-?8h5{)9Bk3{ zrEAT!JJr&k7_KO_ynN`*{CTFi^3uhYcoS^8U9nzst8&!MzlSG}p@^Q%7<4_Ze0yk| zIskHZt|rlGd{oj;MDJ$VmsN1t=-r*b&qQg*8x0I@YtKDGavAD?Rb1To3HgP4kpXmK zwiraW(_Ln(!kKl=q(%?aLTtrF_rcq$$ZOr!@RqmfSh(*D!H07((XlPx=cJ?Nl-{$V zx>qb-W61tAr{zf(hH|i@ z@CKTFN!?7aN_oG|0K-1lFoqMzx7Z5GOz~<_4z}-14Mfa(kfET+uu4=gICSw39gH&^ zzWZ|fU^w$AVhOeOWl@~OkEzyT1 zYlJ09*Cr+3rc2q&e%Ki6J|rD9_%jK?q9j#`%JzCrx)U5zG}MPL^%=8FPYHDH0#gEZ z4Y6B3$*NlZdJ*e+!MoqcB@DqyVvgIZ-dqlDOA$BRSRCbnuow++Y-<~L?*T1jnVK4q zw^>#r6vwq}cDcxS3DeHg%-Xzi3*TB<4XDo8Q5g3+&bJVMz(As)`phex8>v#CQ>%yD*%@~ zQYqDiBU~$>&01MD_<_{LQZTLH>>N0Q=*MIoDg9AbsI2qe=Kd`?W93#-`1-3!On%8~ z`B6k0%XK~(irrR?S`*6&sW);Nh$aw^WgZ}o#mxQey!KJ!xIkqf%V~fv?di|4GcpG1 zABXnRRI{+8>g(~PlYMkEHA%>CG1CRj2o{V+f^3Lma_qq+5By-wH@>%U9&}}zuzAjy z^`NH^+^T@~3wY-_>~so4cD{LncDM6zz0hA!{QtD~m2pw7@1H}6U<(F<0xF=AlG0&- zG>CL5-5^K{jEI4PC@oS-cbB9E(lvlEq)0P>bhFou=j{KtyD#?5{@*Qp&YZ(hW}dj8 z>%QU}Q-uxz%m@2BEg{_}xBc{G?w(YP$#SgTuDF$8>(_U+>BgOfYd>=GDC26WuyrQm z?7<(ISVVAEDDc#xc>?KQZ2En$gEwW0jzRr4)$2HX;Z;@{M)mz`@i@?1e|;u54Q95y55?L48ht^xHvL`?R-@k&zov9N}3!&=lVF zPkm}|DD^1XNl1$X9mjrr^P@@x*a$NPzxB_nh!j<-uz47Wahger(U!fO=y5f*FMmjtKxJ}*sO9}%9m2zX5c)QA> z-fQ7>J@HRYaL|=41--mWupP*wQ?9zZ9XqoN1sb8K^{Ou1DLI{#m_lCFVaRoSi zKbI!$b8P-F%$iLLWBDjt%jx4xPTsTq;3C$&bn)XY!vdKVEp?2x2C=p}3IFR?#+27E z7|GY9_p$R*_9=CrSqC&Zps^J_ko->-2{Uul&-er7eHlR$QridkT;g*S*W+wYgoi|i zdYN_(=2A60o;o0D*QW2?HQ+tENmO(DRl;SzLyj#6>|mrB0eAbwrz1y}y_lbT}0Vh zbRfsepRvZ*!$cP*%~x~$5AuB!-ES2Hfo-ri9NDch1osuVE90 z4~TmWdLc>>oGw9Xa|cY+MvK7EA|?O9z6-Mc7k;9l;cR@Z$F5eecbG$5Y0%t))cmx15s6f0bSOXyK|I&;>xFP!7_j#=uOkivytN`DgyX}Jrz z8>I2egV0oY^cHP-X}Pc?myWy^{tciF2TPCb*Lsk@1bIEilAb44xc|U!MyY{^qI!Xq zi0?`SYi-`aa{i(XuHmqg-bHuf!W>2U_I3xB^RPg6Z}_Q{unAXbZkafLh&f`6+oJw- zpe<4vr|P3encJ=w5TXD*U5PhL`&$G?4goiZt1bq`}DXiv4EmDJ#SrdotVzA zmPk_hrnjCJZh+g1_ARcN>Duw`B8?+BJTa%j!+j&Axxin-#?S3KHDYBClQVP0Ldl-H za5|)ENMj-|KYtX!gkdNgKu!Y02l+Q7J}NLiPxuMzp7QZZUG)s{oPvOD*fEnJ&lMR7 z_Wfz&3coHg`MLstne+_YL2~bD$&|se{V{#Y3om6Ed9r^*JuHl+!^=^TZ4;ZQ!ubwa z1qB5$^6;pFspxqa8?LO_`;ak74IgG=N_4o&)3QWdkg)|yDU|cdrj^|_NnE*1>l$50 zTiw&_CMl>-(uo8|!@ESGxiQYk-gULs>UX}HhgJD*gjn6e)>_OZ-RzMTcD5p>!+iJ6 zWzL?GQP+QKe0R;o)N7c6LaJimeLkgUv}X^_fkLW23D#*G86eM^R@eOfwQ(!Y8*U%_ zqRKl9S1l0|lTPf$9Q1y!T+UiI9_}SaQ8{QP*c&|#%-?-^&YXf(voD1DR)DUj_Im&+ zWYe|Q3@>k(h%sMijEZE&W$~u4-`7<0;mTI#W@S9f-aS3~Q9CneDKIa!RG*I75y zq0sC)n&V~|;`@|aWi|xSe5b< zi4=U4xvH+2tI6@%JRDnKKy>DXiR$7jSw``6iPy#_aZ>_&i*4Qx?*Jg9x45_x@y z`f4?FcMgP5AGU>fT$C5v73zJ(Sw_O9fB5i5k#)>5li~xdStrb&P+OcmUH4lNK9znC zO28_BBkclyuW%t~5gw^@)tZZwJ@SdlL{$_nI{Ym@Rb^!c1RCEXZUFU84!U4WVq%r` zp2#3uh}MANf+Y3YpV+LWd?n)EY+{k1(7D1~i@uH|>+iA5J3&B)a<_bYg6Zx35E>A@ zncolmFyCuWZd9}sUYT6jX(=izFU;E$6>Hzo=5Unm%sB4WoTZilLmjM zgDASjSi3doB#nYdEH{Ptb_|R>_r^26Pu27Uje(fp#9t%vo0W?pEheh;T4xZ3*H65{ zy8Ec|$VEGgv-B2|+udfCI1woK;!ufss3?=3|#&_&ZcmABj*3JY!7Ia{^s8qhZ@a?8=WT)5b4Ge&|BFMG9 z;cZEUVbk^*I`-K=FEs`(+V00Wi|q$25FiaN-e?Mx%mAq&S6LWcGeYTvM!CsR4lj$J zQ}MnantoP8QACF|xg;m3EN55!h*Lowx9-tI6_|~|k;XCaW>9>7mV5lsD%>`SP?Sjd z+=z3&{C>*q{`qD3_SScKsa@Wx{GElX^Z&(2-*I5IS{9`3xv}}VC=AGoYq{i@A$0(D zjES(bH{owkuzbVLs{Ae7ttqPePf}w4hWmsFdwiO>-eL19%A8ww3Vfe0R8XV!C;gYP zSNVAF^l(&@9@9`Ao3tg}=(P249%o+oWm3`PND8B9cnBmAChg3bGn8k}OcI7k4;#EB z@+@_KEGC{R81^R=g@OI{2HG*}N9GP6|8O35{B5n>Z=qNUu>}NxVYrHRw8|4_iBm-D zlic;=LKHf?*I!->YxorMFnM+IZPex6_m5~~u4Aw|(4iJ~0GMYTtzG%~U}l8{2+Plk zjU)`x^xtS`vRz57^}wjhley>DOLzF*)`8ZDxI}|n@Y)Y-WC2`H%)%}JYqT)iXxxvB4nVT^DF@Ip z^f`u2R_5f#to;fM77oiGFEKtmp|wL_~D9NaX{R3Vdg zCxQCvkxc4V)y{)UWuq|;th)|26xAU{rFWEPqUVA>oL^2b2@2Z=(zJHgS#y6}Acff% zh`E?8%6~pMJKj2NZnF1+q|~aVg>0hKv!}8Nf}CX;<6YAYS^0yC9%IUs2QhKiud^`~ zNBiB%z)!k8;8HuV#ZH!?uzf4puy{8_uAs-{=`ANElimm)ON!CC#3pDcOw*tJp zb5^#k^wc*xifD2q4|%?Wz4lnhKc6=wQlGRTzHOJC*ZKNnuwQ7Ve?jJphdJ7Lzr>1+ z!fcX9YaqH6QP0l?cK?h{!6QbhS8xW{0};&oG2}v# zeKNrMN$!oaPFtO@`{4GJY3%o5co2%0vf>Sfz$GNxgiyY7#4ynd5naD3YXNFyS< zziQ=PQ5L2kYb&3TBtFq(+kA1C5#AOv{+0ni0?*{XFmv36$$Xbv4-V)ch%l!^e>6(t z?j4p}`@ZY5YFNpZs()f>-`S|AdJ~!*$5sVfeQ=ng!t0V=RNS2(1& zSp>|h2H~T|i|)MHYV>h!725T?0ki7yo^rCXP*bv5q)^#eB{&S+~XyF zi88pW0p?t$`{>JW#BTI`F?3vb2Ad5ZOSiT~tgM;3R+!%Dyq%0rc_`6PerL9TH6k$;My!o#w!ufmtZ z04vhxz=kd(#6bi-bXvBsu?%G>cb9liGe=8#N9p5r-p3!sPjzj-jQIGAQqHGc(y-lK zi^}f^RYWs0Px{ZIpusYg{RdvjCFW>0%({<5Ba%&Bo>{$!yIq~E%hu#K1wNGZjK`K2 zpz+EVpyvtDt>yr$ac4?E_vV0pn1f0Wz8zNzOcAB3g9{3WW+-Xiio(3 zdRAImCD2*d0(B~0OWbXQc?UfH03Cn1a>YGp)`#>WD9EFm747v&E@UZa3<{>vzXA3V zigZ$8Dha5L+cei~*96q&ycv?k2||oLs{KmEOa63vs|$6Nh-hND|O#2 z>&xi|vld8`oT?m5?tQ(kV4-E8_aJGo>QYtl$C_Zifr!=?Gy3=W%t|qApg9|cwA>cJ zgyP*az#|i&jgf~u3!Kr^23a%55BxmOl=S5AEj%fCVT_Ci#lYH6etPA zlygpaoxw4vQTeE=sHiYs1yXCapeN{yI6xP?0jf0?E-o&H0FEeD3z395!c5wFH}eCVVFP%MtT*3i8ueIu4HN@g4nT zUTE~4J3}qgpRL&DX?yk&52aq4?hlQL{~c5SDeh9H<_n zkX4638z#HChJ2NLYzo~di8zW+Cw*y30Irzo(8SmM0}$hgqdv9)oXvWqYHM1_61}|v zw3Kf1efOc=-S}sU5_E$YN+!pvc7AOAn|anNPX+c}^eB3hu4rZC6|uM)%0fQ7TB;?g z#r1CDUZGJ8m7mP>y2iZi17~NFR|AGln6l4J#jP4cL*;A}N6Mev+9jE=@!HiB-bqxf zuU@@U0AXEV)yv4ra-Ys9!EBfyn`VGR z=z~Hdr%_NiEQ%Bhz3XyVUOUNY^sCwC@^}F=aUn|spq*^GEhw)F3+pT_EHJWVJi3zt z@US~~?py$6TYYMqp?OmPIA2dgMHOa;eWLP%M(EkQkj~^Wk!xgV4&OmR#~@Ce+r?tI zN}QLZQg?t4fb>!2k_~0*W!!l)d70Mfw88J@59wv51q*GhJC=d!dYbBTcOF0uh0pv)o5^!;m`3M&@7^8b7=>jb55XYSFvh4M$>IED^FEMint z(${=y1lPLJvUb6fDVlXryPu`u%dcvEu~gQ|Z7pEC8w1@i4^dg$6JM3uf<_b~P9PG+ zBk%|)vH=Z~`;G&-M$*{SjpdT6#+z&B`4xJOi1;nETH^qEBPIPsN@y@FF#%btBb;>kD@G6eZuWOVo zYGMV=XbpS+fcd$c87LWoP-77gPS!BP?wFf-?WwL#VHOpVV>B<{FyIt+{ldrX3s%Yq}=OcOeHR;X6+RGDd zHxO@b59tAs^4g_z0I2EOmJ#?l1E4a8eQ_$(JwCqlXuH=L&;keo4$nL8cmzkLu+;kl z!-58&hWrkaf?<}E-&}kAMF!V}^9Z1VjbxD2 zlrqby1oyJB1*rcUx35(QklyF8$bsNdkRQLfpV0ke&%eMpS!H zcqgT*f8r0rLoPqvK#dg-fNAJCX*Q_j|1}0QS9w z89|59D#|_MW`lSUvzcV|j`RfU!cuD}WA_ z0UJ?-k+}5j(nblOlUS>EHmuIQ7PyK4*sv@P??&ZK)%=btqjaJPQ9Jp2n$X2)47wOF zLi^aY3fN0bwTrn(E6j=o6-rZH%?O6E{w>y-Gih$hJ-S;$>H~bLwkQL@?i{i6hIzld z&OT2uX2qz>*ZSa{;KDa@qP7 zDC!+qwS%5*LUOX){rmUN`SaWVvw8yJrvFlZQc7WZGB!ic=>nwe^+ikNdt)V4YXOpgTt#p z8uOs$42WhT**T*4MxV_-%3-oeF}ie%xv1$0FeFk5taR?3?<-~ zFome4_gC8`#R_>(WzOA$c3q;vd;U;cWN+UCD047$dw=qI%DMmJS0v^c05cAsO}hk% z$G=}b9q?un8SxzJ_|hs@?K zG*m5i&i#D^gCMBvK%lZ)2dqybpp-c1JwOhI`_Y=cp!-tFdNJ8k@ZY<0_pb3yW^vOy z9%QN7w-GVbnQqqnWFD!uf#=+7o^yzIfYy*?v5zv96Cek8C3S!dyk{&mGh+<{nYYlf zmW4o;paulU6=Q1pX6lcD`y>-c;)MecsUFI#$+xwr zafTt!5ZF^H5jhW8<6PxtR)#^_>Fj|8cfBBxngHn7*2@@u09N7^&t7gt<-;wR0t_%_ z60$=A!z&FOW%4KxJFX+9+vLwK0wM7JToK-uhe^7yu@R!a@>gl}1{m_Ne-AN*(pP(6 z6ls|vbl#Tira%;+Ove@=w?VedrgkL;oDPst=1~$$z-lZQHDD zmMs2+&S(&L*x_>b0@ml#W+?V!lYM25)ipL2=)LyyOMNK3iJw{h^DVb4M18R%sRF@>BZ$FXGOCW#{u_wlN2vA7;74kDdV{RDkH8Tif3P;x=n^Ai z3E}4se@C~_lg`+hLzu_US`0DsVq&kJJW2i~@9i--@3kK@_sfwQQ=T0Y>v?K_=YcOE ztoxG#ly-{S343o7UE4+aF@F>&lJNCEkC{}>CkJ96C8eR;4_bcfl!wC8Hi)MN>{A3&k9;82RV`Z=*T zuZsXq=vM%pEpq=oQtX7w&IBr;4iLh-et}#ld=ZQO`+oJL_W@znbf_6Yhd?CDDvR^V z6)s6UK!}^F5U3OJk?RSmYKfte&N7k;Mt5W>Ay31uEpn z7v~T>IpZIasLS8QG@zV)_hJM?d|d@_>3V`5X;Vs(6a!9t5%>!p@3C3%#9v}0hx2{l z=mwQWJHT71UT4aDh8nT~+_0Ofkdn}Yg9i`Vj2a_jQ-B6jrtm&a^O?`whY@@7?|qe1 z7-^LFym<~&{;K6ZX1>zhYennqLsm#FSJ2!Uk-Z**$5grb@bGYZv7G@b*8pK(UL*OQ z&Is^N4&8b#byt$33@M)=&q?!E1~-uK_zix02PS?qpzB6@QK{w~`i~)7$W zuKvIM_*bE1@6WId*Z=cxUifeLqX#}wQD@g-$Nmj#Aw-U&^d_RbEl+jwBqPfd&nS=t zLPO}DIRGu?=W{E$2^w_YU1>^d$@sJw%QGwSbv2iIx8 zfPhe_xqvzs1~;Pt>ZWKhbx4#N$7hh?cHb;46W&#$?M80^NcYX(#xUfFY%g^bwaAEe zjyVc2G2KG7ilYaFUD*GT6(sV2Txgf$lmWo`bGm)XbX*J=S9*9^OG+jYAUl-a;qk;^ z4XVCTVCkeRKM3Q{v#jiUNa$7TEw-~O+X0!Ml;vC^hE^c?$O9$UdiV9qm)r(4E3^Gp zipZWI3PgbxppKT6mzPhh?SU-^vFu=&owjv6tDxX6N@*5J&O-?c7~vmqIcD3%LOqY5 ztd`rfk-DzFo_qHQ1qBz;ueTSaje%P+wggwV9_fW4Gm=dsq8Z2tv8(81u7`4ni-DDV zSQ(mgXl54#?INW;zeT3+Y8&5gfwn_h-*Q`9n*(6N>%kI_5ji0sp>3NHGOWah#~-}Z z0jAV|S{DdQZNT6>^ApW=xbYM6yD;NvOKmaSZc{Rw))<3p5{|NSIJem`X=2Rd4**!r zDw`Doot$70wiKv{jc*x-a=i{b`z*z}b&74-VF%=?RRG5bP$P1K?pjk3>H-YDd;CcP zXv_nM%UqerNNy;he(V=1I2Q5SX?h$LnaARNz5q#e>@!rv*Ry8x zvc>~K-7I;CVDBB zKj=gK=A^H{W6vRV)#>i+YfgFcObC!4>Wv8Nc*o_lGQLB)e4@1fvUAha53>g4~HwBtEr_DIJLdsgy05{UHBs+^C zr&+u6Bq{dNf8Fb!#~Ha8fIsNVC+F36>$G5h2^D5YgdlxiO;>lu{ymUw+Cx$WL1V7T zno$}meo@sVsp3*7szia^Hg6VuNhu5xk##UdGNJh)oSxS*>E+;NDE$+U6}XF9J=jujBZ$7kOU*qWHDlxzSl3xUjrRl6s+D!sLFL>Fp3G=?P!vV zI>07iW57Ys=^38~_dZ5I?dw*KMOqny3p8ME%vry$fuO$*;^nBy%6q6_cCfIcioAlC zsTx|z6C~|Vj@cH2K(`i1!PmX9N;-=f($~)4iUl9zOfzMO%}PI+#l;Oyv1#5jou8KX zlC~`!J=I}%dsD-l-3e3eH)w@0seZ3tJFiO@q=`K*4EH7TMG`fL$^rt4*TI9DYcRH> zqoc`Wn{NBZ#lRnD%%XE&tBpCa?Sm`H)5CB71DMtLK z#GX5i?Lxl7iD0ONMZLKOcdb=XU?;t&qu9IPLxOu(-S$u(^&8UjeY2>$Fdh;{gjp>E z^bj&59Rp>WO#`aF(haRg5F`<6v1x{p{e9dhbX7;2Kul>VZ~`d3eNbuH?kZ-~cE8)~ zHM=hNV}k7t;Xy8R2zkm3vXM)NO(Bd1(TaA2mpI!-@-KUz2^t+=aW}g;X+FuH!VHNa z?1N3w1X2@+yB+? zs9%6O0WI`@R8r9~x-0#2W||Y&F8h!v-yI zc9-{I+|WTkid?DpxM~agR+3?N!WV5M-pcOik;fd9Us_Z#>m9WJdKk>Xlc~g5mp8LQ z@32dxz%6aN9_N($}7Knl`+h8!k~V z{V!k(^MuC&O$yP5Ew{AF0mstTA;L4FDJmC1BxZ=B z*Z@pqk`g3ME^VHX1&;Q$Zmp^)@GJ1l&Ya3$$gQQzx3Q{FeBUU%D44OaG-=eV0cn|R zo507|&om5aur0U_sSp>U)0jokeju0e?^V1E0TP^$4TVEPLL#OMIZC&ewXM24+qpjl zZgXYWxkr0idRG7|%O}*c%Flay+MqKmBy~+o1kBnyG|ym^i=)=hRzfM00N%rQbamO# zgN{JuMLXgfqHDW5bGz~#?wyd-0^{BIoER}FtDyvDo&&uo8LT}xq?vtJHCkdHv_&!< zO4B<54Lb_O7FW1a%|wi-hcP5D?dclXva4Dc7LIYN;cG&BMAD($xWL;>1w%@e-FLkoLY$Q9MSx+|RZnQF7w z5oiuhJAhwoB&lEfJuVIoMbI9#!B4DCBi>)KVzkJLCT8!L>CiFTs1oy%CG0=nX9z7K z=Bv1C_G*vMW8&Pq&TMuyz(;2|cudRhe0_ctHLDpxe;;&abpr#bFv>Bkv0qVpkb)6t zHYTdgQr=M=XA(fwH5?gk-#V?wh?c0u-J@$4;DhYx_pZ?MZ4oj?N%H5q(S?(;JjT&B zmeCImVC+SnR52gi0!a=lpE_i00IG{qz}aH>X&N|{S`_;gW>8_U*SeX9V69Mu@z~oq z6)0M%-_ZHmTSfOrhJbib$^O{y&}V&ghFwhoPfa*H2)E(c#-(etM4@TKA17FOd`6xSGRsBx4tAdQ8x}*X z7f}p-Fnc}Vjx|Q|$ssjhBqffz%RsgSvY!kZ070h%l~l42Ps>H`lDH_ zg7xFyz~l)BW7+OKF&9MB!Lou%CVLAWrU14(Uf{v2c+DU6+!3|3@&$E{Dcq^G1JE-Z zGUKI=g4`VN^PSFmpydGw7*q)D74ykZ(QBidvkv@3km>!BZX{(70>+$;>tNEw*eU`v z$gwR%nnHeDRe~;UL-R0Z2Q{GP-4aS5a}A2CfAKf$$ERopCH1*$6{DRT*?)5zS%Thy zu5*tG4DrTTYSmyY3H4{LKb@$yc9HcJs4$UU^(4MHd(RylRDwhBJCRmdT&t<$p=~wz zPXYYDvVM5&wzuCw$*>7x4y|N*cGw;^Um8?Rl79=z+Z}ciPpkQQ zU)e&`R)d{|(~CmWl^Z`%$6&83D=OM=ot*u2nGbPl(nZE#*VePO=8G0|hv7jPZ&~|} zTX#>3_9>ZIX-_5S1REfuM^r3ESyskCYhbS|o*U>7(aEdSAXrQ?ZNI%GS30qn^Cf?FXaSg zXc0t+lxZj_@1)V=p#Vzj%msbmJCPwF#;+5Tp|S3mw^Ihqf<&;L9HH1GfbmvQ6G*1Y zz6akX4&W(4UIfRGg>5p>_ZrY%8*m9(6W=l$ZaOKwvpg&08-8wf>|+3f~}6zJ5Xe> zva_%_tZAO+&^50y?pLpcGWcsdC^qAr^37U`Z0tOB)hg_O+N@4<@}x=8CZtiaR=*Gm zbh8EYWfQkEMi9eS4z?O=5BZfj712q@@fVe2?we55$O%83xW_SVb-&+WusE%xq+|&~ z0H=$*=it+4&w6)|!gV`r4ZW|okxwB6gMtEJnHJeB<>@CU!!-@1Z2;z2{Wkl=C|CpF z_gXH~W=eta-aK8aiOWB`I?yB`y?by+5d5aJC(%lD)G99CE$dhvL=)0>D>OW4IZwt& zR&argEs**R^)c)w>Qal(!>A(cV56law3>=In_w()1^!7T6d@DKM}l2=8eVhdNU#^M zuum~i1P2?<4J7%3KXCA%TopiDs7_20Hkhg|^h>SgdRcT~reQc$gh$NC4b*96_)PWr zgV*sc&UTucFtd4&hh1E572H=jZnqJZ5^5U(h+hn%ya}KV6+gBBdNUUI+0^71mi{X5 zs#5(YrBE$~Kuz>+Yy~OpKa&3Zp^}V-e8Q*`t2G$`P-7Lm^pn&+5qA7_C4eRy#d%B5QCcu6UJef`vQq+H>cF8OY`)G2J) zO-Wv!ckUG=)`h&+u63_?or7Y4Ah-_UTk+>0&Dz9tTZR@76}47uFig`zOwC}digX-$ z6uoh%bhGjhb<#P!1xge)wsRfHffS6a9&<@p&U(Y>;Zu3` z_TubihAxRFp%G1zk1sPG2R0Ks@fvk+40Ph1n9WRh-Y*w+Vh7r08m4@1oQtnyb}!qh zR(S_1FF&juq1Ax9ERaz*ZMS>YV!i^hCMw%*dW#InM4ml7t0a5JL$}%*~_M zG4mwab_250P2IkV2ihu+P9fxsX;Wl$PQXe*D;@``Nvxt>;4W|x+C9K-sMrsaipEit z=#>#*g2z98TG#%uU?(5J#2#9U>SfU@hpAyBtJz9`$M_8}lI(|N6jaQ(iM2wvZ9)KeATNDMxxE>S;Qzy$ z2s<+%c}UX(#qzy)o_0#TiGaakAnqIybe@SQMIZo;Ptp9O-tZp^5^0;}H)dZJa5!RibVJ;F{2Zq^71$AVHob>sAL{kfLWJuQlI? z3xf>IK}Xx&tyHM(-Yr6ELFHR^{fU@ObtRA!9H-y`Aryg=`PV3LKfNjBJaaj#{n>c0 zg#PCG`vyTw{4WJXYt{XUpagTL(lZ zG-fMC($WTkVPZaJH-wB#2Wqn_ZF_op#_o1>XMP%)_Ry`>JU1PSlw_`5-tI({CiX_r{yRy|vcT}jOqgUsfwl=iCvbaRKbQ2|; z0aV#XdDc?qV?89Lc=sIKZ_&xg?o5t>(-0b=oES2ALFBjheG>XmdrH$@aS#vvY`x6o#Y1) zlN`JB#96DrCxFAN2T_$oj%eVE7rlF_&78!X&zgC7qVSu4;ptm8G#yq-$JsO6v-E_Y zi=Nj-Y(x#A^aa3g6@=p3xcT`!`+3`bOv(O=H7=iw7l0+E!8b-47}l5uATlH zs!;s?Cw9lWLQ_mZ)iF~qJ5eGn(k2;pBLxYIXsy)ID+O#X-63PMUY;RLp7yLzK$3U1 z=XMjoF=rh$IO?=u3o6)m;WhD1n^U{U>Q0Xu1htUZ3C%WS?=GPRV}Mg(gcgU*YCmvP zkScKFC@g}xB$t~C`Xq;O&`{Svm@n&qNk~tImfN{erVqc{J-oPvly7ulN+uWJ(~1yj zBDT?P>j3If1MQ`^#r$b5CeFo`VzRLsRu?zw?^+YbzIk)6Io-XgWMmmK4XiiHG|=~5 zc=|-10uXDBVw)I@p?*%cm^CjZKplEGf7xFM;k zD3I6b$pbepyp8zwQgEqaRooO?$9_83>dX<;X+(KFb6k7eGHNmzbP-58z_#Y|5_!%w zpyg7Xu#(^hl}Xr6UzO_bW`6d2U7B>bOfRveA86m^99Di|(7J1F)a}GbO_q|S!Y)or@A-LY*@5_bVm<`xb@|rMNDNvRaNz6h7PEj**ciI2k z&(AMO`dL_cZu2~jo@7b-C5!>^UJR*zfH&1blQ+k8xJ985qJO~H*t&SS5eF6N575lK z?{l?c2J9~OB1R<$worYWJY=(JI)|JSdXm651VLZI@l`M^7}F)Ggc;FKqik_lP+JEf zMgQj;6Ued;vla@d@vtJhLH%U*UPX|BOaz6V%ZDi_Ku-=PLhZ2Wmlwpv#r5sTg|S^x z0A(U8W9HduZEbZ}P4qV6bDNu9%$jR@nH^9mJ`=bLJk6M#IW4njwb|{=KshB& zXoF`Oc$>vsG}}cuXyw(vQ6~_ELDC_y3={<8^J@@c3VGq7GS9T=ee>mxnh~LCRWoxj zu=qAT6~B^}!~d+f-Qj{e9%USm-1GC$IY}k!S3=*o$p$7lD?fhH@>w#%y?6_NydThz z-~?SY{-z=$L&F5S!AjGbw8D_v-}5dbiR~yzfRj!DFA!4f=L}^jeP0MhtFb~ z4Lir>8V1cBU80@R^K}cwzksnD^WdY!0KD8%3X}!ijdZx~5&)DT0l=8ANP-FwTMBLr zo-r)(#af$ha0Ofy+l?yGlIa!fmU3pdrv2qSx54AfJhFY-zdgTMsG)oXG~ahM(nqY^ zn4-^pvh5dI2a7AqfjJ>t_NoL?#e%A@&{Q`wy{oe+Q(}o^Uipi|a@xFDH@sR`pNiFP zh?OFN9|*Z)UMw3RKCNIu<7qJMuWaL-o&1u?%yM&s1h9R($M!b2Pp@I(k*1-P@2hAH zv4ps~K9to)VSukX(_)T-{lmwA5woU9lZ_8J98?+#*leOsXdGHT+C%YG`$3xq6M7$- z5g18h7_>t8r|#mtimN;?)Nsd)ecJyP+*IK3T>v!7UFM8-*jgnd-XU8&LVN+Qq5`D+ z@^r9 + + + + + + + 2026-11-10T22:08:55.028824 + image/svg+xml + + + Matplotlib + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/evidence/lab5-smoke.txt b/evidence/lab5-smoke.txt new file mode 100644 index 00000000..c1bf3fc7 --- /dev/null +++ b/evidence/lab5-smoke.txt @@ -0,0 +1,2 @@ +.venv/bin/python tests/smoke.py --base-url "http://localhost:18085" +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/lab5-validation.txt b/evidence/lab5-validation.txt new file mode 100644 index 00000000..3f7f5878 --- /dev/null +++ b/evidence/lab5-validation.txt @@ -0,0 +1,19 @@ +Fresh lab5 validation, 2026-11-10. +Root independent checks: + make check validate: exit 0. + go vet ./...: exit 0. + ./scripts/compose.sh config --quiet: exit 0. + docker stack config -c stack.yaml: exit 0. + sh -n scripts/*.sh infra/*.sh: exit 0. +Worker checks: + go test -race ./...: exit 0. + make test: exit 0, lab5 scenario on port18085. + make test-integration: exit 0 on repeat, including 005-010 up/down-to4/legacy/reset/up. + Integration initial failure was an inherited fixture that dropped the migration journal from an already-lab5 schema. + The fixture now models an actual lab4 schema (down-to4) before legacy-journal adoption. + Fault cases: all 11 groups verified across the initial main phase and --tail-only repeat. + The original full invocation exited 1 due to a test stop-helper race; it is not claimed as a full green run. + make test-persistence: exit 0; own stack restart preserved confirmed data and existing session. + make test-failover: exit 0; one of two API replicas stopped, 123 successful writes, 1 failed attempt, recovery 0.696s. +Public contracts and tests match skeleton276dbda; optional private tests cover added requirements. +Historical evidence/lab4-* is not used as current lab5 evidence. diff --git a/infra/README.md b/infra/README.md index 82e02813..3504ad78 100644 --- a/infra/README.md +++ b/infra/README.md @@ -1,14 +1,24 @@ -# Инфраструктура студента +# Инфраструктура lab5 -Основной стенд использует `stack.yaml`: HAProxy → две API-реплики → PostgreSQL, Redis AOF и приватный MinIO. -Запуск: `make up`. Остановка с сохранением данных: `make down`. -Версии, секреты, ограничения и проверки описаны в [REPORT.md](../REPORT.md). +HAProxy направляет HTTP в две API-реплики. +PostgreSQL хранит сообщения, метаданные, jobs, outbox и audit replay. +Redis хранит сессии и Streams с consumer group `image-workers-v1`. +Redis использует AOF и `appendfsync always`; Pub/Sub не используется. +MinIO хранит приватные staging и конечные варианты. Worker работает отдельным сервисом. -`s3-bootstrap.sh` идемпотентно создаёт bucket и отдельного API-пользователя по `s3-policy.json`. -`minio-entrypoint.sh` читает root keys из Docker secrets. Консоль слушает loopback контейнера. -`Dockerfile.storage` собирает MinIO и mc из исходников с фиксированными commit и SHA-256. -`haproxy.cfg` проверяет readiness реплик. `redis.conf` включает AOF и запрещает eviction. -`migrate.sh` запускает goose без автоматической загрузки файлов окружения. +Основной запуск: `make up`. API: http://localhost:18085. +Остановка с сохранением volumes: `make down`. +Compose запускает те же роли с одной API-репликой. +Скрипты не читают `.env`; bootstrap создаёт Docker secrets. +Redis имеет лимит памяти 384MiB и `maxmemory 256mb`, политика `noeviction`. +Worker имеет лимит 1CPU/1536MiB; API — по 1CPU/1536MiB. -Restart hook перезапускает API, HAProxy, PostgreSQL, Redis и MinIO с сохранением volumes. -Failover hook останавливает API-реплику по `TARGET_INSTANCE_ID`; Swarm создаёт новую task с другим ID. +Миграции goose 005–010 добавляют состояния, jobs, outbox, audit и cleanup. +Миграция 009 переводит прежние оригиналы в устойчивый staging без потери байтов. +При обновлении API останавливаются перед миграциями, затем запускается новая версия. +Swarm configs Redis и S3 policy используют суффикс `_v5`. +Это позволяет обновить прежний стенд без изменения immutable configs. + +Порты PostgreSQL, Redis, MinIO и worker не опубликованы. +S3 policy разрешает API/worker только `original/*` и `staging/*` своего bucket. +Версии образов, протокол отказов и команды находятся в [REPORT.md](../REPORT.md). diff --git a/infra/redis.conf b/infra/redis.conf index c9e470c6..0b8eb4c4 100644 --- a/infra/redis.conf +++ b/infra/redis.conf @@ -5,5 +5,5 @@ dir /data appendonly yes appendfsync always save "" -maxmemory 64mb +maxmemory 256mb maxmemory-policy noeviction diff --git a/infra/s3-policy.json b/infra/s3-policy.json index b9fb3b95..20797ca1 100644 --- a/infra/s3-policy.json +++ b/infra/s3-policy.json @@ -10,12 +10,12 @@ "Effect": "Allow", "Action": ["s3:ListBucket"], "Resource": ["arn:aws:s3:::attachments"], - "Condition": {"StringLike": {"s3:prefix": ["original/*"]}} + "Condition": {"StringLike": {"s3:prefix": ["original/*", "staging/*"]}} }, { "Effect": "Allow", "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], - "Resource": ["arn:aws:s3:::attachments/original/*"] + "Resource": ["arn:aws:s3:::attachments/original/*", "arn:aws:s3:::attachments/staging/*"] } ] } diff --git a/internal/app/chat/attachments.go b/internal/app/chat/attachments.go index f326d707..c345e177 100644 --- a/internal/app/chat/attachments.go +++ b/internal/app/chat/attachments.go @@ -33,7 +33,7 @@ func (h *Chat) uploadAttachment(c echo.Context) error { if err != nil { return h.useCaseError(c, err) } - return c.JSON(http.StatusCreated, a) + return c.JSON(http.StatusAccepted, a) } func (h *Chat) getAttachment(c echo.Context) error { a, err := h.attachments.Get(c.Request().Context(), c.Param("attachment_id"), authenticatedUser(c).ID) @@ -51,8 +51,15 @@ func (h *Chat) attachmentContent(c echo.Context) error { if err != nil { return h.invalid(c, "Некорректный variant") } - if present && variant != "original" { - return h.writeError(c, 404, "not_found", "Вариант не найден") + if a.Status != "ready" { + return h.writeError(c, 409, "conflict", "Вложение не готово") + } + if !present { + variant = "original" + } + a, err = h.attachments.Variant(a, variant) + if err != nil { + return h.useCaseError(c, err) } ctx, cancel := context.WithTimeout(c.Request().Context(), 45*time.Second) defer cancel() diff --git a/internal/app/chat/errors.go b/internal/app/chat/errors.go index c556830e..7c532f81 100644 --- a/internal/app/chat/errors.go +++ b/internal/app/chat/errors.go @@ -35,6 +35,9 @@ func (h *Chat) useCaseError(c echo.Context, err error) error { switch applicationError.Code { case "unsupported_media_type": status = http.StatusUnsupportedMediaType + case "rate_limited": + status = http.StatusTooManyRequests + c.Response().Header().Set("Retry-After", "2") case "payload_too_large": status = http.StatusRequestEntityTooLarge case domain.ErrorUnavailable: diff --git a/internal/attachment/async.go b/internal/attachment/async.go new file mode 100644 index 00000000..3e9faa69 --- /dev/null +++ b/internal/attachment/async.go @@ -0,0 +1,156 @@ +package attachment + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strconv" + "time" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" +) + +type Options struct { + Concurrency, Prefetch, MaxOutstanding, MaxAttempts, PixelBudget int + Timeout, Lease time.Duration +} + +func LoadOptions() (Options, error) { + o := Options{} + for _, v := range []struct { + name string + target *int + def, max int + }{ + {"WORKER_CONCURRENCY", &o.Concurrency, 2, 8}, {"WORKER_PREFETCH", &o.Prefetch, 4, 32}, + {"MAX_OUTSTANDING_JOBS", &o.MaxOutstanding, 200, 10000}, {"WORKER_MAX_ATTEMPTS", &o.MaxAttempts, 5, 20}, + {"WORKER_PIXEL_BUDGET", &o.PixelBudget, 40000000, 80000000}, + } { + n := v.def + if raw := os.Getenv(v.name); raw != "" { + var err error + n, err = strconv.Atoi(raw) + if err != nil { + return o, fmt.Errorf("invalid %s", v.name) + } + } + if n < 1 || n > v.max { + return o, fmt.Errorf("invalid %s", v.name) + } + *v.target = n + } + for _, v := range []struct { + name string + target *time.Duration + def int + }{{"WORKER_TIMEOUT_SECONDS", &o.Timeout, 25}, {"WORKER_LEASE_SECONDS", &o.Lease, 40}} { + n := v.def + if raw := os.Getenv(v.name); raw != "" { + var e error + n, e = strconv.Atoi(raw) + if e != nil { + return o, fmt.Errorf("invalid %s", v.name) + } + } + if n < 1 || n > 300 { + return o, fmt.Errorf("invalid %s", v.name) + } + *v.target = time.Duration(n) * time.Second + } + if o.Lease < o.Timeout+5*time.Second { + return o, fmt.Errorf("worker lease must exceed timeout by at least 5s") + } + if o.PixelBudget < int(MaxPixels) { + return o, fmt.Errorf("WORKER_PIXEL_BUDGET must allow one 40MP image") + } + return o, nil +} +func NewAsync(pool *pgxpool.Pool, store *Store, o Options) *Service { + s := New(pool, store) + s.Async = true + s.Options = o + return s +} +func (s *Service) Upload(ctx context.Context, chatID, userID, name, contentType string, body interface{ Read([]byte) (int, error) }) (Attachment, error) { + if !s.Async { + return s.uploadSync(ctx, chatID, userID, name, contentType, body) + } + if err := ValidateFilename(name); err != nil { + return Attachment{}, err + } + ctx, cancel := context.WithTimeout(ctx, UploadTimeout) + defer cancel() + select { + case s.slots <- struct{}{}: + defer func() { <-s.slots }() + case <-ctx.Done(): + return Attachment{}, unavailable() + } + original, err := Spool(ctx, body, contentType) + if err != nil { + return Attachment{}, err + } + defer original.Close() + tx, err := s.Pool.Begin(ctx) + if err != nil { + return Attachment{}, unavailable() + } + defer tx.Rollback(context.Background()) + // All API replicas serialize admission, including inserts before their COMMIT. + if _, err = tx.Exec(ctx, `SELECT pg_advisory_xact_lock(578903551)`); err != nil { + return Attachment{}, unavailable() + } + var pending int + if err = tx.QueryRow(ctx, `SELECT count(*) FROM attachment_jobs WHERE state IN ('queued','processing')`).Scan(&pending); err != nil { + return Attachment{}, unavailable() + } + if pending >= s.Options.MaxOutstanding { + return Attachment{}, appError("rate_limited", "Очередь заполнена; повторите загрузку позднее") + } + a := Attachment{ID: uuid.NewString(), ChatID: chatID, OwnerID: userID, Filename: name, ContentType: original.ContentType, Size: original.Size, SHA256: original.SHA256, Status: "queued", Variants: []any{}, PipelineVersion: 1} + stage := "staging/" + a.ID + a.Key = effectKey(a.ID, 1, "original") + // The stage lock protects even a slow PUT from the orphan cleaner. + if _, err = tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, stage); err != nil { + return Attachment{}, unavailable() + } + if err = s.put(ctx, stage, original.File, original.Size, original.ContentType); err != nil { + return Attachment{}, unavailable() + } + err = tx.QueryRow(ctx, `INSERT INTO attachments(id,chat_id,uploader_id,filename,object_key,content_type,size_bytes,sha256,width,height,status,staging_key) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,'queued',$11) RETURNING created_at,updated_at`, a.ID, chatID, userID, name, a.Key, a.ContentType, a.Size, a.SHA256, original.Width, original.Height, stage).Scan(&a.CreatedAt, &a.UpdatedAt) + if err != nil { + return Attachment{}, unavailable() + } + job := uuid.NewString() + if _, err = tx.Exec(ctx, `INSERT INTO attachment_jobs(id,attachment_id,state,max_attempts) VALUES($1,$2,'queued',$3)`, job, a.ID, s.Options.MaxAttempts); err != nil { + return Attachment{}, unavailable() + } + if _, err = tx.Exec(ctx, `INSERT INTO attachment_outbox(job_id) VALUES($1)`, job); err != nil { + return Attachment{}, unavailable() + } + if err = tx.Commit(ctx); err != nil { + return Attachment{}, unavailable() + } + return a, nil +} +func effectKey(id string, version int, name string) string { + return fmt.Sprintf("original/%s/v%d/%s", id, version, name) +} +func (s *Service) Variant(a Attachment, name string) (Attachment, error) { + if name == "original" { + return a, nil + } + for _, item := range a.Variants { + raw, _ := json.Marshal(item) + var v Variant + if json.Unmarshal(raw, &v) == nil && v.Name == name { + a.Key = effectKey(a.ID, a.PipelineVersion, name) + a.ContentType = v.ContentType + return a, nil + } + } + return Attachment{}, appError(domain.ErrorNotFound, "Вариант не найден") +} diff --git a/internal/attachment/reconcile.go b/internal/attachment/reconcile.go index 402d7b3e..755338f4 100644 --- a/internal/attachment/reconcile.go +++ b/internal/attachment/reconcile.go @@ -139,7 +139,7 @@ func (s *Service) reconcileObject(ctx context.Context, key string, cutoff time.T defer tx.Rollback(context.Background()) var exists bool r.DBQueries++ - if err = tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM attachments WHERE object_key=$1)`, key).Scan(&exists); err != nil { + if err = tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM attachments WHERE object_key=$1 OR staging_key=$1 OR $1 LIKE 'original/'||id::text||'/v'||pipeline_version::text||'/%')`, key).Scan(&exists); err != nil { return fmt.Errorf("metadata recheck failed") } if exists { @@ -207,9 +207,12 @@ func (s *Service) reconcileMetadata(ctx context.Context, id, key string, cutoff } if apply { r.DBQueries++ - if _, err = tx.Exec(ctx, `UPDATE attachments SET status='failed',error_code='original_missing' WHERE id=$1 AND status='ready'`, id); err != nil { + if _, err = tx.Exec(ctx, `UPDATE attachments SET status='failed',error_code='original_missing',updated_at=clock_timestamp() WHERE id=$1 AND status='ready'`, id); err != nil { return fmt.Errorf("mark missing failed") } + if _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET state='failed',last_error='original_missing',fence=fence+1,updated_at=clock_timestamp() WHERE attachment_id=$1 AND state='ready'`, id); err != nil { + return fmt.Errorf("mark job missing failed") + } if err = tx.Commit(ctx); err != nil { return fmt.Errorf("commit missing failed") } diff --git a/internal/attachment/render.go b/internal/attachment/render.go new file mode 100644 index 00000000..e6cebce9 --- /dev/null +++ b/internal/attachment/render.go @@ -0,0 +1,267 @@ +package attachment + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "image" + "image/color" + "image/jpeg" + "image/png" + "math" + "os" + "os/exec" + "path/filepath" + "syscall" +) + +var ErrInvalidImage = errors.New("invalid image") + +type oriented struct { + image.Image + orientation int +} + +func (o oriented) Bounds() image.Rectangle { + b := o.Image.Bounds() + w, h := b.Dx(), b.Dy() + if o.orientation >= 5 && o.orientation <= 8 { + w, h = h, w + } + return image.Rect(0, 0, w, h) +} +func (o oriented) At(x, y int) color.Color { + b := o.Image.Bounds() + w, h := b.Dx(), b.Dy() + sx, sy := x, y + switch o.orientation { + case 2: + sx = w - 1 - x + case 3: + sx, sy = w-1-x, h-1-y + case 4: + sy = h - 1 - y + case 5: + sx, sy = y, x + case 6: + sx, sy = y, h-1-x + case 7: + sx, sy = w-1-y, h-1-x + case 8: + sx, sy = w-1-y, x + } + r, g, blue, a := o.Image.At(b.Min.X+sx, b.Min.Y+sy).RGBA() + white := uint32(65535) - a + return color.RGBA{uint8((r + white) / 257), uint8((g + white) / 257), uint8((blue + white) / 257), 255} +} + +// EXIF IFD0 orientation; bounded segment parsing does not follow client offsets outside APP1. +func tiffOrientation(t []byte) int { + if len(t) < 8 { + return 1 + } + var order binary.ByteOrder + if string(t[:2]) == "II" { + order = binary.LittleEndian + } else if string(t[:2]) == "MM" { + order = binary.BigEndian + } else { + return 1 + } + if order.Uint16(t[2:4]) != 42 { + return 1 + } + offset := uint64(order.Uint32(t[4:8])) + if offset+2 > uint64(len(t)) { + return 1 + } + count := uint64(order.Uint16(t[offset : offset+2])) + for k := uint64(0); k < count; k++ { + p := offset + 2 + k*12 + if p+12 > uint64(len(t)) { + break + } + entry := t[p : p+12] + if order.Uint16(entry[:2]) == 0x112 && order.Uint16(entry[2:4]) == 3 && order.Uint32(entry[4:8]) == 1 { + v := int(order.Uint16(entry[8:10])) + if v >= 1 && v <= 8 { + return v + } + } + } + return 1 +} +func Orientation(data []byte) int { + if len(data) >= 8 && bytes.Equal(data[:8], []byte{137, 80, 78, 71, 13, 10, 26, 10}) { + for i := 8; i+12 <= len(data); { + n := uint64(binary.BigEndian.Uint32(data[i : i+4])) + if n > uint64(len(data)-i-12) { + return 1 + } + end := i + 8 + int(n) + if string(data[i+4:i+8]) == "eXIf" { + return tiffOrientation(data[i+8 : end]) + } + i = end + 4 + } + return 1 + } + if len(data) < 4 || data[0] != 255 || data[1] != 216 { + return 1 + } + for i := 2; i+4 <= len(data); { + if data[i] != 255 { + return 1 + } + for i+1 < len(data) && data[i+1] == 255 { + i++ + } + if i+3 >= len(data) { + return 1 + } + marker := data[i+1] + i += 2 + if marker == 0xda || marker == 0xd9 { + return 1 + } + if marker == 0x01 || marker >= 0xd0 && marker <= 0xd7 { + continue + } + n := int(binary.BigEndian.Uint16(data[i : i+2])) + if n < 2 || i+n > len(data) { + return 1 + } + body := data[i+2 : i+n] + i += n + if marker == 0xe1 && len(body) >= 14 && bytes.Equal(body[:6], []byte("Exif\x00\x00")) { + return tiffOrientation(body[6:]) + } + } + return 1 +} + +func fit(w, h, maxSide int) (int, int) { + if w <= maxSide && h <= maxSide { + return w, h + } + if w >= h { + return maxSide, max(1, int(int64(h)*int64(maxSide)/int64(w))) + } + return max(1, int(int64(w)*int64(maxSide)/int64(h))), maxSide +} + +func resize(src image.Image, region image.Rectangle, w, h int) *image.RGBA { + out := image.NewRGBA(image.Rect(0, 0, w, h)) + for y := 0; y < h; y++ { + fy := (float64(y)+.5)*float64(region.Dy())/float64(h) - .5 + fy = math.Max(0, fy) + y0 := min(int(fy), region.Dy()-1) + y1 := min(y0+1, region.Dy()-1) + dy := fy - float64(y0) + for x := 0; x < w; x++ { + fx := (float64(x)+.5)*float64(region.Dx())/float64(w) - .5 + fx = math.Max(0, fx) + x0 := min(int(fx), region.Dx()-1) + x1 := min(x0+1, region.Dx()-1) + dx := fx - float64(x0) + colors := [4]color.RGBA{color.RGBAModel.Convert(src.At(region.Min.X+x0, region.Min.Y+y0)).(color.RGBA), color.RGBAModel.Convert(src.At(region.Min.X+x1, region.Min.Y+y0)).(color.RGBA), color.RGBAModel.Convert(src.At(region.Min.X+x0, region.Min.Y+y1)).(color.RGBA), color.RGBAModel.Convert(src.At(region.Min.X+x1, region.Min.Y+y1)).(color.RGBA)} + blend := func(a, b, c, d uint8) uint8 { + return uint8(math.Round(float64(a)*(1-dx)*(1-dy) + float64(b)*dx*(1-dy) + float64(c)*(1-dx)*dy + float64(d)*dx*dy)) + } + out.SetRGBA(x, y, color.RGBA{blend(colors[0].R, colors[1].R, colors[2].R, colors[3].R), blend(colors[0].G, colors[1].G, colors[2].G, colors[3].G), blend(colors[0].B, colors[1].B, colors[2].B, colors[3].B), 255}) + } + } + return out +} + +// Render runs only inside a deadline-killed child, never in the HTTP process. +func Render(ctx context.Context, source, dir, expectedSHA string) error { + raw, err := os.ReadFile(source) + if err != nil { + return err + } + sum := sha256.Sum256(raw) + if int64(len(raw)) > MaxBytes || hex.EncodeToString(sum[:]) != expectedSHA { + return ErrInvalidImage + } + cfg, _, err := image.DecodeConfig(bytes.NewReader(raw)) + if err != nil || cfg.Width < 1 || cfg.Height < 1 || int64(cfg.Width) > MaxPixels/int64(cfg.Height) { + return ErrInvalidImage + } + if os.Getenv("PIPELINE_FAILPOINT") == "renderer_timeout" { + fifo := filepath.Join(dir, "blocked-input.png") + if err = syscall.Mkfifo(fifo, 0600); err != nil { + return err + } + return exec.CommandContext(ctx, "cwebp", "-quiet", fifo, "-o", filepath.Join(dir, "blocked.webp")).Run() + } + decoded, _, err := image.Decode(bytes.NewReader(raw)) + if err != nil { + return ErrInvalidImage + } + src := oriented{decoded, Orientation(raw)} + bounds := src.Bounds() + tw, th := fit(bounds.Dx(), bounds.Dy(), 256) + ow, oh := fit(bounds.Dx(), bounds.Dy(), 1280) + variants := []Variant{} + for _, name := range []string{"thumbnail", "crop", "optimized"} { + w, h := tw, th + region := bounds + if name == "crop" { + w, h = 128, 128 + side := min(bounds.Dx(), bounds.Dy()) + left, top := (bounds.Dx()-side)/2, (bounds.Dy()-side)/2 + region = image.Rect(left, top, left+side, top+side) + } + if name == "optimized" { + w, h = ow, oh + } + img := resize(src, region, w, h) + path := filepath.Join(dir, name) + if name == "optimized" { + input := filepath.Join(dir, "webp-input.png") + f, e := os.Create(input) + if e != nil { + return e + } + e = png.Encode(f, img) + f.Close() + if e != nil { + return e + } + command := exec.CommandContext(ctx, "cwebp", "-quiet", "-q", "80", "-m", "4", "-metadata", "none", input, "-o", path) + if err = command.Run(); err != nil { + return err + } + } else { + f, e := os.Create(path) + if e != nil { + return e + } + e = jpeg.Encode(f, img, &jpeg.Options{Quality: 85}) + f.Close() + if e != nil { + return e + } + } + data, e := os.ReadFile(path) + if e != nil { + return e + } + hash := sha256.Sum256(data) + mime := "image/jpeg" + if name == "optimized" { + mime = "image/webp" + } + variants = append(variants, Variant{Name: name, ContentType: mime, Width: w, Height: h, Size: int64(len(data)), SHA256: hex.EncodeToString(hash[:])}) + } + data, err := json.Marshal(variants) + if err != nil { + return err + } + return os.WriteFile(filepath.Join(dir, "manifest.json"), data, 0600) +} diff --git a/internal/attachment/render_test.go b/internal/attachment/render_test.go new file mode 100644 index 00000000..a5ab246d --- /dev/null +++ b/internal/attachment/render_test.go @@ -0,0 +1,81 @@ +package attachment + +import ( + "bytes" + "encoding/binary" + "image" + "image/color" + "testing" +) + +func TestFitGeometry(t *testing.T) { + for _, c := range []struct{ w, h, limit, x, y int }{{320, 200, 256, 256, 160}, {513, 257, 256, 256, 128}, {257, 513, 256, 128, 256}, {12, 8, 256, 12, 8}, {1, 40000, 256, 1, 256}} { + w, h := fit(c.w, c.h, c.limit) + if w != c.x || h != c.y { + t.Fatalf("fit(%d,%d)=%d,%d", c.w, c.h, w, h) + } + } +} +func TestOrientationsAndAlpha(t *testing.T) { + img := image.NewNRGBA(image.Rect(0, 0, 3, 2)) + for y := 0; y < 2; y++ { + for x := 0; x < 3; x++ { + img.SetNRGBA(x, y, color.NRGBA{R: uint8(1 + x + y*3), A: 255}) + } + } + expected := [][]uint8{{1, 2, 3, 4, 5, 6}, {3, 2, 1, 6, 5, 4}, {6, 5, 4, 3, 2, 1}, {4, 5, 6, 1, 2, 3}, {1, 4, 2, 5, 3, 6}, {4, 1, 5, 2, 6, 3}, {6, 3, 5, 2, 4, 1}, {3, 6, 2, 5, 1, 4}} + for i, want := range expected { + src := oriented{img, i + 1} + got := []uint8{} + for y := 0; y < src.Bounds().Dy(); y++ { + for x := 0; x < src.Bounds().Dx(); x++ { + got = append(got, src.At(x, y).(color.RGBA).R) + } + } + if !bytes.Equal(got, want) { + t.Fatalf("orientation %d: %v want %v", i+1, got, want) + } + } + img.SetNRGBA(0, 0, color.NRGBA{R: 255, A: 128}) + got := oriented{img, 1}.At(0, 0).(color.RGBA) + if got != (color.RGBA{255, 127, 127, 255}) { + t.Fatalf("white alpha: %v", got) + } +} +func TestEXIFOrientation(t *testing.T) { + for _, order := range []binary.ByteOrder{binary.LittleEndian, binary.BigEndian} { + for v := 1; v <= 8; v++ { + tiff := make([]byte, 26) + copy(tiff, "II") + if order == binary.BigEndian { + copy(tiff, "MM") + } + order.PutUint16(tiff[2:], 42) + order.PutUint32(tiff[4:], 8) + order.PutUint16(tiff[8:], 1) + order.PutUint16(tiff[10:], 0x112) + order.PutUint16(tiff[12:], 3) + order.PutUint32(tiff[14:], 1) + order.PutUint16(tiff[18:], uint16(v)) + data := append([]byte{255, 216, 255, 225, 0, 34}, append([]byte("Exif\x00\x00"), tiff...)...) + if Orientation(data) != v { + t.Fatalf("EXIF value %d", v) + } + pngData := []byte{137, 80, 78, 71, 13, 10, 26, 10} + size := make([]byte, 4) + binary.BigEndian.PutUint32(size, uint32(len(tiff))) + pngData = append(pngData, size...) + pngData = append(pngData, []byte("eXIf")...) + pngData = append(pngData, tiff...) + pngData = append(pngData, 0, 0, 0, 0) + if Orientation(pngData) != v { + t.Fatalf("PNG EXIF value %d", v) + } + } + } + for _, b := range [][]byte{nil, {}, {255, 216, 255, 225, 255, 255}, {255, 216, 255, 255, 225, 0, 2}} { + if Orientation(b) != 1 { + t.Fatal("malformed EXIF should be ignored") + } + } +} diff --git a/internal/attachment/service.go b/internal/attachment/service.go index 3156afbb..fb0a441d 100644 --- a/internal/attachment/service.go +++ b/internal/attachment/service.go @@ -2,6 +2,7 @@ package attachment import ( "context" + "encoding/json" "errors" "io" "log" @@ -19,32 +20,35 @@ const UploadTimeout = 45 * time.Second const MinimumGrace = 90 * time.Second type Attachment struct { - ID string `json:"id"` - ChatID string `json:"chat_id"` - OwnerID string `json:"owner_id"` - Filename string `json:"filename"` - ContentType string `json:"content_type"` - Size int64 `json:"size_bytes"` - SHA256 string `json:"sha256"` - Status string `json:"status"` - Variants []any `json:"variants"` - ErrorCode *string `json:"error_code"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` - Key string `json:"-"` + ID string `json:"id"` + ChatID string `json:"chat_id"` + OwnerID string `json:"owner_id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + Size int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Status string `json:"status"` + Variants []any `json:"variants"` + ErrorCode *string `json:"error_code"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + Key string `json:"-"` + PipelineVersion int `json:"-"` } type Service struct { - Pool *pgxpool.Pool - Store *Store - slots chan struct{} + Pool *pgxpool.Pool + Store *Store + slots chan struct{} + Async bool + Options Options } func New(pool *pgxpool.Pool, store *Store) *Service { return &Service{Pool: pool, Store: store, slots: make(chan struct{}, 1)} } func (s *Service) Ready(ctx context.Context) error { return s.Store.Ready(ctx) } -func (s *Service) Upload(ctx context.Context, chatID, userID, name, contentType string, body io.Reader) (Attachment, error) { +func (s *Service) uploadSync(ctx context.Context, chatID, userID, name, contentType string, body io.Reader) (Attachment, error) { if err := ValidateFilename(name); err != nil { return Attachment{}, err } @@ -98,17 +102,19 @@ func (s *Service) Get(ctx context.Context, id, userID string) (Attachment, error ctx, cancel := context.WithTimeout(ctx, 3*time.Second) defer cancel() a := Attachment{Variants: []any{}} - err = s.Pool.QueryRow(ctx, `SELECT a.id::text,a.chat_id::text,a.uploader_id::text,a.filename,a.object_key,a.content_type,a.size_bytes,a.sha256,a.status,a.error_code,a.created_at FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id WHERE a.id=$1 AND m.user_id=$2`, parsed.String(), userID).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.Key, &a.ContentType, &a.Size, &a.SHA256, &a.Status, &a.ErrorCode, &a.CreatedAt) + var variants []byte + err = s.Pool.QueryRow(ctx, `SELECT a.id::text,a.chat_id::text,a.uploader_id::text,a.filename,a.object_key,a.content_type,a.size_bytes,a.sha256,a.status,a.error_code,a.created_at,a.updated_at,a.variants,a.pipeline_version FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id WHERE a.id=$1 AND m.user_id=$2`, parsed.String(), userID).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.Key, &a.ContentType, &a.Size, &a.SHA256, &a.Status, &a.ErrorCode, &a.CreatedAt, &a.UpdatedAt, &variants, &a.PipelineVersion) if errors.Is(err, pgx.ErrNoRows) { return Attachment{}, appError(domain.ErrorNotFound, "Объект не найден") } if err != nil { return Attachment{}, unavailable() } - a.UpdatedAt = a.CreatedAt - // Keep the mandatory lab4 ready-only metadata schema. Failed originals are - // represented by 409; internal status remains inspectable in reconciliation. - if a.Status != "ready" { + if err = json.Unmarshal(variants, &a.Variants); err != nil { + return Attachment{}, unavailable() + } + // The legacy fixture service preserves lab4 errors; production Async exposes all states. + if a.Status != "ready" && !s.Async { return Attachment{}, appError(domain.ErrorConflict, "Оригинал вложения недоступен") } return a, nil diff --git a/internal/attachment/worker.go b/internal/attachment/worker.go new file mode 100644 index 00000000..30c72a7f --- /dev/null +++ b/internal/attachment/worker.go @@ -0,0 +1,618 @@ +package attachment + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "math/rand/v2" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "syscall" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/minio/minio-go/v7" + "github.com/redis/go-redis/v9" + "golang.org/x/sync/semaphore" +) + +const Stream = "attachments:jobs:v1" +const Group = "image-workers-v1" + +type JobMessage struct { + SchemaVersion int `json:"schema_version"` + JobID string `json:"job_id"` + Type string `json:"type"` + AttachmentID string `json:"attachment_id"` + PipelineVersion int `json:"pipeline_version"` + Attempt int `json:"attempt"` + CreatedAt time.Time `json:"created_at"` + Traceparent string `json:"traceparent,omitempty"` + Tracestate string `json:"tracestate,omitempty"` +} +type Variant struct { + Name string `json:"name"` + ContentType string `json:"content_type"` + Width int `json:"width"` + Height int `json:"height"` + Size int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` +} +type claimed struct { + JobMessage + Fence int64 + Stage, Original, MIME, SHA string + Width, Height int + Size int64 +} + +func (s *Service) put(ctx context.Context, key string, r io.Reader, size int64, mime string) error { + _, err := s.Store.Client.PutObject(ctx, s.Store.Bucket, key, r, size, minio.PutObjectOptions{ContentType: mime, DisableMultipart: true}) + return err +} +func (s *Service) Publish(ctx context.Context, r *redis.Client) error { + tx, err := s.Pool.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(context.Background()) + rows, err := tx.Query(ctx, `SELECT j.id::text,j.attachment_id::text,j.pipeline_version,j.attempts+1,j.created_at FROM attachment_outbox o JOIN attachment_jobs j ON j.id=o.job_id WHERE o.published_at IS NULL AND o.available_at<=clock_timestamp() AND j.state='queued' ORDER BY o.available_at LIMIT 32 FOR UPDATE OF o SKIP LOCKED`) + if err != nil { + return err + } + messages := []JobMessage{} + for rows.Next() { + m := JobMessage{SchemaVersion: 1, Type: "attachment.process.v1"} + if err = rows.Scan(&m.JobID, &m.AttachmentID, &m.PipelineVersion, &m.Attempt, &m.CreatedAt); err != nil { + rows.Close() + return err + } + m.CreatedAt = m.CreatedAt.UTC() + messages = append(messages, m) + } + err = rows.Err() + rows.Close() + if err != nil { + return err + } + for _, m := range messages { + raw, _ := json.Marshal(m) + id, e := r.XAdd(ctx, &redis.XAddArgs{Stream: Stream, Values: map[string]interface{}{"job": string(raw)}}).Result() + if e != nil { + return e + } + if _, e = tx.Exec(ctx, `UPDATE attachment_outbox SET published_at=clock_timestamp(),stream_id=$2 WHERE job_id=$1`, m.JobID, id); e != nil { + return e + } + } + return tx.Commit(ctx) +} +func (s *Service) recover(ctx context.Context) error { + tx, err := s.Pool.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(context.Background()) + rows, err := tx.Query(ctx, `SELECT id::text,attachment_id::text,attempts,max_attempts FROM attachment_jobs WHERE state='processing' AND lease_until= l.max { + state = "failed" + v := "retry_exhausted" + code = &v + } + if _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET state=$2,lease_until=NULL,available_at=clock_timestamp(),updated_at=clock_timestamp(),last_error='worker_lost' WHERE id=$1`, l.id, state); err != nil { + return err + } + if _, err = tx.Exec(ctx, `UPDATE attachments SET status=$2,error_code=$3,updated_at=clock_timestamp() WHERE id=$1`, l.att, state, code); err != nil { + return err + } + if state == "queued" { + if _, err = tx.Exec(ctx, `UPDATE attachment_outbox SET published_at=NULL,available_at=clock_timestamp() WHERE job_id=$1`, l.id); err != nil { + return err + } + } + } + return tx.Commit(ctx) +} +func (s *Service) claim(ctx context.Context, m JobMessage) (claimed, bool, error) { + j := claimed{JobMessage: m} + tx, err := s.Pool.Begin(ctx) + if err != nil { + return j, false, err + } + defer tx.Rollback(context.Background()) + var state string + var attempts, max, version int + var available time.Time + err = tx.QueryRow(ctx, `SELECT state,attempts,max_attempts,available_at,pipeline_version FROM attachment_jobs WHERE id=$1 AND attachment_id=$2 FOR UPDATE`, m.JobID, m.AttachmentID).Scan(&state, &attempts, &max, &available, &version) + if errors.Is(err, pgx.ErrNoRows) { + return j, false, nil + } + if err != nil { + return j, false, err + } + if state == "ready" { + var key, mime string + var size int64 + var raw []byte + if err = tx.QueryRow(ctx, `SELECT object_key,content_type,size_bytes,variants FROM attachments WHERE id=$1 AND status='ready' AND pipeline_version=$2`, m.AttachmentID, version).Scan(&key, &mime, &size, &raw); err != nil { + return j, false, err + } + var variants []Variant + if err = json.Unmarshal(raw, &variants); err != nil || len(variants) != 3 { + return j, false, fmt.Errorf("completed effect metadata incomplete") + } + objects := map[string]int64{key: size} + for _, v := range variants { + objects[effectKey(m.AttachmentID, version, v.Name)] = v.Size + } + for object, n := range objects { + info, e := s.Store.Client.StatObject(ctx, s.Store.Bucket, object, minio.StatObjectOptions{}) + if e != nil { + return j, false, e + } + if info.Size != n { + return j, false, fmt.Errorf("completed effect size mismatch") + } + } + return j, false, nil + } + if state != "queued" || available.After(time.Now()) || m.Attempt != attempts+1 || version != m.PipelineVersion { + return j, false, nil + } + if attempts >= max { + return j, false, fmt.Errorf("attempt invariant violated") + } + err = tx.QueryRow(ctx, `UPDATE attachment_jobs SET state='processing',attempts=attempts+1,fence=fence+1,lease_until=clock_timestamp()+$2*interval '1 second',updated_at=clock_timestamp() WHERE id=$1 RETURNING fence`, m.JobID, s.Options.Lease.Seconds()).Scan(&j.Fence) + if err != nil { + return j, false, err + } + err = tx.QueryRow(ctx, `UPDATE attachments SET status='processing',updated_at=clock_timestamp() WHERE id=$1 AND pipeline_version=$2 AND status IN ('queued','processing') RETURNING staging_key,object_key,content_type,size_bytes,sha256,width,height`, m.AttachmentID, version).Scan(&j.Stage, &j.Original, &j.MIME, &j.Size, &j.SHA, &j.Width, &j.Height) + if err != nil { + return j, false, err + } + return j, true, tx.Commit(ctx) +} + +// finish publishes metadata only for the current, unexpired fencing token. +func (s *Service) finish(ctx context.Context, j claimed, variants []Variant, code string, permanent bool) (bool, error) { + tx, err := s.Pool.Begin(ctx) + if err != nil { + return false, err + } + defer tx.Rollback(context.Background()) + var max int + err = tx.QueryRow(ctx, `SELECT max_attempts FROM attachment_jobs WHERE id=$1 AND fence=$2 AND state='processing' AND lease_until>clock_timestamp() AND pipeline_version=$3 FOR UPDATE`, j.JobID, j.Fence, j.PipelineVersion).Scan(&max) + if errors.Is(err, pgx.ErrNoRows) { + return false, nil + } + if err != nil { + return false, err + } + lastError := code + state := "ready" + var failure *string + available := time.Now() + if code != "" { + state = "queued" + if permanent || j.Attempt >= max { + state = "failed" + if !permanent { + code = "retry_exhausted" + } + failure = &code + } + delay := time.Duration(1< 0 { + op, cancel = context.WithTimeout(ctx, time.Second) + streams, e := r.XReadGroup(op, &redis.XReadGroupArgs{Group: Group, Consumer: consumer, Streams: []string{Stream, ">"}, Count: int64(room), Block: 100 * time.Millisecond}).Result() + cancel() + if e == nil { + for _, stream := range streams { + for _, m := range stream.Messages { + select { + case queue <- m: + case <-ctx.Done(): + return nil + } + } + } + } + } + if time.Since(lastCleanup) >= 5*time.Second { + lastCleanup = time.Now() + op, cancel = context.WithTimeout(ctx, 5*time.Second) + s.cleanStages(op) + cancel() + } + } +} +func ack(ctx context.Context, r *redis.Client, id string) { + if r.XAck(ctx, Stream, Group, id).Err() == nil { + r.XDel(ctx, Stream, id) + } +} + +// Replay is an explicit operator action, serialized with admission and job claims. +// It preserves job/effect identity and refuses already successful work. +func (s *Service) Replay(ctx context.Context, id, operator, reason string) error { + if uuid.Validate(id) != nil || len(operator) < 1 || len(operator) > 100 || len(reason) < 1 || len(reason) > 300 { + return fmt.Errorf("job UUID, operator and reason are required") + } + tx, err := s.Pool.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(context.Background()) + if _, err = tx.Exec(ctx, `SELECT pg_advisory_xact_lock(578903551)`); err != nil { + return err + } + var pending int + if err = tx.QueryRow(ctx, `SELECT count(*) FROM attachment_jobs WHERE state IN ('queued','processing')`).Scan(&pending); err != nil { + return err + } + if pending >= s.Options.MaxOutstanding { + return fmt.Errorf("admission limit reached") + } + var att, state, previous string + err = tx.QueryRow(ctx, `SELECT attachment_id::text,state,coalesce(last_error,'processing_failed') FROM attachment_jobs WHERE id=$1 FOR UPDATE`, id).Scan(&att, &state, &previous) + if err != nil { + return err + } + if state != "failed" { + return fmt.Errorf("only failed jobs may be replayed") + } + var stage *string + if err = tx.QueryRow(ctx, `SELECT staging_key FROM attachments WHERE id=$1 FOR UPDATE`, att).Scan(&stage); err != nil { + return err + } + if stage == nil { + return fmt.Errorf("staging retention expired") + } + if _, err = s.Store.Client.StatObject(ctx, s.Store.Bucket, *stage, minio.StatObjectOptions{}); err != nil { + return fmt.Errorf("staging unavailable") + } + if _, err = tx.Exec(ctx, `INSERT INTO attachment_replays(job_id,operator,reason,previous_error) VALUES($1,$2,$3,$4)`, id, operator, reason, previous); err != nil { + return err + } + if _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET state='queued',attempts=0,fence=fence+1,last_error=NULL,available_at=clock_timestamp(),updated_at=clock_timestamp() WHERE id=$1`, id); err != nil { + return err + } + if _, err = tx.Exec(ctx, `UPDATE attachments SET status='queued',error_code=NULL,updated_at=clock_timestamp() WHERE id=$1`, att); err != nil { + return err + } + if _, err = tx.Exec(ctx, `UPDATE attachment_outbox SET published_at=NULL,available_at=clock_timestamp() WHERE job_id=$1`, id); err != nil { + return err + } + return tx.Commit(ctx) +} + +// Keep failed stages for seven days for explicit replay; ready stages are garbage. +func (s *Service) cleanStages(ctx context.Context) { + rows, err := s.Pool.Query(ctx, `SELECT id::text FROM attachments WHERE (status='ready' AND staging_key IS NOT NULL) OR (status='failed' AND updated_at'failed' AND error_code IS NULL)), + ADD COLUMN variants jsonb NOT NULL DEFAULT '[]', + ADD COLUMN updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1, + ADD COLUMN staging_key text; + +-- +goose Down +UPDATE attachments SET status='failed',error_code='pipeline_downgrade' WHERE status IN ('queued','processing'); +ALTER TABLE attachments DROP CONSTRAINT attachments_status_check, + ADD CONSTRAINT attachments_status_check CHECK (status IN ('ready','failed')), + DROP CONSTRAINT attachments_check, + ADD CONSTRAINT attachments_check CHECK ((status='ready' AND error_code IS NULL) OR (status='failed' AND error_code IS NOT NULL)), + DROP COLUMN variants, DROP COLUMN updated_at, DROP COLUMN pipeline_version, DROP COLUMN staging_key; diff --git a/migrations/006_attachment_jobs.sql b/migrations/006_attachment_jobs.sql new file mode 100644 index 00000000..96fe8715 --- /dev/null +++ b/migrations/006_attachment_jobs.sql @@ -0,0 +1,18 @@ +-- +goose Up +CREATE TABLE attachment_jobs ( + id uuid PRIMARY KEY, + attachment_id uuid NOT NULL UNIQUE REFERENCES attachments(id), + pipeline_version integer NOT NULL DEFAULT 1, + state text NOT NULL CHECK (state IN ('queued','processing','ready','failed')), + attempts integer NOT NULL DEFAULT 0, + max_attempts integer NOT NULL CHECK (max_attempts BETWEEN 1 AND 20), + available_at timestamptz NOT NULL DEFAULT clock_timestamp(), + lease_until timestamptz, + fence bigint NOT NULL DEFAULT 0, + last_error text, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +-- +goose Down +DROP TABLE attachment_jobs; diff --git a/migrations/007_attachment_outbox.sql b/migrations/007_attachment_outbox.sql new file mode 100644 index 00000000..4aa021c3 --- /dev/null +++ b/migrations/007_attachment_outbox.sql @@ -0,0 +1,10 @@ +-- +goose Up +CREATE TABLE attachment_outbox ( + job_id uuid PRIMARY KEY REFERENCES attachment_jobs(id), + available_at timestamptz NOT NULL DEFAULT clock_timestamp(), + published_at timestamptz, + stream_id text +); + +-- +goose Down +DROP TABLE attachment_outbox; diff --git a/migrations/008_attachment_replays.sql b/migrations/008_attachment_replays.sql new file mode 100644 index 00000000..2587f83c --- /dev/null +++ b/migrations/008_attachment_replays.sql @@ -0,0 +1,12 @@ +-- +goose Up +CREATE TABLE attachment_replays ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs(id), + operator text NOT NULL CHECK (char_length(operator) BETWEEN 1 AND 100), + reason text NOT NULL CHECK (char_length(reason) BETWEEN 1 AND 300), + previous_error text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +-- +goose Down +DROP TABLE attachment_replays; diff --git a/migrations/009_backfill_pipeline.sql b/migrations/009_backfill_pipeline.sql new file mode 100644 index 00000000..ac2b71e7 --- /dev/null +++ b/migrations/009_backfill_pipeline.sql @@ -0,0 +1,12 @@ +-- +goose Up +-- Re-process accepted lab4 originals; original keys remain durable staging until ready. +INSERT INTO attachment_jobs(id,attachment_id,state,max_attempts) +SELECT id,id,'queued',5 FROM attachments WHERE status='ready' AND variants='[]'::jsonb; +UPDATE attachments SET staging_key=object_key, + object_key='original/'||id::text||'/v1/original',status='queued',updated_at=clock_timestamp() +WHERE id IN (SELECT attachment_id FROM attachment_jobs); +INSERT INTO attachment_outbox(job_id) SELECT id FROM attachment_jobs; + +-- +goose Down +-- Do not erase accepted processing jobs during a downgrade. Schema Down is destructive. +SELECT 1; diff --git a/migrations/010_storage_cleanup.sql b/migrations/010_storage_cleanup.sql new file mode 100644 index 00000000..62cbac50 --- /dev/null +++ b/migrations/010_storage_cleanup.sql @@ -0,0 +1,5 @@ +-- +goose Up +ALTER TABLE attachments ADD COLUMN storage_cleaned_at timestamptz; + +-- +goose Down +ALTER TABLE attachments DROP COLUMN storage_cleaned_at; diff --git a/requirements-research.txt b/requirements-research.txt new file mode 100644 index 00000000..129feb00 --- /dev/null +++ b/requirements-research.txt @@ -0,0 +1,2 @@ +# Only for rendering the measured lab5 research charts. +matplotlib==3.9.4 diff --git a/scripts/benchmark_pipeline.py b/scripts/benchmark_pipeline.py new file mode 100644 index 00000000..191ce11c --- /dev/null +++ b/scripts/benchmark_pipeline.py @@ -0,0 +1,332 @@ +#!/usr/bin/env python3 +"""Repeatable lab5 track 1 experiment against the isolated Swarm stack. + +Uses synthetic users and PNGs. Changes only worker knobs and admission limits, +then restores their original values. Never deletes volumes or accepted jobs. +""" +import argparse +from concurrent.futures import ThreadPoolExecutor +import csv +import datetime as dt +import hashlib +import json +import math +import os +from pathlib import Path +import platform +import random +import struct +import subprocess +import sys +import time +import zlib + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, check + + +def docker(*args): + return subprocess.check_output(['docker', *args], text=True).strip() + + +def service_spec(name): + return json.loads(docker('service', 'inspect', name))[0]['Spec'] + + +def containers(service): + return docker('ps', '--filter', 'label=com.docker.swarm.service.name=' + service, + '--format', '{{.ID}}').split() + + +def wait_service(name, timeout=120): + deadline, stable, previous = time.monotonic() + timeout, None, None + while time.monotonic() < deadline: + inspected = json.loads(docker('service', 'inspect', name))[0] + spec = inspected['Spec'] + update_state = inspected.get('UpdateStatus', {}).get('State') + check(update_state not in ('paused', 'rollback_started', 'rollback_paused', 'rollback_completed'), + 'service configuration update failed: ' + name) + expected = spec['Mode']['Replicated']['Replicas'] + ids = docker('service', 'ps', '--filter', 'desired-state=running', '-q', name).split() + healthy = expected > 0 and len(ids) == expected and update_state in (None, 'completed') + if healthy: + tasks = json.loads(docker('inspect', *ids)) + healthy = all(t['Status']['State'] == 'running' for t in tasks) + if healthy: + states = json.loads(docker('inspect', *[t['Status']['ContainerStatus']['ContainerID'] for t in tasks])) + healthy = all(c['State'].get('Health', {}).get('Status', 'healthy') == 'healthy' for c in states) + now, identity = time.monotonic(), frozenset(ids) + if healthy: + if stable is None or identity != previous: + stable = now + if stable is not None and now - stable >= 5: + return + else: + stable = None + previous = identity + time.sleep(.5) + raise AssertionError('service did not stabilize: ' + name) + + +def update_env(service, values): + args = ['service', 'update', '--detach'] + for name, value in values.items(): + args += ['--env-rm', name] if value is None else ['--env-add', name + '=' + str(value)] + docker(*args, service) + wait_service(service) + + +def old_env(spec, names): + values = dict(entry.split('=', 1) for entry in spec['TaskTemplate']['ContainerSpec'].get('Env', [])) + return {name: values.get(name) for name in names} + + +def png(width, height): + rng = random.Random(20261110 + width) + def chunk(kind, value): + return struct.pack('>I', len(value)) + kind + value + struct.pack('>I', zlib.crc32(kind + value)) + pixels = b''.join(b'\0' + rng.randbytes(width * 3) for _ in range(height)) + return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)) + + chunk(b'IDAT', zlib.compress(pixels)) + chunk(b'IEND', b'')) + + +def percentile(values, fraction): + check(bool(values), 'no observations for percentile') + return sorted(values)[max(0, math.ceil(len(values) * fraction) - 1)] + + +class Experiment: + def __init__(self, args): + self.args = args + self.api = args.stack + '_api' + self.worker = args.stack + '_worker' + self.pg = args.stack + '_postgres' + self.client = Client(args.base_url, timeout=60) + + def sql(self, statement): + ids = containers(self.pg) + check(len(ids) == 1, 'expected one PostgreSQL container') + return docker('exec', ids[0], 'psql', '-U', 'messenger', '-d', 'messenger', + '-At', '-v', 'ON_ERROR_STOP=1', '-c', statement) + + def outstanding(self): + return int(self.sql("SELECT count(*) FROM attachment_jobs WHERE state IN ('queued','processing')")) + + def sample(self, chat): + # chat is a UUID generated by the API, never an arbitrary SQL fragment. + from uuid import UUID + chat_id = str(UUID(chat['id'])) + query = """SELECT json_build_object( + 'outstanding', count(*) FILTER (WHERE j.state IN ('queued','processing')), + 'queued', count(*) FILTER (WHERE j.state='queued'), + 'processing', count(*) FILTER (WHERE j.state='processing'), + 'queue_lag_seconds', COALESCE(EXTRACT(EPOCH FROM clock_timestamp()-min(j.created_at) + FILTER (WHERE j.state='queued')),0)) + FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id + WHERE a.chat_id='""" + chat_id + "'" + result = json.loads(self.sql(query)) + total_rss = 0 + for container in containers(self.worker): + # Only process status counters; no command lines, environment or credentials. + raw = docker('exec', container, 'sh', '-c', + "awk '/^VmRSS:/{s+=$2} END{printf \"%.0f\\n\",s*1024}' /proc/[0-9]*/status 2>/dev/null || true") + total_rss += int(raw or 0) + result['rss_bytes'] = total_rss + return result + + def upload(self, actor, chat, content): + started = time.monotonic() + response = Client(self.args.base_url, timeout=60).call( + 'POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=research.png', + [202, 429], actor=actor, raw=content, headers={'Content-Type': 'image/png'}, schema='Attachment') + return {'started': started, 'accepted': time.monotonic(), 'status': response['status'], + 'instance': response['headers'].get('X-Instance-Id'), + 'attachment': response['json'] if response['status'] == 202 else None} + + def validate_result(self, actor, attachment, content): + check({v['name'] for v in attachment['variants']} == {'thumbnail', 'crop', 'optimized'}, + 'ready must publish all three variants') + original = self.client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', actor=actor) + check(original['bytes'] == content, 'original changed during pipeline') + for variant in attachment['variants']: + response = self.client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content?variant=' + variant['name'], actor=actor) + check(len(response['bytes']) == variant['size_bytes'], 'variant size differs') + check(hashlib.sha256(response['bytes']).hexdigest() == variant['sha256'], 'variant checksum differs') + + def run_case(self, concurrency, prefetch, width, height): + check(self.outstanding() == 0, 'drain other workloads before research') + update_env(self.worker, {'WORKER_CONCURRENCY': concurrency, 'WORKER_PREFETCH': prefetch}) + alice, bob, _, chat = self.client.bootstrap() + content = png(width, height) + warm = self.upload(alice, chat, content) + check(warm['status'] == 202, 'warmup unexpectedly rejected') + self.client.wait_ready(alice, warm['attachment'], wait_seconds=self.args.timeout) + records, samples, rejected = [], [], 0 + started, deadline = time.monotonic(), time.monotonic() + self.args.timeout + baseline = self.sample(chat) + with ThreadPoolExecutor(max_workers=self.args.uploaders) as pool: + pending = {pool.submit(self.upload, alice, chat, content) for _ in range(self.args.requests)} + polling = {} + while pending or polling: + check(time.monotonic() < deadline, 'pipeline did not drain before research timeout') + done = {future for future in pending if future.done()} + pending -= done + for future in done: + row = future.result() + if row['status'] == 429: + rejected += 1 + else: + check(row['attachment']['status'] == 'queued', 'upload must return a queued snapshot') + polling[row['attachment']['id']] = row + for attachment_id, row in list(polling.items()): + attachment = self.client.call('GET', '/api/v1/attachments/' + attachment_id, + actor=bob, schema='Attachment')['json'] + check(attachment['status'] != 'failed', 'research attachment failed') + if attachment['status'] == 'ready': + now = time.monotonic() + row.update(attachment=attachment, upload_ms=(row['accepted']-row['started'])*1000, + ready_ms=(now-row['started'])*1000, + after_accept_ms=(now-row['accepted'])*1000) + records.append(row) + del polling[attachment_id] + sample = self.sample(chat) + samples.append({'seconds': round(time.monotonic() - started, 3), **sample}) + if pending or polling: + time.sleep(self.args.interval) + elapsed = time.monotonic() - started + check(len(records) == self.args.requests and rejected == 0, 'unexpected rejection in comparison matrix') + check(self.outstanding() == 0, 'comparison workload not drained') + self.validate_result(bob, records[-1]['attachment'], content) + result = {'concurrency': concurrency, 'prefetch': prefetch, 'width': width, 'height': height, + 'size_bytes': len(content), 'input_sha256': hashlib.sha256(content).hexdigest(), + 'offered': self.args.requests, 'accepted': len(records), + 'rejected': rejected, 'completed': len(records), 'makespan_seconds': round(elapsed, 3), + 'throughput_rps': round(len(records) / elapsed, 3), + 'ready_p50_ms': round(percentile([r['ready_ms'] for r in records], .5), 3), + 'ready_p95_ms': round(percentile([r['ready_ms'] for r in records], .95), 3), + 'after_accept_p95_ms': round(percentile([r['after_accept_ms'] for r in records], .95), 3), + 'upload_p95_ms': round(percentile([r['upload_ms'] for r in records], .95), 3), + 'queue_lag_peak_seconds': max(s['queue_lag_seconds'] for s in samples), + 'outstanding_peak': max(s['outstanding'] for s in samples), + 'rss_baseline_bytes': baseline['rss_bytes'], + 'rss_peak_bytes': max([baseline['rss_bytes']] + [s['rss_bytes'] for s in samples]), + 'samples': samples, + 'observations': [{'attachment_id': r['attachment']['id'], 'ready_ms': round(r['ready_ms'], 3), + 'upload_ms': round(r['upload_ms'], 3)} for r in records]} + print(json.dumps({k: v for k, v in result.items() if k not in ('samples', 'observations')}), flush=True) + return result + + def overload(self, limit): + check(self.outstanding() == 0, 'drain other workloads before overload test') + docker('service', 'scale', '--detach', self.worker + '=0') + deadline = time.monotonic() + 60 + while containers(self.worker): + check(time.monotonic() < deadline, 'worker did not stop') + time.sleep(.5) + update_env(self.api, {'MAX_OUTSTANDING_JOBS': limit}) + alice, bob, _, chat = self.client.bootstrap() + content = png(320, 200) + with ThreadPoolExecutor(max_workers=self.args.uploaders) as pool: + rows = list(pool.map(lambda _: self.upload(alice, chat, content), range(limit * 2))) + accepted = [r for r in rows if r['status'] == 202] + rejected = len(rows) - len(accepted) + outstanding = self.outstanding() + check(len({row['instance'] for row in rows}) >= 2, 'overload must exercise both API replicas') + check(len(accepted) == limit and rejected == limit and outstanding == limit, + 'admission limit did not bound accepted work while worker was stopped') + self.client.message(alice, chat, 'text messages remain available during overload') + docker('service', 'scale', '--detach', self.worker + '=1') + wait_service(self.worker) + started = time.monotonic() + for row in accepted: + attachment = self.client.wait_ready(bob, row['attachment'], wait_seconds=self.args.timeout) + check(attachment['error_code'] is None, 'accepted work failed after overload') + recovery = time.monotonic() - started + check(self.outstanding() == 0, 'accepted work remained outstanding after recovery') + followup = self.upload(alice, chat, content) + check(followup['status'] == 202, 'API did not resume admission after drain') + self.client.wait_ready(bob, followup['attachment'], wait_seconds=self.args.timeout) + result = {'limit': limit, 'offered': len(rows), 'accepted': len(accepted), 'rejected_429': rejected, + 'observed_api_instances': len({row['instance'] for row in rows}), + 'outstanding_worker_stopped': outstanding, 'ready_after_resume': len(accepted), + 'observed_drain_after_worker_healthy_seconds': round(recovery, 3), + 'outstanding_after_recovery': self.outstanding(), 'followup_upload': 202, + 'text_messages_during_overload': 'PASS'} + print('OVERLOAD ' + json.dumps(result), flush=True) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--base-url', default=os.environ.get('BASE_URL', 'http://localhost:18085')) + parser.add_argument('--stack', default=os.environ.get('STACK_NAME', 'mtusi-lab5')) + parser.add_argument('--requests', type=int, default=20) + parser.add_argument('--uploaders', type=int, default=8) + parser.add_argument('--interval', type=float, default=.25) + parser.add_argument('--timeout', type=int, default=240) + parser.add_argument('--limit', type=int, default=8) + parser.add_argument('--output', type=Path, default=ROOT / 'evidence/lab5-research.json') + args = parser.parse_args() + check(args.requests >= 5 and args.uploaders > 0 and 0 < args.limit <= 100, + 'invalid experiment limits') + check(args.interval > 0 and args.timeout > 0, 'sampling interval and timeout must be positive') + experiment = Experiment(args) + initial_worker, initial_api = service_spec(experiment.worker), service_spec(experiment.api) + original_worker_env = old_env(initial_worker, ('WORKER_CONCURRENCY', 'WORKER_PREFETCH')) + original_api_env = old_env(initial_api, ('MAX_OUTSTANDING_JOBS',)) + original_replicas = initial_worker['Mode']['Replicated']['Replicas'] + check(original_replicas == 1, 'experiment expects one worker service replica; concurrency is internal') + report = {'track': 1, 'started_at': dt.datetime.now(dt.timezone.utc).isoformat(), + 'stack': args.stack, 'worker_image': initial_worker['TaskTemplate']['ContainerSpec']['Image'], + 'worker_image_id': docker('inspect', containers(experiment.worker)[0], '--format', '{{.Image}}'), + 'environment': {'docker_server': docker('version', '--format', '{{.Server.Version}}'), + 'docker_cpus': int(docker('info', '--format', '{{.NCPU}}')), + 'docker_memory_bytes': int(docker('info', '--format', '{{.MemTotal}}')), + 'architecture': platform.machine(), 'python': platform.python_version()}, + 'worker_limits': initial_worker['TaskTemplate'].get('Resources', {}).get('Limits', {}), + 'initial_settings': {'worker': original_worker_env, 'api': original_api_env}, + 'requests_per_case': args.requests, 'upload_concurrency': args.uploaders, + 'sampling_interval_min_seconds': args.interval, + 'method': {'throughput': 'completed / burst makespan including client upload and polling', + 'ready_latency': 'client POST start to first observed ready; nearest-rank p95', + 'queue_lag': 'age of oldest queued PostgreSQL job in this chat; excludes processing', + 'rss': 'sum VmRSS for worker, renderers and cwebp; shared pages may count repeatedly', + 'limitations': 'single host; warmup per case; sampled peaks and polling latency; finite burst, no production SLA'}, + 'cases': []} + args.output.parent.mkdir(parents=True, exist_ok=True) + def save(): + args.output.write_text(json.dumps(report, indent=2, ensure_ascii=False, allow_nan=False) + '\n') + try: + for width, height in ((512, 512), (1536, 1536)): + for concurrency, prefetch in ((1, 1), (2, 4), (4, 8)): + report['cases'].append(experiment.run_case(concurrency, prefetch, width, height)) + save() + report['overload'] = experiment.overload(args.limit) + save() + finally: + # Keep every accepted job recoverable, including when an assertion fails. + try: + if not containers(experiment.worker): + docker('service', 'scale', '--detach', experiment.worker + '=' + str(original_replicas)) + wait_service(experiment.worker) + update_env(experiment.worker, original_worker_env) + finally: + update_env(experiment.api, original_api_env) + report['result'] = 'PASS' + report['restored_initial_settings'] = True + report['finished_at'] = dt.datetime.now(dt.timezone.utc).isoformat() + save() + summary = args.output.with_suffix('.csv') + keys = ['width', 'height', 'concurrency', 'prefetch', 'offered', 'accepted', 'completed', + 'throughput_rps', 'ready_p95_ms', 'after_accept_p95_ms', 'upload_p95_ms', + 'queue_lag_peak_seconds', 'rss_peak_bytes'] + with summary.open('w', newline='') as file: + writer = csv.DictWriter(file, fieldnames=keys, extrasaction='ignore') + writer.writeheader() + writer.writerows(report['cases']) + print('PASS: six measured configurations, bounded overload and recovery; ' + str(args.output)) + + +if __name__ == '__main__': + main() diff --git a/scripts/measure_load.py b/scripts/measure_load.py index 82481f07..891b407c 100644 --- a/scripts/measure_load.py +++ b/scripts/measure_load.py @@ -9,7 +9,7 @@ import sys import time ROOT = Path(__file__).resolve().parents[1] -STACK = os.environ.get('STACK_NAME', 'mtusi-lab4') +STACK = os.environ.get('STACK_NAME', 'mtusi-lab5') def containers(service=None): label = 'com.docker.swarm.service.name='+STACK+'_'+service if service else 'com.docker.stack.namespace='+STACK diff --git a/scripts/measure_uploads.py b/scripts/measure_uploads.py index 7a0de357..07bd16f5 100644 --- a/scripts/measure_uploads.py +++ b/scripts/measure_uploads.py @@ -59,7 +59,7 @@ def main(): elapsed = time.monotonic()-started samples.append({'seconds':round(elapsed,3), 'bytes':memory(ids)}) for response in responses: - att = response['json'] + att = c.wait_ready(bob,response['json']) check(att['sha256']==sha,'concurrent upload checksum mismatch') downloaded = c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob) check(hashlib.sha256(downloaded['bytes']).hexdigest()==sha,'concurrent downloaded bytes changed') @@ -77,7 +77,7 @@ def main(): check(all(value<1536*1024*1024 for value in peak.values()),'API exceeded memory limit') results['runs'].append(run) print(json.dumps({k:v for k,v in run.items() if k!='samples'}),flush=True) - save('lab4-upload-memory.json',results) + save('lab5-upload-memory.json',results) print('PASS: ten simultaneous uploads at two sizes; all originals verified; memory samples saved.') diff --git a/scripts/plot_pipeline.py b/scripts/plot_pipeline.py new file mode 100644 index 00000000..d07a880d --- /dev/null +++ b/scripts/plot_pipeline.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Render measured pipeline results; install requirements-research.txt first.""" +import argparse +import json +import os +from pathlib import Path +import tempfile + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('input', nargs='?', type=Path, default=Path('evidence/lab5-research.json')) + args = parser.parse_args() + os.environ['MPLCONFIGDIR'] = str(Path(tempfile.gettempdir()) / 'mtusi-lab5-matplotlib') + import matplotlib + matplotlib.use('Agg') + import matplotlib.pyplot as plt + + report = json.loads(args.input.read_text()) + assert report.get('result') == 'PASS', 'plot only completed experiments' + cases = report['cases'] + metrics = [ + ('throughput_rps', 'Throughput, images/s', 1), + ('ready_p95_ms', 'p95 POST start → ready, s', .001), + ('queue_lag_peak_seconds', 'Peak queued job age, s', 1), + ('rss_peak_bytes', 'Sampled worker + child RSS, MiB', 1 / 1024**2), + ] + plt.rcParams.update({'font.size': 10, 'axes.spines.top': False, 'axes.spines.right': False}) + figure, axes = plt.subplots(2, 2, figsize=(10, 7), layout='constrained') + for axis, (metric, label, factor) in zip(axes.flat, metrics): + for width in sorted({case['width'] for case in cases}): + series = sorted([case for case in cases if case['width'] == width], key=lambda c: c['concurrency']) + axis.plot(range(len(series)), [case[metric] * factor for case in series], + marker='o', linewidth=2, label=str(width) + ' × ' + str(series[0]['height'])) + axis.set_xticks(range(len(series)), [str(c['concurrency']) + '/' + str(c['prefetch']) for c in series]) + axis.set_xlabel('Concurrency / prefetch') + axis.set_ylabel(label) + axis.set_ylim(bottom=0) + axis.grid(axis='y', alpha=.25) + axis.legend(title='PNG pixels') + figure.suptitle('Redis Streams image pipeline — finite burst on one host\n' + + str(report['requests_per_case']) + ' images per case; sampled RSS and queue lag', fontsize=13) + for suffix in ('.svg', '.png'): + target = args.input.with_suffix(suffix) + figure.savefig(target, dpi=180, metadata={'Creator': 'Matplotlib'}) + print(target) + plt.close(figure) + + +if __name__ == '__main__': + main() diff --git a/scripts/swarm.py b/scripts/swarm.py index a40d801f..e90e26bb 100644 --- a/scripts/swarm.py +++ b/scripts/swarm.py @@ -10,13 +10,13 @@ import urllib.request ROOT = Path(__file__).resolve().parents[1] os.chdir(ROOT) -STACK = os.environ.get('STACK_NAME', 'mtusi-lab4') -API = os.environ.get('API_IMAGE', 'mtusi-messenger:lab4') -MIGRATIONS = os.environ.get('MIGRATIONS_IMAGE', 'mtusi-migrations:lab4') -TESTS = os.environ.get('TESTS_IMAGE', 'mtusi-messenger-tests:lab4') +STACK = os.environ.get('STACK_NAME', 'mtusi-lab5') +API = os.environ.get('API_IMAGE', 'mtusi-messenger:lab5') +MIGRATIONS = os.environ.get('MIGRATIONS_IMAGE', 'mtusi-migrations:lab5') +TESTS = os.environ.get('TESTS_IMAGE', 'mtusi-messenger-tests:lab5') S3 = os.environ.get('S3_IMAGE', 'mtusi-minio:RELEASE.2025-09-07T16-13-09Z') MC = os.environ.get('S3_CLIENT_IMAGE', 'mtusi-mc:RELEASE.2025-08-13T08-35-41Z') -PORT = os.environ.get('PORT', '8080') +PORT = os.environ.get('PORT', '18085') SECRETS = ('postgres_password', 'cursor_key', 's3_root_access_key', 's3_root_secret_key', 's3_access_key', 's3_secret_key') @@ -71,7 +71,7 @@ def job(kind, image, command, integration=False): # Fixed service name is a deployment lock: concurrent runners fail to create it. args = ['service', 'create', '--detach', '--no-resolve-image', '--name', name, '--mode', 'replicated-job', '--replicas', '1', '--restart-condition', 'none', '--network', STACK+'_data', - '--constraint', 'node.labels.mtusi_lab4_storage==true', + '--constraint', 'node.labels.mtusi_lab5_storage==true', '--limit-cpu', '1', '--limit-memory', '1g' if integration else '128m', '--secret', 'source='+STACK+'_postgres_password,target=postgres_password', '--env', 'POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password'] @@ -113,10 +113,10 @@ def bootstrap_s3(): name = STACK+'_s3-bootstrap' args = ['service', 'create', '--detach', '--no-resolve-image', '--name', name, '--mode', 'replicated-job', '--replicas', '1', '--restart-condition', 'none', '--network', STACK+'_data', - '--constraint', 'node.labels.mtusi_lab4_storage==true', + '--constraint', 'node.labels.mtusi_lab5_storage==true', '--limit-cpu', '0.5', '--limit-memory', '512m', '--entrypoint', '/bin/sh', '--config', 'source='+STACK+'_s3_bootstrap,target=/bootstrap.sh', - '--config', 'source='+STACK+'_s3_policy,target=/policy.json'] + '--config', 'source='+STACK+'_s3_policy_v5,target=/policy.json'] for secret in SECRETS[2:]: args += ['--secret', 'source='+STACK+'_'+secret+',target='+secret] docker(*args, MC, '/bootstrap.sh') @@ -131,9 +131,18 @@ def reconcile(): docker('exec', container, '/usr/local/bin/reconcile', *sys.argv[2:]) +def worker_cli(args): + active=[t for t in tasks('worker') if t['Status']['State']=='running'] + if not active: raise RuntimeError('start worker before using its operator CLI') + docker('exec',active[0]['Status']['ContainerStatus']['ContainerID'],'/usr/local/bin/worker',*args) + + def build(): - docker('build', '-f', 'Dockerfile.storage', '--target', 'minio', '-t', S3, '.') - docker('build', '-f', 'Dockerfile.storage', '--target', 'mc', '-t', MC, '.') + if os.environ.get('SKIP_STORAGE_BUILD') != '1': + docker('build', '-f', 'Dockerfile.storage', '--target', 'minio', '-t', S3, '.') + docker('build', '-f', 'Dockerfile.storage', '--target', 'mc', '-t', MC, '.') + else: + docker('image','inspect',S3,MC,capture=True) docker('build', '-t', API, '.') docker('build', '--target', 'migrations', '-t', MIGRATIONS, '.') @@ -147,7 +156,7 @@ def up(): raise RuntimeError('run on the Swarm manager') if len(docker('node', 'ls', '-q', capture=True).split()) != 1: raise RuntimeError('automatic lifecycle supports one Swarm node; see REPORT.md for image distribution') - docker('node', 'update', '--label-add', 'mtusi_lab4_storage=true', swarm['NodeID']) + docker('node', 'update', '--label-add', 'mtusi_lab5_storage=true', swarm['NodeID']) subprocess.run(['./scripts/bootstrap.sh'], check=True) for secret in SECRETS: known = docker('secret', 'ls', '--format', '{{.Name}}', capture=True).split() @@ -158,6 +167,13 @@ def up(): # Upgrade the shared schema before replacing any running API image. fresh = STACK+'_api' not in existing if not fresh: + # A maintenance barrier prevents old lab4 writers inserting ready-only rows + # after the one-shot async backfill but before the new API image starts. + docker('service','scale','--detach',STACK+'_api=0') + deadline=time.monotonic()+45 + while docker('ps','--filter','label=com.docker.swarm.service.name='+STACK+'_api','-q',capture=True).strip(): + if time.monotonic()>deadline: raise RuntimeError('old API did not drain before migrations') + time.sleep(.5) job('migrations', MIGRATIONS, ['up']) # Initial deployment keeps API at zero until the one-shot migration succeeds. env = dict(os.environ) @@ -176,12 +192,13 @@ def up(): wait_api() else: update() + wait_service('worker') print('PASS: Swarm stack ready at http://localhost:'+PORT) def restart(): # Force replaces every task while retaining the same Swarm secrets and volumes. - services = ('postgres', 'redis', 'minio', 'api', 'lb') + services = ('postgres', 'redis', 'minio', 'api', 'lb', 'worker') previous = {name: {t['ID'] for t in tasks(name)} for name in services} for service in services: docker('service', 'update', '--detach', '--force', STACK+'_'+service) @@ -216,7 +233,9 @@ def update(): state = inspect('service', STACK+'_api').get('UpdateStatus', {}).get('State') if state == 'completed' and previous.isdisjoint(t['ID'] for t in tasks('api')): wait_api() - print('PASS: rolling update completed') + docker('service','update','--detach','--force','--no-resolve-image','--image',API,STACK+'_worker') + wait_service('worker') + print('PASS: rolling update completed (API and worker image refreshed)') return if state in ('paused', 'rollback_started', 'rollback_paused', 'rollback_completed'): raise RuntimeError('rolling update: '+state) @@ -226,7 +245,7 @@ def update(): def status(): result = {'stack': STACK, 'docker': docker('version','--format','{{.Server.Version}}',capture=True).strip(), 'services': {}} - for name in ('lb','api','postgres','redis','minio'): + for name in ('lb','api','worker','postgres','redis','minio'): service = inspect('service', STACK+'_'+name) spec = service['Spec'] template = spec['TaskTemplate'] @@ -248,6 +267,17 @@ def main(): elif action == 'build': build() elif action == 'bootstrap-s3': bootstrap_s3() elif action == 'reconcile': reconcile() + elif action == 'worker-stop': + docker('service','scale','--detach',STACK+'_worker=0') + deadline=time.monotonic()+30 + while docker('ps','--filter','label=com.docker.swarm.service.name='+STACK+'_worker','-q',capture=True).strip(): + if time.monotonic()>deadline: raise RuntimeError('worker did not stop') + time.sleep(.2) + elif action == 'worker-start': + docker('service','scale','--detach',STACK+'_worker=1') + wait_service('worker') + elif action == 'failed-jobs': worker_cli(['--failed']) + elif action == 'replay': worker_cli(sys.argv[2:]) elif action == 'wait': wait_api() elif action == 'restart': restart() elif action == 'stop-one': stop_one() diff --git a/stack.yaml b/stack.yaml index da599861..d984126d 100644 --- a/stack.yaml +++ b/stack.yaml @@ -18,7 +18,7 @@ services: deploy: replicas: 1 placement: - constraints: [node.labels.mtusi_lab4_storage == true] + constraints: [node.labels.mtusi_lab5_storage == true] resources: limits: {cpus: '1.0', memory: 512M} update_config: {parallelism: 1, order: stop-first} @@ -39,9 +39,9 @@ services: deploy: replicas: 1 placement: - constraints: [node.labels.mtusi_lab4_storage == true] + constraints: [node.labels.mtusi_lab5_storage == true] resources: - limits: {cpus: '0.5', memory: 128M} + limits: {cpus: '0.5', memory: 384M} update_config: {parallelism: 1, order: stop-first} minio: image: ${S3_IMAGE:-mtusi-minio:RELEASE.2025-09-07T16-13-09Z} @@ -58,13 +58,13 @@ services: deploy: replicas: 1 placement: - constraints: [node.labels.mtusi_lab4_storage == true] + constraints: [node.labels.mtusi_lab5_storage == true] resources: limits: {cpus: '1.0', memory: 512M} update_config: {parallelism: 1, order: stop-first} api: - image: ${API_IMAGE:-mtusi-messenger:lab4} - environment: + image: ${API_IMAGE:-mtusi-messenger:lab5} + environment: &api_environment INSTANCE_ID: '{{.Task.ID}}' POSTGRES_HOST: postgres POSTGRES_USER: messenger @@ -104,6 +104,27 @@ services: monitor: 15s failure_action: rollback rollback_config: {parallelism: 1, order: start-first} + worker: + image: ${API_IMAGE:-mtusi-messenger:lab5} + entrypoint: [/usr/local/bin/worker] + environment: + <<: *api_environment + WORKER_CONCURRENCY: '2' + WORKER_PREFETCH: '4' + WORKER_MAX_ATTEMPTS: '5' + WORKER_TIMEOUT_SECONDS: '25' + WORKER_LEASE_SECONDS: '40' + WORKER_PIXEL_BUDGET: '40000000' + secrets: [postgres_password, cursor_key, s3_access_key, s3_secret_key] + networks: [data] + healthcheck: {disable: true} + stop_grace_period: 10s + deploy: + replicas: 1 + resources: + limits: {cpus: '1.0', memory: 1536M} + restart_policy: {condition: any, delay: 1s} + update_config: {parallelism: 1, order: stop-first} lb: image: haproxy:3.2.6-alpine configs: @@ -112,7 +133,7 @@ services: networks: [backend] ports: - target: 8080 - published: ${PORT:-8080} + published: ${PORT:-18085} protocol: tcp mode: host healthcheck: @@ -138,27 +159,29 @@ configs: haproxy: file: ./infra/haproxy.cfg redis: + name: ${STACK_NAME:-mtusi-lab5}_redis_v5 file: ./infra/redis.conf s3_bootstrap: file: ./infra/s3-bootstrap.sh s3_policy: + name: ${STACK_NAME:-mtusi-lab5}_s3_policy_v5 file: ./infra/s3-policy.json secrets: postgres_password: external: true - name: ${STACK_NAME:-mtusi-lab4}_postgres_password + name: ${STACK_NAME:-mtusi-lab5}_postgres_password cursor_key: external: true - name: ${STACK_NAME:-mtusi-lab4}_cursor_key + name: ${STACK_NAME:-mtusi-lab5}_cursor_key s3_root_access_key: external: true - name: ${STACK_NAME:-mtusi-lab4}_s3_root_access_key + name: ${STACK_NAME:-mtusi-lab5}_s3_root_access_key s3_root_secret_key: external: true - name: ${STACK_NAME:-mtusi-lab4}_s3_root_secret_key + name: ${STACK_NAME:-mtusi-lab5}_s3_root_secret_key s3_access_key: external: true - name: ${STACK_NAME:-mtusi-lab4}_s3_access_key + name: ${STACK_NAME:-mtusi-lab5}_s3_access_key s3_secret_key: external: true - name: ${STACK_NAME:-mtusi-lab4}_s3_secret_key + name: ${STACK_NAME:-mtusi-lab5}_s3_secret_key diff --git a/tests/own/README.md b/tests/own/README.md index b446401a..31e10eb5 100644 --- a/tests/own/README.md +++ b/tests/own/README.md @@ -1,20 +1,20 @@ -# Собственные проверки lab4 +# Проверки lab5 -`make test-lab4` проверяет реальные PNG/JPEG, лимиты, MIME, повреждения, права, приватность bucket и отказы. -Тест останавливает именно API-реплику, которая приняла файл, и проверяет скачивание через другую. -`make measure-uploads` запускает десять клиентов на двух размерах PNG и сохраняет замеры cgroup memory. -`make test-reconcile` внедряет ошибки S3/БД и проверяет grace, блокировки, dry-run, повторное исправление и стоимость обхода. -Эти сценарии используют реальный MinIO. Запускайте их последовательно на учебном Swarm-стенде. +`make test` запускает неизменённый публичный smoke lab5. +`make test-own` запускает Go-тесты с race detector. +`make test-integration` проверяет реальные миграции, транзакции, гонки, пагинацию, fencing и версии. +`make test-lab5` проверяет реальные Redis Streams, MinIO и PostgreSQL. +Сценарий проверяет границы staging/COMMIT/publish/objects/COMMIT/ack и восстановление после остановки компонентов. +Он проверяет retry, failed registry, audit replay, process timeout и cleanup. +Проверки JPEG/WebP, EXIF, alpha и малых изображений используют Pillow из `requirements-research.txt`. +Сценарии изменяют только свой стек и синтетические данные. +Запускайте проверки отказов последовательно. -Результаты находятся в `evidence/lab4-*`. Подробная методика приведена в [REPORT.md](../../REPORT.md). +`make test-lab4` и `make test-reconcile` оставлены как совместимые имена для полного `test-lab5`. +Старые `reconcile.py` и `grade4.py` сохраняют исторические сценарии; напрямую их не запускайте. +`make test-persistence` проверяет общий restart; `make test-failover` проверяет остановку одной API-реплики. -Публичные tests/*.py и обязательный контракт сохранены. - -- `make test-own`: Go tests с race detector. Без INTEGRATION_TESTS=1 PostgreSQL-сценарий пропускается. -- `make test-integration`: отдельный Swarm job, Go 1.26.4 и goose. Временная схема, миграции с нуля/повторно/legacy/reset/up, регистрационная гонка, rollback, 10 000 сообщений с одинаковыми timestamps, EXPLAIN и 16 конкурентных запросов одного ключа. -- `make test-grade4`: `swarm_grade4.py`, ровно две API tasks. Проверяет общий ключ через LB, одинаковый результат, 409, границы ключа, область пользователя/чата, права и logout. По очереди останавливает обе исходные tasks с восстановлением, затем выполняет rolling update под нагрузкой. Проверяет историю, отсутствие дублей и повтор исходного ключа после воздействия. Сохраняет `evidence/swarm-grade4.json`. -- `make test-persistence`: перезапуск всех API, PostgreSQL, Redis, MinIO и LB; volumes/secrets сохраняются. -- `make test-failover ACTION_SCRIPT=scripts/stop-one.sh`: публичный сценарий отказа; `make wait` дожидается восстановления двух реплик. -- `make measure-load`: 3s прогрева + 30s, concurrency 4, метрики и Docker stats в `evidence/swarm-*`. - -Сценарии отказов, persistence и rolling update выполняйте **последовательно**, на локальном учебном стенде. Все данные синтетические; пароль и токены в результаты не записываются. Старый `grade4.py` относится к Compose-стенду lab2 и не является entrypoint текущей работы. +`make benchmark` измеряет очередь под перегрузкой для трёх настроек и двух размеров изображений. +`make plot-research` строит графики по фактическому JSON. +Методика и результаты находятся в [RESEARCH.md](../../RESEARCH.md). +Новые доказательства используют имена `evidence/lab5-*`; исходные lab4 evidence считаются историческими. diff --git a/tests/own/helpers.py b/tests/own/helpers.py index ca4dcfb3..d8032414 100644 --- a/tests/own/helpers.py +++ b/tests/own/helpers.py @@ -14,7 +14,7 @@ ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT / 'tests')) from client import Client, check -STACK = os.environ.get('STACK_NAME', 'mtusi-lab4') +STACK = os.environ.get('STACK_NAME', 'mtusi-lab5') MC = os.environ.get('S3_CLIENT_IMAGE', 'mtusi-mc:RELEASE.2025-08-13T08-35-41Z') @@ -43,11 +43,11 @@ def reconcile(apply=False, grace='90s'): return json.loads(docker(*args).stdout) -def upload(c, actor, chat, content, filename='sample.png', mime='image/png', expected=201): +def upload(c, actor, chat, content, filename='sample.png', mime='image/png', expected=202): from urllib.parse import quote return c.call('POST', '/api/v1/chats/'+chat['id']+'/attachments?filename='+quote(filename, safe=''), expected, actor=actor, raw=content, headers={'Content-Type': mime}, - schema='Attachment' if expected == 201 else None) + schema='Attachment' if expected == 202 else None) def png(width=320, height=200, noisy=False): @@ -72,13 +72,26 @@ def save(name, result): def wait_ready(timeout=90): deadline = time.monotonic()+timeout + stable_since = None + previous = None while time.monotonic() < deadline: try: - Client(timeout=2).call('GET', '/health/ready', schema='Health') - return - except AssertionError: - time.sleep(.5) - raise AssertionError('readiness did not recover') + ids=docker('service','ps','--filter','desired-state=running','-q',STACK+'_api').stdout.decode().split() + wanted=json.loads(docker('service','inspect',STACK+'_api').stdout)[0]['Spec']['Mode']['Replicated']['Replicas'] + check(wanted>0 and len(ids)==wanted,'API count') + ts=json.loads(docker('inspect',*ids).stdout) + check(all(t['Status']['State']=='running' for t in ts),'API task state') + cs=json.loads(docker('inspect',*[t['Status']['ContainerStatus']['ContainerID'] for t in ts]).stdout) + check(all(c['State'].get('Health',{}).get('Status')=='healthy' for c in cs),'API health') + Client(timeout=2).call('GET','/health/ready',schema='Health') + current=frozenset(ids) + if current!=previous: stable_since=time.monotonic() + previous=current + if stable_since is not None and time.monotonic()-stable_since>=5:return + except (AssertionError,subprocess.CalledProcessError,KeyError): + stable_since=None;previous=None + time.sleep(.5) + raise AssertionError('all API replicas and LB did not stabilize') class Probe: diff --git a/tests/own/lab4.py b/tests/own/lab4.py deleted file mode 100644 index 8622723b..00000000 --- a/tests/own/lab4.py +++ /dev/null @@ -1,154 +0,0 @@ -#!/usr/bin/env python3 -"""Real S3/API negative cases and failures; only synthetic lab data.""" -import hashlib -import http.client -import json -import os -from pathlib import Path -import subprocess -import time -import uuid -from urllib.parse import urlsplit - -from helpers import Client, Probe, ROOT, STACK, check, containers, docker, png, oversized_dimensions, save, sql, upload, wait_ready - - -def main(): - c = Client(timeout=55) - alice, bob, eve, chat = c.bootstrap() - data = png() - result = {'negative_cases': [], 'failures': {}, 'checksums': {}} - for name, body, mime, filename, status in ( - ('oversized_bytes', b'x'*(10*1024*1024+1), 'image/png', 'large.png', 413), - ('damaged_png', data[:50], 'image/png', 'broken.png', 415), - ('corrupt_crc', data[:-16]+b'\xff'*16, 'image/png', 'crc.png', 415), - ('mime_spoof', data, 'image/jpeg', 'fake.jpg', 415), - ('not_image', b'plain text', 'image/png', 'fake.png', 415), - ('unsupported_mime', data, 'application/octet-stream', 'file.png', 415), - ('pixel_budget', oversized_dimensions(), 'image/png', 'large-dimensions.png', 413), - ('unsafe_slash', data, 'image/png', '../file.png', 400), - ('unsafe_backslash', data, 'image/png', 'x\\y.png', 400), - ('unsafe_control', data, 'image/png', 'x\x00.png', 400), - ('empty_filename', data, 'image/png', '', 400), - ): - upload(c, alice, chat, body, filename, mime, status) - result['negative_cases'].append({'case': name, 'status': status}) - for content, filename, mime in ((data, 'изображение.png', 'image/png'), - ((ROOT/'tests/own/fixtures/sample.jpg').read_bytes(), 'photo.jpg', 'image/jpeg')): - att = upload(c, alice, chat, content, filename, mime)['json'] - downloaded = c.call('GET', '/api/v1/attachments/'+att['id']+'/content', actor=bob) - check(downloaded['bytes'] == content, 'original bytes changed') - check(att['sha256'] == hashlib.sha256(content).hexdigest(), 'incorrect SHA256') - check(downloaded['headers'].get_content_type() == mime, 'incorrect content MIME') - result['checksums'][mime] = att['sha256'] - att = upload(c, alice, chat, data)['json'] - for suffix in ('', '/content'): - c.call('GET', '/api/v1/attachments/'+att['id']+suffix, 404, actor=eve) - c.call('GET', '/api/v1/attachments/'+str(uuid.uuid4())+suffix, 404, actor=alice) - c.call('GET', '/api/v1/attachments/'+att['id']+suffix, 401) - upload(c, eve, chat, data, expected=404) - c.call('GET', '/api/v1/attachments/'+att['id']+'/content?variant=thumbnail', 404, actor=bob) - other = c.call('POST', '/api/v1/chats', 201, actor=alice, - body={'title':'other chat', 'member_ids':[alice['id'], bob['id']]}, schema='Chat')['json'] - path = '/api/v1/chats/'+other['id']+'/messages' - for ids, status in (([att['id']], 404), ([str(uuid.uuid4())], 404), ([att['id'],att['id']], 400)): - c.call('POST', path, status, actor=alice, body={'text':'', 'attachment_ids':ids}, - headers={'Idempotency-Key':uuid.uuid4().hex}) - result['negative_cases'] += [{'case': x, 'status': s} for x,s in ( - ('foreign_attachment',404), ('foreign_chat_link',404), ('unknown_attachment',404), - ('duplicate_ids',400), ('missing_authentication',401), ('unknown_variant',404))] - - # The proxy must stop oversized chunked bodies even without Content-Length. - address = urlsplit(c.base) - conn = http.client.HTTPConnection(address.hostname, address.port, timeout=55) - conn.request('POST', '/api/v1/chats/'+chat['id']+'/attachments?filename=chunked.png', - body=iter([b'x'*65536]*161), encode_chunked=True, - headers={'Authorization':'Bearer '+alice['token'], 'Content-Type':'image/png'}) - response = conn.getresponse() - check(response.status == 413, 'chunked upload bypassed byte limit') - response.read(); conn.close() - result['negative_cases'].append({'case':'chunked_oversized', 'status':413}) - - # Disconnect halfway through the declared body. No object/metadata can be published. - before = int(sql('SELECT count(*) FROM attachments;')) - conn = http.client.HTTPConnection(address.hostname, address.port, timeout=5) - conn.putrequest('POST', '/api/v1/chats/'+chat['id']+'/attachments?filename=aborted.png') - conn.putheader('Authorization', 'Bearer '+alice['token']) - conn.putheader('Content-Type', 'image/png') - conn.putheader('Content-Length', str(len(data)+10000)) - conn.endheaders(); conn.send(data[:40]); conn.close() - time.sleep(1) - check(int(sql('SELECT count(*) FROM attachments;')) == before, 'aborted upload published metadata') - leftovers = [docker('exec', container, 'find', '/tmp', '-maxdepth', '1', '-name', 'messenger-upload-*').stdout.decode() - for container in containers('api')] - check(not any(leftovers), 'aborted upload left spool files') - result['failures']['client_disconnect'] = {'metadata_added':0, 'spool_files_remaining':0} - - with Probe() as probe: - key = sql("SELECT object_key FROM attachments WHERE id='"+att['id']+"';") - raw = docker('exec', containers('minio')[0], 'wget', '-S', '-O', '/dev/null', - 'http://127.0.0.1:9000/attachments/'+key, check_result=False) - check(b'403' in raw.stderr, 'unauthenticated direct S3 GET was not forbidden') - probe.mc('stat', 'local/attachments/'+key) - probe.mc('mb', 'local/forbidden-'+uuid.uuid4().hex[:8], expected=1) - probe.mc('pipe', 'local/attachments/outside-prefix', data=b'x', expected=1) - result['bucket_security'] = {'anonymous_get_status':403, 'create_bucket_denied':True, - 'write_outside_prefix_denied':True, 'console_published':False} - service = json.loads(docker('service', 'inspect', STACK+'_minio').stdout)[0] - check(not service['Spec'].get('EndpointSpec', {}).get('Ports'), 'MinIO publishes an external port') - for _ in range(2): - subprocess.run(['python3', str(ROOT/'scripts/swarm.py'), 'bootstrap-s3'], check=True, - stdout=subprocess.DEVNULL) - check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==data, - 'repeated bucket bootstrap changed data') - result['bucket_security']['bootstrap_repeats'] = 2 - - # Stop the replica that accepted THIS upload, then read through another one. - accepted = upload(c,alice,chat,data) - source = accepted['headers']['X-Instance-Id'] - env = dict(os.environ, TARGET_INSTANCE_ID=source) - started = time.monotonic() - subprocess.run(['python3',str(ROOT/'scripts/swarm.py'),'stop-one'],env=env,check=True, - stdout=subprocess.DEVNULL) - deadline = time.monotonic()+10 - while True: - try: - downloaded = Client(timeout=2).call('GET','/api/v1/attachments/'+accepted['json']['id']+'/content',actor=bob) - check(downloaded['headers']['X-Instance-Id']!=source,'stopped upload replica still serves bytes') - check(downloaded['bytes']==data,'cross-replica original changed') - break - except AssertionError: - check(time.monotonic()=4 + def observe(): + try: + return ready() + except (AssertionError,subprocess.CalledProcessError,KeyError,IndexError): + stable[:]=[None,None] + raise + until(observe, timeout=120) + + +def control(service, values=None, replicas=None, force=False): + args = ['service','update','--detach'] + for key,value in (values or {}).items(): + args += ['--env-rm',key] if value is None else ['--env-add',key+'='+str(value)] + if replicas is not None: + args += ['--replicas',str(replicas)] + if force: + args += ['--force'] + docker(*args,STACK+'_'+service) + if replicas is None: + spec = json.loads(docker('service','inspect',STACK+'_'+service).stdout)[0] + replicas = spec['Spec']['Mode']['Replicated']['Replicas'] + settled(service,replicas) + + +def redis(*args): + return json.loads(docker('exec',containers('redis')[0],'redis-cli','--json',*args).stdout) + + +def job(att): + return json.loads(sql("SELECT row_to_json(j) FROM attachment_jobs j WHERE attachment_id='"+att['id']+"';")) + + +def wait_state(att, state, timeout=90): + return until(lambda: job(att), lambda j: j['state']==state, timeout) + + +def variant_check(c, actor, att, original, expected_thumb=(256,160)): + ready = c.wait_ready(actor,att) + check(ready['error_code'] is None, 'ready has error') + check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=actor)['bytes']==original,'original changed') + variants = {v['name']:v for v in ready['variants']} + check(set(variants)=={'thumbnail','crop','optimized'},'incomplete variants') + check((variants['thumbnail']['width'],variants['thumbnail']['height'])==expected_thumb,'thumbnail geometry') + check((variants['crop']['width'],variants['crop']['height'])==(128,128),'crop geometry') + for name,v in variants.items(): + response = c.call('GET','/api/v1/attachments/'+att['id']+'/content?variant='+name,actor=actor) + body = response['bytes'] + check(len(body)==v['size_bytes'] and hashlib.sha256(body).hexdigest()==v['sha256'],'variant checksum') + check(response['headers'].get_content_type()==v['content_type'],'variant MIME') + if name=='optimized': + check(body[:4]==b'RIFF' and body[8:12]==b'WEBP','not WebP') + else: + check(jpeg_size(body)==(v['width'],v['height']),'JPEG actual geometry') + return ready + + +def replay(j, expected=True): + result = docker('exec',containers('worker')[0],'/usr/local/bin/worker','--replay',j['id'], + '--operator','synthetic-fault-test','--reason','restore synthetic fixture',check_result=False) + check((result.returncode==0)==expected,'unexpected replay result') + + +def main(): + check(STACK!='mtusi-lab4','refuse source stack') + c = Client(timeout=55) + alice,bob,eve,chat = c.bootstrap() + data = png() + # Inherited lab4 obligations remain public: input validation and access control. + negatives=[] + for name,body,mime,filename,status in ( + ('oversized',b'x'*(10485760+1),'image/png','large.png',413), + ('damaged',data[:45],'image/png','broken.png',415), + ('pixel_budget',oversized_dimensions(),'image/png','huge.png',413), + ('mime_spoof',data,'image/jpeg','fake.jpg',415), + ('filename',data,'image/png','../evil.png',400), + ('unsupported',data,'application/octet-stream','file.png',415)): + upload(c,alice,chat,body,filename,mime,status) + negatives.append({'name':name,'status':status}) + att = upload(c,alice,chat,data)['json'] + variant_check(c,bob,att,data) + c.message(alice,chat,'',attachments=[att['id']]) + c.call('GET','/api/v1/attachments/'+att['id'],404,actor=eve) + c.call('GET','/api/v1/attachments/'+att['id']+'/content',401) + upload(c,eve,chat,data,expected=404) + RESULT['cases']['input_and_acl'] = negatives + # Real small image, alpha and EXIF orientation exercise the actual renderer. + from PIL import Image + for name,im,exif,shape in ( + ('small.png',Image.new('RGBA',(12,8),(255,0,0,0)),None,(12,8)), + ('oriented.jpg',Image.new('RGB',(320,200),(40,120,200)),6,(160,256)), + ('oriented.png',Image.new('RGB',(320,200),(40,120,200)),6,(160,256))): + out=io.BytesIO();kwargs={} + if exif: + tags=Image.Exif();tags[274]=exif;kwargs['exif']=tags + is_jpeg=name.endswith('.jpg') + im.save(out,format='JPEG' if is_jpeg else 'PNG',**kwargs) + body=out.getvalue();item=upload(c,alice,chat,body,name,'image/jpeg' if is_jpeg else 'image/png')['json'] + variant_check(c,bob,item,body,shape) + if name=='small.png': + raw=c.call('GET','/api/v1/attachments/'+item['id']+'/content?variant=thumbnail',actor=bob)['bytes'] + pixel=Image.open(io.BytesIO(raw)).getpixel((0,0));check(min(pixel)>250,'alpha was not white') + RESULT['cases']['small_alpha_exif']='PASS' + print('PASS: input/ACL, three actual formats, geometry, EXIF, alpha',flush=True) + with Probe() as probe: + key=sql("SELECT object_key FROM attachments WHERE id='"+att['id']+"';") + anonymous=docker('exec',containers('minio')[0],'wget','-S','-O','/dev/null','http://127.0.0.1:9000/attachments/'+key,check_result=False) + check(b'403' in anonymous.stderr,'bucket public') + probe.mc('pipe','local/attachments/outside-prefix',data=b'x',expected=1) + RESULT['cases']['private_s3']={'anonymous_get':403,'outside_prefix_denied':True} + + # A stage PUT without metadata COMMIT must never produce a 202 or a job. + until(lambda:sql("SELECT count(*) FROM attachments WHERE status='ready' AND staging_key IS NOT NULL;")=='0',timeout=45) + def stage_keys(): + lines=probe.mc('ls','--recursive','--json','local/attachments/staging/').stdout.decode().splitlines() + return {json.loads(line)['key'] for line in lines if json.loads(line).get('type')=='file'} + old=stage_keys() + filename='db-fault-'+uuid.uuid4().hex+'.png' + sql("CREATE FUNCTION lab5_reject_metadata() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.filename='"+filename+"' THEN RAISE EXCEPTION 'synthetic metadata failure'; END IF; RETURN NEW; END; $$; CREATE TRIGGER lab5_reject_metadata BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION lab5_reject_metadata();") + try: + upload(c,alice,chat,data,filename=filename,expected=503) + finally: + sql('DROP TRIGGER lab5_reject_metadata ON attachments; DROP FUNCTION lab5_reject_metadata();') + unconfirmed=stage_keys()-old + check(len(unconfirmed)==1,'expected one unconfirmed stage') + check(sql("SELECT count(*) FROM attachments WHERE filename='"+filename+"';")=='0','failed upload published metadata') + orphan=next(iter(unconfirmed)) + time.sleep(6) + check(orphan in stage_keys(),'fresh unconfirmed stage deleted before grace') + RESULT['cases']['stage_before_commit']={'upload_status':503,'metadata':0,'fresh_orphan_preserved':True} + + if '--tail-only' not in sys.argv: + # COMMIT without publish, then API and broker restart with worker absent. + control('worker',replicas=0) + control('api',{'PIPELINE_PAUSE_PUBLISH':'1'}) + item=upload(c,alice,chat,data)['json'] + c.message(alice,chat,'',attachments=[item['id']]) + c.call('GET','/api/v1/attachments/'+item['id']+'/content',409,actor=bob) + j=job(item) + check(sql("SELECT published_at IS NULL FROM attachment_outbox WHERE job_id='"+j['id']+"';")=='t','publish not paused') + control('api',force=True) + control('redis',force=True) + settled('api',2) + control('api',{'PIPELINE_PAUSE_PUBLISH':None}) + until(lambda:sql("SELECT published_at IS NOT NULL FROM attachment_outbox WHERE job_id='"+j['id']+"';")=='t') + before=redis('XRANGE',STREAM,'-','+') + check(any(json.loads(row[1][1])['job_id']==j['id'] for row in before),'job not in stream') + control('redis',force=True) + settled('api',2) + check(any(json.loads(row[1][1])['job_id']==j['id'] for row in redis('XRANGE',STREAM,'-','+')),'broker lost durable job') + control('worker',replicas=1) + variant_check(c,bob,item,data) + RESULT['cases']['commit_publish_and_restarts']={'job_id':j['id'],'api_restart':True,'broker_restarts':2,'link_queued':True} + save();print('PASS: COMMIT/publish gap, queued link, API/broker restarts, worker recovery',flush=True) + + # A killed worker leaves deterministic objects before metadata COMMIT/ack. + control('worker',{'WORKER_TIMEOUT_SECONDS':'5','WORKER_LEASE_SECONDS':'12','PIPELINE_FAILPOINT':'after_objects'}) + item=upload(c,alice,chat,data)['json'] + until(lambda:job(item)['state']=='processing') + until(lambda:probe.mc('stat','local/attachments/original/'+item['id']+'/v1/optimized').returncode==0) + control('worker',{'PIPELINE_FAILPOINT':None}) + variant_check(c,bob,item,data) + j=job(item);check(j['attempts']>=2,'expired processing was not retried') + objects=probe.mc('ls','--recursive','--json','local/attachments/original/'+item['id']+'/').stdout.decode().splitlines() + check(sum(json.loads(x).get('type')=='file' for x in objects)==4,'duplicate objects created') + RESULT['cases']['crash_after_objects']={'attempts':j['attempts'],'objects':4,'fence':j['fence']} + save();print('PASS: crash after objects, expired lease/fencing, same four keys',flush=True) + + # Ready COMMIT before ack: redelivery must leave metadata unchanged. + control('worker',{'PIPELINE_FAILPOINT':'after_commit'}) + item=upload(c,alice,chat,data)['json'];wait_state(item,'ready') + snapshot=c.call('GET','/api/v1/attachments/'+item['id'],actor=bob,schema='Attachment')['json'] + check(redis('XPENDING',STREAM,GROUP)[0]>0,'ack happened before crash') + control('worker',{'PIPELINE_FAILPOINT':None}) + until(lambda:redis('XPENDING',STREAM,GROUP)[0]==0,timeout=30) + check(c.call('GET','/api/v1/attachments/'+item['id'],actor=bob,schema='Attachment')['json']==snapshot,'ready replay changed metadata') + j=job(item) + envelope={'schema_version':1,'job_id':j['id'],'type':'attachment.process.v1','attachment_id':item['id'],'pipeline_version':1,'attempt':1,'created_at':j['created_at']} + redis('XADD',STREAM,'*','job',json.dumps(envelope)) + until(lambda:redis('XLEN',STREAM)==0,timeout=30) + check(c.call('GET','/api/v1/attachments/'+item['id'],actor=bob,schema='Attachment')['json']==snapshot,'duplicate changed ready') + replay(j,False) + RESULT['cases']['commit_before_ack_and_duplicate']={'ready_unchanged':True,'ready_replay_rejected':True} + save();print('PASS: COMMIT before ack, duplicate no-op and ready replay rejection',flush=True) + + # Temporary storage error, bounded attempts/backoff, failed registry and audit replay. + control('api',{'WORKER_MAX_ATTEMPTS':'2'}) + control('worker',{'PIPELINE_FAILPOINT':'transient'}) + item=upload(c,alice,chat,data)['json'] + retry=until(lambda:job(item),lambda j:j['state']=='queued' and j['attempts']==1) + delay=(dt.datetime.fromisoformat(retry['available_at'].replace('Z','+00:00'))-dt.datetime.fromisoformat(retry['updated_at'].replace('Z','+00:00'))).total_seconds() + check(1.5<=delay<=2.6,'retry backoff/jitter outside configured range') + failed=wait_state(item,'failed',timeout=30) + check(failed['attempts']==2,'retry limit') + meta=c.call('GET','/api/v1/attachments/'+item['id'],actor=bob,schema='Attachment')['json'] + check(meta['error_code']=='retry_exhausted','unsafe final code') + c.call('GET','/api/v1/attachments/'+item['id']+'/content',409,actor=bob) + c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,body={'text':'','attachment_ids':[item['id']]},headers={'Idempotency-Key':uuid.uuid4().hex}) + registry=json.loads(docker('exec',containers('worker')[0],'/usr/local/bin/worker','--failed').stdout) + check(any(row['job_id']==failed['id'] for row in registry),'failed registry missing') + control('worker',{'PIPELINE_FAILPOINT':None}) + replay(failed) + variant_check(c,bob,item,data) + check(sql("SELECT count(*) FROM attachment_replays WHERE job_id='"+failed['id']+"';")=='1','replay audit missing') + RESULT['cases']['bounded_retry_replay']={'max_attempts':2,'retry_available_at':retry['available_at'],'first_backoff_seconds':round(delay,3),'failed_attempts':failed['attempts'],'audits':1} + save();print('PASS: bounded retry/backoff, failed registry, safe audited replay',flush=True) + + # A permanently corrupted durable stage is not retried endlessly. + control('worker',replicas=0) + item=upload(c,alice,chat,data)['json'] + probe.mc('pipe','local/attachments/staging/'+item['id'],data=b'damaged synthetic stage') + control('worker',replicas=1) + failed=wait_state(item,'failed') + check(failed['attempts']==1,'permanent error retried') + check(c.call('GET','/api/v1/attachments/'+item['id'],actor=bob,schema='Attachment')['json']['error_code']=='invalid_image','permanent classification') + RESULT['cases']['permanent_stage_error']={'attempts':1,'error_code':'invalid_image'} + permanent_att=item + permanent_job=failed + + # Deadline kills the renderer AND its real cwebp child blocked on a FIFO. + control('api',{'WORKER_MAX_ATTEMPTS':'1'}) + control('worker',{'WORKER_TIMEOUT_SECONDS':'2','WORKER_LEASE_SECONDS':'7','PIPELINE_FAILPOINT':'renderer_timeout'}) + item=upload(c,alice,chat,data)['json'];started=time.monotonic() + seen=until(lambda:docker('exec',containers('worker')[0],'ps','-o','pid,args').stdout.decode(),lambda text:'cwebp -quiet' in text,timeout=10) + wait_state(item,'failed',timeout=15) + elapsed=time.monotonic()-started + processes=docker('exec',containers('worker')[0],'ps','-o','pid,args').stdout.decode() + check('cwebp -quiet' not in processes and '/worker render' not in processes,'renderer descendant survived timeout') + RESULT['cases']['renderer_process_group_timeout']={'deadline_seconds':2,'elapsed_seconds':round(elapsed,3),'cwebp_observed':True,'descendants_after_failure':0} + control('worker',{'PIPELINE_FAILPOINT':None,'WORKER_TIMEOUT_SECONDS':'25','WORKER_LEASE_SECONDS':'40'}) + control('api',{'WORKER_MAX_ATTEMPTS':None}) + # Every ready stage is eventually collected; linked final objects remain. + until(lambda:sql("SELECT count(*) FROM attachments WHERE status='ready' AND staging_key IS NOT NULL;")=='0',timeout=45) + check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==data,'cleanup removed ready original') + sql("UPDATE attachments SET created_at=created_at-interval '8 days',updated_at=clock_timestamp()-interval '8 days' WHERE id='"+permanent_att['id']+"';") + until(lambda:sql("SELECT storage_cleaned_at IS NOT NULL AND staging_key IS NULL FROM attachments WHERE id='"+permanent_att['id']+"';")=='t',timeout=45) + replay(permanent_job,False) + until(lambda:orphan not in stage_keys(),timeout=100) + RESULT['cases']['staging_cleanup']={'ready_stages_remaining':0,'linked_original_preserved':True,'failed_retention_days':7,'expired_failed_stage_removed':True,'expired_replay_rejected':True,'unconfirmed_orphan_removed':True} + required={'input_and_acl','small_alpha_exif','private_s3','stage_before_commit','commit_publish_and_restarts','crash_after_objects','commit_before_ack_and_duplicate','bounded_retry_replay','permanent_stage_error','renderer_process_group_timeout','staging_cleanup'} + check(required.issubset(RESULT['cases']),'incomplete fault evidence') + RESULT['finished_at']=time.time();save() + print('PASS: all lab5 fault, retry, replay, durability, rendering and cleanup checks',flush=True) + + +if __name__=='__main__': + try: + main() + finally: + # Restore ordinary service configuration even if an assertion fails. + try: + control('worker',{'PIPELINE_FAILPOINT':None,'WORKER_TIMEOUT_SECONDS':'25','WORKER_LEASE_SECONDS':'40'},replicas=1) + control('api',{'WORKER_MAX_ATTEMPTS':None,'PIPELINE_PAUSE_PUBLISH':None}) + finally: + save() diff --git a/tests/own/swarm_grade4.py b/tests/own/swarm_grade4.py index fd37a5bd..5b9c5bd7 100644 --- a/tests/own/swarm_grade4.py +++ b/tests/own/swarm_grade4.py @@ -15,7 +15,7 @@ sys.path.insert(0, str(ROOT/'tests')) from client import Client, Failure, check, run from resilience import history -STACK = os.environ.get('STACK_NAME', 'mtusi-lab4') +STACK = os.environ.get('STACK_NAME', 'mtusi-lab5') def docker_json(*args): -- 2.54.0