diff --git a/.env.example b/.env.example index 4cfb028..0e85d04 100644 --- a/.env.example +++ b/.env.example @@ -156,3 +156,4 @@ STAGING_FAILED_RETENTION=168h ADMISSION_MAX_PENDING_PER_USER=20 ADMISSION_MAX_PENDING=200 ADMISSION_MAX_STAGED_BYTES=2147483648 +ADMISSION_MAX_CONCURRENT_UPLOADS=8 diff --git a/REPORT.md b/REPORT.md index ebaf336..4490a98 100644 --- a/REPORT.md +++ b/REPORT.md @@ -90,7 +90,8 @@ flowchart LR - Admission control ограничивает число незавершённых rows (включая `pending` presign) на пользователя/стенд и объём staged bytes (429 `rate_limited`). Проверка повторяется внутри транзакции под PostgreSQL advisory lock, поэтому параллельные запросы не обходят cap; presign - использует тот же механизм. + использует тот же механизм. Число одновременных upload-spool на процесс ограничено + `ADMISSION_MAX_CONCURRENT_UPLOADS`, что ограничивает и временный диск API. ### Соответствие `.env.example` реальной конфигурации diff --git a/cmd/messenger/main.go b/cmd/messenger/main.go index 0d57988..45c46e9 100644 --- a/cmd/messenger/main.go +++ b/cmd/messenger/main.go @@ -87,6 +87,7 @@ func run(cfg config.Config, log *slog.Logger) error { MaxPendingPerUser: cfg.Admission.MaxPendingPerUser, MaxPending: cfg.Admission.MaxPending, MaxStagedBytes: cfg.Admission.MaxStagedBytes, + MaxConcurrent: cfg.Admission.MaxConcurrent, }, log, ) diff --git a/compose.yaml b/compose.yaml index ef5d33b..03075f3 100644 --- a/compose.yaml +++ b/compose.yaml @@ -226,6 +226,7 @@ services: ADMISSION_MAX_PENDING_PER_USER: ${ADMISSION_MAX_PENDING_PER_USER:-20} ADMISSION_MAX_PENDING: ${ADMISSION_MAX_PENDING:-200} ADMISSION_MAX_STAGED_BYTES: ${ADMISSION_MAX_STAGED_BYTES:-2147483648} + ADMISSION_MAX_CONCURRENT_UPLOADS: ${ADMISSION_MAX_CONCURRENT_UPLOADS:-8} # Размер пула соединений ОДНОЙ реплики; общий расход = реплики × значение. # Это один из общих ресурсов, который масштабирование расходует первым. POSTGRES_MAX_CONNS: "${POSTGRES_MAX_CONNS:-10}" diff --git a/docker-compose.local.yaml b/docker-compose.local.yaml index ed7ee52..ec17c1a 100644 --- a/docker-compose.local.yaml +++ b/docker-compose.local.yaml @@ -161,6 +161,7 @@ services: ADMISSION_MAX_PENDING_PER_USER: ${ADMISSION_MAX_PENDING_PER_USER:-20} ADMISSION_MAX_PENDING: ${ADMISSION_MAX_PENDING:-200} ADMISSION_MAX_STAGED_BYTES: ${ADMISSION_MAX_STAGED_BYTES:-2147483648} + ADMISSION_MAX_CONCURRENT_UPLOADS: ${ADMISSION_MAX_CONCURRENT_UPLOADS:-8} ports: - "127.0.0.1:${SERVER_PORT:-8080}:8080" depends_on: diff --git a/evidence/lab5/async.txt b/evidence/lab5/async.txt index 82cd79e..fee82a7 100644 --- a/evidence/lab5/async.txt +++ b/evidence/lab5/async.txt @@ -1,4 +1,4 @@ -# lab5_async 2026-10-30T20:08:22Z +# lab5_async 2026-10-30T20:15:54Z worker down: 202 queued, content 409, queued attachment linked, API restarted worker restarted: ready, thumbnail and original downloadable, staging cleared broker down: upload accepted into outbox diff --git a/evidence/lab5/db-state.txt b/evidence/lab5/db-state.txt index 68827e4..c1126aa 100644 --- a/evidence/lab5/db-state.txt +++ b/evidence/lab5/db-state.txt @@ -1,29 +1,29 @@ -# db state 2026-10-30T20:13:30Z +# db state 2026-10-30T20:16:08Z status | count ---------+------- failed | 2 pending | 4 - ready | 25 + ready | 31 (3 rows) status | count --------+------- failed | 2 - ready | 28 + ready | 34 (2 rows) variants ---------- - 84 + 102 (1 row) unpublished | outbox -------------+-------- - 0 | 52 + 0 | 58 (1 row) Timeout: 60.0 seconds ... Listing queues for vhost messenger ... name messages messages_ready messages_unacknowledged consumers attachment.process.dead 4 4 0 0 -attachment.process.v1 0 0 0 1 +attachment.process.v1 0 0 0 0 diff --git a/evidence/lab5/lint.txt b/evidence/lab5/lint.txt index c719053..5341a79 100644 --- a/evidence/lab5/lint.txt +++ b/evidence/lab5/lint.txt @@ -3,18 +3,18 @@ level=info msg="golangci-lint has version 2.13.2 built with go1.27.1 from (unkno level=info msg="[config_reader] Config search paths: [./ /Users/vladislav/Projects/mtusi2026lab5 /Users/vladislav/Projects /Users/vladislav /Users /]" level=info msg="[config_reader] Used config file .golangci.yml" level=info msg="[config_reader] Module name \"mtusi-messenger\"" -level=info msg="[goenv] Read go env for 5.643458ms: map[string]string{\"GOCACHE\":\"/Users/vladislav/Library/Caches/go-build\", \"GOROOT\":\"/opt/homebrew/Cellar/go/1.27.1/libexec\"}" +level=info msg="[goenv] Read go env for 6.899ms: map[string]string{\"GOCACHE\":\"/Users/vladislav/Library/Caches/go-build\", \"GOROOT\":\"/opt/homebrew/Cellar/go/1.27.1/libexec\"}" level=info msg="[lintersdb] Active 50 linters: [asasalint bodyclose canonicalheader containedctx contextcheck copyloopvar depguard durationcheck errcheck errname errorlint exhaustive exptostd fatcontext forbidigo forcetypeassert gocheckcompilerdirectives gocritic gofumpt goimports gomoddirectives gosec govet ineffassign intrange makezero misspell musttag nilerr nilnil noctx nolintlint perfsprint prealloc reassign recvcheck revive rowserrcheck sloglint sqlclosecheck staticcheck testifylint thelper tparallel unconvert unparam unused usestdlibvars usetesting wastedassign]" level=info msg="[loader] Using build tags: [integration]" -level=info msg="[loader] Go packages loading at mode 8767 (files|imports|name|compiled_files|deps|exports_file|types_sizes) took 210.149917ms" -level=info msg="[runner/filename_unadjuster] Pre-built 0 adjustments in 11.759541ms" -level=info msg="[linters_context/goanalysis] analyzers took 0s with no stages" +level=info msg="[loader] Go packages loading at mode 8767 (files|name|compiled_files|deps|exports_file|imports|types_sizes) took 386.58525ms" +level=info msg="[runner/filename_unadjuster] Pre-built 0 adjustments in 10.82225ms" +level=info msg="[linters_context/goanalysis] analyzers took 9.441069744s with top 10 stages: goimports: 2.162295625s, revive: 1.635175292s, gosec: 746.543207ms, unparam: 497.231375ms, wastedassign: 307.470709ms, gocritic: 147.325123ms, buildir: 145.809195ms, gofumpt: 123.647625ms, buildssa: 118.610962ms, nilness: 115.823328ms" level=info msg="[runner/exclusion_rules] Skipped 8 issues by rules: [Path: \"_test\\\\.go\", Linters: \"gosec, prealloc, forcetypeassert, unparam\"]" level=info msg="[runner] Issues before processing: 881, after processing: 0" -level=info msg="[runner] Processors filtering stat (in/out): exclusion_paths: 881/881, nolint_filter: 6/0, path_absoluter: 881/881, invalid_issue: 881/881, generated_file_filter: 881/292, cgo: 881/881, filename_unadjuster: 881/881, path_relativity: 881/881, exclusion_rules: 292/6" -level=info msg="[runner] processing took 2.916079ms with stages: generated_file_filter: 1.448166ms, nolint_filter: 891.875µs, exclusion_rules: 277.041µs, path_relativity: 263.459µs, invalid_issue: 13.082µs, cgo: 11.375µs, path_absoluter: 4.792µs, filename_unadjuster: 2.875µs, max_same_issues: 958ns, sort_results: 750ns, exclusion_paths: 500ns, fixer: 458ns, path_shortener: 167ns, max_per_file_from_linter: 125ns, source_code: 125ns, path_prettifier: 125ns, diff: 83ns, max_from_linter: 41ns, severity-rules: 41ns, uniq_by_line: 41ns" -level=info msg="[runner] linters took 627.484792ms with stages: goanalysis_metalinter: 624.525167ms" +level=info msg="[runner] Processors filtering stat (in/out): cgo: 881/881, exclusion_paths: 881/881, generated_file_filter: 881/292, exclusion_rules: 292/6, path_absoluter: 881/881, invalid_issue: 881/881, filename_unadjuster: 881/881, path_relativity: 881/881, nolint_filter: 6/0" +level=info msg="[runner] processing took 4.266378ms with stages: generated_file_filter: 2.538917ms, nolint_filter: 1.042041ms, exclusion_rules: 367.959µs, path_relativity: 277.792µs, invalid_issue: 13.084µs, cgo: 11.75µs, path_absoluter: 6.125µs, sort_results: 3.792µs, filename_unadjuster: 2.709µs, max_same_issues: 625ns, exclusion_paths: 458ns, fixer: 376ns, path_shortener: 126ns, path_prettifier: 125ns, max_from_linter: 125ns, max_per_file_from_linter: 125ns, uniq_by_line: 84ns, diff: 83ns, severity-rules: 41ns, source_code: 41ns" +level=info msg="[runner] linters took 2.049702792s with stages: goanalysis_metalinter: 2.045391875s" 0 issues. level=info msg="File cache stats: 0 entries of total size 0B" -level=info msg="Memory: 10 samples, avg is 84.0MB, max is 111.1MB" -level=info msg="Execution took 856.003459ms" +level=info msg="Memory: 26 samples, avg is 277.9MB, max is 547.6MB" +level=info msg="Execution took 2.454527125s" diff --git a/evidence/lab5/smoke.txt b/evidence/lab5/smoke.txt index 08f22e9..1b737ec 100644 --- a/evidence/lab5/smoke.txt +++ b/evidence/lab5/smoke.txt @@ -1,3 +1,3 @@ -# make test 2026-10-30T20:08:03Z +# make test 2026-10-30T20:15:52Z env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy NO_PROXY=localhost,127.0.0.1,::1 no_proxy=localhost,127.0.0.1,::1 python3 tests/smoke.py --base-url "http://localhost:8080" PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/lab5/stand.txt b/evidence/lab5/stand.txt index b83aa16..f6eb749 100644 --- a/evidence/lab5/stand.txt +++ b/evidence/lab5/stand.txt @@ -1,16 +1,16 @@ -# stand 2026-10-30T20:13:29Z -NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS -mtusi-messenger-lab5-api-1 mtusi-messenger:lab5 "/usr/local/bin/mess…" api 6 minutes ago Up 46 seconds (healthy) 8080/tcp -mtusi-messenger-lab5-api-2 mtusi-messenger:lab5 "/usr/local/bin/mess…" api 6 minutes ago Up 46 seconds (healthy) 8080/tcp -mtusi-messenger-lab5-broker-1 rabbitmq:4.3.7-management-alpine@sha256:1528a88eebf2bc4d0ab7e9d5d9bc542e9bdf5e1b105e65848fcf8de5c2186528 "docker-entrypoint.s…" broker 6 minutes ago Up About a minute (healthy) 4369/tcp, 5671-5672/tcp, 15671/tcp, 15691-15692/tcp, 25672/tcp, 127.0.0.1:15672->15672/tcp -mtusi-messenger-lab5-lb-1 haproxy:3.2-alpine@sha256:254e6370f01cd654e0744edf607c67f33777c1b6a9dc0d4b28bcba7171460122 "docker-entrypoint.s…" lb 6 minutes ago Up 40 seconds 127.0.0.1:8404->8404/tcp, 127.0.0.1:8080->80/tcp -mtusi-messenger-lab5-postgres-1 postgres:18-alpine@sha256:77f585114c32fbca283dc835b0596f4e52b51b4c6662d7810b2f4084f60a1873 "docker-entrypoint.s…" postgres 6 minutes ago Up About a minute (healthy) 5432/tcp -mtusi-messenger-lab5-redis-1 redis:7.4-alpine@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499 "docker-entrypoint.s…" redis 6 minutes ago Up About a minute (healthy) 6379/tcp -mtusi-messenger-lab5-relay-1 mtusi-messenger:lab5 "/usr/local/bin/relay" relay About a minute ago Up 46 seconds (healthy) 8080/tcp, 127.0.0.1:8082->8082/tcp -mtusi-messenger-lab5-seaweedfs-1 chrislusf/seaweedfs:3.80@sha256:1055999e08eed1789b0ae45d235126e4495e23d3fb9d6396293fd42539b1ae6a "/entrypoint.sh serv…" seaweedfs 6 minutes ago Up About a minute (healthy) 7333/tcp, 8080/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp, 127.0.0.1:8333->8333/tcp -mtusi-messenger-lab5-worker-1 mtusi-messenger:lab5 "/usr/local/bin/work…" worker About a minute ago Up 46 seconds (healthy) 8080/tcp, 127.0.0.1:8081->8081/tcp +# stand 2026-10-30T20:16:07Z +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi-messenger-lab5-api-1 mtusi-messenger:lab5 "/usr/local/bin/mess…" api 42 seconds ago Up Less than a second (health: starting) 8080/tcp +mtusi-messenger-lab5-api-2 mtusi-messenger:lab5 "/usr/local/bin/mess…" api 42 seconds ago Up Less than a second (health: starting) 8080/tcp +mtusi-messenger-lab5-broker-1 rabbitmq:4.3.7-management-alpine@sha256:1528a88eebf2bc4d0ab7e9d5d9bc542e9bdf5e1b105e65848fcf8de5c2186528 "docker-entrypoint.s…" broker 8 minutes ago Up 8 seconds (healthy) 4369/tcp, 5671-5672/tcp, 15671/tcp, 15691-15692/tcp, 25672/tcp, 127.0.0.1:15672->15672/tcp +mtusi-messenger-lab5-lb-1 haproxy:3.2-alpine@sha256:254e6370f01cd654e0744edf607c67f33777c1b6a9dc0d4b28bcba7171460122 "docker-entrypoint.s…" lb 8 minutes ago Up 3 minutes 127.0.0.1:8404->8404/tcp, 127.0.0.1:8080->80/tcp +mtusi-messenger-lab5-postgres-1 postgres:18-alpine@sha256:77f585114c32fbca283dc835b0596f4e52b51b4c6662d7810b2f4084f60a1873 "docker-entrypoint.s…" postgres 8 minutes ago Up 3 minutes (healthy) 5432/tcp +mtusi-messenger-lab5-redis-1 redis:7.4-alpine@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499 "docker-entrypoint.s…" redis 8 minutes ago Up 4 minutes (healthy) 6379/tcp +mtusi-messenger-lab5-relay-1 mtusi-messenger:lab5 "/usr/local/bin/relay" relay 42 seconds ago Up Less than a second (health: starting) 8080/tcp, 127.0.0.1:8082->8082/tcp +mtusi-messenger-lab5-seaweedfs-1 chrislusf/seaweedfs:3.80@sha256:1055999e08eed1789b0ae45d235126e4495e23d3fb9d6396293fd42539b1ae6a "/entrypoint.sh serv…" seaweedfs 8 minutes ago Up 3 minutes (healthy) 7333/tcp, 8080/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp, 127.0.0.1:8333->8333/tcp +mtusi-messenger-lab5-worker-1 mtusi-messenger:lab5 "/usr/local/bin/work…" worker 42 seconds ago Up Less than a second (health: starting) 8080/tcp, 127.0.0.1:8081->8081/tcp -mtusi-messenger:lab5 a637d8b7d2d2 +mtusi-messenger:lab5 10b83ff52c7d rabbitmq:4.3.7-management-alpine 1528a88eebf2 haproxy:3.2-alpine 254e6370f01c redis:7.4-alpine 858f009f9709 diff --git a/evidence/lab5/unit-test.txt b/evidence/lab5/unit-test.txt index 937909f..bb08778 100644 --- a/evidence/lab5/unit-test.txt +++ b/evidence/lab5/unit-test.txt @@ -6,35 +6,35 @@ env -u APP_ENV -u SERVER_HOST -u SERVER_PORT -u SERVER_READ_HEADER_TIMEOUT -u SE ? mtusi-messenger/cmd/s3init [no test files] ? mtusi-messenger/cmd/worker [no test files] ? mtusi-messenger/internal/broker [no test files] -ok mtusi-messenger/internal/config 2.441s -ok mtusi-messenger/internal/domain 1.484s +ok mtusi-messenger/internal/config 1.385s +ok mtusi-messenger/internal/domain 2.358s ? mtusi-messenger/internal/failpoint [no test files] -ok mtusi-messenger/internal/health 2.069s -ok mtusi-messenger/internal/imagecheck 1.809s -ok mtusi-messenger/internal/imageproc 3.094s -ok mtusi-messenger/internal/logger 2.975s +ok mtusi-messenger/internal/health 2.187s +ok mtusi-messenger/internal/imagecheck 2.045s +ok mtusi-messenger/internal/imageproc 3.557s +ok mtusi-messenger/internal/logger 2.942s ? mtusi-messenger/internal/outbox [no test files] -ok mtusi-messenger/internal/platform/cache 3.522s +ok mtusi-messenger/internal/platform/cache 2.970s ? mtusi-messenger/internal/platform/cache/mocks [no test files] -ok mtusi-messenger/internal/platform/objectstore 5.364s -ok mtusi-messenger/internal/platform/password 4.551s -ok mtusi-messenger/internal/platform/token 4.459s -ok mtusi-messenger/internal/service/attachment 5.686s +ok mtusi-messenger/internal/platform/objectstore 3.292s +ok mtusi-messenger/internal/platform/password 4.017s +ok mtusi-messenger/internal/platform/token 2.535s +ok mtusi-messenger/internal/service/attachment 1.665s ? mtusi-messenger/internal/service/attachment/mocks [no test files] -ok mtusi-messenger/internal/service/chat 3.610s +ok mtusi-messenger/internal/service/chat 3.495s ? mtusi-messenger/internal/service/chat/mocks [no test files] -ok mtusi-messenger/internal/service/message 3.931s +ok mtusi-messenger/internal/service/message 1.866s ? mtusi-messenger/internal/service/message/mocks [no test files] -ok mtusi-messenger/internal/service/session 5.387s +ok mtusi-messenger/internal/service/session 4.403s ? mtusi-messenger/internal/service/session/mocks [no test files] -ok mtusi-messenger/internal/service/user 4.965s +ok mtusi-messenger/internal/service/user 4.032s ? mtusi-messenger/internal/service/user/mocks [no test files] -ok mtusi-messenger/internal/storage/postgres 5.965s +ok mtusi-messenger/internal/storage/postgres 4.123s ? mtusi-messenger/internal/storage/postgres/internal/sqlcgen [no test files] ? mtusi-messenger/internal/storage/postgres/migrations [no test files] ? mtusi-messenger/internal/storage/redis [no test files] ? mtusi-messenger/internal/storage/redis/mocks [no test files] -ok mtusi-messenger/internal/transport/rest 6.275s +ok mtusi-messenger/internal/transport/rest 4.446s ? mtusi-messenger/internal/transport/rest/internal/oas [no test files] ? mtusi-messenger/internal/transport/rest/mocks [no test files] ? mtusi-messenger/internal/worker [no test files] diff --git a/internal/config/config.go b/internal/config/config.go index cf1cb55..a1c7a0d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -105,6 +105,7 @@ type Admission struct { MaxPendingPerUser int `mapstructure:"max_pending_per_user" validate:"required,gte=0"` MaxPending int `mapstructure:"max_pending" validate:"required,gte=0"` MaxStagedBytes int64 `mapstructure:"max_staged_bytes" validate:"required,gte=0"` + MaxConcurrent int `mapstructure:"max_concurrent_uploads" validate:"omitempty,gt=0"` } // S3 describes the object store and attachment limits. Access keys are checked diff --git a/internal/service/attachment/service.go b/internal/service/attachment/service.go index 96f9cea..a4d39b4 100644 --- a/internal/service/attachment/service.go +++ b/internal/service/attachment/service.go @@ -17,6 +17,7 @@ import ( "unicode/utf8" "github.com/google/uuid" + "golang.org/x/sync/semaphore" "mtusi-messenger/internal/domain" "mtusi-messenger/internal/imagecheck" @@ -81,6 +82,7 @@ type Config struct { MaxPendingPerUser int MaxPending int MaxStagedBytes int64 + MaxConcurrent int } type Service struct { @@ -88,6 +90,7 @@ type Service struct { store objectstore.Store config Config log *slog.Logger + upload *semaphore.Weighted } func NewService(repo Repository, store objectstore.Store, config Config, log *slog.Logger) *Service { @@ -95,7 +98,12 @@ func NewService(repo Repository, store objectstore.Store, config Config, log *sl log = slog.Default() } - return &Service{repo: repo, store: store, config: config, log: log} + service := &Service{repo: repo, store: store, config: config, log: log} + if config.MaxConcurrent > 0 { + service.upload = semaphore.NewWeighted(int64(config.MaxConcurrent)) + } + + return service } // UploadInput is one raw-body upload. DeclaredContentType comes from the HTTP @@ -111,6 +119,13 @@ type UploadInput struct { // Upload durably stages the original, records the job intent and returns the // queued snapshot. It never waits for the broker or the worker. func (s *Service) Upload(ctx context.Context, input UploadInput) (domain.Attachment, error) { + if s.upload != nil { + if err := s.upload.Acquire(ctx, 1); err != nil { + return domain.Attachment{}, unavailable("acquire upload slot", err) + } + defer s.upload.Release(1) + } + if err := ValidateFilename(input.Filename); err != nil { return domain.Attachment{}, err }