lab5: ограничение одновременных upload-spool и evidence после ревью

Process-level semaphore bounds temporary disk use on the upload path; refresh raw evidence after the review fixes.
This commit is contained in:
Vladislav Kunetskiy
2026-10-30 23:16:12 +03:00
parent 9d7a0acb5a
commit e40a5cf215
13 changed files with 69 additions and 48 deletions
+1
View File
@@ -156,3 +156,4 @@ STAGING_FAILED_RETENTION=168h
ADMISSION_MAX_PENDING_PER_USER=20
ADMISSION_MAX_PENDING=200
ADMISSION_MAX_STAGED_BYTES=2147483648
ADMISSION_MAX_CONCURRENT_UPLOADS=8
+2 -1
View File
@@ -90,7 +90,8 @@ flowchart LR
- Admission control ограничивает число незавершённых rows (включая `pending` presign) на
пользователя/стенд и объём staged bytes (429 `rate_limited`). Проверка повторяется внутри
транзакции под PostgreSQL advisory lock, поэтому параллельные запросы не обходят cap; presign
использует тот же механизм.
использует тот же механизм. Число одновременных upload-spool на процесс ограничено
`ADMISSION_MAX_CONCURRENT_UPLOADS`, что ограничивает и временный диск API.
### Соответствие `.env.example` реальной конфигурации
+1
View File
@@ -87,6 +87,7 @@ func run(cfg config.Config, log *slog.Logger) error {
MaxPendingPerUser: cfg.Admission.MaxPendingPerUser,
MaxPending: cfg.Admission.MaxPending,
MaxStagedBytes: cfg.Admission.MaxStagedBytes,
MaxConcurrent: cfg.Admission.MaxConcurrent,
},
log,
)
+1
View File
@@ -226,6 +226,7 @@ services:
ADMISSION_MAX_PENDING_PER_USER: ${ADMISSION_MAX_PENDING_PER_USER:-20}
ADMISSION_MAX_PENDING: ${ADMISSION_MAX_PENDING:-200}
ADMISSION_MAX_STAGED_BYTES: ${ADMISSION_MAX_STAGED_BYTES:-2147483648}
ADMISSION_MAX_CONCURRENT_UPLOADS: ${ADMISSION_MAX_CONCURRENT_UPLOADS:-8}
# Размер пула соединений ОДНОЙ реплики; общий расход = реплики × значение.
# Это один из общих ресурсов, который масштабирование расходует первым.
POSTGRES_MAX_CONNS: "${POSTGRES_MAX_CONNS:-10}"
+1
View File
@@ -161,6 +161,7 @@ services:
ADMISSION_MAX_PENDING_PER_USER: ${ADMISSION_MAX_PENDING_PER_USER:-20}
ADMISSION_MAX_PENDING: ${ADMISSION_MAX_PENDING:-200}
ADMISSION_MAX_STAGED_BYTES: ${ADMISSION_MAX_STAGED_BYTES:-2147483648}
ADMISSION_MAX_CONCURRENT_UPLOADS: ${ADMISSION_MAX_CONCURRENT_UPLOADS:-8}
ports:
- "127.0.0.1:${SERVER_PORT:-8080}:8080"
depends_on:
+1 -1
View File
@@ -1,4 +1,4 @@
# lab5_async 2026-10-30T20:08:22Z
# lab5_async 2026-10-30T20:15:54Z
worker down: 202 queued, content 409, queued attachment linked, API restarted
worker restarted: ready, thumbnail and original downloadable, staging cleared
broker down: upload accepted into outbox
+6 -6
View File
@@ -1,29 +1,29 @@
# db state 2026-10-30T20:13:30Z
# db state 2026-10-30T20:16:08Z
status | count
---------+-------
failed | 2
pending | 4
ready | 25
ready | 31
(3 rows)
status | count
--------+-------
failed | 2
ready | 28
ready | 34
(2 rows)
variants
----------
84
102
(1 row)
unpublished | outbox
-------------+--------
0 | 52
0 | 58
(1 row)
Timeout: 60.0 seconds ...
Listing queues for vhost messenger ...
name messages messages_ready messages_unacknowledged consumers
attachment.process.dead 4 4 0 0
attachment.process.v1 0 0 0 1
attachment.process.v1 0 0 0 0
+9 -9
View File
@@ -3,18 +3,18 @@ level=info msg="golangci-lint has version 2.13.2 built with go1.27.1 from (unkno
level=info msg="[config_reader] Config search paths: [./ /Users/vladislav/Projects/mtusi2026lab5 /Users/vladislav/Projects /Users/vladislav /Users /]"
level=info msg="[config_reader] Used config file .golangci.yml"
level=info msg="[config_reader] Module name \"mtusi-messenger\""
level=info msg="[goenv] Read go env for 5.643458ms: map[string]string{\"GOCACHE\":\"/Users/vladislav/Library/Caches/go-build\", \"GOROOT\":\"/opt/homebrew/Cellar/go/1.27.1/libexec\"}"
level=info msg="[goenv] Read go env for 6.899ms: map[string]string{\"GOCACHE\":\"/Users/vladislav/Library/Caches/go-build\", \"GOROOT\":\"/opt/homebrew/Cellar/go/1.27.1/libexec\"}"
level=info msg="[lintersdb] Active 50 linters: [asasalint bodyclose canonicalheader containedctx contextcheck copyloopvar depguard durationcheck errcheck errname errorlint exhaustive exptostd fatcontext forbidigo forcetypeassert gocheckcompilerdirectives gocritic gofumpt goimports gomoddirectives gosec govet ineffassign intrange makezero misspell musttag nilerr nilnil noctx nolintlint perfsprint prealloc reassign recvcheck revive rowserrcheck sloglint sqlclosecheck staticcheck testifylint thelper tparallel unconvert unparam unused usestdlibvars usetesting wastedassign]"
level=info msg="[loader] Using build tags: [integration]"
level=info msg="[loader] Go packages loading at mode 8767 (files|imports|name|compiled_files|deps|exports_file|types_sizes) took 210.149917ms"
level=info msg="[runner/filename_unadjuster] Pre-built 0 adjustments in 11.759541ms"
level=info msg="[linters_context/goanalysis] analyzers took 0s with no stages"
level=info msg="[loader] Go packages loading at mode 8767 (files|name|compiled_files|deps|exports_file|imports|types_sizes) took 386.58525ms"
level=info msg="[runner/filename_unadjuster] Pre-built 0 adjustments in 10.82225ms"
level=info msg="[linters_context/goanalysis] analyzers took 9.441069744s with top 10 stages: goimports: 2.162295625s, revive: 1.635175292s, gosec: 746.543207ms, unparam: 497.231375ms, wastedassign: 307.470709ms, gocritic: 147.325123ms, buildir: 145.809195ms, gofumpt: 123.647625ms, buildssa: 118.610962ms, nilness: 115.823328ms"
level=info msg="[runner/exclusion_rules] Skipped 8 issues by rules: [Path: \"_test\\\\.go\", Linters: \"gosec, prealloc, forcetypeassert, unparam\"]"
level=info msg="[runner] Issues before processing: 881, after processing: 0"
level=info msg="[runner] Processors filtering stat (in/out): exclusion_paths: 881/881, nolint_filter: 6/0, path_absoluter: 881/881, invalid_issue: 881/881, generated_file_filter: 881/292, cgo: 881/881, filename_unadjuster: 881/881, path_relativity: 881/881, exclusion_rules: 292/6"
level=info msg="[runner] processing took 2.916079ms with stages: generated_file_filter: 1.448166ms, nolint_filter: 891.875µs, exclusion_rules: 277.041µs, path_relativity: 263.459µs, invalid_issue: 13.082µs, cgo: 11.375µs, path_absoluter: 4.792µs, filename_unadjuster: 2.875µs, max_same_issues: 958ns, sort_results: 750ns, exclusion_paths: 500ns, fixer: 458ns, path_shortener: 167ns, max_per_file_from_linter: 125ns, source_code: 125ns, path_prettifier: 125ns, diff: 83ns, max_from_linter: 41ns, severity-rules: 41ns, uniq_by_line: 41ns"
level=info msg="[runner] linters took 627.484792ms with stages: goanalysis_metalinter: 624.525167ms"
level=info msg="[runner] Processors filtering stat (in/out): cgo: 881/881, exclusion_paths: 881/881, generated_file_filter: 881/292, exclusion_rules: 292/6, path_absoluter: 881/881, invalid_issue: 881/881, filename_unadjuster: 881/881, path_relativity: 881/881, nolint_filter: 6/0"
level=info msg="[runner] processing took 4.266378ms with stages: generated_file_filter: 2.538917ms, nolint_filter: 1.042041ms, exclusion_rules: 367.959µs, path_relativity: 277.792µs, invalid_issue: 13.084µs, cgo: 11.75µs, path_absoluter: 6.125µs, sort_results: 3.792µs, filename_unadjuster: 2.709µs, max_same_issues: 625ns, exclusion_paths: 458ns, fixer: 376ns, path_shortener: 126ns, path_prettifier: 125ns, max_from_linter: 125ns, max_per_file_from_linter: 125ns, uniq_by_line: 84ns, diff: 83ns, severity-rules: 41ns, source_code: 41ns"
level=info msg="[runner] linters took 2.049702792s with stages: goanalysis_metalinter: 2.045391875s"
0 issues.
level=info msg="File cache stats: 0 entries of total size 0B"
level=info msg="Memory: 10 samples, avg is 84.0MB, max is 111.1MB"
level=info msg="Execution took 856.003459ms"
level=info msg="Memory: 26 samples, avg is 277.9MB, max is 547.6MB"
level=info msg="Execution took 2.454527125s"
+1 -1
View File
@@ -1,3 +1,3 @@
# make test 2026-10-30T20:08:03Z
# make test 2026-10-30T20:15:52Z
env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy NO_PROXY=localhost,127.0.0.1,::1 no_proxy=localhost,127.0.0.1,::1 python3 tests/smoke.py --base-url "http://localhost:8080"
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
+12 -12
View File
@@ -1,16 +1,16 @@
# stand 2026-10-30T20:13:29Z
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi-messenger-lab5-api-1 mtusi-messenger:lab5 "/usr/local/bin/mess…" api 6 minutes ago Up 46 seconds (healthy) 8080/tcp
mtusi-messenger-lab5-api-2 mtusi-messenger:lab5 "/usr/local/bin/mess…" api 6 minutes ago Up 46 seconds (healthy) 8080/tcp
mtusi-messenger-lab5-broker-1 rabbitmq:4.3.7-management-alpine@sha256:1528a88eebf2bc4d0ab7e9d5d9bc542e9bdf5e1b105e65848fcf8de5c2186528 "docker-entrypoint.s…" broker 6 minutes ago Up About a minute (healthy) 4369/tcp, 5671-5672/tcp, 15671/tcp, 15691-15692/tcp, 25672/tcp, 127.0.0.1:15672->15672/tcp
mtusi-messenger-lab5-lb-1 haproxy:3.2-alpine@sha256:254e6370f01cd654e0744edf607c67f33777c1b6a9dc0d4b28bcba7171460122 "docker-entrypoint.s…" lb 6 minutes ago Up 40 seconds 127.0.0.1:8404->8404/tcp, 127.0.0.1:8080->80/tcp
mtusi-messenger-lab5-postgres-1 postgres:18-alpine@sha256:77f585114c32fbca283dc835b0596f4e52b51b4c6662d7810b2f4084f60a1873 "docker-entrypoint.s…" postgres 6 minutes ago Up About a minute (healthy) 5432/tcp
mtusi-messenger-lab5-redis-1 redis:7.4-alpine@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499 "docker-entrypoint.s…" redis 6 minutes ago Up About a minute (healthy) 6379/tcp
mtusi-messenger-lab5-relay-1 mtusi-messenger:lab5 "/usr/local/bin/relay" relay About a minute ago Up 46 seconds (healthy) 8080/tcp, 127.0.0.1:8082->8082/tcp
mtusi-messenger-lab5-seaweedfs-1 chrislusf/seaweedfs:3.80@sha256:1055999e08eed1789b0ae45d235126e4495e23d3fb9d6396293fd42539b1ae6a "/entrypoint.sh serv…" seaweedfs 6 minutes ago Up About a minute (healthy) 7333/tcp, 8080/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp, 127.0.0.1:8333->8333/tcp
mtusi-messenger-lab5-worker-1 mtusi-messenger:lab5 "/usr/local/bin/work…" worker About a minute ago Up 46 seconds (healthy) 8080/tcp, 127.0.0.1:8081->8081/tcp
# stand 2026-10-30T20:16:07Z
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi-messenger-lab5-api-1 mtusi-messenger:lab5 "/usr/local/bin/mess…" api 42 seconds ago Up Less than a second (health: starting) 8080/tcp
mtusi-messenger-lab5-api-2 mtusi-messenger:lab5 "/usr/local/bin/mess…" api 42 seconds ago Up Less than a second (health: starting) 8080/tcp
mtusi-messenger-lab5-broker-1 rabbitmq:4.3.7-management-alpine@sha256:1528a88eebf2bc4d0ab7e9d5d9bc542e9bdf5e1b105e65848fcf8de5c2186528 "docker-entrypoint.s…" broker 8 minutes ago Up 8 seconds (healthy) 4369/tcp, 5671-5672/tcp, 15671/tcp, 15691-15692/tcp, 25672/tcp, 127.0.0.1:15672->15672/tcp
mtusi-messenger-lab5-lb-1 haproxy:3.2-alpine@sha256:254e6370f01cd654e0744edf607c67f33777c1b6a9dc0d4b28bcba7171460122 "docker-entrypoint.s…" lb 8 minutes ago Up 3 minutes 127.0.0.1:8404->8404/tcp, 127.0.0.1:8080->80/tcp
mtusi-messenger-lab5-postgres-1 postgres:18-alpine@sha256:77f585114c32fbca283dc835b0596f4e52b51b4c6662d7810b2f4084f60a1873 "docker-entrypoint.s…" postgres 8 minutes ago Up 3 minutes (healthy) 5432/tcp
mtusi-messenger-lab5-redis-1 redis:7.4-alpine@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499 "docker-entrypoint.s…" redis 8 minutes ago Up 4 minutes (healthy) 6379/tcp
mtusi-messenger-lab5-relay-1 mtusi-messenger:lab5 "/usr/local/bin/relay" relay 42 seconds ago Up Less than a second (health: starting) 8080/tcp, 127.0.0.1:8082->8082/tcp
mtusi-messenger-lab5-seaweedfs-1 chrislusf/seaweedfs:3.80@sha256:1055999e08eed1789b0ae45d235126e4495e23d3fb9d6396293fd42539b1ae6a "/entrypoint.sh serv…" seaweedfs 8 minutes ago Up 3 minutes (healthy) 7333/tcp, 8080/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp, 127.0.0.1:8333->8333/tcp
mtusi-messenger-lab5-worker-1 mtusi-messenger:lab5 "/usr/local/bin/work…" worker 42 seconds ago Up Less than a second (health: starting) 8080/tcp, 127.0.0.1:8081->8081/tcp
mtusi-messenger:lab5 a637d8b7d2d2
mtusi-messenger:lab5 10b83ff52c7d
rabbitmq:4.3.7-management-alpine 1528a88eebf2
haproxy:3.2-alpine 254e6370f01c
redis:7.4-alpine 858f009f9709
+17 -17
View File
@@ -6,35 +6,35 @@ env -u APP_ENV -u SERVER_HOST -u SERVER_PORT -u SERVER_READ_HEADER_TIMEOUT -u SE
? mtusi-messenger/cmd/s3init [no test files]
? mtusi-messenger/cmd/worker [no test files]
? mtusi-messenger/internal/broker [no test files]
ok mtusi-messenger/internal/config 2.441s
ok mtusi-messenger/internal/domain 1.484s
ok mtusi-messenger/internal/config 1.385s
ok mtusi-messenger/internal/domain 2.358s
? mtusi-messenger/internal/failpoint [no test files]
ok mtusi-messenger/internal/health 2.069s
ok mtusi-messenger/internal/imagecheck 1.809s
ok mtusi-messenger/internal/imageproc 3.094s
ok mtusi-messenger/internal/logger 2.975s
ok mtusi-messenger/internal/health 2.187s
ok mtusi-messenger/internal/imagecheck 2.045s
ok mtusi-messenger/internal/imageproc 3.557s
ok mtusi-messenger/internal/logger 2.942s
? mtusi-messenger/internal/outbox [no test files]
ok mtusi-messenger/internal/platform/cache 3.522s
ok mtusi-messenger/internal/platform/cache 2.970s
? mtusi-messenger/internal/platform/cache/mocks [no test files]
ok mtusi-messenger/internal/platform/objectstore 5.364s
ok mtusi-messenger/internal/platform/password 4.551s
ok mtusi-messenger/internal/platform/token 4.459s
ok mtusi-messenger/internal/service/attachment 5.686s
ok mtusi-messenger/internal/platform/objectstore 3.292s
ok mtusi-messenger/internal/platform/password 4.017s
ok mtusi-messenger/internal/platform/token 2.535s
ok mtusi-messenger/internal/service/attachment 1.665s
? mtusi-messenger/internal/service/attachment/mocks [no test files]
ok mtusi-messenger/internal/service/chat 3.610s
ok mtusi-messenger/internal/service/chat 3.495s
? mtusi-messenger/internal/service/chat/mocks [no test files]
ok mtusi-messenger/internal/service/message 3.931s
ok mtusi-messenger/internal/service/message 1.866s
? mtusi-messenger/internal/service/message/mocks [no test files]
ok mtusi-messenger/internal/service/session 5.387s
ok mtusi-messenger/internal/service/session 4.403s
? mtusi-messenger/internal/service/session/mocks [no test files]
ok mtusi-messenger/internal/service/user 4.965s
ok mtusi-messenger/internal/service/user 4.032s
? mtusi-messenger/internal/service/user/mocks [no test files]
ok mtusi-messenger/internal/storage/postgres 5.965s
ok mtusi-messenger/internal/storage/postgres 4.123s
? mtusi-messenger/internal/storage/postgres/internal/sqlcgen [no test files]
? mtusi-messenger/internal/storage/postgres/migrations [no test files]
? mtusi-messenger/internal/storage/redis [no test files]
? mtusi-messenger/internal/storage/redis/mocks [no test files]
ok mtusi-messenger/internal/transport/rest 6.275s
ok mtusi-messenger/internal/transport/rest 4.446s
? mtusi-messenger/internal/transport/rest/internal/oas [no test files]
? mtusi-messenger/internal/transport/rest/mocks [no test files]
? mtusi-messenger/internal/worker [no test files]
+1
View File
@@ -105,6 +105,7 @@ type Admission struct {
MaxPendingPerUser int `mapstructure:"max_pending_per_user" validate:"required,gte=0"`
MaxPending int `mapstructure:"max_pending" validate:"required,gte=0"`
MaxStagedBytes int64 `mapstructure:"max_staged_bytes" validate:"required,gte=0"`
MaxConcurrent int `mapstructure:"max_concurrent_uploads" validate:"omitempty,gt=0"`
}
// S3 describes the object store and attachment limits. Access keys are checked
+16 -1
View File
@@ -17,6 +17,7 @@ import (
"unicode/utf8"
"github.com/google/uuid"
"golang.org/x/sync/semaphore"
"mtusi-messenger/internal/domain"
"mtusi-messenger/internal/imagecheck"
@@ -81,6 +82,7 @@ type Config struct {
MaxPendingPerUser int
MaxPending int
MaxStagedBytes int64
MaxConcurrent int
}
type Service struct {
@@ -88,6 +90,7 @@ type Service struct {
store objectstore.Store
config Config
log *slog.Logger
upload *semaphore.Weighted
}
func NewService(repo Repository, store objectstore.Store, config Config, log *slog.Logger) *Service {
@@ -95,7 +98,12 @@ func NewService(repo Repository, store objectstore.Store, config Config, log *sl
log = slog.Default()
}
return &Service{repo: repo, store: store, config: config, log: log}
service := &Service{repo: repo, store: store, config: config, log: log}
if config.MaxConcurrent > 0 {
service.upload = semaphore.NewWeighted(int64(config.MaxConcurrent))
}
return service
}
// UploadInput is one raw-body upload. DeclaredContentType comes from the HTTP
@@ -111,6 +119,13 @@ type UploadInput struct {
// Upload durably stages the original, records the job intent and returns the
// queued snapshot. It never waits for the broker or the worker.
func (s *Service) Upload(ctx context.Context, input UploadInput) (domain.Attachment, error) {
if s.upload != nil {
if err := s.upload.Acquire(ctx, 1); err != nil {
return domain.Attachment{}, unavailable("acquire upload slot", err)
}
defer s.upload.Release(1)
}
if err := ValidateFilename(input.Filename); err != nil {
return domain.Attachment{}, err
}