forked from mtusi2026/mtusi2026lab5
lab5: ограничение одновременных upload-spool и evidence после ревью
Process-level semaphore bounds temporary disk use on the upload path; refresh raw evidence after the review fixes.
This commit is contained in:
@@ -156,3 +156,4 @@ STAGING_FAILED_RETENTION=168h
|
||||
ADMISSION_MAX_PENDING_PER_USER=20
|
||||
ADMISSION_MAX_PENDING=200
|
||||
ADMISSION_MAX_STAGED_BYTES=2147483648
|
||||
ADMISSION_MAX_CONCURRENT_UPLOADS=8
|
||||
|
||||
@@ -90,7 +90,8 @@ flowchart LR
|
||||
- Admission control ограничивает число незавершённых rows (включая `pending` presign) на
|
||||
пользователя/стенд и объём staged bytes (429 `rate_limited`). Проверка повторяется внутри
|
||||
транзакции под PostgreSQL advisory lock, поэтому параллельные запросы не обходят cap; presign
|
||||
использует тот же механизм.
|
||||
использует тот же механизм. Число одновременных upload-spool на процесс ограничено
|
||||
`ADMISSION_MAX_CONCURRENT_UPLOADS`, что ограничивает и временный диск API.
|
||||
|
||||
### Соответствие `.env.example` реальной конфигурации
|
||||
|
||||
|
||||
@@ -87,6 +87,7 @@ func run(cfg config.Config, log *slog.Logger) error {
|
||||
MaxPendingPerUser: cfg.Admission.MaxPendingPerUser,
|
||||
MaxPending: cfg.Admission.MaxPending,
|
||||
MaxStagedBytes: cfg.Admission.MaxStagedBytes,
|
||||
MaxConcurrent: cfg.Admission.MaxConcurrent,
|
||||
},
|
||||
log,
|
||||
)
|
||||
|
||||
@@ -226,6 +226,7 @@ services:
|
||||
ADMISSION_MAX_PENDING_PER_USER: ${ADMISSION_MAX_PENDING_PER_USER:-20}
|
||||
ADMISSION_MAX_PENDING: ${ADMISSION_MAX_PENDING:-200}
|
||||
ADMISSION_MAX_STAGED_BYTES: ${ADMISSION_MAX_STAGED_BYTES:-2147483648}
|
||||
ADMISSION_MAX_CONCURRENT_UPLOADS: ${ADMISSION_MAX_CONCURRENT_UPLOADS:-8}
|
||||
# Размер пула соединений ОДНОЙ реплики; общий расход = реплики × значение.
|
||||
# Это один из общих ресурсов, который масштабирование расходует первым.
|
||||
POSTGRES_MAX_CONNS: "${POSTGRES_MAX_CONNS:-10}"
|
||||
|
||||
@@ -161,6 +161,7 @@ services:
|
||||
ADMISSION_MAX_PENDING_PER_USER: ${ADMISSION_MAX_PENDING_PER_USER:-20}
|
||||
ADMISSION_MAX_PENDING: ${ADMISSION_MAX_PENDING:-200}
|
||||
ADMISSION_MAX_STAGED_BYTES: ${ADMISSION_MAX_STAGED_BYTES:-2147483648}
|
||||
ADMISSION_MAX_CONCURRENT_UPLOADS: ${ADMISSION_MAX_CONCURRENT_UPLOADS:-8}
|
||||
ports:
|
||||
- "127.0.0.1:${SERVER_PORT:-8080}:8080"
|
||||
depends_on:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# lab5_async 2026-10-30T20:08:22Z
|
||||
# lab5_async 2026-10-30T20:15:54Z
|
||||
worker down: 202 queued, content 409, queued attachment linked, API restarted
|
||||
worker restarted: ready, thumbnail and original downloadable, staging cleared
|
||||
broker down: upload accepted into outbox
|
||||
|
||||
@@ -1,29 +1,29 @@
|
||||
# db state 2026-10-30T20:13:30Z
|
||||
# db state 2026-10-30T20:16:08Z
|
||||
status | count
|
||||
---------+-------
|
||||
failed | 2
|
||||
pending | 4
|
||||
ready | 25
|
||||
ready | 31
|
||||
(3 rows)
|
||||
|
||||
status | count
|
||||
--------+-------
|
||||
failed | 2
|
||||
ready | 28
|
||||
ready | 34
|
||||
(2 rows)
|
||||
|
||||
variants
|
||||
----------
|
||||
84
|
||||
102
|
||||
(1 row)
|
||||
|
||||
unpublished | outbox
|
||||
-------------+--------
|
||||
0 | 52
|
||||
0 | 58
|
||||
(1 row)
|
||||
|
||||
Timeout: 60.0 seconds ...
|
||||
Listing queues for vhost messenger ...
|
||||
name messages messages_ready messages_unacknowledged consumers
|
||||
attachment.process.dead 4 4 0 0
|
||||
attachment.process.v1 0 0 0 1
|
||||
attachment.process.v1 0 0 0 0
|
||||
|
||||
@@ -3,18 +3,18 @@ level=info msg="golangci-lint has version 2.13.2 built with go1.27.1 from (unkno
|
||||
level=info msg="[config_reader] Config search paths: [./ /Users/vladislav/Projects/mtusi2026lab5 /Users/vladislav/Projects /Users/vladislav /Users /]"
|
||||
level=info msg="[config_reader] Used config file .golangci.yml"
|
||||
level=info msg="[config_reader] Module name \"mtusi-messenger\""
|
||||
level=info msg="[goenv] Read go env for 5.643458ms: map[string]string{\"GOCACHE\":\"/Users/vladislav/Library/Caches/go-build\", \"GOROOT\":\"/opt/homebrew/Cellar/go/1.27.1/libexec\"}"
|
||||
level=info msg="[goenv] Read go env for 6.899ms: map[string]string{\"GOCACHE\":\"/Users/vladislav/Library/Caches/go-build\", \"GOROOT\":\"/opt/homebrew/Cellar/go/1.27.1/libexec\"}"
|
||||
level=info msg="[lintersdb] Active 50 linters: [asasalint bodyclose canonicalheader containedctx contextcheck copyloopvar depguard durationcheck errcheck errname errorlint exhaustive exptostd fatcontext forbidigo forcetypeassert gocheckcompilerdirectives gocritic gofumpt goimports gomoddirectives gosec govet ineffassign intrange makezero misspell musttag nilerr nilnil noctx nolintlint perfsprint prealloc reassign recvcheck revive rowserrcheck sloglint sqlclosecheck staticcheck testifylint thelper tparallel unconvert unparam unused usestdlibvars usetesting wastedassign]"
|
||||
level=info msg="[loader] Using build tags: [integration]"
|
||||
level=info msg="[loader] Go packages loading at mode 8767 (files|imports|name|compiled_files|deps|exports_file|types_sizes) took 210.149917ms"
|
||||
level=info msg="[runner/filename_unadjuster] Pre-built 0 adjustments in 11.759541ms"
|
||||
level=info msg="[linters_context/goanalysis] analyzers took 0s with no stages"
|
||||
level=info msg="[loader] Go packages loading at mode 8767 (files|name|compiled_files|deps|exports_file|imports|types_sizes) took 386.58525ms"
|
||||
level=info msg="[runner/filename_unadjuster] Pre-built 0 adjustments in 10.82225ms"
|
||||
level=info msg="[linters_context/goanalysis] analyzers took 9.441069744s with top 10 stages: goimports: 2.162295625s, revive: 1.635175292s, gosec: 746.543207ms, unparam: 497.231375ms, wastedassign: 307.470709ms, gocritic: 147.325123ms, buildir: 145.809195ms, gofumpt: 123.647625ms, buildssa: 118.610962ms, nilness: 115.823328ms"
|
||||
level=info msg="[runner/exclusion_rules] Skipped 8 issues by rules: [Path: \"_test\\\\.go\", Linters: \"gosec, prealloc, forcetypeassert, unparam\"]"
|
||||
level=info msg="[runner] Issues before processing: 881, after processing: 0"
|
||||
level=info msg="[runner] Processors filtering stat (in/out): exclusion_paths: 881/881, nolint_filter: 6/0, path_absoluter: 881/881, invalid_issue: 881/881, generated_file_filter: 881/292, cgo: 881/881, filename_unadjuster: 881/881, path_relativity: 881/881, exclusion_rules: 292/6"
|
||||
level=info msg="[runner] processing took 2.916079ms with stages: generated_file_filter: 1.448166ms, nolint_filter: 891.875µs, exclusion_rules: 277.041µs, path_relativity: 263.459µs, invalid_issue: 13.082µs, cgo: 11.375µs, path_absoluter: 4.792µs, filename_unadjuster: 2.875µs, max_same_issues: 958ns, sort_results: 750ns, exclusion_paths: 500ns, fixer: 458ns, path_shortener: 167ns, max_per_file_from_linter: 125ns, source_code: 125ns, path_prettifier: 125ns, diff: 83ns, max_from_linter: 41ns, severity-rules: 41ns, uniq_by_line: 41ns"
|
||||
level=info msg="[runner] linters took 627.484792ms with stages: goanalysis_metalinter: 624.525167ms"
|
||||
level=info msg="[runner] Processors filtering stat (in/out): cgo: 881/881, exclusion_paths: 881/881, generated_file_filter: 881/292, exclusion_rules: 292/6, path_absoluter: 881/881, invalid_issue: 881/881, filename_unadjuster: 881/881, path_relativity: 881/881, nolint_filter: 6/0"
|
||||
level=info msg="[runner] processing took 4.266378ms with stages: generated_file_filter: 2.538917ms, nolint_filter: 1.042041ms, exclusion_rules: 367.959µs, path_relativity: 277.792µs, invalid_issue: 13.084µs, cgo: 11.75µs, path_absoluter: 6.125µs, sort_results: 3.792µs, filename_unadjuster: 2.709µs, max_same_issues: 625ns, exclusion_paths: 458ns, fixer: 376ns, path_shortener: 126ns, path_prettifier: 125ns, max_from_linter: 125ns, max_per_file_from_linter: 125ns, uniq_by_line: 84ns, diff: 83ns, severity-rules: 41ns, source_code: 41ns"
|
||||
level=info msg="[runner] linters took 2.049702792s with stages: goanalysis_metalinter: 2.045391875s"
|
||||
0 issues.
|
||||
level=info msg="File cache stats: 0 entries of total size 0B"
|
||||
level=info msg="Memory: 10 samples, avg is 84.0MB, max is 111.1MB"
|
||||
level=info msg="Execution took 856.003459ms"
|
||||
level=info msg="Memory: 26 samples, avg is 277.9MB, max is 547.6MB"
|
||||
level=info msg="Execution took 2.454527125s"
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
# make test 2026-10-30T20:08:03Z
|
||||
# make test 2026-10-30T20:15:52Z
|
||||
env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy NO_PROXY=localhost,127.0.0.1,::1 no_proxy=localhost,127.0.0.1,::1 python3 tests/smoke.py --base-url "http://localhost:8080"
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
|
||||
+12
-12
@@ -1,16 +1,16 @@
|
||||
# stand 2026-10-30T20:13:29Z
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
mtusi-messenger-lab5-api-1 mtusi-messenger:lab5 "/usr/local/bin/mess…" api 6 minutes ago Up 46 seconds (healthy) 8080/tcp
|
||||
mtusi-messenger-lab5-api-2 mtusi-messenger:lab5 "/usr/local/bin/mess…" api 6 minutes ago Up 46 seconds (healthy) 8080/tcp
|
||||
mtusi-messenger-lab5-broker-1 rabbitmq:4.3.7-management-alpine@sha256:1528a88eebf2bc4d0ab7e9d5d9bc542e9bdf5e1b105e65848fcf8de5c2186528 "docker-entrypoint.s…" broker 6 minutes ago Up About a minute (healthy) 4369/tcp, 5671-5672/tcp, 15671/tcp, 15691-15692/tcp, 25672/tcp, 127.0.0.1:15672->15672/tcp
|
||||
mtusi-messenger-lab5-lb-1 haproxy:3.2-alpine@sha256:254e6370f01cd654e0744edf607c67f33777c1b6a9dc0d4b28bcba7171460122 "docker-entrypoint.s…" lb 6 minutes ago Up 40 seconds 127.0.0.1:8404->8404/tcp, 127.0.0.1:8080->80/tcp
|
||||
mtusi-messenger-lab5-postgres-1 postgres:18-alpine@sha256:77f585114c32fbca283dc835b0596f4e52b51b4c6662d7810b2f4084f60a1873 "docker-entrypoint.s…" postgres 6 minutes ago Up About a minute (healthy) 5432/tcp
|
||||
mtusi-messenger-lab5-redis-1 redis:7.4-alpine@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499 "docker-entrypoint.s…" redis 6 minutes ago Up About a minute (healthy) 6379/tcp
|
||||
mtusi-messenger-lab5-relay-1 mtusi-messenger:lab5 "/usr/local/bin/relay" relay About a minute ago Up 46 seconds (healthy) 8080/tcp, 127.0.0.1:8082->8082/tcp
|
||||
mtusi-messenger-lab5-seaweedfs-1 chrislusf/seaweedfs:3.80@sha256:1055999e08eed1789b0ae45d235126e4495e23d3fb9d6396293fd42539b1ae6a "/entrypoint.sh serv…" seaweedfs 6 minutes ago Up About a minute (healthy) 7333/tcp, 8080/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp, 127.0.0.1:8333->8333/tcp
|
||||
mtusi-messenger-lab5-worker-1 mtusi-messenger:lab5 "/usr/local/bin/work…" worker About a minute ago Up 46 seconds (healthy) 8080/tcp, 127.0.0.1:8081->8081/tcp
|
||||
# stand 2026-10-30T20:16:07Z
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
mtusi-messenger-lab5-api-1 mtusi-messenger:lab5 "/usr/local/bin/mess…" api 42 seconds ago Up Less than a second (health: starting) 8080/tcp
|
||||
mtusi-messenger-lab5-api-2 mtusi-messenger:lab5 "/usr/local/bin/mess…" api 42 seconds ago Up Less than a second (health: starting) 8080/tcp
|
||||
mtusi-messenger-lab5-broker-1 rabbitmq:4.3.7-management-alpine@sha256:1528a88eebf2bc4d0ab7e9d5d9bc542e9bdf5e1b105e65848fcf8de5c2186528 "docker-entrypoint.s…" broker 8 minutes ago Up 8 seconds (healthy) 4369/tcp, 5671-5672/tcp, 15671/tcp, 15691-15692/tcp, 25672/tcp, 127.0.0.1:15672->15672/tcp
|
||||
mtusi-messenger-lab5-lb-1 haproxy:3.2-alpine@sha256:254e6370f01cd654e0744edf607c67f33777c1b6a9dc0d4b28bcba7171460122 "docker-entrypoint.s…" lb 8 minutes ago Up 3 minutes 127.0.0.1:8404->8404/tcp, 127.0.0.1:8080->80/tcp
|
||||
mtusi-messenger-lab5-postgres-1 postgres:18-alpine@sha256:77f585114c32fbca283dc835b0596f4e52b51b4c6662d7810b2f4084f60a1873 "docker-entrypoint.s…" postgres 8 minutes ago Up 3 minutes (healthy) 5432/tcp
|
||||
mtusi-messenger-lab5-redis-1 redis:7.4-alpine@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499 "docker-entrypoint.s…" redis 8 minutes ago Up 4 minutes (healthy) 6379/tcp
|
||||
mtusi-messenger-lab5-relay-1 mtusi-messenger:lab5 "/usr/local/bin/relay" relay 42 seconds ago Up Less than a second (health: starting) 8080/tcp, 127.0.0.1:8082->8082/tcp
|
||||
mtusi-messenger-lab5-seaweedfs-1 chrislusf/seaweedfs:3.80@sha256:1055999e08eed1789b0ae45d235126e4495e23d3fb9d6396293fd42539b1ae6a "/entrypoint.sh serv…" seaweedfs 8 minutes ago Up 3 minutes (healthy) 7333/tcp, 8080/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp, 127.0.0.1:8333->8333/tcp
|
||||
mtusi-messenger-lab5-worker-1 mtusi-messenger:lab5 "/usr/local/bin/work…" worker 42 seconds ago Up Less than a second (health: starting) 8080/tcp, 127.0.0.1:8081->8081/tcp
|
||||
|
||||
mtusi-messenger:lab5 a637d8b7d2d2
|
||||
mtusi-messenger:lab5 10b83ff52c7d
|
||||
rabbitmq:4.3.7-management-alpine 1528a88eebf2
|
||||
haproxy:3.2-alpine 254e6370f01c
|
||||
redis:7.4-alpine 858f009f9709
|
||||
|
||||
+17
-17
@@ -6,35 +6,35 @@ env -u APP_ENV -u SERVER_HOST -u SERVER_PORT -u SERVER_READ_HEADER_TIMEOUT -u SE
|
||||
? mtusi-messenger/cmd/s3init [no test files]
|
||||
? mtusi-messenger/cmd/worker [no test files]
|
||||
? mtusi-messenger/internal/broker [no test files]
|
||||
ok mtusi-messenger/internal/config 2.441s
|
||||
ok mtusi-messenger/internal/domain 1.484s
|
||||
ok mtusi-messenger/internal/config 1.385s
|
||||
ok mtusi-messenger/internal/domain 2.358s
|
||||
? mtusi-messenger/internal/failpoint [no test files]
|
||||
ok mtusi-messenger/internal/health 2.069s
|
||||
ok mtusi-messenger/internal/imagecheck 1.809s
|
||||
ok mtusi-messenger/internal/imageproc 3.094s
|
||||
ok mtusi-messenger/internal/logger 2.975s
|
||||
ok mtusi-messenger/internal/health 2.187s
|
||||
ok mtusi-messenger/internal/imagecheck 2.045s
|
||||
ok mtusi-messenger/internal/imageproc 3.557s
|
||||
ok mtusi-messenger/internal/logger 2.942s
|
||||
? mtusi-messenger/internal/outbox [no test files]
|
||||
ok mtusi-messenger/internal/platform/cache 3.522s
|
||||
ok mtusi-messenger/internal/platform/cache 2.970s
|
||||
? mtusi-messenger/internal/platform/cache/mocks [no test files]
|
||||
ok mtusi-messenger/internal/platform/objectstore 5.364s
|
||||
ok mtusi-messenger/internal/platform/password 4.551s
|
||||
ok mtusi-messenger/internal/platform/token 4.459s
|
||||
ok mtusi-messenger/internal/service/attachment 5.686s
|
||||
ok mtusi-messenger/internal/platform/objectstore 3.292s
|
||||
ok mtusi-messenger/internal/platform/password 4.017s
|
||||
ok mtusi-messenger/internal/platform/token 2.535s
|
||||
ok mtusi-messenger/internal/service/attachment 1.665s
|
||||
? mtusi-messenger/internal/service/attachment/mocks [no test files]
|
||||
ok mtusi-messenger/internal/service/chat 3.610s
|
||||
ok mtusi-messenger/internal/service/chat 3.495s
|
||||
? mtusi-messenger/internal/service/chat/mocks [no test files]
|
||||
ok mtusi-messenger/internal/service/message 3.931s
|
||||
ok mtusi-messenger/internal/service/message 1.866s
|
||||
? mtusi-messenger/internal/service/message/mocks [no test files]
|
||||
ok mtusi-messenger/internal/service/session 5.387s
|
||||
ok mtusi-messenger/internal/service/session 4.403s
|
||||
? mtusi-messenger/internal/service/session/mocks [no test files]
|
||||
ok mtusi-messenger/internal/service/user 4.965s
|
||||
ok mtusi-messenger/internal/service/user 4.032s
|
||||
? mtusi-messenger/internal/service/user/mocks [no test files]
|
||||
ok mtusi-messenger/internal/storage/postgres 5.965s
|
||||
ok mtusi-messenger/internal/storage/postgres 4.123s
|
||||
? mtusi-messenger/internal/storage/postgres/internal/sqlcgen [no test files]
|
||||
? mtusi-messenger/internal/storage/postgres/migrations [no test files]
|
||||
? mtusi-messenger/internal/storage/redis [no test files]
|
||||
? mtusi-messenger/internal/storage/redis/mocks [no test files]
|
||||
ok mtusi-messenger/internal/transport/rest 6.275s
|
||||
ok mtusi-messenger/internal/transport/rest 4.446s
|
||||
? mtusi-messenger/internal/transport/rest/internal/oas [no test files]
|
||||
? mtusi-messenger/internal/transport/rest/mocks [no test files]
|
||||
? mtusi-messenger/internal/worker [no test files]
|
||||
|
||||
@@ -105,6 +105,7 @@ type Admission struct {
|
||||
MaxPendingPerUser int `mapstructure:"max_pending_per_user" validate:"required,gte=0"`
|
||||
MaxPending int `mapstructure:"max_pending" validate:"required,gte=0"`
|
||||
MaxStagedBytes int64 `mapstructure:"max_staged_bytes" validate:"required,gte=0"`
|
||||
MaxConcurrent int `mapstructure:"max_concurrent_uploads" validate:"omitempty,gt=0"`
|
||||
}
|
||||
|
||||
// S3 describes the object store and attachment limits. Access keys are checked
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"golang.org/x/sync/semaphore"
|
||||
|
||||
"mtusi-messenger/internal/domain"
|
||||
"mtusi-messenger/internal/imagecheck"
|
||||
@@ -81,6 +82,7 @@ type Config struct {
|
||||
MaxPendingPerUser int
|
||||
MaxPending int
|
||||
MaxStagedBytes int64
|
||||
MaxConcurrent int
|
||||
}
|
||||
|
||||
type Service struct {
|
||||
@@ -88,6 +90,7 @@ type Service struct {
|
||||
store objectstore.Store
|
||||
config Config
|
||||
log *slog.Logger
|
||||
upload *semaphore.Weighted
|
||||
}
|
||||
|
||||
func NewService(repo Repository, store objectstore.Store, config Config, log *slog.Logger) *Service {
|
||||
@@ -95,7 +98,12 @@ func NewService(repo Repository, store objectstore.Store, config Config, log *sl
|
||||
log = slog.Default()
|
||||
}
|
||||
|
||||
return &Service{repo: repo, store: store, config: config, log: log}
|
||||
service := &Service{repo: repo, store: store, config: config, log: log}
|
||||
if config.MaxConcurrent > 0 {
|
||||
service.upload = semaphore.NewWeighted(int64(config.MaxConcurrent))
|
||||
}
|
||||
|
||||
return service
|
||||
}
|
||||
|
||||
// UploadInput is one raw-body upload. DeclaredContentType comes from the HTTP
|
||||
@@ -111,6 +119,13 @@ type UploadInput struct {
|
||||
// Upload durably stages the original, records the job intent and returns the
|
||||
// queued snapshot. It never waits for the broker or the worker.
|
||||
func (s *Service) Upload(ctx context.Context, input UploadInput) (domain.Attachment, error) {
|
||||
if s.upload != nil {
|
||||
if err := s.upload.Acquire(ctx, 1); err != nil {
|
||||
return domain.Attachment{}, unavailable("acquire upload slot", err)
|
||||
}
|
||||
defer s.upload.Release(1)
|
||||
}
|
||||
|
||||
if err := ValidateFilename(input.Filename); err != nil {
|
||||
return domain.Attachment{}, err
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user