forked from mtusi2026/mtusi2026lab5
fix: enforce pipeline lifecycle limits
This commit is contained in:
@@ -169,6 +169,7 @@ func newExperiment(ctx context.Context) (*experiment, func(), error) {
|
||||
dispatcher, err := outbox.New(repository, broker, nil, outbox.Config{
|
||||
BatchSize: 32, LeaseDuration: 10 * time.Second, PollInterval: time.Second,
|
||||
RecoveryInterval: 10 * time.Second, RetryDelay: time.Second,
|
||||
Retention: 24 * time.Hour, CleanupInterval: time.Hour, CleanupBatchSize: 1000,
|
||||
})
|
||||
if err != nil {
|
||||
pool.Close()
|
||||
@@ -185,9 +186,10 @@ func newExperiment(ctx context.Context) (*experiment, func(), error) {
|
||||
slog.New(slog.NewTextHandler(io.Discard, nil)), repository, broker, objects, processor, nil,
|
||||
worker.Config{
|
||||
WorkerID: consumer, Concurrency: 1,
|
||||
LeaseDuration: experimentLease, ProcessingTimeout: time.Second,
|
||||
ReclaimInterval: experimentLease,
|
||||
MaxAttempts: maxAttempts, RetryBase: time.Second, RetryMax: 4 * time.Second,
|
||||
LeaseDuration: experimentLease, HeartbeatInterval: 500 * time.Millisecond,
|
||||
ProcessingTimeout: time.Second,
|
||||
ReclaimInterval: experimentLease,
|
||||
MaxAttempts: maxAttempts, RetryBase: time.Second, RetryMax: 4 * time.Second,
|
||||
MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels,
|
||||
MemoryBytes: settings.MemoryBytes,
|
||||
},
|
||||
|
||||
+21
-103
@@ -1,3 +1,9 @@
|
||||
x-messenger-config-volume: &messenger-config-volume
|
||||
type: bind
|
||||
source: ./config/messenger.yaml
|
||||
target: /etc/messenger/messenger.yaml
|
||||
read_only: true
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17.6-alpine3.22
|
||||
@@ -115,14 +121,10 @@ services:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
command: ["/app/messenger", "migrate"]
|
||||
environment:
|
||||
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
|
||||
DATABASE_URL: ${DATABASE_URL}
|
||||
DB_MIN_CONNS: ${DB_MIN_CONNS}
|
||||
DB_MAX_CONNS: ${DB_MAX_CONNS}
|
||||
DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS}
|
||||
DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS}
|
||||
DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS}
|
||||
DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS}
|
||||
MIGRATION_TIMEOUT_SECONDS: ${MIGRATION_TIMEOUT_SECONDS}
|
||||
volumes:
|
||||
- *messenger-config-volume
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
@@ -137,43 +139,19 @@ services:
|
||||
build: .
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
environment:
|
||||
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
|
||||
APP_HOST: ${APP_HOST}
|
||||
APP_PORT: ${APP_PORT}
|
||||
DATABASE_URL: ${DATABASE_URL}
|
||||
CACHE_URL: ${CACHE_URL}
|
||||
SESSION_STORE: ${SESSION_STORE}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS}
|
||||
DB_MIN_CONNS: ${DB_MIN_CONNS}
|
||||
DB_MAX_CONNS: ${DB_MAX_CONNS}
|
||||
DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS}
|
||||
DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS}
|
||||
DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS}
|
||||
DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS}
|
||||
REQUEST_TIMEOUT_SECONDS: ${REQUEST_TIMEOUT_SECONDS}
|
||||
HTTP_READ_HEADER_TIMEOUT_SECONDS: ${HTTP_READ_HEADER_TIMEOUT_SECONDS}
|
||||
HTTP_READ_TIMEOUT_SECONDS: ${HTTP_READ_TIMEOUT_SECONDS}
|
||||
HTTP_WRITE_TIMEOUT_SECONDS: ${HTTP_WRITE_TIMEOUT_SECONDS}
|
||||
HTTP_IDLE_TIMEOUT_SECONDS: ${HTTP_IDLE_TIMEOUT_SECONDS}
|
||||
HTTP_SHUTDOWN_TIMEOUT_SECONDS: ${HTTP_SHUTDOWN_TIMEOUT_SECONDS}
|
||||
PASSWORD_MEMORY_KIB: ${PASSWORD_MEMORY_KIB}
|
||||
PASSWORD_ITERATIONS: ${PASSWORD_ITERATIONS}
|
||||
PASSWORD_PARALLELISM: ${PASSWORD_PARALLELISM}
|
||||
PASSWORD_SALT_LENGTH: ${PASSWORD_SALT_LENGTH}
|
||||
PASSWORD_KEY_LENGTH: ${PASSWORD_KEY_LENGTH}
|
||||
SWAGGER_PORT: ${SWAGGER_PORT}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT}
|
||||
S3_REGION: ${S3_REGION}
|
||||
S3_BUCKET: ${S3_BUCKET}
|
||||
S3_ACCESS_KEY: ${S3_ACCESS_KEY}
|
||||
S3_SECRET_KEY: ${S3_SECRET_KEY}
|
||||
S3_USE_TLS: ${S3_USE_TLS}
|
||||
S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS}
|
||||
ATTACHMENT_MAX_BYTES: ${ATTACHMENT_MAX_BYTES}
|
||||
ATTACHMENT_MAX_PIXELS: ${ATTACHMENT_MAX_PIXELS}
|
||||
ATTACHMENT_BODY_MEMORY_BYTES: ${ATTACHMENT_BODY_MEMORY_BYTES}
|
||||
ATTACHMENT_DECODE_MEMORY_BYTES: ${ATTACHMENT_DECODE_MEMORY_BYTES}
|
||||
PIPELINE_VERSION: ${PIPELINE_VERSION}
|
||||
volumes:
|
||||
- *messenger-config-volume
|
||||
expose:
|
||||
- "8080"
|
||||
depends_on:
|
||||
@@ -201,25 +179,11 @@ services:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
command: ["/app/messenger", "dispatch"]
|
||||
environment:
|
||||
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
|
||||
DATABASE_URL: ${DATABASE_URL}
|
||||
DB_MIN_CONNS: ${DB_MIN_CONNS}
|
||||
DB_MAX_CONNS: ${DB_MAX_CONNS}
|
||||
DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS}
|
||||
DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS}
|
||||
DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS}
|
||||
DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS}
|
||||
BROKER_URL: ${BROKER_URL}
|
||||
BROKER_STREAM: ${BROKER_STREAM}
|
||||
BROKER_CONSUMER_GROUP: ${BROKER_CONSUMER_GROUP}
|
||||
BROKER_FAILED_STREAM: ${BROKER_FAILED_STREAM}
|
||||
BROKER_FAILED_MAX_LENGTH: ${BROKER_FAILED_MAX_LENGTH}
|
||||
OUTBOX_BATCH_SIZE: ${OUTBOX_BATCH_SIZE}
|
||||
OUTBOX_LEASE_SECONDS: ${OUTBOX_LEASE_SECONDS}
|
||||
OUTBOX_POLL_SECONDS: ${OUTBOX_POLL_SECONDS}
|
||||
OUTBOX_RECOVERY_SECONDS: ${OUTBOX_RECOVERY_SECONDS}
|
||||
OUTBOX_RETRY_SECONDS: ${OUTBOX_RETRY_SECONDS}
|
||||
PIPELINE_METRICS_HOST: 0.0.0.0
|
||||
PIPELINE_METRICS_PORT: 9091
|
||||
volumes:
|
||||
- *messenger-config-volume
|
||||
expose:
|
||||
- "9091"
|
||||
depends_on:
|
||||
@@ -241,39 +205,16 @@ services:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
command: ["/app/messenger", "worker"]
|
||||
environment:
|
||||
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
|
||||
DATABASE_URL: ${DATABASE_URL}
|
||||
DB_MIN_CONNS: ${DB_MIN_CONNS}
|
||||
DB_MAX_CONNS: ${DB_MAX_CONNS}
|
||||
DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS}
|
||||
DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS}
|
||||
DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS}
|
||||
DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT}
|
||||
S3_REGION: ${S3_REGION}
|
||||
S3_BUCKET: ${S3_BUCKET}
|
||||
S3_ACCESS_KEY: ${S3_ACCESS_KEY}
|
||||
S3_SECRET_KEY: ${S3_SECRET_KEY}
|
||||
S3_USE_TLS: ${S3_USE_TLS}
|
||||
S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS}
|
||||
BROKER_URL: ${BROKER_URL}
|
||||
BROKER_STREAM: ${BROKER_STREAM}
|
||||
BROKER_CONSUMER_GROUP: ${BROKER_CONSUMER_GROUP}
|
||||
BROKER_FAILED_STREAM: ${BROKER_FAILED_STREAM}
|
||||
BROKER_FAILED_MAX_LENGTH: ${BROKER_FAILED_MAX_LENGTH}
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY}
|
||||
WORKER_PREFETCH: ${WORKER_PREFETCH}
|
||||
WORKER_LEASE_SECONDS: ${WORKER_LEASE_SECONDS}
|
||||
WORKER_PROCESSING_TIMEOUT_SECONDS: ${WORKER_PROCESSING_TIMEOUT_SECONDS}
|
||||
WORKER_READ_BLOCK_SECONDS: ${WORKER_READ_BLOCK_SECONDS}
|
||||
WORKER_RECLAIM_SECONDS: ${WORKER_RECLAIM_SECONDS}
|
||||
WORKER_MAX_ATTEMPTS: ${WORKER_MAX_ATTEMPTS}
|
||||
WORKER_RETRY_BASE_SECONDS: ${WORKER_RETRY_BASE_SECONDS}
|
||||
WORKER_RETRY_MAX_SECONDS: ${WORKER_RETRY_MAX_SECONDS}
|
||||
ATTACHMENT_MAX_BYTES: ${ATTACHMENT_MAX_BYTES}
|
||||
ATTACHMENT_MAX_PIXELS: ${ATTACHMENT_MAX_PIXELS}
|
||||
WORKER_MEMORY_BYTES: ${WORKER_MEMORY_BYTES}
|
||||
PIPELINE_METRICS_HOST: 0.0.0.0
|
||||
PIPELINE_METRICS_PORT: 9091
|
||||
volumes:
|
||||
- *messenger-config-volume
|
||||
expose:
|
||||
- "9091"
|
||||
depends_on:
|
||||
@@ -351,39 +292,16 @@ services:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
command: ["/app/lab5experiment"]
|
||||
environment:
|
||||
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
|
||||
DATABASE_URL: ${DATABASE_URL}
|
||||
DB_MIN_CONNS: ${DB_MIN_CONNS}
|
||||
DB_MAX_CONNS: ${DB_MAX_CONNS}
|
||||
DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS}
|
||||
DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS}
|
||||
DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS}
|
||||
DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT}
|
||||
S3_REGION: ${S3_REGION}
|
||||
S3_BUCKET: ${S3_BUCKET}
|
||||
S3_ACCESS_KEY: ${S3_ACCESS_KEY}
|
||||
S3_SECRET_KEY: ${S3_SECRET_KEY}
|
||||
S3_USE_TLS: ${S3_USE_TLS}
|
||||
S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS}
|
||||
BROKER_URL: ${BROKER_URL}
|
||||
BROKER_STREAM: ${BROKER_STREAM}
|
||||
BROKER_CONSUMER_GROUP: ${BROKER_CONSUMER_GROUP}
|
||||
BROKER_FAILED_STREAM: ${BROKER_FAILED_STREAM}
|
||||
BROKER_FAILED_MAX_LENGTH: ${BROKER_FAILED_MAX_LENGTH}
|
||||
WORKER_CONCURRENCY: 1
|
||||
WORKER_PREFETCH: 32
|
||||
WORKER_LEASE_SECONDS: 2
|
||||
WORKER_PROCESSING_TIMEOUT_SECONDS: 1
|
||||
WORKER_READ_BLOCK_SECONDS: 1
|
||||
WORKER_RECLAIM_SECONDS: 2
|
||||
WORKER_MAX_ATTEMPTS: ${WORKER_MAX_ATTEMPTS}
|
||||
WORKER_RETRY_BASE_SECONDS: 1
|
||||
WORKER_RETRY_MAX_SECONDS: 4
|
||||
ATTACHMENT_MAX_BYTES: ${ATTACHMENT_MAX_BYTES}
|
||||
ATTACHMENT_MAX_PIXELS: ${ATTACHMENT_MAX_PIXELS}
|
||||
WORKER_MEMORY_BYTES: ${WORKER_MEMORY_BYTES}
|
||||
PIPELINE_METRICS_HOST: 0.0.0.0
|
||||
PIPELINE_METRICS_PORT: 9091
|
||||
volumes:
|
||||
- *messenger-config-volume
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
version: 1
|
||||
|
||||
server:
|
||||
session_store: valkey
|
||||
session_ttl: 1h
|
||||
idempotency_ttl: 24h
|
||||
request_timeout: 5s
|
||||
http:
|
||||
read_header_timeout: 5s
|
||||
read_timeout: 10s
|
||||
write_timeout: 10s
|
||||
idle_timeout: 1m
|
||||
shutdown_timeout: 5s
|
||||
password:
|
||||
memory_kib: 65536
|
||||
iterations: 3
|
||||
parallelism: 2
|
||||
salt_length: 16
|
||||
key_length: 32
|
||||
|
||||
database:
|
||||
min_connections: 1
|
||||
max_connections: 10
|
||||
max_connection_age: 30m
|
||||
max_connection_idle: 5m
|
||||
health_check_period: 30s
|
||||
dependency_timeout: 5s
|
||||
|
||||
migration:
|
||||
timeout: 30s
|
||||
|
||||
object_storage:
|
||||
use_tls: false
|
||||
timeout: 5s
|
||||
|
||||
attachments:
|
||||
max_bytes: 10485760
|
||||
max_pixels: 40000000
|
||||
body_memory_bytes: 67108864
|
||||
decode_memory_bytes: 402653184
|
||||
|
||||
pipeline:
|
||||
version: 1
|
||||
|
||||
broker:
|
||||
stream: attachment-jobs
|
||||
consumer_group: attachment-workers
|
||||
failed_stream: attachment-jobs-failed
|
||||
failed_stream_max_length: 10000
|
||||
|
||||
outbox:
|
||||
batch_size: 32
|
||||
lease: 15s
|
||||
poll_interval: 1s
|
||||
recovery_interval: 10s
|
||||
retry_delay: 2s
|
||||
retention: 168h
|
||||
cleanup_interval: 1m
|
||||
cleanup_batch_size: 1000
|
||||
|
||||
worker:
|
||||
concurrency: 2
|
||||
prefetch: 4
|
||||
lease: 30s
|
||||
heartbeat_interval: 5s
|
||||
processing_timeout: 20s
|
||||
read_block: 2s
|
||||
reclaim_interval: 5s
|
||||
max_attempts: 5
|
||||
retry:
|
||||
base: 1s
|
||||
max: 30s
|
||||
memory_bytes: 738197504
|
||||
|
||||
monitoring:
|
||||
metrics_address: 0.0.0.0:9091
|
||||
@@ -55,7 +55,8 @@ func RunDispatcher(logger *slog.Logger) error {
|
||||
dispatcher, err := outbox.New(postgres.New(pool), broker, metrics, outbox.Config{
|
||||
BatchSize: settings.BatchSize, LeaseDuration: settings.LeaseDuration,
|
||||
PollInterval: settings.PollInterval, RecoveryInterval: settings.RecoveryInterval,
|
||||
RetryDelay: settings.RetryDelay,
|
||||
RetryDelay: settings.RetryDelay, Retention: settings.Retention,
|
||||
CleanupInterval: settings.CleanupInterval, CleanupBatchSize: settings.CleanupBatchSize,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -112,8 +113,8 @@ func RunWorker(logger *slog.Logger) error {
|
||||
logger, postgres.New(pool), broker, objects, processor, metrics,
|
||||
worker.Config{
|
||||
WorkerID: consumer, Concurrency: settings.Concurrency, LeaseDuration: settings.LeaseDuration,
|
||||
ProcessingTimeout: settings.ProcessingTimeout,
|
||||
ReclaimInterval: settings.ReclaimInterval, MaxAttempts: settings.MaxAttempts,
|
||||
HeartbeatInterval: settings.HeartbeatInterval, ProcessingTimeout: settings.ProcessingTimeout,
|
||||
ReclaimInterval: settings.ReclaimInterval, MaxAttempts: settings.MaxAttempts,
|
||||
RetryBase: settings.RetryBase, RetryMax: settings.RetryMax,
|
||||
MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels,
|
||||
MemoryBytes: settings.MemoryBytes,
|
||||
|
||||
+61
-51
@@ -98,14 +98,16 @@ type objectStorageConfig struct {
|
||||
}
|
||||
|
||||
func LoadServer() (ServerConfig, error) {
|
||||
values := viper.New()
|
||||
values.AutomaticEnv()
|
||||
|
||||
host, err := requiredString(values, "APP_HOST")
|
||||
values, err := loadValues()
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
port, err := requiredPort(values, "APP_PORT")
|
||||
|
||||
host, err := requiredString(values, "server.host")
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
port, err := requiredPort(values, "server.port")
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
@@ -113,71 +115,71 @@ func LoadServer() (ServerConfig, error) {
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
cacheURL, err := requiredString(values, "CACHE_URL")
|
||||
cacheURL, err := requiredString(values, "cache.url")
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
store, err := requiredString(values, "SESSION_STORE")
|
||||
store, err := requiredString(values, "server.session_store")
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
if store != "valkey" {
|
||||
return ServerConfig{}, errors.New("SESSION_STORE must be valkey")
|
||||
return ServerConfig{}, errors.New("server.session_store must be valkey")
|
||||
}
|
||||
|
||||
sessionTTL, err := seconds(values, "SESSION_TTL_SECONDS", 1, 86400*30)
|
||||
sessionTTL, err := duration(values, "server.session_ttl", time.Second, 30*24*time.Hour)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
idempotencyTTL, err := seconds(values, "IDEMPOTENCY_TTL_SECONDS", 86400, 86400*30)
|
||||
idempotencyTTL, err := duration(values, "server.idempotency_ttl", 24*time.Hour, 30*24*time.Hour)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
requestTimeout, err := seconds(values, "REQUEST_TIMEOUT_SECONDS", 1, 300)
|
||||
requestTimeout, err := duration(values, "server.request_timeout", time.Second, 5*time.Minute)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
readHeaderTimeout, err := seconds(values, "HTTP_READ_HEADER_TIMEOUT_SECONDS", 1, 300)
|
||||
readHeaderTimeout, err := duration(values, "server.http.read_header_timeout", time.Second, 5*time.Minute)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
readTimeout, err := seconds(values, "HTTP_READ_TIMEOUT_SECONDS", 1, 300)
|
||||
readTimeout, err := duration(values, "server.http.read_timeout", time.Second, 5*time.Minute)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
writeTimeout, err := seconds(values, "HTTP_WRITE_TIMEOUT_SECONDS", 1, 300)
|
||||
writeTimeout, err := duration(values, "server.http.write_timeout", time.Second, 5*time.Minute)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
idleTimeout, err := seconds(values, "HTTP_IDLE_TIMEOUT_SECONDS", 1, 3600)
|
||||
idleTimeout, err := duration(values, "server.http.idle_timeout", time.Second, time.Hour)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
shutdownTimeout, err := seconds(values, "HTTP_SHUTDOWN_TIMEOUT_SECONDS", 1, 300)
|
||||
shutdownTimeout, err := duration(values, "server.http.shutdown_timeout", time.Second, 5*time.Minute)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
passwordMemory, err := integer(values, "PASSWORD_MEMORY_KIB", 8*1024, 1024*1024)
|
||||
passwordMemory, err := integer(values, "server.password.memory_kib", 8*1024, 1024*1024)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
passwordIterations, err := integer(values, "PASSWORD_ITERATIONS", 1, 20)
|
||||
passwordIterations, err := integer(values, "server.password.iterations", 1, 20)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
passwordParallelism, err := integer(values, "PASSWORD_PARALLELISM", 1, 255)
|
||||
passwordParallelism, err := integer(values, "server.password.parallelism", 1, 255)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
passwordSaltLength, err := integer(values, "PASSWORD_SALT_LENGTH", 8, 1024)
|
||||
passwordSaltLength, err := integer(values, "server.password.salt_length", 8, 1024)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
passwordKeyLength, err := integer(values, "PASSWORD_KEY_LENGTH", 16, 1024)
|
||||
passwordKeyLength, err := integer(values, "server.password.key_length", 16, 1024)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
swaggerPort, err := requiredPort(values, "SWAGGER_PORT")
|
||||
swaggerPort, err := requiredPort(values, "server.swagger_port")
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
@@ -185,30 +187,30 @@ func LoadServer() (ServerConfig, error) {
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
attachmentMaxBytes, err := integer(values, "ATTACHMENT_MAX_BYTES", 1, 10*1024*1024)
|
||||
attachmentMaxBytes, err := integer(values, "attachments.max_bytes", 1, 10*1024*1024)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
attachmentMaxPixels, err := integer(values, "ATTACHMENT_MAX_PIXELS", 1, 40_000_000)
|
||||
attachmentMaxPixels, err := integer(values, "attachments.max_pixels", 1, 40_000_000)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
attachmentBodyMemory, err := integer(values, "ATTACHMENT_BODY_MEMORY_BYTES", 1, 1024*1024*1024)
|
||||
attachmentBodyMemory, err := integer(values, "attachments.body_memory_bytes", 1, 1024*1024*1024)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
if int64(attachmentBodyMemory) < int64(attachmentMaxBytes) {
|
||||
return ServerConfig{}, fmt.Errorf("ATTACHMENT_BODY_MEMORY_BYTES: must be at least %d", attachmentMaxBytes)
|
||||
return ServerConfig{}, fmt.Errorf("attachments.body_memory_bytes: must be at least %d", attachmentMaxBytes)
|
||||
}
|
||||
attachmentDecodeMemory, err := integer(values, "ATTACHMENT_DECODE_MEMORY_BYTES", 1, 1024*1024*1024)
|
||||
attachmentDecodeMemory, err := integer(values, "attachments.decode_memory_bytes", 1, 1024*1024*1024)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
minimumDecodeMemory := int64(attachmentMaxPixels) * 8
|
||||
if int64(attachmentDecodeMemory) < minimumDecodeMemory {
|
||||
return ServerConfig{}, fmt.Errorf("ATTACHMENT_DECODE_MEMORY_BYTES: must be at least %d", minimumDecodeMemory)
|
||||
return ServerConfig{}, fmt.Errorf("attachments.decode_memory_bytes: must be at least %d", minimumDecodeMemory)
|
||||
}
|
||||
pipelineVersion, err := integer(values, "PIPELINE_VERSION", 1, 1000)
|
||||
pipelineVersion, err := integer(values, "pipeline.version", 1, 1000)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
@@ -236,13 +238,15 @@ func LoadServer() (ServerConfig, error) {
|
||||
}
|
||||
|
||||
func LoadMigration() (MigrationConfig, error) {
|
||||
values := viper.New()
|
||||
values.AutomaticEnv()
|
||||
values, err := loadValues()
|
||||
if err != nil {
|
||||
return MigrationConfig{}, err
|
||||
}
|
||||
database, err := loadDatabaseConfig(values)
|
||||
if err != nil {
|
||||
return MigrationConfig{}, err
|
||||
}
|
||||
migrationTimeout, err := seconds(values, "MIGRATION_TIMEOUT_SECONDS", 1, 600)
|
||||
migrationTimeout, err := duration(values, "migration.timeout", time.Second, 10*time.Minute)
|
||||
if err != nil {
|
||||
return MigrationConfig{}, err
|
||||
}
|
||||
@@ -255,8 +259,10 @@ func LoadMigration() (MigrationConfig, error) {
|
||||
}
|
||||
|
||||
func LoadReconciliation() (ReconciliationConfig, error) {
|
||||
values := viper.New()
|
||||
values.AutomaticEnv()
|
||||
values, err := loadValues()
|
||||
if err != nil {
|
||||
return ReconciliationConfig{}, err
|
||||
}
|
||||
database, err := loadDatabaseConfig(values)
|
||||
if err != nil {
|
||||
return ReconciliationConfig{}, err
|
||||
@@ -276,31 +282,31 @@ func LoadReconciliation() (ReconciliationConfig, error) {
|
||||
}
|
||||
|
||||
func loadObjectStorageConfig(values *viper.Viper) (objectStorageConfig, error) {
|
||||
endpoint, err := requiredString(values, "S3_ENDPOINT")
|
||||
endpoint, err := requiredString(values, "object_storage.endpoint")
|
||||
if err != nil {
|
||||
return objectStorageConfig{}, err
|
||||
}
|
||||
region, err := requiredString(values, "S3_REGION")
|
||||
region, err := requiredString(values, "object_storage.region")
|
||||
if err != nil {
|
||||
return objectStorageConfig{}, err
|
||||
}
|
||||
bucket, err := requiredString(values, "S3_BUCKET")
|
||||
bucket, err := requiredString(values, "object_storage.bucket")
|
||||
if err != nil {
|
||||
return objectStorageConfig{}, err
|
||||
}
|
||||
accessKey, err := requiredString(values, "S3_ACCESS_KEY")
|
||||
accessKey, err := requiredString(values, "object_storage.access_key")
|
||||
if err != nil {
|
||||
return objectStorageConfig{}, err
|
||||
}
|
||||
secretKey, err := requiredString(values, "S3_SECRET_KEY")
|
||||
secretKey, err := requiredString(values, "object_storage.secret_key")
|
||||
if err != nil {
|
||||
return objectStorageConfig{}, err
|
||||
}
|
||||
useTLS, err := boolean(values, "S3_USE_TLS")
|
||||
useTLS, err := boolean(values, "object_storage.use_tls")
|
||||
if err != nil {
|
||||
return objectStorageConfig{}, err
|
||||
}
|
||||
timeout, err := seconds(values, "S3_TIMEOUT_SECONDS", 1, 300)
|
||||
timeout, err := duration(values, "object_storage.timeout", time.Second, 5*time.Minute)
|
||||
if err != nil {
|
||||
return objectStorageConfig{}, err
|
||||
}
|
||||
@@ -311,31 +317,31 @@ func loadObjectStorageConfig(values *viper.Viper) (objectStorageConfig, error) {
|
||||
}
|
||||
|
||||
func loadDatabaseConfig(values *viper.Viper) (databaseConfig, error) {
|
||||
databaseURL, err := requiredString(values, "DATABASE_URL")
|
||||
databaseURL, err := requiredString(values, "database.url")
|
||||
if err != nil {
|
||||
return databaseConfig{}, err
|
||||
}
|
||||
maxConns, err := integer(values, "DB_MAX_CONNS", 1, 100)
|
||||
maxConns, err := integer(values, "database.max_connections", 1, 100)
|
||||
if err != nil {
|
||||
return databaseConfig{}, err
|
||||
}
|
||||
minConns, err := integer(values, "DB_MIN_CONNS", 0, maxConns)
|
||||
minConns, err := integer(values, "database.min_connections", 0, maxConns)
|
||||
if err != nil {
|
||||
return databaseConfig{}, err
|
||||
}
|
||||
maxConnectionAge, err := seconds(values, "DB_MAX_CONNECTION_AGE_SECONDS", 1, 86400)
|
||||
maxConnectionAge, err := duration(values, "database.max_connection_age", time.Second, 24*time.Hour)
|
||||
if err != nil {
|
||||
return databaseConfig{}, err
|
||||
}
|
||||
maxConnectionIdle, err := seconds(values, "DB_MAX_CONNECTION_IDLE_SECONDS", 1, 86400)
|
||||
maxConnectionIdle, err := duration(values, "database.max_connection_idle", time.Second, 24*time.Hour)
|
||||
if err != nil {
|
||||
return databaseConfig{}, err
|
||||
}
|
||||
healthCheckPeriod, err := seconds(values, "DB_HEALTH_CHECK_SECONDS", 1, 3600)
|
||||
healthCheckPeriod, err := duration(values, "database.health_check_period", time.Second, time.Hour)
|
||||
if err != nil {
|
||||
return databaseConfig{}, err
|
||||
}
|
||||
dependencyTimeout, err := seconds(values, "DEPENDENCY_TIMEOUT_SECONDS", 1, 60)
|
||||
dependencyTimeout, err := duration(values, "database.dependency_timeout", time.Second, time.Minute)
|
||||
if err != nil {
|
||||
return databaseConfig{}, err
|
||||
}
|
||||
@@ -365,12 +371,16 @@ func requiredPort(values *viper.Viper, name string) (string, error) {
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func seconds(values *viper.Viper, name string, minimum, maximum int) (time.Duration, error) {
|
||||
value, err := integer(values, name, minimum, maximum)
|
||||
func duration(values *viper.Viper, name string, minimum, maximum time.Duration) (time.Duration, error) {
|
||||
value, err := requiredString(values, name)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return time.Duration(value) * time.Second, nil
|
||||
parsed, err := time.ParseDuration(value)
|
||||
if err != nil || parsed < minimum || parsed > maximum {
|
||||
return 0, fmt.Errorf("%s: must be between %s and %s", name, minimum, maximum)
|
||||
}
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
func integer(values *viper.Viper, name string, minimum, maximum int) (int, error) {
|
||||
|
||||
+114
-12
@@ -1,24 +1,17 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestLoadReconciliationDoesNotRequireServerSettings(t *testing.T) {
|
||||
values := map[string]string{
|
||||
"DATABASE_URL": "postgres://user:password@postgres:5432/messenger",
|
||||
"DB_MIN_CONNS": "0", "DB_MAX_CONNS": "4",
|
||||
"DB_MAX_CONNECTION_AGE_SECONDS": "300", "DB_MAX_CONNECTION_IDLE_SECONDS": "60",
|
||||
"DB_HEALTH_CHECK_SECONDS": "30", "DEPENDENCY_TIMEOUT_SECONDS": "5",
|
||||
"S3_ENDPOINT": "s3:9000", "S3_REGION": "us-east-1", "S3_BUCKET": "attachments",
|
||||
"S3_ACCESS_KEY": "application", "S3_SECRET_KEY": "application-secret",
|
||||
"S3_USE_TLS": "false", "S3_TIMEOUT_SECONDS": "5",
|
||||
}
|
||||
for name, value := range values {
|
||||
t.Setenv(name, value)
|
||||
}
|
||||
setConfigEnvironment(t, validTestConfig)
|
||||
t.Setenv("APP_HOST", "")
|
||||
t.Setenv("CACHE_URL", "")
|
||||
t.Setenv("SWAGGER_PORT", "")
|
||||
@@ -27,3 +20,112 @@ func TestLoadReconciliationDoesNotRequireServerSettings(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "attachments", settings.S3Bucket)
|
||||
}
|
||||
|
||||
func TestLoadWorkerUsesYAMLForRuntimeSettings(t *testing.T) {
|
||||
setConfigEnvironment(t, validTestConfig)
|
||||
t.Setenv("WORKER_CONCURRENCY", "64")
|
||||
|
||||
settings, err := LoadWorker()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 2, settings.Concurrency)
|
||||
require.Equal(t, 5*time.Second, settings.HeartbeatInterval)
|
||||
}
|
||||
|
||||
func TestLoadWorkerRejectsHeartbeatAtLeaseBoundary(t *testing.T) {
|
||||
content := strings.Replace(validTestConfig, "heartbeat_interval: 5s", "heartbeat_interval: 30s", 1)
|
||||
setConfigEnvironment(t, content)
|
||||
|
||||
_, err := LoadWorker()
|
||||
require.ErrorContains(t, err, "heartbeat_interval must be less than worker.lease")
|
||||
}
|
||||
|
||||
func TestLoadValuesRejectsUnknownVersion(t *testing.T) {
|
||||
setConfigEnvironment(t, "version: 2\n")
|
||||
|
||||
_, err := loadValues()
|
||||
require.ErrorContains(t, err, "version: must be 1")
|
||||
}
|
||||
|
||||
func setConfigEnvironment(t *testing.T, content string) {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "messenger.yaml")
|
||||
require.NoError(t, os.WriteFile(path, []byte(content), 0o600))
|
||||
t.Setenv("MESSENGER_CONFIG_PATH", path)
|
||||
for name, value := range map[string]string{
|
||||
"APP_HOST": "0.0.0.0", "APP_PORT": "8080", "SWAGGER_PORT": "8081",
|
||||
"DATABASE_URL": "postgres://user:password@postgres:5432/messenger",
|
||||
"CACHE_URL": "redis://valkey:6379/0", "BROKER_URL": "redis://redis-broker:6379/0",
|
||||
"S3_ENDPOINT": "s3:9000", "S3_REGION": "us-east-1", "S3_BUCKET": "attachments",
|
||||
"S3_ACCESS_KEY": "application", "S3_SECRET_KEY": "application-secret",
|
||||
} {
|
||||
t.Setenv(name, value)
|
||||
}
|
||||
}
|
||||
|
||||
const validTestConfig = `version: 1
|
||||
server:
|
||||
session_store: valkey
|
||||
session_ttl: 1h
|
||||
idempotency_ttl: 24h
|
||||
request_timeout: 5s
|
||||
http:
|
||||
read_header_timeout: 5s
|
||||
read_timeout: 10s
|
||||
write_timeout: 10s
|
||||
idle_timeout: 1m
|
||||
shutdown_timeout: 5s
|
||||
password:
|
||||
memory_kib: 65536
|
||||
iterations: 3
|
||||
parallelism: 2
|
||||
salt_length: 16
|
||||
key_length: 32
|
||||
database:
|
||||
min_connections: 0
|
||||
max_connections: 4
|
||||
max_connection_age: 5m
|
||||
max_connection_idle: 1m
|
||||
health_check_period: 30s
|
||||
dependency_timeout: 5s
|
||||
migration:
|
||||
timeout: 30s
|
||||
object_storage:
|
||||
use_tls: false
|
||||
timeout: 5s
|
||||
attachments:
|
||||
max_bytes: 10485760
|
||||
max_pixels: 40000000
|
||||
body_memory_bytes: 67108864
|
||||
decode_memory_bytes: 402653184
|
||||
pipeline:
|
||||
version: 1
|
||||
broker:
|
||||
stream: attachment-jobs
|
||||
consumer_group: attachment-workers
|
||||
failed_stream: attachment-jobs-failed
|
||||
failed_stream_max_length: 10000
|
||||
outbox:
|
||||
batch_size: 32
|
||||
lease: 15s
|
||||
poll_interval: 1s
|
||||
recovery_interval: 10s
|
||||
retry_delay: 2s
|
||||
retention: 168h
|
||||
cleanup_interval: 1m
|
||||
cleanup_batch_size: 1000
|
||||
worker:
|
||||
concurrency: 2
|
||||
prefetch: 4
|
||||
lease: 30s
|
||||
heartbeat_interval: 5s
|
||||
processing_timeout: 20s
|
||||
read_block: 2s
|
||||
reclaim_interval: 5s
|
||||
max_attempts: 5
|
||||
retry:
|
||||
base: 1s
|
||||
max: 30s
|
||||
memory_bytes: 738197504
|
||||
monitoring:
|
||||
metrics_address: 0.0.0.0:9091
|
||||
`
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
const supportedConfigVersion = 1
|
||||
|
||||
var environmentKeys = map[string]string{
|
||||
"server.host": "APP_HOST",
|
||||
"server.port": "APP_PORT",
|
||||
"server.swagger_port": "SWAGGER_PORT",
|
||||
"database.url": "DATABASE_URL",
|
||||
"cache.url": "CACHE_URL",
|
||||
"object_storage.endpoint": "S3_ENDPOINT",
|
||||
"object_storage.region": "S3_REGION",
|
||||
"object_storage.bucket": "S3_BUCKET",
|
||||
"object_storage.access_key": "S3_ACCESS_KEY",
|
||||
"object_storage.secret_key": "S3_SECRET_KEY",
|
||||
"broker.url": "BROKER_URL",
|
||||
}
|
||||
|
||||
func loadValues() (*viper.Viper, error) {
|
||||
path := strings.TrimSpace(os.Getenv("MESSENGER_CONFIG_PATH"))
|
||||
if path == "" {
|
||||
return nil, errors.New("MESSENGER_CONFIG_PATH is required")
|
||||
}
|
||||
values := viper.New()
|
||||
values.SetConfigFile(path)
|
||||
if err := values.ReadInConfig(); err != nil {
|
||||
return nil, fmt.Errorf("read messenger config: %w", err)
|
||||
}
|
||||
if version := values.GetInt("version"); version != supportedConfigVersion {
|
||||
return nil, fmt.Errorf("version: must be %d", supportedConfigVersion)
|
||||
}
|
||||
for key, name := range environmentKeys {
|
||||
if err := values.BindEnv(key, name); err != nil {
|
||||
return nil, fmt.Errorf("bind %s: %w", name, err)
|
||||
}
|
||||
}
|
||||
return values, nil
|
||||
}
|
||||
+72
-38
@@ -2,6 +2,7 @@ package config
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"time"
|
||||
|
||||
@@ -26,6 +27,9 @@ type DispatcherConfig struct {
|
||||
PollInterval time.Duration
|
||||
RecoveryInterval time.Duration
|
||||
RetryDelay time.Duration
|
||||
Retention time.Duration
|
||||
CleanupInterval time.Duration
|
||||
CleanupBatchSize int
|
||||
MetricsAddress string
|
||||
}
|
||||
|
||||
@@ -52,6 +56,7 @@ type WorkerConfig struct {
|
||||
Concurrency int
|
||||
Prefetch int
|
||||
LeaseDuration time.Duration
|
||||
HeartbeatInterval time.Duration
|
||||
ProcessingTimeout time.Duration
|
||||
ReadBlock time.Duration
|
||||
ReclaimInterval time.Duration
|
||||
@@ -83,8 +88,10 @@ type brokerConfig struct {
|
||||
}
|
||||
|
||||
func LoadJobAdmin() (JobAdminConfig, error) {
|
||||
values := viper.New()
|
||||
values.AutomaticEnv()
|
||||
values, err := loadValues()
|
||||
if err != nil {
|
||||
return JobAdminConfig{}, err
|
||||
}
|
||||
database, err := loadDatabaseConfig(values)
|
||||
if err != nil {
|
||||
return JobAdminConfig{}, err
|
||||
@@ -97,8 +104,10 @@ func LoadJobAdmin() (JobAdminConfig, error) {
|
||||
}
|
||||
|
||||
func LoadDispatcher() (DispatcherConfig, error) {
|
||||
values := viper.New()
|
||||
values.AutomaticEnv()
|
||||
values, err := loadValues()
|
||||
if err != nil {
|
||||
return DispatcherConfig{}, err
|
||||
}
|
||||
database, err := loadDatabaseConfig(values)
|
||||
if err != nil {
|
||||
return DispatcherConfig{}, err
|
||||
@@ -107,23 +116,35 @@ func LoadDispatcher() (DispatcherConfig, error) {
|
||||
if err != nil {
|
||||
return DispatcherConfig{}, err
|
||||
}
|
||||
batchSize, err := integer(values, "OUTBOX_BATCH_SIZE", 1, 1000)
|
||||
batchSize, err := integer(values, "outbox.batch_size", 1, 1000)
|
||||
if err != nil {
|
||||
return DispatcherConfig{}, err
|
||||
}
|
||||
leaseDuration, err := seconds(values, "OUTBOX_LEASE_SECONDS", 1, 600)
|
||||
leaseDuration, err := duration(values, "outbox.lease", time.Second, 10*time.Minute)
|
||||
if err != nil {
|
||||
return DispatcherConfig{}, err
|
||||
}
|
||||
pollInterval, err := seconds(values, "OUTBOX_POLL_SECONDS", 1, 60)
|
||||
pollInterval, err := duration(values, "outbox.poll_interval", time.Millisecond, time.Minute)
|
||||
if err != nil {
|
||||
return DispatcherConfig{}, err
|
||||
}
|
||||
recoveryInterval, err := seconds(values, "OUTBOX_RECOVERY_SECONDS", 1, 3600)
|
||||
recoveryInterval, err := duration(values, "outbox.recovery_interval", time.Second, time.Hour)
|
||||
if err != nil {
|
||||
return DispatcherConfig{}, err
|
||||
}
|
||||
retryDelay, err := seconds(values, "OUTBOX_RETRY_SECONDS", 1, 3600)
|
||||
retryDelay, err := duration(values, "outbox.retry_delay", time.Second, time.Hour)
|
||||
if err != nil {
|
||||
return DispatcherConfig{}, err
|
||||
}
|
||||
retention, err := duration(values, "outbox.retention", time.Hour, 365*24*time.Hour)
|
||||
if err != nil {
|
||||
return DispatcherConfig{}, err
|
||||
}
|
||||
cleanupInterval, err := duration(values, "outbox.cleanup_interval", time.Minute, 24*time.Hour)
|
||||
if err != nil {
|
||||
return DispatcherConfig{}, err
|
||||
}
|
||||
cleanupBatchSize, err := integer(values, "outbox.cleanup_batch_size", 1, 1000)
|
||||
if err != nil {
|
||||
return DispatcherConfig{}, err
|
||||
}
|
||||
@@ -138,13 +159,17 @@ func LoadDispatcher() (DispatcherConfig, error) {
|
||||
BrokerURL: broker.url, Stream: broker.stream, ConsumerGroup: broker.consumerGroup,
|
||||
FailedStream: broker.failedStream, FailedStreamMaxLength: int64(broker.failedStreamMaxLength),
|
||||
BatchSize: batchSize, LeaseDuration: leaseDuration, PollInterval: pollInterval,
|
||||
RecoveryInterval: recoveryInterval, RetryDelay: retryDelay, MetricsAddress: metricsAddress,
|
||||
RecoveryInterval: recoveryInterval, RetryDelay: retryDelay,
|
||||
Retention: retention, CleanupInterval: cleanupInterval, CleanupBatchSize: cleanupBatchSize,
|
||||
MetricsAddress: metricsAddress,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func LoadWorker() (WorkerConfig, error) {
|
||||
values := viper.New()
|
||||
values.AutomaticEnv()
|
||||
values, err := loadValues()
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
database, err := loadDatabaseConfig(values)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
@@ -157,57 +182,64 @@ func LoadWorker() (WorkerConfig, error) {
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
concurrency, err := integer(values, "WORKER_CONCURRENCY", 1, 64)
|
||||
concurrency, err := integer(values, "worker.concurrency", 1, 64)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
prefetch, err := integer(values, "WORKER_PREFETCH", concurrency, 1024)
|
||||
prefetch, err := integer(values, "worker.prefetch", concurrency, 1024)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
leaseDuration, err := seconds(values, "WORKER_LEASE_SECONDS", 1, 3600)
|
||||
leaseDuration, err := duration(values, "worker.lease", time.Second, time.Hour)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
processingTimeout, err := seconds(values, "WORKER_PROCESSING_TIMEOUT_SECONDS", 1, 3600)
|
||||
heartbeatInterval, err := duration(values, "worker.heartbeat_interval", 100*time.Millisecond, time.Hour)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
if heartbeatInterval >= leaseDuration {
|
||||
return WorkerConfig{}, errors.New("worker.heartbeat_interval must be less than worker.lease")
|
||||
}
|
||||
processingTimeout, err := duration(values, "worker.processing_timeout", time.Second, time.Hour)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
if processingTimeout >= leaseDuration {
|
||||
return WorkerConfig{}, errors.New("WORKER_PROCESSING_TIMEOUT_SECONDS must be less than WORKER_LEASE_SECONDS")
|
||||
return WorkerConfig{}, errors.New("worker.processing_timeout must be less than worker.lease")
|
||||
}
|
||||
readBlock, err := seconds(values, "WORKER_READ_BLOCK_SECONDS", 1, 60)
|
||||
readBlock, err := duration(values, "worker.read_block", 100*time.Millisecond, time.Minute)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
reclaimInterval, err := seconds(values, "WORKER_RECLAIM_SECONDS", 1, 3600)
|
||||
reclaimInterval, err := duration(values, "worker.reclaim_interval", time.Second, time.Hour)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
maxAttempts, err := integer(values, "WORKER_MAX_ATTEMPTS", 1, 100)
|
||||
maxAttempts, err := integer(values, "worker.max_attempts", 1, 100)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
retryBase, err := seconds(values, "WORKER_RETRY_BASE_SECONDS", 1, 3600)
|
||||
retryBase, err := duration(values, "worker.retry.base", time.Second, time.Hour)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
retryMax, err := seconds(values, "WORKER_RETRY_MAX_SECONDS", 1, 86400)
|
||||
retryMax, err := duration(values, "worker.retry.max", time.Second, 24*time.Hour)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
if retryMax < retryBase {
|
||||
return WorkerConfig{}, errors.New("WORKER_RETRY_MAX_SECONDS must be at least WORKER_RETRY_BASE_SECONDS")
|
||||
return WorkerConfig{}, errors.New("worker.retry.max must be at least worker.retry.base")
|
||||
}
|
||||
attachmentMaxBytes, err := integer(values, "ATTACHMENT_MAX_BYTES", 1, 10*1024*1024)
|
||||
attachmentMaxBytes, err := integer(values, "attachments.max_bytes", 1, 10*1024*1024)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
attachmentMaxPixels, err := integer(values, "ATTACHMENT_MAX_PIXELS", 1, 40_000_000)
|
||||
attachmentMaxPixels, err := integer(values, "attachments.max_pixels", 1, 40_000_000)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
memoryBytes, err := integer(values, "WORKER_MEMORY_BYTES", attachmentMaxBytes, 2*1024*1024*1024)
|
||||
memoryBytes, err := integer(values, "worker.memory_bytes", attachmentMaxBytes, 2*1024*1024*1024)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
@@ -225,44 +257,46 @@ func LoadWorker() (WorkerConfig, error) {
|
||||
ConsumerGroup: broker.consumerGroup, FailedStream: broker.failedStream,
|
||||
FailedStreamMaxLength: int64(broker.failedStreamMaxLength),
|
||||
Concurrency: concurrency, Prefetch: prefetch, LeaseDuration: leaseDuration,
|
||||
ProcessingTimeout: processingTimeout, ReadBlock: readBlock,
|
||||
HeartbeatInterval: heartbeatInterval, ProcessingTimeout: processingTimeout, ReadBlock: readBlock,
|
||||
ReclaimInterval: reclaimInterval, MaxAttempts: maxAttempts, RetryBase: retryBase, RetryMax: retryMax,
|
||||
AttachmentMaxBytes: int64(attachmentMaxBytes), AttachmentMaxPixels: int64(attachmentMaxPixels),
|
||||
MemoryBytes: int64(memoryBytes),
|
||||
MetricsAddress: metricsAddress,
|
||||
MemoryBytes: int64(memoryBytes), MetricsAddress: metricsAddress,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func metricsAddress(values *viper.Viper) (string, error) {
|
||||
host, err := requiredString(values, "PIPELINE_METRICS_HOST")
|
||||
address, err := requiredString(values, "monitoring.metrics_address")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
port, err := requiredPort(values, "PIPELINE_METRICS_PORT")
|
||||
_, port, err := net.SplitHostPort(address)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return "", fmt.Errorf("monitoring.metrics_address: %w", err)
|
||||
}
|
||||
return net.JoinHostPort(host, port), nil
|
||||
if _, err := integerValue(port, 1, 65535); err != nil {
|
||||
return "", fmt.Errorf("monitoring.metrics_address: %w", err)
|
||||
}
|
||||
return address, nil
|
||||
}
|
||||
|
||||
func loadBrokerConfig(values *viper.Viper) (brokerConfig, error) {
|
||||
url, err := requiredString(values, "BROKER_URL")
|
||||
url, err := requiredString(values, "broker.url")
|
||||
if err != nil {
|
||||
return brokerConfig{}, err
|
||||
}
|
||||
stream, err := requiredString(values, "BROKER_STREAM")
|
||||
stream, err := requiredString(values, "broker.stream")
|
||||
if err != nil {
|
||||
return brokerConfig{}, err
|
||||
}
|
||||
consumerGroup, err := requiredString(values, "BROKER_CONSUMER_GROUP")
|
||||
consumerGroup, err := requiredString(values, "broker.consumer_group")
|
||||
if err != nil {
|
||||
return brokerConfig{}, err
|
||||
}
|
||||
failedStream, err := requiredString(values, "BROKER_FAILED_STREAM")
|
||||
failedStream, err := requiredString(values, "broker.failed_stream")
|
||||
if err != nil {
|
||||
return brokerConfig{}, err
|
||||
}
|
||||
failedStreamMaxLength, err := integer(values, "BROKER_FAILED_MAX_LENGTH", 100, 10_000_000)
|
||||
failedStreamMaxLength, err := integer(values, "broker.failed_stream_max_length", 100, 10_000_000)
|
||||
if err != nil {
|
||||
return brokerConfig{}, err
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package transform
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"image"
|
||||
@@ -9,11 +10,14 @@ import (
|
||||
|
||||
var errInvalidEXIF = errors.New("invalid EXIF metadata")
|
||||
|
||||
func jpegOrientation(data []byte) (int, error) {
|
||||
func jpegOrientation(ctx context.Context, data []byte) (int, error) {
|
||||
if len(data) < 2 || data[0] != 0xff || data[1] != 0xd8 {
|
||||
return 0, errInvalidEXIF
|
||||
}
|
||||
for offset := 2; offset < len(data); {
|
||||
if err := contextError(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if data[offset] != 0xff {
|
||||
return 0, errInvalidEXIF
|
||||
}
|
||||
@@ -43,12 +47,12 @@ func jpegOrientation(data []byte) (int, error) {
|
||||
if marker != 0xe1 || len(payload) < 6 || string(payload[:6]) != "Exif\x00\x00" {
|
||||
continue
|
||||
}
|
||||
return tiffOrientation(payload[6:])
|
||||
return tiffOrientation(ctx, payload[6:])
|
||||
}
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
func tiffOrientation(data []byte) (int, error) {
|
||||
func tiffOrientation(ctx context.Context, data []byte) (int, error) {
|
||||
if len(data) < 8 {
|
||||
return 0, errInvalidEXIF
|
||||
}
|
||||
@@ -74,6 +78,9 @@ func tiffOrientation(data []byte) (int, error) {
|
||||
return 0, errInvalidEXIF
|
||||
}
|
||||
for index := uint64(0); index < count; index++ {
|
||||
if err := contextError(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
entry := data[entriesStart+index*12 : entriesStart+(index+1)*12]
|
||||
if order.Uint16(entry[:2]) != 0x0112 {
|
||||
continue
|
||||
@@ -90,7 +97,7 @@ func tiffOrientation(data []byte) (int, error) {
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
func applyOrientation(source image.Image, orientation int) *image.NRGBA {
|
||||
func applyOrientation(ctx context.Context, source image.Image, orientation int) (*image.NRGBA, error) {
|
||||
bounds := source.Bounds()
|
||||
width, height := bounds.Dx(), bounds.Dy()
|
||||
outputWidth, outputHeight := width, height
|
||||
@@ -99,12 +106,20 @@ func applyOrientation(source image.Image, orientation int) *image.NRGBA {
|
||||
}
|
||||
output := image.NewNRGBA(image.Rect(0, 0, outputWidth, outputHeight))
|
||||
for y := 0; y < outputHeight; y++ {
|
||||
if err := contextError(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for x := 0; x < outputWidth; x++ {
|
||||
if x&255 == 0 {
|
||||
if err := contextError(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
sourceX, sourceY := orientedSourcePoint(x, y, width, height, orientation)
|
||||
output.Set(x, y, source.At(bounds.Min.X+sourceX, bounds.Min.Y+sourceY))
|
||||
}
|
||||
}
|
||||
return output
|
||||
return output, nil
|
||||
}
|
||||
|
||||
func orientedSourcePoint(x, y, width, height, orientation int) (int, int) {
|
||||
@@ -128,11 +143,19 @@ func orientedSourcePoint(x, y, width, height, orientation int) (int, int) {
|
||||
}
|
||||
}
|
||||
|
||||
func flattenWhite(source image.Image) *image.NRGBA {
|
||||
func flattenWhite(ctx context.Context, source image.Image) (*image.NRGBA, error) {
|
||||
bounds := source.Bounds()
|
||||
output := image.NewNRGBA(image.Rect(0, 0, bounds.Dx(), bounds.Dy()))
|
||||
for y := 0; y < bounds.Dy(); y++ {
|
||||
if err := contextError(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for x := 0; x < bounds.Dx(); x++ {
|
||||
if x&255 == 0 {
|
||||
if err := contextError(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
r, g, b, a := source.At(bounds.Min.X+x, bounds.Min.Y+y).RGBA()
|
||||
output.SetNRGBA(x, y, color.NRGBA{
|
||||
R: uint8((r + 0xffff - a) / 0x101),
|
||||
@@ -142,5 +165,5 @@ func flattenWhite(source image.Image) *image.NRGBA {
|
||||
})
|
||||
}
|
||||
}
|
||||
return output
|
||||
return output, nil
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"image"
|
||||
"image/jpeg"
|
||||
_ "image/png"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
|
||||
@@ -78,7 +79,7 @@ func (p *Processor) Process(ctx context.Context, data []byte, declaredContentTyp
|
||||
if err := contextError(ctx); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
img, format, width, height, orientation, err := p.decode(data, declaredContentType)
|
||||
img, format, width, height, orientation, err := p.decode(ctx, data, declaredContentType)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
@@ -86,12 +87,12 @@ func (p *Processor) Process(ctx context.Context, data []byte, declaredContentTyp
|
||||
return Result{}, err
|
||||
}
|
||||
|
||||
oriented := applyOrientation(img, orientation)
|
||||
if err := contextError(ctx); err != nil {
|
||||
oriented, err := applyOrientation(ctx, img, orientation)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
opaque := flattenWhite(oriented)
|
||||
if err := contextError(ctx); err != nil {
|
||||
opaque, err := flattenWhite(ctx, oriented)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
result := Result{
|
||||
@@ -99,7 +100,10 @@ func (p *Processor) Process(ctx context.Context, data []byte, declaredContentTyp
|
||||
Orientation: orientation,
|
||||
}
|
||||
|
||||
thumbnail := fit(opaque, 256, 256, false)
|
||||
thumbnail, err := fitContext(ctx, opaque, 256, 256, false)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
result.Thumbnail, err = encodeJPEG(thumbnail, p.config.JPEGQuality)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
@@ -109,7 +113,10 @@ func (p *Processor) Process(ctx context.Context, data []byte, declaredContentTyp
|
||||
}
|
||||
|
||||
crop := centeredSquare(opaque)
|
||||
crop = resize(crop, 128, 128)
|
||||
crop, err = resizeContext(ctx, crop, 128, 128)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
result.Crop, err = encodeJPEG(crop, p.config.JPEGQuality)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
@@ -118,7 +125,10 @@ func (p *Processor) Process(ctx context.Context, data []byte, declaredContentTyp
|
||||
return Result{}, err
|
||||
}
|
||||
|
||||
optimized := fit(opaque, 1280, 1280, false)
|
||||
optimized, err := fitContext(ctx, opaque, 1280, 1280, false)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
result.Optimized, err = encodeWebP(optimized, p.config.WebPQuality, p.config.WebPMethod)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
@@ -129,7 +139,11 @@ func (p *Processor) Process(ctx context.Context, data []byte, declaredContentTyp
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (p *Processor) decode(data []byte, declaredContentType string) (image.Image, string, int, int, int, error) {
|
||||
func (p *Processor) decode(
|
||||
ctx context.Context,
|
||||
data []byte,
|
||||
declaredContentType string,
|
||||
) (image.Image, string, int, int, int, error) {
|
||||
if declaredContentType != "image/jpeg" && declaredContentType != "image/png" {
|
||||
return nil, "", 0, 0, 0, ErrUnsupportedMedia
|
||||
}
|
||||
@@ -142,27 +156,48 @@ func (p *Processor) decode(data []byte, declaredContentType string) (image.Image
|
||||
if http.DetectContentType(data) != declaredContentType {
|
||||
return nil, "", 0, 0, 0, ErrUnsupportedMedia
|
||||
}
|
||||
config, format, err := image.DecodeConfig(bytes.NewReader(data))
|
||||
config, format, err := image.DecodeConfig(contextReader{ctx: ctx, reader: bytes.NewReader(data)})
|
||||
if contextErr := contextError(ctx); contextErr != nil {
|
||||
return nil, "", 0, 0, 0, contextErr
|
||||
}
|
||||
if err != nil || contentType(format) != declaredContentType || config.Width < 1 || config.Height < 1 {
|
||||
return nil, "", 0, 0, 0, ErrInvalidImage
|
||||
}
|
||||
if int64(config.Width) > p.config.MaxPixels/int64(config.Height) {
|
||||
return nil, "", 0, 0, 0, ErrPayloadTooLarge
|
||||
}
|
||||
img, decodedFormat, err := image.Decode(bytes.NewReader(data))
|
||||
img, decodedFormat, err := image.Decode(contextReader{ctx: ctx, reader: bytes.NewReader(data)})
|
||||
if contextErr := contextError(ctx); contextErr != nil {
|
||||
return nil, "", 0, 0, 0, contextErr
|
||||
}
|
||||
if err != nil || decodedFormat != format {
|
||||
return nil, "", 0, 0, 0, ErrInvalidImage
|
||||
}
|
||||
orientation := 1
|
||||
if format == "jpeg" {
|
||||
orientation, err = jpegOrientation(data)
|
||||
orientation, err = jpegOrientation(ctx, data)
|
||||
if err != nil {
|
||||
if contextErr := contextError(ctx); contextErr != nil {
|
||||
return nil, "", 0, 0, 0, contextErr
|
||||
}
|
||||
return nil, "", 0, 0, 0, ErrInvalidImage
|
||||
}
|
||||
}
|
||||
return img, declaredContentType, config.Width, config.Height, orientation, nil
|
||||
}
|
||||
|
||||
type contextReader struct {
|
||||
ctx context.Context
|
||||
reader io.Reader
|
||||
}
|
||||
|
||||
func (r contextReader) Read(buffer []byte) (int, error) {
|
||||
if err := contextError(r.ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return r.reader.Read(buffer)
|
||||
}
|
||||
|
||||
func contextError(ctx context.Context) error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
|
||||
@@ -64,7 +64,10 @@ func TestFlattenWhiteCompositesAlpha(t *testing.T) {
|
||||
source.SetNRGBA(0, 0, color.NRGBA{R: 10, G: 20, B: 30, A: 0})
|
||||
source.SetNRGBA(1, 0, color.NRGBA{R: 100, G: 50, B: 0, A: 128})
|
||||
|
||||
result := flattenWhite(source)
|
||||
result, err := flattenWhite(context.Background(), source)
|
||||
if err != nil {
|
||||
t.Fatalf("flatten image: %v", err)
|
||||
}
|
||||
if got := result.NRGBAAt(0, 0); got != (color.NRGBA{R: 255, G: 255, B: 255, A: 255}) {
|
||||
t.Fatalf("transparent pixel: got %#v", got)
|
||||
}
|
||||
@@ -154,6 +157,28 @@ func TestProcessHonorsCanceledContext(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResizeStopsAfterCancellation(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
source := cancelingImage{
|
||||
Image: image.NewNRGBA(image.Rect(0, 0, 256, 256)),
|
||||
cancel: cancel,
|
||||
}
|
||||
_, err := resizeContext(ctx, source, 128, 128)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("error: got %v, want %v", err, context.Canceled)
|
||||
}
|
||||
}
|
||||
|
||||
type cancelingImage struct {
|
||||
image.Image
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func (i cancelingImage) At(x, y int) color.Color {
|
||||
i.cancel()
|
||||
return i.Image.At(x, y)
|
||||
}
|
||||
|
||||
func TestEstimateMemoryWeight(t *testing.T) {
|
||||
weight, err := EstimateMemoryWeight(1000, 320, 200)
|
||||
if err != nil {
|
||||
|
||||
@@ -1,14 +1,21 @@
|
||||
package transform
|
||||
|
||||
import (
|
||||
"context"
|
||||
"image"
|
||||
"image/color"
|
||||
)
|
||||
|
||||
func fit(source image.Image, maxWidth, maxHeight int, upscale bool) *image.NRGBA {
|
||||
func fitContext(
|
||||
ctx context.Context,
|
||||
source image.Image,
|
||||
maxWidth int,
|
||||
maxHeight int,
|
||||
upscale bool,
|
||||
) (*image.NRGBA, error) {
|
||||
bounds := source.Bounds()
|
||||
width, height := fitDimensions(bounds.Dx(), bounds.Dy(), maxWidth, maxHeight, upscale)
|
||||
return resize(source, width, height)
|
||||
return resizeContext(ctx, source, width, height)
|
||||
}
|
||||
|
||||
func fitDimensions(width, height, maxWidth, maxHeight int, upscale bool) (int, int) {
|
||||
@@ -43,15 +50,23 @@ func (i croppedImage) At(x, y int) color.Color {
|
||||
return i.source.At(i.offset.X+x, i.offset.Y+y)
|
||||
}
|
||||
|
||||
func resize(source image.Image, width, height int) *image.NRGBA {
|
||||
func resizeContext(ctx context.Context, source image.Image, width, height int) (*image.NRGBA, error) {
|
||||
bounds := source.Bounds()
|
||||
if bounds.Dx() == width && bounds.Dy() == height {
|
||||
return copyNRGBA(source)
|
||||
return copyNRGBAContext(ctx, source)
|
||||
}
|
||||
output := image.NewNRGBA(image.Rect(0, 0, width, height))
|
||||
for y := 0; y < height; y++ {
|
||||
if err := contextError(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
y0, y1, fy := sampleCoordinate(y, height, bounds.Dy())
|
||||
for x := 0; x < width; x++ {
|
||||
if x&255 == 0 {
|
||||
if err := contextError(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
x0, x1, fx := sampleCoordinate(x, width, bounds.Dx())
|
||||
c00 := color.NRGBAModel.Convert(source.At(bounds.Min.X+x0, bounds.Min.Y+y0)).(color.NRGBA)
|
||||
c10 := color.NRGBAModel.Convert(source.At(bounds.Min.X+x1, bounds.Min.Y+y0)).(color.NRGBA)
|
||||
@@ -60,7 +75,7 @@ func resize(source image.Image, width, height int) *image.NRGBA {
|
||||
output.SetNRGBA(x, y, interpolate(c00, c10, c01, c11, fx, fy))
|
||||
}
|
||||
}
|
||||
return output
|
||||
return output, nil
|
||||
}
|
||||
|
||||
func sampleCoordinate(destination, destinationSize, sourceSize int) (int, int, uint32) {
|
||||
@@ -93,13 +108,21 @@ func interpolateChannel(c00, c10, c01, c11 uint8, fx, fy uint32) uint8 {
|
||||
return uint8((value + 1<<31) >> 32)
|
||||
}
|
||||
|
||||
func copyNRGBA(source image.Image) *image.NRGBA {
|
||||
func copyNRGBAContext(ctx context.Context, source image.Image) (*image.NRGBA, error) {
|
||||
bounds := source.Bounds()
|
||||
output := image.NewNRGBA(image.Rect(0, 0, bounds.Dx(), bounds.Dy()))
|
||||
for y := 0; y < bounds.Dy(); y++ {
|
||||
if err := contextError(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for x := 0; x < bounds.Dx(); x++ {
|
||||
if x&255 == 0 {
|
||||
if err := contextError(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
output.Set(x, y, source.At(bounds.Min.X+x, bounds.Min.Y+y))
|
||||
}
|
||||
}
|
||||
return output
|
||||
return output, nil
|
||||
}
|
||||
|
||||
@@ -27,6 +27,7 @@ import (
|
||||
type Repository interface {
|
||||
AttachmentJob(context.Context, string) (model.AttachmentJob, error)
|
||||
ClaimAttachmentJob(context.Context, string, string, int, int, string, time.Duration) (model.AttachmentJobLease, error)
|
||||
RenewAttachmentJobLease(context.Context, model.AttachmentJobLease, time.Duration) (bool, error)
|
||||
CompleteAttachmentJob(context.Context, model.AttachmentJobLease, []model.Variant) (model.JobMutationResult, error)
|
||||
RetryAttachmentJob(context.Context, model.AttachmentJobLease, string, time.Time) (model.JobMutationResult, error)
|
||||
FailAttachmentJob(context.Context, model.AttachmentJobLease, string) (model.JobMutationResult, error)
|
||||
@@ -59,6 +60,7 @@ type Config struct {
|
||||
WorkerID string
|
||||
Concurrency int
|
||||
LeaseDuration time.Duration
|
||||
HeartbeatInterval time.Duration
|
||||
ProcessingTimeout time.Duration
|
||||
ReclaimInterval time.Duration
|
||||
MaxAttempts int
|
||||
@@ -119,6 +121,7 @@ func New(
|
||||
|
||||
func validateConfig(config Config) error {
|
||||
if config.WorkerID == "" || config.Concurrency < 1 || config.LeaseDuration <= 0 ||
|
||||
config.HeartbeatInterval <= 0 || config.HeartbeatInterval >= config.LeaseDuration ||
|
||||
config.ProcessingTimeout <= 0 || config.ProcessingTimeout >= config.LeaseDuration ||
|
||||
config.ReclaimInterval <= 0 || config.MaxAttempts < 1 || config.RetryBase <= 0 ||
|
||||
config.RetryMax < config.RetryBase || config.MaxBytes < 1 || config.MaxPixels < 1 || config.MemoryBytes < 1 {
|
||||
@@ -218,9 +221,17 @@ func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Deliv
|
||||
return ErrInjectedFailure
|
||||
}
|
||||
|
||||
processingCtx, cancel := context.WithTimeout(ctx, w.config.ProcessingTimeout)
|
||||
processingCtx, cancelProcessing := context.WithTimeout(ctx, w.config.ProcessingTimeout)
|
||||
heartbeatCtx, cancelHeartbeat := context.WithCancel(ctx)
|
||||
heartbeatDone := make(chan error, 1)
|
||||
go func() { heartbeatDone <- w.maintainLease(heartbeatCtx, cancelProcessing, lease) }()
|
||||
variants, err := w.process(processingCtx, lease)
|
||||
cancel()
|
||||
cancelProcessing()
|
||||
cancelHeartbeat()
|
||||
heartbeatErr := <-heartbeatDone
|
||||
if heartbeatErr != nil {
|
||||
return heartbeatErr
|
||||
}
|
||||
if err != nil {
|
||||
result, err = w.resolveProcessingFailure(ctx, delivery, lease, err)
|
||||
return err
|
||||
@@ -252,6 +263,34 @@ func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Deliv
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Worker) maintainLease(
|
||||
ctx context.Context,
|
||||
cancelProcessing context.CancelFunc,
|
||||
lease model.AttachmentJobLease,
|
||||
) error {
|
||||
ticker := time.NewTicker(w.config.HeartbeatInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-ticker.C:
|
||||
renewed, err := w.repository.RenewAttachmentJobLease(ctx, lease, w.config.LeaseDuration)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
cancelProcessing()
|
||||
return fmt.Errorf("renew attachment job lease: %w", err)
|
||||
}
|
||||
if !renewed {
|
||||
cancelProcessing()
|
||||
return model.ErrStaleLease
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) resolveClaim(
|
||||
ctx context.Context,
|
||||
delivery redisstream.Delivery,
|
||||
@@ -295,15 +334,18 @@ func (w *Worker) process(ctx context.Context, lease model.AttachmentJobLease) ([
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = body.Close() }()
|
||||
data, err := io.ReadAll(io.LimitReader(body, w.config.MaxBytes+1))
|
||||
data, err := io.ReadAll(io.LimitReader(contextReader{ctx: ctx, reader: body}, w.config.MaxBytes+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if int64(len(data)) > w.config.MaxBytes {
|
||||
return nil, transform.ErrPayloadTooLarge
|
||||
}
|
||||
imageConfig, _, err := image.DecodeConfig(bytes.NewReader(data))
|
||||
imageConfig, _, err := image.DecodeConfig(contextReader{ctx: ctx, reader: bytes.NewReader(data)})
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
return nil, transform.ErrInvalidImage
|
||||
}
|
||||
weight, err := transform.EstimateMemoryWeight(int64(len(data)), imageConfig.Width, imageConfig.Height)
|
||||
@@ -346,6 +388,20 @@ func (w *Worker) process(ctx context.Context, lease model.AttachmentJobLease) ([
|
||||
return result, nil
|
||||
}
|
||||
|
||||
type contextReader struct {
|
||||
ctx context.Context
|
||||
reader io.Reader
|
||||
}
|
||||
|
||||
func (r contextReader) Read(buffer []byte) (int, error) {
|
||||
select {
|
||||
case <-r.ctx.Done():
|
||||
return 0, r.ctx.Err()
|
||||
default:
|
||||
return r.reader.Read(buffer)
|
||||
}
|
||||
}
|
||||
|
||||
var errWorkerCapacity = errors.New("image exceeds worker memory budget")
|
||||
|
||||
func variantObjectKey(attachmentID string, pipelineVersion int, name string, value transform.Variant) string {
|
||||
|
||||
@@ -1,13 +1,16 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
|
||||
)
|
||||
|
||||
@@ -46,11 +49,61 @@ func TestProcessingErrorClassification(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLeaseHeartbeatRenewsUntilCanceled(t *testing.T) {
|
||||
var renewals atomic.Int32
|
||||
repository := &repositoryStub{renew: func(context.Context, model.AttachmentJobLease, time.Duration) (bool, error) {
|
||||
renewals.Add(1)
|
||||
return true, nil
|
||||
}}
|
||||
config := validConfig()
|
||||
config.HeartbeatInterval = 5 * time.Millisecond
|
||||
worker := &Worker{repository: repository, config: config}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- worker.maintainLease(ctx, cancel, testLease()) }()
|
||||
require.Eventually(t, func() bool { return renewals.Load() >= 2 }, time.Second, time.Millisecond)
|
||||
cancel()
|
||||
require.NoError(t, <-done)
|
||||
}
|
||||
|
||||
func TestLeaseHeartbeatCancelsProcessingWhenLeaseIsLost(t *testing.T) {
|
||||
repository := &repositoryStub{renew: func(context.Context, model.AttachmentJobLease, time.Duration) (bool, error) {
|
||||
return false, nil
|
||||
}}
|
||||
config := validConfig()
|
||||
config.HeartbeatInterval = time.Millisecond
|
||||
worker := &Worker{repository: repository, config: config}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
err := worker.maintainLease(ctx, cancel, testLease())
|
||||
require.ErrorIs(t, err, model.ErrStaleLease)
|
||||
require.ErrorIs(t, ctx.Err(), context.Canceled)
|
||||
}
|
||||
|
||||
type repositoryStub struct {
|
||||
Repository
|
||||
renew func(context.Context, model.AttachmentJobLease, time.Duration) (bool, error)
|
||||
}
|
||||
|
||||
func (s *repositoryStub) RenewAttachmentJobLease(
|
||||
ctx context.Context,
|
||||
lease model.AttachmentJobLease,
|
||||
duration time.Duration,
|
||||
) (bool, error) {
|
||||
return s.renew(ctx, lease, duration)
|
||||
}
|
||||
|
||||
func testLease() model.AttachmentJobLease {
|
||||
return model.AttachmentJobLease{Job: model.AttachmentJob{
|
||||
ID: "11111111-1111-4111-8111-111111111111", LeaseToken: "lease-token", LeaseOwner: "worker-1",
|
||||
}}
|
||||
}
|
||||
|
||||
func validConfig() Config {
|
||||
return Config{
|
||||
WorkerID: "worker-1", Concurrency: 2, LeaseDuration: 30 * time.Second,
|
||||
ProcessingTimeout: 20 * time.Second, ReclaimInterval: 5 * time.Second,
|
||||
MaxAttempts: 5, RetryBase: time.Second, RetryMax: 30 * time.Second,
|
||||
HeartbeatInterval: 5 * time.Second, ProcessingTimeout: 20 * time.Second,
|
||||
ReclaimInterval: 5 * time.Second,
|
||||
MaxAttempts: 5, RetryBase: time.Second, RetryMax: 30 * time.Second,
|
||||
MaxBytes: 10 * 1024 * 1024, MaxPixels: 40_000_000, MemoryBytes: 704 * 1024 * 1024,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ type Store interface {
|
||||
MarkOutboxPublished(context.Context, string, string) (bool, error)
|
||||
ReleaseOutboxEvent(context.Context, string, string, string, time.Time) (bool, error)
|
||||
RecoverExpiredOutboxEvents(context.Context, time.Time) (int, error)
|
||||
DeletePublishedOutboxEvents(context.Context, time.Time, int) (int, error)
|
||||
}
|
||||
|
||||
type Publisher interface {
|
||||
@@ -34,6 +35,9 @@ type Config struct {
|
||||
PollInterval time.Duration
|
||||
RecoveryInterval time.Duration
|
||||
RetryDelay time.Duration
|
||||
Retention time.Duration
|
||||
CleanupInterval time.Duration
|
||||
CleanupBatchSize int
|
||||
}
|
||||
|
||||
type Dispatcher struct {
|
||||
@@ -72,6 +76,12 @@ func (c Config) validate() error {
|
||||
return errors.New("outbox recovery interval must be positive")
|
||||
case c.RetryDelay <= 0:
|
||||
return errors.New("outbox retry delay must be positive")
|
||||
case c.Retention <= 0:
|
||||
return errors.New("outbox retention must be positive")
|
||||
case c.CleanupInterval <= 0:
|
||||
return errors.New("outbox cleanup interval must be positive")
|
||||
case c.CleanupBatchSize <= 0 || c.CleanupBatchSize > 1000:
|
||||
return errors.New("outbox cleanup batch size must be between 1 and 1000")
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
@@ -84,8 +94,10 @@ func (d *Dispatcher) Run(ctx context.Context) error {
|
||||
|
||||
pollTicker := time.NewTicker(d.config.PollInterval)
|
||||
recoveryTicker := time.NewTicker(d.config.RecoveryInterval)
|
||||
cleanupTicker := time.NewTicker(d.config.CleanupInterval)
|
||||
defer pollTicker.Stop()
|
||||
defer recoveryTicker.Stop()
|
||||
defer cleanupTicker.Stop()
|
||||
|
||||
if err := d.RecoverExpired(ctx); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
@@ -112,10 +124,27 @@ func (d *Dispatcher) Run(ctx context.Context) error {
|
||||
}
|
||||
return err
|
||||
}
|
||||
case <-cleanupTicker.C:
|
||||
if _, err := d.CleanupPublished(ctx); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Dispatcher) CleanupPublished(ctx context.Context) (int, error) {
|
||||
deleted, err := d.store.DeletePublishedOutboxEvents(
|
||||
ctx, d.now().Add(-d.config.Retention), d.config.CleanupBatchSize,
|
||||
)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("delete published outbox events: %w", err)
|
||||
}
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
func (d *Dispatcher) DispatchOnce(ctx context.Context) (int, error) {
|
||||
events, err := d.store.ClaimOutboxEvents(ctx, d.config.BatchSize, d.config.LeaseDuration)
|
||||
if err != nil {
|
||||
|
||||
@@ -70,6 +70,18 @@ func TestDispatcherRejectsStaleLeaseAfterPublish(t *testing.T) {
|
||||
require.Zero(t, published)
|
||||
}
|
||||
|
||||
func TestDispatcherCleansPublishedEventsBeforeRetentionCutoff(t *testing.T) {
|
||||
store := &storeStub{deleted: 2}
|
||||
dispatcher := newTestDispatcher(t, store, &publisherStub{})
|
||||
dispatcher.now = func() time.Time { return time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC) }
|
||||
|
||||
deleted, err := dispatcher.CleanupPublished(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 2, deleted)
|
||||
require.Equal(t, dispatcher.now().Add(-24*time.Hour), store.publishedBefore)
|
||||
require.Equal(t, 100, store.cleanupLimit)
|
||||
}
|
||||
|
||||
func TestDispatcherStopsAfterCancellation(t *testing.T) {
|
||||
store := &storeStub{}
|
||||
publisher := &publisherStub{}
|
||||
@@ -96,6 +108,9 @@ func newTestDispatcher(t *testing.T, store Store, publisher Publisher) *Dispatch
|
||||
PollInterval: time.Millisecond,
|
||||
RecoveryInterval: time.Second,
|
||||
RetryDelay: time.Second,
|
||||
Retention: 24 * time.Hour,
|
||||
CleanupInterval: time.Hour,
|
||||
CleanupBatchSize: 100,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
return dispatcher
|
||||
@@ -114,14 +129,17 @@ func testPayload() json.RawMessage {
|
||||
}
|
||||
|
||||
type storeStub struct {
|
||||
mu sync.Mutex
|
||||
events []model.OutboxEvent
|
||||
markErr error
|
||||
markLost bool
|
||||
marked []string
|
||||
released string
|
||||
nextAttemptAt time.Time
|
||||
recoveredCalls int
|
||||
mu sync.Mutex
|
||||
events []model.OutboxEvent
|
||||
markErr error
|
||||
markLost bool
|
||||
marked []string
|
||||
released string
|
||||
nextAttemptAt time.Time
|
||||
recoveredCalls int
|
||||
deleted int
|
||||
publishedBefore time.Time
|
||||
cleanupLimit int
|
||||
}
|
||||
|
||||
func (s *storeStub) ClaimOutboxEvents(context.Context, int, time.Duration) ([]model.OutboxEvent, error) {
|
||||
@@ -164,6 +182,14 @@ func (s *storeStub) RecoverExpiredOutboxEvents(context.Context, time.Time) (int,
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func (s *storeStub) DeletePublishedOutboxEvents(_ context.Context, before time.Time, limit int) (int, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.publishedBefore = before
|
||||
s.cleanupLimit = limit
|
||||
return s.deleted, nil
|
||||
}
|
||||
|
||||
type publisherStub struct {
|
||||
mu sync.Mutex
|
||||
messageID string
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
)
|
||||
|
||||
func (s *Store) RenewAttachmentJobLease(
|
||||
ctx context.Context,
|
||||
lease model.AttachmentJobLease,
|
||||
duration time.Duration,
|
||||
) (bool, error) {
|
||||
if duration <= 0 || lease.Job.ID == "" || lease.Job.LeaseToken == "" || lease.Job.LeaseOwner == "" {
|
||||
return false, model.ErrInvalidInput
|
||||
}
|
||||
const query = `
|
||||
UPDATE attachment_jobs
|
||||
SET lease_until = clock_timestamp() + ($4::bigint * interval '1 microsecond'),
|
||||
updated_at = clock_timestamp()
|
||||
WHERE id = $1 AND lease_token = $2 AND lease_owner = $3
|
||||
AND status = 'processing' AND lease_until > clock_timestamp()`
|
||||
result, err := s.pool.Exec(
|
||||
ctx, query, lease.Job.ID, lease.Job.LeaseToken, lease.Job.LeaseOwner, duration.Microseconds(),
|
||||
)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("renew attachment job lease: %w", err)
|
||||
}
|
||||
return result.RowsAffected() == 1, nil
|
||||
}
|
||||
@@ -111,3 +111,26 @@ func (s *Store) RecoverExpiredOutboxEvents(ctx context.Context, now time.Time) (
|
||||
}
|
||||
return int(result.RowsAffected()), nil
|
||||
}
|
||||
|
||||
func (s *Store) DeletePublishedOutboxEvents(ctx context.Context, publishedBefore time.Time, limit int) (int, error) {
|
||||
if publishedBefore.IsZero() || limit < 1 || limit > 1000 {
|
||||
return 0, model.ErrInvalidInput
|
||||
}
|
||||
const query = `
|
||||
WITH candidates AS (
|
||||
SELECT id
|
||||
FROM outbox_events
|
||||
WHERE published_at < $1
|
||||
ORDER BY published_at, id
|
||||
LIMIT $2
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
DELETE FROM outbox_events event
|
||||
USING candidates
|
||||
WHERE event.id = candidates.id`
|
||||
result, err := s.pool.Exec(ctx, query, publishedBefore.UTC(), limit)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("delete published outbox events: %w", err)
|
||||
}
|
||||
return int(result.RowsAffected()), nil
|
||||
}
|
||||
|
||||
+1
-57
@@ -1,6 +1,7 @@
|
||||
# Адрес и внутренний порт HTTP-сервера.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
MESSENGER_CONFIG_PATH=config/messenger.yaml
|
||||
|
||||
# Адреса публикации сервисов на локальной машине.
|
||||
API_BIND_ADDRESS=127.0.0.1:8080
|
||||
@@ -8,11 +9,6 @@ SWAGGER_BIND_ADDRESS=127.0.0.1:8081
|
||||
PROMETHEUS_BIND_ADDRESS=127.0.0.1:9090
|
||||
GRAFANA_BIND_ADDRESS=127.0.0.1:3000
|
||||
|
||||
# Время жизни сессии, идемпотентности и используемое хранилище.
|
||||
SESSION_TTL_SECONDS=3600
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
SESSION_STORE=valkey
|
||||
|
||||
# Число API-реплик за Nginx.
|
||||
API_REPLICAS=2
|
||||
|
||||
@@ -29,28 +25,6 @@ CACHE_URL=redis://default:local-valkey-password@valkey:6379/0
|
||||
# Redis Streams используется только как устойчивый брокер заданий.
|
||||
REDIS_BROKER_PASSWORD=local-redis-broker-password
|
||||
BROKER_URL=redis://default:local-redis-broker-password@redis-broker:6379/0
|
||||
BROKER_STREAM=attachment-jobs
|
||||
BROKER_CONSUMER_GROUP=attachment-workers
|
||||
BROKER_FAILED_STREAM=attachment-jobs-failed
|
||||
BROKER_FAILED_MAX_LENGTH=10000
|
||||
|
||||
# Версия pipeline, outbox и ограниченные повторы worker.
|
||||
PIPELINE_VERSION=1
|
||||
OUTBOX_BATCH_SIZE=32
|
||||
OUTBOX_LEASE_SECONDS=15
|
||||
OUTBOX_POLL_SECONDS=1
|
||||
OUTBOX_RECOVERY_SECONDS=10
|
||||
OUTBOX_RETRY_SECONDS=2
|
||||
WORKER_CONCURRENCY=2
|
||||
WORKER_PREFETCH=4
|
||||
WORKER_LEASE_SECONDS=30
|
||||
WORKER_PROCESSING_TIMEOUT_SECONDS=20
|
||||
WORKER_READ_BLOCK_SECONDS=2
|
||||
WORKER_RECLAIM_SECONDS=5
|
||||
WORKER_MAX_ATTEMPTS=5
|
||||
WORKER_RETRY_BASE_SECONDS=1
|
||||
WORKER_RETRY_MAX_SECONDS=30
|
||||
WORKER_MEMORY_BYTES=738197504
|
||||
|
||||
# Административные данные MinIO используются только при bootstrap.
|
||||
MINIO_ROOT_USER=local-minio-admin
|
||||
@@ -62,36 +36,6 @@ S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY=messenger-app
|
||||
S3_SECRET_KEY=local-messenger-s3-password
|
||||
S3_USE_TLS=false
|
||||
S3_TIMEOUT_SECONDS=5
|
||||
ATTACHMENT_MAX_BYTES=10485760
|
||||
ATTACHMENT_MAX_PIXELS=40000000
|
||||
ATTACHMENT_BODY_MEMORY_BYTES=67108864
|
||||
ATTACHMENT_DECODE_MEMORY_BYTES=402653184
|
||||
|
||||
# Размер пула и обслуживание соединений PostgreSQL.
|
||||
DB_MIN_CONNS=1
|
||||
DB_MAX_CONNS=10
|
||||
DB_MAX_CONNECTION_AGE_SECONDS=1800
|
||||
DB_MAX_CONNECTION_IDLE_SECONDS=300
|
||||
DB_HEALTH_CHECK_SECONDS=30
|
||||
|
||||
# Таймауты зависимостей, миграций и HTTP-запросов.
|
||||
DEPENDENCY_TIMEOUT_SECONDS=5
|
||||
MIGRATION_TIMEOUT_SECONDS=30
|
||||
REQUEST_TIMEOUT_SECONDS=5
|
||||
HTTP_READ_HEADER_TIMEOUT_SECONDS=5
|
||||
HTTP_READ_TIMEOUT_SECONDS=10
|
||||
HTTP_WRITE_TIMEOUT_SECONDS=10
|
||||
HTTP_IDLE_TIMEOUT_SECONDS=60
|
||||
HTTP_SHUTDOWN_TIMEOUT_SECONDS=5
|
||||
|
||||
# Параметры Argon2id для новых хешей паролей.
|
||||
PASSWORD_MEMORY_KIB=65536
|
||||
PASSWORD_ITERATIONS=3
|
||||
PASSWORD_PARALLELISM=2
|
||||
PASSWORD_SALT_LENGTH=16
|
||||
PASSWORD_KEY_LENGTH=32
|
||||
|
||||
# Порт Swagger внутри конфигурации CORS API.
|
||||
SWAGGER_PORT=8081
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
CREATE INDEX outbox_events_published_at_idx
|
||||
ON outbox_events (published_at, id)
|
||||
WHERE published_at IS NOT NULL;
|
||||
Reference in New Issue
Block a user