fix: enforce pipeline lifecycle limits

This commit is contained in:
2026-10-30 13:45:49 +03:00
parent 3835106125
commit aa5d6a6e16
20 changed files with 770 additions and 308 deletions
+5 -3
View File
@@ -169,6 +169,7 @@ func newExperiment(ctx context.Context) (*experiment, func(), error) {
dispatcher, err := outbox.New(repository, broker, nil, outbox.Config{
BatchSize: 32, LeaseDuration: 10 * time.Second, PollInterval: time.Second,
RecoveryInterval: 10 * time.Second, RetryDelay: time.Second,
Retention: 24 * time.Hour, CleanupInterval: time.Hour, CleanupBatchSize: 1000,
})
if err != nil {
pool.Close()
@@ -185,9 +186,10 @@ func newExperiment(ctx context.Context) (*experiment, func(), error) {
slog.New(slog.NewTextHandler(io.Discard, nil)), repository, broker, objects, processor, nil,
worker.Config{
WorkerID: consumer, Concurrency: 1,
LeaseDuration: experimentLease, ProcessingTimeout: time.Second,
ReclaimInterval: experimentLease,
MaxAttempts: maxAttempts, RetryBase: time.Second, RetryMax: 4 * time.Second,
LeaseDuration: experimentLease, HeartbeatInterval: 500 * time.Millisecond,
ProcessingTimeout: time.Second,
ReclaimInterval: experimentLease,
MaxAttempts: maxAttempts, RetryBase: time.Second, RetryMax: 4 * time.Second,
MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels,
MemoryBytes: settings.MemoryBytes,
},
+21 -103
View File
@@ -1,3 +1,9 @@
x-messenger-config-volume: &messenger-config-volume
type: bind
source: ./config/messenger.yaml
target: /etc/messenger/messenger.yaml
read_only: true
services:
postgres:
image: postgres:17.6-alpine3.22
@@ -115,14 +121,10 @@ services:
image: ${API_IMAGE:-mtusi-messenger:lab5}
command: ["/app/messenger", "migrate"]
environment:
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
DATABASE_URL: ${DATABASE_URL}
DB_MIN_CONNS: ${DB_MIN_CONNS}
DB_MAX_CONNS: ${DB_MAX_CONNS}
DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS}
DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS}
DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS}
DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS}
MIGRATION_TIMEOUT_SECONDS: ${MIGRATION_TIMEOUT_SECONDS}
volumes:
- *messenger-config-volume
depends_on:
postgres:
condition: service_healthy
@@ -137,43 +139,19 @@ services:
build: .
image: ${API_IMAGE:-mtusi-messenger:lab5}
environment:
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
APP_HOST: ${APP_HOST}
APP_PORT: ${APP_PORT}
DATABASE_URL: ${DATABASE_URL}
CACHE_URL: ${CACHE_URL}
SESSION_STORE: ${SESSION_STORE}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS}
DB_MIN_CONNS: ${DB_MIN_CONNS}
DB_MAX_CONNS: ${DB_MAX_CONNS}
DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS}
DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS}
DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS}
DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS}
REQUEST_TIMEOUT_SECONDS: ${REQUEST_TIMEOUT_SECONDS}
HTTP_READ_HEADER_TIMEOUT_SECONDS: ${HTTP_READ_HEADER_TIMEOUT_SECONDS}
HTTP_READ_TIMEOUT_SECONDS: ${HTTP_READ_TIMEOUT_SECONDS}
HTTP_WRITE_TIMEOUT_SECONDS: ${HTTP_WRITE_TIMEOUT_SECONDS}
HTTP_IDLE_TIMEOUT_SECONDS: ${HTTP_IDLE_TIMEOUT_SECONDS}
HTTP_SHUTDOWN_TIMEOUT_SECONDS: ${HTTP_SHUTDOWN_TIMEOUT_SECONDS}
PASSWORD_MEMORY_KIB: ${PASSWORD_MEMORY_KIB}
PASSWORD_ITERATIONS: ${PASSWORD_ITERATIONS}
PASSWORD_PARALLELISM: ${PASSWORD_PARALLELISM}
PASSWORD_SALT_LENGTH: ${PASSWORD_SALT_LENGTH}
PASSWORD_KEY_LENGTH: ${PASSWORD_KEY_LENGTH}
SWAGGER_PORT: ${SWAGGER_PORT}
S3_ENDPOINT: ${S3_ENDPOINT}
S3_REGION: ${S3_REGION}
S3_BUCKET: ${S3_BUCKET}
S3_ACCESS_KEY: ${S3_ACCESS_KEY}
S3_SECRET_KEY: ${S3_SECRET_KEY}
S3_USE_TLS: ${S3_USE_TLS}
S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS}
ATTACHMENT_MAX_BYTES: ${ATTACHMENT_MAX_BYTES}
ATTACHMENT_MAX_PIXELS: ${ATTACHMENT_MAX_PIXELS}
ATTACHMENT_BODY_MEMORY_BYTES: ${ATTACHMENT_BODY_MEMORY_BYTES}
ATTACHMENT_DECODE_MEMORY_BYTES: ${ATTACHMENT_DECODE_MEMORY_BYTES}
PIPELINE_VERSION: ${PIPELINE_VERSION}
volumes:
- *messenger-config-volume
expose:
- "8080"
depends_on:
@@ -201,25 +179,11 @@ services:
image: ${API_IMAGE:-mtusi-messenger:lab5}
command: ["/app/messenger", "dispatch"]
environment:
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
DATABASE_URL: ${DATABASE_URL}
DB_MIN_CONNS: ${DB_MIN_CONNS}
DB_MAX_CONNS: ${DB_MAX_CONNS}
DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS}
DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS}
DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS}
DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS}
BROKER_URL: ${BROKER_URL}
BROKER_STREAM: ${BROKER_STREAM}
BROKER_CONSUMER_GROUP: ${BROKER_CONSUMER_GROUP}
BROKER_FAILED_STREAM: ${BROKER_FAILED_STREAM}
BROKER_FAILED_MAX_LENGTH: ${BROKER_FAILED_MAX_LENGTH}
OUTBOX_BATCH_SIZE: ${OUTBOX_BATCH_SIZE}
OUTBOX_LEASE_SECONDS: ${OUTBOX_LEASE_SECONDS}
OUTBOX_POLL_SECONDS: ${OUTBOX_POLL_SECONDS}
OUTBOX_RECOVERY_SECONDS: ${OUTBOX_RECOVERY_SECONDS}
OUTBOX_RETRY_SECONDS: ${OUTBOX_RETRY_SECONDS}
PIPELINE_METRICS_HOST: 0.0.0.0
PIPELINE_METRICS_PORT: 9091
volumes:
- *messenger-config-volume
expose:
- "9091"
depends_on:
@@ -241,39 +205,16 @@ services:
image: ${API_IMAGE:-mtusi-messenger:lab5}
command: ["/app/messenger", "worker"]
environment:
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
DATABASE_URL: ${DATABASE_URL}
DB_MIN_CONNS: ${DB_MIN_CONNS}
DB_MAX_CONNS: ${DB_MAX_CONNS}
DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS}
DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS}
DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS}
DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS}
S3_ENDPOINT: ${S3_ENDPOINT}
S3_REGION: ${S3_REGION}
S3_BUCKET: ${S3_BUCKET}
S3_ACCESS_KEY: ${S3_ACCESS_KEY}
S3_SECRET_KEY: ${S3_SECRET_KEY}
S3_USE_TLS: ${S3_USE_TLS}
S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS}
BROKER_URL: ${BROKER_URL}
BROKER_STREAM: ${BROKER_STREAM}
BROKER_CONSUMER_GROUP: ${BROKER_CONSUMER_GROUP}
BROKER_FAILED_STREAM: ${BROKER_FAILED_STREAM}
BROKER_FAILED_MAX_LENGTH: ${BROKER_FAILED_MAX_LENGTH}
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY}
WORKER_PREFETCH: ${WORKER_PREFETCH}
WORKER_LEASE_SECONDS: ${WORKER_LEASE_SECONDS}
WORKER_PROCESSING_TIMEOUT_SECONDS: ${WORKER_PROCESSING_TIMEOUT_SECONDS}
WORKER_READ_BLOCK_SECONDS: ${WORKER_READ_BLOCK_SECONDS}
WORKER_RECLAIM_SECONDS: ${WORKER_RECLAIM_SECONDS}
WORKER_MAX_ATTEMPTS: ${WORKER_MAX_ATTEMPTS}
WORKER_RETRY_BASE_SECONDS: ${WORKER_RETRY_BASE_SECONDS}
WORKER_RETRY_MAX_SECONDS: ${WORKER_RETRY_MAX_SECONDS}
ATTACHMENT_MAX_BYTES: ${ATTACHMENT_MAX_BYTES}
ATTACHMENT_MAX_PIXELS: ${ATTACHMENT_MAX_PIXELS}
WORKER_MEMORY_BYTES: ${WORKER_MEMORY_BYTES}
PIPELINE_METRICS_HOST: 0.0.0.0
PIPELINE_METRICS_PORT: 9091
volumes:
- *messenger-config-volume
expose:
- "9091"
depends_on:
@@ -351,39 +292,16 @@ services:
image: ${API_IMAGE:-mtusi-messenger:lab5}
command: ["/app/lab5experiment"]
environment:
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
DATABASE_URL: ${DATABASE_URL}
DB_MIN_CONNS: ${DB_MIN_CONNS}
DB_MAX_CONNS: ${DB_MAX_CONNS}
DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS}
DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS}
DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS}
DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS}
S3_ENDPOINT: ${S3_ENDPOINT}
S3_REGION: ${S3_REGION}
S3_BUCKET: ${S3_BUCKET}
S3_ACCESS_KEY: ${S3_ACCESS_KEY}
S3_SECRET_KEY: ${S3_SECRET_KEY}
S3_USE_TLS: ${S3_USE_TLS}
S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS}
BROKER_URL: ${BROKER_URL}
BROKER_STREAM: ${BROKER_STREAM}
BROKER_CONSUMER_GROUP: ${BROKER_CONSUMER_GROUP}
BROKER_FAILED_STREAM: ${BROKER_FAILED_STREAM}
BROKER_FAILED_MAX_LENGTH: ${BROKER_FAILED_MAX_LENGTH}
WORKER_CONCURRENCY: 1
WORKER_PREFETCH: 32
WORKER_LEASE_SECONDS: 2
WORKER_PROCESSING_TIMEOUT_SECONDS: 1
WORKER_READ_BLOCK_SECONDS: 1
WORKER_RECLAIM_SECONDS: 2
WORKER_MAX_ATTEMPTS: ${WORKER_MAX_ATTEMPTS}
WORKER_RETRY_BASE_SECONDS: 1
WORKER_RETRY_MAX_SECONDS: 4
ATTACHMENT_MAX_BYTES: ${ATTACHMENT_MAX_BYTES}
ATTACHMENT_MAX_PIXELS: ${ATTACHMENT_MAX_PIXELS}
WORKER_MEMORY_BYTES: ${WORKER_MEMORY_BYTES}
PIPELINE_METRICS_HOST: 0.0.0.0
PIPELINE_METRICS_PORT: 9091
volumes:
- *messenger-config-volume
depends_on:
migrate:
condition: service_completed_successfully
+76
View File
@@ -0,0 +1,76 @@
version: 1
server:
session_store: valkey
session_ttl: 1h
idempotency_ttl: 24h
request_timeout: 5s
http:
read_header_timeout: 5s
read_timeout: 10s
write_timeout: 10s
idle_timeout: 1m
shutdown_timeout: 5s
password:
memory_kib: 65536
iterations: 3
parallelism: 2
salt_length: 16
key_length: 32
database:
min_connections: 1
max_connections: 10
max_connection_age: 30m
max_connection_idle: 5m
health_check_period: 30s
dependency_timeout: 5s
migration:
timeout: 30s
object_storage:
use_tls: false
timeout: 5s
attachments:
max_bytes: 10485760
max_pixels: 40000000
body_memory_bytes: 67108864
decode_memory_bytes: 402653184
pipeline:
version: 1
broker:
stream: attachment-jobs
consumer_group: attachment-workers
failed_stream: attachment-jobs-failed
failed_stream_max_length: 10000
outbox:
batch_size: 32
lease: 15s
poll_interval: 1s
recovery_interval: 10s
retry_delay: 2s
retention: 168h
cleanup_interval: 1m
cleanup_batch_size: 1000
worker:
concurrency: 2
prefetch: 4
lease: 30s
heartbeat_interval: 5s
processing_timeout: 20s
read_block: 2s
reclaim_interval: 5s
max_attempts: 5
retry:
base: 1s
max: 30s
memory_bytes: 738197504
monitoring:
metrics_address: 0.0.0.0:9091
+4 -3
View File
@@ -55,7 +55,8 @@ func RunDispatcher(logger *slog.Logger) error {
dispatcher, err := outbox.New(postgres.New(pool), broker, metrics, outbox.Config{
BatchSize: settings.BatchSize, LeaseDuration: settings.LeaseDuration,
PollInterval: settings.PollInterval, RecoveryInterval: settings.RecoveryInterval,
RetryDelay: settings.RetryDelay,
RetryDelay: settings.RetryDelay, Retention: settings.Retention,
CleanupInterval: settings.CleanupInterval, CleanupBatchSize: settings.CleanupBatchSize,
})
if err != nil {
return err
@@ -112,8 +113,8 @@ func RunWorker(logger *slog.Logger) error {
logger, postgres.New(pool), broker, objects, processor, metrics,
worker.Config{
WorkerID: consumer, Concurrency: settings.Concurrency, LeaseDuration: settings.LeaseDuration,
ProcessingTimeout: settings.ProcessingTimeout,
ReclaimInterval: settings.ReclaimInterval, MaxAttempts: settings.MaxAttempts,
HeartbeatInterval: settings.HeartbeatInterval, ProcessingTimeout: settings.ProcessingTimeout,
ReclaimInterval: settings.ReclaimInterval, MaxAttempts: settings.MaxAttempts,
RetryBase: settings.RetryBase, RetryMax: settings.RetryMax,
MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels,
MemoryBytes: settings.MemoryBytes,
+61 -51
View File
@@ -98,14 +98,16 @@ type objectStorageConfig struct {
}
func LoadServer() (ServerConfig, error) {
values := viper.New()
values.AutomaticEnv()
host, err := requiredString(values, "APP_HOST")
values, err := loadValues()
if err != nil {
return ServerConfig{}, err
}
port, err := requiredPort(values, "APP_PORT")
host, err := requiredString(values, "server.host")
if err != nil {
return ServerConfig{}, err
}
port, err := requiredPort(values, "server.port")
if err != nil {
return ServerConfig{}, err
}
@@ -113,71 +115,71 @@ func LoadServer() (ServerConfig, error) {
if err != nil {
return ServerConfig{}, err
}
cacheURL, err := requiredString(values, "CACHE_URL")
cacheURL, err := requiredString(values, "cache.url")
if err != nil {
return ServerConfig{}, err
}
store, err := requiredString(values, "SESSION_STORE")
store, err := requiredString(values, "server.session_store")
if err != nil {
return ServerConfig{}, err
}
if store != "valkey" {
return ServerConfig{}, errors.New("SESSION_STORE must be valkey")
return ServerConfig{}, errors.New("server.session_store must be valkey")
}
sessionTTL, err := seconds(values, "SESSION_TTL_SECONDS", 1, 86400*30)
sessionTTL, err := duration(values, "server.session_ttl", time.Second, 30*24*time.Hour)
if err != nil {
return ServerConfig{}, err
}
idempotencyTTL, err := seconds(values, "IDEMPOTENCY_TTL_SECONDS", 86400, 86400*30)
idempotencyTTL, err := duration(values, "server.idempotency_ttl", 24*time.Hour, 30*24*time.Hour)
if err != nil {
return ServerConfig{}, err
}
requestTimeout, err := seconds(values, "REQUEST_TIMEOUT_SECONDS", 1, 300)
requestTimeout, err := duration(values, "server.request_timeout", time.Second, 5*time.Minute)
if err != nil {
return ServerConfig{}, err
}
readHeaderTimeout, err := seconds(values, "HTTP_READ_HEADER_TIMEOUT_SECONDS", 1, 300)
readHeaderTimeout, err := duration(values, "server.http.read_header_timeout", time.Second, 5*time.Minute)
if err != nil {
return ServerConfig{}, err
}
readTimeout, err := seconds(values, "HTTP_READ_TIMEOUT_SECONDS", 1, 300)
readTimeout, err := duration(values, "server.http.read_timeout", time.Second, 5*time.Minute)
if err != nil {
return ServerConfig{}, err
}
writeTimeout, err := seconds(values, "HTTP_WRITE_TIMEOUT_SECONDS", 1, 300)
writeTimeout, err := duration(values, "server.http.write_timeout", time.Second, 5*time.Minute)
if err != nil {
return ServerConfig{}, err
}
idleTimeout, err := seconds(values, "HTTP_IDLE_TIMEOUT_SECONDS", 1, 3600)
idleTimeout, err := duration(values, "server.http.idle_timeout", time.Second, time.Hour)
if err != nil {
return ServerConfig{}, err
}
shutdownTimeout, err := seconds(values, "HTTP_SHUTDOWN_TIMEOUT_SECONDS", 1, 300)
shutdownTimeout, err := duration(values, "server.http.shutdown_timeout", time.Second, 5*time.Minute)
if err != nil {
return ServerConfig{}, err
}
passwordMemory, err := integer(values, "PASSWORD_MEMORY_KIB", 8*1024, 1024*1024)
passwordMemory, err := integer(values, "server.password.memory_kib", 8*1024, 1024*1024)
if err != nil {
return ServerConfig{}, err
}
passwordIterations, err := integer(values, "PASSWORD_ITERATIONS", 1, 20)
passwordIterations, err := integer(values, "server.password.iterations", 1, 20)
if err != nil {
return ServerConfig{}, err
}
passwordParallelism, err := integer(values, "PASSWORD_PARALLELISM", 1, 255)
passwordParallelism, err := integer(values, "server.password.parallelism", 1, 255)
if err != nil {
return ServerConfig{}, err
}
passwordSaltLength, err := integer(values, "PASSWORD_SALT_LENGTH", 8, 1024)
passwordSaltLength, err := integer(values, "server.password.salt_length", 8, 1024)
if err != nil {
return ServerConfig{}, err
}
passwordKeyLength, err := integer(values, "PASSWORD_KEY_LENGTH", 16, 1024)
passwordKeyLength, err := integer(values, "server.password.key_length", 16, 1024)
if err != nil {
return ServerConfig{}, err
}
swaggerPort, err := requiredPort(values, "SWAGGER_PORT")
swaggerPort, err := requiredPort(values, "server.swagger_port")
if err != nil {
return ServerConfig{}, err
}
@@ -185,30 +187,30 @@ func LoadServer() (ServerConfig, error) {
if err != nil {
return ServerConfig{}, err
}
attachmentMaxBytes, err := integer(values, "ATTACHMENT_MAX_BYTES", 1, 10*1024*1024)
attachmentMaxBytes, err := integer(values, "attachments.max_bytes", 1, 10*1024*1024)
if err != nil {
return ServerConfig{}, err
}
attachmentMaxPixels, err := integer(values, "ATTACHMENT_MAX_PIXELS", 1, 40_000_000)
attachmentMaxPixels, err := integer(values, "attachments.max_pixels", 1, 40_000_000)
if err != nil {
return ServerConfig{}, err
}
attachmentBodyMemory, err := integer(values, "ATTACHMENT_BODY_MEMORY_BYTES", 1, 1024*1024*1024)
attachmentBodyMemory, err := integer(values, "attachments.body_memory_bytes", 1, 1024*1024*1024)
if err != nil {
return ServerConfig{}, err
}
if int64(attachmentBodyMemory) < int64(attachmentMaxBytes) {
return ServerConfig{}, fmt.Errorf("ATTACHMENT_BODY_MEMORY_BYTES: must be at least %d", attachmentMaxBytes)
return ServerConfig{}, fmt.Errorf("attachments.body_memory_bytes: must be at least %d", attachmentMaxBytes)
}
attachmentDecodeMemory, err := integer(values, "ATTACHMENT_DECODE_MEMORY_BYTES", 1, 1024*1024*1024)
attachmentDecodeMemory, err := integer(values, "attachments.decode_memory_bytes", 1, 1024*1024*1024)
if err != nil {
return ServerConfig{}, err
}
minimumDecodeMemory := int64(attachmentMaxPixels) * 8
if int64(attachmentDecodeMemory) < minimumDecodeMemory {
return ServerConfig{}, fmt.Errorf("ATTACHMENT_DECODE_MEMORY_BYTES: must be at least %d", minimumDecodeMemory)
return ServerConfig{}, fmt.Errorf("attachments.decode_memory_bytes: must be at least %d", minimumDecodeMemory)
}
pipelineVersion, err := integer(values, "PIPELINE_VERSION", 1, 1000)
pipelineVersion, err := integer(values, "pipeline.version", 1, 1000)
if err != nil {
return ServerConfig{}, err
}
@@ -236,13 +238,15 @@ func LoadServer() (ServerConfig, error) {
}
func LoadMigration() (MigrationConfig, error) {
values := viper.New()
values.AutomaticEnv()
values, err := loadValues()
if err != nil {
return MigrationConfig{}, err
}
database, err := loadDatabaseConfig(values)
if err != nil {
return MigrationConfig{}, err
}
migrationTimeout, err := seconds(values, "MIGRATION_TIMEOUT_SECONDS", 1, 600)
migrationTimeout, err := duration(values, "migration.timeout", time.Second, 10*time.Minute)
if err != nil {
return MigrationConfig{}, err
}
@@ -255,8 +259,10 @@ func LoadMigration() (MigrationConfig, error) {
}
func LoadReconciliation() (ReconciliationConfig, error) {
values := viper.New()
values.AutomaticEnv()
values, err := loadValues()
if err != nil {
return ReconciliationConfig{}, err
}
database, err := loadDatabaseConfig(values)
if err != nil {
return ReconciliationConfig{}, err
@@ -276,31 +282,31 @@ func LoadReconciliation() (ReconciliationConfig, error) {
}
func loadObjectStorageConfig(values *viper.Viper) (objectStorageConfig, error) {
endpoint, err := requiredString(values, "S3_ENDPOINT")
endpoint, err := requiredString(values, "object_storage.endpoint")
if err != nil {
return objectStorageConfig{}, err
}
region, err := requiredString(values, "S3_REGION")
region, err := requiredString(values, "object_storage.region")
if err != nil {
return objectStorageConfig{}, err
}
bucket, err := requiredString(values, "S3_BUCKET")
bucket, err := requiredString(values, "object_storage.bucket")
if err != nil {
return objectStorageConfig{}, err
}
accessKey, err := requiredString(values, "S3_ACCESS_KEY")
accessKey, err := requiredString(values, "object_storage.access_key")
if err != nil {
return objectStorageConfig{}, err
}
secretKey, err := requiredString(values, "S3_SECRET_KEY")
secretKey, err := requiredString(values, "object_storage.secret_key")
if err != nil {
return objectStorageConfig{}, err
}
useTLS, err := boolean(values, "S3_USE_TLS")
useTLS, err := boolean(values, "object_storage.use_tls")
if err != nil {
return objectStorageConfig{}, err
}
timeout, err := seconds(values, "S3_TIMEOUT_SECONDS", 1, 300)
timeout, err := duration(values, "object_storage.timeout", time.Second, 5*time.Minute)
if err != nil {
return objectStorageConfig{}, err
}
@@ -311,31 +317,31 @@ func loadObjectStorageConfig(values *viper.Viper) (objectStorageConfig, error) {
}
func loadDatabaseConfig(values *viper.Viper) (databaseConfig, error) {
databaseURL, err := requiredString(values, "DATABASE_URL")
databaseURL, err := requiredString(values, "database.url")
if err != nil {
return databaseConfig{}, err
}
maxConns, err := integer(values, "DB_MAX_CONNS", 1, 100)
maxConns, err := integer(values, "database.max_connections", 1, 100)
if err != nil {
return databaseConfig{}, err
}
minConns, err := integer(values, "DB_MIN_CONNS", 0, maxConns)
minConns, err := integer(values, "database.min_connections", 0, maxConns)
if err != nil {
return databaseConfig{}, err
}
maxConnectionAge, err := seconds(values, "DB_MAX_CONNECTION_AGE_SECONDS", 1, 86400)
maxConnectionAge, err := duration(values, "database.max_connection_age", time.Second, 24*time.Hour)
if err != nil {
return databaseConfig{}, err
}
maxConnectionIdle, err := seconds(values, "DB_MAX_CONNECTION_IDLE_SECONDS", 1, 86400)
maxConnectionIdle, err := duration(values, "database.max_connection_idle", time.Second, 24*time.Hour)
if err != nil {
return databaseConfig{}, err
}
healthCheckPeriod, err := seconds(values, "DB_HEALTH_CHECK_SECONDS", 1, 3600)
healthCheckPeriod, err := duration(values, "database.health_check_period", time.Second, time.Hour)
if err != nil {
return databaseConfig{}, err
}
dependencyTimeout, err := seconds(values, "DEPENDENCY_TIMEOUT_SECONDS", 1, 60)
dependencyTimeout, err := duration(values, "database.dependency_timeout", time.Second, time.Minute)
if err != nil {
return databaseConfig{}, err
}
@@ -365,12 +371,16 @@ func requiredPort(values *viper.Viper, name string) (string, error) {
return value, nil
}
func seconds(values *viper.Viper, name string, minimum, maximum int) (time.Duration, error) {
value, err := integer(values, name, minimum, maximum)
func duration(values *viper.Viper, name string, minimum, maximum time.Duration) (time.Duration, error) {
value, err := requiredString(values, name)
if err != nil {
return 0, err
}
return time.Duration(value) * time.Second, nil
parsed, err := time.ParseDuration(value)
if err != nil || parsed < minimum || parsed > maximum {
return 0, fmt.Errorf("%s: must be between %s and %s", name, minimum, maximum)
}
return parsed, nil
}
func integer(values *viper.Viper, name string, minimum, maximum int) (int, error) {
+114 -12
View File
@@ -1,24 +1,17 @@
package config
import (
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/stretchr/testify/require"
)
func TestLoadReconciliationDoesNotRequireServerSettings(t *testing.T) {
values := map[string]string{
"DATABASE_URL": "postgres://user:password@postgres:5432/messenger",
"DB_MIN_CONNS": "0", "DB_MAX_CONNS": "4",
"DB_MAX_CONNECTION_AGE_SECONDS": "300", "DB_MAX_CONNECTION_IDLE_SECONDS": "60",
"DB_HEALTH_CHECK_SECONDS": "30", "DEPENDENCY_TIMEOUT_SECONDS": "5",
"S3_ENDPOINT": "s3:9000", "S3_REGION": "us-east-1", "S3_BUCKET": "attachments",
"S3_ACCESS_KEY": "application", "S3_SECRET_KEY": "application-secret",
"S3_USE_TLS": "false", "S3_TIMEOUT_SECONDS": "5",
}
for name, value := range values {
t.Setenv(name, value)
}
setConfigEnvironment(t, validTestConfig)
t.Setenv("APP_HOST", "")
t.Setenv("CACHE_URL", "")
t.Setenv("SWAGGER_PORT", "")
@@ -27,3 +20,112 @@ func TestLoadReconciliationDoesNotRequireServerSettings(t *testing.T) {
require.NoError(t, err)
require.Equal(t, "attachments", settings.S3Bucket)
}
func TestLoadWorkerUsesYAMLForRuntimeSettings(t *testing.T) {
setConfigEnvironment(t, validTestConfig)
t.Setenv("WORKER_CONCURRENCY", "64")
settings, err := LoadWorker()
require.NoError(t, err)
require.Equal(t, 2, settings.Concurrency)
require.Equal(t, 5*time.Second, settings.HeartbeatInterval)
}
func TestLoadWorkerRejectsHeartbeatAtLeaseBoundary(t *testing.T) {
content := strings.Replace(validTestConfig, "heartbeat_interval: 5s", "heartbeat_interval: 30s", 1)
setConfigEnvironment(t, content)
_, err := LoadWorker()
require.ErrorContains(t, err, "heartbeat_interval must be less than worker.lease")
}
func TestLoadValuesRejectsUnknownVersion(t *testing.T) {
setConfigEnvironment(t, "version: 2\n")
_, err := loadValues()
require.ErrorContains(t, err, "version: must be 1")
}
func setConfigEnvironment(t *testing.T, content string) {
t.Helper()
path := filepath.Join(t.TempDir(), "messenger.yaml")
require.NoError(t, os.WriteFile(path, []byte(content), 0o600))
t.Setenv("MESSENGER_CONFIG_PATH", path)
for name, value := range map[string]string{
"APP_HOST": "0.0.0.0", "APP_PORT": "8080", "SWAGGER_PORT": "8081",
"DATABASE_URL": "postgres://user:password@postgres:5432/messenger",
"CACHE_URL": "redis://valkey:6379/0", "BROKER_URL": "redis://redis-broker:6379/0",
"S3_ENDPOINT": "s3:9000", "S3_REGION": "us-east-1", "S3_BUCKET": "attachments",
"S3_ACCESS_KEY": "application", "S3_SECRET_KEY": "application-secret",
} {
t.Setenv(name, value)
}
}
const validTestConfig = `version: 1
server:
session_store: valkey
session_ttl: 1h
idempotency_ttl: 24h
request_timeout: 5s
http:
read_header_timeout: 5s
read_timeout: 10s
write_timeout: 10s
idle_timeout: 1m
shutdown_timeout: 5s
password:
memory_kib: 65536
iterations: 3
parallelism: 2
salt_length: 16
key_length: 32
database:
min_connections: 0
max_connections: 4
max_connection_age: 5m
max_connection_idle: 1m
health_check_period: 30s
dependency_timeout: 5s
migration:
timeout: 30s
object_storage:
use_tls: false
timeout: 5s
attachments:
max_bytes: 10485760
max_pixels: 40000000
body_memory_bytes: 67108864
decode_memory_bytes: 402653184
pipeline:
version: 1
broker:
stream: attachment-jobs
consumer_group: attachment-workers
failed_stream: attachment-jobs-failed
failed_stream_max_length: 10000
outbox:
batch_size: 32
lease: 15s
poll_interval: 1s
recovery_interval: 10s
retry_delay: 2s
retention: 168h
cleanup_interval: 1m
cleanup_batch_size: 1000
worker:
concurrency: 2
prefetch: 4
lease: 30s
heartbeat_interval: 5s
processing_timeout: 20s
read_block: 2s
reclaim_interval: 5s
max_attempts: 5
retry:
base: 1s
max: 30s
memory_bytes: 738197504
monitoring:
metrics_address: 0.0.0.0:9091
`
+47
View File
@@ -0,0 +1,47 @@
package config
import (
"errors"
"fmt"
"os"
"strings"
"github.com/spf13/viper"
)
const supportedConfigVersion = 1
var environmentKeys = map[string]string{
"server.host": "APP_HOST",
"server.port": "APP_PORT",
"server.swagger_port": "SWAGGER_PORT",
"database.url": "DATABASE_URL",
"cache.url": "CACHE_URL",
"object_storage.endpoint": "S3_ENDPOINT",
"object_storage.region": "S3_REGION",
"object_storage.bucket": "S3_BUCKET",
"object_storage.access_key": "S3_ACCESS_KEY",
"object_storage.secret_key": "S3_SECRET_KEY",
"broker.url": "BROKER_URL",
}
func loadValues() (*viper.Viper, error) {
path := strings.TrimSpace(os.Getenv("MESSENGER_CONFIG_PATH"))
if path == "" {
return nil, errors.New("MESSENGER_CONFIG_PATH is required")
}
values := viper.New()
values.SetConfigFile(path)
if err := values.ReadInConfig(); err != nil {
return nil, fmt.Errorf("read messenger config: %w", err)
}
if version := values.GetInt("version"); version != supportedConfigVersion {
return nil, fmt.Errorf("version: must be %d", supportedConfigVersion)
}
for key, name := range environmentKeys {
if err := values.BindEnv(key, name); err != nil {
return nil, fmt.Errorf("bind %s: %w", name, err)
}
}
return values, nil
}
+72 -38
View File
@@ -2,6 +2,7 @@ package config
import (
"errors"
"fmt"
"net"
"time"
@@ -26,6 +27,9 @@ type DispatcherConfig struct {
PollInterval time.Duration
RecoveryInterval time.Duration
RetryDelay time.Duration
Retention time.Duration
CleanupInterval time.Duration
CleanupBatchSize int
MetricsAddress string
}
@@ -52,6 +56,7 @@ type WorkerConfig struct {
Concurrency int
Prefetch int
LeaseDuration time.Duration
HeartbeatInterval time.Duration
ProcessingTimeout time.Duration
ReadBlock time.Duration
ReclaimInterval time.Duration
@@ -83,8 +88,10 @@ type brokerConfig struct {
}
func LoadJobAdmin() (JobAdminConfig, error) {
values := viper.New()
values.AutomaticEnv()
values, err := loadValues()
if err != nil {
return JobAdminConfig{}, err
}
database, err := loadDatabaseConfig(values)
if err != nil {
return JobAdminConfig{}, err
@@ -97,8 +104,10 @@ func LoadJobAdmin() (JobAdminConfig, error) {
}
func LoadDispatcher() (DispatcherConfig, error) {
values := viper.New()
values.AutomaticEnv()
values, err := loadValues()
if err != nil {
return DispatcherConfig{}, err
}
database, err := loadDatabaseConfig(values)
if err != nil {
return DispatcherConfig{}, err
@@ -107,23 +116,35 @@ func LoadDispatcher() (DispatcherConfig, error) {
if err != nil {
return DispatcherConfig{}, err
}
batchSize, err := integer(values, "OUTBOX_BATCH_SIZE", 1, 1000)
batchSize, err := integer(values, "outbox.batch_size", 1, 1000)
if err != nil {
return DispatcherConfig{}, err
}
leaseDuration, err := seconds(values, "OUTBOX_LEASE_SECONDS", 1, 600)
leaseDuration, err := duration(values, "outbox.lease", time.Second, 10*time.Minute)
if err != nil {
return DispatcherConfig{}, err
}
pollInterval, err := seconds(values, "OUTBOX_POLL_SECONDS", 1, 60)
pollInterval, err := duration(values, "outbox.poll_interval", time.Millisecond, time.Minute)
if err != nil {
return DispatcherConfig{}, err
}
recoveryInterval, err := seconds(values, "OUTBOX_RECOVERY_SECONDS", 1, 3600)
recoveryInterval, err := duration(values, "outbox.recovery_interval", time.Second, time.Hour)
if err != nil {
return DispatcherConfig{}, err
}
retryDelay, err := seconds(values, "OUTBOX_RETRY_SECONDS", 1, 3600)
retryDelay, err := duration(values, "outbox.retry_delay", time.Second, time.Hour)
if err != nil {
return DispatcherConfig{}, err
}
retention, err := duration(values, "outbox.retention", time.Hour, 365*24*time.Hour)
if err != nil {
return DispatcherConfig{}, err
}
cleanupInterval, err := duration(values, "outbox.cleanup_interval", time.Minute, 24*time.Hour)
if err != nil {
return DispatcherConfig{}, err
}
cleanupBatchSize, err := integer(values, "outbox.cleanup_batch_size", 1, 1000)
if err != nil {
return DispatcherConfig{}, err
}
@@ -138,13 +159,17 @@ func LoadDispatcher() (DispatcherConfig, error) {
BrokerURL: broker.url, Stream: broker.stream, ConsumerGroup: broker.consumerGroup,
FailedStream: broker.failedStream, FailedStreamMaxLength: int64(broker.failedStreamMaxLength),
BatchSize: batchSize, LeaseDuration: leaseDuration, PollInterval: pollInterval,
RecoveryInterval: recoveryInterval, RetryDelay: retryDelay, MetricsAddress: metricsAddress,
RecoveryInterval: recoveryInterval, RetryDelay: retryDelay,
Retention: retention, CleanupInterval: cleanupInterval, CleanupBatchSize: cleanupBatchSize,
MetricsAddress: metricsAddress,
}, nil
}
func LoadWorker() (WorkerConfig, error) {
values := viper.New()
values.AutomaticEnv()
values, err := loadValues()
if err != nil {
return WorkerConfig{}, err
}
database, err := loadDatabaseConfig(values)
if err != nil {
return WorkerConfig{}, err
@@ -157,57 +182,64 @@ func LoadWorker() (WorkerConfig, error) {
if err != nil {
return WorkerConfig{}, err
}
concurrency, err := integer(values, "WORKER_CONCURRENCY", 1, 64)
concurrency, err := integer(values, "worker.concurrency", 1, 64)
if err != nil {
return WorkerConfig{}, err
}
prefetch, err := integer(values, "WORKER_PREFETCH", concurrency, 1024)
prefetch, err := integer(values, "worker.prefetch", concurrency, 1024)
if err != nil {
return WorkerConfig{}, err
}
leaseDuration, err := seconds(values, "WORKER_LEASE_SECONDS", 1, 3600)
leaseDuration, err := duration(values, "worker.lease", time.Second, time.Hour)
if err != nil {
return WorkerConfig{}, err
}
processingTimeout, err := seconds(values, "WORKER_PROCESSING_TIMEOUT_SECONDS", 1, 3600)
heartbeatInterval, err := duration(values, "worker.heartbeat_interval", 100*time.Millisecond, time.Hour)
if err != nil {
return WorkerConfig{}, err
}
if heartbeatInterval >= leaseDuration {
return WorkerConfig{}, errors.New("worker.heartbeat_interval must be less than worker.lease")
}
processingTimeout, err := duration(values, "worker.processing_timeout", time.Second, time.Hour)
if err != nil {
return WorkerConfig{}, err
}
if processingTimeout >= leaseDuration {
return WorkerConfig{}, errors.New("WORKER_PROCESSING_TIMEOUT_SECONDS must be less than WORKER_LEASE_SECONDS")
return WorkerConfig{}, errors.New("worker.processing_timeout must be less than worker.lease")
}
readBlock, err := seconds(values, "WORKER_READ_BLOCK_SECONDS", 1, 60)
readBlock, err := duration(values, "worker.read_block", 100*time.Millisecond, time.Minute)
if err != nil {
return WorkerConfig{}, err
}
reclaimInterval, err := seconds(values, "WORKER_RECLAIM_SECONDS", 1, 3600)
reclaimInterval, err := duration(values, "worker.reclaim_interval", time.Second, time.Hour)
if err != nil {
return WorkerConfig{}, err
}
maxAttempts, err := integer(values, "WORKER_MAX_ATTEMPTS", 1, 100)
maxAttempts, err := integer(values, "worker.max_attempts", 1, 100)
if err != nil {
return WorkerConfig{}, err
}
retryBase, err := seconds(values, "WORKER_RETRY_BASE_SECONDS", 1, 3600)
retryBase, err := duration(values, "worker.retry.base", time.Second, time.Hour)
if err != nil {
return WorkerConfig{}, err
}
retryMax, err := seconds(values, "WORKER_RETRY_MAX_SECONDS", 1, 86400)
retryMax, err := duration(values, "worker.retry.max", time.Second, 24*time.Hour)
if err != nil {
return WorkerConfig{}, err
}
if retryMax < retryBase {
return WorkerConfig{}, errors.New("WORKER_RETRY_MAX_SECONDS must be at least WORKER_RETRY_BASE_SECONDS")
return WorkerConfig{}, errors.New("worker.retry.max must be at least worker.retry.base")
}
attachmentMaxBytes, err := integer(values, "ATTACHMENT_MAX_BYTES", 1, 10*1024*1024)
attachmentMaxBytes, err := integer(values, "attachments.max_bytes", 1, 10*1024*1024)
if err != nil {
return WorkerConfig{}, err
}
attachmentMaxPixels, err := integer(values, "ATTACHMENT_MAX_PIXELS", 1, 40_000_000)
attachmentMaxPixels, err := integer(values, "attachments.max_pixels", 1, 40_000_000)
if err != nil {
return WorkerConfig{}, err
}
memoryBytes, err := integer(values, "WORKER_MEMORY_BYTES", attachmentMaxBytes, 2*1024*1024*1024)
memoryBytes, err := integer(values, "worker.memory_bytes", attachmentMaxBytes, 2*1024*1024*1024)
if err != nil {
return WorkerConfig{}, err
}
@@ -225,44 +257,46 @@ func LoadWorker() (WorkerConfig, error) {
ConsumerGroup: broker.consumerGroup, FailedStream: broker.failedStream,
FailedStreamMaxLength: int64(broker.failedStreamMaxLength),
Concurrency: concurrency, Prefetch: prefetch, LeaseDuration: leaseDuration,
ProcessingTimeout: processingTimeout, ReadBlock: readBlock,
HeartbeatInterval: heartbeatInterval, ProcessingTimeout: processingTimeout, ReadBlock: readBlock,
ReclaimInterval: reclaimInterval, MaxAttempts: maxAttempts, RetryBase: retryBase, RetryMax: retryMax,
AttachmentMaxBytes: int64(attachmentMaxBytes), AttachmentMaxPixels: int64(attachmentMaxPixels),
MemoryBytes: int64(memoryBytes),
MetricsAddress: metricsAddress,
MemoryBytes: int64(memoryBytes), MetricsAddress: metricsAddress,
}, nil
}
func metricsAddress(values *viper.Viper) (string, error) {
host, err := requiredString(values, "PIPELINE_METRICS_HOST")
address, err := requiredString(values, "monitoring.metrics_address")
if err != nil {
return "", err
}
port, err := requiredPort(values, "PIPELINE_METRICS_PORT")
_, port, err := net.SplitHostPort(address)
if err != nil {
return "", err
return "", fmt.Errorf("monitoring.metrics_address: %w", err)
}
return net.JoinHostPort(host, port), nil
if _, err := integerValue(port, 1, 65535); err != nil {
return "", fmt.Errorf("monitoring.metrics_address: %w", err)
}
return address, nil
}
func loadBrokerConfig(values *viper.Viper) (brokerConfig, error) {
url, err := requiredString(values, "BROKER_URL")
url, err := requiredString(values, "broker.url")
if err != nil {
return brokerConfig{}, err
}
stream, err := requiredString(values, "BROKER_STREAM")
stream, err := requiredString(values, "broker.stream")
if err != nil {
return brokerConfig{}, err
}
consumerGroup, err := requiredString(values, "BROKER_CONSUMER_GROUP")
consumerGroup, err := requiredString(values, "broker.consumer_group")
if err != nil {
return brokerConfig{}, err
}
failedStream, err := requiredString(values, "BROKER_FAILED_STREAM")
failedStream, err := requiredString(values, "broker.failed_stream")
if err != nil {
return brokerConfig{}, err
}
failedStreamMaxLength, err := integer(values, "BROKER_FAILED_MAX_LENGTH", 100, 10_000_000)
failedStreamMaxLength, err := integer(values, "broker.failed_stream_max_length", 100, 10_000_000)
if err != nil {
return brokerConfig{}, err
}
@@ -1,6 +1,7 @@
package transform
import (
"context"
"encoding/binary"
"errors"
"image"
@@ -9,11 +10,14 @@ import (
var errInvalidEXIF = errors.New("invalid EXIF metadata")
func jpegOrientation(data []byte) (int, error) {
func jpegOrientation(ctx context.Context, data []byte) (int, error) {
if len(data) < 2 || data[0] != 0xff || data[1] != 0xd8 {
return 0, errInvalidEXIF
}
for offset := 2; offset < len(data); {
if err := contextError(ctx); err != nil {
return 0, err
}
if data[offset] != 0xff {
return 0, errInvalidEXIF
}
@@ -43,12 +47,12 @@ func jpegOrientation(data []byte) (int, error) {
if marker != 0xe1 || len(payload) < 6 || string(payload[:6]) != "Exif\x00\x00" {
continue
}
return tiffOrientation(payload[6:])
return tiffOrientation(ctx, payload[6:])
}
return 1, nil
}
func tiffOrientation(data []byte) (int, error) {
func tiffOrientation(ctx context.Context, data []byte) (int, error) {
if len(data) < 8 {
return 0, errInvalidEXIF
}
@@ -74,6 +78,9 @@ func tiffOrientation(data []byte) (int, error) {
return 0, errInvalidEXIF
}
for index := uint64(0); index < count; index++ {
if err := contextError(ctx); err != nil {
return 0, err
}
entry := data[entriesStart+index*12 : entriesStart+(index+1)*12]
if order.Uint16(entry[:2]) != 0x0112 {
continue
@@ -90,7 +97,7 @@ func tiffOrientation(data []byte) (int, error) {
return 1, nil
}
func applyOrientation(source image.Image, orientation int) *image.NRGBA {
func applyOrientation(ctx context.Context, source image.Image, orientation int) (*image.NRGBA, error) {
bounds := source.Bounds()
width, height := bounds.Dx(), bounds.Dy()
outputWidth, outputHeight := width, height
@@ -99,12 +106,20 @@ func applyOrientation(source image.Image, orientation int) *image.NRGBA {
}
output := image.NewNRGBA(image.Rect(0, 0, outputWidth, outputHeight))
for y := 0; y < outputHeight; y++ {
if err := contextError(ctx); err != nil {
return nil, err
}
for x := 0; x < outputWidth; x++ {
if x&255 == 0 {
if err := contextError(ctx); err != nil {
return nil, err
}
}
sourceX, sourceY := orientedSourcePoint(x, y, width, height, orientation)
output.Set(x, y, source.At(bounds.Min.X+sourceX, bounds.Min.Y+sourceY))
}
}
return output
return output, nil
}
func orientedSourcePoint(x, y, width, height, orientation int) (int, int) {
@@ -128,11 +143,19 @@ func orientedSourcePoint(x, y, width, height, orientation int) (int, int) {
}
}
func flattenWhite(source image.Image) *image.NRGBA {
func flattenWhite(ctx context.Context, source image.Image) (*image.NRGBA, error) {
bounds := source.Bounds()
output := image.NewNRGBA(image.Rect(0, 0, bounds.Dx(), bounds.Dy()))
for y := 0; y < bounds.Dy(); y++ {
if err := contextError(ctx); err != nil {
return nil, err
}
for x := 0; x < bounds.Dx(); x++ {
if x&255 == 0 {
if err := contextError(ctx); err != nil {
return nil, err
}
}
r, g, b, a := source.At(bounds.Min.X+x, bounds.Min.Y+y).RGBA()
output.SetNRGBA(x, y, color.NRGBA{
R: uint8((r + 0xffff - a) / 0x101),
@@ -142,5 +165,5 @@ func flattenWhite(source image.Image) *image.NRGBA {
})
}
}
return output
return output, nil
}
@@ -9,6 +9,7 @@ import (
"image"
"image/jpeg"
_ "image/png"
"io"
"math"
"net/http"
@@ -78,7 +79,7 @@ func (p *Processor) Process(ctx context.Context, data []byte, declaredContentTyp
if err := contextError(ctx); err != nil {
return Result{}, err
}
img, format, width, height, orientation, err := p.decode(data, declaredContentType)
img, format, width, height, orientation, err := p.decode(ctx, data, declaredContentType)
if err != nil {
return Result{}, err
}
@@ -86,12 +87,12 @@ func (p *Processor) Process(ctx context.Context, data []byte, declaredContentTyp
return Result{}, err
}
oriented := applyOrientation(img, orientation)
if err := contextError(ctx); err != nil {
oriented, err := applyOrientation(ctx, img, orientation)
if err != nil {
return Result{}, err
}
opaque := flattenWhite(oriented)
if err := contextError(ctx); err != nil {
opaque, err := flattenWhite(ctx, oriented)
if err != nil {
return Result{}, err
}
result := Result{
@@ -99,7 +100,10 @@ func (p *Processor) Process(ctx context.Context, data []byte, declaredContentTyp
Orientation: orientation,
}
thumbnail := fit(opaque, 256, 256, false)
thumbnail, err := fitContext(ctx, opaque, 256, 256, false)
if err != nil {
return Result{}, err
}
result.Thumbnail, err = encodeJPEG(thumbnail, p.config.JPEGQuality)
if err != nil {
return Result{}, err
@@ -109,7 +113,10 @@ func (p *Processor) Process(ctx context.Context, data []byte, declaredContentTyp
}
crop := centeredSquare(opaque)
crop = resize(crop, 128, 128)
crop, err = resizeContext(ctx, crop, 128, 128)
if err != nil {
return Result{}, err
}
result.Crop, err = encodeJPEG(crop, p.config.JPEGQuality)
if err != nil {
return Result{}, err
@@ -118,7 +125,10 @@ func (p *Processor) Process(ctx context.Context, data []byte, declaredContentTyp
return Result{}, err
}
optimized := fit(opaque, 1280, 1280, false)
optimized, err := fitContext(ctx, opaque, 1280, 1280, false)
if err != nil {
return Result{}, err
}
result.Optimized, err = encodeWebP(optimized, p.config.WebPQuality, p.config.WebPMethod)
if err != nil {
return Result{}, err
@@ -129,7 +139,11 @@ func (p *Processor) Process(ctx context.Context, data []byte, declaredContentTyp
return result, nil
}
func (p *Processor) decode(data []byte, declaredContentType string) (image.Image, string, int, int, int, error) {
func (p *Processor) decode(
ctx context.Context,
data []byte,
declaredContentType string,
) (image.Image, string, int, int, int, error) {
if declaredContentType != "image/jpeg" && declaredContentType != "image/png" {
return nil, "", 0, 0, 0, ErrUnsupportedMedia
}
@@ -142,27 +156,48 @@ func (p *Processor) decode(data []byte, declaredContentType string) (image.Image
if http.DetectContentType(data) != declaredContentType {
return nil, "", 0, 0, 0, ErrUnsupportedMedia
}
config, format, err := image.DecodeConfig(bytes.NewReader(data))
config, format, err := image.DecodeConfig(contextReader{ctx: ctx, reader: bytes.NewReader(data)})
if contextErr := contextError(ctx); contextErr != nil {
return nil, "", 0, 0, 0, contextErr
}
if err != nil || contentType(format) != declaredContentType || config.Width < 1 || config.Height < 1 {
return nil, "", 0, 0, 0, ErrInvalidImage
}
if int64(config.Width) > p.config.MaxPixels/int64(config.Height) {
return nil, "", 0, 0, 0, ErrPayloadTooLarge
}
img, decodedFormat, err := image.Decode(bytes.NewReader(data))
img, decodedFormat, err := image.Decode(contextReader{ctx: ctx, reader: bytes.NewReader(data)})
if contextErr := contextError(ctx); contextErr != nil {
return nil, "", 0, 0, 0, contextErr
}
if err != nil || decodedFormat != format {
return nil, "", 0, 0, 0, ErrInvalidImage
}
orientation := 1
if format == "jpeg" {
orientation, err = jpegOrientation(data)
orientation, err = jpegOrientation(ctx, data)
if err != nil {
if contextErr := contextError(ctx); contextErr != nil {
return nil, "", 0, 0, 0, contextErr
}
return nil, "", 0, 0, 0, ErrInvalidImage
}
}
return img, declaredContentType, config.Width, config.Height, orientation, nil
}
type contextReader struct {
ctx context.Context
reader io.Reader
}
func (r contextReader) Read(buffer []byte) (int, error) {
if err := contextError(r.ctx); err != nil {
return 0, err
}
return r.reader.Read(buffer)
}
func contextError(ctx context.Context) error {
select {
case <-ctx.Done():
@@ -64,7 +64,10 @@ func TestFlattenWhiteCompositesAlpha(t *testing.T) {
source.SetNRGBA(0, 0, color.NRGBA{R: 10, G: 20, B: 30, A: 0})
source.SetNRGBA(1, 0, color.NRGBA{R: 100, G: 50, B: 0, A: 128})
result := flattenWhite(source)
result, err := flattenWhite(context.Background(), source)
if err != nil {
t.Fatalf("flatten image: %v", err)
}
if got := result.NRGBAAt(0, 0); got != (color.NRGBA{R: 255, G: 255, B: 255, A: 255}) {
t.Fatalf("transparent pixel: got %#v", got)
}
@@ -154,6 +157,28 @@ func TestProcessHonorsCanceledContext(t *testing.T) {
}
}
func TestResizeStopsAfterCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
source := cancelingImage{
Image: image.NewNRGBA(image.Rect(0, 0, 256, 256)),
cancel: cancel,
}
_, err := resizeContext(ctx, source, 128, 128)
if !errors.Is(err, context.Canceled) {
t.Fatalf("error: got %v, want %v", err, context.Canceled)
}
}
type cancelingImage struct {
image.Image
cancel context.CancelFunc
}
func (i cancelingImage) At(x, y int) color.Color {
i.cancel()
return i.Image.At(x, y)
}
func TestEstimateMemoryWeight(t *testing.T) {
weight, err := EstimateMemoryWeight(1000, 320, 200)
if err != nil {
@@ -1,14 +1,21 @@
package transform
import (
"context"
"image"
"image/color"
)
func fit(source image.Image, maxWidth, maxHeight int, upscale bool) *image.NRGBA {
func fitContext(
ctx context.Context,
source image.Image,
maxWidth int,
maxHeight int,
upscale bool,
) (*image.NRGBA, error) {
bounds := source.Bounds()
width, height := fitDimensions(bounds.Dx(), bounds.Dy(), maxWidth, maxHeight, upscale)
return resize(source, width, height)
return resizeContext(ctx, source, width, height)
}
func fitDimensions(width, height, maxWidth, maxHeight int, upscale bool) (int, int) {
@@ -43,15 +50,23 @@ func (i croppedImage) At(x, y int) color.Color {
return i.source.At(i.offset.X+x, i.offset.Y+y)
}
func resize(source image.Image, width, height int) *image.NRGBA {
func resizeContext(ctx context.Context, source image.Image, width, height int) (*image.NRGBA, error) {
bounds := source.Bounds()
if bounds.Dx() == width && bounds.Dy() == height {
return copyNRGBA(source)
return copyNRGBAContext(ctx, source)
}
output := image.NewNRGBA(image.Rect(0, 0, width, height))
for y := 0; y < height; y++ {
if err := contextError(ctx); err != nil {
return nil, err
}
y0, y1, fy := sampleCoordinate(y, height, bounds.Dy())
for x := 0; x < width; x++ {
if x&255 == 0 {
if err := contextError(ctx); err != nil {
return nil, err
}
}
x0, x1, fx := sampleCoordinate(x, width, bounds.Dx())
c00 := color.NRGBAModel.Convert(source.At(bounds.Min.X+x0, bounds.Min.Y+y0)).(color.NRGBA)
c10 := color.NRGBAModel.Convert(source.At(bounds.Min.X+x1, bounds.Min.Y+y0)).(color.NRGBA)
@@ -60,7 +75,7 @@ func resize(source image.Image, width, height int) *image.NRGBA {
output.SetNRGBA(x, y, interpolate(c00, c10, c01, c11, fx, fy))
}
}
return output
return output, nil
}
func sampleCoordinate(destination, destinationSize, sourceSize int) (int, int, uint32) {
@@ -93,13 +108,21 @@ func interpolateChannel(c00, c10, c01, c11 uint8, fx, fy uint32) uint8 {
return uint8((value + 1<<31) >> 32)
}
func copyNRGBA(source image.Image) *image.NRGBA {
func copyNRGBAContext(ctx context.Context, source image.Image) (*image.NRGBA, error) {
bounds := source.Bounds()
output := image.NewNRGBA(image.Rect(0, 0, bounds.Dx(), bounds.Dy()))
for y := 0; y < bounds.Dy(); y++ {
if err := contextError(ctx); err != nil {
return nil, err
}
for x := 0; x < bounds.Dx(); x++ {
if x&255 == 0 {
if err := contextError(ctx); err != nil {
return nil, err
}
}
output.Set(x, y, source.At(bounds.Min.X+x, bounds.Min.Y+y))
}
}
return output
return output, nil
}
+60 -4
View File
@@ -27,6 +27,7 @@ import (
type Repository interface {
AttachmentJob(context.Context, string) (model.AttachmentJob, error)
ClaimAttachmentJob(context.Context, string, string, int, int, string, time.Duration) (model.AttachmentJobLease, error)
RenewAttachmentJobLease(context.Context, model.AttachmentJobLease, time.Duration) (bool, error)
CompleteAttachmentJob(context.Context, model.AttachmentJobLease, []model.Variant) (model.JobMutationResult, error)
RetryAttachmentJob(context.Context, model.AttachmentJobLease, string, time.Time) (model.JobMutationResult, error)
FailAttachmentJob(context.Context, model.AttachmentJobLease, string) (model.JobMutationResult, error)
@@ -59,6 +60,7 @@ type Config struct {
WorkerID string
Concurrency int
LeaseDuration time.Duration
HeartbeatInterval time.Duration
ProcessingTimeout time.Duration
ReclaimInterval time.Duration
MaxAttempts int
@@ -119,6 +121,7 @@ func New(
func validateConfig(config Config) error {
if config.WorkerID == "" || config.Concurrency < 1 || config.LeaseDuration <= 0 ||
config.HeartbeatInterval <= 0 || config.HeartbeatInterval >= config.LeaseDuration ||
config.ProcessingTimeout <= 0 || config.ProcessingTimeout >= config.LeaseDuration ||
config.ReclaimInterval <= 0 || config.MaxAttempts < 1 || config.RetryBase <= 0 ||
config.RetryMax < config.RetryBase || config.MaxBytes < 1 || config.MaxPixels < 1 || config.MemoryBytes < 1 {
@@ -218,9 +221,17 @@ func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Deliv
return ErrInjectedFailure
}
processingCtx, cancel := context.WithTimeout(ctx, w.config.ProcessingTimeout)
processingCtx, cancelProcessing := context.WithTimeout(ctx, w.config.ProcessingTimeout)
heartbeatCtx, cancelHeartbeat := context.WithCancel(ctx)
heartbeatDone := make(chan error, 1)
go func() { heartbeatDone <- w.maintainLease(heartbeatCtx, cancelProcessing, lease) }()
variants, err := w.process(processingCtx, lease)
cancel()
cancelProcessing()
cancelHeartbeat()
heartbeatErr := <-heartbeatDone
if heartbeatErr != nil {
return heartbeatErr
}
if err != nil {
result, err = w.resolveProcessingFailure(ctx, delivery, lease, err)
return err
@@ -252,6 +263,34 @@ func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Deliv
return nil
}
func (w *Worker) maintainLease(
ctx context.Context,
cancelProcessing context.CancelFunc,
lease model.AttachmentJobLease,
) error {
ticker := time.NewTicker(w.config.HeartbeatInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return nil
case <-ticker.C:
renewed, err := w.repository.RenewAttachmentJobLease(ctx, lease, w.config.LeaseDuration)
if err != nil {
if ctx.Err() != nil {
return nil
}
cancelProcessing()
return fmt.Errorf("renew attachment job lease: %w", err)
}
if !renewed {
cancelProcessing()
return model.ErrStaleLease
}
}
}
}
func (w *Worker) resolveClaim(
ctx context.Context,
delivery redisstream.Delivery,
@@ -295,15 +334,18 @@ func (w *Worker) process(ctx context.Context, lease model.AttachmentJobLease) ([
return nil, err
}
defer func() { _ = body.Close() }()
data, err := io.ReadAll(io.LimitReader(body, w.config.MaxBytes+1))
data, err := io.ReadAll(io.LimitReader(contextReader{ctx: ctx, reader: body}, w.config.MaxBytes+1))
if err != nil {
return nil, err
}
if int64(len(data)) > w.config.MaxBytes {
return nil, transform.ErrPayloadTooLarge
}
imageConfig, _, err := image.DecodeConfig(bytes.NewReader(data))
imageConfig, _, err := image.DecodeConfig(contextReader{ctx: ctx, reader: bytes.NewReader(data)})
if err != nil {
if ctx.Err() != nil {
return nil, ctx.Err()
}
return nil, transform.ErrInvalidImage
}
weight, err := transform.EstimateMemoryWeight(int64(len(data)), imageConfig.Width, imageConfig.Height)
@@ -346,6 +388,20 @@ func (w *Worker) process(ctx context.Context, lease model.AttachmentJobLease) ([
return result, nil
}
type contextReader struct {
ctx context.Context
reader io.Reader
}
func (r contextReader) Read(buffer []byte) (int, error) {
select {
case <-r.ctx.Done():
return 0, r.ctx.Err()
default:
return r.reader.Read(buffer)
}
}
var errWorkerCapacity = errors.New("image exceeds worker memory budget")
func variantObjectKey(attachmentID string, pipelineVersion int, name string, value transform.Variant) string {
@@ -1,13 +1,16 @@
package worker
import (
"context"
"errors"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/require"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
)
@@ -46,11 +49,61 @@ func TestProcessingErrorClassification(t *testing.T) {
}
}
func TestLeaseHeartbeatRenewsUntilCanceled(t *testing.T) {
var renewals atomic.Int32
repository := &repositoryStub{renew: func(context.Context, model.AttachmentJobLease, time.Duration) (bool, error) {
renewals.Add(1)
return true, nil
}}
config := validConfig()
config.HeartbeatInterval = 5 * time.Millisecond
worker := &Worker{repository: repository, config: config}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- worker.maintainLease(ctx, cancel, testLease()) }()
require.Eventually(t, func() bool { return renewals.Load() >= 2 }, time.Second, time.Millisecond)
cancel()
require.NoError(t, <-done)
}
func TestLeaseHeartbeatCancelsProcessingWhenLeaseIsLost(t *testing.T) {
repository := &repositoryStub{renew: func(context.Context, model.AttachmentJobLease, time.Duration) (bool, error) {
return false, nil
}}
config := validConfig()
config.HeartbeatInterval = time.Millisecond
worker := &Worker{repository: repository, config: config}
ctx, cancel := context.WithCancel(context.Background())
err := worker.maintainLease(ctx, cancel, testLease())
require.ErrorIs(t, err, model.ErrStaleLease)
require.ErrorIs(t, ctx.Err(), context.Canceled)
}
type repositoryStub struct {
Repository
renew func(context.Context, model.AttachmentJobLease, time.Duration) (bool, error)
}
func (s *repositoryStub) RenewAttachmentJobLease(
ctx context.Context,
lease model.AttachmentJobLease,
duration time.Duration,
) (bool, error) {
return s.renew(ctx, lease, duration)
}
func testLease() model.AttachmentJobLease {
return model.AttachmentJobLease{Job: model.AttachmentJob{
ID: "11111111-1111-4111-8111-111111111111", LeaseToken: "lease-token", LeaseOwner: "worker-1",
}}
}
func validConfig() Config {
return Config{
WorkerID: "worker-1", Concurrency: 2, LeaseDuration: 30 * time.Second,
ProcessingTimeout: 20 * time.Second, ReclaimInterval: 5 * time.Second,
MaxAttempts: 5, RetryBase: time.Second, RetryMax: 30 * time.Second,
HeartbeatInterval: 5 * time.Second, ProcessingTimeout: 20 * time.Second,
ReclaimInterval: 5 * time.Second,
MaxAttempts: 5, RetryBase: time.Second, RetryMax: 30 * time.Second,
MaxBytes: 10 * 1024 * 1024, MaxPixels: 40_000_000, MemoryBytes: 704 * 1024 * 1024,
}
}
+29
View File
@@ -16,6 +16,7 @@ type Store interface {
MarkOutboxPublished(context.Context, string, string) (bool, error)
ReleaseOutboxEvent(context.Context, string, string, string, time.Time) (bool, error)
RecoverExpiredOutboxEvents(context.Context, time.Time) (int, error)
DeletePublishedOutboxEvents(context.Context, time.Time, int) (int, error)
}
type Publisher interface {
@@ -34,6 +35,9 @@ type Config struct {
PollInterval time.Duration
RecoveryInterval time.Duration
RetryDelay time.Duration
Retention time.Duration
CleanupInterval time.Duration
CleanupBatchSize int
}
type Dispatcher struct {
@@ -72,6 +76,12 @@ func (c Config) validate() error {
return errors.New("outbox recovery interval must be positive")
case c.RetryDelay <= 0:
return errors.New("outbox retry delay must be positive")
case c.Retention <= 0:
return errors.New("outbox retention must be positive")
case c.CleanupInterval <= 0:
return errors.New("outbox cleanup interval must be positive")
case c.CleanupBatchSize <= 0 || c.CleanupBatchSize > 1000:
return errors.New("outbox cleanup batch size must be between 1 and 1000")
default:
return nil
}
@@ -84,8 +94,10 @@ func (d *Dispatcher) Run(ctx context.Context) error {
pollTicker := time.NewTicker(d.config.PollInterval)
recoveryTicker := time.NewTicker(d.config.RecoveryInterval)
cleanupTicker := time.NewTicker(d.config.CleanupInterval)
defer pollTicker.Stop()
defer recoveryTicker.Stop()
defer cleanupTicker.Stop()
if err := d.RecoverExpired(ctx); err != nil {
if ctx.Err() != nil {
@@ -112,10 +124,27 @@ func (d *Dispatcher) Run(ctx context.Context) error {
}
return err
}
case <-cleanupTicker.C:
if _, err := d.CleanupPublished(ctx); err != nil {
if ctx.Err() != nil {
return nil
}
return err
}
}
}
}
func (d *Dispatcher) CleanupPublished(ctx context.Context) (int, error) {
deleted, err := d.store.DeletePublishedOutboxEvents(
ctx, d.now().Add(-d.config.Retention), d.config.CleanupBatchSize,
)
if err != nil {
return 0, fmt.Errorf("delete published outbox events: %w", err)
}
return deleted, nil
}
func (d *Dispatcher) DispatchOnce(ctx context.Context) (int, error) {
events, err := d.store.ClaimOutboxEvents(ctx, d.config.BatchSize, d.config.LeaseDuration)
if err != nil {
+34 -8
View File
@@ -70,6 +70,18 @@ func TestDispatcherRejectsStaleLeaseAfterPublish(t *testing.T) {
require.Zero(t, published)
}
func TestDispatcherCleansPublishedEventsBeforeRetentionCutoff(t *testing.T) {
store := &storeStub{deleted: 2}
dispatcher := newTestDispatcher(t, store, &publisherStub{})
dispatcher.now = func() time.Time { return time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC) }
deleted, err := dispatcher.CleanupPublished(context.Background())
require.NoError(t, err)
require.Equal(t, 2, deleted)
require.Equal(t, dispatcher.now().Add(-24*time.Hour), store.publishedBefore)
require.Equal(t, 100, store.cleanupLimit)
}
func TestDispatcherStopsAfterCancellation(t *testing.T) {
store := &storeStub{}
publisher := &publisherStub{}
@@ -96,6 +108,9 @@ func newTestDispatcher(t *testing.T, store Store, publisher Publisher) *Dispatch
PollInterval: time.Millisecond,
RecoveryInterval: time.Second,
RetryDelay: time.Second,
Retention: 24 * time.Hour,
CleanupInterval: time.Hour,
CleanupBatchSize: 100,
})
require.NoError(t, err)
return dispatcher
@@ -114,14 +129,17 @@ func testPayload() json.RawMessage {
}
type storeStub struct {
mu sync.Mutex
events []model.OutboxEvent
markErr error
markLost bool
marked []string
released string
nextAttemptAt time.Time
recoveredCalls int
mu sync.Mutex
events []model.OutboxEvent
markErr error
markLost bool
marked []string
released string
nextAttemptAt time.Time
recoveredCalls int
deleted int
publishedBefore time.Time
cleanupLimit int
}
func (s *storeStub) ClaimOutboxEvents(context.Context, int, time.Duration) ([]model.OutboxEvent, error) {
@@ -164,6 +182,14 @@ func (s *storeStub) RecoverExpiredOutboxEvents(context.Context, time.Time) (int,
return 0, nil
}
func (s *storeStub) DeletePublishedOutboxEvents(_ context.Context, before time.Time, limit int) (int, error) {
s.mu.Lock()
defer s.mu.Unlock()
s.publishedBefore = before
s.cleanupLimit = limit
return s.deleted, nil
}
type publisherStub struct {
mu sync.Mutex
messageID string
+32
View File
@@ -0,0 +1,32 @@
package postgres
import (
"context"
"fmt"
"time"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
)
func (s *Store) RenewAttachmentJobLease(
ctx context.Context,
lease model.AttachmentJobLease,
duration time.Duration,
) (bool, error) {
if duration <= 0 || lease.Job.ID == "" || lease.Job.LeaseToken == "" || lease.Job.LeaseOwner == "" {
return false, model.ErrInvalidInput
}
const query = `
UPDATE attachment_jobs
SET lease_until = clock_timestamp() + ($4::bigint * interval '1 microsecond'),
updated_at = clock_timestamp()
WHERE id = $1 AND lease_token = $2 AND lease_owner = $3
AND status = 'processing' AND lease_until > clock_timestamp()`
result, err := s.pool.Exec(
ctx, query, lease.Job.ID, lease.Job.LeaseToken, lease.Job.LeaseOwner, duration.Microseconds(),
)
if err != nil {
return false, fmt.Errorf("renew attachment job lease: %w", err)
}
return result.RowsAffected() == 1, nil
}
+23
View File
@@ -111,3 +111,26 @@ func (s *Store) RecoverExpiredOutboxEvents(ctx context.Context, now time.Time) (
}
return int(result.RowsAffected()), nil
}
func (s *Store) DeletePublishedOutboxEvents(ctx context.Context, publishedBefore time.Time, limit int) (int, error) {
if publishedBefore.IsZero() || limit < 1 || limit > 1000 {
return 0, model.ErrInvalidInput
}
const query = `
WITH candidates AS (
SELECT id
FROM outbox_events
WHERE published_at < $1
ORDER BY published_at, id
LIMIT $2
FOR UPDATE SKIP LOCKED
)
DELETE FROM outbox_events event
USING candidates
WHERE event.id = candidates.id`
result, err := s.pool.Exec(ctx, query, publishedBefore.UTC(), limit)
if err != nil {
return 0, fmt.Errorf("delete published outbox events: %w", err)
}
return int(result.RowsAffected()), nil
}
+1 -57
View File
@@ -1,6 +1,7 @@
# Адрес и внутренний порт HTTP-сервера.
APP_HOST=0.0.0.0
APP_PORT=8080
MESSENGER_CONFIG_PATH=config/messenger.yaml
# Адреса публикации сервисов на локальной машине.
API_BIND_ADDRESS=127.0.0.1:8080
@@ -8,11 +9,6 @@ SWAGGER_BIND_ADDRESS=127.0.0.1:8081
PROMETHEUS_BIND_ADDRESS=127.0.0.1:9090
GRAFANA_BIND_ADDRESS=127.0.0.1:3000
# Время жизни сессии, идемпотентности и используемое хранилище.
SESSION_TTL_SECONDS=3600
IDEMPOTENCY_TTL_SECONDS=86400
SESSION_STORE=valkey
# Число API-реплик за Nginx.
API_REPLICAS=2
@@ -29,28 +25,6 @@ CACHE_URL=redis://default:local-valkey-password@valkey:6379/0
# Redis Streams используется только как устойчивый брокер заданий.
REDIS_BROKER_PASSWORD=local-redis-broker-password
BROKER_URL=redis://default:local-redis-broker-password@redis-broker:6379/0
BROKER_STREAM=attachment-jobs
BROKER_CONSUMER_GROUP=attachment-workers
BROKER_FAILED_STREAM=attachment-jobs-failed
BROKER_FAILED_MAX_LENGTH=10000
# Версия pipeline, outbox и ограниченные повторы worker.
PIPELINE_VERSION=1
OUTBOX_BATCH_SIZE=32
OUTBOX_LEASE_SECONDS=15
OUTBOX_POLL_SECONDS=1
OUTBOX_RECOVERY_SECONDS=10
OUTBOX_RETRY_SECONDS=2
WORKER_CONCURRENCY=2
WORKER_PREFETCH=4
WORKER_LEASE_SECONDS=30
WORKER_PROCESSING_TIMEOUT_SECONDS=20
WORKER_READ_BLOCK_SECONDS=2
WORKER_RECLAIM_SECONDS=5
WORKER_MAX_ATTEMPTS=5
WORKER_RETRY_BASE_SECONDS=1
WORKER_RETRY_MAX_SECONDS=30
WORKER_MEMORY_BYTES=738197504
# Административные данные MinIO используются только при bootstrap.
MINIO_ROOT_USER=local-minio-admin
@@ -62,36 +36,6 @@ S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY=messenger-app
S3_SECRET_KEY=local-messenger-s3-password
S3_USE_TLS=false
S3_TIMEOUT_SECONDS=5
ATTACHMENT_MAX_BYTES=10485760
ATTACHMENT_MAX_PIXELS=40000000
ATTACHMENT_BODY_MEMORY_BYTES=67108864
ATTACHMENT_DECODE_MEMORY_BYTES=402653184
# Размер пула и обслуживание соединений PostgreSQL.
DB_MIN_CONNS=1
DB_MAX_CONNS=10
DB_MAX_CONNECTION_AGE_SECONDS=1800
DB_MAX_CONNECTION_IDLE_SECONDS=300
DB_HEALTH_CHECK_SECONDS=30
# Таймауты зависимостей, миграций и HTTP-запросов.
DEPENDENCY_TIMEOUT_SECONDS=5
MIGRATION_TIMEOUT_SECONDS=30
REQUEST_TIMEOUT_SECONDS=5
HTTP_READ_HEADER_TIMEOUT_SECONDS=5
HTTP_READ_TIMEOUT_SECONDS=10
HTTP_WRITE_TIMEOUT_SECONDS=10
HTTP_IDLE_TIMEOUT_SECONDS=60
HTTP_SHUTDOWN_TIMEOUT_SECONDS=5
# Параметры Argon2id для новых хешей паролей.
PASSWORD_MEMORY_KIB=65536
PASSWORD_ITERATIONS=3
PASSWORD_PARALLELISM=2
PASSWORD_SALT_LENGTH=16
PASSWORD_KEY_LENGTH=32
# Порт Swagger внутри конфигурации CORS API.
SWAGGER_PORT=8081
+3
View File
@@ -0,0 +1,3 @@
CREATE INDEX outbox_events_published_at_idx
ON outbox_events (published_at, id)
WHERE published_at IS NOT NULL;