From aa5d6a6e163a0940d4e6c40482c18d6f998eb2a3 Mon Sep 17 00:00:00 2001 From: dizi Date: Fri, 30 Oct 2026 13:45:49 +0300 Subject: [PATCH] fix: enforce pipeline lifecycle limits --- cmd/lab5experiment/main.go | 8 +- compose.yaml | 124 +++-------------- config/messenger.yaml | 76 +++++++++++ internal/bootstrap/pipeline.go | 7 +- internal/config/config.go | 112 +++++++++------- internal/config/config_test.go | 126 ++++++++++++++++-- internal/config/loader.go | 47 +++++++ internal/config/pipeline.go | 110 +++++++++------ .../messenger/attachment/transform/exif.go | 37 ++++- .../attachment/transform/processor.go | 59 ++++++-- .../attachment/transform/processor_test.go | 27 +++- .../messenger/attachment/transform/resize.go | 37 ++++- .../messenger/attachment/worker/worker.go | 64 ++++++++- .../attachment/worker/worker_test.go | 57 +++++++- internal/outbox/dispatcher.go | 29 ++++ internal/outbox/dispatcher_test.go | 42 ++++-- internal/postgres/attachment_lease.go | 32 +++++ internal/postgres/outbox.go | 23 ++++ lab5.env.example | 58 +------- migrations/0007_outbox_retention.sql | 3 + 20 files changed, 770 insertions(+), 308 deletions(-) create mode 100644 config/messenger.yaml create mode 100644 internal/config/loader.go create mode 100644 internal/postgres/attachment_lease.go create mode 100644 migrations/0007_outbox_retention.sql diff --git a/cmd/lab5experiment/main.go b/cmd/lab5experiment/main.go index b6532d6d..3749235f 100644 --- a/cmd/lab5experiment/main.go +++ b/cmd/lab5experiment/main.go @@ -169,6 +169,7 @@ func newExperiment(ctx context.Context) (*experiment, func(), error) { dispatcher, err := outbox.New(repository, broker, nil, outbox.Config{ BatchSize: 32, LeaseDuration: 10 * time.Second, PollInterval: time.Second, RecoveryInterval: 10 * time.Second, RetryDelay: time.Second, + Retention: 24 * time.Hour, CleanupInterval: time.Hour, CleanupBatchSize: 1000, }) if err != nil { pool.Close() @@ -185,9 +186,10 @@ func newExperiment(ctx context.Context) (*experiment, func(), error) { slog.New(slog.NewTextHandler(io.Discard, nil)), repository, broker, objects, processor, nil, worker.Config{ WorkerID: consumer, Concurrency: 1, - LeaseDuration: experimentLease, ProcessingTimeout: time.Second, - ReclaimInterval: experimentLease, - MaxAttempts: maxAttempts, RetryBase: time.Second, RetryMax: 4 * time.Second, + LeaseDuration: experimentLease, HeartbeatInterval: 500 * time.Millisecond, + ProcessingTimeout: time.Second, + ReclaimInterval: experimentLease, + MaxAttempts: maxAttempts, RetryBase: time.Second, RetryMax: 4 * time.Second, MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels, MemoryBytes: settings.MemoryBytes, }, diff --git a/compose.yaml b/compose.yaml index b2137b4f..48ad2db0 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,3 +1,9 @@ +x-messenger-config-volume: &messenger-config-volume + type: bind + source: ./config/messenger.yaml + target: /etc/messenger/messenger.yaml + read_only: true + services: postgres: image: postgres:17.6-alpine3.22 @@ -115,14 +121,10 @@ services: image: ${API_IMAGE:-mtusi-messenger:lab5} command: ["/app/messenger", "migrate"] environment: + MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml DATABASE_URL: ${DATABASE_URL} - DB_MIN_CONNS: ${DB_MIN_CONNS} - DB_MAX_CONNS: ${DB_MAX_CONNS} - DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS} - DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS} - DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS} - DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS} - MIGRATION_TIMEOUT_SECONDS: ${MIGRATION_TIMEOUT_SECONDS} + volumes: + - *messenger-config-volume depends_on: postgres: condition: service_healthy @@ -137,43 +139,19 @@ services: build: . image: ${API_IMAGE:-mtusi-messenger:lab5} environment: + MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml APP_HOST: ${APP_HOST} APP_PORT: ${APP_PORT} DATABASE_URL: ${DATABASE_URL} CACHE_URL: ${CACHE_URL} - SESSION_STORE: ${SESSION_STORE} - SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS} - IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS} - DB_MIN_CONNS: ${DB_MIN_CONNS} - DB_MAX_CONNS: ${DB_MAX_CONNS} - DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS} - DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS} - DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS} - DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS} - REQUEST_TIMEOUT_SECONDS: ${REQUEST_TIMEOUT_SECONDS} - HTTP_READ_HEADER_TIMEOUT_SECONDS: ${HTTP_READ_HEADER_TIMEOUT_SECONDS} - HTTP_READ_TIMEOUT_SECONDS: ${HTTP_READ_TIMEOUT_SECONDS} - HTTP_WRITE_TIMEOUT_SECONDS: ${HTTP_WRITE_TIMEOUT_SECONDS} - HTTP_IDLE_TIMEOUT_SECONDS: ${HTTP_IDLE_TIMEOUT_SECONDS} - HTTP_SHUTDOWN_TIMEOUT_SECONDS: ${HTTP_SHUTDOWN_TIMEOUT_SECONDS} - PASSWORD_MEMORY_KIB: ${PASSWORD_MEMORY_KIB} - PASSWORD_ITERATIONS: ${PASSWORD_ITERATIONS} - PASSWORD_PARALLELISM: ${PASSWORD_PARALLELISM} - PASSWORD_SALT_LENGTH: ${PASSWORD_SALT_LENGTH} - PASSWORD_KEY_LENGTH: ${PASSWORD_KEY_LENGTH} SWAGGER_PORT: ${SWAGGER_PORT} S3_ENDPOINT: ${S3_ENDPOINT} S3_REGION: ${S3_REGION} S3_BUCKET: ${S3_BUCKET} S3_ACCESS_KEY: ${S3_ACCESS_KEY} S3_SECRET_KEY: ${S3_SECRET_KEY} - S3_USE_TLS: ${S3_USE_TLS} - S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS} - ATTACHMENT_MAX_BYTES: ${ATTACHMENT_MAX_BYTES} - ATTACHMENT_MAX_PIXELS: ${ATTACHMENT_MAX_PIXELS} - ATTACHMENT_BODY_MEMORY_BYTES: ${ATTACHMENT_BODY_MEMORY_BYTES} - ATTACHMENT_DECODE_MEMORY_BYTES: ${ATTACHMENT_DECODE_MEMORY_BYTES} - PIPELINE_VERSION: ${PIPELINE_VERSION} + volumes: + - *messenger-config-volume expose: - "8080" depends_on: @@ -201,25 +179,11 @@ services: image: ${API_IMAGE:-mtusi-messenger:lab5} command: ["/app/messenger", "dispatch"] environment: + MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml DATABASE_URL: ${DATABASE_URL} - DB_MIN_CONNS: ${DB_MIN_CONNS} - DB_MAX_CONNS: ${DB_MAX_CONNS} - DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS} - DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS} - DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS} - DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS} BROKER_URL: ${BROKER_URL} - BROKER_STREAM: ${BROKER_STREAM} - BROKER_CONSUMER_GROUP: ${BROKER_CONSUMER_GROUP} - BROKER_FAILED_STREAM: ${BROKER_FAILED_STREAM} - BROKER_FAILED_MAX_LENGTH: ${BROKER_FAILED_MAX_LENGTH} - OUTBOX_BATCH_SIZE: ${OUTBOX_BATCH_SIZE} - OUTBOX_LEASE_SECONDS: ${OUTBOX_LEASE_SECONDS} - OUTBOX_POLL_SECONDS: ${OUTBOX_POLL_SECONDS} - OUTBOX_RECOVERY_SECONDS: ${OUTBOX_RECOVERY_SECONDS} - OUTBOX_RETRY_SECONDS: ${OUTBOX_RETRY_SECONDS} - PIPELINE_METRICS_HOST: 0.0.0.0 - PIPELINE_METRICS_PORT: 9091 + volumes: + - *messenger-config-volume expose: - "9091" depends_on: @@ -241,39 +205,16 @@ services: image: ${API_IMAGE:-mtusi-messenger:lab5} command: ["/app/messenger", "worker"] environment: + MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml DATABASE_URL: ${DATABASE_URL} - DB_MIN_CONNS: ${DB_MIN_CONNS} - DB_MAX_CONNS: ${DB_MAX_CONNS} - DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS} - DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS} - DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS} - DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS} S3_ENDPOINT: ${S3_ENDPOINT} S3_REGION: ${S3_REGION} S3_BUCKET: ${S3_BUCKET} S3_ACCESS_KEY: ${S3_ACCESS_KEY} S3_SECRET_KEY: ${S3_SECRET_KEY} - S3_USE_TLS: ${S3_USE_TLS} - S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS} BROKER_URL: ${BROKER_URL} - BROKER_STREAM: ${BROKER_STREAM} - BROKER_CONSUMER_GROUP: ${BROKER_CONSUMER_GROUP} - BROKER_FAILED_STREAM: ${BROKER_FAILED_STREAM} - BROKER_FAILED_MAX_LENGTH: ${BROKER_FAILED_MAX_LENGTH} - WORKER_CONCURRENCY: ${WORKER_CONCURRENCY} - WORKER_PREFETCH: ${WORKER_PREFETCH} - WORKER_LEASE_SECONDS: ${WORKER_LEASE_SECONDS} - WORKER_PROCESSING_TIMEOUT_SECONDS: ${WORKER_PROCESSING_TIMEOUT_SECONDS} - WORKER_READ_BLOCK_SECONDS: ${WORKER_READ_BLOCK_SECONDS} - WORKER_RECLAIM_SECONDS: ${WORKER_RECLAIM_SECONDS} - WORKER_MAX_ATTEMPTS: ${WORKER_MAX_ATTEMPTS} - WORKER_RETRY_BASE_SECONDS: ${WORKER_RETRY_BASE_SECONDS} - WORKER_RETRY_MAX_SECONDS: ${WORKER_RETRY_MAX_SECONDS} - ATTACHMENT_MAX_BYTES: ${ATTACHMENT_MAX_BYTES} - ATTACHMENT_MAX_PIXELS: ${ATTACHMENT_MAX_PIXELS} - WORKER_MEMORY_BYTES: ${WORKER_MEMORY_BYTES} - PIPELINE_METRICS_HOST: 0.0.0.0 - PIPELINE_METRICS_PORT: 9091 + volumes: + - *messenger-config-volume expose: - "9091" depends_on: @@ -351,39 +292,16 @@ services: image: ${API_IMAGE:-mtusi-messenger:lab5} command: ["/app/lab5experiment"] environment: + MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml DATABASE_URL: ${DATABASE_URL} - DB_MIN_CONNS: ${DB_MIN_CONNS} - DB_MAX_CONNS: ${DB_MAX_CONNS} - DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS} - DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS} - DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS} - DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS} S3_ENDPOINT: ${S3_ENDPOINT} S3_REGION: ${S3_REGION} S3_BUCKET: ${S3_BUCKET} S3_ACCESS_KEY: ${S3_ACCESS_KEY} S3_SECRET_KEY: ${S3_SECRET_KEY} - S3_USE_TLS: ${S3_USE_TLS} - S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS} BROKER_URL: ${BROKER_URL} - BROKER_STREAM: ${BROKER_STREAM} - BROKER_CONSUMER_GROUP: ${BROKER_CONSUMER_GROUP} - BROKER_FAILED_STREAM: ${BROKER_FAILED_STREAM} - BROKER_FAILED_MAX_LENGTH: ${BROKER_FAILED_MAX_LENGTH} - WORKER_CONCURRENCY: 1 - WORKER_PREFETCH: 32 - WORKER_LEASE_SECONDS: 2 - WORKER_PROCESSING_TIMEOUT_SECONDS: 1 - WORKER_READ_BLOCK_SECONDS: 1 - WORKER_RECLAIM_SECONDS: 2 - WORKER_MAX_ATTEMPTS: ${WORKER_MAX_ATTEMPTS} - WORKER_RETRY_BASE_SECONDS: 1 - WORKER_RETRY_MAX_SECONDS: 4 - ATTACHMENT_MAX_BYTES: ${ATTACHMENT_MAX_BYTES} - ATTACHMENT_MAX_PIXELS: ${ATTACHMENT_MAX_PIXELS} - WORKER_MEMORY_BYTES: ${WORKER_MEMORY_BYTES} - PIPELINE_METRICS_HOST: 0.0.0.0 - PIPELINE_METRICS_PORT: 9091 + volumes: + - *messenger-config-volume depends_on: migrate: condition: service_completed_successfully diff --git a/config/messenger.yaml b/config/messenger.yaml new file mode 100644 index 00000000..031e1522 --- /dev/null +++ b/config/messenger.yaml @@ -0,0 +1,76 @@ +version: 1 + +server: + session_store: valkey + session_ttl: 1h + idempotency_ttl: 24h + request_timeout: 5s + http: + read_header_timeout: 5s + read_timeout: 10s + write_timeout: 10s + idle_timeout: 1m + shutdown_timeout: 5s + password: + memory_kib: 65536 + iterations: 3 + parallelism: 2 + salt_length: 16 + key_length: 32 + +database: + min_connections: 1 + max_connections: 10 + max_connection_age: 30m + max_connection_idle: 5m + health_check_period: 30s + dependency_timeout: 5s + +migration: + timeout: 30s + +object_storage: + use_tls: false + timeout: 5s + +attachments: + max_bytes: 10485760 + max_pixels: 40000000 + body_memory_bytes: 67108864 + decode_memory_bytes: 402653184 + +pipeline: + version: 1 + +broker: + stream: attachment-jobs + consumer_group: attachment-workers + failed_stream: attachment-jobs-failed + failed_stream_max_length: 10000 + +outbox: + batch_size: 32 + lease: 15s + poll_interval: 1s + recovery_interval: 10s + retry_delay: 2s + retention: 168h + cleanup_interval: 1m + cleanup_batch_size: 1000 + +worker: + concurrency: 2 + prefetch: 4 + lease: 30s + heartbeat_interval: 5s + processing_timeout: 20s + read_block: 2s + reclaim_interval: 5s + max_attempts: 5 + retry: + base: 1s + max: 30s + memory_bytes: 738197504 + +monitoring: + metrics_address: 0.0.0.0:9091 diff --git a/internal/bootstrap/pipeline.go b/internal/bootstrap/pipeline.go index 2f794c50..97ce3ec6 100644 --- a/internal/bootstrap/pipeline.go +++ b/internal/bootstrap/pipeline.go @@ -55,7 +55,8 @@ func RunDispatcher(logger *slog.Logger) error { dispatcher, err := outbox.New(postgres.New(pool), broker, metrics, outbox.Config{ BatchSize: settings.BatchSize, LeaseDuration: settings.LeaseDuration, PollInterval: settings.PollInterval, RecoveryInterval: settings.RecoveryInterval, - RetryDelay: settings.RetryDelay, + RetryDelay: settings.RetryDelay, Retention: settings.Retention, + CleanupInterval: settings.CleanupInterval, CleanupBatchSize: settings.CleanupBatchSize, }) if err != nil { return err @@ -112,8 +113,8 @@ func RunWorker(logger *slog.Logger) error { logger, postgres.New(pool), broker, objects, processor, metrics, worker.Config{ WorkerID: consumer, Concurrency: settings.Concurrency, LeaseDuration: settings.LeaseDuration, - ProcessingTimeout: settings.ProcessingTimeout, - ReclaimInterval: settings.ReclaimInterval, MaxAttempts: settings.MaxAttempts, + HeartbeatInterval: settings.HeartbeatInterval, ProcessingTimeout: settings.ProcessingTimeout, + ReclaimInterval: settings.ReclaimInterval, MaxAttempts: settings.MaxAttempts, RetryBase: settings.RetryBase, RetryMax: settings.RetryMax, MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels, MemoryBytes: settings.MemoryBytes, diff --git a/internal/config/config.go b/internal/config/config.go index b27025c5..977abc54 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -98,14 +98,16 @@ type objectStorageConfig struct { } func LoadServer() (ServerConfig, error) { - values := viper.New() - values.AutomaticEnv() - - host, err := requiredString(values, "APP_HOST") + values, err := loadValues() if err != nil { return ServerConfig{}, err } - port, err := requiredPort(values, "APP_PORT") + + host, err := requiredString(values, "server.host") + if err != nil { + return ServerConfig{}, err + } + port, err := requiredPort(values, "server.port") if err != nil { return ServerConfig{}, err } @@ -113,71 +115,71 @@ func LoadServer() (ServerConfig, error) { if err != nil { return ServerConfig{}, err } - cacheURL, err := requiredString(values, "CACHE_URL") + cacheURL, err := requiredString(values, "cache.url") if err != nil { return ServerConfig{}, err } - store, err := requiredString(values, "SESSION_STORE") + store, err := requiredString(values, "server.session_store") if err != nil { return ServerConfig{}, err } if store != "valkey" { - return ServerConfig{}, errors.New("SESSION_STORE must be valkey") + return ServerConfig{}, errors.New("server.session_store must be valkey") } - sessionTTL, err := seconds(values, "SESSION_TTL_SECONDS", 1, 86400*30) + sessionTTL, err := duration(values, "server.session_ttl", time.Second, 30*24*time.Hour) if err != nil { return ServerConfig{}, err } - idempotencyTTL, err := seconds(values, "IDEMPOTENCY_TTL_SECONDS", 86400, 86400*30) + idempotencyTTL, err := duration(values, "server.idempotency_ttl", 24*time.Hour, 30*24*time.Hour) if err != nil { return ServerConfig{}, err } - requestTimeout, err := seconds(values, "REQUEST_TIMEOUT_SECONDS", 1, 300) + requestTimeout, err := duration(values, "server.request_timeout", time.Second, 5*time.Minute) if err != nil { return ServerConfig{}, err } - readHeaderTimeout, err := seconds(values, "HTTP_READ_HEADER_TIMEOUT_SECONDS", 1, 300) + readHeaderTimeout, err := duration(values, "server.http.read_header_timeout", time.Second, 5*time.Minute) if err != nil { return ServerConfig{}, err } - readTimeout, err := seconds(values, "HTTP_READ_TIMEOUT_SECONDS", 1, 300) + readTimeout, err := duration(values, "server.http.read_timeout", time.Second, 5*time.Minute) if err != nil { return ServerConfig{}, err } - writeTimeout, err := seconds(values, "HTTP_WRITE_TIMEOUT_SECONDS", 1, 300) + writeTimeout, err := duration(values, "server.http.write_timeout", time.Second, 5*time.Minute) if err != nil { return ServerConfig{}, err } - idleTimeout, err := seconds(values, "HTTP_IDLE_TIMEOUT_SECONDS", 1, 3600) + idleTimeout, err := duration(values, "server.http.idle_timeout", time.Second, time.Hour) if err != nil { return ServerConfig{}, err } - shutdownTimeout, err := seconds(values, "HTTP_SHUTDOWN_TIMEOUT_SECONDS", 1, 300) + shutdownTimeout, err := duration(values, "server.http.shutdown_timeout", time.Second, 5*time.Minute) if err != nil { return ServerConfig{}, err } - passwordMemory, err := integer(values, "PASSWORD_MEMORY_KIB", 8*1024, 1024*1024) + passwordMemory, err := integer(values, "server.password.memory_kib", 8*1024, 1024*1024) if err != nil { return ServerConfig{}, err } - passwordIterations, err := integer(values, "PASSWORD_ITERATIONS", 1, 20) + passwordIterations, err := integer(values, "server.password.iterations", 1, 20) if err != nil { return ServerConfig{}, err } - passwordParallelism, err := integer(values, "PASSWORD_PARALLELISM", 1, 255) + passwordParallelism, err := integer(values, "server.password.parallelism", 1, 255) if err != nil { return ServerConfig{}, err } - passwordSaltLength, err := integer(values, "PASSWORD_SALT_LENGTH", 8, 1024) + passwordSaltLength, err := integer(values, "server.password.salt_length", 8, 1024) if err != nil { return ServerConfig{}, err } - passwordKeyLength, err := integer(values, "PASSWORD_KEY_LENGTH", 16, 1024) + passwordKeyLength, err := integer(values, "server.password.key_length", 16, 1024) if err != nil { return ServerConfig{}, err } - swaggerPort, err := requiredPort(values, "SWAGGER_PORT") + swaggerPort, err := requiredPort(values, "server.swagger_port") if err != nil { return ServerConfig{}, err } @@ -185,30 +187,30 @@ func LoadServer() (ServerConfig, error) { if err != nil { return ServerConfig{}, err } - attachmentMaxBytes, err := integer(values, "ATTACHMENT_MAX_BYTES", 1, 10*1024*1024) + attachmentMaxBytes, err := integer(values, "attachments.max_bytes", 1, 10*1024*1024) if err != nil { return ServerConfig{}, err } - attachmentMaxPixels, err := integer(values, "ATTACHMENT_MAX_PIXELS", 1, 40_000_000) + attachmentMaxPixels, err := integer(values, "attachments.max_pixels", 1, 40_000_000) if err != nil { return ServerConfig{}, err } - attachmentBodyMemory, err := integer(values, "ATTACHMENT_BODY_MEMORY_BYTES", 1, 1024*1024*1024) + attachmentBodyMemory, err := integer(values, "attachments.body_memory_bytes", 1, 1024*1024*1024) if err != nil { return ServerConfig{}, err } if int64(attachmentBodyMemory) < int64(attachmentMaxBytes) { - return ServerConfig{}, fmt.Errorf("ATTACHMENT_BODY_MEMORY_BYTES: must be at least %d", attachmentMaxBytes) + return ServerConfig{}, fmt.Errorf("attachments.body_memory_bytes: must be at least %d", attachmentMaxBytes) } - attachmentDecodeMemory, err := integer(values, "ATTACHMENT_DECODE_MEMORY_BYTES", 1, 1024*1024*1024) + attachmentDecodeMemory, err := integer(values, "attachments.decode_memory_bytes", 1, 1024*1024*1024) if err != nil { return ServerConfig{}, err } minimumDecodeMemory := int64(attachmentMaxPixels) * 8 if int64(attachmentDecodeMemory) < minimumDecodeMemory { - return ServerConfig{}, fmt.Errorf("ATTACHMENT_DECODE_MEMORY_BYTES: must be at least %d", minimumDecodeMemory) + return ServerConfig{}, fmt.Errorf("attachments.decode_memory_bytes: must be at least %d", minimumDecodeMemory) } - pipelineVersion, err := integer(values, "PIPELINE_VERSION", 1, 1000) + pipelineVersion, err := integer(values, "pipeline.version", 1, 1000) if err != nil { return ServerConfig{}, err } @@ -236,13 +238,15 @@ func LoadServer() (ServerConfig, error) { } func LoadMigration() (MigrationConfig, error) { - values := viper.New() - values.AutomaticEnv() + values, err := loadValues() + if err != nil { + return MigrationConfig{}, err + } database, err := loadDatabaseConfig(values) if err != nil { return MigrationConfig{}, err } - migrationTimeout, err := seconds(values, "MIGRATION_TIMEOUT_SECONDS", 1, 600) + migrationTimeout, err := duration(values, "migration.timeout", time.Second, 10*time.Minute) if err != nil { return MigrationConfig{}, err } @@ -255,8 +259,10 @@ func LoadMigration() (MigrationConfig, error) { } func LoadReconciliation() (ReconciliationConfig, error) { - values := viper.New() - values.AutomaticEnv() + values, err := loadValues() + if err != nil { + return ReconciliationConfig{}, err + } database, err := loadDatabaseConfig(values) if err != nil { return ReconciliationConfig{}, err @@ -276,31 +282,31 @@ func LoadReconciliation() (ReconciliationConfig, error) { } func loadObjectStorageConfig(values *viper.Viper) (objectStorageConfig, error) { - endpoint, err := requiredString(values, "S3_ENDPOINT") + endpoint, err := requiredString(values, "object_storage.endpoint") if err != nil { return objectStorageConfig{}, err } - region, err := requiredString(values, "S3_REGION") + region, err := requiredString(values, "object_storage.region") if err != nil { return objectStorageConfig{}, err } - bucket, err := requiredString(values, "S3_BUCKET") + bucket, err := requiredString(values, "object_storage.bucket") if err != nil { return objectStorageConfig{}, err } - accessKey, err := requiredString(values, "S3_ACCESS_KEY") + accessKey, err := requiredString(values, "object_storage.access_key") if err != nil { return objectStorageConfig{}, err } - secretKey, err := requiredString(values, "S3_SECRET_KEY") + secretKey, err := requiredString(values, "object_storage.secret_key") if err != nil { return objectStorageConfig{}, err } - useTLS, err := boolean(values, "S3_USE_TLS") + useTLS, err := boolean(values, "object_storage.use_tls") if err != nil { return objectStorageConfig{}, err } - timeout, err := seconds(values, "S3_TIMEOUT_SECONDS", 1, 300) + timeout, err := duration(values, "object_storage.timeout", time.Second, 5*time.Minute) if err != nil { return objectStorageConfig{}, err } @@ -311,31 +317,31 @@ func loadObjectStorageConfig(values *viper.Viper) (objectStorageConfig, error) { } func loadDatabaseConfig(values *viper.Viper) (databaseConfig, error) { - databaseURL, err := requiredString(values, "DATABASE_URL") + databaseURL, err := requiredString(values, "database.url") if err != nil { return databaseConfig{}, err } - maxConns, err := integer(values, "DB_MAX_CONNS", 1, 100) + maxConns, err := integer(values, "database.max_connections", 1, 100) if err != nil { return databaseConfig{}, err } - minConns, err := integer(values, "DB_MIN_CONNS", 0, maxConns) + minConns, err := integer(values, "database.min_connections", 0, maxConns) if err != nil { return databaseConfig{}, err } - maxConnectionAge, err := seconds(values, "DB_MAX_CONNECTION_AGE_SECONDS", 1, 86400) + maxConnectionAge, err := duration(values, "database.max_connection_age", time.Second, 24*time.Hour) if err != nil { return databaseConfig{}, err } - maxConnectionIdle, err := seconds(values, "DB_MAX_CONNECTION_IDLE_SECONDS", 1, 86400) + maxConnectionIdle, err := duration(values, "database.max_connection_idle", time.Second, 24*time.Hour) if err != nil { return databaseConfig{}, err } - healthCheckPeriod, err := seconds(values, "DB_HEALTH_CHECK_SECONDS", 1, 3600) + healthCheckPeriod, err := duration(values, "database.health_check_period", time.Second, time.Hour) if err != nil { return databaseConfig{}, err } - dependencyTimeout, err := seconds(values, "DEPENDENCY_TIMEOUT_SECONDS", 1, 60) + dependencyTimeout, err := duration(values, "database.dependency_timeout", time.Second, time.Minute) if err != nil { return databaseConfig{}, err } @@ -365,12 +371,16 @@ func requiredPort(values *viper.Viper, name string) (string, error) { return value, nil } -func seconds(values *viper.Viper, name string, minimum, maximum int) (time.Duration, error) { - value, err := integer(values, name, minimum, maximum) +func duration(values *viper.Viper, name string, minimum, maximum time.Duration) (time.Duration, error) { + value, err := requiredString(values, name) if err != nil { return 0, err } - return time.Duration(value) * time.Second, nil + parsed, err := time.ParseDuration(value) + if err != nil || parsed < minimum || parsed > maximum { + return 0, fmt.Errorf("%s: must be between %s and %s", name, minimum, maximum) + } + return parsed, nil } func integer(values *viper.Viper, name string, minimum, maximum int) (int, error) { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 863746ce..80d22161 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1,24 +1,17 @@ package config import ( + "os" + "path/filepath" + "strings" "testing" + "time" "github.com/stretchr/testify/require" ) func TestLoadReconciliationDoesNotRequireServerSettings(t *testing.T) { - values := map[string]string{ - "DATABASE_URL": "postgres://user:password@postgres:5432/messenger", - "DB_MIN_CONNS": "0", "DB_MAX_CONNS": "4", - "DB_MAX_CONNECTION_AGE_SECONDS": "300", "DB_MAX_CONNECTION_IDLE_SECONDS": "60", - "DB_HEALTH_CHECK_SECONDS": "30", "DEPENDENCY_TIMEOUT_SECONDS": "5", - "S3_ENDPOINT": "s3:9000", "S3_REGION": "us-east-1", "S3_BUCKET": "attachments", - "S3_ACCESS_KEY": "application", "S3_SECRET_KEY": "application-secret", - "S3_USE_TLS": "false", "S3_TIMEOUT_SECONDS": "5", - } - for name, value := range values { - t.Setenv(name, value) - } + setConfigEnvironment(t, validTestConfig) t.Setenv("APP_HOST", "") t.Setenv("CACHE_URL", "") t.Setenv("SWAGGER_PORT", "") @@ -27,3 +20,112 @@ func TestLoadReconciliationDoesNotRequireServerSettings(t *testing.T) { require.NoError(t, err) require.Equal(t, "attachments", settings.S3Bucket) } + +func TestLoadWorkerUsesYAMLForRuntimeSettings(t *testing.T) { + setConfigEnvironment(t, validTestConfig) + t.Setenv("WORKER_CONCURRENCY", "64") + + settings, err := LoadWorker() + require.NoError(t, err) + require.Equal(t, 2, settings.Concurrency) + require.Equal(t, 5*time.Second, settings.HeartbeatInterval) +} + +func TestLoadWorkerRejectsHeartbeatAtLeaseBoundary(t *testing.T) { + content := strings.Replace(validTestConfig, "heartbeat_interval: 5s", "heartbeat_interval: 30s", 1) + setConfigEnvironment(t, content) + + _, err := LoadWorker() + require.ErrorContains(t, err, "heartbeat_interval must be less than worker.lease") +} + +func TestLoadValuesRejectsUnknownVersion(t *testing.T) { + setConfigEnvironment(t, "version: 2\n") + + _, err := loadValues() + require.ErrorContains(t, err, "version: must be 1") +} + +func setConfigEnvironment(t *testing.T, content string) { + t.Helper() + path := filepath.Join(t.TempDir(), "messenger.yaml") + require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) + t.Setenv("MESSENGER_CONFIG_PATH", path) + for name, value := range map[string]string{ + "APP_HOST": "0.0.0.0", "APP_PORT": "8080", "SWAGGER_PORT": "8081", + "DATABASE_URL": "postgres://user:password@postgres:5432/messenger", + "CACHE_URL": "redis://valkey:6379/0", "BROKER_URL": "redis://redis-broker:6379/0", + "S3_ENDPOINT": "s3:9000", "S3_REGION": "us-east-1", "S3_BUCKET": "attachments", + "S3_ACCESS_KEY": "application", "S3_SECRET_KEY": "application-secret", + } { + t.Setenv(name, value) + } +} + +const validTestConfig = `version: 1 +server: + session_store: valkey + session_ttl: 1h + idempotency_ttl: 24h + request_timeout: 5s + http: + read_header_timeout: 5s + read_timeout: 10s + write_timeout: 10s + idle_timeout: 1m + shutdown_timeout: 5s + password: + memory_kib: 65536 + iterations: 3 + parallelism: 2 + salt_length: 16 + key_length: 32 +database: + min_connections: 0 + max_connections: 4 + max_connection_age: 5m + max_connection_idle: 1m + health_check_period: 30s + dependency_timeout: 5s +migration: + timeout: 30s +object_storage: + use_tls: false + timeout: 5s +attachments: + max_bytes: 10485760 + max_pixels: 40000000 + body_memory_bytes: 67108864 + decode_memory_bytes: 402653184 +pipeline: + version: 1 +broker: + stream: attachment-jobs + consumer_group: attachment-workers + failed_stream: attachment-jobs-failed + failed_stream_max_length: 10000 +outbox: + batch_size: 32 + lease: 15s + poll_interval: 1s + recovery_interval: 10s + retry_delay: 2s + retention: 168h + cleanup_interval: 1m + cleanup_batch_size: 1000 +worker: + concurrency: 2 + prefetch: 4 + lease: 30s + heartbeat_interval: 5s + processing_timeout: 20s + read_block: 2s + reclaim_interval: 5s + max_attempts: 5 + retry: + base: 1s + max: 30s + memory_bytes: 738197504 +monitoring: + metrics_address: 0.0.0.0:9091 +` diff --git a/internal/config/loader.go b/internal/config/loader.go new file mode 100644 index 00000000..6cd115c1 --- /dev/null +++ b/internal/config/loader.go @@ -0,0 +1,47 @@ +package config + +import ( + "errors" + "fmt" + "os" + "strings" + + "github.com/spf13/viper" +) + +const supportedConfigVersion = 1 + +var environmentKeys = map[string]string{ + "server.host": "APP_HOST", + "server.port": "APP_PORT", + "server.swagger_port": "SWAGGER_PORT", + "database.url": "DATABASE_URL", + "cache.url": "CACHE_URL", + "object_storage.endpoint": "S3_ENDPOINT", + "object_storage.region": "S3_REGION", + "object_storage.bucket": "S3_BUCKET", + "object_storage.access_key": "S3_ACCESS_KEY", + "object_storage.secret_key": "S3_SECRET_KEY", + "broker.url": "BROKER_URL", +} + +func loadValues() (*viper.Viper, error) { + path := strings.TrimSpace(os.Getenv("MESSENGER_CONFIG_PATH")) + if path == "" { + return nil, errors.New("MESSENGER_CONFIG_PATH is required") + } + values := viper.New() + values.SetConfigFile(path) + if err := values.ReadInConfig(); err != nil { + return nil, fmt.Errorf("read messenger config: %w", err) + } + if version := values.GetInt("version"); version != supportedConfigVersion { + return nil, fmt.Errorf("version: must be %d", supportedConfigVersion) + } + for key, name := range environmentKeys { + if err := values.BindEnv(key, name); err != nil { + return nil, fmt.Errorf("bind %s: %w", name, err) + } + } + return values, nil +} diff --git a/internal/config/pipeline.go b/internal/config/pipeline.go index 572d59bf..76b4f5d8 100644 --- a/internal/config/pipeline.go +++ b/internal/config/pipeline.go @@ -2,6 +2,7 @@ package config import ( "errors" + "fmt" "net" "time" @@ -26,6 +27,9 @@ type DispatcherConfig struct { PollInterval time.Duration RecoveryInterval time.Duration RetryDelay time.Duration + Retention time.Duration + CleanupInterval time.Duration + CleanupBatchSize int MetricsAddress string } @@ -52,6 +56,7 @@ type WorkerConfig struct { Concurrency int Prefetch int LeaseDuration time.Duration + HeartbeatInterval time.Duration ProcessingTimeout time.Duration ReadBlock time.Duration ReclaimInterval time.Duration @@ -83,8 +88,10 @@ type brokerConfig struct { } func LoadJobAdmin() (JobAdminConfig, error) { - values := viper.New() - values.AutomaticEnv() + values, err := loadValues() + if err != nil { + return JobAdminConfig{}, err + } database, err := loadDatabaseConfig(values) if err != nil { return JobAdminConfig{}, err @@ -97,8 +104,10 @@ func LoadJobAdmin() (JobAdminConfig, error) { } func LoadDispatcher() (DispatcherConfig, error) { - values := viper.New() - values.AutomaticEnv() + values, err := loadValues() + if err != nil { + return DispatcherConfig{}, err + } database, err := loadDatabaseConfig(values) if err != nil { return DispatcherConfig{}, err @@ -107,23 +116,35 @@ func LoadDispatcher() (DispatcherConfig, error) { if err != nil { return DispatcherConfig{}, err } - batchSize, err := integer(values, "OUTBOX_BATCH_SIZE", 1, 1000) + batchSize, err := integer(values, "outbox.batch_size", 1, 1000) if err != nil { return DispatcherConfig{}, err } - leaseDuration, err := seconds(values, "OUTBOX_LEASE_SECONDS", 1, 600) + leaseDuration, err := duration(values, "outbox.lease", time.Second, 10*time.Minute) if err != nil { return DispatcherConfig{}, err } - pollInterval, err := seconds(values, "OUTBOX_POLL_SECONDS", 1, 60) + pollInterval, err := duration(values, "outbox.poll_interval", time.Millisecond, time.Minute) if err != nil { return DispatcherConfig{}, err } - recoveryInterval, err := seconds(values, "OUTBOX_RECOVERY_SECONDS", 1, 3600) + recoveryInterval, err := duration(values, "outbox.recovery_interval", time.Second, time.Hour) if err != nil { return DispatcherConfig{}, err } - retryDelay, err := seconds(values, "OUTBOX_RETRY_SECONDS", 1, 3600) + retryDelay, err := duration(values, "outbox.retry_delay", time.Second, time.Hour) + if err != nil { + return DispatcherConfig{}, err + } + retention, err := duration(values, "outbox.retention", time.Hour, 365*24*time.Hour) + if err != nil { + return DispatcherConfig{}, err + } + cleanupInterval, err := duration(values, "outbox.cleanup_interval", time.Minute, 24*time.Hour) + if err != nil { + return DispatcherConfig{}, err + } + cleanupBatchSize, err := integer(values, "outbox.cleanup_batch_size", 1, 1000) if err != nil { return DispatcherConfig{}, err } @@ -138,13 +159,17 @@ func LoadDispatcher() (DispatcherConfig, error) { BrokerURL: broker.url, Stream: broker.stream, ConsumerGroup: broker.consumerGroup, FailedStream: broker.failedStream, FailedStreamMaxLength: int64(broker.failedStreamMaxLength), BatchSize: batchSize, LeaseDuration: leaseDuration, PollInterval: pollInterval, - RecoveryInterval: recoveryInterval, RetryDelay: retryDelay, MetricsAddress: metricsAddress, + RecoveryInterval: recoveryInterval, RetryDelay: retryDelay, + Retention: retention, CleanupInterval: cleanupInterval, CleanupBatchSize: cleanupBatchSize, + MetricsAddress: metricsAddress, }, nil } func LoadWorker() (WorkerConfig, error) { - values := viper.New() - values.AutomaticEnv() + values, err := loadValues() + if err != nil { + return WorkerConfig{}, err + } database, err := loadDatabaseConfig(values) if err != nil { return WorkerConfig{}, err @@ -157,57 +182,64 @@ func LoadWorker() (WorkerConfig, error) { if err != nil { return WorkerConfig{}, err } - concurrency, err := integer(values, "WORKER_CONCURRENCY", 1, 64) + concurrency, err := integer(values, "worker.concurrency", 1, 64) if err != nil { return WorkerConfig{}, err } - prefetch, err := integer(values, "WORKER_PREFETCH", concurrency, 1024) + prefetch, err := integer(values, "worker.prefetch", concurrency, 1024) if err != nil { return WorkerConfig{}, err } - leaseDuration, err := seconds(values, "WORKER_LEASE_SECONDS", 1, 3600) + leaseDuration, err := duration(values, "worker.lease", time.Second, time.Hour) if err != nil { return WorkerConfig{}, err } - processingTimeout, err := seconds(values, "WORKER_PROCESSING_TIMEOUT_SECONDS", 1, 3600) + heartbeatInterval, err := duration(values, "worker.heartbeat_interval", 100*time.Millisecond, time.Hour) + if err != nil { + return WorkerConfig{}, err + } + if heartbeatInterval >= leaseDuration { + return WorkerConfig{}, errors.New("worker.heartbeat_interval must be less than worker.lease") + } + processingTimeout, err := duration(values, "worker.processing_timeout", time.Second, time.Hour) if err != nil { return WorkerConfig{}, err } if processingTimeout >= leaseDuration { - return WorkerConfig{}, errors.New("WORKER_PROCESSING_TIMEOUT_SECONDS must be less than WORKER_LEASE_SECONDS") + return WorkerConfig{}, errors.New("worker.processing_timeout must be less than worker.lease") } - readBlock, err := seconds(values, "WORKER_READ_BLOCK_SECONDS", 1, 60) + readBlock, err := duration(values, "worker.read_block", 100*time.Millisecond, time.Minute) if err != nil { return WorkerConfig{}, err } - reclaimInterval, err := seconds(values, "WORKER_RECLAIM_SECONDS", 1, 3600) + reclaimInterval, err := duration(values, "worker.reclaim_interval", time.Second, time.Hour) if err != nil { return WorkerConfig{}, err } - maxAttempts, err := integer(values, "WORKER_MAX_ATTEMPTS", 1, 100) + maxAttempts, err := integer(values, "worker.max_attempts", 1, 100) if err != nil { return WorkerConfig{}, err } - retryBase, err := seconds(values, "WORKER_RETRY_BASE_SECONDS", 1, 3600) + retryBase, err := duration(values, "worker.retry.base", time.Second, time.Hour) if err != nil { return WorkerConfig{}, err } - retryMax, err := seconds(values, "WORKER_RETRY_MAX_SECONDS", 1, 86400) + retryMax, err := duration(values, "worker.retry.max", time.Second, 24*time.Hour) if err != nil { return WorkerConfig{}, err } if retryMax < retryBase { - return WorkerConfig{}, errors.New("WORKER_RETRY_MAX_SECONDS must be at least WORKER_RETRY_BASE_SECONDS") + return WorkerConfig{}, errors.New("worker.retry.max must be at least worker.retry.base") } - attachmentMaxBytes, err := integer(values, "ATTACHMENT_MAX_BYTES", 1, 10*1024*1024) + attachmentMaxBytes, err := integer(values, "attachments.max_bytes", 1, 10*1024*1024) if err != nil { return WorkerConfig{}, err } - attachmentMaxPixels, err := integer(values, "ATTACHMENT_MAX_PIXELS", 1, 40_000_000) + attachmentMaxPixels, err := integer(values, "attachments.max_pixels", 1, 40_000_000) if err != nil { return WorkerConfig{}, err } - memoryBytes, err := integer(values, "WORKER_MEMORY_BYTES", attachmentMaxBytes, 2*1024*1024*1024) + memoryBytes, err := integer(values, "worker.memory_bytes", attachmentMaxBytes, 2*1024*1024*1024) if err != nil { return WorkerConfig{}, err } @@ -225,44 +257,46 @@ func LoadWorker() (WorkerConfig, error) { ConsumerGroup: broker.consumerGroup, FailedStream: broker.failedStream, FailedStreamMaxLength: int64(broker.failedStreamMaxLength), Concurrency: concurrency, Prefetch: prefetch, LeaseDuration: leaseDuration, - ProcessingTimeout: processingTimeout, ReadBlock: readBlock, + HeartbeatInterval: heartbeatInterval, ProcessingTimeout: processingTimeout, ReadBlock: readBlock, ReclaimInterval: reclaimInterval, MaxAttempts: maxAttempts, RetryBase: retryBase, RetryMax: retryMax, AttachmentMaxBytes: int64(attachmentMaxBytes), AttachmentMaxPixels: int64(attachmentMaxPixels), - MemoryBytes: int64(memoryBytes), - MetricsAddress: metricsAddress, + MemoryBytes: int64(memoryBytes), MetricsAddress: metricsAddress, }, nil } func metricsAddress(values *viper.Viper) (string, error) { - host, err := requiredString(values, "PIPELINE_METRICS_HOST") + address, err := requiredString(values, "monitoring.metrics_address") if err != nil { return "", err } - port, err := requiredPort(values, "PIPELINE_METRICS_PORT") + _, port, err := net.SplitHostPort(address) if err != nil { - return "", err + return "", fmt.Errorf("monitoring.metrics_address: %w", err) } - return net.JoinHostPort(host, port), nil + if _, err := integerValue(port, 1, 65535); err != nil { + return "", fmt.Errorf("monitoring.metrics_address: %w", err) + } + return address, nil } func loadBrokerConfig(values *viper.Viper) (brokerConfig, error) { - url, err := requiredString(values, "BROKER_URL") + url, err := requiredString(values, "broker.url") if err != nil { return brokerConfig{}, err } - stream, err := requiredString(values, "BROKER_STREAM") + stream, err := requiredString(values, "broker.stream") if err != nil { return brokerConfig{}, err } - consumerGroup, err := requiredString(values, "BROKER_CONSUMER_GROUP") + consumerGroup, err := requiredString(values, "broker.consumer_group") if err != nil { return brokerConfig{}, err } - failedStream, err := requiredString(values, "BROKER_FAILED_STREAM") + failedStream, err := requiredString(values, "broker.failed_stream") if err != nil { return brokerConfig{}, err } - failedStreamMaxLength, err := integer(values, "BROKER_FAILED_MAX_LENGTH", 100, 10_000_000) + failedStreamMaxLength, err := integer(values, "broker.failed_stream_max_length", 100, 10_000_000) if err != nil { return brokerConfig{}, err } diff --git a/internal/messenger/attachment/transform/exif.go b/internal/messenger/attachment/transform/exif.go index 35a5c664..53e532be 100644 --- a/internal/messenger/attachment/transform/exif.go +++ b/internal/messenger/attachment/transform/exif.go @@ -1,6 +1,7 @@ package transform import ( + "context" "encoding/binary" "errors" "image" @@ -9,11 +10,14 @@ import ( var errInvalidEXIF = errors.New("invalid EXIF metadata") -func jpegOrientation(data []byte) (int, error) { +func jpegOrientation(ctx context.Context, data []byte) (int, error) { if len(data) < 2 || data[0] != 0xff || data[1] != 0xd8 { return 0, errInvalidEXIF } for offset := 2; offset < len(data); { + if err := contextError(ctx); err != nil { + return 0, err + } if data[offset] != 0xff { return 0, errInvalidEXIF } @@ -43,12 +47,12 @@ func jpegOrientation(data []byte) (int, error) { if marker != 0xe1 || len(payload) < 6 || string(payload[:6]) != "Exif\x00\x00" { continue } - return tiffOrientation(payload[6:]) + return tiffOrientation(ctx, payload[6:]) } return 1, nil } -func tiffOrientation(data []byte) (int, error) { +func tiffOrientation(ctx context.Context, data []byte) (int, error) { if len(data) < 8 { return 0, errInvalidEXIF } @@ -74,6 +78,9 @@ func tiffOrientation(data []byte) (int, error) { return 0, errInvalidEXIF } for index := uint64(0); index < count; index++ { + if err := contextError(ctx); err != nil { + return 0, err + } entry := data[entriesStart+index*12 : entriesStart+(index+1)*12] if order.Uint16(entry[:2]) != 0x0112 { continue @@ -90,7 +97,7 @@ func tiffOrientation(data []byte) (int, error) { return 1, nil } -func applyOrientation(source image.Image, orientation int) *image.NRGBA { +func applyOrientation(ctx context.Context, source image.Image, orientation int) (*image.NRGBA, error) { bounds := source.Bounds() width, height := bounds.Dx(), bounds.Dy() outputWidth, outputHeight := width, height @@ -99,12 +106,20 @@ func applyOrientation(source image.Image, orientation int) *image.NRGBA { } output := image.NewNRGBA(image.Rect(0, 0, outputWidth, outputHeight)) for y := 0; y < outputHeight; y++ { + if err := contextError(ctx); err != nil { + return nil, err + } for x := 0; x < outputWidth; x++ { + if x&255 == 0 { + if err := contextError(ctx); err != nil { + return nil, err + } + } sourceX, sourceY := orientedSourcePoint(x, y, width, height, orientation) output.Set(x, y, source.At(bounds.Min.X+sourceX, bounds.Min.Y+sourceY)) } } - return output + return output, nil } func orientedSourcePoint(x, y, width, height, orientation int) (int, int) { @@ -128,11 +143,19 @@ func orientedSourcePoint(x, y, width, height, orientation int) (int, int) { } } -func flattenWhite(source image.Image) *image.NRGBA { +func flattenWhite(ctx context.Context, source image.Image) (*image.NRGBA, error) { bounds := source.Bounds() output := image.NewNRGBA(image.Rect(0, 0, bounds.Dx(), bounds.Dy())) for y := 0; y < bounds.Dy(); y++ { + if err := contextError(ctx); err != nil { + return nil, err + } for x := 0; x < bounds.Dx(); x++ { + if x&255 == 0 { + if err := contextError(ctx); err != nil { + return nil, err + } + } r, g, b, a := source.At(bounds.Min.X+x, bounds.Min.Y+y).RGBA() output.SetNRGBA(x, y, color.NRGBA{ R: uint8((r + 0xffff - a) / 0x101), @@ -142,5 +165,5 @@ func flattenWhite(source image.Image) *image.NRGBA { }) } } - return output + return output, nil } diff --git a/internal/messenger/attachment/transform/processor.go b/internal/messenger/attachment/transform/processor.go index a8dde17d..8a6dd08f 100644 --- a/internal/messenger/attachment/transform/processor.go +++ b/internal/messenger/attachment/transform/processor.go @@ -9,6 +9,7 @@ import ( "image" "image/jpeg" _ "image/png" + "io" "math" "net/http" @@ -78,7 +79,7 @@ func (p *Processor) Process(ctx context.Context, data []byte, declaredContentTyp if err := contextError(ctx); err != nil { return Result{}, err } - img, format, width, height, orientation, err := p.decode(data, declaredContentType) + img, format, width, height, orientation, err := p.decode(ctx, data, declaredContentType) if err != nil { return Result{}, err } @@ -86,12 +87,12 @@ func (p *Processor) Process(ctx context.Context, data []byte, declaredContentTyp return Result{}, err } - oriented := applyOrientation(img, orientation) - if err := contextError(ctx); err != nil { + oriented, err := applyOrientation(ctx, img, orientation) + if err != nil { return Result{}, err } - opaque := flattenWhite(oriented) - if err := contextError(ctx); err != nil { + opaque, err := flattenWhite(ctx, oriented) + if err != nil { return Result{}, err } result := Result{ @@ -99,7 +100,10 @@ func (p *Processor) Process(ctx context.Context, data []byte, declaredContentTyp Orientation: orientation, } - thumbnail := fit(opaque, 256, 256, false) + thumbnail, err := fitContext(ctx, opaque, 256, 256, false) + if err != nil { + return Result{}, err + } result.Thumbnail, err = encodeJPEG(thumbnail, p.config.JPEGQuality) if err != nil { return Result{}, err @@ -109,7 +113,10 @@ func (p *Processor) Process(ctx context.Context, data []byte, declaredContentTyp } crop := centeredSquare(opaque) - crop = resize(crop, 128, 128) + crop, err = resizeContext(ctx, crop, 128, 128) + if err != nil { + return Result{}, err + } result.Crop, err = encodeJPEG(crop, p.config.JPEGQuality) if err != nil { return Result{}, err @@ -118,7 +125,10 @@ func (p *Processor) Process(ctx context.Context, data []byte, declaredContentTyp return Result{}, err } - optimized := fit(opaque, 1280, 1280, false) + optimized, err := fitContext(ctx, opaque, 1280, 1280, false) + if err != nil { + return Result{}, err + } result.Optimized, err = encodeWebP(optimized, p.config.WebPQuality, p.config.WebPMethod) if err != nil { return Result{}, err @@ -129,7 +139,11 @@ func (p *Processor) Process(ctx context.Context, data []byte, declaredContentTyp return result, nil } -func (p *Processor) decode(data []byte, declaredContentType string) (image.Image, string, int, int, int, error) { +func (p *Processor) decode( + ctx context.Context, + data []byte, + declaredContentType string, +) (image.Image, string, int, int, int, error) { if declaredContentType != "image/jpeg" && declaredContentType != "image/png" { return nil, "", 0, 0, 0, ErrUnsupportedMedia } @@ -142,27 +156,48 @@ func (p *Processor) decode(data []byte, declaredContentType string) (image.Image if http.DetectContentType(data) != declaredContentType { return nil, "", 0, 0, 0, ErrUnsupportedMedia } - config, format, err := image.DecodeConfig(bytes.NewReader(data)) + config, format, err := image.DecodeConfig(contextReader{ctx: ctx, reader: bytes.NewReader(data)}) + if contextErr := contextError(ctx); contextErr != nil { + return nil, "", 0, 0, 0, contextErr + } if err != nil || contentType(format) != declaredContentType || config.Width < 1 || config.Height < 1 { return nil, "", 0, 0, 0, ErrInvalidImage } if int64(config.Width) > p.config.MaxPixels/int64(config.Height) { return nil, "", 0, 0, 0, ErrPayloadTooLarge } - img, decodedFormat, err := image.Decode(bytes.NewReader(data)) + img, decodedFormat, err := image.Decode(contextReader{ctx: ctx, reader: bytes.NewReader(data)}) + if contextErr := contextError(ctx); contextErr != nil { + return nil, "", 0, 0, 0, contextErr + } if err != nil || decodedFormat != format { return nil, "", 0, 0, 0, ErrInvalidImage } orientation := 1 if format == "jpeg" { - orientation, err = jpegOrientation(data) + orientation, err = jpegOrientation(ctx, data) if err != nil { + if contextErr := contextError(ctx); contextErr != nil { + return nil, "", 0, 0, 0, contextErr + } return nil, "", 0, 0, 0, ErrInvalidImage } } return img, declaredContentType, config.Width, config.Height, orientation, nil } +type contextReader struct { + ctx context.Context + reader io.Reader +} + +func (r contextReader) Read(buffer []byte) (int, error) { + if err := contextError(r.ctx); err != nil { + return 0, err + } + return r.reader.Read(buffer) +} + func contextError(ctx context.Context) error { select { case <-ctx.Done(): diff --git a/internal/messenger/attachment/transform/processor_test.go b/internal/messenger/attachment/transform/processor_test.go index 972cb6d6..7b595dd9 100644 --- a/internal/messenger/attachment/transform/processor_test.go +++ b/internal/messenger/attachment/transform/processor_test.go @@ -64,7 +64,10 @@ func TestFlattenWhiteCompositesAlpha(t *testing.T) { source.SetNRGBA(0, 0, color.NRGBA{R: 10, G: 20, B: 30, A: 0}) source.SetNRGBA(1, 0, color.NRGBA{R: 100, G: 50, B: 0, A: 128}) - result := flattenWhite(source) + result, err := flattenWhite(context.Background(), source) + if err != nil { + t.Fatalf("flatten image: %v", err) + } if got := result.NRGBAAt(0, 0); got != (color.NRGBA{R: 255, G: 255, B: 255, A: 255}) { t.Fatalf("transparent pixel: got %#v", got) } @@ -154,6 +157,28 @@ func TestProcessHonorsCanceledContext(t *testing.T) { } } +func TestResizeStopsAfterCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + source := cancelingImage{ + Image: image.NewNRGBA(image.Rect(0, 0, 256, 256)), + cancel: cancel, + } + _, err := resizeContext(ctx, source, 128, 128) + if !errors.Is(err, context.Canceled) { + t.Fatalf("error: got %v, want %v", err, context.Canceled) + } +} + +type cancelingImage struct { + image.Image + cancel context.CancelFunc +} + +func (i cancelingImage) At(x, y int) color.Color { + i.cancel() + return i.Image.At(x, y) +} + func TestEstimateMemoryWeight(t *testing.T) { weight, err := EstimateMemoryWeight(1000, 320, 200) if err != nil { diff --git a/internal/messenger/attachment/transform/resize.go b/internal/messenger/attachment/transform/resize.go index 4526b5ef..2d86bb83 100644 --- a/internal/messenger/attachment/transform/resize.go +++ b/internal/messenger/attachment/transform/resize.go @@ -1,14 +1,21 @@ package transform import ( + "context" "image" "image/color" ) -func fit(source image.Image, maxWidth, maxHeight int, upscale bool) *image.NRGBA { +func fitContext( + ctx context.Context, + source image.Image, + maxWidth int, + maxHeight int, + upscale bool, +) (*image.NRGBA, error) { bounds := source.Bounds() width, height := fitDimensions(bounds.Dx(), bounds.Dy(), maxWidth, maxHeight, upscale) - return resize(source, width, height) + return resizeContext(ctx, source, width, height) } func fitDimensions(width, height, maxWidth, maxHeight int, upscale bool) (int, int) { @@ -43,15 +50,23 @@ func (i croppedImage) At(x, y int) color.Color { return i.source.At(i.offset.X+x, i.offset.Y+y) } -func resize(source image.Image, width, height int) *image.NRGBA { +func resizeContext(ctx context.Context, source image.Image, width, height int) (*image.NRGBA, error) { bounds := source.Bounds() if bounds.Dx() == width && bounds.Dy() == height { - return copyNRGBA(source) + return copyNRGBAContext(ctx, source) } output := image.NewNRGBA(image.Rect(0, 0, width, height)) for y := 0; y < height; y++ { + if err := contextError(ctx); err != nil { + return nil, err + } y0, y1, fy := sampleCoordinate(y, height, bounds.Dy()) for x := 0; x < width; x++ { + if x&255 == 0 { + if err := contextError(ctx); err != nil { + return nil, err + } + } x0, x1, fx := sampleCoordinate(x, width, bounds.Dx()) c00 := color.NRGBAModel.Convert(source.At(bounds.Min.X+x0, bounds.Min.Y+y0)).(color.NRGBA) c10 := color.NRGBAModel.Convert(source.At(bounds.Min.X+x1, bounds.Min.Y+y0)).(color.NRGBA) @@ -60,7 +75,7 @@ func resize(source image.Image, width, height int) *image.NRGBA { output.SetNRGBA(x, y, interpolate(c00, c10, c01, c11, fx, fy)) } } - return output + return output, nil } func sampleCoordinate(destination, destinationSize, sourceSize int) (int, int, uint32) { @@ -93,13 +108,21 @@ func interpolateChannel(c00, c10, c01, c11 uint8, fx, fy uint32) uint8 { return uint8((value + 1<<31) >> 32) } -func copyNRGBA(source image.Image) *image.NRGBA { +func copyNRGBAContext(ctx context.Context, source image.Image) (*image.NRGBA, error) { bounds := source.Bounds() output := image.NewNRGBA(image.Rect(0, 0, bounds.Dx(), bounds.Dy())) for y := 0; y < bounds.Dy(); y++ { + if err := contextError(ctx); err != nil { + return nil, err + } for x := 0; x < bounds.Dx(); x++ { + if x&255 == 0 { + if err := contextError(ctx); err != nil { + return nil, err + } + } output.Set(x, y, source.At(bounds.Min.X+x, bounds.Min.Y+y)) } } - return output + return output, nil } diff --git a/internal/messenger/attachment/worker/worker.go b/internal/messenger/attachment/worker/worker.go index 9fa0082a..f94f525d 100644 --- a/internal/messenger/attachment/worker/worker.go +++ b/internal/messenger/attachment/worker/worker.go @@ -27,6 +27,7 @@ import ( type Repository interface { AttachmentJob(context.Context, string) (model.AttachmentJob, error) ClaimAttachmentJob(context.Context, string, string, int, int, string, time.Duration) (model.AttachmentJobLease, error) + RenewAttachmentJobLease(context.Context, model.AttachmentJobLease, time.Duration) (bool, error) CompleteAttachmentJob(context.Context, model.AttachmentJobLease, []model.Variant) (model.JobMutationResult, error) RetryAttachmentJob(context.Context, model.AttachmentJobLease, string, time.Time) (model.JobMutationResult, error) FailAttachmentJob(context.Context, model.AttachmentJobLease, string) (model.JobMutationResult, error) @@ -59,6 +60,7 @@ type Config struct { WorkerID string Concurrency int LeaseDuration time.Duration + HeartbeatInterval time.Duration ProcessingTimeout time.Duration ReclaimInterval time.Duration MaxAttempts int @@ -119,6 +121,7 @@ func New( func validateConfig(config Config) error { if config.WorkerID == "" || config.Concurrency < 1 || config.LeaseDuration <= 0 || + config.HeartbeatInterval <= 0 || config.HeartbeatInterval >= config.LeaseDuration || config.ProcessingTimeout <= 0 || config.ProcessingTimeout >= config.LeaseDuration || config.ReclaimInterval <= 0 || config.MaxAttempts < 1 || config.RetryBase <= 0 || config.RetryMax < config.RetryBase || config.MaxBytes < 1 || config.MaxPixels < 1 || config.MemoryBytes < 1 { @@ -218,9 +221,17 @@ func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Deliv return ErrInjectedFailure } - processingCtx, cancel := context.WithTimeout(ctx, w.config.ProcessingTimeout) + processingCtx, cancelProcessing := context.WithTimeout(ctx, w.config.ProcessingTimeout) + heartbeatCtx, cancelHeartbeat := context.WithCancel(ctx) + heartbeatDone := make(chan error, 1) + go func() { heartbeatDone <- w.maintainLease(heartbeatCtx, cancelProcessing, lease) }() variants, err := w.process(processingCtx, lease) - cancel() + cancelProcessing() + cancelHeartbeat() + heartbeatErr := <-heartbeatDone + if heartbeatErr != nil { + return heartbeatErr + } if err != nil { result, err = w.resolveProcessingFailure(ctx, delivery, lease, err) return err @@ -252,6 +263,34 @@ func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Deliv return nil } +func (w *Worker) maintainLease( + ctx context.Context, + cancelProcessing context.CancelFunc, + lease model.AttachmentJobLease, +) error { + ticker := time.NewTicker(w.config.HeartbeatInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + renewed, err := w.repository.RenewAttachmentJobLease(ctx, lease, w.config.LeaseDuration) + if err != nil { + if ctx.Err() != nil { + return nil + } + cancelProcessing() + return fmt.Errorf("renew attachment job lease: %w", err) + } + if !renewed { + cancelProcessing() + return model.ErrStaleLease + } + } + } +} + func (w *Worker) resolveClaim( ctx context.Context, delivery redisstream.Delivery, @@ -295,15 +334,18 @@ func (w *Worker) process(ctx context.Context, lease model.AttachmentJobLease) ([ return nil, err } defer func() { _ = body.Close() }() - data, err := io.ReadAll(io.LimitReader(body, w.config.MaxBytes+1)) + data, err := io.ReadAll(io.LimitReader(contextReader{ctx: ctx, reader: body}, w.config.MaxBytes+1)) if err != nil { return nil, err } if int64(len(data)) > w.config.MaxBytes { return nil, transform.ErrPayloadTooLarge } - imageConfig, _, err := image.DecodeConfig(bytes.NewReader(data)) + imageConfig, _, err := image.DecodeConfig(contextReader{ctx: ctx, reader: bytes.NewReader(data)}) if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } return nil, transform.ErrInvalidImage } weight, err := transform.EstimateMemoryWeight(int64(len(data)), imageConfig.Width, imageConfig.Height) @@ -346,6 +388,20 @@ func (w *Worker) process(ctx context.Context, lease model.AttachmentJobLease) ([ return result, nil } +type contextReader struct { + ctx context.Context + reader io.Reader +} + +func (r contextReader) Read(buffer []byte) (int, error) { + select { + case <-r.ctx.Done(): + return 0, r.ctx.Err() + default: + return r.reader.Read(buffer) + } +} + var errWorkerCapacity = errors.New("image exceeds worker memory budget") func variantObjectKey(attachmentID string, pipelineVersion int, name string, value transform.Variant) string { diff --git a/internal/messenger/attachment/worker/worker_test.go b/internal/messenger/attachment/worker/worker_test.go index 77a1d5f5..7a573f1d 100644 --- a/internal/messenger/attachment/worker/worker_test.go +++ b/internal/messenger/attachment/worker/worker_test.go @@ -1,13 +1,16 @@ package worker import ( + "context" "errors" + "sync/atomic" "testing" "time" "github.com/stretchr/testify/require" "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore" ) @@ -46,11 +49,61 @@ func TestProcessingErrorClassification(t *testing.T) { } } +func TestLeaseHeartbeatRenewsUntilCanceled(t *testing.T) { + var renewals atomic.Int32 + repository := &repositoryStub{renew: func(context.Context, model.AttachmentJobLease, time.Duration) (bool, error) { + renewals.Add(1) + return true, nil + }} + config := validConfig() + config.HeartbeatInterval = 5 * time.Millisecond + worker := &Worker{repository: repository, config: config} + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- worker.maintainLease(ctx, cancel, testLease()) }() + require.Eventually(t, func() bool { return renewals.Load() >= 2 }, time.Second, time.Millisecond) + cancel() + require.NoError(t, <-done) +} + +func TestLeaseHeartbeatCancelsProcessingWhenLeaseIsLost(t *testing.T) { + repository := &repositoryStub{renew: func(context.Context, model.AttachmentJobLease, time.Duration) (bool, error) { + return false, nil + }} + config := validConfig() + config.HeartbeatInterval = time.Millisecond + worker := &Worker{repository: repository, config: config} + ctx, cancel := context.WithCancel(context.Background()) + err := worker.maintainLease(ctx, cancel, testLease()) + require.ErrorIs(t, err, model.ErrStaleLease) + require.ErrorIs(t, ctx.Err(), context.Canceled) +} + +type repositoryStub struct { + Repository + renew func(context.Context, model.AttachmentJobLease, time.Duration) (bool, error) +} + +func (s *repositoryStub) RenewAttachmentJobLease( + ctx context.Context, + lease model.AttachmentJobLease, + duration time.Duration, +) (bool, error) { + return s.renew(ctx, lease, duration) +} + +func testLease() model.AttachmentJobLease { + return model.AttachmentJobLease{Job: model.AttachmentJob{ + ID: "11111111-1111-4111-8111-111111111111", LeaseToken: "lease-token", LeaseOwner: "worker-1", + }} +} + func validConfig() Config { return Config{ WorkerID: "worker-1", Concurrency: 2, LeaseDuration: 30 * time.Second, - ProcessingTimeout: 20 * time.Second, ReclaimInterval: 5 * time.Second, - MaxAttempts: 5, RetryBase: time.Second, RetryMax: 30 * time.Second, + HeartbeatInterval: 5 * time.Second, ProcessingTimeout: 20 * time.Second, + ReclaimInterval: 5 * time.Second, + MaxAttempts: 5, RetryBase: time.Second, RetryMax: 30 * time.Second, MaxBytes: 10 * 1024 * 1024, MaxPixels: 40_000_000, MemoryBytes: 704 * 1024 * 1024, } } diff --git a/internal/outbox/dispatcher.go b/internal/outbox/dispatcher.go index 17778e3c..22aa75de 100644 --- a/internal/outbox/dispatcher.go +++ b/internal/outbox/dispatcher.go @@ -16,6 +16,7 @@ type Store interface { MarkOutboxPublished(context.Context, string, string) (bool, error) ReleaseOutboxEvent(context.Context, string, string, string, time.Time) (bool, error) RecoverExpiredOutboxEvents(context.Context, time.Time) (int, error) + DeletePublishedOutboxEvents(context.Context, time.Time, int) (int, error) } type Publisher interface { @@ -34,6 +35,9 @@ type Config struct { PollInterval time.Duration RecoveryInterval time.Duration RetryDelay time.Duration + Retention time.Duration + CleanupInterval time.Duration + CleanupBatchSize int } type Dispatcher struct { @@ -72,6 +76,12 @@ func (c Config) validate() error { return errors.New("outbox recovery interval must be positive") case c.RetryDelay <= 0: return errors.New("outbox retry delay must be positive") + case c.Retention <= 0: + return errors.New("outbox retention must be positive") + case c.CleanupInterval <= 0: + return errors.New("outbox cleanup interval must be positive") + case c.CleanupBatchSize <= 0 || c.CleanupBatchSize > 1000: + return errors.New("outbox cleanup batch size must be between 1 and 1000") default: return nil } @@ -84,8 +94,10 @@ func (d *Dispatcher) Run(ctx context.Context) error { pollTicker := time.NewTicker(d.config.PollInterval) recoveryTicker := time.NewTicker(d.config.RecoveryInterval) + cleanupTicker := time.NewTicker(d.config.CleanupInterval) defer pollTicker.Stop() defer recoveryTicker.Stop() + defer cleanupTicker.Stop() if err := d.RecoverExpired(ctx); err != nil { if ctx.Err() != nil { @@ -112,10 +124,27 @@ func (d *Dispatcher) Run(ctx context.Context) error { } return err } + case <-cleanupTicker.C: + if _, err := d.CleanupPublished(ctx); err != nil { + if ctx.Err() != nil { + return nil + } + return err + } } } } +func (d *Dispatcher) CleanupPublished(ctx context.Context) (int, error) { + deleted, err := d.store.DeletePublishedOutboxEvents( + ctx, d.now().Add(-d.config.Retention), d.config.CleanupBatchSize, + ) + if err != nil { + return 0, fmt.Errorf("delete published outbox events: %w", err) + } + return deleted, nil +} + func (d *Dispatcher) DispatchOnce(ctx context.Context) (int, error) { events, err := d.store.ClaimOutboxEvents(ctx, d.config.BatchSize, d.config.LeaseDuration) if err != nil { diff --git a/internal/outbox/dispatcher_test.go b/internal/outbox/dispatcher_test.go index 036741e8..206790d2 100644 --- a/internal/outbox/dispatcher_test.go +++ b/internal/outbox/dispatcher_test.go @@ -70,6 +70,18 @@ func TestDispatcherRejectsStaleLeaseAfterPublish(t *testing.T) { require.Zero(t, published) } +func TestDispatcherCleansPublishedEventsBeforeRetentionCutoff(t *testing.T) { + store := &storeStub{deleted: 2} + dispatcher := newTestDispatcher(t, store, &publisherStub{}) + dispatcher.now = func() time.Time { return time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC) } + + deleted, err := dispatcher.CleanupPublished(context.Background()) + require.NoError(t, err) + require.Equal(t, 2, deleted) + require.Equal(t, dispatcher.now().Add(-24*time.Hour), store.publishedBefore) + require.Equal(t, 100, store.cleanupLimit) +} + func TestDispatcherStopsAfterCancellation(t *testing.T) { store := &storeStub{} publisher := &publisherStub{} @@ -96,6 +108,9 @@ func newTestDispatcher(t *testing.T, store Store, publisher Publisher) *Dispatch PollInterval: time.Millisecond, RecoveryInterval: time.Second, RetryDelay: time.Second, + Retention: 24 * time.Hour, + CleanupInterval: time.Hour, + CleanupBatchSize: 100, }) require.NoError(t, err) return dispatcher @@ -114,14 +129,17 @@ func testPayload() json.RawMessage { } type storeStub struct { - mu sync.Mutex - events []model.OutboxEvent - markErr error - markLost bool - marked []string - released string - nextAttemptAt time.Time - recoveredCalls int + mu sync.Mutex + events []model.OutboxEvent + markErr error + markLost bool + marked []string + released string + nextAttemptAt time.Time + recoveredCalls int + deleted int + publishedBefore time.Time + cleanupLimit int } func (s *storeStub) ClaimOutboxEvents(context.Context, int, time.Duration) ([]model.OutboxEvent, error) { @@ -164,6 +182,14 @@ func (s *storeStub) RecoverExpiredOutboxEvents(context.Context, time.Time) (int, return 0, nil } +func (s *storeStub) DeletePublishedOutboxEvents(_ context.Context, before time.Time, limit int) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.publishedBefore = before + s.cleanupLimit = limit + return s.deleted, nil +} + type publisherStub struct { mu sync.Mutex messageID string diff --git a/internal/postgres/attachment_lease.go b/internal/postgres/attachment_lease.go new file mode 100644 index 00000000..b6883eb0 --- /dev/null +++ b/internal/postgres/attachment_lease.go @@ -0,0 +1,32 @@ +package postgres + +import ( + "context" + "fmt" + "time" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" +) + +func (s *Store) RenewAttachmentJobLease( + ctx context.Context, + lease model.AttachmentJobLease, + duration time.Duration, +) (bool, error) { + if duration <= 0 || lease.Job.ID == "" || lease.Job.LeaseToken == "" || lease.Job.LeaseOwner == "" { + return false, model.ErrInvalidInput + } + const query = ` + UPDATE attachment_jobs + SET lease_until = clock_timestamp() + ($4::bigint * interval '1 microsecond'), + updated_at = clock_timestamp() + WHERE id = $1 AND lease_token = $2 AND lease_owner = $3 + AND status = 'processing' AND lease_until > clock_timestamp()` + result, err := s.pool.Exec( + ctx, query, lease.Job.ID, lease.Job.LeaseToken, lease.Job.LeaseOwner, duration.Microseconds(), + ) + if err != nil { + return false, fmt.Errorf("renew attachment job lease: %w", err) + } + return result.RowsAffected() == 1, nil +} diff --git a/internal/postgres/outbox.go b/internal/postgres/outbox.go index 6bfcc107..9eda8656 100644 --- a/internal/postgres/outbox.go +++ b/internal/postgres/outbox.go @@ -111,3 +111,26 @@ func (s *Store) RecoverExpiredOutboxEvents(ctx context.Context, now time.Time) ( } return int(result.RowsAffected()), nil } + +func (s *Store) DeletePublishedOutboxEvents(ctx context.Context, publishedBefore time.Time, limit int) (int, error) { + if publishedBefore.IsZero() || limit < 1 || limit > 1000 { + return 0, model.ErrInvalidInput + } + const query = ` + WITH candidates AS ( + SELECT id + FROM outbox_events + WHERE published_at < $1 + ORDER BY published_at, id + LIMIT $2 + FOR UPDATE SKIP LOCKED + ) + DELETE FROM outbox_events event + USING candidates + WHERE event.id = candidates.id` + result, err := s.pool.Exec(ctx, query, publishedBefore.UTC(), limit) + if err != nil { + return 0, fmt.Errorf("delete published outbox events: %w", err) + } + return int(result.RowsAffected()), nil +} diff --git a/lab5.env.example b/lab5.env.example index c9d7b28f..5d3809a1 100644 --- a/lab5.env.example +++ b/lab5.env.example @@ -1,6 +1,7 @@ # Адрес и внутренний порт HTTP-сервера. APP_HOST=0.0.0.0 APP_PORT=8080 +MESSENGER_CONFIG_PATH=config/messenger.yaml # Адреса публикации сервисов на локальной машине. API_BIND_ADDRESS=127.0.0.1:8080 @@ -8,11 +9,6 @@ SWAGGER_BIND_ADDRESS=127.0.0.1:8081 PROMETHEUS_BIND_ADDRESS=127.0.0.1:9090 GRAFANA_BIND_ADDRESS=127.0.0.1:3000 -# Время жизни сессии, идемпотентности и используемое хранилище. -SESSION_TTL_SECONDS=3600 -IDEMPOTENCY_TTL_SECONDS=86400 -SESSION_STORE=valkey - # Число API-реплик за Nginx. API_REPLICAS=2 @@ -29,28 +25,6 @@ CACHE_URL=redis://default:local-valkey-password@valkey:6379/0 # Redis Streams используется только как устойчивый брокер заданий. REDIS_BROKER_PASSWORD=local-redis-broker-password BROKER_URL=redis://default:local-redis-broker-password@redis-broker:6379/0 -BROKER_STREAM=attachment-jobs -BROKER_CONSUMER_GROUP=attachment-workers -BROKER_FAILED_STREAM=attachment-jobs-failed -BROKER_FAILED_MAX_LENGTH=10000 - -# Версия pipeline, outbox и ограниченные повторы worker. -PIPELINE_VERSION=1 -OUTBOX_BATCH_SIZE=32 -OUTBOX_LEASE_SECONDS=15 -OUTBOX_POLL_SECONDS=1 -OUTBOX_RECOVERY_SECONDS=10 -OUTBOX_RETRY_SECONDS=2 -WORKER_CONCURRENCY=2 -WORKER_PREFETCH=4 -WORKER_LEASE_SECONDS=30 -WORKER_PROCESSING_TIMEOUT_SECONDS=20 -WORKER_READ_BLOCK_SECONDS=2 -WORKER_RECLAIM_SECONDS=5 -WORKER_MAX_ATTEMPTS=5 -WORKER_RETRY_BASE_SECONDS=1 -WORKER_RETRY_MAX_SECONDS=30 -WORKER_MEMORY_BYTES=738197504 # Административные данные MinIO используются только при bootstrap. MINIO_ROOT_USER=local-minio-admin @@ -62,36 +36,6 @@ S3_REGION=us-east-1 S3_BUCKET=messenger-attachments S3_ACCESS_KEY=messenger-app S3_SECRET_KEY=local-messenger-s3-password -S3_USE_TLS=false -S3_TIMEOUT_SECONDS=5 -ATTACHMENT_MAX_BYTES=10485760 -ATTACHMENT_MAX_PIXELS=40000000 -ATTACHMENT_BODY_MEMORY_BYTES=67108864 -ATTACHMENT_DECODE_MEMORY_BYTES=402653184 - -# Размер пула и обслуживание соединений PostgreSQL. -DB_MIN_CONNS=1 -DB_MAX_CONNS=10 -DB_MAX_CONNECTION_AGE_SECONDS=1800 -DB_MAX_CONNECTION_IDLE_SECONDS=300 -DB_HEALTH_CHECK_SECONDS=30 - -# Таймауты зависимостей, миграций и HTTP-запросов. -DEPENDENCY_TIMEOUT_SECONDS=5 -MIGRATION_TIMEOUT_SECONDS=30 -REQUEST_TIMEOUT_SECONDS=5 -HTTP_READ_HEADER_TIMEOUT_SECONDS=5 -HTTP_READ_TIMEOUT_SECONDS=10 -HTTP_WRITE_TIMEOUT_SECONDS=10 -HTTP_IDLE_TIMEOUT_SECONDS=60 -HTTP_SHUTDOWN_TIMEOUT_SECONDS=5 - -# Параметры Argon2id для новых хешей паролей. -PASSWORD_MEMORY_KIB=65536 -PASSWORD_ITERATIONS=3 -PASSWORD_PARALLELISM=2 -PASSWORD_SALT_LENGTH=16 -PASSWORD_KEY_LENGTH=32 # Порт Swagger внутри конфигурации CORS API. SWAGGER_PORT=8081 diff --git a/migrations/0007_outbox_retention.sql b/migrations/0007_outbox_retention.sql new file mode 100644 index 00000000..cc936f62 --- /dev/null +++ b/migrations/0007_outbox_retention.sql @@ -0,0 +1,3 @@ +CREATE INDEX outbox_events_published_at_idx + ON outbox_events (published_at, id) + WHERE published_at IS NOT NULL;