Files
mtusi2026lab6/internal/config/pipeline.go
T
2026-11-01 14:46:14 +03:00

308 lines
10 KiB
Go

package config
import (
"errors"
"fmt"
"net"
"time"
"github.com/spf13/viper"
)
type DispatcherConfig struct {
DatabaseURL string
DBMaxConns int32
DBMinConns int32
DBMaxConnectionAge time.Duration
DBMaxConnectionIdle time.Duration
DBHealthCheckPeriod time.Duration
DependencyTimeout time.Duration
BrokerURL string
Stream string
ConsumerGroup string
FailedStream string
FailedStreamMaxLength int64
BatchSize int
LeaseDuration time.Duration
PollInterval time.Duration
RecoveryInterval time.Duration
RetryDelay time.Duration
Retention time.Duration
CleanupInterval time.Duration
CleanupBatchSize int
MetricsAddress string
}
type WorkerConfig struct {
DatabaseURL string
DBMaxConns int32
DBMinConns int32
DBMaxConnectionAge time.Duration
DBMaxConnectionIdle time.Duration
DBHealthCheckPeriod time.Duration
DependencyTimeout time.Duration
S3Endpoint string
S3Region string
S3Bucket string
S3AccessKey string
S3SecretKey string
S3UseTLS bool
S3Timeout time.Duration
BrokerURL string
Stream string
ConsumerGroup string
FailedStream string
FailedStreamMaxLength int64
Concurrency int
Prefetch int
LeaseDuration time.Duration
HeartbeatInterval time.Duration
ProcessingTimeout time.Duration
ReadBlock time.Duration
ReclaimInterval time.Duration
MaxAttempts int
RetryBase time.Duration
RetryMax time.Duration
AttachmentMaxBytes int64
AttachmentMaxPixels int64
MemoryBytes int64
MetricsAddress string
}
type JobAdminConfig struct {
DatabaseURL string
DBMaxConns int32
DBMinConns int32
DBMaxConnectionAge time.Duration
DBMaxConnectionIdle time.Duration
DBHealthCheckPeriod time.Duration
DependencyTimeout time.Duration
}
type brokerConfig struct {
url string
stream string
consumerGroup string
failedStream string
failedStreamMaxLength int
}
func LoadJobAdmin() (JobAdminConfig, error) {
values, err := loadValues()
if err != nil {
return JobAdminConfig{}, err
}
database, err := loadDatabaseConfig(values)
if err != nil {
return JobAdminConfig{}, err
}
return JobAdminConfig{
DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns),
DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle,
DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout,
}, nil
}
func LoadDispatcher() (DispatcherConfig, error) {
values, err := loadValues()
if err != nil {
return DispatcherConfig{}, err
}
database, err := loadDatabaseConfig(values)
if err != nil {
return DispatcherConfig{}, err
}
broker, err := loadBrokerConfig(values)
if err != nil {
return DispatcherConfig{}, err
}
batchSize, err := integer(values, "outbox.batch_size", 1, 1000)
if err != nil {
return DispatcherConfig{}, err
}
leaseDuration, err := duration(values, "outbox.lease", time.Second, 10*time.Minute)
if err != nil {
return DispatcherConfig{}, err
}
pollInterval, err := duration(values, "outbox.poll_interval", time.Millisecond, time.Minute)
if err != nil {
return DispatcherConfig{}, err
}
recoveryInterval, err := duration(values, "outbox.recovery_interval", time.Second, time.Hour)
if err != nil {
return DispatcherConfig{}, err
}
retryDelay, err := duration(values, "outbox.retry_delay", time.Second, time.Hour)
if err != nil {
return DispatcherConfig{}, err
}
retention, err := duration(values, "outbox.retention", time.Hour, 365*24*time.Hour)
if err != nil {
return DispatcherConfig{}, err
}
cleanupInterval, err := duration(values, "outbox.cleanup_interval", time.Minute, 24*time.Hour)
if err != nil {
return DispatcherConfig{}, err
}
cleanupBatchSize, err := integer(values, "outbox.cleanup_batch_size", 1, 1000)
if err != nil {
return DispatcherConfig{}, err
}
metricsAddress, err := metricsAddress(values)
if err != nil {
return DispatcherConfig{}, err
}
return DispatcherConfig{
DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns),
DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle,
DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout,
BrokerURL: broker.url, Stream: broker.stream, ConsumerGroup: broker.consumerGroup,
FailedStream: broker.failedStream, FailedStreamMaxLength: int64(broker.failedStreamMaxLength),
BatchSize: batchSize, LeaseDuration: leaseDuration, PollInterval: pollInterval,
RecoveryInterval: recoveryInterval, RetryDelay: retryDelay,
Retention: retention, CleanupInterval: cleanupInterval, CleanupBatchSize: cleanupBatchSize,
MetricsAddress: metricsAddress,
}, nil
}
func LoadWorker() (WorkerConfig, error) {
values, err := loadValues()
if err != nil {
return WorkerConfig{}, err
}
database, err := loadDatabaseConfig(values)
if err != nil {
return WorkerConfig{}, err
}
storage, err := loadObjectStorageConfig(values)
if err != nil {
return WorkerConfig{}, err
}
broker, err := loadBrokerConfig(values)
if err != nil {
return WorkerConfig{}, err
}
concurrency, err := integer(values, "worker.concurrency", 1, 64)
if err != nil {
return WorkerConfig{}, err
}
prefetch, err := integer(values, "worker.prefetch", concurrency, 1024)
if err != nil {
return WorkerConfig{}, err
}
leaseDuration, err := duration(values, "worker.lease", time.Second, time.Hour)
if err != nil {
return WorkerConfig{}, err
}
heartbeatInterval, err := duration(values, "worker.heartbeat_interval", 100*time.Millisecond, time.Hour)
if err != nil {
return WorkerConfig{}, err
}
if heartbeatInterval >= leaseDuration {
return WorkerConfig{}, errors.New("worker.heartbeat_interval must be less than worker.lease")
}
processingTimeout, err := duration(values, "worker.processing_timeout", time.Second, time.Hour)
if err != nil {
return WorkerConfig{}, err
}
if processingTimeout >= leaseDuration {
return WorkerConfig{}, errors.New("worker.processing_timeout must be less than worker.lease")
}
readBlock, err := duration(values, "worker.read_block", 100*time.Millisecond, time.Minute)
if err != nil {
return WorkerConfig{}, err
}
reclaimInterval, err := duration(values, "worker.reclaim_interval", time.Second, time.Hour)
if err != nil {
return WorkerConfig{}, err
}
maxAttempts, err := integer(values, "worker.max_attempts", 1, 100)
if err != nil {
return WorkerConfig{}, err
}
retryBase, err := duration(values, "worker.retry.base", time.Second, time.Hour)
if err != nil {
return WorkerConfig{}, err
}
retryMax, err := duration(values, "worker.retry.max", time.Second, 24*time.Hour)
if err != nil {
return WorkerConfig{}, err
}
if retryMax < retryBase {
return WorkerConfig{}, errors.New("worker.retry.max must be at least worker.retry.base")
}
attachmentMaxBytes, err := integer(values, "attachments.max_bytes", 1, 10*1024*1024)
if err != nil {
return WorkerConfig{}, err
}
attachmentMaxPixels, err := integer(values, "attachments.max_pixels", 1, 40_000_000)
if err != nil {
return WorkerConfig{}, err
}
memoryBytes, err := integer(values, "worker.memory_bytes", attachmentMaxBytes, 2*1024*1024*1024)
if err != nil {
return WorkerConfig{}, err
}
metricsAddress, err := metricsAddress(values)
if err != nil {
return WorkerConfig{}, err
}
return WorkerConfig{
DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns),
DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle,
DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout,
S3Endpoint: storage.endpoint, S3Region: storage.region, S3Bucket: storage.bucket,
S3AccessKey: storage.accessKey, S3SecretKey: storage.secretKey, S3UseTLS: storage.useTLS,
S3Timeout: storage.timeout, BrokerURL: broker.url, Stream: broker.stream,
ConsumerGroup: broker.consumerGroup, FailedStream: broker.failedStream,
FailedStreamMaxLength: int64(broker.failedStreamMaxLength),
Concurrency: concurrency, Prefetch: prefetch, LeaseDuration: leaseDuration,
HeartbeatInterval: heartbeatInterval, ProcessingTimeout: processingTimeout, ReadBlock: readBlock,
ReclaimInterval: reclaimInterval, MaxAttempts: maxAttempts, RetryBase: retryBase, RetryMax: retryMax,
AttachmentMaxBytes: int64(attachmentMaxBytes), AttachmentMaxPixels: int64(attachmentMaxPixels),
MemoryBytes: int64(memoryBytes), MetricsAddress: metricsAddress,
}, nil
}
func metricsAddress(values *viper.Viper) (string, error) {
address, err := requiredString(values, "monitoring.metrics_address")
if err != nil {
return "", err
}
_, port, err := net.SplitHostPort(address)
if err != nil {
return "", fmt.Errorf("monitoring.metrics_address: %w", err)
}
if _, err := integerValue(port, 1, 65535); err != nil {
return "", fmt.Errorf("monitoring.metrics_address: %w", err)
}
return address, nil
}
func loadBrokerConfig(values *viper.Viper) (brokerConfig, error) {
url, err := requiredString(values, "broker.url")
if err != nil {
return brokerConfig{}, err
}
stream, err := requiredString(values, "broker.stream")
if err != nil {
return brokerConfig{}, err
}
consumerGroup, err := requiredString(values, "broker.consumer_group")
if err != nil {
return brokerConfig{}, err
}
failedStream, err := requiredString(values, "broker.failed_stream")
if err != nil {
return brokerConfig{}, err
}
failedStreamMaxLength, err := integer(values, "broker.failed_stream_max_length", 100, 10_000_000)
if err != nil {
return brokerConfig{}, err
}
return brokerConfig{
url: url, stream: stream, consumerGroup: consumerGroup,
failedStream: failedStream, failedStreamMaxLength: failedStreamMaxLength,
}, nil
}