forked from mtusi2026/mtusi2026lab6
308 lines
10 KiB
Go
308 lines
10 KiB
Go
package config
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"net"
|
|
"time"
|
|
|
|
"github.com/spf13/viper"
|
|
)
|
|
|
|
type DispatcherConfig struct {
|
|
DatabaseURL string
|
|
DBMaxConns int32
|
|
DBMinConns int32
|
|
DBMaxConnectionAge time.Duration
|
|
DBMaxConnectionIdle time.Duration
|
|
DBHealthCheckPeriod time.Duration
|
|
DependencyTimeout time.Duration
|
|
BrokerURL string
|
|
Stream string
|
|
ConsumerGroup string
|
|
FailedStream string
|
|
FailedStreamMaxLength int64
|
|
BatchSize int
|
|
LeaseDuration time.Duration
|
|
PollInterval time.Duration
|
|
RecoveryInterval time.Duration
|
|
RetryDelay time.Duration
|
|
Retention time.Duration
|
|
CleanupInterval time.Duration
|
|
CleanupBatchSize int
|
|
MetricsAddress string
|
|
}
|
|
|
|
type WorkerConfig struct {
|
|
DatabaseURL string
|
|
DBMaxConns int32
|
|
DBMinConns int32
|
|
DBMaxConnectionAge time.Duration
|
|
DBMaxConnectionIdle time.Duration
|
|
DBHealthCheckPeriod time.Duration
|
|
DependencyTimeout time.Duration
|
|
S3Endpoint string
|
|
S3Region string
|
|
S3Bucket string
|
|
S3AccessKey string
|
|
S3SecretKey string
|
|
S3UseTLS bool
|
|
S3Timeout time.Duration
|
|
BrokerURL string
|
|
Stream string
|
|
ConsumerGroup string
|
|
FailedStream string
|
|
FailedStreamMaxLength int64
|
|
Concurrency int
|
|
Prefetch int
|
|
LeaseDuration time.Duration
|
|
HeartbeatInterval time.Duration
|
|
ProcessingTimeout time.Duration
|
|
ReadBlock time.Duration
|
|
ReclaimInterval time.Duration
|
|
MaxAttempts int
|
|
RetryBase time.Duration
|
|
RetryMax time.Duration
|
|
AttachmentMaxBytes int64
|
|
AttachmentMaxPixels int64
|
|
MemoryBytes int64
|
|
MetricsAddress string
|
|
}
|
|
|
|
type JobAdminConfig struct {
|
|
DatabaseURL string
|
|
DBMaxConns int32
|
|
DBMinConns int32
|
|
DBMaxConnectionAge time.Duration
|
|
DBMaxConnectionIdle time.Duration
|
|
DBHealthCheckPeriod time.Duration
|
|
DependencyTimeout time.Duration
|
|
}
|
|
|
|
type brokerConfig struct {
|
|
url string
|
|
stream string
|
|
consumerGroup string
|
|
failedStream string
|
|
failedStreamMaxLength int
|
|
}
|
|
|
|
func LoadJobAdmin() (JobAdminConfig, error) {
|
|
values, err := loadValues()
|
|
if err != nil {
|
|
return JobAdminConfig{}, err
|
|
}
|
|
database, err := loadDatabaseConfig(values)
|
|
if err != nil {
|
|
return JobAdminConfig{}, err
|
|
}
|
|
return JobAdminConfig{
|
|
DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns),
|
|
DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle,
|
|
DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout,
|
|
}, nil
|
|
}
|
|
|
|
func LoadDispatcher() (DispatcherConfig, error) {
|
|
values, err := loadValues()
|
|
if err != nil {
|
|
return DispatcherConfig{}, err
|
|
}
|
|
database, err := loadDatabaseConfig(values)
|
|
if err != nil {
|
|
return DispatcherConfig{}, err
|
|
}
|
|
broker, err := loadBrokerConfig(values)
|
|
if err != nil {
|
|
return DispatcherConfig{}, err
|
|
}
|
|
batchSize, err := integer(values, "outbox.batch_size", 1, 1000)
|
|
if err != nil {
|
|
return DispatcherConfig{}, err
|
|
}
|
|
leaseDuration, err := duration(values, "outbox.lease", time.Second, 10*time.Minute)
|
|
if err != nil {
|
|
return DispatcherConfig{}, err
|
|
}
|
|
pollInterval, err := duration(values, "outbox.poll_interval", time.Millisecond, time.Minute)
|
|
if err != nil {
|
|
return DispatcherConfig{}, err
|
|
}
|
|
recoveryInterval, err := duration(values, "outbox.recovery_interval", time.Second, time.Hour)
|
|
if err != nil {
|
|
return DispatcherConfig{}, err
|
|
}
|
|
retryDelay, err := duration(values, "outbox.retry_delay", time.Second, time.Hour)
|
|
if err != nil {
|
|
return DispatcherConfig{}, err
|
|
}
|
|
retention, err := duration(values, "outbox.retention", time.Hour, 365*24*time.Hour)
|
|
if err != nil {
|
|
return DispatcherConfig{}, err
|
|
}
|
|
cleanupInterval, err := duration(values, "outbox.cleanup_interval", time.Minute, 24*time.Hour)
|
|
if err != nil {
|
|
return DispatcherConfig{}, err
|
|
}
|
|
cleanupBatchSize, err := integer(values, "outbox.cleanup_batch_size", 1, 1000)
|
|
if err != nil {
|
|
return DispatcherConfig{}, err
|
|
}
|
|
metricsAddress, err := metricsAddress(values)
|
|
if err != nil {
|
|
return DispatcherConfig{}, err
|
|
}
|
|
return DispatcherConfig{
|
|
DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns),
|
|
DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle,
|
|
DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout,
|
|
BrokerURL: broker.url, Stream: broker.stream, ConsumerGroup: broker.consumerGroup,
|
|
FailedStream: broker.failedStream, FailedStreamMaxLength: int64(broker.failedStreamMaxLength),
|
|
BatchSize: batchSize, LeaseDuration: leaseDuration, PollInterval: pollInterval,
|
|
RecoveryInterval: recoveryInterval, RetryDelay: retryDelay,
|
|
Retention: retention, CleanupInterval: cleanupInterval, CleanupBatchSize: cleanupBatchSize,
|
|
MetricsAddress: metricsAddress,
|
|
}, nil
|
|
}
|
|
|
|
func LoadWorker() (WorkerConfig, error) {
|
|
values, err := loadValues()
|
|
if err != nil {
|
|
return WorkerConfig{}, err
|
|
}
|
|
database, err := loadDatabaseConfig(values)
|
|
if err != nil {
|
|
return WorkerConfig{}, err
|
|
}
|
|
storage, err := loadObjectStorageConfig(values)
|
|
if err != nil {
|
|
return WorkerConfig{}, err
|
|
}
|
|
broker, err := loadBrokerConfig(values)
|
|
if err != nil {
|
|
return WorkerConfig{}, err
|
|
}
|
|
concurrency, err := integer(values, "worker.concurrency", 1, 64)
|
|
if err != nil {
|
|
return WorkerConfig{}, err
|
|
}
|
|
prefetch, err := integer(values, "worker.prefetch", concurrency, 1024)
|
|
if err != nil {
|
|
return WorkerConfig{}, err
|
|
}
|
|
leaseDuration, err := duration(values, "worker.lease", time.Second, time.Hour)
|
|
if err != nil {
|
|
return WorkerConfig{}, err
|
|
}
|
|
heartbeatInterval, err := duration(values, "worker.heartbeat_interval", 100*time.Millisecond, time.Hour)
|
|
if err != nil {
|
|
return WorkerConfig{}, err
|
|
}
|
|
if heartbeatInterval >= leaseDuration {
|
|
return WorkerConfig{}, errors.New("worker.heartbeat_interval must be less than worker.lease")
|
|
}
|
|
processingTimeout, err := duration(values, "worker.processing_timeout", time.Second, time.Hour)
|
|
if err != nil {
|
|
return WorkerConfig{}, err
|
|
}
|
|
if processingTimeout >= leaseDuration {
|
|
return WorkerConfig{}, errors.New("worker.processing_timeout must be less than worker.lease")
|
|
}
|
|
readBlock, err := duration(values, "worker.read_block", 100*time.Millisecond, time.Minute)
|
|
if err != nil {
|
|
return WorkerConfig{}, err
|
|
}
|
|
reclaimInterval, err := duration(values, "worker.reclaim_interval", time.Second, time.Hour)
|
|
if err != nil {
|
|
return WorkerConfig{}, err
|
|
}
|
|
maxAttempts, err := integer(values, "worker.max_attempts", 1, 100)
|
|
if err != nil {
|
|
return WorkerConfig{}, err
|
|
}
|
|
retryBase, err := duration(values, "worker.retry.base", time.Second, time.Hour)
|
|
if err != nil {
|
|
return WorkerConfig{}, err
|
|
}
|
|
retryMax, err := duration(values, "worker.retry.max", time.Second, 24*time.Hour)
|
|
if err != nil {
|
|
return WorkerConfig{}, err
|
|
}
|
|
if retryMax < retryBase {
|
|
return WorkerConfig{}, errors.New("worker.retry.max must be at least worker.retry.base")
|
|
}
|
|
attachmentMaxBytes, err := integer(values, "attachments.max_bytes", 1, 10*1024*1024)
|
|
if err != nil {
|
|
return WorkerConfig{}, err
|
|
}
|
|
attachmentMaxPixels, err := integer(values, "attachments.max_pixels", 1, 40_000_000)
|
|
if err != nil {
|
|
return WorkerConfig{}, err
|
|
}
|
|
memoryBytes, err := integer(values, "worker.memory_bytes", attachmentMaxBytes, 2*1024*1024*1024)
|
|
if err != nil {
|
|
return WorkerConfig{}, err
|
|
}
|
|
metricsAddress, err := metricsAddress(values)
|
|
if err != nil {
|
|
return WorkerConfig{}, err
|
|
}
|
|
return WorkerConfig{
|
|
DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns),
|
|
DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle,
|
|
DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout,
|
|
S3Endpoint: storage.endpoint, S3Region: storage.region, S3Bucket: storage.bucket,
|
|
S3AccessKey: storage.accessKey, S3SecretKey: storage.secretKey, S3UseTLS: storage.useTLS,
|
|
S3Timeout: storage.timeout, BrokerURL: broker.url, Stream: broker.stream,
|
|
ConsumerGroup: broker.consumerGroup, FailedStream: broker.failedStream,
|
|
FailedStreamMaxLength: int64(broker.failedStreamMaxLength),
|
|
Concurrency: concurrency, Prefetch: prefetch, LeaseDuration: leaseDuration,
|
|
HeartbeatInterval: heartbeatInterval, ProcessingTimeout: processingTimeout, ReadBlock: readBlock,
|
|
ReclaimInterval: reclaimInterval, MaxAttempts: maxAttempts, RetryBase: retryBase, RetryMax: retryMax,
|
|
AttachmentMaxBytes: int64(attachmentMaxBytes), AttachmentMaxPixels: int64(attachmentMaxPixels),
|
|
MemoryBytes: int64(memoryBytes), MetricsAddress: metricsAddress,
|
|
}, nil
|
|
}
|
|
|
|
func metricsAddress(values *viper.Viper) (string, error) {
|
|
address, err := requiredString(values, "monitoring.metrics_address")
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
_, port, err := net.SplitHostPort(address)
|
|
if err != nil {
|
|
return "", fmt.Errorf("monitoring.metrics_address: %w", err)
|
|
}
|
|
if _, err := integerValue(port, 1, 65535); err != nil {
|
|
return "", fmt.Errorf("monitoring.metrics_address: %w", err)
|
|
}
|
|
return address, nil
|
|
}
|
|
|
|
func loadBrokerConfig(values *viper.Viper) (brokerConfig, error) {
|
|
url, err := requiredString(values, "broker.url")
|
|
if err != nil {
|
|
return brokerConfig{}, err
|
|
}
|
|
stream, err := requiredString(values, "broker.stream")
|
|
if err != nil {
|
|
return brokerConfig{}, err
|
|
}
|
|
consumerGroup, err := requiredString(values, "broker.consumer_group")
|
|
if err != nil {
|
|
return brokerConfig{}, err
|
|
}
|
|
failedStream, err := requiredString(values, "broker.failed_stream")
|
|
if err != nil {
|
|
return brokerConfig{}, err
|
|
}
|
|
failedStreamMaxLength, err := integer(values, "broker.failed_stream_max_length", 100, 10_000_000)
|
|
if err != nil {
|
|
return brokerConfig{}, err
|
|
}
|
|
return brokerConfig{
|
|
url: url, stream: stream, consumerGroup: consumerGroup,
|
|
failedStream: failedStream, failedStreamMaxLength: failedStreamMaxLength,
|
|
}, nil
|
|
}
|