forked from mtusi2026/mtusi2026lab7
272 lines
9.4 KiB
Go
272 lines
9.4 KiB
Go
package bootstrap
|
|
|
|
import (
|
|
"context"
|
|
"crypto/tls"
|
|
"errors"
|
|
"log/slog"
|
|
"net"
|
|
"net/http"
|
|
"os"
|
|
"os/signal"
|
|
"syscall"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/redis/go-redis/v9"
|
|
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab7/internal/broker/redisstream"
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab7/internal/config"
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab7/internal/database"
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab7/internal/messenger/attachment/pipeline"
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab7/internal/messenger/attachment/transform"
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab7/internal/messenger/attachment/worker"
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab7/internal/objectstore"
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab7/internal/outbox"
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab7/internal/postgres"
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab7/internal/security/tlsconfig"
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab7/internal/telemetry"
|
|
)
|
|
|
|
func RunDispatcher() error {
|
|
settings, err := config.LoadDispatcher()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
instanceID, err := os.Hostname()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
observability, logger, err := startTelemetry("messenger-dispatcher", instanceID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer shutdownTelemetry(observability)
|
|
pool, err := database.Open(context.Background(), dispatcherDatabaseConfig(settings))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer pool.Close()
|
|
if err := telemetry.ObserveDatabasePool(observability.MeterProvider().Meter("messenger-dispatcher/database"), pool); err != nil {
|
|
return err
|
|
}
|
|
brokerTLSConfiguration, err := loadClientTLS(settings.BrokerTLS)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
client, err := newRedisClient(settings.BrokerURL, brokerTLSConfiguration)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer func() { _ = client.Close() }()
|
|
broker, err := redisstream.New(client, redisstream.Config{
|
|
Stream: settings.Stream, Group: settings.ConsumerGroup, Consumer: "dispatcher",
|
|
FailedStream: settings.FailedStream, FailedMaxLen: settings.FailedStreamMaxLength,
|
|
ClaimMinIdle: settings.LeaseDuration, ReadCount: int64(settings.BatchSize),
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
metrics := pipeline.NewMetrics(observability.MeterProvider().Meter("messenger-dispatcher"), observability.MetricsHandler(), 0)
|
|
dispatcher, err := outbox.New(postgres.New(pool), broker, metrics, outbox.Config{
|
|
BatchSize: settings.BatchSize, LeaseDuration: settings.LeaseDuration,
|
|
PollInterval: settings.PollInterval, RecoveryInterval: settings.RecoveryInterval,
|
|
RetryDelay: settings.RetryDelay, Retention: settings.Retention,
|
|
CleanupInterval: settings.CleanupInterval, CleanupBatchSize: settings.CleanupBatchSize,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
metricsTLSConfiguration, err := loadServerTLS(settings.MetricsTLS)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
logger.Info("outbox dispatcher started", "stream", settings.Stream)
|
|
return runPipelineProcess(
|
|
logger, settings.MetricsAddress, settings.HealthAddress, metricsTLSConfiguration, metrics.Handler(), dispatcher.Run,
|
|
)
|
|
}
|
|
|
|
func RunWorker() error {
|
|
settings, err := config.LoadWorker()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
hostname, err := os.Hostname()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
observability, logger, err := startTelemetry("messenger-worker", hostname)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer shutdownTelemetry(observability)
|
|
pool, err := database.Open(context.Background(), workerDatabaseConfig(settings))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer pool.Close()
|
|
if err := telemetry.ObserveDatabasePool(observability.MeterProvider().Meter("messenger-worker/database"), pool); err != nil {
|
|
return err
|
|
}
|
|
objects, err := objectstore.New(objectstore.Config{
|
|
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
|
|
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
|
|
UseTLS: settings.S3UseTLS, CAPath: settings.S3CAPath, Timeout: settings.S3Timeout,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
brokerTLSConfiguration, err := loadClientTLS(settings.BrokerTLS)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
client, err := newRedisClient(settings.BrokerURL, brokerTLSConfiguration)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer func() { _ = client.Close() }()
|
|
consumer := hostname + "-" + uuid.NewString()
|
|
broker, err := redisstream.New(client, redisstream.Config{
|
|
Stream: settings.Stream, Group: settings.ConsumerGroup, Consumer: consumer,
|
|
FailedStream: settings.FailedStream, FailedMaxLen: settings.FailedStreamMaxLength, Block: settings.ReadBlock,
|
|
ClaimMinIdle: settings.LeaseDuration, ReadCount: int64(settings.Prefetch),
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := broker.Ensure(context.Background()); err != nil {
|
|
return err
|
|
}
|
|
processor, err := transform.NewProcessor(transform.DefaultConfig(settings.AttachmentMaxBytes, settings.AttachmentMaxPixels))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
metrics := pipeline.NewMetrics(observability.MeterProvider().Meter("messenger-worker"), observability.MetricsHandler(), settings.Concurrency)
|
|
processorWorker, err := worker.New(
|
|
logger, postgres.New(pool), broker, objects, processor, metrics,
|
|
worker.Config{
|
|
WorkerID: consumer, Concurrency: settings.Concurrency, LeaseDuration: settings.LeaseDuration,
|
|
HeartbeatInterval: settings.HeartbeatInterval, ProcessingTimeout: settings.ProcessingTimeout,
|
|
ReclaimInterval: settings.ReclaimInterval, MaxAttempts: settings.MaxAttempts,
|
|
RetryBase: settings.RetryBase, RetryMax: settings.RetryMax,
|
|
MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels,
|
|
MemoryBytes: settings.MemoryBytes,
|
|
},
|
|
)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
metricsTLSConfiguration, err := loadServerTLS(settings.MetricsTLS)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
logger.Info("attachment worker started", "consumer", consumer, "concurrency", settings.Concurrency)
|
|
return runPipelineProcess(
|
|
logger, settings.MetricsAddress, settings.HealthAddress, metricsTLSConfiguration, metrics.Handler(), processorWorker.Run,
|
|
)
|
|
}
|
|
|
|
func loadClientTLS(config config.ClientTLSConfig) (*tls.Config, error) {
|
|
return tlsconfig.LoadClient(tlsconfig.ClientConfig{
|
|
CAPath: config.CAPath, CertificatePath: config.CertificatePath,
|
|
PrivateKeyPath: config.PrivateKeyPath, ServerName: config.ServerName,
|
|
})
|
|
}
|
|
|
|
func loadServerTLS(config config.ServerTLSConfig) (*tls.Config, error) {
|
|
return tlsconfig.LoadServer(tlsconfig.ServerConfig{
|
|
CAPath: config.CAPath, CertificatePath: config.CertificatePath,
|
|
PrivateKeyPath: config.PrivateKeyPath, AllowedClientURI: config.AllowedClientURI,
|
|
})
|
|
}
|
|
|
|
func newRedisClient(rawURL string, tlsConfiguration *tls.Config) (*redis.Client, error) {
|
|
options, err := redis.ParseURL(rawURL)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
options.TLSConfig = tlsConfiguration.Clone()
|
|
client := redis.NewClient(options)
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
if err := client.Ping(ctx).Err(); err != nil {
|
|
_ = client.Close()
|
|
return nil, err
|
|
}
|
|
return client, nil
|
|
}
|
|
|
|
func runPipelineProcess(
|
|
logger *slog.Logger,
|
|
metricsAddress string,
|
|
healthAddress string,
|
|
metricsTLSConfiguration *tls.Config,
|
|
metricsHandler http.Handler,
|
|
run func(context.Context) error,
|
|
) error {
|
|
metricsMux := http.NewServeMux()
|
|
metricsMux.Handle("/metrics", metricsHandler)
|
|
metricsServer := &http.Server{
|
|
Addr: metricsAddress, Handler: metricsMux, ReadHeaderTimeout: 5 * time.Second,
|
|
TLSConfig: metricsTLSConfiguration,
|
|
}
|
|
healthMux := http.NewServeMux()
|
|
healthMux.HandleFunc("/health/live", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) })
|
|
healthServer := &http.Server{Addr: healthAddress, Handler: healthMux, ReadHeaderTimeout: 5 * time.Second}
|
|
metricsListener, err := net.Listen("tcp", metricsAddress)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
healthListener, err := net.Listen("tcp", healthAddress)
|
|
if err != nil {
|
|
_ = metricsListener.Close()
|
|
return err
|
|
}
|
|
serveResult := make(chan error, 2)
|
|
go func() { serveResult <- metricsServer.Serve(tls.NewListener(metricsListener, metricsTLSConfiguration)) }()
|
|
go func() { serveResult <- healthServer.Serve(healthListener) }()
|
|
|
|
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
|
defer stop()
|
|
runResult := make(chan error, 1)
|
|
go func() { runResult <- run(ctx) }()
|
|
|
|
var result error
|
|
select {
|
|
case result = <-runResult:
|
|
stop()
|
|
case result = <-serveResult:
|
|
stop()
|
|
case <-ctx.Done():
|
|
}
|
|
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
for _, server := range []*http.Server{metricsServer, healthServer} {
|
|
if err := server.Shutdown(shutdownCtx); err != nil && result == nil {
|
|
result = err
|
|
}
|
|
}
|
|
if errors.Is(result, http.ErrServerClosed) || errors.Is(result, context.Canceled) {
|
|
return nil
|
|
}
|
|
logger.Info("pipeline process stopped")
|
|
return result
|
|
}
|
|
|
|
func dispatcherDatabaseConfig(settings config.DispatcherConfig) database.Config {
|
|
return database.Config{
|
|
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
|
|
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
|
|
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
|
|
}
|
|
}
|
|
|
|
func workerDatabaseConfig(settings config.WorkerConfig) database.Config {
|
|
return database.Config{
|
|
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
|
|
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
|
|
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
|
|
}
|
|
}
|