Files
mtusi2026lab6/internal/bootstrap/pipeline.go
T

223 lines
7.7 KiB
Go

package bootstrap
import (
"context"
"errors"
"log/slog"
"net"
"net/http"
"os"
"os/signal"
"syscall"
"time"
"github.com/google/uuid"
"github.com/redis/go-redis/v9"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/broker/redisstream"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/pipeline"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/worker"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/outbox"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry"
)
func RunDispatcher(logger *slog.Logger) error {
settings, err := config.LoadDispatcher()
if err != nil {
return err
}
instanceID, err := os.Hostname()
if err != nil {
return err
}
observability, logger, err := startTelemetry("messenger-dispatcher", instanceID)
if err != nil {
return err
}
defer shutdownTelemetry(observability)
pool, err := database.Open(context.Background(), dispatcherDatabaseConfig(settings))
if err != nil {
return err
}
defer pool.Close()
if err := telemetry.ObserveDatabasePool(observability.MeterProvider().Meter("messenger-dispatcher/database"), pool); err != nil {
return err
}
client, err := newRedisClient(settings.BrokerURL)
if err != nil {
return err
}
defer func() { _ = client.Close() }()
broker, err := redisstream.New(client, redisstream.Config{
Stream: settings.Stream, Group: settings.ConsumerGroup, Consumer: "dispatcher",
FailedStream: settings.FailedStream, FailedMaxLen: settings.FailedStreamMaxLength,
ClaimMinIdle: settings.LeaseDuration, ReadCount: int64(settings.BatchSize),
})
if err != nil {
return err
}
if err := broker.Ensure(context.Background()); err != nil {
return err
}
metrics := pipeline.NewMetrics(observability.MeterProvider().Meter("messenger-dispatcher"), observability.MetricsHandler(), 0)
dispatcher, err := outbox.New(postgres.New(pool), broker, metrics, outbox.Config{
BatchSize: settings.BatchSize, LeaseDuration: settings.LeaseDuration,
PollInterval: settings.PollInterval, RecoveryInterval: settings.RecoveryInterval,
RetryDelay: settings.RetryDelay, Retention: settings.Retention,
CleanupInterval: settings.CleanupInterval, CleanupBatchSize: settings.CleanupBatchSize,
})
if err != nil {
return err
}
logger.Info("outbox dispatcher started", "stream", settings.Stream)
return runPipelineProcess(logger, settings.MetricsAddress, metrics.Handler(), dispatcher.Run)
}
func RunWorker(logger *slog.Logger) error {
settings, err := config.LoadWorker()
if err != nil {
return err
}
hostname, err := os.Hostname()
if err != nil {
return err
}
observability, logger, err := startTelemetry("messenger-worker", hostname)
if err != nil {
return err
}
defer shutdownTelemetry(observability)
pool, err := database.Open(context.Background(), workerDatabaseConfig(settings))
if err != nil {
return err
}
defer pool.Close()
if err := telemetry.ObserveDatabasePool(observability.MeterProvider().Meter("messenger-worker/database"), pool); err != nil {
return err
}
objects, err := objectstore.New(objectstore.Config{
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout,
})
if err != nil {
return err
}
client, err := newRedisClient(settings.BrokerURL)
if err != nil {
return err
}
defer func() { _ = client.Close() }()
consumer := hostname + "-" + uuid.NewString()
broker, err := redisstream.New(client, redisstream.Config{
Stream: settings.Stream, Group: settings.ConsumerGroup, Consumer: consumer,
FailedStream: settings.FailedStream, FailedMaxLen: settings.FailedStreamMaxLength, Block: settings.ReadBlock,
ClaimMinIdle: settings.LeaseDuration, ReadCount: int64(settings.Prefetch),
})
if err != nil {
return err
}
if err := broker.Ensure(context.Background()); err != nil {
return err
}
processor, err := transform.NewProcessor(transform.DefaultConfig(settings.AttachmentMaxBytes, settings.AttachmentMaxPixels))
if err != nil {
return err
}
metrics := pipeline.NewMetrics(observability.MeterProvider().Meter("messenger-worker"), observability.MetricsHandler(), settings.Concurrency)
processorWorker, err := worker.New(
logger, postgres.New(pool), broker, objects, processor, metrics,
worker.Config{
WorkerID: consumer, Concurrency: settings.Concurrency, LeaseDuration: settings.LeaseDuration,
HeartbeatInterval: settings.HeartbeatInterval, ProcessingTimeout: settings.ProcessingTimeout,
ReclaimInterval: settings.ReclaimInterval, MaxAttempts: settings.MaxAttempts,
RetryBase: settings.RetryBase, RetryMax: settings.RetryMax,
MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels,
MemoryBytes: settings.MemoryBytes,
},
)
if err != nil {
return err
}
logger.Info("attachment worker started", "consumer", consumer, "concurrency", settings.Concurrency)
return runPipelineProcess(logger, settings.MetricsAddress, metrics.Handler(), processorWorker.Run)
}
func newRedisClient(rawURL string) (*redis.Client, error) {
options, err := redis.ParseURL(rawURL)
if err != nil {
return nil, err
}
client := redis.NewClient(options)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := client.Ping(ctx).Err(); err != nil {
_ = client.Close()
return nil, err
}
return client, nil
}
func runPipelineProcess(
logger *slog.Logger,
metricsAddress string,
metricsHandler http.Handler,
run func(context.Context) error,
) error {
mux := http.NewServeMux()
mux.Handle("/metrics", metricsHandler)
mux.HandleFunc("/health/live", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) })
server := &http.Server{Addr: metricsAddress, Handler: mux, ReadHeaderTimeout: 5 * time.Second}
listener, err := net.Listen("tcp", metricsAddress)
if err != nil {
return err
}
serveResult := make(chan error, 1)
go func() { serveResult <- server.Serve(listener) }()
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
runResult := make(chan error, 1)
go func() { runResult <- run(ctx) }()
var result error
select {
case result = <-runResult:
stop()
case result = <-serveResult:
stop()
case <-ctx.Done():
}
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := server.Shutdown(shutdownCtx); err != nil && result == nil {
result = err
}
if errors.Is(result, http.ErrServerClosed) || errors.Is(result, context.Canceled) {
return nil
}
logger.Info("pipeline process stopped")
return result
}
func dispatcherDatabaseConfig(settings config.DispatcherConfig) database.Config {
return database.Config{
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
}
}
func workerDatabaseConfig(settings config.WorkerConfig) database.Config {
return database.Config{
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
}
}