forked from mtusi2026/mtusi2026lab6
223 lines
7.7 KiB
Go
223 lines
7.7 KiB
Go
package bootstrap
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"log/slog"
|
|
"net"
|
|
"net/http"
|
|
"os"
|
|
"os/signal"
|
|
"syscall"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/redis/go-redis/v9"
|
|
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/broker/redisstream"
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config"
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database"
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/pipeline"
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform"
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/worker"
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore"
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/outbox"
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres"
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry"
|
|
)
|
|
|
|
func RunDispatcher(logger *slog.Logger) error {
|
|
settings, err := config.LoadDispatcher()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
instanceID, err := os.Hostname()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
observability, logger, err := startTelemetry("messenger-dispatcher", instanceID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer shutdownTelemetry(observability)
|
|
pool, err := database.Open(context.Background(), dispatcherDatabaseConfig(settings))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer pool.Close()
|
|
if err := telemetry.ObserveDatabasePool(observability.MeterProvider().Meter("messenger-dispatcher/database"), pool); err != nil {
|
|
return err
|
|
}
|
|
client, err := newRedisClient(settings.BrokerURL)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer func() { _ = client.Close() }()
|
|
broker, err := redisstream.New(client, redisstream.Config{
|
|
Stream: settings.Stream, Group: settings.ConsumerGroup, Consumer: "dispatcher",
|
|
FailedStream: settings.FailedStream, FailedMaxLen: settings.FailedStreamMaxLength,
|
|
ClaimMinIdle: settings.LeaseDuration, ReadCount: int64(settings.BatchSize),
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := broker.Ensure(context.Background()); err != nil {
|
|
return err
|
|
}
|
|
metrics := pipeline.NewMetrics(observability.MeterProvider().Meter("messenger-dispatcher"), observability.MetricsHandler(), 0)
|
|
dispatcher, err := outbox.New(postgres.New(pool), broker, metrics, outbox.Config{
|
|
BatchSize: settings.BatchSize, LeaseDuration: settings.LeaseDuration,
|
|
PollInterval: settings.PollInterval, RecoveryInterval: settings.RecoveryInterval,
|
|
RetryDelay: settings.RetryDelay, Retention: settings.Retention,
|
|
CleanupInterval: settings.CleanupInterval, CleanupBatchSize: settings.CleanupBatchSize,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
logger.Info("outbox dispatcher started", "stream", settings.Stream)
|
|
return runPipelineProcess(logger, settings.MetricsAddress, metrics.Handler(), dispatcher.Run)
|
|
}
|
|
|
|
func RunWorker(logger *slog.Logger) error {
|
|
settings, err := config.LoadWorker()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
hostname, err := os.Hostname()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
observability, logger, err := startTelemetry("messenger-worker", hostname)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer shutdownTelemetry(observability)
|
|
pool, err := database.Open(context.Background(), workerDatabaseConfig(settings))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer pool.Close()
|
|
if err := telemetry.ObserveDatabasePool(observability.MeterProvider().Meter("messenger-worker/database"), pool); err != nil {
|
|
return err
|
|
}
|
|
objects, err := objectstore.New(objectstore.Config{
|
|
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
|
|
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
|
|
UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
client, err := newRedisClient(settings.BrokerURL)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer func() { _ = client.Close() }()
|
|
consumer := hostname + "-" + uuid.NewString()
|
|
broker, err := redisstream.New(client, redisstream.Config{
|
|
Stream: settings.Stream, Group: settings.ConsumerGroup, Consumer: consumer,
|
|
FailedStream: settings.FailedStream, FailedMaxLen: settings.FailedStreamMaxLength, Block: settings.ReadBlock,
|
|
ClaimMinIdle: settings.LeaseDuration, ReadCount: int64(settings.Prefetch),
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := broker.Ensure(context.Background()); err != nil {
|
|
return err
|
|
}
|
|
processor, err := transform.NewProcessor(transform.DefaultConfig(settings.AttachmentMaxBytes, settings.AttachmentMaxPixels))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
metrics := pipeline.NewMetrics(observability.MeterProvider().Meter("messenger-worker"), observability.MetricsHandler(), settings.Concurrency)
|
|
processorWorker, err := worker.New(
|
|
logger, postgres.New(pool), broker, objects, processor, metrics,
|
|
worker.Config{
|
|
WorkerID: consumer, Concurrency: settings.Concurrency, LeaseDuration: settings.LeaseDuration,
|
|
HeartbeatInterval: settings.HeartbeatInterval, ProcessingTimeout: settings.ProcessingTimeout,
|
|
ReclaimInterval: settings.ReclaimInterval, MaxAttempts: settings.MaxAttempts,
|
|
RetryBase: settings.RetryBase, RetryMax: settings.RetryMax,
|
|
MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels,
|
|
MemoryBytes: settings.MemoryBytes,
|
|
},
|
|
)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
logger.Info("attachment worker started", "consumer", consumer, "concurrency", settings.Concurrency)
|
|
return runPipelineProcess(logger, settings.MetricsAddress, metrics.Handler(), processorWorker.Run)
|
|
}
|
|
|
|
func newRedisClient(rawURL string) (*redis.Client, error) {
|
|
options, err := redis.ParseURL(rawURL)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
client := redis.NewClient(options)
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
if err := client.Ping(ctx).Err(); err != nil {
|
|
_ = client.Close()
|
|
return nil, err
|
|
}
|
|
return client, nil
|
|
}
|
|
|
|
func runPipelineProcess(
|
|
logger *slog.Logger,
|
|
metricsAddress string,
|
|
metricsHandler http.Handler,
|
|
run func(context.Context) error,
|
|
) error {
|
|
mux := http.NewServeMux()
|
|
mux.Handle("/metrics", metricsHandler)
|
|
mux.HandleFunc("/health/live", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) })
|
|
server := &http.Server{Addr: metricsAddress, Handler: mux, ReadHeaderTimeout: 5 * time.Second}
|
|
listener, err := net.Listen("tcp", metricsAddress)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
serveResult := make(chan error, 1)
|
|
go func() { serveResult <- server.Serve(listener) }()
|
|
|
|
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
|
defer stop()
|
|
runResult := make(chan error, 1)
|
|
go func() { runResult <- run(ctx) }()
|
|
|
|
var result error
|
|
select {
|
|
case result = <-runResult:
|
|
stop()
|
|
case result = <-serveResult:
|
|
stop()
|
|
case <-ctx.Done():
|
|
}
|
|
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
if err := server.Shutdown(shutdownCtx); err != nil && result == nil {
|
|
result = err
|
|
}
|
|
if errors.Is(result, http.ErrServerClosed) || errors.Is(result, context.Canceled) {
|
|
return nil
|
|
}
|
|
logger.Info("pipeline process stopped")
|
|
return result
|
|
}
|
|
|
|
func dispatcherDatabaseConfig(settings config.DispatcherConfig) database.Config {
|
|
return database.Config{
|
|
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
|
|
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
|
|
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
|
|
}
|
|
}
|
|
|
|
func workerDatabaseConfig(settings config.WorkerConfig) database.Config {
|
|
return database.Config{
|
|
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
|
|
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
|
|
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
|
|
}
|
|
}
|