Files

367 lines
9.0 KiB
Go

package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"log/slog"
"net"
"net/http"
"os"
"os/signal"
"sync"
"syscall"
"time"
"mtusi-messenger/internal/broker"
"mtusi-messenger/internal/config"
"mtusi-messenger/internal/logger"
"mtusi-messenger/internal/platform/bao"
"mtusi-messenger/internal/platform/objectstore"
"mtusi-messenger/internal/service/attachment"
"mtusi-messenger/internal/storage/postgres"
"mtusi-messenger/internal/telemetry"
"mtusi-messenger/internal/worker"
)
var version = "dev"
func main() {
if err := execute(); err != nil {
logger.New().Error("worker stopped with error", slog.Any("error", err))
os.Exit(1)
}
}
func execute() (err error) {
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
serviceCtx, manager, err := bao.Start(ctx, "worker")
if err != nil {
return fmt.Errorf("start secret manager: %w", err)
}
var shutdown func(context.Context) error
defer func() {
manager.Stop()
shutdownCtx, cancel := manager.ShutdownContext(serviceCtx)
defer cancel()
if shutdown != nil {
err = errors.Join(err, shutdown(shutdownCtx))
}
err = errors.Join(err, manager.Close(serviceCtx))
if errors.Is(context.Cause(serviceCtx), bao.ErrExpired) {
err = errors.Join(err, bao.ErrExpired)
}
}()
cfg, err := config.Load()
if err != nil {
return fmt.Errorf("load config: %w", err)
}
log := logger.New(
logger.WithLevel(cfg.SlogLevel()),
logger.WithFormat(cfg.LoggerFormat()),
)
shutdown, err = telemetry.Setup(serviceCtx, "messenger-worker", version)
if err != nil {
return fmt.Errorf("setup telemetry: %w", err)
}
log = telemetry.WrapLogger(log, "messenger-worker", cfg.SlogLevel())
slog.SetDefault(log)
return run(
serviceCtx,
cfg,
log,
manager,
)
}
func run(
ctx context.Context,
cfg config.Config,
log *slog.Logger,
manager *bao.Manager,
) (err error) {
if cfg.Broker.URL == "" {
return errors.New("broker url is required for worker")
}
if err := requireLoopback(cfg.Worker.HealthAddr); err != nil {
return err
}
store, err := postgres.New(
ctx,
cfg.Postgres,
time.Duration(cfg.Idempotency.TTLSeconds)*time.Second,
postgres.WithQueryTracer(telemetry.NewPGXTracer()),
)
if err != nil {
return fmt.Errorf("connect database: %w", err)
}
defer func() {
err = errors.Join(err, finish(ctx, manager, func(context.Context) error {
store.Close()
return nil
}))
}()
if err := telemetry.RegisterDBPool(store.PoolStats); err != nil {
log.WarnContext(ctx, "register db pool metrics", slog.Any("error", err))
}
if err := telemetry.RegisterProcessMetrics(); err != nil {
log.WarnContext(ctx, "register process metrics", slog.Any("error", err))
}
objectStoreConcrete, err := objectstore.New(objectstore.Config{
Endpoint: cfg.S3.Endpoint,
PublicEndpoint: cfg.S3.PublicEndpoint,
Region: cfg.S3.Region,
Bucket: cfg.S3.Bucket,
AccessKeyID: cfg.S3.AccessKeyID,
SecretAccessKey: cfg.S3.SecretAccessKey,
UsePathStyle: cfg.S3.UsePathStyle,
Timeout: cfg.S3.Timeout,
})
if err != nil {
return fmt.Errorf("connect object store: %w", err)
}
defer objectStoreConcrete.Close()
objectStore := telemetry.WrapStore(objectStoreConcrete)
workerInstance := worker.New(
store.Pipeline(),
objectStore,
worker.Config{
Concurrency: cfg.Worker.Concurrency,
LeaseTTL: cfg.Processing.LeaseTTL,
Heartbeat: cfg.Processing.Heartbeat,
JobTimeout: cfg.Processing.JobTimeout,
RetryBase: cfg.Processing.RetryBase,
RetryCap: cfg.Processing.RetryCap,
MaxUploadBytes: cfg.S3.MaxUploadBytes,
MaxImagePixels: cfg.S3.MaxImagePixels,
},
log,
)
if err := telemetry.RegisterWorkerMetrics(workerInstance.Busy, int64(cfg.Worker.Concurrency)); err != nil {
log.WarnContext(ctx, "register worker metrics", slog.Any("error", err))
}
sweeper := attachment.NewSweeper(
store.Pipeline(),
objectStore,
store,
cfg.S3.SweepGrace,
cfg.Staging.OrphanGrace,
cfg.Staging.FailedRetention,
cfg.S3.SweepInterval,
log,
)
sweepDone := make(chan struct{})
go func() {
defer close(sweepDone)
sweeper.Run(ctx)
}()
consumerDone := make(chan error, 1)
go func() {
consumerDone <- consumeWithReconnect(
ctx,
cfg,
workerInstance,
log,
)
}()
server := healthServer(
ctx,
cfg.Worker.HealthAddr,
store,
objectStore,
workerInstance,
)
serverDone := make(chan error, 1)
go func() { serverDone <- server.ListenAndServe() }()
defer func() {
err = errors.Join(err, finish(ctx, manager, func(shutdownCtx context.Context) error {
serverErr := server.Shutdown(shutdownCtx)
if serverErr != nil {
serverErr = errors.Join(serverErr, server.Close())
}
select {
case <-sweepDone:
case <-shutdownCtx.Done():
return errors.Join(serverErr, shutdownCtx.Err())
}
select {
case consumerErr := <-consumerDone:
return errors.Join(serverErr, consumerErr)
case <-shutdownCtx.Done():
return errors.Join(serverErr, shutdownCtx.Err())
}
}))
}()
select {
case <-ctx.Done():
return nil
case serverErr := <-serverDone:
return fmt.Errorf("health listener: %w", serverErr)
}
}
func consumeWithReconnect(
ctx context.Context,
cfg config.Config,
workerInstance *worker.Worker,
log *slog.Logger,
) error {
for ctx.Err() == nil {
client, err := broker.Dial(brokerConfig(cfg))
if err != nil {
log.WarnContext(ctx, "broker dial failed, retrying", slog.Any("error", err))
if !sleepCtx(ctx, 2*time.Second) {
return nil
}
continue
}
var closeOnce sync.Once
var closeErr error
closeClient := func() {
closeOnce.Do(func() { closeErr = client.Close() })
}
stopClose := context.AfterFunc(ctx, closeClient)
err = workerInstance.Run(ctx, client)
stopClose()
closeClient()
if closeErr != nil && ctx.Err() == nil {
log.WarnContext(ctx, "close broker failed", slog.Any("error", closeErr))
}
if ctx.Err() != nil {
return closeErr
}
if err != nil {
log.WarnContext(ctx, "consumer stopped, reconnecting", slog.Any("error", err))
}
if !sleepCtx(ctx, 2*time.Second) {
return nil
}
}
return nil
}
func sleepCtx(ctx context.Context, d time.Duration) bool {
timer := time.NewTimer(d)
defer timer.Stop()
select {
case <-ctx.Done():
return false
case <-timer.C:
return true
}
}
func brokerConfig(cfg config.Config) broker.Config {
return broker.Config{
URL: cfg.Broker.URL,
Exchange: cfg.Broker.Exchange,
Queue: cfg.Broker.Queue,
ConnectTimeout: cfg.Broker.ConnectTimeout,
ConfirmTimeout: cfg.Broker.ConfirmTimeout,
Heartbeat: cfg.Broker.Heartbeat,
Prefetch: cfg.Worker.Prefetch,
Role: "worker",
Producer: false,
}
}
func healthServer(
serviceCtx context.Context,
addr string,
store *postgres.Store,
objectStore objectstore.Store,
workerInstance *worker.Worker,
) *http.Server {
mux := http.NewServeMux()
mux.HandleFunc("GET /health/live", func(w http.ResponseWriter, _ *http.Request) {
writeHealth(w, http.StatusOK, map[string]any{"status": "ok"})
})
mux.HandleFunc("GET /health/ready", func(w http.ResponseWriter, _ *http.Request) {
if serviceCtx.Err() != nil {
writeHealth(w, http.StatusServiceUnavailable, map[string]any{"status": "unavailable"})
return
}
ctx, cancel := context.WithTimeout(serviceCtx, 5*time.Second)
defer cancel()
if err := store.Ping(ctx); err != nil {
writeHealth(w, http.StatusServiceUnavailable, map[string]any{"status": "unavailable"})
return
}
if err := objectStore.Ping(ctx); err != nil {
writeHealth(w, http.StatusServiceUnavailable, map[string]any{"status": "unavailable"})
return
}
running, connected := workerInstance.Ready()
if !running || !connected || serviceCtx.Err() != nil {
writeHealth(w, http.StatusServiceUnavailable, map[string]any{"status": "unavailable"})
return
}
writeHealth(w, http.StatusOK, map[string]any{"status": "ok"})
})
return &http.Server{
Addr: addr,
Handler: mux,
ReadHeaderTimeout: 5 * time.Second,
ReadTimeout: 5 * time.Second,
WriteTimeout: 6 * time.Second,
BaseContext: func(net.Listener) context.Context { return serviceCtx },
}
}
func writeHealth(w http.ResponseWriter, status int, body map[string]any) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(body)
}
func requireLoopback(addr string) error {
host, _, err := net.SplitHostPort(addr)
if err != nil {
return fmt.Errorf("health listener address: %w", err)
}
ip := net.ParseIP(host)
if ip == nil || !ip.IsLoopback() {
return errors.New("health listener must bind a loopback ip")
}
return nil
}
func finish(parent context.Context, manager *bao.Manager, cleanup func(context.Context) error) error {
manager.Stop()
ctx, cancel := manager.ShutdownContext(parent)
defer cancel()
done := make(chan error, 1)
go func() { done <- cleanup(ctx) }()
select {
case err := <-done:
return err
case <-ctx.Done():
return fmt.Errorf("shutdown deadline: %w", ctx.Err())
}
}