forked from mtusi2026/mtusi2026lab7
367 lines
9.0 KiB
Go
367 lines
9.0 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"net"
|
|
"net/http"
|
|
"os"
|
|
"os/signal"
|
|
"sync"
|
|
"syscall"
|
|
"time"
|
|
|
|
"mtusi-messenger/internal/broker"
|
|
"mtusi-messenger/internal/config"
|
|
"mtusi-messenger/internal/logger"
|
|
"mtusi-messenger/internal/platform/bao"
|
|
"mtusi-messenger/internal/platform/objectstore"
|
|
"mtusi-messenger/internal/service/attachment"
|
|
"mtusi-messenger/internal/storage/postgres"
|
|
"mtusi-messenger/internal/telemetry"
|
|
"mtusi-messenger/internal/worker"
|
|
)
|
|
|
|
var version = "dev"
|
|
|
|
func main() {
|
|
if err := execute(); err != nil {
|
|
logger.New().Error("worker stopped with error", slog.Any("error", err))
|
|
os.Exit(1)
|
|
}
|
|
}
|
|
|
|
func execute() (err error) {
|
|
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
|
defer stop()
|
|
serviceCtx, manager, err := bao.Start(ctx, "worker")
|
|
if err != nil {
|
|
return fmt.Errorf("start secret manager: %w", err)
|
|
}
|
|
var shutdown func(context.Context) error
|
|
defer func() {
|
|
manager.Stop()
|
|
shutdownCtx, cancel := manager.ShutdownContext(serviceCtx)
|
|
defer cancel()
|
|
if shutdown != nil {
|
|
err = errors.Join(err, shutdown(shutdownCtx))
|
|
}
|
|
err = errors.Join(err, manager.Close(serviceCtx))
|
|
if errors.Is(context.Cause(serviceCtx), bao.ErrExpired) {
|
|
err = errors.Join(err, bao.ErrExpired)
|
|
}
|
|
}()
|
|
cfg, err := config.Load()
|
|
if err != nil {
|
|
return fmt.Errorf("load config: %w", err)
|
|
}
|
|
log := logger.New(
|
|
logger.WithLevel(cfg.SlogLevel()),
|
|
logger.WithFormat(cfg.LoggerFormat()),
|
|
)
|
|
shutdown, err = telemetry.Setup(serviceCtx, "messenger-worker", version)
|
|
if err != nil {
|
|
return fmt.Errorf("setup telemetry: %w", err)
|
|
}
|
|
log = telemetry.WrapLogger(log, "messenger-worker", cfg.SlogLevel())
|
|
slog.SetDefault(log)
|
|
return run(
|
|
serviceCtx,
|
|
cfg,
|
|
log,
|
|
manager,
|
|
)
|
|
}
|
|
|
|
func run(
|
|
ctx context.Context,
|
|
cfg config.Config,
|
|
log *slog.Logger,
|
|
manager *bao.Manager,
|
|
) (err error) {
|
|
if cfg.Broker.URL == "" {
|
|
return errors.New("broker url is required for worker")
|
|
}
|
|
if err := requireLoopback(cfg.Worker.HealthAddr); err != nil {
|
|
return err
|
|
}
|
|
store, err := postgres.New(
|
|
ctx,
|
|
cfg.Postgres,
|
|
time.Duration(cfg.Idempotency.TTLSeconds)*time.Second,
|
|
postgres.WithQueryTracer(telemetry.NewPGXTracer()),
|
|
)
|
|
if err != nil {
|
|
return fmt.Errorf("connect database: %w", err)
|
|
}
|
|
defer func() {
|
|
err = errors.Join(err, finish(ctx, manager, func(context.Context) error {
|
|
store.Close()
|
|
return nil
|
|
}))
|
|
}()
|
|
|
|
if err := telemetry.RegisterDBPool(store.PoolStats); err != nil {
|
|
log.WarnContext(ctx, "register db pool metrics", slog.Any("error", err))
|
|
}
|
|
if err := telemetry.RegisterProcessMetrics(); err != nil {
|
|
log.WarnContext(ctx, "register process metrics", slog.Any("error", err))
|
|
}
|
|
|
|
objectStoreConcrete, err := objectstore.New(objectstore.Config{
|
|
Endpoint: cfg.S3.Endpoint,
|
|
PublicEndpoint: cfg.S3.PublicEndpoint,
|
|
Region: cfg.S3.Region,
|
|
Bucket: cfg.S3.Bucket,
|
|
AccessKeyID: cfg.S3.AccessKeyID,
|
|
SecretAccessKey: cfg.S3.SecretAccessKey,
|
|
UsePathStyle: cfg.S3.UsePathStyle,
|
|
Timeout: cfg.S3.Timeout,
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("connect object store: %w", err)
|
|
}
|
|
defer objectStoreConcrete.Close()
|
|
|
|
objectStore := telemetry.WrapStore(objectStoreConcrete)
|
|
|
|
workerInstance := worker.New(
|
|
store.Pipeline(),
|
|
objectStore,
|
|
worker.Config{
|
|
Concurrency: cfg.Worker.Concurrency,
|
|
LeaseTTL: cfg.Processing.LeaseTTL,
|
|
Heartbeat: cfg.Processing.Heartbeat,
|
|
JobTimeout: cfg.Processing.JobTimeout,
|
|
RetryBase: cfg.Processing.RetryBase,
|
|
RetryCap: cfg.Processing.RetryCap,
|
|
MaxUploadBytes: cfg.S3.MaxUploadBytes,
|
|
MaxImagePixels: cfg.S3.MaxImagePixels,
|
|
},
|
|
log,
|
|
)
|
|
|
|
if err := telemetry.RegisterWorkerMetrics(workerInstance.Busy, int64(cfg.Worker.Concurrency)); err != nil {
|
|
log.WarnContext(ctx, "register worker metrics", slog.Any("error", err))
|
|
}
|
|
|
|
sweeper := attachment.NewSweeper(
|
|
store.Pipeline(),
|
|
objectStore,
|
|
store,
|
|
cfg.S3.SweepGrace,
|
|
cfg.Staging.OrphanGrace,
|
|
cfg.Staging.FailedRetention,
|
|
cfg.S3.SweepInterval,
|
|
log,
|
|
)
|
|
sweepDone := make(chan struct{})
|
|
go func() {
|
|
defer close(sweepDone)
|
|
sweeper.Run(ctx)
|
|
}()
|
|
consumerDone := make(chan error, 1)
|
|
go func() {
|
|
consumerDone <- consumeWithReconnect(
|
|
ctx,
|
|
cfg,
|
|
workerInstance,
|
|
log,
|
|
)
|
|
}()
|
|
server := healthServer(
|
|
ctx,
|
|
cfg.Worker.HealthAddr,
|
|
store,
|
|
objectStore,
|
|
workerInstance,
|
|
)
|
|
serverDone := make(chan error, 1)
|
|
go func() { serverDone <- server.ListenAndServe() }()
|
|
defer func() {
|
|
err = errors.Join(err, finish(ctx, manager, func(shutdownCtx context.Context) error {
|
|
serverErr := server.Shutdown(shutdownCtx)
|
|
if serverErr != nil {
|
|
serverErr = errors.Join(serverErr, server.Close())
|
|
}
|
|
select {
|
|
case <-sweepDone:
|
|
case <-shutdownCtx.Done():
|
|
return errors.Join(serverErr, shutdownCtx.Err())
|
|
}
|
|
select {
|
|
case consumerErr := <-consumerDone:
|
|
return errors.Join(serverErr, consumerErr)
|
|
case <-shutdownCtx.Done():
|
|
return errors.Join(serverErr, shutdownCtx.Err())
|
|
}
|
|
}))
|
|
}()
|
|
select {
|
|
case <-ctx.Done():
|
|
return nil
|
|
case serverErr := <-serverDone:
|
|
return fmt.Errorf("health listener: %w", serverErr)
|
|
}
|
|
}
|
|
|
|
func consumeWithReconnect(
|
|
ctx context.Context,
|
|
cfg config.Config,
|
|
workerInstance *worker.Worker,
|
|
log *slog.Logger,
|
|
) error {
|
|
for ctx.Err() == nil {
|
|
client, err := broker.Dial(brokerConfig(cfg))
|
|
if err != nil {
|
|
log.WarnContext(ctx, "broker dial failed, retrying", slog.Any("error", err))
|
|
|
|
if !sleepCtx(ctx, 2*time.Second) {
|
|
return nil
|
|
}
|
|
|
|
continue
|
|
}
|
|
|
|
var closeOnce sync.Once
|
|
var closeErr error
|
|
closeClient := func() {
|
|
closeOnce.Do(func() { closeErr = client.Close() })
|
|
}
|
|
stopClose := context.AfterFunc(ctx, closeClient)
|
|
err = workerInstance.Run(ctx, client)
|
|
stopClose()
|
|
closeClient()
|
|
if closeErr != nil && ctx.Err() == nil {
|
|
log.WarnContext(ctx, "close broker failed", slog.Any("error", closeErr))
|
|
}
|
|
|
|
if ctx.Err() != nil {
|
|
return closeErr
|
|
}
|
|
|
|
if err != nil {
|
|
log.WarnContext(ctx, "consumer stopped, reconnecting", slog.Any("error", err))
|
|
}
|
|
|
|
if !sleepCtx(ctx, 2*time.Second) {
|
|
return nil
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func sleepCtx(ctx context.Context, d time.Duration) bool {
|
|
timer := time.NewTimer(d)
|
|
defer timer.Stop()
|
|
|
|
select {
|
|
case <-ctx.Done():
|
|
return false
|
|
case <-timer.C:
|
|
return true
|
|
}
|
|
}
|
|
|
|
func brokerConfig(cfg config.Config) broker.Config {
|
|
return broker.Config{
|
|
URL: cfg.Broker.URL,
|
|
Exchange: cfg.Broker.Exchange,
|
|
Queue: cfg.Broker.Queue,
|
|
ConnectTimeout: cfg.Broker.ConnectTimeout,
|
|
ConfirmTimeout: cfg.Broker.ConfirmTimeout,
|
|
Heartbeat: cfg.Broker.Heartbeat,
|
|
Prefetch: cfg.Worker.Prefetch,
|
|
Role: "worker",
|
|
Producer: false,
|
|
}
|
|
}
|
|
|
|
func healthServer(
|
|
serviceCtx context.Context,
|
|
addr string,
|
|
store *postgres.Store,
|
|
objectStore objectstore.Store,
|
|
workerInstance *worker.Worker,
|
|
) *http.Server {
|
|
mux := http.NewServeMux()
|
|
|
|
mux.HandleFunc("GET /health/live", func(w http.ResponseWriter, _ *http.Request) {
|
|
writeHealth(w, http.StatusOK, map[string]any{"status": "ok"})
|
|
})
|
|
|
|
mux.HandleFunc("GET /health/ready", func(w http.ResponseWriter, _ *http.Request) {
|
|
if serviceCtx.Err() != nil {
|
|
writeHealth(w, http.StatusServiceUnavailable, map[string]any{"status": "unavailable"})
|
|
return
|
|
}
|
|
ctx, cancel := context.WithTimeout(serviceCtx, 5*time.Second)
|
|
defer cancel()
|
|
|
|
if err := store.Ping(ctx); err != nil {
|
|
writeHealth(w, http.StatusServiceUnavailable, map[string]any{"status": "unavailable"})
|
|
|
|
return
|
|
}
|
|
|
|
if err := objectStore.Ping(ctx); err != nil {
|
|
writeHealth(w, http.StatusServiceUnavailable, map[string]any{"status": "unavailable"})
|
|
|
|
return
|
|
}
|
|
|
|
running, connected := workerInstance.Ready()
|
|
if !running || !connected || serviceCtx.Err() != nil {
|
|
writeHealth(w, http.StatusServiceUnavailable, map[string]any{"status": "unavailable"})
|
|
|
|
return
|
|
}
|
|
|
|
writeHealth(w, http.StatusOK, map[string]any{"status": "ok"})
|
|
})
|
|
|
|
return &http.Server{
|
|
Addr: addr,
|
|
Handler: mux,
|
|
ReadHeaderTimeout: 5 * time.Second,
|
|
ReadTimeout: 5 * time.Second,
|
|
WriteTimeout: 6 * time.Second,
|
|
BaseContext: func(net.Listener) context.Context { return serviceCtx },
|
|
}
|
|
}
|
|
|
|
func writeHealth(w http.ResponseWriter, status int, body map[string]any) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(status)
|
|
_ = json.NewEncoder(w).Encode(body)
|
|
}
|
|
|
|
func requireLoopback(addr string) error {
|
|
host, _, err := net.SplitHostPort(addr)
|
|
if err != nil {
|
|
return fmt.Errorf("health listener address: %w", err)
|
|
}
|
|
ip := net.ParseIP(host)
|
|
if ip == nil || !ip.IsLoopback() {
|
|
return errors.New("health listener must bind a loopback ip")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func finish(parent context.Context, manager *bao.Manager, cleanup func(context.Context) error) error {
|
|
manager.Stop()
|
|
ctx, cancel := manager.ShutdownContext(parent)
|
|
defer cancel()
|
|
done := make(chan error, 1)
|
|
go func() { done <- cleanup(ctx) }()
|
|
select {
|
|
case err := <-done:
|
|
return err
|
|
case <-ctx.Done():
|
|
return fmt.Errorf("shutdown deadline: %w", ctx.Err())
|
|
}
|
|
}
|