Durable staging handoff, transactional outbox, at-least-once worker with lease fencing, idempotent versioned pipeline, three variants, operator tooling, contract tests and evidence. Includes the review fixes: reprocess source selection and failure settlement, replay guards, stale-job retirement, PNG structural/raster validation, migration rollback safety and audit events.
232 lines
5.3 KiB
Go
232 lines
5.3 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"net/http"
|
|
"os"
|
|
"os/signal"
|
|
"sync"
|
|
"syscall"
|
|
"time"
|
|
|
|
"mtusi-messenger/internal/broker"
|
|
"mtusi-messenger/internal/config"
|
|
"mtusi-messenger/internal/logger"
|
|
"mtusi-messenger/internal/platform/objectstore"
|
|
"mtusi-messenger/internal/storage/postgres"
|
|
"mtusi-messenger/internal/worker"
|
|
)
|
|
|
|
func main() {
|
|
log := logger.New()
|
|
|
|
cfg, err := config.Load()
|
|
if err != nil {
|
|
log.Error("load config", slog.Any("error", err))
|
|
os.Exit(1)
|
|
}
|
|
|
|
log = logger.New(
|
|
logger.WithLevel(cfg.SlogLevel()),
|
|
logger.WithFormat(cfg.LoggerFormat()),
|
|
)
|
|
slog.SetDefault(log)
|
|
|
|
if err := run(cfg, log); err != nil {
|
|
log.Error("shutdown with error", slog.Any("error", err))
|
|
os.Exit(1)
|
|
}
|
|
}
|
|
|
|
func run(cfg config.Config, log *slog.Logger) error {
|
|
if cfg.Broker.URL == "" {
|
|
return errors.New("BROKER_URL is required for worker")
|
|
}
|
|
|
|
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
|
defer stop()
|
|
|
|
store, err := postgres.New(ctx, cfg.Postgres, time.Duration(cfg.Idempotency.TTLSeconds)*time.Second)
|
|
if err != nil {
|
|
return fmt.Errorf("connect database: %w", err)
|
|
}
|
|
defer store.Close()
|
|
|
|
objectStore, err := objectstore.New(objectstore.Config{
|
|
Endpoint: cfg.S3.Endpoint,
|
|
PublicEndpoint: cfg.S3.PublicEndpoint,
|
|
Region: cfg.S3.Region,
|
|
Bucket: cfg.S3.Bucket,
|
|
AccessKeyID: cfg.S3.AccessKeyID,
|
|
SecretAccessKey: cfg.S3.SecretAccessKey,
|
|
UsePathStyle: cfg.S3.UsePathStyle,
|
|
Timeout: cfg.S3.Timeout,
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("connect object store: %w", err)
|
|
}
|
|
|
|
workerInstance := worker.New(store.Pipeline(), objectStore, worker.Config{
|
|
Concurrency: cfg.Worker.Concurrency,
|
|
LeaseTTL: cfg.Processing.LeaseTTL,
|
|
Heartbeat: cfg.Processing.Heartbeat,
|
|
JobTimeout: cfg.Processing.JobTimeout,
|
|
RetryBase: cfg.Processing.RetryBase,
|
|
RetryCap: cfg.Processing.RetryCap,
|
|
MaxUploadBytes: cfg.S3.MaxUploadBytes,
|
|
MaxImagePixels: cfg.S3.MaxImagePixels,
|
|
}, log)
|
|
|
|
server := healthServer(cfg.Worker.HealthAddr, store, objectStore, workerInstance)
|
|
|
|
var serverWait sync.WaitGroup
|
|
serverWait.Add(1)
|
|
|
|
go func() {
|
|
defer serverWait.Done()
|
|
|
|
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
|
log.Error("health server failed", slog.Any("error", err))
|
|
}
|
|
}()
|
|
|
|
runCtx, cancelRun := context.WithCancel(ctx)
|
|
defer cancelRun()
|
|
|
|
consumerDone := make(chan struct{})
|
|
|
|
go func() {
|
|
defer close(consumerDone)
|
|
|
|
consumeWithReconnect(runCtx, cfg, workerInstance, log)
|
|
}()
|
|
|
|
<-ctx.Done()
|
|
cancelRun()
|
|
<-consumerDone
|
|
|
|
shutdownCtx, cancel := context.WithTimeout(context.Background(), cfg.Worker.ShutdownTimeout)
|
|
defer cancel()
|
|
|
|
_ = server.Shutdown(shutdownCtx)
|
|
serverWait.Wait()
|
|
|
|
return nil
|
|
}
|
|
|
|
func consumeWithReconnect(
|
|
ctx context.Context,
|
|
cfg config.Config,
|
|
workerInstance *worker.Worker,
|
|
log *slog.Logger,
|
|
) {
|
|
for ctx.Err() == nil {
|
|
client, err := broker.Dial(brokerConfig(cfg))
|
|
if err != nil {
|
|
log.WarnContext(ctx, "broker dial failed, retrying", slog.Any("error", err))
|
|
|
|
if !sleepCtx(ctx, 2*time.Second) {
|
|
return
|
|
}
|
|
|
|
continue
|
|
}
|
|
|
|
err = workerInstance.Run(ctx, client)
|
|
_ = client.Close()
|
|
|
|
if ctx.Err() != nil {
|
|
return
|
|
}
|
|
|
|
if err != nil {
|
|
log.WarnContext(ctx, "consumer stopped, reconnecting", slog.Any("error", err))
|
|
}
|
|
|
|
if !sleepCtx(ctx, 2*time.Second) {
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
func sleepCtx(ctx context.Context, d time.Duration) bool {
|
|
timer := time.NewTimer(d)
|
|
defer timer.Stop()
|
|
|
|
select {
|
|
case <-ctx.Done():
|
|
return false
|
|
case <-timer.C:
|
|
return true
|
|
}
|
|
}
|
|
|
|
func brokerConfig(cfg config.Config) broker.Config {
|
|
return broker.Config{
|
|
URL: cfg.Broker.URL,
|
|
Exchange: cfg.Broker.Exchange,
|
|
DLX: cfg.Broker.DLX,
|
|
Queue: cfg.Broker.Queue,
|
|
DLQ: cfg.Broker.DLQ,
|
|
ConnectTimeout: cfg.Broker.ConnectTimeout,
|
|
ConfirmTimeout: cfg.Broker.ConfirmTimeout,
|
|
Heartbeat: cfg.Broker.Heartbeat,
|
|
Prefetch: cfg.Worker.Prefetch,
|
|
}
|
|
}
|
|
|
|
func healthServer(
|
|
addr string,
|
|
store *postgres.Store,
|
|
objectStore objectstore.Store,
|
|
workerInstance *worker.Worker,
|
|
) *http.Server {
|
|
mux := http.NewServeMux()
|
|
|
|
mux.HandleFunc("/health/live", func(w http.ResponseWriter, _ *http.Request) {
|
|
writeHealth(w, http.StatusOK, map[string]any{"status": "ok"})
|
|
})
|
|
|
|
mux.HandleFunc("/health/ready", func(w http.ResponseWriter, r *http.Request) {
|
|
ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second)
|
|
defer cancel()
|
|
|
|
if err := store.Ping(ctx); err != nil {
|
|
writeHealth(w, http.StatusServiceUnavailable, map[string]any{"status": "unavailable"})
|
|
|
|
return
|
|
}
|
|
|
|
if err := objectStore.Ping(ctx); err != nil {
|
|
writeHealth(w, http.StatusServiceUnavailable, map[string]any{"status": "unavailable"})
|
|
|
|
return
|
|
}
|
|
|
|
running, connected := workerInstance.Ready()
|
|
if !running || !connected {
|
|
writeHealth(w, http.StatusServiceUnavailable, map[string]any{"status": "unavailable"})
|
|
|
|
return
|
|
}
|
|
|
|
writeHealth(w, http.StatusOK, map[string]any{"status": "ok"})
|
|
})
|
|
|
|
return &http.Server{
|
|
Addr: addr,
|
|
Handler: mux,
|
|
ReadHeaderTimeout: 5 * time.Second,
|
|
}
|
|
}
|
|
|
|
func writeHealth(w http.ResponseWriter, status int, body map[string]any) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(status)
|
|
_ = json.NewEncoder(w).Encode(body)
|
|
}
|