forked from mtusi2026/mtusi2026lab5
- RabbitMQ quorum queue, persistent messages and publisher confirms - S3 staging plus PostgreSQL attachments/jobs outbox transaction - worker variants (thumbnail, crop, optimized WebP), DB lease and lease sweeper - jittered retry, permanent failures, explicit replay and admission cap - deterministic crash hooks, resilience script and evidence for grade 4 - async-aware custom tests, oversized pixel case, docs and changelog
105 lines
3.3 KiB
Go
105 lines
3.3 KiB
Go
// Command worker consumes durable attachment jobs, writes variants and acks
|
|
// only after the ready snapshot commits. WORKER_ONLY=true skips Redis setup.
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"flag"
|
|
"log"
|
|
"log/slog"
|
|
"os"
|
|
"os/signal"
|
|
"syscall"
|
|
|
|
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/broker"
|
|
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/config"
|
|
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/database"
|
|
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/objectstore"
|
|
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/outbox"
|
|
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/store"
|
|
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/worker"
|
|
"github.com/google/uuid"
|
|
)
|
|
|
|
func main() {
|
|
replay := flag.String("replay", "", "explicitly requeue one failed attachment UUID and exit")
|
|
flag.Parse()
|
|
|
|
cfg, err := config.Load()
|
|
if err != nil {
|
|
log.Fatalf("configuration: %v", err)
|
|
}
|
|
|
|
rootCtx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
|
defer stop()
|
|
|
|
pool, err := database.Connect(rootCtx, cfg.DatabaseURL, cfg.DBMaxConns, cfg.StartupWait)
|
|
if err != nil {
|
|
log.Fatalf("database: %v", err)
|
|
}
|
|
defer pool.Close()
|
|
st := store.New(pool, cfg.DBTimeout, cfg.IdempotencyTTL)
|
|
|
|
if *replay != "" {
|
|
attachmentID, parseErr := uuid.Parse(*replay)
|
|
if parseErr != nil {
|
|
log.Fatalf("replay: invalid attachment UUID: %v", parseErr)
|
|
}
|
|
changed, replayErr := st.ReplayFailed(rootCtx, attachmentID)
|
|
if replayErr != nil {
|
|
log.Fatalf("replay: %v", replayErr)
|
|
}
|
|
if !changed {
|
|
log.Printf("replay: attachment %s is not failed; nothing changed", attachmentID)
|
|
return
|
|
}
|
|
// Publish immediately so a replay works even when no API replica is
|
|
// currently running its outbox loop. Duplicate publication is safe.
|
|
publisher := broker.NewPublisher(cfg.BrokerURL, cfg.JobQueue)
|
|
defer publisher.Close()
|
|
if replayErr = outbox.New(st, publisher, nil).PublishBatch(rootCtx); replayErr != nil {
|
|
log.Fatalf("replay publish: %v", replayErr)
|
|
}
|
|
log.Printf("replay: attachment %s requeued", attachmentID)
|
|
return
|
|
}
|
|
|
|
objects, err := objectstore.New(objectstore.Config{
|
|
Endpoint: cfg.S3Endpoint,
|
|
Region: cfg.S3Region,
|
|
Bucket: cfg.S3Bucket,
|
|
AccessKey: cfg.S3AccessKey,
|
|
SecretKey: cfg.S3SecretKey,
|
|
})
|
|
if err != nil {
|
|
log.Fatalf("object storage: %v", err)
|
|
}
|
|
|
|
logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo}))
|
|
slog.SetDefault(logger)
|
|
|
|
processor := worker.New(cfg, st, objects, logger)
|
|
go processor.StartLeaseSweeper(rootCtx)
|
|
|
|
// The worker also drains the outbox. This keeps retries/replay moving even
|
|
// if every API replica is down. Two publishers only create duplicates that
|
|
// the job lease collapses.
|
|
publisher := broker.NewPublisher(cfg.BrokerURL, cfg.JobQueue)
|
|
defer publisher.Close()
|
|
go outbox.New(st, publisher, logger).Run(rootCtx)
|
|
|
|
logger.Info("worker starting",
|
|
"queue", cfg.JobQueue,
|
|
"concurrency", cfg.WorkerConcurrency,
|
|
"prefetch", cfg.WorkerPrefetch,
|
|
"max_attempts", cfg.JobMaxAttempts,
|
|
"job_timeout", cfg.JobTimeout,
|
|
"fault_mode", cfg.WorkerFaultMode)
|
|
|
|
consumer := broker.NewConsumer(cfg.BrokerURL, cfg.JobQueue, logger)
|
|
if err = consumer.Run(rootCtx, cfg.WorkerConcurrency, cfg.WorkerPrefetch, processor.Handle); err != nil {
|
|
log.Fatalf("worker: %v", err)
|
|
}
|
|
logger.Info("worker stopped")
|
|
}
|