forked from mtusi2026/mtusi2026lab5
feat(lab5): durable async attachment pipeline through grade 4
- RabbitMQ quorum queue, persistent messages and publisher confirms - S3 staging plus PostgreSQL attachments/jobs outbox transaction - worker variants (thumbnail, crop, optimized WebP), DB lease and lease sweeper - jittered retry, permanent failures, explicit replay and admission cap - deterministic crash hooks, resilience script and evidence for grade 4 - async-aware custom tests, oversized pixel case, docs and changelog
This commit is contained in:
@@ -0,0 +1,104 @@
|
||||
// Command worker consumes durable attachment jobs, writes variants and acks
|
||||
// only after the ready snapshot commits. WORKER_ONLY=true skips Redis setup.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"log"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/broker"
|
||||
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/config"
|
||||
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/database"
|
||||
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/objectstore"
|
||||
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/outbox"
|
||||
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/store"
|
||||
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/worker"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func main() {
|
||||
replay := flag.String("replay", "", "explicitly requeue one failed attachment UUID and exit")
|
||||
flag.Parse()
|
||||
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
log.Fatalf("configuration: %v", err)
|
||||
}
|
||||
|
||||
rootCtx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
pool, err := database.Connect(rootCtx, cfg.DatabaseURL, cfg.DBMaxConns, cfg.StartupWait)
|
||||
if err != nil {
|
||||
log.Fatalf("database: %v", err)
|
||||
}
|
||||
defer pool.Close()
|
||||
st := store.New(pool, cfg.DBTimeout, cfg.IdempotencyTTL)
|
||||
|
||||
if *replay != "" {
|
||||
attachmentID, parseErr := uuid.Parse(*replay)
|
||||
if parseErr != nil {
|
||||
log.Fatalf("replay: invalid attachment UUID: %v", parseErr)
|
||||
}
|
||||
changed, replayErr := st.ReplayFailed(rootCtx, attachmentID)
|
||||
if replayErr != nil {
|
||||
log.Fatalf("replay: %v", replayErr)
|
||||
}
|
||||
if !changed {
|
||||
log.Printf("replay: attachment %s is not failed; nothing changed", attachmentID)
|
||||
return
|
||||
}
|
||||
// Publish immediately so a replay works even when no API replica is
|
||||
// currently running its outbox loop. Duplicate publication is safe.
|
||||
publisher := broker.NewPublisher(cfg.BrokerURL, cfg.JobQueue)
|
||||
defer publisher.Close()
|
||||
if replayErr = outbox.New(st, publisher, nil).PublishBatch(rootCtx); replayErr != nil {
|
||||
log.Fatalf("replay publish: %v", replayErr)
|
||||
}
|
||||
log.Printf("replay: attachment %s requeued", attachmentID)
|
||||
return
|
||||
}
|
||||
|
||||
objects, err := objectstore.New(objectstore.Config{
|
||||
Endpoint: cfg.S3Endpoint,
|
||||
Region: cfg.S3Region,
|
||||
Bucket: cfg.S3Bucket,
|
||||
AccessKey: cfg.S3AccessKey,
|
||||
SecretKey: cfg.S3SecretKey,
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatalf("object storage: %v", err)
|
||||
}
|
||||
|
||||
logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo}))
|
||||
slog.SetDefault(logger)
|
||||
|
||||
processor := worker.New(cfg, st, objects, logger)
|
||||
go processor.StartLeaseSweeper(rootCtx)
|
||||
|
||||
// The worker also drains the outbox. This keeps retries/replay moving even
|
||||
// if every API replica is down. Two publishers only create duplicates that
|
||||
// the job lease collapses.
|
||||
publisher := broker.NewPublisher(cfg.BrokerURL, cfg.JobQueue)
|
||||
defer publisher.Close()
|
||||
go outbox.New(st, publisher, logger).Run(rootCtx)
|
||||
|
||||
logger.Info("worker starting",
|
||||
"queue", cfg.JobQueue,
|
||||
"concurrency", cfg.WorkerConcurrency,
|
||||
"prefetch", cfg.WorkerPrefetch,
|
||||
"max_attempts", cfg.JobMaxAttempts,
|
||||
"job_timeout", cfg.JobTimeout,
|
||||
"fault_mode", cfg.WorkerFaultMode)
|
||||
|
||||
consumer := broker.NewConsumer(cfg.BrokerURL, cfg.JobQueue, logger)
|
||||
if err = consumer.Run(rootCtx, cfg.WorkerConcurrency, cfg.WorkerPrefetch, processor.Handle); err != nil {
|
||||
log.Fatalf("worker: %v", err)
|
||||
}
|
||||
logger.Info("worker stopped")
|
||||
}
|
||||
Reference in New Issue
Block a user