Files
mtusi2026lab5/cmd/worker/main.go
2026-11-10 16:21:46 +03:00

241 lines
6.7 KiB
Go

package main
import (
"context"
"errors"
"fmt"
"log"
"os"
"os/signal"
"sync"
"syscall"
"time"
"mtusi2026lab5/internal/adapter/broker"
"mtusi2026lab5/internal/adapter/objectstore"
"mtusi2026lab5/internal/adapter/postgres"
"mtusi2026lab5/internal/config"
"mtusi2026lab5/internal/database"
"mtusi2026lab5/internal/domain"
"mtusi2026lab5/internal/worker"
)
const (
defaultBeforeObjectsFaultFile = "/tmp/mtusi-fault-before-objects"
defaultAfterObjectsFaultFile = "/tmp/mtusi-fault-after-objects"
)
func main() {
cfg, err := config.LoadWorker()
if err != nil {
log.Fatalf("[FATAL] Invalid configuration: %v", err)
}
if cfg.BrokerURL == "" {
log.Fatal("[FATAL] BROKER_URL or BROKER_URL_FILE is required")
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
pool, err := database.Open(ctx, cfg)
if err != nil {
log.Fatalf("[FATAL] PostgreSQL initialization failed: %v", err)
}
defer pool.Close()
store := postgres.NewPostgreSQLStore(pool, cfg.DatabaseQueryTimeout)
objects, err := objectstore.NewMinIOStore(
cfg.S3Endpoint,
cfg.S3Region,
cfg.S3Bucket,
cfg.S3AccessKeyID,
cfg.S3SecretAccessKey,
cfg.S3OperationTimeout,
)
if err != nil {
log.Fatalf("[FATAL] Object storage initialization failed: %v", err)
}
beforeFaultFile := os.Getenv("WORKER_FAULT_BEFORE_OBJECTS_FILE")
if beforeFaultFile == "" {
beforeFaultFile = defaultBeforeObjectsFaultFile
}
afterFaultFile := os.Getenv("WORKER_FAULT_AFTER_OBJECTS_FILE")
if afterFaultFile == "" {
afterFaultFile = defaultAfterObjectsFaultFile
}
processor, err := worker.NewProcessor(store, objects, worker.ProcessorHooks{
BeforeObjects: func(job domain.AttachmentJob) error {
removeErr := os.Remove(beforeFaultFile)
if errors.Is(removeErr, os.ErrNotExist) {
return nil
}
if removeErr != nil {
return fmt.Errorf("consume before-objects fault marker: %w", removeErr)
}
log.Printf("[FAULT] exiting before object writes for job %s", job.JobID)
os.Exit(85)
return nil
},
AfterObjects: func(job domain.AttachmentJob) error {
removeErr := os.Remove(afterFaultFile)
if errors.Is(removeErr, os.ErrNotExist) {
return nil
}
if removeErr != nil {
return fmt.Errorf("consume after-objects fault marker: %w", removeErr)
}
log.Printf("[FAULT] exiting after object writes and before DB commit for job %s", job.JobID)
os.Exit(86)
return nil
},
})
if err != nil {
log.Fatalf("[FATAL] Worker processor initialization failed: %v", err)
}
cleaner, err := worker.NewStagingCleaner(store, objects, cfg.StagingRetention)
if err != nil {
log.Fatalf("[FATAL] Staging cleaner initialization failed: %v", err)
}
var maintenance sync.WaitGroup
maintenance.Add(1)
go func() {
defer maintenance.Done()
runCleanup := func() {
cleanupContext, cancel := context.WithTimeout(ctx, cfg.S3OperationTimeout+cfg.DatabaseQueryTimeout)
deleted, cleanupErr := cleaner.RunOnce(cleanupContext, time.Now().UTC())
cancel()
if cleanupErr != nil && ctx.Err() == nil {
log.Printf("[WARN] Staging cleanup failed: %v", cleanupErr)
} else if deleted > 0 {
log.Printf("[INFO] Staging cleanup removed %d object(s)", deleted)
}
}
runCleanup()
ticker := time.NewTicker(cfg.StagingCleanupInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
runCleanup()
}
}
}()
client, err := broker.Open(cfg.BrokerURL, cfg.WorkerConcurrency)
if err != nil {
log.Fatalf("[FATAL] RabbitMQ initialization failed: %v", err)
}
defer client.Close()
deliveries, err := client.Consume(cfg.InstanceID)
if err != nil {
log.Fatalf("[FATAL] RabbitMQ consume failed: %v", err)
}
var group sync.WaitGroup
for index := 0; index < cfg.WorkerConcurrency; index++ {
group.Add(1)
go func(workerIndex int) {
defer group.Done()
workerID := fmt.Sprintf("%s-%d", cfg.InstanceID, workerIndex)
for delivery := range deliveries {
message, decodeErr := broker.Decode(delivery)
if decodeErr != nil {
log.Printf("[WARN] Reject invalid broker message: %v", decodeErr)
_ = delivery.Reject(false)
continue
}
now := time.Now().UTC()
job, claimed, claimErr := store.ClaimAttachmentJob(
ctx,
message,
workerID,
now.Add(cfg.JobLease),
)
if errors.Is(claimErr, postgres.ErrNotFound) {
log.Printf("[INFO] Stale broker message for missing job %s acknowledged", message.JobID)
_ = delivery.Ack(false)
continue
}
if claimErr != nil {
log.Printf("[WARN] Claim job %s failed: %v", message.JobID, claimErr)
_ = delivery.Nack(false, true)
continue
}
if !claimed {
_ = delivery.Ack(false)
continue
}
jobContext, cancel := context.WithTimeout(ctx, cfg.JobTimeout)
processErr := processor.Process(jobContext, job)
cancel()
if processErr == nil {
_ = delivery.Ack(false)
continue
}
var permanent *worker.PermanentError
if errors.As(processErr, &permanent) {
if failErr := store.FailAttachmentJob(
ctx,
job.JobID,
permanent.Code,
time.Now().UTC(),
); failErr != nil {
log.Printf("[WARN] Persist permanent failure for job %s failed: %v", job.JobID, failErr)
_ = delivery.Nack(false, true)
} else {
log.Printf("[WARN] Job %s failed permanently with %s", job.JobID, permanent.Code)
_ = delivery.Ack(false)
}
continue
}
retryAt := time.Now().UTC()
delay := worker.RetryDelay(
job.JobID,
job.Attempt+1,
cfg.JobRetryBase,
cfg.JobRetryMax,
)
retried, final, retryErr := store.RetryAttachmentJob(
ctx,
job.JobID,
"processing_failed",
retryAt.Add(delay),
retryAt,
"retry_scheduled",
)
if retryErr != nil {
log.Printf("[WARN] Schedule retry for job %s failed: %v", job.JobID, retryErr)
_ = delivery.Nack(false, true)
continue
}
if final && retried.Status == domain.AttachmentJobFailed {
log.Printf("[WARN] Job %s exhausted retries", job.JobID)
} else if final && retried.Status == domain.AttachmentJobSucceeded {
log.Printf("[INFO] Duplicate delivery for completed job %s acknowledged", job.JobID)
} else if final && retried.Status == domain.AttachmentJobSuperseded {
log.Printf("[INFO] Stale pipeline job %s acknowledged as superseded", job.JobID)
} else {
log.Printf("[INFO] Job %s retry %d scheduled after %s", job.JobID, retried.Attempt, delay)
}
_ = delivery.Ack(false)
}
}(index)
}
log.Printf(
"[INFO] Attachment worker started with concurrency=%d prefetch=%d lease=%s",
cfg.WorkerConcurrency,
cfg.WorkerConcurrency,
cfg.JobLease,
)
<-ctx.Done()
_ = client.Close()
group.Wait()
maintenance.Wait()
log.Println("[INFO] Attachment worker stopped")
}