forked from mtusi2026/mtusi2026lab6
274 lines
7.9 KiB
Go
274 lines
7.9 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"os"
|
|
"os/signal"
|
|
"sync"
|
|
"syscall"
|
|
"time"
|
|
|
|
"mtusi2026lab6/internal/adapter/broker"
|
|
"mtusi2026lab6/internal/adapter/objectstore"
|
|
"mtusi2026lab6/internal/adapter/postgres"
|
|
"mtusi2026lab6/internal/config"
|
|
"mtusi2026lab6/internal/database"
|
|
"mtusi2026lab6/internal/domain"
|
|
apptelemetry "mtusi2026lab6/internal/telemetry"
|
|
"mtusi2026lab6/internal/worker"
|
|
)
|
|
|
|
const (
|
|
defaultBeforeObjectsFaultFile = "/tmp/mtusi-fault-before-objects"
|
|
defaultAfterObjectsFaultFile = "/tmp/mtusi-fault-after-objects"
|
|
)
|
|
|
|
func main() {
|
|
cfg, err := config.LoadWorker()
|
|
if err != nil {
|
|
log.Fatalf("[FATAL] Invalid configuration: %v", err)
|
|
}
|
|
if cfg.BrokerURL == "" {
|
|
log.Fatal("[FATAL] BROKER_URL or BROKER_URL_FILE is required")
|
|
}
|
|
|
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
|
defer stop()
|
|
telemetryRuntime, err := apptelemetry.Start(
|
|
ctx,
|
|
"messenger-worker",
|
|
cfg.InstanceID,
|
|
"lab",
|
|
)
|
|
if err != nil {
|
|
log.Fatalf("[FATAL] OpenTelemetry initialization failed: %v", err)
|
|
}
|
|
defer func() {
|
|
shutdownContext, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
if shutdownErr := telemetryRuntime.Shutdown(shutdownContext); shutdownErr != nil {
|
|
log.Printf("[WARN] OpenTelemetry shutdown failed: %v", shutdownErr)
|
|
}
|
|
}()
|
|
jobRecorder, err := apptelemetry.NewJobRecorder(telemetryRuntime, cfg.WorkerConcurrency)
|
|
if err != nil {
|
|
log.Fatalf("[FATAL] Worker observability initialization failed: %v", err)
|
|
}
|
|
pool, err := database.Open(ctx, cfg)
|
|
if err != nil {
|
|
log.Fatalf("[FATAL] PostgreSQL initialization failed: %v", err)
|
|
}
|
|
defer pool.Close()
|
|
if err = apptelemetry.RegisterDBPool(telemetryRuntime, pool); err != nil {
|
|
log.Fatalf("[FATAL] DB pool observability initialization failed: %v", err)
|
|
}
|
|
store := postgres.NewPostgreSQLStore(pool, cfg.DatabaseQueryTimeout)
|
|
objects, err := objectstore.NewMinIOStore(
|
|
cfg.S3Endpoint,
|
|
cfg.S3Region,
|
|
cfg.S3Bucket,
|
|
cfg.S3AccessKeyID,
|
|
cfg.S3SecretAccessKey,
|
|
cfg.S3OperationTimeout,
|
|
)
|
|
if err != nil {
|
|
log.Fatalf("[FATAL] Object storage initialization failed: %v", err)
|
|
}
|
|
|
|
beforeFaultFile := os.Getenv("WORKER_FAULT_BEFORE_OBJECTS_FILE")
|
|
if beforeFaultFile == "" {
|
|
beforeFaultFile = defaultBeforeObjectsFaultFile
|
|
}
|
|
afterFaultFile := os.Getenv("WORKER_FAULT_AFTER_OBJECTS_FILE")
|
|
if afterFaultFile == "" {
|
|
afterFaultFile = defaultAfterObjectsFaultFile
|
|
}
|
|
processor, err := worker.NewProcessor(store, objects, worker.ProcessorHooks{
|
|
BeforeObjects: func(job domain.AttachmentJob) error {
|
|
removeErr := os.Remove(beforeFaultFile)
|
|
if errors.Is(removeErr, os.ErrNotExist) {
|
|
return nil
|
|
}
|
|
if removeErr != nil {
|
|
return fmt.Errorf("consume before-objects fault marker: %w", removeErr)
|
|
}
|
|
log.Printf("[FAULT] exiting before object writes for job %s", job.JobID)
|
|
os.Exit(85)
|
|
return nil
|
|
},
|
|
AfterObjects: func(job domain.AttachmentJob) error {
|
|
removeErr := os.Remove(afterFaultFile)
|
|
if errors.Is(removeErr, os.ErrNotExist) {
|
|
return nil
|
|
}
|
|
if removeErr != nil {
|
|
return fmt.Errorf("consume after-objects fault marker: %w", removeErr)
|
|
}
|
|
log.Printf("[FAULT] exiting after object writes and before DB commit for job %s", job.JobID)
|
|
os.Exit(86)
|
|
return nil
|
|
},
|
|
})
|
|
if err != nil {
|
|
log.Fatalf("[FATAL] Worker processor initialization failed: %v", err)
|
|
}
|
|
|
|
cleaner, err := worker.NewStagingCleaner(store, objects, cfg.StagingRetention)
|
|
if err != nil {
|
|
log.Fatalf("[FATAL] Staging cleaner initialization failed: %v", err)
|
|
}
|
|
var maintenance sync.WaitGroup
|
|
maintenance.Add(1)
|
|
go func() {
|
|
defer maintenance.Done()
|
|
runCleanup := func() {
|
|
cleanupContext, cancel := context.WithTimeout(ctx, cfg.S3OperationTimeout+cfg.DatabaseQueryTimeout)
|
|
deleted, cleanupErr := cleaner.RunOnce(cleanupContext, time.Now().UTC())
|
|
cancel()
|
|
if cleanupErr != nil && ctx.Err() == nil {
|
|
log.Printf("[WARN] Staging cleanup failed: %v", cleanupErr)
|
|
} else if deleted > 0 {
|
|
log.Printf("[INFO] Staging cleanup removed %d object(s)", deleted)
|
|
}
|
|
}
|
|
runCleanup()
|
|
ticker := time.NewTicker(cfg.StagingCleanupInterval)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
runCleanup()
|
|
}
|
|
}
|
|
}()
|
|
|
|
client, err := broker.Open(cfg.BrokerURL, cfg.WorkerConcurrency)
|
|
if err != nil {
|
|
log.Fatalf("[FATAL] RabbitMQ initialization failed: %v", err)
|
|
}
|
|
defer client.Close()
|
|
deliveries, err := client.Consume(cfg.InstanceID)
|
|
if err != nil {
|
|
log.Fatalf("[FATAL] RabbitMQ consume failed: %v", err)
|
|
}
|
|
|
|
var group sync.WaitGroup
|
|
for index := 0; index < cfg.WorkerConcurrency; index++ {
|
|
group.Add(1)
|
|
go func(workerIndex int) {
|
|
defer group.Done()
|
|
workerID := fmt.Sprintf("%s-%d", cfg.InstanceID, workerIndex)
|
|
for delivery := range deliveries {
|
|
deliveryContext := broker.ContextFromDelivery(ctx, delivery)
|
|
message, decodeErr := broker.Decode(delivery)
|
|
if decodeErr != nil {
|
|
log.Printf("[WARN] Reject invalid broker message: %v", decodeErr)
|
|
_ = delivery.Reject(false)
|
|
continue
|
|
}
|
|
now := time.Now().UTC()
|
|
job, claimed, claimErr := store.ClaimAttachmentJob(
|
|
deliveryContext,
|
|
message,
|
|
workerID,
|
|
now.Add(cfg.JobLease),
|
|
)
|
|
if errors.Is(claimErr, postgres.ErrNotFound) {
|
|
log.Printf("[INFO] Stale broker message for missing job %s acknowledged", message.JobID)
|
|
_ = delivery.Ack(false)
|
|
continue
|
|
}
|
|
if claimErr != nil {
|
|
log.Printf("[WARN] Claim job %s failed: %v", message.JobID, claimErr)
|
|
_ = delivery.Nack(false, true)
|
|
continue
|
|
}
|
|
if !claimed {
|
|
_ = delivery.Ack(false)
|
|
continue
|
|
}
|
|
|
|
jobContext, cancel := context.WithTimeout(deliveryContext, cfg.JobTimeout)
|
|
jobContext, observedAttempt := jobRecorder.Start(
|
|
jobContext,
|
|
domain.AttachmentJobType,
|
|
broker.QueueName,
|
|
job.Attempt,
|
|
job.CreatedAt,
|
|
)
|
|
processErr := processor.Process(jobContext, job)
|
|
observedAttempt.End(processErr)
|
|
cancel()
|
|
if processErr == nil {
|
|
_ = delivery.Ack(false)
|
|
continue
|
|
}
|
|
|
|
var permanent *worker.PermanentError
|
|
if errors.As(processErr, &permanent) {
|
|
if failErr := store.FailAttachmentJob(
|
|
ctx,
|
|
job.JobID,
|
|
permanent.Code,
|
|
time.Now().UTC(),
|
|
); failErr != nil {
|
|
log.Printf("[WARN] Persist permanent failure for job %s failed: %v", job.JobID, failErr)
|
|
_ = delivery.Nack(false, true)
|
|
} else {
|
|
log.Printf("[WARN] Job %s failed permanently with %s", job.JobID, permanent.Code)
|
|
_ = delivery.Ack(false)
|
|
}
|
|
continue
|
|
}
|
|
|
|
retryAt := time.Now().UTC()
|
|
delay := worker.RetryDelay(
|
|
job.JobID,
|
|
job.Attempt+1,
|
|
cfg.JobRetryBase,
|
|
cfg.JobRetryMax,
|
|
)
|
|
retried, final, retryErr := store.RetryAttachmentJob(
|
|
ctx,
|
|
job.JobID,
|
|
"processing_failed",
|
|
retryAt.Add(delay),
|
|
retryAt,
|
|
"retry_scheduled",
|
|
)
|
|
if retryErr != nil {
|
|
log.Printf("[WARN] Schedule retry for job %s failed: %v", job.JobID, retryErr)
|
|
_ = delivery.Nack(false, true)
|
|
continue
|
|
}
|
|
if final && retried.Status == domain.AttachmentJobFailed {
|
|
log.Printf("[WARN] Job %s exhausted retries", job.JobID)
|
|
} else if final && retried.Status == domain.AttachmentJobSucceeded {
|
|
log.Printf("[INFO] Duplicate delivery for completed job %s acknowledged", job.JobID)
|
|
} else if final && retried.Status == domain.AttachmentJobSuperseded {
|
|
log.Printf("[INFO] Stale pipeline job %s acknowledged as superseded", job.JobID)
|
|
} else {
|
|
log.Printf("[INFO] Job %s retry %d scheduled after %s", job.JobID, retried.Attempt, delay)
|
|
}
|
|
_ = delivery.Ack(false)
|
|
}
|
|
}(index)
|
|
}
|
|
log.Printf(
|
|
"[INFO] Attachment worker started with concurrency=%d prefetch=%d lease=%s",
|
|
cfg.WorkerConcurrency,
|
|
cfg.WorkerConcurrency,
|
|
cfg.JobLease,
|
|
)
|
|
<-ctx.Done()
|
|
_ = client.Close()
|
|
group.Wait()
|
|
maintenance.Wait()
|
|
log.Println("[INFO] Attachment worker stopped")
|
|
}
|