Files
mtusi2026lab6/src/backend/workers/pipeline.go
T

221 lines
6.6 KiB
Go

package workers
import (
"context"
"encoding/json"
"errors"
"log"
"os"
"os/signal"
"strings"
"syscall"
"time"
"go.opentelemetry.io/otel/trace"
"messenger/src/backend/internal/faults"
"messenger/src/backend/repositories"
"messenger/src/backend/services"
"messenger/src/backend/telemetry"
)
func RunPipeline(mode string, args []string) error {
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
db, err := repositories.OpenDatabase(ctx)
if err != nil {
return err
}
defer db.Close()
store := repositories.NewAttachmentStore(db)
data, err := os.ReadFile(os.Getenv("BROKER_URL_FILE"))
if err != nil {
return errors.New("cannot read broker URL")
}
address := strings.TrimSpace(string(data))
if mode == "broker-publish" {
var job repositories.Job
if len(args) != 1 || json.Unmarshal([]byte(args[0]), &job) != nil || job.SchemaVersion != 1 || job.Type != "attachment.process.v1" {
return errors.New("valid internal job JSON required")
}
broker, e := repositories.NewJobBroker(ctx, address)
if e != nil {
return e
}
defer broker.Close()
publishCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
return broker.Publish(publishCtx, []byte(args[0]))
}
maxAttempts, err := positiveEnv("JOB_MAX_ATTEMPTS", 5)
if err != nil {
return err
}
seconds, err := positiveEnv("JOB_TIMEOUT_SECONDS", 60)
if err != nil {
return err
}
timeout := time.Duration(seconds) * time.Second
objects, err := repositories.NewObjectStoreFromEnvironment()
if err != nil {
return err
}
var signals *telemetry.Signals
if mode == "worker" || mode == "dispatcher" {
service := "messenger-" + mode
var stopTelemetry func()
signals, stopTelemetry, err = telemetry.Start(service)
if err != nil {
return err
}
defer stopTelemetry()
if err := signals.ObservePool(db); err != nil {
return err
}
if mode == "dispatcher" {
if err := signals.ObserveBacklog(store.PendingJobs); err != nil {
return err
}
}
}
for ctx.Err() == nil {
broker, e := repositories.NewJobBroker(ctx, address)
if e == nil {
if mode == "dispatcher" {
e = dispatchJobs(ctx, store, broker, signals)
} else {
e = consumeJobs(ctx, store, objects, broker, maxAttempts, timeout, signals)
}
broker.Close()
}
if e != nil && ctx.Err() == nil {
log.Print("pipeline: broker session interrupted; reconnecting")
}
select {
case <-ctx.Done():
case <-time.After(time.Second):
}
}
return nil
}
func dispatchJobs(ctx context.Context, store *repositories.AttachmentStore, broker *repositories.JobBroker, signals *telemetry.Signals) error {
for ctx.Err() == nil {
jobs, err := store.QueuedJobs(ctx)
if err != nil {
return err
}
for _, job := range jobs {
faults.Crash("after_commit_before_publish", job.AttachmentID)
payload, err := json.Marshal(job)
if err != nil {
return err
}
publishCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
publishCtx = telemetry.WithTraceHeaders(publishCtx, map[string]string{
"traceparent": job.Traceparent, "tracestate": job.Tracestate,
})
err = signals.Publish(publishCtx, func(ctx context.Context) error {
return broker.Publish(ctx, payload)
})
cancel()
if err != nil {
return err
}
// Only confirmed publications are marked. Uncertainty causes safe duplicates.
if err = store.MarkPublished(ctx, job.ID); err != nil {
return err
}
}
select {
case <-ctx.Done():
case <-time.After(time.Second):
}
}
return nil
}
func consumeJobs(ctx context.Context, store *repositories.AttachmentStore, objects services.AttachmentObjects, broker *repositories.JobBroker, maxAttempts int, timeout time.Duration, signals *telemetry.Signals) error {
deliveries, err := broker.Consume()
if err != nil {
return errors.New("RabbitMQ consumer unavailable")
}
for {
select {
case <-ctx.Done():
return nil
case message, ok := <-deliveries:
if !ok {
return errors.New("RabbitMQ consumer disconnected")
}
var job repositories.Job
if json.Unmarshal(message.Body, &job) != nil || job.SchemaVersion != 1 || job.Type != "attachment.process.v1" {
log.Print("worker: invalid internal job rejected")
if err = message.Reject(false); err != nil {
return err
}
continue
}
task, cancel := context.WithTimeout(ctx, timeout+15*time.Second)
parent, _ := message.Headers["traceparent"].(string)
state, _ := message.Headers["tracestate"].(string)
task = telemetry.WithTraceHeaders(task, map[string]string{"traceparent": parent, "tracestate": state})
task, finish := signals.Job(task, job.Type, job.AttachmentID, job.Attempt, job.CreatedAt)
var imageErr error
token, workErr := store.StartAttempt(task, job, timeout+15*time.Second, maxAttempts)
if workErr == nil && token != "" {
faults.Crash("before_effect", job.AttachmentID)
faults.Pause(task, "after_claim", job.AttachmentID)
workErr = store.ProcessJob(task, job, token, maxAttempts, func(a repositories.Attachment, staging string) ([]repositories.Variant, string, error) {
imageCtx, finishImage := telemetry.Operation(task, "s3 read staging and write variants", trace.SpanKindClient)
variants, code, err := processImage(imageCtx, objects, a, staging)
finishImage(err)
imageErr = err
return variants, code, err
})
}
cancel()
if errors.Is(workErr, repositories.ErrStaleJob) {
faults.Observe("stale_rejected", job.AttachmentID)
}
completed := token != "" && workErr == nil
if workErr == nil && token == "" || errors.Is(workErr, repositories.ErrStaleJob) {
completed, workErr = store.JobTerminal(ctx, job.ID)
}
outcome := "success"
switch {
case workErr != nil || imageErr != nil:
outcome = "error"
case token == "" || !completed:
outcome = "skipped"
}
finish(outcome)
// Ack after COMMIT of ready/failed or durable retry intent, or terminal no-op.
// A closed connection automatically returns unacked deliveries to RabbitMQ.
if completed {
faults.Crash("after_commit_before_ack", job.AttachmentID)
if objectStore, ok := objects.(*repositories.ObjectStore); ok {
cleanupCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
_, cleanupErr := store.CleanupJobStaging(cleanupCtx, job.ID, 24*time.Hour, false, objectStore.Delete)
cancel()
if cleanupErr != nil {
log.Print("worker: staging cleanup deferred")
}
}
if err = message.Ack(false); err != nil {
return err
}
} else {
if err = message.Nack(false, true); err != nil {
return err
}
select {
case <-ctx.Done():
case <-time.After(time.Second):
}
}
}
}
}