221 lines
6.6 KiB
Go
221 lines
6.6 KiB
Go
package workers
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"log"
|
|
"os"
|
|
"os/signal"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
|
|
"go.opentelemetry.io/otel/trace"
|
|
|
|
"messenger/src/backend/internal/faults"
|
|
"messenger/src/backend/repositories"
|
|
"messenger/src/backend/services"
|
|
"messenger/src/backend/telemetry"
|
|
)
|
|
|
|
func RunPipeline(mode string, args []string) error {
|
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
|
defer stop()
|
|
db, err := repositories.OpenDatabase(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer db.Close()
|
|
store := repositories.NewAttachmentStore(db)
|
|
data, err := os.ReadFile(os.Getenv("BROKER_URL_FILE"))
|
|
if err != nil {
|
|
return errors.New("cannot read broker URL")
|
|
}
|
|
|
|
address := strings.TrimSpace(string(data))
|
|
if mode == "broker-publish" {
|
|
var job repositories.Job
|
|
if len(args) != 1 || json.Unmarshal([]byte(args[0]), &job) != nil || job.SchemaVersion != 1 || job.Type != "attachment.process.v1" {
|
|
return errors.New("valid internal job JSON required")
|
|
}
|
|
broker, e := repositories.NewJobBroker(ctx, address)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
defer broker.Close()
|
|
publishCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
|
defer cancel()
|
|
return broker.Publish(publishCtx, []byte(args[0]))
|
|
}
|
|
maxAttempts, err := positiveEnv("JOB_MAX_ATTEMPTS", 5)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
seconds, err := positiveEnv("JOB_TIMEOUT_SECONDS", 60)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
timeout := time.Duration(seconds) * time.Second
|
|
objects, err := repositories.NewObjectStoreFromEnvironment()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var signals *telemetry.Signals
|
|
if mode == "worker" || mode == "dispatcher" {
|
|
service := "messenger-" + mode
|
|
var stopTelemetry func()
|
|
signals, stopTelemetry, err = telemetry.Start(service)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer stopTelemetry()
|
|
if err := signals.ObservePool(db); err != nil {
|
|
return err
|
|
}
|
|
if mode == "dispatcher" {
|
|
if err := signals.ObserveBacklog(store.PendingJobs); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
for ctx.Err() == nil {
|
|
broker, e := repositories.NewJobBroker(ctx, address)
|
|
if e == nil {
|
|
if mode == "dispatcher" {
|
|
e = dispatchJobs(ctx, store, broker, signals)
|
|
} else {
|
|
e = consumeJobs(ctx, store, objects, broker, maxAttempts, timeout, signals)
|
|
}
|
|
broker.Close()
|
|
}
|
|
if e != nil && ctx.Err() == nil {
|
|
log.Print("pipeline: broker session interrupted; reconnecting")
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
case <-time.After(time.Second):
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func dispatchJobs(ctx context.Context, store *repositories.AttachmentStore, broker *repositories.JobBroker, signals *telemetry.Signals) error {
|
|
for ctx.Err() == nil {
|
|
jobs, err := store.QueuedJobs(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, job := range jobs {
|
|
faults.Crash("after_commit_before_publish", job.AttachmentID)
|
|
payload, err := json.Marshal(job)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
publishCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
|
publishCtx = telemetry.WithTraceHeaders(publishCtx, map[string]string{
|
|
"traceparent": job.Traceparent, "tracestate": job.Tracestate,
|
|
})
|
|
err = signals.Publish(publishCtx, func(ctx context.Context) error {
|
|
return broker.Publish(ctx, payload)
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// Only confirmed publications are marked. Uncertainty causes safe duplicates.
|
|
if err = store.MarkPublished(ctx, job.ID); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
case <-time.After(time.Second):
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func consumeJobs(ctx context.Context, store *repositories.AttachmentStore, objects services.AttachmentObjects, broker *repositories.JobBroker, maxAttempts int, timeout time.Duration, signals *telemetry.Signals) error {
|
|
deliveries, err := broker.Consume()
|
|
if err != nil {
|
|
return errors.New("RabbitMQ consumer unavailable")
|
|
}
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return nil
|
|
case message, ok := <-deliveries:
|
|
if !ok {
|
|
return errors.New("RabbitMQ consumer disconnected")
|
|
}
|
|
var job repositories.Job
|
|
if json.Unmarshal(message.Body, &job) != nil || job.SchemaVersion != 1 || job.Type != "attachment.process.v1" {
|
|
log.Print("worker: invalid internal job rejected")
|
|
if err = message.Reject(false); err != nil {
|
|
return err
|
|
}
|
|
continue
|
|
}
|
|
task, cancel := context.WithTimeout(ctx, timeout+15*time.Second)
|
|
parent, _ := message.Headers["traceparent"].(string)
|
|
state, _ := message.Headers["tracestate"].(string)
|
|
task = telemetry.WithTraceHeaders(task, map[string]string{"traceparent": parent, "tracestate": state})
|
|
task, finish := signals.Job(task, job.Type, job.AttachmentID, job.Attempt, job.CreatedAt)
|
|
var imageErr error
|
|
token, workErr := store.StartAttempt(task, job, timeout+15*time.Second, maxAttempts)
|
|
if workErr == nil && token != "" {
|
|
faults.Crash("before_effect", job.AttachmentID)
|
|
faults.Pause(task, "after_claim", job.AttachmentID)
|
|
workErr = store.ProcessJob(task, job, token, maxAttempts, func(a repositories.Attachment, staging string) ([]repositories.Variant, string, error) {
|
|
imageCtx, finishImage := telemetry.Operation(task, "s3 read staging and write variants", trace.SpanKindClient)
|
|
variants, code, err := processImage(imageCtx, objects, a, staging)
|
|
finishImage(err)
|
|
imageErr = err
|
|
return variants, code, err
|
|
})
|
|
}
|
|
cancel()
|
|
if errors.Is(workErr, repositories.ErrStaleJob) {
|
|
faults.Observe("stale_rejected", job.AttachmentID)
|
|
}
|
|
completed := token != "" && workErr == nil
|
|
if workErr == nil && token == "" || errors.Is(workErr, repositories.ErrStaleJob) {
|
|
completed, workErr = store.JobTerminal(ctx, job.ID)
|
|
}
|
|
outcome := "success"
|
|
switch {
|
|
case workErr != nil || imageErr != nil:
|
|
outcome = "error"
|
|
case token == "" || !completed:
|
|
outcome = "skipped"
|
|
}
|
|
finish(outcome)
|
|
// Ack after COMMIT of ready/failed or durable retry intent, or terminal no-op.
|
|
// A closed connection automatically returns unacked deliveries to RabbitMQ.
|
|
if completed {
|
|
faults.Crash("after_commit_before_ack", job.AttachmentID)
|
|
if objectStore, ok := objects.(*repositories.ObjectStore); ok {
|
|
cleanupCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
|
_, cleanupErr := store.CleanupJobStaging(cleanupCtx, job.ID, 24*time.Hour, false, objectStore.Delete)
|
|
cancel()
|
|
if cleanupErr != nil {
|
|
log.Print("worker: staging cleanup deferred")
|
|
}
|
|
}
|
|
if err = message.Ack(false); err != nil {
|
|
return err
|
|
}
|
|
} else {
|
|
if err = message.Nack(false, true); err != nil {
|
|
return err
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
case <-time.After(time.Second):
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|