Files
mtusi2026lab6/cmd/pipeline/main.go
T

372 lines
14 KiB
Go

// Command pipeline — асинхронная обработка вложений (lab5) с телеметрией (lab6).
//
// pipeline worker потребитель broker (сервис worker)
// pipeline relay outbox → broker, возврат истёкших lease, очистка staging
// pipeline failed реестр failed-заданий (JSON)
// pipeline replay <attachment_id> [--version N] [--actor NAME] явный повтор failed
// pipeline reprocess <attachment_id> --version N работа новой версии (трек 5)
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"log/slog"
"os"
"os/signal"
"strings"
"sync"
"syscall"
"time"
"github.com/charawein/mtusi2026lab6/internal/broker"
"github.com/charawein/mtusi2026lab6/internal/config"
"github.com/charawein/mtusi2026lab6/internal/imaging"
"github.com/charawein/mtusi2026lab6/internal/repository/postgres"
"github.com/charawein/mtusi2026lab6/internal/storage"
"github.com/charawein/mtusi2026lab6/internal/telemetry"
"github.com/charawein/mtusi2026lab6/internal/worker"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/metric"
"go.opentelemetry.io/otel/trace"
)
func main() {
slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, nil)))
if len(os.Args) < 2 {
fmt.Fprintln(os.Stderr, "usage: pipeline worker|relay|failed|replay|reprocess")
os.Exit(2)
}
cfg := config.Load()
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
defer stop()
// Телеметрия только у долгоживущих процессов; служебные команды (failed,
// replay) пишут JSON в stdout. service.name: messenger-worker / messenger-relay
// (OTEL_SERVICE_NAME из compose имеет приоритет).
if os.Args[1] == "worker" || os.Args[1] == "relay" {
host, _ := os.Hostname()
shutdown, err := telemetry.Setup(ctx, "messenger-"+os.Args[1], host, slog.LevelInfo)
if err != nil {
slog.Error("telemetry setup", "err", err)
os.Exit(1)
}
defer func() {
sctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_ = shutdown(sctx)
}()
}
pool, err := postgres.NewPool(ctx, cfg.DatabaseURL, int32(cfg.WorkerConcurrency+4))
if err != nil {
slog.Error("connect to postgres", "err", err)
os.Exit(1)
}
defer pool.Close()
repo := postgres.NewPipelineRepository(pool)
switch os.Args[1] {
case "worker":
code := runWorker(ctx, cfg, repo)
if code != 0 {
os.Exit(code)
}
case "relay":
code := runRelay(ctx, cfg, repo)
if code != 0 {
os.Exit(code)
}
case "failed":
jobs, err := repo.FailedJobs(ctx, 200)
if err != nil {
slog.Error("failed jobs", "err", err)
os.Exit(1)
}
out, _ := json.MarshalIndent(jobs, "", " ")
fmt.Println(string(out))
case "replay", "reprocess":
fs := flag.NewFlagSet(os.Args[1], flag.ExitOnError)
version := fs.Int("version", cfg.PipelineVersion, "pipeline version")
actor := fs.String("actor", "operator", "who requested the replay (audit)")
if len(os.Args) < 3 {
fmt.Fprintln(os.Stderr, "attachment_id required")
os.Exit(2)
}
_ = fs.Parse(os.Args[3:])
var jobID string
if os.Args[1] == "replay" {
jobID, err = repo.Replay(ctx, os.Args[2], *version, *actor)
} else {
jobID, err = repo.Reprocess(ctx, os.Args[2], *version, cfg.JobMaxAttempts)
}
out, _ := json.Marshal(map[string]any{"command": os.Args[1], "attachment_id": os.Args[2], "version": *version,
"job_id": jobID, "ok": err == nil, "error": errString(err)})
fmt.Println(string(out))
if err != nil {
os.Exit(3)
}
default:
fmt.Fprintln(os.Stderr, "unknown command", os.Args[1])
os.Exit(2)
}
}
func errString(err error) string {
if err == nil {
return ""
}
return err.Error()
}
func faultPoints() map[string]bool {
m := map[string]bool{}
for _, p := range strings.Split(os.Getenv("FAULT_POINTS"), ",") {
if p = strings.TrimSpace(p); p != "" {
m[p] = true
}
}
return m
}
func newStore(cfg config.Config) storage.Store {
store, err := storage.NewS3(cfg.S3Endpoint, cfg.S3Region, cfg.S3Bucket, cfg.S3AccessKey, cfg.S3SecretKey)
if err != nil {
slog.Error("s3 client", "err", err)
os.Exit(1)
}
return store
}
func runWorker(ctx context.Context, cfg config.Config, repo *postgres.PipelineRepository) int {
host, _ := os.Hostname()
owner := fmt.Sprintf("%s:%d", host, os.Getpid())
w := worker.New(repo, newStore(cfg), imaging.CWebP, worker.Config{
Owner: owner, Lease: cfg.JobTimeout, JobTimeout: cfg.JobTimeout,
MaxBytes: cfg.MaxUploadBytes, MaxPixels: cfg.MaxImagePixels, PixelBudget: cfg.PixelBudget,
WebPQuality: cfg.WebPQuality, RetryBase: cfg.RetryBaseDelay, RetryMax: cfg.RetryMaxDelay,
FaultPoints: faultPoints(), FaultDir: os.TempDir(), DeleteStaging: true,
})
slog.Info("worker starting", "event", "worker.start", "owner", owner, "concurrency", cfg.WorkerConcurrency, "lease", cfg.JobTimeout)
// Ёмкость worker (saturation = messenger.worker.busy / capacity).
capacity, _ := telemetry.Meter().Int64ObservableGauge("messenger.worker.capacity", metric.WithUnit("{job}"),
metric.WithDescription("Concurrent job slots of this worker (prefetch)"))
_, _ = telemetry.Meter().RegisterCallback(func(_ context.Context, o metric.Observer) error {
o.ObserveInt64(capacity, int64(cfg.WorkerConcurrency))
return nil
}, capacity)
tracer := telemetry.Tracer()
backoff := time.Second
for ctx.Err() == nil {
cons, deliveries, err := broker.NewConsumer(cfg.BrokerURL, cfg.WorkerConcurrency, "worker-"+owner)
if err != nil {
slog.Warn("broker unavailable, retrying", "err", err, "in", backoff)
select {
case <-ctx.Done():
case <-time.After(backoff):
}
backoff = min(backoff*2, 30*time.Second)
continue
}
backoff = time.Second
var wg sync.WaitGroup
for i := 0; i < cfg.WorkerConcurrency; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for d := range deliveries {
// Consumer span — дочерний для producer span relay (W3C traceparent
// из заголовков AMQP), тот — для upload: одна трасса от HTTP до S3/DB.
mctx := otel.GetTextMapPropagator().Extract(context.WithoutCancel(ctx), broker.HeaderCarrier(d.Headers))
mctx, span := tracer.Start(mctx, broker.Queue+" process", trace.WithSpanKind(trace.SpanKindConsumer),
trace.WithAttributes(
attribute.String("messaging.system", "rabbitmq"),
attribute.String("messaging.operation.type", "process"),
attribute.String("messaging.destination.name", broker.Queue),
attribute.String("messaging.message.id", d.MessageId),
attribute.Bool("messaging.rabbitmq.redelivered", d.Redelivered),
))
decision := w.Handle(mctx, d.Body)
span.SetAttributes(attribute.String("messenger.delivery.decision", decision.String()))
span.End()
switch decision {
case worker.Ack:
_ = d.Ack(false)
case worker.Requeue:
time.Sleep(time.Second)
_ = d.Nack(false, true)
case worker.Reject:
_ = d.Nack(false, false) // → DLX attachment.dead
}
}
}()
}
select {
case <-ctx.Done():
// Graceful: прекращаем получать новые сообщения, дорабатываем текущие.
slog.Info("shutdown: cancel consumer, finishing in-flight jobs")
cons.Cancel("worker-" + owner)
done := make(chan struct{})
go func() { wg.Wait(); close(done) }()
select {
case <-done:
case <-time.After(cfg.JobTimeout):
slog.Warn("shutdown timeout: unacked jobs will be redelivered")
}
cons.Close()
return 0
case <-waitAll(&wg):
slog.Warn("broker connection lost, reconnecting")
cons.Close()
}
}
return 0
}
func waitAll(wg *sync.WaitGroup) <-chan struct{} {
ch := make(chan struct{})
go func() { wg.Wait(); close(ch) }()
return ch
}
func runRelay(ctx context.Context, cfg config.Config, repo *postgres.PipelineRepository) int {
store := newStore(cfg)
points := faultPoints()
afterPublish := func() { worker.Fault(points, os.TempDir(), "relay.after_publish", os.Exit) }
var pub *broker.Publisher
defer func() {
if pub != nil {
pub.Close()
}
}()
tPublish := time.NewTicker(300 * time.Millisecond)
tSweep := time.NewTicker(5 * time.Second)
tClean := time.NewTicker(time.Minute)
defer tPublish.Stop()
defer tSweep.Stop()
defer tClean.Stop()
slog.Info("relay starting", "event", "relay.start")
registerQueueGauges(repo)
tracer := telemetry.Tracer()
published, _ := telemetry.Meter().Int64Counter("messenger.outbox.published", metric.WithUnit("{message}"),
metric.WithDescription("Outbox messages confirmed by broker"))
publishDur, _ := telemetry.Meter().Float64Histogram("messenger.outbox.publish.duration", metric.WithUnit("s"),
metric.WithDescription("Publish + publisher confirm latency"))
for {
select {
case <-ctx.Done():
return 0
case <-tPublish.C:
if pub == nil || pub.Closed() {
p, err := broker.NewPublisher(cfg.BrokerURL)
if err != nil {
slog.Warn("broker unavailable; outbox keeps messages", "err", err)
continue
}
pub = p
}
for {
// Цикл опроса outbox не трассируется (Unsampled); producer span каждого
// сообщения — дочерний для span, записавшего его в outbox (upload/попытка).
n, err := repo.PublishDue(telemetry.Unsampled(ctx), 100, func(c context.Context, m postgres.OutboxMessage) error {
pctx := telemetry.ExtractMap(context.WithoutCancel(ctx), m.Trace)
pctx, span := tracer.Start(pctx, broker.Queue+" publish", trace.WithSpanKind(trace.SpanKindProducer),
trace.WithAttributes(
attribute.String("messaging.system", "rabbitmq"),
attribute.String("messaging.operation.type", "publish"),
attribute.String("messaging.destination.name", broker.Exchange),
attribute.String("messaging.message.id", m.JobID),
attribute.Float64("messenger.outbox.wait_seconds", time.Since(m.Created).Seconds()),
))
start := time.Now()
err := pub.Publish(pctx, m.JobID, m.Payload)
res := "ok"
if err != nil {
res = "error"
span.RecordError(err)
span.SetStatus(codes.Error, "publish failed")
}
publishDur.Record(pctx, time.Since(start).Seconds(), metric.WithAttributes(attribute.String("outcome", res)))
if err == nil {
published.Add(pctx, 1)
}
span.End()
return err
}, afterPublish)
if err != nil {
slog.Warn("outbox publish failed; will retry", "err", err)
break
}
if n > 0 {
slog.Info("outbox published", "messages", n)
}
if n < 100 {
break
}
}
case <-tSweep.C:
if n, err := repo.SweepExpiredLeases(telemetry.Unsampled(ctx)); err != nil {
slog.Warn("lease sweep failed", "err", err)
} else if n > 0 {
slog.Warn("expired leases requeued", "jobs", n)
}
case <-tClean.C:
cleanStaging(telemetry.Unsampled(ctx), cfg, repo, store)
}
}
}
// cleanStaging удаляет staging после безопасной публикации (ready, ≥10 мин назад —
// fallback, если worker не удалил сразу) и у failed старше STAGING_RETENTION.
func cleanStaging(ctx context.Context, cfg config.Config, repo *postgres.PipelineRepository, store storage.Store) {
cands, err := repo.StagingToClean(ctx, 10*time.Minute, cfg.StagingRetention, 200)
if err != nil {
slog.Warn("staging cleanup query failed", "err", err)
return
}
for _, c := range cands {
if err := store.Delete(ctx, c.Key); err != nil && !errors.Is(err, storage.ErrNotFound) {
slog.Warn("staging delete failed", "key", c.Key, "err", err)
continue
}
_ = repo.ClearStaging(ctx, c.AttachmentID, c.Key)
}
if len(cands) > 0 {
slog.Info("staging cleaned", "objects", len(cands))
}
}
// registerQueueGauges — глубина очереди и возраст старейшей работы по PostgreSQL
// (источник истины; broker-метрики RabbitMQ собираются Prometheus отдельно).
func registerQueueGauges(repo *postgres.PipelineRepository) {
m := telemetry.Meter()
jobs, _ := m.Int64ObservableGauge("messenger.jobs.backlog", metric.WithUnit("{job}"),
metric.WithDescription("Jobs by state (queued, processing, failed)"))
oldest, _ := m.Float64ObservableGauge("messenger.jobs.oldest_age", metric.WithUnit("s"),
metric.WithDescription("Age of the oldest unfinished job (queued or processing)"))
outbox, _ := m.Int64ObservableGauge("messenger.outbox.pending", metric.WithUnit("{message}"),
metric.WithDescription("Due outbox messages not yet confirmed by broker"))
outboxAge, _ := m.Float64ObservableGauge("messenger.outbox.oldest_age", metric.WithUnit("s"),
metric.WithDescription("Age of the oldest due unpublished outbox message"))
q := attribute.String("queue", broker.Queue)
_, _ = m.RegisterCallback(func(ctx context.Context, o metric.Observer) error {
qctx, cancel := context.WithTimeout(telemetry.Unsampled(ctx), 2*time.Second)
defer cancel()
st, err := repo.QueueStats(qctx)
if err != nil {
return nil // нет данных — пропуск точки, а не ошибка экспорта
}
o.ObserveInt64(jobs, st.Queued, metric.WithAttributes(q, attribute.String("state", "queued")))
o.ObserveInt64(jobs, st.Processing, metric.WithAttributes(q, attribute.String("state", "processing")))
o.ObserveInt64(jobs, st.Failed, metric.WithAttributes(q, attribute.String("state", "failed")))
o.ObserveFloat64(oldest, st.OldestQueuedAge, metric.WithAttributes(q))
o.ObserveInt64(outbox, st.OutboxPending)
o.ObserveFloat64(outboxAge, st.OldestOutboxAge)
return nil
}, jobs, oldest, outbox, outboxAge)
}