372 lines
14 KiB
Go
372 lines
14 KiB
Go
// Command pipeline — асинхронная обработка вложений (lab5) с телеметрией (lab6).
|
|
//
|
|
// pipeline worker потребитель broker (сервис worker)
|
|
// pipeline relay outbox → broker, возврат истёкших lease, очистка staging
|
|
// pipeline failed реестр failed-заданий (JSON)
|
|
// pipeline replay <attachment_id> [--version N] [--actor NAME] явный повтор failed
|
|
// pipeline reprocess <attachment_id> --version N работа новой версии (трек 5)
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"log/slog"
|
|
"os"
|
|
"os/signal"
|
|
"strings"
|
|
"sync"
|
|
"syscall"
|
|
"time"
|
|
|
|
"github.com/charawein/mtusi2026lab6/internal/broker"
|
|
"github.com/charawein/mtusi2026lab6/internal/config"
|
|
"github.com/charawein/mtusi2026lab6/internal/imaging"
|
|
"github.com/charawein/mtusi2026lab6/internal/repository/postgres"
|
|
"github.com/charawein/mtusi2026lab6/internal/storage"
|
|
"github.com/charawein/mtusi2026lab6/internal/telemetry"
|
|
"github.com/charawein/mtusi2026lab6/internal/worker"
|
|
"go.opentelemetry.io/otel"
|
|
"go.opentelemetry.io/otel/attribute"
|
|
"go.opentelemetry.io/otel/codes"
|
|
"go.opentelemetry.io/otel/metric"
|
|
"go.opentelemetry.io/otel/trace"
|
|
)
|
|
|
|
func main() {
|
|
slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, nil)))
|
|
if len(os.Args) < 2 {
|
|
fmt.Fprintln(os.Stderr, "usage: pipeline worker|relay|failed|replay|reprocess")
|
|
os.Exit(2)
|
|
}
|
|
cfg := config.Load()
|
|
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
|
|
defer stop()
|
|
|
|
// Телеметрия только у долгоживущих процессов; служебные команды (failed,
|
|
// replay) пишут JSON в stdout. service.name: messenger-worker / messenger-relay
|
|
// (OTEL_SERVICE_NAME из compose имеет приоритет).
|
|
if os.Args[1] == "worker" || os.Args[1] == "relay" {
|
|
host, _ := os.Hostname()
|
|
shutdown, err := telemetry.Setup(ctx, "messenger-"+os.Args[1], host, slog.LevelInfo)
|
|
if err != nil {
|
|
slog.Error("telemetry setup", "err", err)
|
|
os.Exit(1)
|
|
}
|
|
defer func() {
|
|
sctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
_ = shutdown(sctx)
|
|
}()
|
|
}
|
|
|
|
pool, err := postgres.NewPool(ctx, cfg.DatabaseURL, int32(cfg.WorkerConcurrency+4))
|
|
if err != nil {
|
|
slog.Error("connect to postgres", "err", err)
|
|
os.Exit(1)
|
|
}
|
|
defer pool.Close()
|
|
repo := postgres.NewPipelineRepository(pool)
|
|
|
|
switch os.Args[1] {
|
|
case "worker":
|
|
code := runWorker(ctx, cfg, repo)
|
|
if code != 0 {
|
|
os.Exit(code)
|
|
}
|
|
case "relay":
|
|
code := runRelay(ctx, cfg, repo)
|
|
if code != 0 {
|
|
os.Exit(code)
|
|
}
|
|
case "failed":
|
|
jobs, err := repo.FailedJobs(ctx, 200)
|
|
if err != nil {
|
|
slog.Error("failed jobs", "err", err)
|
|
os.Exit(1)
|
|
}
|
|
out, _ := json.MarshalIndent(jobs, "", " ")
|
|
fmt.Println(string(out))
|
|
case "replay", "reprocess":
|
|
fs := flag.NewFlagSet(os.Args[1], flag.ExitOnError)
|
|
version := fs.Int("version", cfg.PipelineVersion, "pipeline version")
|
|
actor := fs.String("actor", "operator", "who requested the replay (audit)")
|
|
if len(os.Args) < 3 {
|
|
fmt.Fprintln(os.Stderr, "attachment_id required")
|
|
os.Exit(2)
|
|
}
|
|
_ = fs.Parse(os.Args[3:])
|
|
var jobID string
|
|
if os.Args[1] == "replay" {
|
|
jobID, err = repo.Replay(ctx, os.Args[2], *version, *actor)
|
|
} else {
|
|
jobID, err = repo.Reprocess(ctx, os.Args[2], *version, cfg.JobMaxAttempts)
|
|
}
|
|
out, _ := json.Marshal(map[string]any{"command": os.Args[1], "attachment_id": os.Args[2], "version": *version,
|
|
"job_id": jobID, "ok": err == nil, "error": errString(err)})
|
|
fmt.Println(string(out))
|
|
if err != nil {
|
|
os.Exit(3)
|
|
}
|
|
default:
|
|
fmt.Fprintln(os.Stderr, "unknown command", os.Args[1])
|
|
os.Exit(2)
|
|
}
|
|
}
|
|
|
|
func errString(err error) string {
|
|
if err == nil {
|
|
return ""
|
|
}
|
|
return err.Error()
|
|
}
|
|
|
|
func faultPoints() map[string]bool {
|
|
m := map[string]bool{}
|
|
for _, p := range strings.Split(os.Getenv("FAULT_POINTS"), ",") {
|
|
if p = strings.TrimSpace(p); p != "" {
|
|
m[p] = true
|
|
}
|
|
}
|
|
return m
|
|
}
|
|
|
|
func newStore(cfg config.Config) storage.Store {
|
|
store, err := storage.NewS3(cfg.S3Endpoint, cfg.S3Region, cfg.S3Bucket, cfg.S3AccessKey, cfg.S3SecretKey)
|
|
if err != nil {
|
|
slog.Error("s3 client", "err", err)
|
|
os.Exit(1)
|
|
}
|
|
return store
|
|
}
|
|
|
|
func runWorker(ctx context.Context, cfg config.Config, repo *postgres.PipelineRepository) int {
|
|
host, _ := os.Hostname()
|
|
owner := fmt.Sprintf("%s:%d", host, os.Getpid())
|
|
w := worker.New(repo, newStore(cfg), imaging.CWebP, worker.Config{
|
|
Owner: owner, Lease: cfg.JobTimeout, JobTimeout: cfg.JobTimeout,
|
|
MaxBytes: cfg.MaxUploadBytes, MaxPixels: cfg.MaxImagePixels, PixelBudget: cfg.PixelBudget,
|
|
WebPQuality: cfg.WebPQuality, RetryBase: cfg.RetryBaseDelay, RetryMax: cfg.RetryMaxDelay,
|
|
FaultPoints: faultPoints(), FaultDir: os.TempDir(), DeleteStaging: true,
|
|
})
|
|
slog.Info("worker starting", "event", "worker.start", "owner", owner, "concurrency", cfg.WorkerConcurrency, "lease", cfg.JobTimeout)
|
|
// Ёмкость worker (saturation = messenger.worker.busy / capacity).
|
|
capacity, _ := telemetry.Meter().Int64ObservableGauge("messenger.worker.capacity", metric.WithUnit("{job}"),
|
|
metric.WithDescription("Concurrent job slots of this worker (prefetch)"))
|
|
_, _ = telemetry.Meter().RegisterCallback(func(_ context.Context, o metric.Observer) error {
|
|
o.ObserveInt64(capacity, int64(cfg.WorkerConcurrency))
|
|
return nil
|
|
}, capacity)
|
|
tracer := telemetry.Tracer()
|
|
backoff := time.Second
|
|
for ctx.Err() == nil {
|
|
cons, deliveries, err := broker.NewConsumer(cfg.BrokerURL, cfg.WorkerConcurrency, "worker-"+owner)
|
|
if err != nil {
|
|
slog.Warn("broker unavailable, retrying", "err", err, "in", backoff)
|
|
select {
|
|
case <-ctx.Done():
|
|
case <-time.After(backoff):
|
|
}
|
|
backoff = min(backoff*2, 30*time.Second)
|
|
continue
|
|
}
|
|
backoff = time.Second
|
|
var wg sync.WaitGroup
|
|
for i := 0; i < cfg.WorkerConcurrency; i++ {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
for d := range deliveries {
|
|
// Consumer span — дочерний для producer span relay (W3C traceparent
|
|
// из заголовков AMQP), тот — для upload: одна трасса от HTTP до S3/DB.
|
|
mctx := otel.GetTextMapPropagator().Extract(context.WithoutCancel(ctx), broker.HeaderCarrier(d.Headers))
|
|
mctx, span := tracer.Start(mctx, broker.Queue+" process", trace.WithSpanKind(trace.SpanKindConsumer),
|
|
trace.WithAttributes(
|
|
attribute.String("messaging.system", "rabbitmq"),
|
|
attribute.String("messaging.operation.type", "process"),
|
|
attribute.String("messaging.destination.name", broker.Queue),
|
|
attribute.String("messaging.message.id", d.MessageId),
|
|
attribute.Bool("messaging.rabbitmq.redelivered", d.Redelivered),
|
|
))
|
|
decision := w.Handle(mctx, d.Body)
|
|
span.SetAttributes(attribute.String("messenger.delivery.decision", decision.String()))
|
|
span.End()
|
|
switch decision {
|
|
case worker.Ack:
|
|
_ = d.Ack(false)
|
|
case worker.Requeue:
|
|
time.Sleep(time.Second)
|
|
_ = d.Nack(false, true)
|
|
case worker.Reject:
|
|
_ = d.Nack(false, false) // → DLX attachment.dead
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
// Graceful: прекращаем получать новые сообщения, дорабатываем текущие.
|
|
slog.Info("shutdown: cancel consumer, finishing in-flight jobs")
|
|
cons.Cancel("worker-" + owner)
|
|
done := make(chan struct{})
|
|
go func() { wg.Wait(); close(done) }()
|
|
select {
|
|
case <-done:
|
|
case <-time.After(cfg.JobTimeout):
|
|
slog.Warn("shutdown timeout: unacked jobs will be redelivered")
|
|
}
|
|
cons.Close()
|
|
return 0
|
|
case <-waitAll(&wg):
|
|
slog.Warn("broker connection lost, reconnecting")
|
|
cons.Close()
|
|
}
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func waitAll(wg *sync.WaitGroup) <-chan struct{} {
|
|
ch := make(chan struct{})
|
|
go func() { wg.Wait(); close(ch) }()
|
|
return ch
|
|
}
|
|
|
|
func runRelay(ctx context.Context, cfg config.Config, repo *postgres.PipelineRepository) int {
|
|
store := newStore(cfg)
|
|
points := faultPoints()
|
|
afterPublish := func() { worker.Fault(points, os.TempDir(), "relay.after_publish", os.Exit) }
|
|
var pub *broker.Publisher
|
|
defer func() {
|
|
if pub != nil {
|
|
pub.Close()
|
|
}
|
|
}()
|
|
tPublish := time.NewTicker(300 * time.Millisecond)
|
|
tSweep := time.NewTicker(5 * time.Second)
|
|
tClean := time.NewTicker(time.Minute)
|
|
defer tPublish.Stop()
|
|
defer tSweep.Stop()
|
|
defer tClean.Stop()
|
|
slog.Info("relay starting", "event", "relay.start")
|
|
registerQueueGauges(repo)
|
|
tracer := telemetry.Tracer()
|
|
published, _ := telemetry.Meter().Int64Counter("messenger.outbox.published", metric.WithUnit("{message}"),
|
|
metric.WithDescription("Outbox messages confirmed by broker"))
|
|
publishDur, _ := telemetry.Meter().Float64Histogram("messenger.outbox.publish.duration", metric.WithUnit("s"),
|
|
metric.WithDescription("Publish + publisher confirm latency"))
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return 0
|
|
case <-tPublish.C:
|
|
if pub == nil || pub.Closed() {
|
|
p, err := broker.NewPublisher(cfg.BrokerURL)
|
|
if err != nil {
|
|
slog.Warn("broker unavailable; outbox keeps messages", "err", err)
|
|
continue
|
|
}
|
|
pub = p
|
|
}
|
|
for {
|
|
// Цикл опроса outbox не трассируется (Unsampled); producer span каждого
|
|
// сообщения — дочерний для span, записавшего его в outbox (upload/попытка).
|
|
n, err := repo.PublishDue(telemetry.Unsampled(ctx), 100, func(c context.Context, m postgres.OutboxMessage) error {
|
|
pctx := telemetry.ExtractMap(context.WithoutCancel(ctx), m.Trace)
|
|
pctx, span := tracer.Start(pctx, broker.Queue+" publish", trace.WithSpanKind(trace.SpanKindProducer),
|
|
trace.WithAttributes(
|
|
attribute.String("messaging.system", "rabbitmq"),
|
|
attribute.String("messaging.operation.type", "publish"),
|
|
attribute.String("messaging.destination.name", broker.Exchange),
|
|
attribute.String("messaging.message.id", m.JobID),
|
|
attribute.Float64("messenger.outbox.wait_seconds", time.Since(m.Created).Seconds()),
|
|
))
|
|
start := time.Now()
|
|
err := pub.Publish(pctx, m.JobID, m.Payload)
|
|
res := "ok"
|
|
if err != nil {
|
|
res = "error"
|
|
span.RecordError(err)
|
|
span.SetStatus(codes.Error, "publish failed")
|
|
}
|
|
publishDur.Record(pctx, time.Since(start).Seconds(), metric.WithAttributes(attribute.String("outcome", res)))
|
|
if err == nil {
|
|
published.Add(pctx, 1)
|
|
}
|
|
span.End()
|
|
return err
|
|
}, afterPublish)
|
|
if err != nil {
|
|
slog.Warn("outbox publish failed; will retry", "err", err)
|
|
break
|
|
}
|
|
if n > 0 {
|
|
slog.Info("outbox published", "messages", n)
|
|
}
|
|
if n < 100 {
|
|
break
|
|
}
|
|
}
|
|
case <-tSweep.C:
|
|
if n, err := repo.SweepExpiredLeases(telemetry.Unsampled(ctx)); err != nil {
|
|
slog.Warn("lease sweep failed", "err", err)
|
|
} else if n > 0 {
|
|
slog.Warn("expired leases requeued", "jobs", n)
|
|
}
|
|
case <-tClean.C:
|
|
cleanStaging(telemetry.Unsampled(ctx), cfg, repo, store)
|
|
}
|
|
}
|
|
}
|
|
|
|
// cleanStaging удаляет staging после безопасной публикации (ready, ≥10 мин назад —
|
|
// fallback, если worker не удалил сразу) и у failed старше STAGING_RETENTION.
|
|
func cleanStaging(ctx context.Context, cfg config.Config, repo *postgres.PipelineRepository, store storage.Store) {
|
|
cands, err := repo.StagingToClean(ctx, 10*time.Minute, cfg.StagingRetention, 200)
|
|
if err != nil {
|
|
slog.Warn("staging cleanup query failed", "err", err)
|
|
return
|
|
}
|
|
for _, c := range cands {
|
|
if err := store.Delete(ctx, c.Key); err != nil && !errors.Is(err, storage.ErrNotFound) {
|
|
slog.Warn("staging delete failed", "key", c.Key, "err", err)
|
|
continue
|
|
}
|
|
_ = repo.ClearStaging(ctx, c.AttachmentID, c.Key)
|
|
}
|
|
if len(cands) > 0 {
|
|
slog.Info("staging cleaned", "objects", len(cands))
|
|
}
|
|
}
|
|
|
|
// registerQueueGauges — глубина очереди и возраст старейшей работы по PostgreSQL
|
|
// (источник истины; broker-метрики RabbitMQ собираются Prometheus отдельно).
|
|
func registerQueueGauges(repo *postgres.PipelineRepository) {
|
|
m := telemetry.Meter()
|
|
jobs, _ := m.Int64ObservableGauge("messenger.jobs.backlog", metric.WithUnit("{job}"),
|
|
metric.WithDescription("Jobs by state (queued, processing, failed)"))
|
|
oldest, _ := m.Float64ObservableGauge("messenger.jobs.oldest_age", metric.WithUnit("s"),
|
|
metric.WithDescription("Age of the oldest unfinished job (queued or processing)"))
|
|
outbox, _ := m.Int64ObservableGauge("messenger.outbox.pending", metric.WithUnit("{message}"),
|
|
metric.WithDescription("Due outbox messages not yet confirmed by broker"))
|
|
outboxAge, _ := m.Float64ObservableGauge("messenger.outbox.oldest_age", metric.WithUnit("s"),
|
|
metric.WithDescription("Age of the oldest due unpublished outbox message"))
|
|
q := attribute.String("queue", broker.Queue)
|
|
_, _ = m.RegisterCallback(func(ctx context.Context, o metric.Observer) error {
|
|
qctx, cancel := context.WithTimeout(telemetry.Unsampled(ctx), 2*time.Second)
|
|
defer cancel()
|
|
st, err := repo.QueueStats(qctx)
|
|
if err != nil {
|
|
return nil // нет данных — пропуск точки, а не ошибка экспорта
|
|
}
|
|
o.ObserveInt64(jobs, st.Queued, metric.WithAttributes(q, attribute.String("state", "queued")))
|
|
o.ObserveInt64(jobs, st.Processing, metric.WithAttributes(q, attribute.String("state", "processing")))
|
|
o.ObserveInt64(jobs, st.Failed, metric.WithAttributes(q, attribute.String("state", "failed")))
|
|
o.ObserveFloat64(oldest, st.OldestQueuedAge, metric.WithAttributes(q))
|
|
o.ObserveInt64(outbox, st.OutboxPending)
|
|
o.ObserveFloat64(outboxAge, st.OldestOutboxAge)
|
|
return nil
|
|
}, jobs, oldest, outbox, outboxAge)
|
|
}
|