Files
mtusi2026lab7/internal/worker/trace_integration_test.go
T

137 lines
5.4 KiB
Go

package worker_test
// Перенос W3C trace context через outbox (lab6) на настоящем PostgreSQL:
// upload span → outbox.trace_context → (relay) → consumer span → retry-сообщение
// несёт контекст попытки; сообщение от sweeper (без span) — контекст исходного
// upload из jobs.trace_context. Плюс метрики worker по исходам.
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"strings"
"testing"
"time"
"github.com/google/uuid"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/propagation"
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
"go.opentelemetry.io/otel/sdk/metric/metricdata"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/trace"
"github.com/charawein/mtusi2026lab7/internal/domain"
"github.com/charawein/mtusi2026lab7/internal/repository/postgres"
"github.com/charawein/mtusi2026lab7/internal/telemetry"
)
func traceparent(m map[string]string) (traceID, spanID string) {
parts := strings.Split(m["traceparent"], "-")
if len(parts) != 4 {
return "", ""
}
return parts[1], parts[2]
}
func TestPipelineTracePropagationThroughOutbox(t *testing.T) {
e := setup(t)
otel.SetTextMapPropagator(propagation.TraceContext{})
tp := sdktrace.NewTracerProvider(sdktrace.WithSampler(sdktrace.AlwaysSample()))
reader := sdkmetric.NewManualReader()
otel.SetMeterProvider(sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)))
tr := tp.Tracer("test")
// 1. Upload в контексте server span.
uctx, upSpan := tr.Start(context.Background(), "POST /api/v1/chats/{chat_id}/attachments", trace.WithSpanKind(trace.SpanKindServer))
data := pngBytes(120, 80, 3)
id := uuid.NewString()
sum := sha256.Sum256(data)
a := &domain.Attachment{ID: id, ChatID: e.chat, OwnerID: e.owner, Filename: "f.png",
ObjectKey: "s/" + e.chat + "/" + id, CreatedAt: time.Now(), UpdatedAt: time.Now(),
ContentType: "image/png", SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(sum[:])}
if err := e.att.CreateIntent(uctx, a); err != nil {
t.Fatal(err)
}
_ = e.store.Put(uctx, a.ObjectKey, bytes.NewReader(data), int64(len(data)), "image/png")
if _, err := e.att.MarkQueued(uctx, a, 1, 3); err != nil {
t.Fatal(err)
}
upSpan.End()
upTrace, upSpanID := upSpan.SpanContext().TraceID().String(), upSpan.SpanContext().SpanID().String()
publish := func() []postgres.OutboxMessage {
var got []postgres.OutboxMessage
if _, err := e.repo.PublishDue(context.Background(), 10, func(_ context.Context, m postgres.OutboxMessage) error {
got = append(got, m)
return nil
}, nil); err != nil {
t.Fatal(err)
}
return got
}
msgs := publish()
expect(t, len(msgs) == 1, "outbox: %d", len(msgs))
tid, sid := traceparent(msgs[0].Trace)
expect(t, tid == upTrace && sid == upSpanID, "outbox must carry upload span: %v (want %s/%s)", msgs[0].Trace, upTrace, upSpanID)
// 2. Consumer span из контекста сообщения; первая попытка падает (S3) → retry.
cctx := telemetry.ExtractMap(context.Background(), msgs[0].Trace)
cctx, consumer := tr.Start(cctx, "attachment.process process", trace.WithSpanKind(trace.SpanKindConsumer))
expect(t, consumer.SpanContext().TraceID().String() == upTrace, "consumer must continue upload trace")
e.store.failPut = 1
w := e.worker(5 * time.Second)
if d := w.Handle(cctx, msgs[0].Payload); d.String() != "ack" {
t.Fatalf("decision %v", d)
}
consumer.End()
time.Sleep(500 * time.Millisecond) // backoff (RetryBase 200 ms)
retry := publish()
expect(t, len(retry) == 1, "retry message: %d", len(retry))
tid, sid = traceparent(retry[0].Trace)
expect(t, tid == upTrace && sid == consumer.SpanContext().SpanID().String(),
"retry message must be a child of the failed attempt: %v", retry[0].Trace)
// 3. Вторая попытка успешна.
c2 := telemetry.ExtractMap(context.Background(), retry[0].Trace)
c2, consumer2 := tr.Start(c2, "attachment.process process", trace.WithSpanKind(trace.SpanKindConsumer))
w.Handle(c2, retry[0].Payload)
consumer2.End()
expect(t, e.state(a).Status == "ready", "state %+v", e.state(a))
// 4. Сообщение без активного span (sweeper/replay) берёт контекст upload из jobs.
var jobTrace map[string]string
if err := e.pool.QueryRow(e.ctx, `SELECT trace_context FROM jobs WHERE attachment_id = $1`, a.ID).Scan(&jobTrace); err != nil {
t.Fatal(err)
}
tid, sid = traceparent(jobTrace)
expect(t, tid == upTrace && sid == upSpanID, "jobs.trace_context: %v", jobTrace)
// 5. Метрики worker: retry и done, время до ready записано.
var rm metricdata.ResourceMetrics
if err := reader.Collect(context.Background(), &rm); err != nil {
t.Fatal(err)
}
outcomes := map[string]int64{}
var ttr uint64
for _, sm := range rm.ScopeMetrics {
for _, m := range sm.Metrics {
switch m.Name {
case "messenger.jobs.processed":
for _, dp := range m.Data.(metricdata.Sum[int64]).DataPoints {
v, _ := dp.Attributes.Value(attribute.Key("outcome"))
outcomes[v.AsString()] += dp.Value
}
case "messenger.attachment.time_to_ready":
for _, dp := range m.Data.(metricdata.Histogram[float64]).DataPoints {
ttr += dp.Count
}
}
}
}
expect(t, outcomes["retry"] == 1 && outcomes["done"] == 1, "outcomes %v", outcomes)
expect(t, ttr == 1, "time_to_ready count %d", ttr)
}