137 lines
5.4 KiB
Go
137 lines
5.4 KiB
Go
package worker_test
|
|
|
|
// Перенос W3C trace context через outbox (lab6) на настоящем PostgreSQL:
|
|
// upload span → outbox.trace_context → (relay) → consumer span → retry-сообщение
|
|
// несёт контекст попытки; сообщение от sweeper (без span) — контекст исходного
|
|
// upload из jobs.trace_context. Плюс метрики worker по исходам.
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"go.opentelemetry.io/otel"
|
|
"go.opentelemetry.io/otel/attribute"
|
|
"go.opentelemetry.io/otel/propagation"
|
|
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
|
|
"go.opentelemetry.io/otel/sdk/metric/metricdata"
|
|
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
|
"go.opentelemetry.io/otel/trace"
|
|
|
|
"github.com/charawein/mtusi2026lab7/internal/domain"
|
|
"github.com/charawein/mtusi2026lab7/internal/repository/postgres"
|
|
"github.com/charawein/mtusi2026lab7/internal/telemetry"
|
|
)
|
|
|
|
func traceparent(m map[string]string) (traceID, spanID string) {
|
|
parts := strings.Split(m["traceparent"], "-")
|
|
if len(parts) != 4 {
|
|
return "", ""
|
|
}
|
|
return parts[1], parts[2]
|
|
}
|
|
|
|
func TestPipelineTracePropagationThroughOutbox(t *testing.T) {
|
|
e := setup(t)
|
|
otel.SetTextMapPropagator(propagation.TraceContext{})
|
|
tp := sdktrace.NewTracerProvider(sdktrace.WithSampler(sdktrace.AlwaysSample()))
|
|
reader := sdkmetric.NewManualReader()
|
|
otel.SetMeterProvider(sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)))
|
|
tr := tp.Tracer("test")
|
|
|
|
// 1. Upload в контексте server span.
|
|
uctx, upSpan := tr.Start(context.Background(), "POST /api/v1/chats/{chat_id}/attachments", trace.WithSpanKind(trace.SpanKindServer))
|
|
data := pngBytes(120, 80, 3)
|
|
id := uuid.NewString()
|
|
sum := sha256.Sum256(data)
|
|
a := &domain.Attachment{ID: id, ChatID: e.chat, OwnerID: e.owner, Filename: "f.png",
|
|
ObjectKey: "s/" + e.chat + "/" + id, CreatedAt: time.Now(), UpdatedAt: time.Now(),
|
|
ContentType: "image/png", SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(sum[:])}
|
|
if err := e.att.CreateIntent(uctx, a); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_ = e.store.Put(uctx, a.ObjectKey, bytes.NewReader(data), int64(len(data)), "image/png")
|
|
if _, err := e.att.MarkQueued(uctx, a, 1, 3); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
upSpan.End()
|
|
upTrace, upSpanID := upSpan.SpanContext().TraceID().String(), upSpan.SpanContext().SpanID().String()
|
|
|
|
publish := func() []postgres.OutboxMessage {
|
|
var got []postgres.OutboxMessage
|
|
if _, err := e.repo.PublishDue(context.Background(), 10, func(_ context.Context, m postgres.OutboxMessage) error {
|
|
got = append(got, m)
|
|
return nil
|
|
}, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return got
|
|
}
|
|
msgs := publish()
|
|
expect(t, len(msgs) == 1, "outbox: %d", len(msgs))
|
|
tid, sid := traceparent(msgs[0].Trace)
|
|
expect(t, tid == upTrace && sid == upSpanID, "outbox must carry upload span: %v (want %s/%s)", msgs[0].Trace, upTrace, upSpanID)
|
|
|
|
// 2. Consumer span из контекста сообщения; первая попытка падает (S3) → retry.
|
|
cctx := telemetry.ExtractMap(context.Background(), msgs[0].Trace)
|
|
cctx, consumer := tr.Start(cctx, "attachment.process process", trace.WithSpanKind(trace.SpanKindConsumer))
|
|
expect(t, consumer.SpanContext().TraceID().String() == upTrace, "consumer must continue upload trace")
|
|
e.store.failPut = 1
|
|
w := e.worker(5 * time.Second)
|
|
if d := w.Handle(cctx, msgs[0].Payload); d.String() != "ack" {
|
|
t.Fatalf("decision %v", d)
|
|
}
|
|
consumer.End()
|
|
time.Sleep(500 * time.Millisecond) // backoff (RetryBase 200 ms)
|
|
retry := publish()
|
|
expect(t, len(retry) == 1, "retry message: %d", len(retry))
|
|
tid, sid = traceparent(retry[0].Trace)
|
|
expect(t, tid == upTrace && sid == consumer.SpanContext().SpanID().String(),
|
|
"retry message must be a child of the failed attempt: %v", retry[0].Trace)
|
|
|
|
// 3. Вторая попытка успешна.
|
|
c2 := telemetry.ExtractMap(context.Background(), retry[0].Trace)
|
|
c2, consumer2 := tr.Start(c2, "attachment.process process", trace.WithSpanKind(trace.SpanKindConsumer))
|
|
w.Handle(c2, retry[0].Payload)
|
|
consumer2.End()
|
|
expect(t, e.state(a).Status == "ready", "state %+v", e.state(a))
|
|
|
|
// 4. Сообщение без активного span (sweeper/replay) берёт контекст upload из jobs.
|
|
var jobTrace map[string]string
|
|
if err := e.pool.QueryRow(e.ctx, `SELECT trace_context FROM jobs WHERE attachment_id = $1`, a.ID).Scan(&jobTrace); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
tid, sid = traceparent(jobTrace)
|
|
expect(t, tid == upTrace && sid == upSpanID, "jobs.trace_context: %v", jobTrace)
|
|
|
|
// 5. Метрики worker: retry и done, время до ready записано.
|
|
var rm metricdata.ResourceMetrics
|
|
if err := reader.Collect(context.Background(), &rm); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
outcomes := map[string]int64{}
|
|
var ttr uint64
|
|
for _, sm := range rm.ScopeMetrics {
|
|
for _, m := range sm.Metrics {
|
|
switch m.Name {
|
|
case "messenger.jobs.processed":
|
|
for _, dp := range m.Data.(metricdata.Sum[int64]).DataPoints {
|
|
v, _ := dp.Attributes.Value(attribute.Key("outcome"))
|
|
outcomes[v.AsString()] += dp.Value
|
|
}
|
|
case "messenger.attachment.time_to_ready":
|
|
for _, dp := range m.Data.(metricdata.Histogram[float64]).DataPoints {
|
|
ttr += dp.Count
|
|
}
|
|
}
|
|
}
|
|
}
|
|
expect(t, outcomes["retry"] == 1 && outcomes["done"] == 1, "outcomes %v", outcomes)
|
|
expect(t, ttr == 1, "time_to_ready count %d", ttr)
|
|
}
|