forked from mtusi2026/mtusi2026lab6
feat(worker): lab6 record job outcomes and end-to-end readiness latency
This commit is contained in:
@@ -108,7 +108,6 @@ func (s *AttachmentService) Upload(
|
||||
sha := sha256.Sum256(buf)
|
||||
id := uuid.NewString()
|
||||
stagingKey := "staging/" + id
|
||||
now := time.Now().UTC()
|
||||
|
||||
a := &domain.Attachment{
|
||||
ID: id,
|
||||
@@ -122,8 +121,6 @@ func (s *AttachmentService) Upload(
|
||||
Status: domain.AttachmentStatusQueued,
|
||||
StagingKey: stagingKey,
|
||||
PipelineVersion: 1,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
if err := s.storage.Put(ctx, stagingKey, bytes.NewReader(buf), int64(len(buf)), contentType); err != nil {
|
||||
@@ -140,6 +137,8 @@ func (s *AttachmentService) Upload(
|
||||
}
|
||||
|
||||
err = s.tx.WithTx(ctx, func(tx domain.Tx) error {
|
||||
a.CreatedAt = time.Now().UTC()
|
||||
a.UpdatedAt = a.CreatedAt
|
||||
if err := s.attachments.CreateInTx(ctx, tx, a); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -200,10 +200,10 @@ func (r *AttachmentRepo) ClaimForProcessing(ctx context.Context, id string, vers
|
||||
UPDATE attachments
|
||||
SET status = 'processing', locked_until = now() + $2, updated_at = now(), attempts = attempts + 1
|
||||
WHERE id = $1::uuid AND status = 'queued' AND pipeline_version = $3
|
||||
RETURNING id, chat_id, staging_key, pipeline_version, object_key, content_type, attempts
|
||||
RETURNING id, chat_id, staging_key, pipeline_version, object_key, content_type, attempts, created_at
|
||||
`
|
||||
var a domain.Attachment
|
||||
err := r.pool.QueryRow(ctx, q, id, lease, version).Scan(&a.ID, &a.ChatID, &a.StagingKey, &a.PipelineVersion, &a.ObjectKey, &a.ContentType, &a.Attempts)
|
||||
err := r.pool.QueryRow(ctx, q, id, lease, version).Scan(&a.ID, &a.ChatID, &a.StagingKey, &a.PipelineVersion, &a.ObjectKey, &a.ContentType, &a.Attempts, &a.CreatedAt)
|
||||
if err != nil {
|
||||
if err == pgx.ErrNoRows {
|
||||
return nil, nil
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"high-loud-apps-labs/internal/domain"
|
||||
"high-loud-apps-labs/internal/telemetry"
|
||||
)
|
||||
|
||||
type jobMetrics struct {
|
||||
processed metric.Int64Counter
|
||||
duration metric.Float64Histogram
|
||||
ready metric.Float64Histogram
|
||||
}
|
||||
|
||||
func newJobMetrics() jobMetrics {
|
||||
meter := telemetry.Meter("worker.pipeline")
|
||||
processed, err := meter.Int64Counter("jobs_processed_total")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
unit := metric.WithUnit("s")
|
||||
duration, err := meter.Float64Histogram("job_processing_duration_seconds", unit,
|
||||
metric.WithExplicitBucketBoundaries(.01, .05, .1, .25, .5, 1, 2, 5, 10, 30, 60))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
ready, err := meter.Float64Histogram("end_to_end_ready_seconds", unit,
|
||||
metric.WithExplicitBucketBoundaries(.1, .5, 1, 2, 5, 10, 30, 60, 120, 300, 900, 3600))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return jobMetrics{processed: processed, duration: duration, ready: ready}
|
||||
}
|
||||
|
||||
func (m jobMetrics) recordAttempt(ctx context.Context, start time.Time, outcome string) {
|
||||
attrs := metric.WithAttributes(attribute.String("job.type", domain.OutboxEventProcess), attribute.String("outcome", outcome))
|
||||
m.duration.Record(ctx, time.Since(start).Seconds(), attrs)
|
||||
m.processed.Add(ctx, 1, attrs)
|
||||
|
||||
span := trace.SpanFromContext(ctx)
|
||||
span.SetAttributes(attribute.String("job.outcome", outcome))
|
||||
level := slog.LevelInfo
|
||||
switch outcome {
|
||||
case "failed", "error", "retry":
|
||||
level = slog.LevelWarn
|
||||
span.SetStatus(codes.Error, outcome)
|
||||
}
|
||||
slog.Log(ctx, level, "Попытка обработки завершена", "event", "job.processed", "outcome", outcome)
|
||||
}
|
||||
|
||||
func (m jobMetrics) recordReady(ctx context.Context, createdAt time.Time) {
|
||||
if createdAt.IsZero() {
|
||||
return
|
||||
}
|
||||
m.ready.Record(ctx, max(0, time.Since(createdAt).Seconds()),
|
||||
metric.WithAttributes(attribute.String("job.type", domain.OutboxEventProcess)))
|
||||
}
|
||||
@@ -26,6 +26,7 @@ type Pipeline struct {
|
||||
outbox domain.OutboxRepository
|
||||
lease time.Duration
|
||||
maxRetries int
|
||||
metrics jobMetrics
|
||||
}
|
||||
|
||||
func NewPipeline(
|
||||
@@ -43,10 +44,15 @@ func NewPipeline(
|
||||
outbox: outbox,
|
||||
lease: 5 * time.Minute,
|
||||
maxRetries: 5,
|
||||
metrics: newJobMetrics(),
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Pipeline) Process(ctx context.Context, attachmentID string, pipelineVersion int) error {
|
||||
start := time.Now()
|
||||
outcome := "error"
|
||||
defer func() { p.metrics.recordAttempt(ctx, start, outcome) }()
|
||||
|
||||
log.Printf("worker: начало обработки %s (версия пайплайна %d)", attachmentID, pipelineVersion)
|
||||
|
||||
jobCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
@@ -61,23 +67,29 @@ func (p *Pipeline) Process(ctx context.Context, attachmentID string, pipelineVer
|
||||
return fmt.Errorf("захват задачи: %w", err)
|
||||
}
|
||||
if a == nil {
|
||||
outcome = "skipped"
|
||||
log.Printf("worker: задача %s уже взята, выполнена или версия устарела", attachmentID)
|
||||
return nil // no-op
|
||||
}
|
||||
|
||||
fail := func(err error, permanent bool) error {
|
||||
outcome = p.handleFailure(a, err, permanent)
|
||||
return nil
|
||||
}
|
||||
|
||||
// 2. Скачиваем оригинал из staging
|
||||
getCtx, getSpan := tracer.Start(jobCtx, "job.s3.get")
|
||||
rc, _, _, err := p.store.Get(getCtx, a.StagingKey)
|
||||
if err != nil {
|
||||
endPipelineSpan(getSpan, err)
|
||||
return p.handleFailure(a, fmt.Errorf("скачивание staging: %w", err), false)
|
||||
return fail(fmt.Errorf("скачивание staging: %w", err), false)
|
||||
}
|
||||
defer rc.Close()
|
||||
|
||||
buf, err := io.ReadAll(rc)
|
||||
endPipelineSpan(getSpan, err)
|
||||
if err != nil {
|
||||
return p.handleFailure(a, fmt.Errorf("чтение staging: %w", err), false)
|
||||
return fail(fmt.Errorf("чтение staging: %w", err), false)
|
||||
}
|
||||
|
||||
// 3. Генерируем все варианты (Thumbnail, Crop, Optimized)
|
||||
@@ -86,26 +98,26 @@ func (p *Pipeline) Process(ctx context.Context, attachmentID string, pipelineVer
|
||||
thumbMeta, thumbData, err := imaging.Thumbnail(a.ID, buf)
|
||||
if err != nil {
|
||||
endPipelineSpan(imageSpan, err)
|
||||
return p.handleFailure(a, fmt.Errorf("генерация thumbnail: %w", err), true)
|
||||
return fail(fmt.Errorf("генерация thumbnail: %w", err), true)
|
||||
}
|
||||
|
||||
cropMeta, cropData, err := imaging.Crop(a.ID, buf)
|
||||
if err != nil {
|
||||
endPipelineSpan(imageSpan, err)
|
||||
return p.handleFailure(a, fmt.Errorf("генерация crop: %w", err), true)
|
||||
return fail(fmt.Errorf("генерация crop: %w", err), true)
|
||||
}
|
||||
|
||||
webpMeta, webpData, err := imaging.Optimized(a.ID, buf)
|
||||
endPipelineSpan(imageSpan, err)
|
||||
if err != nil {
|
||||
return p.handleFailure(a, fmt.Errorf("генерация webp: %w", err), true)
|
||||
return fail(fmt.Errorf("генерация webp: %w", err), true)
|
||||
}
|
||||
|
||||
// 4. Сохраняем оригинал в финальный prefix (chats/...)
|
||||
putCtx, putSpan := tracer.Start(jobCtx, "job.s3.put")
|
||||
if err := p.store.Put(putCtx, a.ObjectKey, bytes.NewReader(buf), int64(len(buf)), a.ContentType); err != nil {
|
||||
endPipelineSpan(putSpan, err)
|
||||
return p.handleFailure(a, fmt.Errorf("загрузка оригинала: %w", err), false)
|
||||
return fail(fmt.Errorf("загрузка оригинала: %w", err), false)
|
||||
}
|
||||
|
||||
// 5. Загружаем все сгенерированные варианты в S3
|
||||
@@ -127,7 +139,7 @@ func (p *Pipeline) Process(ctx context.Context, attachmentID string, pipelineVer
|
||||
|
||||
if err := p.store.Put(putCtx, key, bytes.NewReader(data), v.SizeBytes, v.ContentType); err != nil {
|
||||
endPipelineSpan(putSpan, err)
|
||||
return p.handleFailure(a, fmt.Errorf("загрузка варианта %s: %w", v.Name, err), false)
|
||||
return fail(fmt.Errorf("загрузка варианта %s: %w", v.Name, err), false)
|
||||
}
|
||||
v.ObjectKey = key
|
||||
}
|
||||
@@ -139,13 +151,16 @@ func (p *Pipeline) Process(ctx context.Context, attachmentID string, pipelineVer
|
||||
ok, err := p.jobs.CompleteJob(completeCtx, a.ID, a.PipelineVersion, variants)
|
||||
endPipelineSpan(completeSpan, err)
|
||||
if err != nil {
|
||||
return p.handleFailure(a, fmt.Errorf("фиксация в БД: %w", err), false)
|
||||
return fail(fmt.Errorf("фиксация в БД: %w", err), false)
|
||||
}
|
||||
if !ok {
|
||||
outcome = "skipped"
|
||||
log.Printf("worker: версия устарела или статус изменился, пропускаем %s", a.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
outcome = "ready"
|
||||
p.metrics.recordReady(ctx, a.CreatedAt)
|
||||
log.Printf("worker: успешно завершено %s", a.ID)
|
||||
return nil
|
||||
}
|
||||
@@ -157,7 +172,7 @@ func endPipelineSpan(span trace.Span, err error) {
|
||||
span.End()
|
||||
}
|
||||
|
||||
func (p *Pipeline) handleFailure(a *domain.Attachment, err error, isPermanent bool) error {
|
||||
func (p *Pipeline) handleFailure(a *domain.Attachment, err error, isPermanent bool) string {
|
||||
// Если исчерпаны попытки или ошибка постоянная (например, битый файл)
|
||||
if isPermanent || a.Attempts >= p.maxRetries {
|
||||
reason := domain.JobErrorInternal
|
||||
@@ -170,13 +185,16 @@ func (p *Pipeline) handleFailure(a *domain.Attachment, err error, isPermanent bo
|
||||
log.Printf("worker: финальная ошибка для %s (попытка %d): %v", a.ID, a.Attempts, err)
|
||||
|
||||
// Используем context.Background(), чтобы запись в БД прошла даже при истекшем jobCtx
|
||||
_ = p.jobs.FailJob(context.Background(), a.ID, reason)
|
||||
_ = p.failedJobs.Record(context.Background(), &domain.FailedJob{
|
||||
updateErr := p.jobs.FailJob(context.Background(), a.ID, reason)
|
||||
recordErr := p.failedJobs.Record(context.Background(), &domain.FailedJob{
|
||||
AttachmentID: a.ID,
|
||||
ReasonCode: reason,
|
||||
Attempts: a.Attempts,
|
||||
})
|
||||
return nil
|
||||
if updateErr != nil || recordErr != nil {
|
||||
return "error"
|
||||
}
|
||||
return "failed"
|
||||
}
|
||||
|
||||
// Временная ошибка - делаем Retry с backoff
|
||||
@@ -187,7 +205,7 @@ func (p *Pipeline) handleFailure(a *domain.Attachment, err error, isPermanent bo
|
||||
jitter := time.Duration(rand.Intn(1000)) * time.Millisecond
|
||||
nextAt := time.Now().UTC().Add(backoff + jitter)
|
||||
|
||||
_ = p.jobs.ScheduleRetry(context.Background(), a.ID, nextAt, "")
|
||||
scheduleErr := p.jobs.ScheduleRetry(context.Background(), a.ID, nextAt, "")
|
||||
|
||||
// Кладем в Outbox отложенное сообщение
|
||||
payload, _ := json.Marshal(domain.JobMessage{
|
||||
@@ -195,7 +213,7 @@ func (p *Pipeline) handleFailure(a *domain.Attachment, err error, isPermanent bo
|
||||
PipelineVersion: a.PipelineVersion,
|
||||
Attempt: a.Attempts + 1,
|
||||
})
|
||||
_ = p.outbox.EnqueueDirect(context.Background(), &domain.OutboxEntry{
|
||||
enqueueErr := p.outbox.EnqueueDirect(context.Background(), &domain.OutboxEntry{
|
||||
AggregateType: domain.OutboxAggregateAttachment,
|
||||
AggregateID: a.ID,
|
||||
EventType: domain.OutboxEventProcess,
|
||||
@@ -203,5 +221,8 @@ func (p *Pipeline) handleFailure(a *domain.Attachment, err error, isPermanent bo
|
||||
CreatedAt: nextAt,
|
||||
})
|
||||
|
||||
return nil
|
||||
if scheduleErr != nil || enqueueErr != nil {
|
||||
return "error"
|
||||
}
|
||||
return "retry"
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user