From 44c755f00c8f7bb46b0faf86d018599af817402e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=93=D0=BE=D0=BB=D0=B8=D0=BA=D0=BE=D0=B2=20Maximillian?= =?UTF-8?q?=20=D0=90=D0=BD=D1=82=D0=BE=D0=BD?= Date: Mon, 16 Nov 2026 21:55:01 +0300 Subject: [PATCH] feat(worker): lab6 record job outcomes and end-to-end readiness latency --- backend/internal/service/attachment.go | 5 +- .../internal/storage/postgres/attachments.go | 4 +- backend/internal/worker/metrics.go | 65 +++++++++++++++++++ backend/internal/worker/pipeline.go | 51 ++++++++++----- 4 files changed, 105 insertions(+), 20 deletions(-) create mode 100644 backend/internal/worker/metrics.go diff --git a/backend/internal/service/attachment.go b/backend/internal/service/attachment.go index b0bd3a5..b656d57 100644 --- a/backend/internal/service/attachment.go +++ b/backend/internal/service/attachment.go @@ -108,7 +108,6 @@ func (s *AttachmentService) Upload( sha := sha256.Sum256(buf) id := uuid.NewString() stagingKey := "staging/" + id - now := time.Now().UTC() a := &domain.Attachment{ ID: id, @@ -122,8 +121,6 @@ func (s *AttachmentService) Upload( Status: domain.AttachmentStatusQueued, StagingKey: stagingKey, PipelineVersion: 1, - CreatedAt: now, - UpdatedAt: now, } if err := s.storage.Put(ctx, stagingKey, bytes.NewReader(buf), int64(len(buf)), contentType); err != nil { @@ -140,6 +137,8 @@ func (s *AttachmentService) Upload( } err = s.tx.WithTx(ctx, func(tx domain.Tx) error { + a.CreatedAt = time.Now().UTC() + a.UpdatedAt = a.CreatedAt if err := s.attachments.CreateInTx(ctx, tx, a); err != nil { return err } diff --git a/backend/internal/storage/postgres/attachments.go b/backend/internal/storage/postgres/attachments.go index 56ebc65..8caf791 100644 --- a/backend/internal/storage/postgres/attachments.go +++ b/backend/internal/storage/postgres/attachments.go @@ -200,10 +200,10 @@ func (r *AttachmentRepo) ClaimForProcessing(ctx context.Context, id string, vers UPDATE attachments SET status = 'processing', locked_until = now() + $2, updated_at = now(), attempts = attempts + 1 WHERE id = $1::uuid AND status = 'queued' AND pipeline_version = $3 - RETURNING id, chat_id, staging_key, pipeline_version, object_key, content_type, attempts + RETURNING id, chat_id, staging_key, pipeline_version, object_key, content_type, attempts, created_at ` var a domain.Attachment - err := r.pool.QueryRow(ctx, q, id, lease, version).Scan(&a.ID, &a.ChatID, &a.StagingKey, &a.PipelineVersion, &a.ObjectKey, &a.ContentType, &a.Attempts) + err := r.pool.QueryRow(ctx, q, id, lease, version).Scan(&a.ID, &a.ChatID, &a.StagingKey, &a.PipelineVersion, &a.ObjectKey, &a.ContentType, &a.Attempts, &a.CreatedAt) if err != nil { if err == pgx.ErrNoRows { return nil, nil diff --git a/backend/internal/worker/metrics.go b/backend/internal/worker/metrics.go new file mode 100644 index 0000000..3257c63 --- /dev/null +++ b/backend/internal/worker/metrics.go @@ -0,0 +1,65 @@ +package worker + +import ( + "context" + "log/slog" + "time" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" + + "high-loud-apps-labs/internal/domain" + "high-loud-apps-labs/internal/telemetry" +) + +type jobMetrics struct { + processed metric.Int64Counter + duration metric.Float64Histogram + ready metric.Float64Histogram +} + +func newJobMetrics() jobMetrics { + meter := telemetry.Meter("worker.pipeline") + processed, err := meter.Int64Counter("jobs_processed_total") + if err != nil { + panic(err) + } + unit := metric.WithUnit("s") + duration, err := meter.Float64Histogram("job_processing_duration_seconds", unit, + metric.WithExplicitBucketBoundaries(.01, .05, .1, .25, .5, 1, 2, 5, 10, 30, 60)) + if err != nil { + panic(err) + } + ready, err := meter.Float64Histogram("end_to_end_ready_seconds", unit, + metric.WithExplicitBucketBoundaries(.1, .5, 1, 2, 5, 10, 30, 60, 120, 300, 900, 3600)) + if err != nil { + panic(err) + } + return jobMetrics{processed: processed, duration: duration, ready: ready} +} + +func (m jobMetrics) recordAttempt(ctx context.Context, start time.Time, outcome string) { + attrs := metric.WithAttributes(attribute.String("job.type", domain.OutboxEventProcess), attribute.String("outcome", outcome)) + m.duration.Record(ctx, time.Since(start).Seconds(), attrs) + m.processed.Add(ctx, 1, attrs) + + span := trace.SpanFromContext(ctx) + span.SetAttributes(attribute.String("job.outcome", outcome)) + level := slog.LevelInfo + switch outcome { + case "failed", "error", "retry": + level = slog.LevelWarn + span.SetStatus(codes.Error, outcome) + } + slog.Log(ctx, level, "Попытка обработки завершена", "event", "job.processed", "outcome", outcome) +} + +func (m jobMetrics) recordReady(ctx context.Context, createdAt time.Time) { + if createdAt.IsZero() { + return + } + m.ready.Record(ctx, max(0, time.Since(createdAt).Seconds()), + metric.WithAttributes(attribute.String("job.type", domain.OutboxEventProcess))) +} diff --git a/backend/internal/worker/pipeline.go b/backend/internal/worker/pipeline.go index 533f7c4..0258264 100644 --- a/backend/internal/worker/pipeline.go +++ b/backend/internal/worker/pipeline.go @@ -26,6 +26,7 @@ type Pipeline struct { outbox domain.OutboxRepository lease time.Duration maxRetries int + metrics jobMetrics } func NewPipeline( @@ -43,10 +44,15 @@ func NewPipeline( outbox: outbox, lease: 5 * time.Minute, maxRetries: 5, + metrics: newJobMetrics(), } } func (p *Pipeline) Process(ctx context.Context, attachmentID string, pipelineVersion int) error { + start := time.Now() + outcome := "error" + defer func() { p.metrics.recordAttempt(ctx, start, outcome) }() + log.Printf("worker: начало обработки %s (версия пайплайна %d)", attachmentID, pipelineVersion) jobCtx, cancel := context.WithTimeout(ctx, 30*time.Second) @@ -61,23 +67,29 @@ func (p *Pipeline) Process(ctx context.Context, attachmentID string, pipelineVer return fmt.Errorf("захват задачи: %w", err) } if a == nil { + outcome = "skipped" log.Printf("worker: задача %s уже взята, выполнена или версия устарела", attachmentID) return nil // no-op } + fail := func(err error, permanent bool) error { + outcome = p.handleFailure(a, err, permanent) + return nil + } + // 2. Скачиваем оригинал из staging getCtx, getSpan := tracer.Start(jobCtx, "job.s3.get") rc, _, _, err := p.store.Get(getCtx, a.StagingKey) if err != nil { endPipelineSpan(getSpan, err) - return p.handleFailure(a, fmt.Errorf("скачивание staging: %w", err), false) + return fail(fmt.Errorf("скачивание staging: %w", err), false) } defer rc.Close() buf, err := io.ReadAll(rc) endPipelineSpan(getSpan, err) if err != nil { - return p.handleFailure(a, fmt.Errorf("чтение staging: %w", err), false) + return fail(fmt.Errorf("чтение staging: %w", err), false) } // 3. Генерируем все варианты (Thumbnail, Crop, Optimized) @@ -86,26 +98,26 @@ func (p *Pipeline) Process(ctx context.Context, attachmentID string, pipelineVer thumbMeta, thumbData, err := imaging.Thumbnail(a.ID, buf) if err != nil { endPipelineSpan(imageSpan, err) - return p.handleFailure(a, fmt.Errorf("генерация thumbnail: %w", err), true) + return fail(fmt.Errorf("генерация thumbnail: %w", err), true) } cropMeta, cropData, err := imaging.Crop(a.ID, buf) if err != nil { endPipelineSpan(imageSpan, err) - return p.handleFailure(a, fmt.Errorf("генерация crop: %w", err), true) + return fail(fmt.Errorf("генерация crop: %w", err), true) } webpMeta, webpData, err := imaging.Optimized(a.ID, buf) endPipelineSpan(imageSpan, err) if err != nil { - return p.handleFailure(a, fmt.Errorf("генерация webp: %w", err), true) + return fail(fmt.Errorf("генерация webp: %w", err), true) } // 4. Сохраняем оригинал в финальный prefix (chats/...) putCtx, putSpan := tracer.Start(jobCtx, "job.s3.put") if err := p.store.Put(putCtx, a.ObjectKey, bytes.NewReader(buf), int64(len(buf)), a.ContentType); err != nil { endPipelineSpan(putSpan, err) - return p.handleFailure(a, fmt.Errorf("загрузка оригинала: %w", err), false) + return fail(fmt.Errorf("загрузка оригинала: %w", err), false) } // 5. Загружаем все сгенерированные варианты в S3 @@ -127,7 +139,7 @@ func (p *Pipeline) Process(ctx context.Context, attachmentID string, pipelineVer if err := p.store.Put(putCtx, key, bytes.NewReader(data), v.SizeBytes, v.ContentType); err != nil { endPipelineSpan(putSpan, err) - return p.handleFailure(a, fmt.Errorf("загрузка варианта %s: %w", v.Name, err), false) + return fail(fmt.Errorf("загрузка варианта %s: %w", v.Name, err), false) } v.ObjectKey = key } @@ -139,13 +151,16 @@ func (p *Pipeline) Process(ctx context.Context, attachmentID string, pipelineVer ok, err := p.jobs.CompleteJob(completeCtx, a.ID, a.PipelineVersion, variants) endPipelineSpan(completeSpan, err) if err != nil { - return p.handleFailure(a, fmt.Errorf("фиксация в БД: %w", err), false) + return fail(fmt.Errorf("фиксация в БД: %w", err), false) } if !ok { + outcome = "skipped" log.Printf("worker: версия устарела или статус изменился, пропускаем %s", a.ID) return nil } + outcome = "ready" + p.metrics.recordReady(ctx, a.CreatedAt) log.Printf("worker: успешно завершено %s", a.ID) return nil } @@ -157,7 +172,7 @@ func endPipelineSpan(span trace.Span, err error) { span.End() } -func (p *Pipeline) handleFailure(a *domain.Attachment, err error, isPermanent bool) error { +func (p *Pipeline) handleFailure(a *domain.Attachment, err error, isPermanent bool) string { // Если исчерпаны попытки или ошибка постоянная (например, битый файл) if isPermanent || a.Attempts >= p.maxRetries { reason := domain.JobErrorInternal @@ -170,13 +185,16 @@ func (p *Pipeline) handleFailure(a *domain.Attachment, err error, isPermanent bo log.Printf("worker: финальная ошибка для %s (попытка %d): %v", a.ID, a.Attempts, err) // Используем context.Background(), чтобы запись в БД прошла даже при истекшем jobCtx - _ = p.jobs.FailJob(context.Background(), a.ID, reason) - _ = p.failedJobs.Record(context.Background(), &domain.FailedJob{ + updateErr := p.jobs.FailJob(context.Background(), a.ID, reason) + recordErr := p.failedJobs.Record(context.Background(), &domain.FailedJob{ AttachmentID: a.ID, ReasonCode: reason, Attempts: a.Attempts, }) - return nil + if updateErr != nil || recordErr != nil { + return "error" + } + return "failed" } // Временная ошибка - делаем Retry с backoff @@ -187,7 +205,7 @@ func (p *Pipeline) handleFailure(a *domain.Attachment, err error, isPermanent bo jitter := time.Duration(rand.Intn(1000)) * time.Millisecond nextAt := time.Now().UTC().Add(backoff + jitter) - _ = p.jobs.ScheduleRetry(context.Background(), a.ID, nextAt, "") + scheduleErr := p.jobs.ScheduleRetry(context.Background(), a.ID, nextAt, "") // Кладем в Outbox отложенное сообщение payload, _ := json.Marshal(domain.JobMessage{ @@ -195,7 +213,7 @@ func (p *Pipeline) handleFailure(a *domain.Attachment, err error, isPermanent bo PipelineVersion: a.PipelineVersion, Attempt: a.Attempts + 1, }) - _ = p.outbox.EnqueueDirect(context.Background(), &domain.OutboxEntry{ + enqueueErr := p.outbox.EnqueueDirect(context.Background(), &domain.OutboxEntry{ AggregateType: domain.OutboxAggregateAttachment, AggregateID: a.ID, EventType: domain.OutboxEventProcess, @@ -203,5 +221,8 @@ func (p *Pipeline) handleFailure(a *domain.Attachment, err error, isPermanent bo CreatedAt: nextAt, }) - return nil + if scheduleErr != nil || enqueueErr != nil { + return "error" + } + return "retry" }