forked from mtusi2026/mtusi2026lab5
255 lines
6.0 KiB
Go
255 lines
6.0 KiB
Go
// Package broker wraps RabbitMQ delivery and publisher confirms. PostgreSQL is
|
|
// still the source of truth: a message carries only a job id and a snapshot of
|
|
// the durable job intent.
|
|
package broker
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"sync"
|
|
"time"
|
|
|
|
amqp "github.com/rabbitmq/amqp091-go"
|
|
|
|
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/model"
|
|
)
|
|
|
|
// ErrRequeue asks the consumer to nack and requeue. The worker uses it only
|
|
// when its durable state could not be written (for example PostgreSQL is down).
|
|
var ErrRequeue = errors.New("requeue delivery")
|
|
|
|
func dial(url string) (*amqp.Connection, error) {
|
|
return amqp.DialConfig(url, amqp.Config{
|
|
Heartbeat: 10 * time.Second,
|
|
Locale: "en_US",
|
|
})
|
|
}
|
|
|
|
func declareQueue(ch *amqp.Channel, queue string) error {
|
|
// A quorum queue uses a durable replicated log on the broker. Delivery is
|
|
// still at least once; idempotency lives in the job effect key.
|
|
_, err := ch.QueueDeclare(queue, true, false, false, false, amqp.Table{
|
|
"x-queue-type": "quorum",
|
|
})
|
|
return err
|
|
}
|
|
|
|
// Publisher is safe for concurrent use. It keeps one confirm-mode channel and
|
|
// reconnects lazily after a broker restart.
|
|
type Publisher struct {
|
|
url string
|
|
queue string
|
|
mu sync.Mutex
|
|
conn *amqp.Connection
|
|
ch *amqp.Channel
|
|
}
|
|
|
|
func NewPublisher(url, queue string) *Publisher {
|
|
return &Publisher{url: url, queue: queue}
|
|
}
|
|
|
|
func (p *Publisher) ensure() error {
|
|
if p.ch != nil && !p.ch.IsClosed() {
|
|
return nil
|
|
}
|
|
p.closeLocked()
|
|
conn, err := dial(p.url)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
ch, err := conn.Channel()
|
|
if err != nil {
|
|
_ = conn.Close()
|
|
return err
|
|
}
|
|
if err = ch.Confirm(false); err != nil {
|
|
_ = ch.Close()
|
|
_ = conn.Close()
|
|
return err
|
|
}
|
|
if err = declareQueue(ch, p.queue); err != nil {
|
|
_ = ch.Close()
|
|
_ = conn.Close()
|
|
return err
|
|
}
|
|
p.conn, p.ch = conn, ch
|
|
return nil
|
|
}
|
|
|
|
// Publish waits for the broker's publisher confirm. A nil return means the
|
|
// broker accepted the message, not that a worker executed it.
|
|
func (p *Publisher) Publish(ctx context.Context, job model.Job) error {
|
|
body, err := json.Marshal(job)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
p.mu.Lock()
|
|
defer p.mu.Unlock()
|
|
if err = p.ensure(); err != nil {
|
|
return err
|
|
}
|
|
|
|
confirmation, err := p.ch.PublishWithDeferredConfirmWithContext(ctx, "", p.queue, true, false, amqp.Publishing{
|
|
DeliveryMode: amqp.Persistent,
|
|
ContentType: "application/json",
|
|
MessageId: job.JobID.String(),
|
|
Timestamp: time.Now().UTC(),
|
|
Type: job.Type,
|
|
Body: body,
|
|
})
|
|
if err != nil {
|
|
p.closeLocked()
|
|
return err
|
|
}
|
|
confirmed, err := confirmation.WaitContext(ctx)
|
|
if err != nil {
|
|
p.closeLocked()
|
|
return err
|
|
}
|
|
if !confirmed {
|
|
p.closeLocked()
|
|
return fmt.Errorf("publisher confirm was negative")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (p *Publisher) Close() {
|
|
p.mu.Lock()
|
|
defer p.mu.Unlock()
|
|
p.closeLocked()
|
|
}
|
|
|
|
func (p *Publisher) closeLocked() {
|
|
if p.ch != nil {
|
|
_ = p.ch.Close()
|
|
}
|
|
if p.conn != nil {
|
|
_ = p.conn.Close()
|
|
}
|
|
p.ch, p.conn = nil, nil
|
|
}
|
|
|
|
// Handler must return nil after the durable effect (and optional ack); the
|
|
// consumer acks then. ErrRequeue leaves the message for redelivery.
|
|
type Handler func(context.Context, model.Job) error
|
|
|
|
type Consumer struct {
|
|
url string
|
|
queue string
|
|
logger *slog.Logger
|
|
}
|
|
|
|
func NewConsumer(url, queue string, logger *slog.Logger) *Consumer {
|
|
if logger == nil {
|
|
logger = slog.Default()
|
|
}
|
|
return &Consumer{url: url, queue: queue, logger: logger}
|
|
}
|
|
|
|
// Run reconnects until ctx is cancelled. concurrency bounds simultaneous
|
|
// handlers in this process; prefetch bounds unacked messages the broker may
|
|
// push. The two knobs are intentionally separate so throughput and in-flight
|
|
// memory can be measured independently.
|
|
func (c *Consumer) Run(ctx context.Context, concurrency, prefetch int, handler Handler) error {
|
|
for {
|
|
if ctx.Err() != nil {
|
|
return nil
|
|
}
|
|
err := c.consumeOnce(ctx, concurrency, prefetch, handler)
|
|
if ctx.Err() != nil {
|
|
return nil
|
|
}
|
|
c.logger.Error("rabbitmq consumer disconnected", "err", err)
|
|
select {
|
|
case <-ctx.Done():
|
|
return nil
|
|
case <-time.After(2 * time.Second):
|
|
}
|
|
}
|
|
}
|
|
|
|
func (c *Consumer) consumeOnce(ctx context.Context, concurrency, prefetch int, handler Handler) error {
|
|
conn, err := dial(c.url)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer conn.Close()
|
|
|
|
ch, err := conn.Channel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer ch.Close()
|
|
if err = declareQueue(ch, c.queue); err != nil {
|
|
return err
|
|
}
|
|
if err = ch.Qos(prefetch, 0, false); err != nil {
|
|
return err
|
|
}
|
|
deliveries, err := ch.Consume(c.queue, "", false, false, false, false, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
sem := make(chan struct{}, concurrency)
|
|
var workers sync.WaitGroup
|
|
|
|
stop := make(chan struct{})
|
|
go func() {
|
|
select {
|
|
case <-ctx.Done():
|
|
case <-stop:
|
|
}
|
|
_ = ch.Cancel("", false)
|
|
}()
|
|
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
close(stop)
|
|
workers.Wait()
|
|
return nil
|
|
case delivery, ok := <-deliveries:
|
|
if !ok {
|
|
close(stop)
|
|
workers.Wait()
|
|
return errors.New("delivery channel closed")
|
|
}
|
|
var job model.Job
|
|
if err = json.Unmarshal(delivery.Body, &job); err != nil {
|
|
c.logger.Error("drop malformed job message", "err", err)
|
|
_ = delivery.Ack(false)
|
|
continue
|
|
}
|
|
|
|
sem <- struct{}{}
|
|
workers.Add(1)
|
|
go func(delivery amqp.Delivery, job model.Job) {
|
|
defer workers.Done()
|
|
defer func() { <-sem }()
|
|
handlerErr := handler(ctx, job)
|
|
switch {
|
|
case handlerErr == nil:
|
|
if err := delivery.Ack(false); err != nil {
|
|
c.logger.Error("ack failed", "job_id", job.JobID, "err", err)
|
|
}
|
|
case errors.Is(handlerErr, ErrRequeue):
|
|
// A short sleep avoids a tight redelivery loop while a
|
|
// dependency is down. The lease/outbox still own the job.
|
|
time.Sleep(time.Second)
|
|
if err := delivery.Nack(false, true); err != nil {
|
|
c.logger.Error("nack failed", "job_id", job.JobID, "err", err)
|
|
}
|
|
default:
|
|
c.logger.Error("job handler failed", "job_id", job.JobID, "err", handlerErr)
|
|
_ = delivery.Ack(false)
|
|
}
|
|
}(delivery, job)
|
|
}
|
|
}
|
|
}
|