forked from mtusi2026/mtusi2026lab5
376 lines
11 KiB
Go
376 lines
11 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"image"
|
|
|
|
_ "image/jpeg"
|
|
_ "image/png"
|
|
"io"
|
|
"log"
|
|
"math/rand/v2"
|
|
"os"
|
|
"os/exec"
|
|
"os/signal"
|
|
"strconv"
|
|
"sync"
|
|
"syscall"
|
|
"time"
|
|
|
|
"errors"
|
|
"gitflic.ru/project/relationskatie/mtusi2026lab1-ashchkl/internal/config"
|
|
"gitflic.ru/project/relationskatie/mtusi2026lab1-ashchkl/internal/domain"
|
|
"gitflic.ru/project/relationskatie/mtusi2026lab1-ashchkl/internal/objectstore"
|
|
"gitflic.ru/project/relationskatie/mtusi2026lab1-ashchkl/internal/pipeline"
|
|
"gitflic.ru/project/relationskatie/mtusi2026lab1-ashchkl/internal/store/db"
|
|
"github.com/google/uuid"
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
"github.com/redis/go-redis/v9"
|
|
)
|
|
|
|
func main() {
|
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
|
defer stop()
|
|
cfg, err := config.Load()
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
pool, err := db.Connect(ctx, cfg.Database)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
defer pool.Close()
|
|
objects, err := objectstore.NewS3(cfg.S3)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
if len(os.Args) == 4 && os.Args[1] == "render" {
|
|
conn, e := pool.Acquire(ctx)
|
|
if e != nil {
|
|
os.Exit(1)
|
|
}
|
|
defer conn.Release()
|
|
attempt, _ := strconv.Atoi(os.Args[3])
|
|
e = render(ctx, conn, objects, os.Args[2], attempt)
|
|
if e != nil {
|
|
var pe permanentError
|
|
if errors.As(e, &pe) {
|
|
os.Exit(42)
|
|
}
|
|
os.Exit(1)
|
|
}
|
|
return
|
|
}
|
|
if len(os.Args) == 3 && os.Args[1] == "replay" {
|
|
if err := replay(ctx, pool, objects, os.Args[2]); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
log.Print("replay queued")
|
|
return
|
|
}
|
|
if len(os.Args) == 2 && os.Args[1] == "failed" {
|
|
rows, e := pool.Query(ctx, `SELECT id::text,error_code,attempts FROM attachments WHERE status='failed' ORDER BY updated_at DESC LIMIT 100`)
|
|
if e != nil {
|
|
log.Fatal(e)
|
|
}
|
|
defer rows.Close()
|
|
for rows.Next() {
|
|
var id, code string
|
|
var n int
|
|
if e = rows.Scan(&id, &code, &n); e != nil {
|
|
log.Fatal(e)
|
|
}
|
|
fmt.Println(id, code, n)
|
|
}
|
|
if rows.Err() != nil {
|
|
log.Fatal(rows.Err())
|
|
}
|
|
return
|
|
}
|
|
go pipeline.Dispatch(ctx, pool)
|
|
c := pipeline.Client()
|
|
defer c.Close()
|
|
maxAttempts := 5
|
|
if s := os.Getenv("JOB_MAX_ATTEMPTS"); s != "" {
|
|
n, e := strconv.Atoi(s)
|
|
if e != nil || n < 1 {
|
|
log.Fatal("invalid JOB_MAX_ATTEMPTS")
|
|
}
|
|
maxAttempts = n
|
|
}
|
|
// Один worker держит сессионную блокировку PostgreSQL. После его смерти новый
|
|
// worker возвращает неподтверждённые сообщения в очередь.
|
|
lock, err := pool.Acquire(ctx)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
defer lock.Release()
|
|
if _, err = lock.Exec(ctx, "SELECT pg_advisory_lock(505001)"); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
for ctx.Err() == nil {
|
|
_, err = c.RPopLPush(ctx, pipeline.Active, pipeline.Pending).Result()
|
|
if errors.Is(err, redis.Nil) {
|
|
break
|
|
}
|
|
if err != nil {
|
|
log.Fatal("queue recovery failed")
|
|
}
|
|
}
|
|
concurrency := positiveEnv("WORKER_CONCURRENCY", 1, 8)
|
|
prefetch := positiveEnv("WORKER_PREFETCH", concurrency, 64)
|
|
if prefetch < concurrency {
|
|
log.Fatal("WORKER_PREFETCH must be >= WORKER_CONCURRENCY")
|
|
}
|
|
if int(cfg.Database.MaxConns) < concurrency+2 {
|
|
log.Fatal("DB_MAX_CONNS must be >= WORKER_CONCURRENCY+2")
|
|
}
|
|
slots := make(chan struct{}, prefetch)
|
|
jobs := make(chan string, prefetch)
|
|
var workers sync.WaitGroup
|
|
for i := 0; i < concurrency; i++ {
|
|
workers.Add(1)
|
|
go func() {
|
|
defer workers.Done()
|
|
for raw := range jobs {
|
|
func() {
|
|
defer func() { <-slots }()
|
|
conn, e := pool.Acquire(ctx)
|
|
if e != nil {
|
|
return
|
|
} // Active entry is recovered on restart; outbox also retries.
|
|
defer conn.Release()
|
|
var job pipeline.Job
|
|
if json.Unmarshal([]byte(raw), &job) != nil || !validJob(job) {
|
|
log.Print("invalid job")
|
|
if err := c.Eval(ctx, `redis.call("LPUSH", KEYS[2], ARGV[1]); return redis.call("LREM", KEYS[1], 1, ARGV[1])`, []string{pipeline.Active, "attachments:invalid"}, raw).Err(); err != nil {
|
|
return
|
|
}
|
|
return
|
|
}
|
|
for ctx.Err() == nil {
|
|
done, err := process(ctx, conn, objects, job.AttachmentID, maxAttempts)
|
|
if err != nil {
|
|
log.Printf("job %s: %v", job.JobID, err)
|
|
if conn.Conn().IsClosed() {
|
|
log.Fatal("worker database session lost; restart required")
|
|
}
|
|
}
|
|
if done {
|
|
if os.Getenv("FAULT_BEFORE_ACK") == "1" {
|
|
faultOnce("/tmp/fault-before-ack")
|
|
}
|
|
if err = c.LRem(ctx, pipeline.Active, 1, raw).Err(); err != nil {
|
|
log.Fatal("queue ack failed; restart required")
|
|
}
|
|
break
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-time.After(time.Second):
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
}()
|
|
}
|
|
defer func() { close(jobs); workers.Wait() }()
|
|
for ctx.Err() == nil {
|
|
// The owner session alone holds the advisory lock and performs maintenance.
|
|
rows, e := lock.Query(ctx, `SELECT o.attachment_id::text FROM attachment_outbox o JOIN attachments a ON a.id=o.attachment_id WHERE a.status='ready' LIMIT 16`)
|
|
if e != nil && lock.Conn().IsClosed() {
|
|
log.Fatal("worker database session lost; restart required")
|
|
}
|
|
if e == nil {
|
|
var ids []string
|
|
for rows.Next() {
|
|
var id string
|
|
if rows.Scan(&id) == nil {
|
|
ids = append(ids, id)
|
|
}
|
|
}
|
|
rows.Close()
|
|
for _, id := range ids {
|
|
_ = cleanup(ctx, lock, objects, id)
|
|
}
|
|
}
|
|
select {
|
|
case slots <- struct{}{}:
|
|
case <-ctx.Done():
|
|
return
|
|
case <-time.After(200 * time.Millisecond):
|
|
continue
|
|
}
|
|
raw, err := c.BRPopLPush(ctx, pipeline.Pending, pipeline.Active, time.Second).Result()
|
|
if err != nil {
|
|
<-slots
|
|
continue
|
|
}
|
|
select {
|
|
case jobs <- raw:
|
|
case <-ctx.Done():
|
|
<-slots
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
func positiveEnv(name string, fallback, maximum int) int {
|
|
value := os.Getenv(name)
|
|
if value == "" {
|
|
return fallback
|
|
}
|
|
n, err := strconv.Atoi(value)
|
|
if err != nil || n < 1 || n > maximum {
|
|
log.Fatalf("invalid %s (1..%d)", name, maximum)
|
|
}
|
|
return n
|
|
}
|
|
|
|
// One owner with bounded concurrent render processes. A killed render subprocess cannot outlive its deadline.
|
|
func process(ctx context.Context, p *pgxpool.Conn, s objectstore.Store, id string, max int) (bool, error) {
|
|
var attempt int
|
|
err := p.QueryRow(ctx, `UPDATE attachments SET status='processing',attempts=attempts+1,
|
|
lease_until=now()+interval '60 seconds',processing_started_at=COALESCE(processing_started_at,now()),updated_at=now()
|
|
WHERE id=$1 AND attempts<$2 AND ((status='queued' AND next_attempt_at<=now()) OR
|
|
(status='processing' AND lease_until<now())) RETURNING attempts`, id, max).Scan(&attempt)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
_, err = p.Exec(ctx, `UPDATE attachments SET status='failed',error_code='retry_exhausted',lease_until=NULL,updated_at=now()
|
|
WHERE id=$1 AND attempts >= $2 AND (status='queued' OR (status='processing' AND lease_until<now()))`, id, max)
|
|
return err == nil, err
|
|
}
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
deadline, cancel := context.WithTimeout(ctx, 45*time.Second)
|
|
defer cancel()
|
|
executable, err := os.Executable()
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
cmd := exec.CommandContext(deadline, executable, "render", id, strconv.Itoa(attempt))
|
|
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true, Pdeathsig: syscall.SIGKILL}
|
|
cmd.Cancel = func() error { return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) }
|
|
cmd.WaitDelay = time.Second
|
|
err = cmd.Run()
|
|
if err == nil {
|
|
return true, nil
|
|
}
|
|
var ee *exec.ExitError
|
|
if errors.As(err, &ee) && ee.ExitCode() == 86 {
|
|
os.Exit(86)
|
|
}
|
|
code := "processing_error"
|
|
status := "queued"
|
|
if errors.As(err, &ee) && ee.ExitCode() == 42 {
|
|
code = "invalid_image"
|
|
status = "failed"
|
|
}
|
|
if attempt >= max {
|
|
code = "retry_exhausted"
|
|
status = "failed"
|
|
}
|
|
if deadline.Err() != nil {
|
|
code = "processing_timeout"
|
|
}
|
|
delay := retryDelay(attempt)
|
|
_, e := p.Exec(ctx, `UPDATE attachments SET status=$3,error_code=$4,lease_until=NULL,
|
|
next_attempt_at=now()+($5 * interval '1 millisecond'),updated_at=now()
|
|
WHERE id=$1 AND attempts=$2 AND status='processing'`, id, attempt, status, code, delay.Milliseconds())
|
|
// The outbox is still present. Ack only after durable retry/failed state.
|
|
return e == nil, e
|
|
}
|
|
func retryDelay(attempt int) time.Duration {
|
|
capMs := int64(1000) << min(max(attempt-1, 0), 6)
|
|
return time.Duration(capMs/2+rand.Int64N(capMs/2+1)) * time.Millisecond
|
|
}
|
|
|
|
type permanentError struct{ error }
|
|
|
|
func faultOnce(path string) {
|
|
f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
|
|
if err == nil {
|
|
f.Close()
|
|
os.Exit(86)
|
|
}
|
|
}
|
|
func render(ctx context.Context, p *pgxpool.Conn, s objectstore.Store, id string, attempt int) error {
|
|
var key, ct string
|
|
if err := p.QueryRow(ctx, `SELECT object_key,content_type FROM attachments WHERE id=$1 AND attempts=$2 AND status='processing'`, id, attempt).Scan(&key, &ct); err != nil {
|
|
return err
|
|
}
|
|
r, _, err := s.Get(ctx, key)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
data, err := io.ReadAll(io.LimitReader(r, 10485761))
|
|
r.Close()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(data) > 10485760 {
|
|
return permanentError{fmt.Errorf("image too large")}
|
|
}
|
|
cfg, _, err := image.DecodeConfig(bytes.NewReader(data))
|
|
if err != nil {
|
|
return permanentError{err}
|
|
}
|
|
if cfg.Width <= 0 || cfg.Height <= 0 || int64(cfg.Width)*int64(cfg.Height) > 40000000 {
|
|
return permanentError{fmt.Errorf("invalid image dimensions")}
|
|
}
|
|
im, _, err := image.Decode(bytes.NewReader(data))
|
|
if err != nil {
|
|
return permanentError{err}
|
|
}
|
|
im = orient(im, data)
|
|
outputs, err := variants(ctx, im)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
original := "originals/" + id
|
|
if err = s.Put(ctx, original, ct, bytes.NewReader(data), int64(len(data))); err != nil {
|
|
return err
|
|
}
|
|
var metadata []domain.Variant
|
|
for _, o := range outputs {
|
|
if err = s.Put(ctx, o.meta.Name+"s/"+id, o.meta.ContentType, bytes.NewReader(o.data), int64(len(o.data))); err != nil {
|
|
return err
|
|
}
|
|
metadata = append(metadata, o.meta)
|
|
}
|
|
if os.Getenv("FAULT_AFTER_OBJECTS") == "1" {
|
|
faultOnce("/tmp/fault-after-objects")
|
|
}
|
|
meta, _ := json.Marshal(metadata)
|
|
thumb, _ := json.Marshal(metadata[0])
|
|
result, err := p.Exec(ctx, `UPDATE attachments SET status='ready',object_key=$2,thumbnail=$3,variants=$4,
|
|
error_code=NULL,lease_until=NULL,updated_at=now() WHERE id=$1 AND attempts=$5 AND status='processing'`, id, original, thumb, meta, attempt)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if result.RowsAffected() != 1 {
|
|
return fmt.Errorf("lost lease")
|
|
}
|
|
// Terminal state first; cleanup is retried by the worker maintenance loop.
|
|
return cleanup(ctx, p, s, id)
|
|
}
|
|
func cleanup(ctx context.Context, p *pgxpool.Conn, s objectstore.Store, id string) error {
|
|
if err := s.Delete(ctx, "staging/"+id); err != nil {
|
|
return err
|
|
}
|
|
_, err := p.Exec(ctx, `DELETE FROM attachment_outbox WHERE attachment_id=$1 AND EXISTS(SELECT 1 FROM attachments WHERE id=$1 AND status='ready')`, id)
|
|
|
|
return err
|
|
}
|
|
|
|
func validJob(j pipeline.Job) bool {
|
|
_, e1 := uuid.Parse(j.JobID)
|
|
_, e2 := uuid.Parse(j.AttachmentID)
|
|
return e1 == nil && e2 == nil && j.SchemaVersion == 1 && j.PipelineVersion == 1 && j.Type == "attachment.process.v1" && j.Attempt >= 1 && !j.CreatedAt.IsZero()
|
|
}
|