156 lines
3.8 KiB
Go
156 lines
3.8 KiB
Go
// Command worker runs image consumers or an explicit operator action.
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
|
|
"log/slog"
|
|
"os"
|
|
"os/signal"
|
|
|
|
"strconv"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
|
|
"messenger/internal/attachment"
|
|
"messenger/internal/broker"
|
|
"messenger/internal/imaging"
|
|
"messenger/internal/pipeline"
|
|
"messenger/internal/platform/config"
|
|
"messenger/internal/platform/postgres"
|
|
"messenger/internal/platform/s3store"
|
|
"messenger/internal/processing"
|
|
)
|
|
|
|
func main() {
|
|
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
|
|
defer stop()
|
|
if err := run(ctx, os.Args[1:]); err != nil {
|
|
slog.Error("worker stopped", "error", err)
|
|
os.Exit(1)
|
|
}
|
|
}
|
|
|
|
func run(ctx context.Context, args []string) error {
|
|
if len(args) == 1 && args[0] == "--prepare-large-png" {
|
|
return imaging.PrepareLargePNG(os.Stdin)
|
|
}
|
|
db, err := config.LoadDatabaseURL(os.LookupEnv)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
cfg, err := config.LoadJobs(os.LookupEnv)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
s3, err := config.LoadS3(os.LookupEnv)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
blobs, err := s3store.New(s3)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
pool, err := postgres.NewPool(ctx, db, 5)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer pool.Close()
|
|
store := pipeline.NewStore(pool)
|
|
w := processing.Worker{Store: store, Blobs: blobs, Config: cfg, Log: slog.Default()}
|
|
if len(args) == 1 && strings.HasPrefix(args[0], "--barriers=") {
|
|
dir := strings.TrimPrefix(args[0], "--barriers=")
|
|
if err := os.MkdirAll(dir, 0700); err != nil {
|
|
return err
|
|
}
|
|
w.Checkpoint = processing.FileBarrier(dir)
|
|
args = nil
|
|
}
|
|
if len(args) == 0 {
|
|
w.Run(ctx)
|
|
return nil
|
|
}
|
|
op, cancel := context.WithTimeout(ctx, 15*time.Second)
|
|
defer cancel()
|
|
if len(args) == 1 && args[0] == "failed" {
|
|
jobs, err := store.Failed(op)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return json.NewEncoder(os.Stdout).Encode(jobs)
|
|
}
|
|
const usage = "use failed, replay <id> <operator>, reprocess <id> <version>, or duplicate <id> <version> <attempt>"
|
|
if len(args) < 3 {
|
|
return errors.New(usage)
|
|
}
|
|
id, err := uuid.Parse(args[1])
|
|
if err != nil {
|
|
return errors.New("invalid attachment UUID")
|
|
}
|
|
exists := func(ctx context.Context, key string) (bool, error) {
|
|
_, err := blobs.Stat(ctx, key)
|
|
if errors.Is(err, attachment.ErrObjectMissing) {
|
|
return false, nil
|
|
}
|
|
return err == nil, err
|
|
}
|
|
switch args[0] {
|
|
case "replay":
|
|
if len(args) != 3 {
|
|
return errors.New(usage)
|
|
}
|
|
return store.Replay(op, id, args[2], exists)
|
|
case "reprocess":
|
|
if len(args) != 3 {
|
|
return errors.New(usage)
|
|
}
|
|
v, err := strconv.Atoi(args[2])
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return store.Reprocess(op, id, v, exists)
|
|
case "duplicate":
|
|
if len(args) != 4 {
|
|
return errors.New(usage)
|
|
}
|
|
version, err := strconv.Atoi(args[2])
|
|
if err != nil {
|
|
return err
|
|
}
|
|
attempt, err := strconv.Atoi(args[3])
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return duplicate(op, pool, cfg.URL, id, version, attempt)
|
|
default:
|
|
return errors.New(usage)
|
|
}
|
|
}
|
|
|
|
// duplicate sends a previously issued envelope through the actual broker. It
|
|
// cannot change SQL state or manufacture a future attempt.
|
|
func duplicate(ctx context.Context, pool *pgxpool.Pool, url string, id uuid.UUID, version, attempt int) error {
|
|
j := pipeline.Job{SchemaVersion: 1, Type: "attachment.process.v1", AttachmentID: id, PipelineVersion: version, Attempt: attempt}
|
|
err := pool.QueryRow(ctx, `SELECT job_id,created_at FROM attachment_jobs
|
|
WHERE attachment_id=$1 AND pipeline_version=$2 AND $3::integer BETWEEN 1 AND attempt`, id, version, attempt).Scan(&j.JobID, &j.CreatedAt)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
p, err := broker.NewPublisher(ctx, url)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer p.Close()
|
|
body, err := json.Marshal(j)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return p.Publish(ctx, body)
|
|
}
|