Добавить версии pipeline и защиту от устаревших попыток
This commit is contained in:
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
@@ -14,7 +15,7 @@ import (
|
||||
|
||||
func runJobAdmin(args []string) error {
|
||||
if len(args) < 1 {
|
||||
return errors.New("usage: jobs list-failed | replay JOB_ID REASON | cleanup-staging JOB_ID")
|
||||
return errors.New("usage: jobs list-failed | replay JOB_ID REASON | cleanup-staging JOB_ID | upgrade ATTACHMENT_ID 2 REASON")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
@@ -43,6 +44,18 @@ func runJobAdmin(args []string) error {
|
||||
}
|
||||
var changed bool
|
||||
switch {
|
||||
case args[0] == "upgrade" && len(args) == 4:
|
||||
version, e := strconv.Atoi(args[2])
|
||||
if e != nil {
|
||||
return errors.New("invalid pipeline version")
|
||||
}
|
||||
changed, err = store.UpgradePipeline(ctx, args[1], version, args[3], func(ctx context.Context, key string) error {
|
||||
r, e := objects.Get(ctx, key)
|
||||
if e == nil {
|
||||
r.Close()
|
||||
}
|
||||
return e
|
||||
})
|
||||
case args[0] == "replay" && len(args) == 3:
|
||||
changed, err = store.ReplayJob(ctx, args[1], args[2], func(ctx context.Context, key string) error {
|
||||
r, e := objects.Get(ctx, key)
|
||||
@@ -54,7 +67,7 @@ func runJobAdmin(args []string) error {
|
||||
case args[0] == "cleanup-staging" && len(args) == 2:
|
||||
changed, err = store.CleanupJobStaging(ctx, args[1], 24*time.Hour, false, objects.Delete)
|
||||
default:
|
||||
return errors.New("usage: jobs list-failed | replay JOB_ID REASON | cleanup-staging JOB_ID")
|
||||
return errors.New("usage: jobs list-failed | replay JOB_ID REASON | cleanup-staging JOB_ID | upgrade ATTACHMENT_ID 2 REASON")
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"github.com/google/uuid"
|
||||
"io"
|
||||
"log"
|
||||
"messenger/src/backend/repositories"
|
||||
"messenger/src/backend/services"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@@ -120,11 +121,7 @@ func attachmentHandler(attachments *services.AttachmentService) http.HandlerFunc
|
||||
found := false
|
||||
for _, raw := range a.Variants {
|
||||
if v, ok := raw.(map[string]any); ok && v["name"] == variants[0] {
|
||||
extension := ".jpg"
|
||||
if variants[0] == "optimized" {
|
||||
extension = ".webp"
|
||||
}
|
||||
a.ObjectKey = "variants/" + a.ID + "/v1/" + variants[0] + extension
|
||||
a.ObjectKey = repositories.VariantObjectKey(a.ID, a.PipelineVersion, variants[0])
|
||||
a.ContentType = v["content_type"].(string)
|
||||
a.SizeBytes = int64(v["size_bytes"].(float64))
|
||||
found = true
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
package faults
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
@@ -27,3 +29,36 @@ func Crash(point, id string) {
|
||||
os.Exit(86)
|
||||
}
|
||||
}
|
||||
|
||||
// Observe records a test-only branch without exposing an API or a lease token.
|
||||
func Observe(point, id string) {
|
||||
directory := os.Getenv("PIPELINE_FAULT_DIR")
|
||||
if directory == "" {
|
||||
return
|
||||
}
|
||||
if _, err := uuid.Parse(id); err != nil {
|
||||
return
|
||||
}
|
||||
_ = os.WriteFile(filepath.Join(directory, point+"."+id+".observed"), []byte("observed\n"), 0644)
|
||||
}
|
||||
|
||||
// Pause is operator-only and ends on release or context expiry.
|
||||
func Pause(ctx context.Context, point, id string) {
|
||||
if !Take(point, id) {
|
||||
return
|
||||
}
|
||||
release := filepath.Join(os.Getenv("PIPELINE_FAULT_DIR"), point+"."+id+".release")
|
||||
ticker := time.NewTicker(50 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if _, err := os.Stat(release); err == nil {
|
||||
_ = os.WriteFile(release+".observed", []byte("released\n"), 0644)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,19 +8,20 @@ import (
|
||||
)
|
||||
|
||||
type Attachment struct {
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
OwnerID string `json:"owner_id"`
|
||||
Filename string `json:"filename"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Status string `json:"status"`
|
||||
Variants []any `json:"variants"`
|
||||
ErrorCode *string `json:"error_code"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
ObjectKey string `json:"-"`
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
OwnerID string `json:"owner_id"`
|
||||
Filename string `json:"filename"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Status string `json:"status"`
|
||||
Variants []any `json:"variants"`
|
||||
ErrorCode *string `json:"error_code"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
ObjectKey string `json:"-"`
|
||||
PipelineVersion int `json:"-"`
|
||||
}
|
||||
|
||||
type AttachmentStore struct{ db *pgxpool.Pool }
|
||||
@@ -37,7 +38,7 @@ func (s *AttachmentStore) GetForMember(ctx context.Context, id, userID string) (
|
||||
defer cancel()
|
||||
a := Attachment{Variants: []any{}}
|
||||
var variants []byte
|
||||
err := s.db.QueryRow(ctx, `SELECT a.id,a.chat_id,a.owner_id,a.filename,a.object_key,a.content_type,a.size_bytes,a.sha256,a.created_at,a.updated_at,a.status,a.variants,a.error_code FROM attachments a JOIN chat_members cm ON cm.chat_id=a.chat_id WHERE a.id=$1 AND cm.user_id=$2`, id, userID).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ObjectKey, &a.ContentType, &a.SizeBytes, &a.SHA256, &a.CreatedAt, &a.UpdatedAt, &a.Status, &variants, &a.ErrorCode)
|
||||
err := s.db.QueryRow(ctx, `SELECT a.id,a.chat_id,a.owner_id,a.filename,a.object_key,a.content_type,a.size_bytes,a.sha256,a.created_at,a.updated_at,a.status,a.variants,a.error_code,a.published_pipeline_version FROM attachments a JOIN chat_members cm ON cm.chat_id=a.chat_id WHERE a.id=$1 AND cm.user_id=$2`, id, userID).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ObjectKey, &a.ContentType, &a.SizeBytes, &a.SHA256, &a.CreatedAt, &a.UpdatedAt, &a.Status, &variants, &a.ErrorCode, &a.PipelineVersion)
|
||||
if err == nil {
|
||||
err = json.Unmarshal(variants, &a.Variants)
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ type FailedJob struct {
|
||||
}
|
||||
|
||||
func (s *AttachmentStore) FailedJobs(ctx context.Context) ([]FailedJob, error) {
|
||||
rows, err := s.db.Query(ctx, `SELECT j.id,j.attachment_id,f.error_code,f.attempts,f.failed_at FROM attachment_job_failures f JOIN attachment_jobs j ON j.id=f.job_id JOIN attachments a ON a.id=j.attachment_id WHERE a.status='failed' ORDER BY f.failed_at DESC LIMIT 100`)
|
||||
rows, err := s.db.Query(ctx, `SELECT j.id,j.attachment_id,f.error_code,f.attempts,f.failed_at FROM attachment_job_failures f JOIN attachment_jobs j ON j.id=f.job_id JOIN attachments a ON a.id=j.attachment_id WHERE a.status='failed' AND j.pipeline_version=a.pipeline_version AND j.superseded_at IS NULL ORDER BY f.failed_at DESC LIMIT 100`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -45,12 +45,12 @@ func (s *AttachmentStore) ReplayJob(ctx context.Context, id, reason string, exis
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
var status, key, attachment string
|
||||
var deleted bool
|
||||
err = tx.QueryRow(ctx, `SELECT a.status,j.staging_key,a.id,j.staging_deleted_at IS NOT NULL FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1 FOR UPDATE OF j,a`, id).Scan(&status, &key, &attachment, &deleted)
|
||||
var deleted, obsolete bool
|
||||
err = tx.QueryRow(ctx, `SELECT a.status,j.staging_key,a.id,j.staging_deleted_at IS NOT NULL,j.pipeline_version<>a.pipeline_version OR j.superseded_at IS NOT NULL FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1 FOR UPDATE OF j,a`, id).Scan(&status, &key, &attachment, &deleted, &obsolete)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if status != "failed" {
|
||||
if obsolete || status != "failed" {
|
||||
return false, nil
|
||||
}
|
||||
if deleted {
|
||||
@@ -84,7 +84,7 @@ func (s *AttachmentStore) CleanupJobStaging(ctx context.Context, id string, rete
|
||||
defer tx.Rollback(context.Background())
|
||||
var key, attachment, status string
|
||||
var eligible bool
|
||||
err = tx.QueryRow(ctx, `SELECT j.staging_key,a.id,a.status,j.staging_deleted_at IS NULL AND (a.status='ready' AND jsonb_array_length(a.variants)>=3 OR a.status='failed' AND COALESCE(f.failed_at,a.updated_at)<now()-$2*interval '1 second') FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id LEFT JOIN attachment_job_failures f ON f.job_id=j.id WHERE j.id=$1 FOR UPDATE OF j,a SKIP LOCKED`, id, retention.Seconds()).Scan(&key, &attachment, &status, &eligible)
|
||||
err = tx.QueryRow(ctx, `SELECT j.staging_key,a.id,a.status,j.staging_deleted_at IS NULL AND (a.status='ready' AND jsonb_array_length(a.variants)>=3 OR a.status='failed' AND j.pipeline_version=a.pipeline_version AND COALESCE(f.failed_at,a.updated_at)<now()-$2*interval '1 second') FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id LEFT JOIN attachment_job_failures f ON f.job_id=j.id WHERE j.id=$1 FOR UPDATE OF j,a SKIP LOCKED`, id, retention.Seconds()).Scan(&key, &attachment, &status, &eligible)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return false, nil
|
||||
}
|
||||
@@ -110,7 +110,7 @@ func (s *AttachmentStore) CleanupJobStaging(ctx context.Context, id string, rete
|
||||
}
|
||||
|
||||
func (s *AttachmentStore) CleanupJobs(ctx context.Context, retention time.Duration, dryRun bool, remove func(context.Context, string) error) (int, error) {
|
||||
rows, err := s.db.Query(ctx, `SELECT j.id FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id LEFT JOIN attachment_job_failures f ON f.job_id=j.id WHERE j.staging_deleted_at IS NULL AND j.staging_key='staging/jobs/'||j.attachment_id::text AND (a.status='ready' AND jsonb_array_length(a.variants)>=3 OR a.status='failed' AND COALESCE(f.failed_at,a.updated_at)<now()-$1*interval '1 second') ORDER BY j.created_at LIMIT 100`, retention.Seconds())
|
||||
rows, err := s.db.Query(ctx, `SELECT j.id FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id LEFT JOIN attachment_job_failures f ON f.job_id=j.id WHERE j.staging_deleted_at IS NULL AND j.staging_key='staging/jobs/'||j.attachment_id::text AND (a.status='ready' AND jsonb_array_length(a.variants)>=3 OR a.status='failed' AND j.pipeline_version=a.pipeline_version AND COALESCE(f.failed_at,a.updated_at)<now()-$1*interval '1 second') ORDER BY j.created_at LIMIT 100`, retention.Seconds())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
@@ -32,7 +32,7 @@ type Job struct {
|
||||
// QueuedJobs is also the recovery source when publish/confirm is uncertain.
|
||||
// Re-publication is intentional: a durable intent is never removed by publish.
|
||||
func (s *AttachmentStore) QueuedJobs(ctx context.Context) ([]Job, error) {
|
||||
rows, err := s.db.Query(ctx, `SELECT j.id,j.attachment_id,j.pipeline_version,j.attempts+1,j.created_at FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE a.status IN ('queued','processing') AND j.next_attempt_at<=now() AND (j.lease_until IS NULL OR j.lease_until<now()) AND (j.published_at IS NULL OR j.published_at<now()-interval '30 seconds') ORDER BY j.created_at LIMIT 100`)
|
||||
rows, err := s.db.Query(ctx, `SELECT j.id,j.attachment_id,j.pipeline_version,j.attempts+1,j.created_at FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.superseded_at IS NULL AND j.pipeline_version=a.pipeline_version AND a.status IN ('queued','processing') AND j.next_attempt_at<=now() AND (j.lease_until IS NULL OR j.lease_until<now()) AND (j.published_at IS NULL OR j.published_at<now()-interval '30 seconds') ORDER BY j.created_at LIMIT 100`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -64,7 +64,7 @@ func (s *AttachmentStore) StartAttempt(ctx context.Context, j Job, timeout time.
|
||||
var status string
|
||||
var attempts, generation int
|
||||
var available bool
|
||||
err = tx.QueryRow(ctx, `SELECT a.status,j.attempts,j.generation,j.next_attempt_at<=now() AND (j.lease_until IS NULL OR j.lease_until<now()) FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1 AND j.attachment_id=$2 AND j.pipeline_version=$3 FOR UPDATE OF j,a SKIP LOCKED`, j.ID, j.AttachmentID, j.PipelineVersion).Scan(&status, &attempts, &generation, &available)
|
||||
err = tx.QueryRow(ctx, `SELECT a.status,j.attempts,j.generation,j.next_attempt_at<=now() AND (j.lease_until IS NULL OR j.lease_until<now()) FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1 AND j.attachment_id=$2 AND j.pipeline_version=$3 AND j.superseded_at IS NULL AND a.pipeline_version=j.pipeline_version FOR UPDATE OF j,a SKIP LOCKED`, j.ID, j.AttachmentID, j.PipelineVersion).Scan(&status, &attempts, &generation, &available)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return "", nil
|
||||
}
|
||||
@@ -98,6 +98,8 @@ func (s *AttachmentStore) StartAttempt(ctx context.Context, j Job, timeout time.
|
||||
return token, tx.Commit(ctx)
|
||||
}
|
||||
|
||||
var ErrStaleJob = errors.New("stale job lease or version")
|
||||
|
||||
var ErrProcessingTimeout = errors.New("processing timeout")
|
||||
|
||||
// ProcessJob holds the row lock across deterministic object writes. The lock and
|
||||
@@ -111,9 +113,9 @@ func (s *AttachmentStore) ProcessJob(ctx context.Context, j Job, token string, m
|
||||
var staging string
|
||||
var attempts, generation int
|
||||
var a Attachment
|
||||
err = tx.QueryRow(ctx, `SELECT a.id,a.object_key,a.content_type,a.size_bytes,a.sha256,a.status,j.staging_key,j.attempts,j.generation FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1 AND j.attachment_id=$2 AND j.pipeline_version=$3 AND j.lease_token=$4 AND j.lease_until>now() FOR UPDATE OF j,a`, j.ID, j.AttachmentID, j.PipelineVersion, token).Scan(&a.ID, &a.ObjectKey, &a.ContentType, &a.SizeBytes, &a.SHA256, &a.Status, &staging, &attempts, &generation)
|
||||
err = tx.QueryRow(ctx, `SELECT a.id,a.object_key,a.content_type,a.size_bytes,a.sha256,a.status,j.staging_key,j.attempts,j.generation FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1 AND j.attachment_id=$2 AND j.pipeline_version=$3 AND j.lease_token=$4 AND j.lease_until>now() AND j.superseded_at IS NULL AND a.pipeline_version=j.pipeline_version FOR UPDATE OF j,a`, j.ID, j.AttachmentID, j.PipelineVersion, token).Scan(&a.ID, &a.ObjectKey, &a.ContentType, &a.SizeBytes, &a.SHA256, &a.Status, &staging, &attempts, &generation)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return errors.New("stale job lease")
|
||||
return ErrStaleJob
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -121,6 +123,7 @@ func (s *AttachmentStore) ProcessJob(ctx context.Context, j Job, token string, m
|
||||
if a.Status != "processing" {
|
||||
return errors.New("job is not processing")
|
||||
}
|
||||
a.PipelineVersion = j.PipelineVersion
|
||||
variants, code, workErr := process(a, staging)
|
||||
var next *time.Time
|
||||
var auditCode *string
|
||||
@@ -129,7 +132,10 @@ func (s *AttachmentStore) ProcessJob(ctx context.Context, j Job, token string, m
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
_, err = tx.Exec(ctx, `UPDATE attachments SET status='ready',variants=$2,error_code=NULL,updated_at=now() WHERE id=$1`, a.ID, data)
|
||||
_, err = tx.Exec(ctx, `UPDATE attachments SET status='ready',variants=$2,published_pipeline_version=$3,error_code=NULL,updated_at=now() WHERE id=$1 AND pipeline_version=$3`, a.ID, data, j.PipelineVersion)
|
||||
if err == nil {
|
||||
_, err = tx.Exec(ctx, `INSERT INTO attachment_publications(attachment_id,job_id,pipeline_version,variants) VALUES($1,$2,$3,$4)`, a.ID, j.ID, j.PipelineVersion, data)
|
||||
}
|
||||
} else {
|
||||
reason := "temporary_failure"
|
||||
if errors.Is(workErr, ErrProcessingTimeout) {
|
||||
@@ -202,6 +208,6 @@ func enqueueJob(ctx context.Context, tx pgx.Tx, a Attachment, staging string) er
|
||||
|
||||
func (s *AttachmentStore) JobTerminal(ctx context.Context, id string) (bool, error) {
|
||||
var done bool
|
||||
err := s.db.QueryRow(ctx, `SELECT a.status IN ('ready','failed') FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1`, id).Scan(&done)
|
||||
err := s.db.QueryRow(ctx, `SELECT j.superseded_at IS NOT NULL OR j.pipeline_version<a.pipeline_version OR a.status IN ('ready','failed') FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1`, id).Scan(&done)
|
||||
return done, err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
-- Downgrade is intentionally refused if multiple versions exist: never drop jobs.
|
||||
DO $$ BEGIN
|
||||
IF EXISTS(SELECT 1 FROM attachment_jobs WHERE pipeline_version<>1) THEN
|
||||
RAISE EXCEPTION 'cannot downgrade while versioned jobs exist';
|
||||
END IF;
|
||||
END $$;
|
||||
DROP TABLE attachment_publications,attachment_pipeline_upgrades;
|
||||
ALTER TABLE attachment_jobs DROP CONSTRAINT attachment_jobs_version_key;
|
||||
ALTER TABLE attachment_jobs ADD CONSTRAINT attachment_jobs_attachment_id_key UNIQUE(attachment_id);
|
||||
ALTER TABLE attachment_jobs DROP COLUMN superseded_at;
|
||||
ALTER TABLE attachments DROP COLUMN pipeline_version,DROP COLUMN published_pipeline_version;
|
||||
@@ -0,0 +1,19 @@
|
||||
ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK(pipeline_version BETWEEN 1 AND 2);
|
||||
ALTER TABLE attachments ADD COLUMN published_pipeline_version integer NOT NULL DEFAULT 0 CHECK(published_pipeline_version BETWEEN 0 AND 2);
|
||||
UPDATE attachments SET published_pipeline_version=1 WHERE status='ready';
|
||||
ALTER TABLE attachment_jobs DROP CONSTRAINT attachment_jobs_attachment_id_key;
|
||||
ALTER TABLE attachment_jobs ADD CONSTRAINT attachment_jobs_version_key UNIQUE(attachment_id,pipeline_version);
|
||||
ALTER TABLE attachment_jobs ADD COLUMN superseded_at timestamptz;
|
||||
CREATE TABLE attachment_pipeline_upgrades (
|
||||
id bigserial PRIMARY KEY, attachment_id uuid NOT NULL REFERENCES attachments(id),
|
||||
from_version integer NOT NULL,to_version integer NOT NULL,reason text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
CREATE TABLE attachment_publications (
|
||||
id bigserial PRIMARY KEY,attachment_id uuid NOT NULL REFERENCES attachments(id),
|
||||
job_id uuid NOT NULL REFERENCES attachment_jobs(id),pipeline_version integer NOT NULL,
|
||||
variants jsonb NOT NULL,published_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
INSERT INTO attachment_publications(attachment_id,job_id,pipeline_version,variants)
|
||||
SELECT a.id,j.id,1,a.variants FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id
|
||||
WHERE a.status='ready';
|
||||
@@ -0,0 +1,74 @@
|
||||
package repositories
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/google/uuid"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// VariantObjectKey separates effects of different pipelines, including retries.
|
||||
func VariantObjectKey(id string, version int, name string) string {
|
||||
if version < 1 {
|
||||
version = 1
|
||||
}
|
||||
extension := ".jpg"
|
||||
if name == "optimized" {
|
||||
extension = ".webp"
|
||||
}
|
||||
return fmt.Sprintf("variants/%s/v%d/%s%s", id, version, name, extension)
|
||||
}
|
||||
|
||||
// UpgradePipeline is an explicit operator action. It never downgrades a result.
|
||||
// The row lock serializes this change with publication, replay and cleanup.
|
||||
func (s *AttachmentStore) UpgradePipeline(ctx context.Context, id string, version int, reason string, exists func(context.Context, string) error) (bool, error) {
|
||||
reason = strings.TrimSpace(reason)
|
||||
if reason == "" || utf8.RuneCountInString(reason) > 200 {
|
||||
return false, errors.New("upgrade reason must contain 1-200 characters")
|
||||
}
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
var current int
|
||||
var status, original, source string
|
||||
err = tx.QueryRow(ctx, `SELECT a.pipeline_version,a.status,a.object_key,j.staging_key FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE a.id=$1 AND j.pipeline_version=a.pipeline_version FOR UPDATE OF j,a`, id).Scan(¤t, &status, &original, &source)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if version <= current {
|
||||
return false, nil
|
||||
}
|
||||
if version != current+1 || version > 2 {
|
||||
return false, errors.New("only sequential upgrade to pipeline 2 is supported")
|
||||
}
|
||||
|
||||
if status == "ready" {
|
||||
source = original
|
||||
}
|
||||
if source != "staging/jobs/"+id && source != "originals/"+id {
|
||||
return false, errors.New("invalid pipeline source")
|
||||
}
|
||||
if err = exists(ctx, source); err != nil {
|
||||
return false, errors.New("pipeline source unavailable")
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET superseded_at=clock_timestamp(),lease_token=NULL,lease_until=NULL WHERE attachment_id=$1 AND pipeline_version<$2`, id, version); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `UPDATE attachment_job_attempts SET finished_at=clock_timestamp(),error_code='superseded' WHERE job_id IN (SELECT id FROM attachment_jobs WHERE attachment_id=$1 AND pipeline_version<$2) AND finished_at IS NULL`, id, version); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `INSERT INTO attachment_jobs(id,attachment_id,staging_key,pipeline_version) VALUES($1,$2,$3,$4)`, uuid.NewString(), id, source, version); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `UPDATE attachments SET pipeline_version=$2,status='queued',error_code=NULL,updated_at=clock_timestamp() WHERE id=$1`, id, version); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `INSERT INTO attachment_pipeline_upgrades(attachment_id,from_version,to_version,reason) VALUES($1,$2,$3,$4)`, id, current, version, reason); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, tx.Commit(ctx)
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"github.com/google/uuid"
|
||||
"messenger/src/backend/repositories"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestPipelineUpgradeFencesOldLeaseAndPublishesV2(t *testing.T) {
|
||||
f := newTestFixture(t)
|
||||
ctx := context.Background()
|
||||
store := repositories.NewAttachmentStore(f.db)
|
||||
user, _ := createTestUser(t, f.users, "versions", "Versions")
|
||||
chat := createTestChat(t, f.chats, "Versions", []string{user.ID})
|
||||
id, oldID := uuid.NewString(), uuid.NewString()
|
||||
if err := store.Create(ctx, repositories.Attachment{ID: id, ChatID: chat.ID, OwnerID: user.ID, Filename: "v.png", ObjectKey: "originals/" + id, ContentType: "image/png", SizeBytes: 1, SHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", CreatedAt: time.Now(), UpdatedAt: time.Now()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := f.db.Exec(ctx, `UPDATE attachments SET status='queued' WHERE id=$1`, id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := f.db.Exec(ctx, `INSERT INTO attachment_jobs(id,attachment_id,staging_key) VALUES($1,$2,$3)`, oldID, id, "staging/jobs/"+id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
old := repositories.Job{ID: oldID, AttachmentID: id, PipelineVersion: 1}
|
||||
token, err := store.StartAttempt(ctx, old, time.Minute, 5)
|
||||
if err != nil || token == "" {
|
||||
t.Fatal(err)
|
||||
}
|
||||
changed, err := store.UpgradePipeline(ctx, id, 2, "unit test upgrade", func(context.Context, string) error { return nil })
|
||||
if err != nil || !changed {
|
||||
t.Fatal(err)
|
||||
}
|
||||
effects := 0
|
||||
callback := func(a repositories.Attachment, _ string) ([]repositories.Variant, string, error) {
|
||||
effects++
|
||||
if a.PipelineVersion != 2 {
|
||||
t.Fatal("old version reached effects")
|
||||
}
|
||||
return []repositories.Variant{{Name: "thumbnail"}, {Name: "crop"}, {Name: "optimized"}}, "", nil
|
||||
}
|
||||
if err = store.ProcessJob(ctx, old, token, 5, callback); !errors.Is(err, repositories.ErrStaleJob) || effects != 0 {
|
||||
t.Fatal("old lease not fenced", err)
|
||||
}
|
||||
if terminal, err := store.JobTerminal(ctx, oldID); err != nil || !terminal {
|
||||
t.Fatal("superseded delivery not terminal", err)
|
||||
}
|
||||
var newID string
|
||||
if err = f.db.QueryRow(ctx, `SELECT id FROM attachment_jobs WHERE attachment_id=$1 AND pipeline_version=2`, id).Scan(&newID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
next := repositories.Job{ID: newID, AttachmentID: id, PipelineVersion: 2}
|
||||
token, err = store.StartAttempt(ctx, next, time.Minute, 5)
|
||||
if err != nil || token == "" {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = store.ProcessJob(ctx, next, token, 5, callback); err != nil || effects != 1 {
|
||||
t.Fatal(err)
|
||||
}
|
||||
published, err := store.GetForMember(ctx, id, user.ID)
|
||||
if err != nil || published.Status != "ready" || published.PipelineVersion != 2 {
|
||||
t.Fatal("wrong active publication", err)
|
||||
}
|
||||
if changed, err = store.UpgradePipeline(ctx, id, 1, "no downgrade", nil); err != nil || changed {
|
||||
t.Fatal("version rollback allowed", err)
|
||||
}
|
||||
if changed, err = store.ReplayJob(ctx, oldID, "obsolete replay", nil); err != nil || changed {
|
||||
t.Fatal("old job reopened", err)
|
||||
}
|
||||
var count int
|
||||
if err = f.db.QueryRow(ctx, `SELECT count(*) FROM attachment_publications WHERE attachment_id=$1 AND pipeline_version=2`, id).Scan(&count); err != nil || count != 1 {
|
||||
t.Fatal("publication count", count, err)
|
||||
}
|
||||
}
|
||||
@@ -121,6 +121,13 @@ func newTestFixture(t *testing.T) *testFixture {
|
||||
if _, err = db.Exec(ctx, string(ddl)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ddl, err = os.ReadFile("../repositories/migrations/000009_pipeline_versions.up.sql")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = db.Exec(ctx, string(ddl)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = redisClient.Close() })
|
||||
if err := redisClient.Ping(ctx).Err(); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -15,7 +15,7 @@ import (
|
||||
|
||||
func processImage(ctx context.Context, objects services.AttachmentObjects, a repositories.Attachment, staging string) ([]repositories.Variant, string, error) {
|
||||
return publishImage(ctx, objects, a, staging, func(ctx context.Context, data []byte) ([]encodedVariant, string, error) {
|
||||
return renderInSubprocess(ctx, a.ID, data)
|
||||
return renderInSubprocess(ctx, a.ID, a.PipelineVersion, data)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -42,11 +42,7 @@ func publishImage(ctx context.Context, objects services.AttachmentObjects, a rep
|
||||
}
|
||||
metadata := []repositories.Variant{}
|
||||
for _, variant := range variants {
|
||||
extension := ".jpg"
|
||||
if variant.Metadata.Name == "optimized" {
|
||||
extension = ".webp"
|
||||
}
|
||||
key := "variants/" + a.ID + "/v1/" + variant.Metadata.Name + extension
|
||||
key := repositories.VariantObjectKey(a.ID, a.PipelineVersion, variant.Metadata.Name)
|
||||
if err = objects.Put(ctx, key, bytes.NewReader(variant.Data), int64(len(variant.Data)), variant.Metadata.ContentType); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
@@ -158,3 +158,44 @@ func TestOptimizedBoundsAndCenteredCrop(t *testing.T) {
|
||||
t.Fatal("crop did not preserve the green center", pixel)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVersionedRenderingHasDistinctImmutableResults(t *testing.T) {
|
||||
if _, err := exec.LookPath("cwebp"); err != nil {
|
||||
t.Skip("cwebp: use Docker builder tests")
|
||||
}
|
||||
imageData := image.NewNRGBA(image.Rect(0, 0, 48, 32))
|
||||
for y := 0; y < 32; y++ {
|
||||
for x := 0; x < 48; x++ {
|
||||
imageData.SetNRGBA(x, y, color.NRGBA{R: uint8(x*17 + y), G: uint8(y*31 + x), B: uint8(x * y * 3), A: 255})
|
||||
}
|
||||
}
|
||||
var input bytes.Buffer
|
||||
if err := png.Encode(&input, imageData); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
original := append([]byte{}, input.Bytes()...)
|
||||
first, _, err := renderVersion(context.Background(), input.Bytes(), 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, _, err := renderVersion(context.Background(), input.Bytes(), 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(first) != 3 || len(second) != 3 {
|
||||
t.Fatal("incomplete version")
|
||||
}
|
||||
changed := false
|
||||
for i := range first {
|
||||
if first[i].Metadata.Width != second[i].Metadata.Width || first[i].Metadata.Height != second[i].Metadata.Height {
|
||||
t.Fatal("version changed geometry")
|
||||
}
|
||||
changed = changed || first[i].Metadata.SHA256 != second[i].Metadata.SHA256
|
||||
if repositories.VariantObjectKey("sample", 1, first[i].Metadata.Name) == repositories.VariantObjectKey("sample", 2, second[i].Metadata.Name) {
|
||||
t.Fatal("versions share an object key")
|
||||
}
|
||||
}
|
||||
if !changed || !bytes.Equal(original, input.Bytes()) {
|
||||
t.Fatal("versions indistinguishable or source mutated")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"messenger/src/backend/internal/faults"
|
||||
@@ -24,7 +25,7 @@ func (b *boundedOutput) Write(data []byte) (int, error) {
|
||||
return b.Buffer.Write(data)
|
||||
}
|
||||
|
||||
func renderInSubprocess(ctx context.Context, id string, data []byte) ([]encodedVariant, string, error) {
|
||||
func renderInSubprocess(ctx context.Context, id string, version int, data []byte) ([]encodedVariant, string, error) {
|
||||
seconds, err := positiveEnv("JOB_TIMEOUT_SECONDS", 60)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
@@ -41,7 +42,10 @@ func renderInSubprocess(ctx context.Context, id string, data []byte) ([]encodedV
|
||||
return nil, "", err
|
||||
}
|
||||
defer os.RemoveAll(directory)
|
||||
command := exec.CommandContext(task, executable, "image-process", id)
|
||||
if version < 1 {
|
||||
version = 1
|
||||
}
|
||||
command := exec.CommandContext(task, executable, "image-process", id, strconv.Itoa(version))
|
||||
// No database, broker or S3 credentials are needed by a decoder.
|
||||
command.Env = []string{"PATH=/usr/bin:/bin", "GOMAXPROCS=1", "GOMEMLIMIT=256MiB", "WORKER_MAX_PIXELS=" + workerPixelLimit(), "PIPELINE_FAULT_DIR=" + os.Getenv("PIPELINE_FAULT_DIR")}
|
||||
command.Env = append(command.Env, "TMPDIR="+directory)
|
||||
@@ -77,7 +81,7 @@ func workerPixelLimit() string {
|
||||
}
|
||||
|
||||
func RunImageChild(args []string) error {
|
||||
if len(args) != 1 {
|
||||
if len(args) < 1 || len(args) > 2 {
|
||||
return errors.New("image id required")
|
||||
}
|
||||
if faults.Take("transform_delay", args[0]) {
|
||||
@@ -90,7 +94,14 @@ func RunImageChild(args []string) error {
|
||||
if len(data) > int(services.MaxUploadBytes) {
|
||||
return json.NewEncoder(os.Stdout).Encode(imageResult{ErrorCode: "invalid_image"})
|
||||
}
|
||||
variants, code, err := renderImage(context.Background(), data)
|
||||
version := 1
|
||||
if len(args) == 2 {
|
||||
version, err = strconv.Atoi(args[1])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
variants, code, err := renderVersion(context.Background(), data, version)
|
||||
if err != nil && code == "" {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -53,6 +53,16 @@ func opaque(source image.Image) image.Image {
|
||||
// renderImage runs only in the disposable image child in production. DecodeConfig
|
||||
// bounds pixels before allocation; the container bounds total memory to 512 MiB.
|
||||
func renderImage(ctx context.Context, data []byte) ([]encodedVariant, string, error) {
|
||||
return renderVersion(ctx, data, 1)
|
||||
}
|
||||
|
||||
func renderVersion(ctx context.Context, data []byte, version int) ([]encodedVariant, string, error) {
|
||||
jpegQuality, webpQuality := 85, "80"
|
||||
if version == 2 {
|
||||
jpegQuality, webpQuality = 75, "65"
|
||||
} else if version != 1 {
|
||||
return nil, "invalid_pipeline", errors.New("unsupported pipeline")
|
||||
}
|
||||
pixels, err := strconv.Atoi(workerPixelLimit())
|
||||
if err != nil || pixels < 1 || int64(pixels) > services.MaxImagePixels {
|
||||
return nil, "", errors.New("invalid WORKER_MAX_PIXELS")
|
||||
@@ -73,13 +83,13 @@ func renderImage(ctx context.Context, data []byte) ([]encodedVariant, string, er
|
||||
w, h := fitSize(source.Bounds().Dx(), source.Bounds().Dy(), 256)
|
||||
thumbnail := opaque(imaging.Resize(source, w, h, imaging.Lanczos))
|
||||
var body bytes.Buffer
|
||||
if err = jpeg.Encode(&body, thumbnail, &jpeg.Options{Quality: 85}); err != nil {
|
||||
if err = jpeg.Encode(&body, thumbnail, &jpeg.Options{Quality: jpegQuality}); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
appendVariant("thumbnail", "image/jpeg", thumbnail, body.Bytes())
|
||||
crop := opaque(imaging.Fill(source, 128, 128, imaging.Center, imaging.Lanczos))
|
||||
body = bytes.Buffer{}
|
||||
if err = jpeg.Encode(&body, crop, &jpeg.Options{Quality: 85}); err != nil {
|
||||
if err = jpeg.Encode(&body, crop, &jpeg.Options{Quality: jpegQuality}); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
appendVariant("crop", "image/jpeg", crop, body.Bytes())
|
||||
@@ -103,7 +113,7 @@ func renderImage(ctx context.Context, data []byte) ([]encodedVariant, string, er
|
||||
if closeErr != nil {
|
||||
return nil, "", closeErr
|
||||
}
|
||||
command := exec.CommandContext(ctx, "cwebp", "-quiet", "-q", "80", "-m", "4", "-metadata", "none", input, "-o", "-")
|
||||
command := exec.CommandContext(ctx, "cwebp", "-quiet", "-q", webpQuality, "-m", "4", "-metadata", "none", input, "-o", "-")
|
||||
webp, err := command.Output()
|
||||
if err != nil {
|
||||
return nil, "", errors.New("WebP encoding failed")
|
||||
|
||||
@@ -134,13 +134,18 @@ func consumeJobs(ctx context.Context, store *repositories.AttachmentStore, objec
|
||||
task, cancel := context.WithTimeout(ctx, timeout+15*time.Second)
|
||||
token, workErr := store.StartAttempt(task, job, timeout+15*time.Second, maxAttempts)
|
||||
if workErr == nil && token != "" {
|
||||
faults.Crash("before_effect", job.AttachmentID)
|
||||
faults.Pause(task, "after_claim", job.AttachmentID)
|
||||
workErr = store.ProcessJob(task, job, token, maxAttempts, func(a repositories.Attachment, staging string) ([]repositories.Variant, string, error) {
|
||||
return processImage(task, objects, a, staging)
|
||||
})
|
||||
}
|
||||
cancel()
|
||||
if errors.Is(workErr, repositories.ErrStaleJob) {
|
||||
faults.Observe("stale_rejected", job.AttachmentID)
|
||||
}
|
||||
completed := token != "" && workErr == nil
|
||||
if workErr == nil && token == "" {
|
||||
if workErr == nil && token == "" || errors.Is(workErr, repositories.ErrStaleJob) {
|
||||
completed, workErr = store.JobTerminal(ctx, job.ID)
|
||||
}
|
||||
// Ack after COMMIT of ready/failed or durable retry intent, or terminal no-op.
|
||||
|
||||
Reference in New Issue
Block a user