5
This commit is contained in:
@@ -15,19 +15,22 @@ public record AttachmentJob(
|
||||
@JsonProperty("created_at") Instant createdAt) {
|
||||
|
||||
public static final int SCHEMA_VERSION = 1;
|
||||
public static final int PIPELINE_VERSION = 1;
|
||||
public static final int MIN_PIPELINE_VERSION = 1;
|
||||
public static final int CURRENT_PIPELINE_VERSION = 2;
|
||||
public static final String TYPE = "attachment.process.v1";
|
||||
|
||||
public AttachmentJob {
|
||||
if (schemaVersion != SCHEMA_VERSION || !TYPE.equals(type)
|
||||
|| pipelineVersion < 1 || attempt < 1) {
|
||||
|| pipelineVersion < MIN_PIPELINE_VERSION
|
||||
|| pipelineVersion > CURRENT_PIPELINE_VERSION || attempt < 1) {
|
||||
throw new IllegalArgumentException("Unsupported attachment job message");
|
||||
}
|
||||
}
|
||||
|
||||
public static AttachmentJob first(UUID jobId, UUID attachmentId, Instant createdAt) {
|
||||
public static AttachmentJob first(
|
||||
UUID jobId, UUID attachmentId, int pipelineVersion, Instant createdAt) {
|
||||
return new AttachmentJob(SCHEMA_VERSION, jobId, TYPE, attachmentId,
|
||||
PIPELINE_VERSION, 1, createdAt);
|
||||
pipelineVersion, 1, createdAt);
|
||||
}
|
||||
|
||||
public AttachmentJob nextAttempt(int nextAttempt) {
|
||||
|
||||
@@ -38,11 +38,13 @@ public class ExpiredLeaseReaper {
|
||||
try {
|
||||
transaction.executeWithoutResult(status -> {
|
||||
if (job.attempt() >= maxAttempts) {
|
||||
attachments.failExpired(job.jobId(), job.attachmentId(), "retry_exhausted");
|
||||
attachments.failExpired(job.jobId(), job.attachmentId(),
|
||||
job.pipelineVersion(), "retry_exhausted");
|
||||
} else {
|
||||
int nextAttempt = job.attempt() + 1;
|
||||
attachments.requeueExpired(
|
||||
job.jobId(), job.attachmentId(), nextAttempt, "processing_timeout",
|
||||
job.jobId(), job.attachmentId(), job.pipelineVersion(),
|
||||
nextAttempt, "processing_timeout",
|
||||
clock.instant().plus(retryPolicy.delayForAttempt(nextAttempt)));
|
||||
}
|
||||
});
|
||||
|
||||
@@ -21,16 +21,20 @@ public interface AttachmentRepository {
|
||||
boolean markPublishFaultInjected(UUID jobId);
|
||||
List<AttachmentJob> findUnpublishedJobs(int limit);
|
||||
int currentAttempt(UUID jobId);
|
||||
boolean claim(UUID jobId, UUID attachmentId, int attempt, UUID leaseToken, Duration leaseDuration);
|
||||
boolean markQueuedForRetry(UUID jobId, UUID attachmentId, UUID leaseToken,
|
||||
boolean claim(UUID jobId, UUID attachmentId, int pipelineVersion,
|
||||
int attempt, UUID leaseToken, Duration leaseDuration);
|
||||
boolean markBeforeEffectFaultInjected(UUID jobId, UUID leaseToken);
|
||||
boolean markAfterEffectFaultInjected(UUID jobId, UUID leaseToken);
|
||||
boolean markQueuedForRetry(UUID jobId, UUID attachmentId, int pipelineVersion, UUID leaseToken,
|
||||
int nextAttempt, String errorCode, Instant nextAttemptAt);
|
||||
CompletionResult markReady(UUID jobId, UUID attachmentId, UUID leaseToken, int pipelineVersion,
|
||||
List<AttachmentVariant> variants, boolean injectCrashAfterCommit);
|
||||
boolean markFailed(UUID jobId, UUID attachmentId, UUID leaseToken, String errorCode);
|
||||
boolean markFailed(UUID jobId, UUID attachmentId, int pipelineVersion,
|
||||
UUID leaseToken, String errorCode);
|
||||
List<AttachmentJob> findExpiredJobs(int limit);
|
||||
boolean requeueExpired(UUID jobId, UUID attachmentId, int nextAttempt,
|
||||
boolean requeueExpired(UUID jobId, UUID attachmentId, int pipelineVersion, int nextAttempt,
|
||||
String errorCode, Instant nextAttemptAt);
|
||||
boolean failExpired(UUID jobId, UUID attachmentId, String errorCode);
|
||||
boolean failExpired(UUID jobId, UUID attachmentId, int pipelineVersion, String errorCode);
|
||||
|
||||
record CompletionResult(boolean completed, boolean crashAfterCommit) {
|
||||
}
|
||||
|
||||
+83
-39
@@ -35,11 +35,13 @@ public class PostgresAttachmentRepository implements AttachmentRepository {
|
||||
jdbc.update("""
|
||||
INSERT INTO attachments
|
||||
(id, chat_id, owner_id, filename, content_type, size_bytes,
|
||||
sha256, object_key, staging_key, status, error_code, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'queued', NULL, ?, ?)
|
||||
sha256, object_key, staging_key, pipeline_version,
|
||||
status, error_code, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'queued', NULL, ?, ?)
|
||||
""", attachment.id(), attachment.chatId(), attachment.ownerId(),
|
||||
attachment.filename(), attachment.contentType(), attachment.sizeBytes(),
|
||||
attachment.sha256(), attachment.objectKey(), attachment.stagingKey(),
|
||||
job.pipelineVersion(),
|
||||
Timestamp.from(attachment.createdAt()), Timestamp.from(attachment.updatedAt()));
|
||||
jdbc.update("""
|
||||
INSERT INTO attachment_jobs
|
||||
@@ -129,7 +131,7 @@ public class PostgresAttachmentRepository implements AttachmentRepository {
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean claim(UUID jobId, UUID attachmentId, int attempt,
|
||||
public boolean claim(UUID jobId, UUID attachmentId, int pipelineVersion, int attempt,
|
||||
UUID leaseToken, Duration leaseDuration) {
|
||||
int claimed = jdbc.update("""
|
||||
UPDATE attachment_jobs
|
||||
@@ -138,22 +140,50 @@ public class PostgresAttachmentRepository implements AttachmentRepository {
|
||||
lease_until = clock_timestamp() + (? * interval '1 millisecond'),
|
||||
updated_at = clock_timestamp()
|
||||
WHERE id = ? AND attachment_id = ?
|
||||
AND pipeline_version = ?
|
||||
AND (status = 'queued'
|
||||
OR (status = 'processing' AND lease_until < clock_timestamp()))
|
||||
""", attempt, leaseToken, leaseDuration.toMillis(), jobId, attachmentId);
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM attachments a
|
||||
WHERE a.id = ? AND a.pipeline_version = ?
|
||||
AND a.status IN ('queued', 'processing'))
|
||||
""", attempt, leaseToken, leaseDuration.toMillis(), jobId, attachmentId,
|
||||
pipelineVersion, attachmentId, pipelineVersion);
|
||||
if (claimed == 0) {
|
||||
return false;
|
||||
}
|
||||
jdbc.update("""
|
||||
UPDATE attachments
|
||||
SET status = 'processing', error_code = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = ? AND status NOT IN ('ready', 'failed')
|
||||
""", attachmentId);
|
||||
WHERE id = ? AND pipeline_version = ?
|
||||
AND status IN ('queued', 'processing')
|
||||
""", attachmentId, pipelineVersion);
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean markQueuedForRetry(UUID jobId, UUID attachmentId, UUID leaseToken,
|
||||
public boolean markBeforeEffectFaultInjected(UUID jobId, UUID leaseToken) {
|
||||
return jdbc.update("""
|
||||
UPDATE attachment_jobs
|
||||
SET before_effect_fault_injected = true, updated_at = clock_timestamp()
|
||||
WHERE id = ? AND status = 'processing' AND lease_token = ?
|
||||
AND before_effect_fault_injected = false
|
||||
""", jobId, leaseToken) == 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean markAfterEffectFaultInjected(UUID jobId, UUID leaseToken) {
|
||||
return jdbc.update("""
|
||||
UPDATE attachment_jobs
|
||||
SET after_effect_fault_injected = true, updated_at = clock_timestamp()
|
||||
WHERE id = ? AND status = 'processing' AND lease_token = ?
|
||||
AND after_effect_fault_injected = false
|
||||
""", jobId, leaseToken) == 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean markQueuedForRetry(UUID jobId, UUID attachmentId, int pipelineVersion,
|
||||
UUID leaseToken,
|
||||
int nextAttempt, String errorCode, Instant nextAttemptAt) {
|
||||
int updated = jdbc.update("""
|
||||
UPDATE attachment_jobs
|
||||
@@ -161,17 +191,18 @@ public class PostgresAttachmentRepository implements AttachmentRepository {
|
||||
last_error_code = ?, next_attempt_at = ?,
|
||||
lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = ? AND attachment_id = ? AND status = 'processing'
|
||||
AND lease_token = ?
|
||||
AND pipeline_version = ? AND lease_token = ?
|
||||
""", nextAttempt, errorCode, Timestamp.from(nextAttemptAt),
|
||||
jobId, attachmentId, leaseToken);
|
||||
jobId, attachmentId, pipelineVersion, leaseToken);
|
||||
if (updated == 0) {
|
||||
return false;
|
||||
}
|
||||
jdbc.update("""
|
||||
UPDATE attachments
|
||||
SET status = 'queued', error_code = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = ? AND status NOT IN ('ready', 'failed')
|
||||
""", attachmentId);
|
||||
WHERE id = ? AND pipeline_version = ?
|
||||
AND status IN ('queued', 'processing')
|
||||
""", attachmentId, pipelineVersion);
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -192,8 +223,13 @@ public class PostgresAttachmentRepository implements AttachmentRepository {
|
||||
result_fault_injected = result_fault_injected OR ?,
|
||||
updated_at = clock_timestamp()
|
||||
WHERE id = ? AND attachment_id = ? AND status = 'processing'
|
||||
AND lease_token = ?
|
||||
""", triggerCrash, jobId, attachmentId, leaseToken);
|
||||
AND pipeline_version = ? AND lease_token = ?
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM attachments a
|
||||
WHERE a.id = ? AND a.pipeline_version = ?
|
||||
AND a.status = 'processing')
|
||||
""", triggerCrash, jobId, attachmentId, pipelineVersion, leaseToken,
|
||||
attachmentId, pipelineVersion);
|
||||
if (completed == 0) {
|
||||
return new CompletionResult(false, false);
|
||||
}
|
||||
@@ -203,13 +239,7 @@ public class PostgresAttachmentRepository implements AttachmentRepository {
|
||||
(attachment_id, pipeline_version, name, content_type, size_bytes,
|
||||
width, height, sha256, object_key, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, clock_timestamp())
|
||||
ON CONFLICT (attachment_id, pipeline_version, name) DO UPDATE SET
|
||||
content_type = EXCLUDED.content_type,
|
||||
size_bytes = EXCLUDED.size_bytes,
|
||||
width = EXCLUDED.width,
|
||||
height = EXCLUDED.height,
|
||||
sha256 = EXCLUDED.sha256,
|
||||
object_key = EXCLUDED.object_key
|
||||
ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING
|
||||
""", attachmentId, pipelineVersion, variant.name(), variant.contentType(),
|
||||
variant.sizeBytes(), variant.width(), variant.height(), variant.sha256(),
|
||||
variant.objectKey());
|
||||
@@ -217,29 +247,33 @@ public class PostgresAttachmentRepository implements AttachmentRepository {
|
||||
jdbc.update("""
|
||||
UPDATE attachments
|
||||
SET status = 'ready', error_code = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = ? AND status <> 'failed'
|
||||
""", attachmentId);
|
||||
WHERE id = ? AND pipeline_version = ? AND status = 'processing'
|
||||
""", attachmentId, pipelineVersion);
|
||||
return new CompletionResult(true, triggerCrash);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean markFailed(UUID jobId, UUID attachmentId, UUID leaseToken, String errorCode) {
|
||||
public boolean markFailed(UUID jobId, UUID attachmentId, int pipelineVersion,
|
||||
UUID leaseToken, String errorCode) {
|
||||
int updated = jdbc.update("""
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'failed', published = true, last_error_code = ?,
|
||||
lease_token = NULL, lease_until = NULL,
|
||||
updated_at = clock_timestamp()
|
||||
WHERE id = ? AND attachment_id = ? AND status = 'processing'
|
||||
AND lease_token = ?
|
||||
""", errorCode, jobId, attachmentId, leaseToken);
|
||||
AND pipeline_version = ? AND lease_token = ?
|
||||
AND EXISTS (SELECT 1 FROM attachments a
|
||||
WHERE a.id = ? AND a.pipeline_version = ?)
|
||||
""", errorCode, jobId, attachmentId, pipelineVersion, leaseToken,
|
||||
attachmentId, pipelineVersion);
|
||||
if (updated == 0) {
|
||||
return false;
|
||||
}
|
||||
jdbc.update("""
|
||||
UPDATE attachments
|
||||
SET status = 'failed', error_code = ?, updated_at = clock_timestamp()
|
||||
WHERE id = ? AND status <> 'ready'
|
||||
""", errorCode, attachmentId);
|
||||
WHERE id = ? AND pipeline_version = ? AND status <> 'ready'
|
||||
""", errorCode, attachmentId, pipelineVersion);
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -255,7 +289,8 @@ public class PostgresAttachmentRepository implements AttachmentRepository {
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean requeueExpired(UUID jobId, UUID attachmentId, int nextAttempt,
|
||||
public boolean requeueExpired(UUID jobId, UUID attachmentId, int pipelineVersion,
|
||||
int nextAttempt,
|
||||
String errorCode, Instant nextAttemptAt) {
|
||||
int updated = jdbc.update("""
|
||||
UPDATE attachment_jobs
|
||||
@@ -263,33 +298,40 @@ public class PostgresAttachmentRepository implements AttachmentRepository {
|
||||
last_error_code = ?, next_attempt_at = ?,
|
||||
lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = ? AND attachment_id = ? AND status = 'processing'
|
||||
AND lease_until < clock_timestamp()
|
||||
""", nextAttempt, errorCode, Timestamp.from(nextAttemptAt), jobId, attachmentId);
|
||||
AND pipeline_version = ? AND lease_until < clock_timestamp()
|
||||
AND EXISTS (SELECT 1 FROM attachments a
|
||||
WHERE a.id = ? AND a.pipeline_version = ?)
|
||||
""", nextAttempt, errorCode, Timestamp.from(nextAttemptAt), jobId, attachmentId,
|
||||
pipelineVersion, attachmentId, pipelineVersion);
|
||||
if (updated == 1) {
|
||||
jdbc.update("""
|
||||
UPDATE attachments SET status = 'queued', error_code = NULL,
|
||||
updated_at = clock_timestamp()
|
||||
WHERE id = ? AND status = 'processing'
|
||||
""", attachmentId);
|
||||
WHERE id = ? AND pipeline_version = ? AND status = 'processing'
|
||||
""", attachmentId, pipelineVersion);
|
||||
}
|
||||
return updated == 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean failExpired(UUID jobId, UUID attachmentId, String errorCode) {
|
||||
public boolean failExpired(UUID jobId, UUID attachmentId, int pipelineVersion,
|
||||
String errorCode) {
|
||||
int updated = jdbc.update("""
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'failed', published = true, last_error_code = ?,
|
||||
lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = ? AND attachment_id = ? AND status = 'processing'
|
||||
AND lease_until < clock_timestamp()
|
||||
""", errorCode, jobId, attachmentId);
|
||||
AND pipeline_version = ? AND lease_until < clock_timestamp()
|
||||
AND EXISTS (SELECT 1 FROM attachments a
|
||||
WHERE a.id = ? AND a.pipeline_version = ?)
|
||||
""", errorCode, jobId, attachmentId, pipelineVersion,
|
||||
attachmentId, pipelineVersion);
|
||||
if (updated == 1) {
|
||||
jdbc.update("""
|
||||
UPDATE attachments SET status = 'failed', error_code = ?,
|
||||
updated_at = clock_timestamp()
|
||||
WHERE id = ? AND status <> 'ready'
|
||||
""", errorCode, attachmentId);
|
||||
WHERE id = ? AND pipeline_version = ? AND status <> 'ready'
|
||||
""", errorCode, attachmentId, pipelineVersion);
|
||||
}
|
||||
return updated == 1;
|
||||
}
|
||||
@@ -317,12 +359,14 @@ public class PostgresAttachmentRepository implements AttachmentRepository {
|
||||
return jdbc.query("""
|
||||
SELECT name, content_type, size_bytes, width, height, sha256, object_key
|
||||
FROM attachment_variants
|
||||
WHERE attachment_id = ? AND pipeline_version = 1
|
||||
WHERE attachment_id = ?
|
||||
AND pipeline_version = (
|
||||
SELECT pipeline_version FROM attachments WHERE id = ?)
|
||||
ORDER BY name
|
||||
""", (rs, rowNum) -> new AttachmentVariant(
|
||||
rs.getString("name"), rs.getString("content_type"), rs.getLong("size_bytes"),
|
||||
rs.getInt("width"), rs.getInt("height"), rs.getString("sha256"),
|
||||
rs.getString("object_key")), attachmentId);
|
||||
rs.getString("object_key")), attachmentId, attachmentId);
|
||||
}
|
||||
|
||||
private static AttachmentJob mapJob(ResultSet rs) throws SQLException {
|
||||
|
||||
@@ -44,6 +44,7 @@ public class AttachmentService {
|
||||
private final Semaphore uploadPermits;
|
||||
private final long uploadPermitWaitMillis;
|
||||
private final boolean injectCrashAfterOutboxCommit;
|
||||
private final int pipelineVersion;
|
||||
|
||||
public AttachmentService(
|
||||
AttachmentRepository attachments,
|
||||
@@ -55,7 +56,8 @@ public class AttachmentService {
|
||||
@Value("${messenger.attachments.max-image-pixels:40000000}") long maxImagePixels,
|
||||
@Value("${messenger.attachments.max-concurrent-uploads:3}") int maxConcurrentUploads,
|
||||
@Value("${messenger.attachments.upload-permit-wait-millis:10000}") long uploadPermitWaitMillis,
|
||||
@Value("${messenger.jobs.fault-crash-after-outbox-commit:false}") boolean injectCrashAfterOutboxCommit
|
||||
@Value("${messenger.jobs.fault-crash-after-outbox-commit:false}") boolean injectCrashAfterOutboxCommit,
|
||||
@Value("${messenger.jobs.pipeline-version:2}") int pipelineVersion
|
||||
) {
|
||||
this.attachments = attachments;
|
||||
this.chats = chats;
|
||||
@@ -70,6 +72,11 @@ public class AttachmentService {
|
||||
this.uploadPermits = new Semaphore(maxConcurrentUploads, true);
|
||||
this.uploadPermitWaitMillis = uploadPermitWaitMillis;
|
||||
this.injectCrashAfterOutboxCommit = injectCrashAfterOutboxCommit;
|
||||
if (pipelineVersion < AttachmentJob.MIN_PIPELINE_VERSION
|
||||
|| pipelineVersion > AttachmentJob.CURRENT_PIPELINE_VERSION) {
|
||||
throw new IllegalArgumentException("Unsupported attachment pipeline version");
|
||||
}
|
||||
this.pipelineVersion = pipelineVersion;
|
||||
}
|
||||
|
||||
public Attachment upload(UUID chatId,
|
||||
@@ -119,7 +126,7 @@ public class AttachmentService {
|
||||
String objectKey = "attachments/" + id + "/original";
|
||||
String stagingKey = "staging/" + id + "/original";
|
||||
Instant now = Instant.now().truncatedTo(ChronoUnit.MICROS);
|
||||
AttachmentJob job = AttachmentJob.first(UUID.randomUUID(), id, now);
|
||||
AttachmentJob job = AttachmentJob.first(UUID.randomUUID(), id, pipelineVersion, now);
|
||||
Attachment attachment = new Attachment(id, chatId, ownerId, safeFilename, contentType,
|
||||
bytes.length, sha256(bytes), objectKey, stagingKey, "queued", List.of(),
|
||||
null, now, now);
|
||||
|
||||
@@ -45,9 +45,13 @@ public class ReadinessService {
|
||||
}
|
||||
|
||||
try {
|
||||
// The schema-level probe also proves that the separate lab4 migration
|
||||
// completed before this replica is allowed to serve business traffic.
|
||||
jdbc.queryForObject("SELECT count(*) FROM attachments", Long.class);
|
||||
// The schema-level probe proves that both the shared idempotency
|
||||
// schema (lab3) and the attachment schema (lab4/lab5) completed
|
||||
// before this replica is allowed to serve business traffic.
|
||||
jdbc.queryForObject("""
|
||||
SELECT (SELECT count(*) FROM message_idempotency)
|
||||
+ (SELECT count(*) FROM attachments)
|
||||
""", Long.class);
|
||||
markDependenciesAvailable();
|
||||
return true;
|
||||
} catch (DataAccessException e) {
|
||||
|
||||
@@ -6,6 +6,7 @@ import com.example.mtusi_messenger.model.AttachmentVariant;
|
||||
import com.example.mtusi_messenger.queue.RetryPolicy;
|
||||
import com.example.mtusi_messenger.repository.AttachmentRepository;
|
||||
import com.example.mtusi_messenger.storage.ObjectStorage;
|
||||
import com.example.mtusi_messenger.storage.ObjectStorageException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.rabbitmq.client.Channel;
|
||||
import jakarta.annotation.PreDestroy;
|
||||
@@ -47,6 +48,8 @@ public class AttachmentWorker {
|
||||
private final int maxAttempts;
|
||||
private final Duration jobTimeout;
|
||||
private final Duration leaseDuration;
|
||||
private final boolean injectCrashBeforeSideEffect;
|
||||
private final boolean injectCrashAfterSideEffect;
|
||||
private final boolean injectCrashAfterResultCommit;
|
||||
private final ExecutorService processingPool;
|
||||
private final Clock clock = Clock.systemUTC();
|
||||
@@ -61,6 +64,8 @@ public class AttachmentWorker {
|
||||
@Value("${messenger.jobs.max-attempts:5}") int maxAttempts,
|
||||
@Value("${messenger.jobs.timeout:60s}") Duration jobTimeout,
|
||||
@Value("${messenger.jobs.lease-duration:90s}") Duration leaseDuration,
|
||||
@Value("${messenger.jobs.fault-crash-before-side-effect:false}") boolean injectCrashBeforeSideEffect,
|
||||
@Value("${messenger.jobs.fault-crash-after-side-effect:false}") boolean injectCrashAfterSideEffect,
|
||||
@Value("${messenger.jobs.fault-crash-after-result-commit:false}") boolean injectCrashAfterResultCommit,
|
||||
@Value("${spring.rabbitmq.listener.simple.concurrency:2}") int concurrency) {
|
||||
if (maxAttempts < 1 || jobTimeout.isZero() || jobTimeout.isNegative()
|
||||
@@ -76,6 +81,8 @@ public class AttachmentWorker {
|
||||
this.maxAttempts = maxAttempts;
|
||||
this.jobTimeout = jobTimeout;
|
||||
this.leaseDuration = leaseDuration;
|
||||
this.injectCrashBeforeSideEffect = injectCrashBeforeSideEffect;
|
||||
this.injectCrashAfterSideEffect = injectCrashAfterSideEffect;
|
||||
this.injectCrashAfterResultCommit = injectCrashAfterResultCommit;
|
||||
this.processingPool = Executors.newFixedThreadPool(concurrency,
|
||||
Thread.ofPlatform().name("attachment-image-", 0).factory());
|
||||
@@ -103,7 +110,8 @@ public class AttachmentWorker {
|
||||
int attempt = Math.max(job.attempt(), attachments.currentAttempt(job.jobId()));
|
||||
UUID leaseToken = UUID.randomUUID();
|
||||
boolean claimed = Boolean.TRUE.equals(transaction.execute(status ->
|
||||
attachments.claim(job.jobId(), job.attachmentId(), attempt, leaseToken, leaseDuration)));
|
||||
attachments.claim(job.jobId(), job.attachmentId(), job.pipelineVersion(),
|
||||
attempt, leaseToken, leaseDuration)));
|
||||
if (!claimed) {
|
||||
// A live worker owns the lease, or a terminal state won the race.
|
||||
// If the owner died, the DB lease reaper will create a new outbox delivery.
|
||||
@@ -140,17 +148,23 @@ public class AttachmentWorker {
|
||||
}
|
||||
|
||||
private ProcessOutcome process(AttachmentJob job, UUID leaseToken) {
|
||||
if (injectCrashBeforeSideEffect
|
||||
&& attachments.markBeforeEffectFaultInjected(job.jobId(), leaseToken)) {
|
||||
log.error("Fault injection: halting worker before object-storage side effect");
|
||||
Runtime.getRuntime().halt(84);
|
||||
}
|
||||
Attachment attachment = attachments.findById(job.attachmentId())
|
||||
.orElseThrow(() -> new ImageVariantProcessor.ProcessingException(
|
||||
"attachment_missing", true, "Attachment metadata is missing", null));
|
||||
byte[] original = storage.get(attachment.stagingKey());
|
||||
byte[] original = loadOriginal(attachment);
|
||||
if (original.length != attachment.sizeBytes() || !sha256(original).equals(attachment.sha256())) {
|
||||
throw new ImageVariantProcessor.ProcessingException(
|
||||
"invalid_image", true, "Staging checksum mismatch", null);
|
||||
}
|
||||
|
||||
storage.put(attachment.objectKey(), original, attachment.contentType());
|
||||
List<ImageVariantProcessor.GeneratedVariant> generated = images.createVariants(original);
|
||||
List<ImageVariantProcessor.GeneratedVariant> generated =
|
||||
images.createVariants(original, job.pipelineVersion());
|
||||
List<AttachmentVariant> variants = new ArrayList<>(generated.size());
|
||||
for (ImageVariantProcessor.GeneratedVariant item : generated) {
|
||||
String objectKey = "attachments/" + attachment.id() + "/"
|
||||
@@ -161,6 +175,12 @@ public class AttachmentWorker {
|
||||
item.width(), item.height(), sha256(item.bytes()), objectKey));
|
||||
}
|
||||
|
||||
if (injectCrashAfterSideEffect
|
||||
&& attachments.markAfterEffectFaultInjected(job.jobId(), leaseToken)) {
|
||||
log.error("Fault injection: halting worker after object-storage side effect before result commit");
|
||||
Runtime.getRuntime().halt(85);
|
||||
}
|
||||
|
||||
AttachmentRepository.CompletionResult completion = transaction.execute(status ->
|
||||
attachments.markReady(job.jobId(), job.attachmentId(), leaseToken,
|
||||
job.pipelineVersion(), variants, injectCrashAfterResultCommit));
|
||||
@@ -175,11 +195,27 @@ public class AttachmentWorker {
|
||||
return new ProcessOutcome(completion.crashAfterCommit());
|
||||
}
|
||||
|
||||
private byte[] loadOriginal(Attachment attachment) {
|
||||
try {
|
||||
return storage.get(attachment.stagingKey());
|
||||
} catch (ObjectStorageException missingStaging) {
|
||||
try {
|
||||
// A pipeline promotion reuses the immutable final original after
|
||||
// the previous version safely removed its staging object.
|
||||
return storage.get(attachment.objectKey());
|
||||
} catch (ObjectStorageException missingOriginal) {
|
||||
missingOriginal.addSuppressed(missingStaging);
|
||||
throw missingOriginal;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void failPermanent(
|
||||
AttachmentJob job, UUID leaseToken, String errorCode,
|
||||
long deliveryTag, Channel channel) throws IOException {
|
||||
boolean failed = Boolean.TRUE.equals(transaction.execute(status ->
|
||||
attachments.markFailed(job.jobId(), job.attachmentId(), leaseToken, errorCode)));
|
||||
attachments.markFailed(job.jobId(), job.attachmentId(), job.pipelineVersion(),
|
||||
leaseToken, errorCode)));
|
||||
if (failed) {
|
||||
channel.basicReject(deliveryTag, false);
|
||||
} else {
|
||||
@@ -193,8 +229,8 @@ public class AttachmentWorker {
|
||||
try {
|
||||
if (attempt >= maxAttempts) {
|
||||
boolean failed = Boolean.TRUE.equals(transaction.execute(status ->
|
||||
attachments.markFailed(job.jobId(), job.attachmentId(), leaseToken,
|
||||
"retry_exhausted")));
|
||||
attachments.markFailed(job.jobId(), job.attachmentId(),
|
||||
job.pipelineVersion(), leaseToken, "retry_exhausted")));
|
||||
if (failed) {
|
||||
log.error("Attachment job {} exhausted {} attempts: {}",
|
||||
job.jobId(), attempt, errorCode);
|
||||
@@ -208,7 +244,8 @@ public class AttachmentWorker {
|
||||
int nextAttempt = attempt + 1;
|
||||
boolean queued = Boolean.TRUE.equals(transaction.execute(status ->
|
||||
attachments.markQueuedForRetry(
|
||||
job.jobId(), job.attachmentId(), leaseToken, nextAttempt, errorCode,
|
||||
job.jobId(), job.attachmentId(), job.pipelineVersion(),
|
||||
leaseToken, nextAttempt, errorCode,
|
||||
clock.instant().plus(retryPolicy.delayForAttempt(nextAttempt)))));
|
||||
// The DB outbox is now the durable retry carrier; the current broker
|
||||
// delivery can be acknowledged without a DB/broker dual write.
|
||||
|
||||
@@ -45,7 +45,8 @@ public class ImageVariantProcessor {
|
||||
this.memoryWait = memoryWait;
|
||||
}
|
||||
|
||||
public List<GeneratedVariant> createVariants(byte[] original) {
|
||||
public List<GeneratedVariant> createVariants(byte[] original, int pipelineVersion) {
|
||||
PipelineProfile profile = PipelineProfile.forVersion(pipelineVersion);
|
||||
ImageInfo info = inspect(original);
|
||||
long estimatedBytes;
|
||||
try {
|
||||
@@ -70,7 +71,7 @@ public class ImageVariantProcessor {
|
||||
}
|
||||
|
||||
try {
|
||||
return transform(original);
|
||||
return transform(original, profile);
|
||||
} finally {
|
||||
memoryPermits.release(permits);
|
||||
}
|
||||
@@ -106,7 +107,7 @@ public class ImageVariantProcessor {
|
||||
}
|
||||
}
|
||||
|
||||
private List<GeneratedVariant> transform(byte[] original) {
|
||||
private List<GeneratedVariant> transform(byte[] original, PipelineProfile profile) {
|
||||
try {
|
||||
BufferedImage oriented = Thumbnails.of(new ByteArrayInputStream(original))
|
||||
.useExifOrientation(true)
|
||||
@@ -118,17 +119,20 @@ public class ImageVariantProcessor {
|
||||
try {
|
||||
List<GeneratedVariant> variants = new ArrayList<>(3);
|
||||
BufferedImage thumbnail = fit(oriented, 256, BufferedImage.TYPE_INT_RGB, true);
|
||||
variants.add(encode("thumbnail", "image/jpeg", "jpg", thumbnail, "jpeg", 0.85f));
|
||||
variants.add(encode("thumbnail", "image/jpeg", "jpg", thumbnail, "jpeg",
|
||||
profile.thumbnailQuality()));
|
||||
thumbnail.flush();
|
||||
|
||||
BufferedImage crop = cropSquare(oriented, 128);
|
||||
variants.add(encode("crop", "image/jpeg", "jpg", crop, "jpeg", 0.88f));
|
||||
variants.add(encode("crop", "image/jpeg", "jpg", crop, "jpeg",
|
||||
profile.cropQuality()));
|
||||
crop.flush();
|
||||
|
||||
int webpType = oriented.getColorModel().hasAlpha()
|
||||
? BufferedImage.TYPE_INT_ARGB : BufferedImage.TYPE_INT_RGB;
|
||||
BufferedImage optimized = fit(oriented, 1280, webpType, !oriented.getColorModel().hasAlpha());
|
||||
variants.add(encode("optimized", "image/webp", "webp", optimized, "webp", 0.82f));
|
||||
variants.add(encode("optimized", "image/webp", "webp", optimized, "webp",
|
||||
profile.webpQuality()));
|
||||
optimized.flush();
|
||||
return List.copyOf(variants);
|
||||
} finally {
|
||||
@@ -228,6 +232,18 @@ public class ImageVariantProcessor {
|
||||
private record ImageInfo(int width, int height, long pixels) {
|
||||
}
|
||||
|
||||
private record PipelineProfile(
|
||||
int version, float thumbnailQuality, float cropQuality, float webpQuality) {
|
||||
private static PipelineProfile forVersion(int version) {
|
||||
return switch (version) {
|
||||
case 1 -> new PipelineProfile(1, 0.85f, 0.88f, 0.82f);
|
||||
case 2 -> new PipelineProfile(2, 0.87f, 0.90f, 0.84f);
|
||||
default -> throw permanent(
|
||||
"unsupported_pipeline", "Unsupported pipeline version " + version, null);
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
public record GeneratedVariant(String name, String contentType, String extension,
|
||||
byte[] bytes, int width, int height) {
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user