This commit is contained in:
2026-11-20 22:48:28 +03:00
parent 276dbdad8a
commit 8e1f96b70b
46 changed files with 2345 additions and 49 deletions
@@ -0,0 +1,119 @@
package ru.mtusi.messenger;
import jakarta.annotation.PostConstruct;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.data.redis.connection.stream.MapRecord;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.sql.Timestamp;
import java.time.Instant;
import java.util.List;
import java.util.UUID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@Component
@ConditionalOnProperty(name = "WORKER_ONLY", havingValue = "true")
public class AttachmentWorker {
private static final Logger LOG = LoggerFactory.getLogger(AttachmentWorker.class);
private final JdbcTemplate jdbc;
private final JobBroker broker;
private final S3Storage s3;
private final String consumer = "worker-" + UUID.randomUUID();
public AttachmentWorker(JdbcTemplate jdbc, JobBroker broker, S3Storage s3) { this.jdbc = jdbc; this.broker = broker; this.s3 = s3; }
@PostConstruct
void start() { LOG.info("attachment worker started consumer={}", consumer); }
@Scheduled(fixedDelayString = "${WORKER_POLL_MS:500}", initialDelayString = "${WORKER_POLL_INITIAL_DELAY_MS:500}")
public void poll() {
try {
MapRecord<String, Object, Object> record = broker.read(consumer);
if (record != null) {
UUID jobId = UUID.fromString(String.valueOf(record.getValue().get("job_id")));
LOG.info("attachment job received job_id={}", jobId);
try { process(jobId); } finally { broker.ack(record.getId()); }
} else {
processQueuedFallback();
}
recoverLeases();
} catch (Exception exception) {
LOG.warn("worker poll error", exception);
}
}
private void processQueuedFallback() {
List<UUID> jobs = jdbc.query("SELECT job_id FROM attachment_jobs WHERE status='queued' AND next_attempt_at <= now() ORDER BY created_at LIMIT 1", (rs, row) -> rs.getObject("job_id", UUID.class));
if (!jobs.isEmpty()) { LOG.info("processing queued fallback job_id={}", jobs.getFirst()); process(jobs.getFirst()); }
}
@Scheduled(fixedDelayString = "${WORKER_LEASE_SCAN_MS:1000}")
public void recoverLeases() {
jdbc.update("UPDATE attachment_jobs SET status='queued', published_at=NULL, lease_until=NULL, updated_at=now() WHERE status='processing' AND lease_until < now() AND attempt < 5");
}
private void process(UUID jobId) {
Job job = claim(jobId);
if (job == null) return;
LOG.info("processing attachment job_id={} attempt={}", job.jobId(), job.attempt());
Path temp = null;
try {
temp = Files.createTempFile("messenger-worker-", ".bin");
try (InputStream input = s3.get(job.stagingKey()); var output = Files.newOutputStream(temp)) { input.transferTo(output); }
AttachmentService.Dimensions dimensions = AttachmentService.validateImage(temp);
List<AttachmentImageProcessor.Rendered> variants = AttachmentImageProcessor.render(temp);
try (InputStream original = Files.newInputStream(temp)) {
s3.put(job.originalKey(), original, Files.size(temp), job.contentType());
}
for (AttachmentImageProcessor.Rendered variant : variants) {
String key = "variants/" + job.attachmentId() + "/" + variant.name();
s3.put(key, new ByteArrayInputStream(variant.bytes()), variant.bytes().length, variant.contentType());
}
Instant now = Instant.now();
for (AttachmentImageProcessor.Rendered variant : variants) {
String key = "variants/" + job.attachmentId() + "/" + variant.name();
jdbc.update("""
INSERT INTO attachment_variants(attachment_id, name, content_type, size_bytes, width, height, sha256, object_key)
VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (attachment_id, name) DO NOTHING
""", job.attachmentId(), variant.name(), variant.contentType(), variant.bytes().length,
variant.width(), variant.height(), sha256(variant.bytes()), key);
}
jdbc.update("UPDATE attachments SET status='ready', error_code=NULL, updated_at=? WHERE id=?", Timestamp.from(now), job.attachmentId());
jdbc.update("UPDATE attachment_jobs SET status='ready', lease_until=NULL, updated_at=? WHERE job_id=?", Timestamp.from(now), job.jobId());
s3.delete(job.stagingKey());
LOG.info("attachment ready job_id={} attachment_id={}", job.jobId(), job.attachmentId());
} catch (Exception exception) {
LOG.warn("attachment processing failed job_id={} attempt={}", job.jobId(), job.attempt(), exception);
failOrRetry(job, exception);
} finally { if (temp != null) try { Files.deleteIfExists(temp); } catch (Exception ignored) {} LOG.info("attachment job finished job_id={}", job.jobId()); }
}
private Job claim(UUID jobId) {
int changed = jdbc.update("UPDATE attachment_jobs SET status='processing', attempt=attempt+1, lease_until=now()+interval '30 seconds', updated_at=now() WHERE job_id=? AND (status='queued' OR (status='processing' AND lease_until < now())) AND attempt < 5", jobId);
if (changed == 0) return null;
return jdbc.queryForObject("SELECT j.job_id, j.attachment_id, a.staging_key, a.object_key, a.content_type, j.attempt FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.job_id=?", (rs, row) -> new Job(rs.getObject("job_id", UUID.class), rs.getObject("attachment_id", UUID.class), rs.getString("staging_key"), rs.getString("object_key"), rs.getString("content_type"), rs.getInt("attempt")), jobId);
}
private void failOrRetry(Job job, Exception exception) {
boolean permanent = exception instanceof ApiException api && (api.status() == 413 || api.status() == 415);
if (permanent || job.attempt() >= 5) {
String code = permanent ? "invalid_image" : "retry_exhausted";
jdbc.update("UPDATE attachments SET status='failed', error_code=?, updated_at=now() WHERE id=?", code, job.attachmentId());
jdbc.update("UPDATE attachment_jobs SET status='failed', error_code=?, lease_until=NULL, updated_at=now() WHERE job_id=?", code, job.jobId());
} else {
long delay = Math.min(5000L, 200L * (1L << Math.min(job.attempt(), 4))) + (long) (Math.random() * 300);
jdbc.update("UPDATE attachment_jobs SET status='queued', published_at=NULL, lease_until=NULL, next_attempt_at=now() + (? * interval '1 millisecond'), error_code=?, updated_at=now() WHERE job_id=?", delay, "temporary_error", job.jobId());
jdbc.update("UPDATE attachments SET status='queued', error_code=NULL, updated_at=now() WHERE id=?", job.attachmentId());
}
}
private static String sha256(byte[] bytes) throws Exception { var digest = java.security.MessageDigest.getInstance("SHA-256"); byte[] hash = digest.digest(bytes); StringBuilder s = new StringBuilder(); for (byte b : hash) s.append(String.format("%02x", b)); return s.toString(); }
private record Job(UUID jobId, UUID attachmentId, String stagingKey, String originalKey, String contentType, int attempt) {}
}