forked from mtusi2026/mtusi2026lab5
Автор / группа: ivanstorozhev, БВТ2304 Целевая оценка (3/4/5): 4 Трек 5, если заявлен: не заявлен Что реализовано и что перенесено: из lab4 перенесены Spring Boot API, PostgreSQL, Redis-кэш сессий, Argon2id, Liquibase через одноразовый сервис migrate, HAProxy без sticky session, две реплики, X-Instance-Id, обязательный Idempotency-Key, scripts/stop-one.sh и scripts/restart.sh, приватный SeaweedFS, таблица attachments и скачивание оригинала через API. Ключ объекта назначает сервер. У API отдельный ключ S3 с правами Read/Write/List; Admin используется только чтобы создать bucket. Объект без строки в базе старше двух минут удаляет фоновая очистка. Добавлены ответ 202, staging, transactional outbox, отдельный worker, Kafka 3.9.1 (KRaft, volume kafka-data), состояния queued/processing/ready/failed, JPEG thumbnail и crop, lossy WebP, retry с backoff и jitter, топик attachments.failed, явный replay и lease. Сообщение брокера содержит только идентификаторы. Подтверждение записи брокером (acks=all) и commit смещения потребителя разделены. Команды запуска из чистого clone: cp .env.example .env docker compose --env-file .env config --quiet docker compose --env-file .env up -d --build curl -fsS http://127.0.0.1:8080/health/ready make test make test-persistence ACTION_SCRIPT=scripts/restart.sh make test-failover ACTION_SCRIPT=scripts/stop-one.sh docker compose --env-file .env start api python3 scripts/grade4-evidence.py Результаты проверок и ссылка на REPORT.md: подробности и числа в REPORT.md make check — PASS: offline skeleton structure, local refs and Python syntax. make test — PASS: lab5 public API scenario. make test-persistence ACTION_SCRIPT=scripts/restart.sh — PASS: persistence preserved confirmed data and existing session. Хук перезапускает и kafka. make test-failover ACTION_SCRIPT=scripts/stop-one.sh — PASS: две реплики до отказа, восстановление записи за 1.795 с, 117 успешных записей и 1 ошибка перехода. python3 scripts/grade4-evidence.py — PASS. Worker остановлен: после рестарта API вложение осталось queued, затем стало ready; thumbnail JPEG 256×160, crop JPEG 128×128, optimized WebP 320×200, оригинал 111817 байт совпал. Конец топика attachments.process вырос 2→3 и после restart kafka остался 3. Пауза outbox пережила kill API, unpublished=1, затем тот же id стал ready. System.exit после записи объектов: RestartCount 0→1, один набор вариантов. Постоянный сбой processing_failed, топик attachments.failed, replay доводит то же вложение до ready, второй replay остаётся ready. mvn -q test в backend/ — код выхода 0, 22 теста: 202 и варианты, слишком большой файл 413, порча и подмена MIME 415, чужой id и другой чат 404, сирота в S3 удаляется. Анонимный GET объекта в сети стенда — HTTP 403. Ограничения / известные ошибки: трек 5 не делался. Один Docker-хост, единственные PostgreSQL, Redis, SeaweedFS, Kafka и HAProxy остаются точками отказа. Replication factor топиков равен 1. Доставка at least once: повтор перезаписывает те же ключи, но это не exactly-once на стороне S3. Если процесс умер после PUT staging и до COMMIT, объект без строки живёт не дольше grace period, затем удаляется. Staging failed-задания хранится сутки; позже replay уже не находит байты. Пока HAProxy не снял backend, часть запросов на останавливаемую реплику завершается ошибкой соединения; повтор с тем же Idempotency-Key безопасен для отправки сообщения. Источники и использование ИИ: контракт contracts/openapi.json, contracts/README.md, contracts/attachment-states.md, contracts/jobs.schema.json; код lab4; Spring Boot 4.1.1, Spring Data JPA, spring-security-crypto, Spring Data Redis, Spring Kafka, Liquibase, Actuator, MapStruct 1.6.3, OpenAPI Generator 7.17.0, AWS SDK for Java 2.31.50, Thumbnailator 0.4.20, webp-imageio 0.11.0, Bouncy Castle, PostgreSQL 16.10, Redis 7.4.6, HAProxy 3.2.6, SeaweedFS 4.52, Apache Kafka 3.9.1, Testcontainers. Реализация собрана в Cursor. Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. Контракт и публичные тесты не ослаблены. make check и make test выполнены; приложен реальный вывод. Сценарии отказов своего уровня воспроизведены. Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. README задания сохранён, REPORT.md содержит инструкцию именно моего решения. Co-authored-by: Cursor <cursoragent@cursor.com>
394 lines
17 KiB
Java
394 lines
17 KiB
Java
package ru.mtuci.messenger.job;
|
|
|
|
import org.slf4j.Logger;
|
|
import org.slf4j.LoggerFactory;
|
|
import org.springframework.beans.factory.ObjectProvider;
|
|
import org.springframework.beans.factory.annotation.Value;
|
|
import org.springframework.stereotype.Service;
|
|
import org.springframework.transaction.PlatformTransactionManager;
|
|
import org.springframework.transaction.support.TransactionTemplate;
|
|
import ru.mtuci.messenger.domain.AttachmentEntity;
|
|
import ru.mtuci.messenger.domain.AttachmentVariantEntity;
|
|
import ru.mtuci.messenger.domain.FailedJobEntity;
|
|
import ru.mtuci.messenger.domain.OutboxMessage;
|
|
import ru.mtuci.messenger.exception.ConflictException;
|
|
import ru.mtuci.messenger.exception.NotFoundException;
|
|
import ru.mtuci.messenger.exception.PayloadTooLargeException;
|
|
import ru.mtuci.messenger.exception.UnavailableException;
|
|
import ru.mtuci.messenger.exception.UnsupportedMediaException;
|
|
import ru.mtuci.messenger.repository.AttachmentRepository;
|
|
import ru.mtuci.messenger.repository.AttachmentVariantRepository;
|
|
import ru.mtuci.messenger.repository.FailedJobRepository;
|
|
import ru.mtuci.messenger.repository.OutboxRepository;
|
|
import ru.mtuci.messenger.service.ImageFiles;
|
|
import ru.mtuci.messenger.service.ImageVariants;
|
|
import ru.mtuci.messenger.service.RetryDelay;
|
|
import ru.mtuci.messenger.storage.ObjectStorage;
|
|
import tools.jackson.databind.json.JsonMapper;
|
|
|
|
import java.io.IOException;
|
|
import java.io.InputStream;
|
|
import java.nio.file.Files;
|
|
import java.nio.file.Path;
|
|
import java.security.MessageDigest;
|
|
import java.security.NoSuchAlgorithmException;
|
|
import java.time.Duration;
|
|
import java.time.Instant;
|
|
import java.time.temporal.ChronoUnit;
|
|
import java.util.HexFormat;
|
|
import java.util.Set;
|
|
import java.util.UUID;
|
|
import java.util.concurrent.ConcurrentHashMap;
|
|
|
|
@Service
|
|
public class AttachmentJobs {
|
|
|
|
private static final Logger log = LoggerFactory.getLogger(AttachmentJobs.class);
|
|
private static final Path CRASH_AFTER_WRITE = Path.of("/tmp/messenger-crash-after-write");
|
|
private static final Path FAIL_PERMANENT = Path.of("/tmp/messenger-fail-permanent");
|
|
|
|
private final AttachmentRepository attachments;
|
|
private final AttachmentVariantRepository variants;
|
|
private final OutboxRepository outbox;
|
|
private final FailedJobRepository failedJobs;
|
|
private final ObjectStorage storage;
|
|
private final JsonMapper json;
|
|
private final TransactionTemplate transactions;
|
|
private final ObjectProvider<BrokerPublisher> brokers;
|
|
private final int maxAttempts;
|
|
private final Duration timeout;
|
|
private final long backoffBaseMs;
|
|
private final long backoffCapMs;
|
|
private final int maxBytes;
|
|
private final long maxPixels;
|
|
private final float jpegQuality;
|
|
private final float webpQuality;
|
|
private final Set<UUID> localLeases = ConcurrentHashMap.newKeySet();
|
|
|
|
public AttachmentJobs(AttachmentRepository attachments, AttachmentVariantRepository variants, OutboxRepository outbox,
|
|
FailedJobRepository failedJobs, ObjectStorage storage, JsonMapper json,
|
|
PlatformTransactionManager transactionManager, ObjectProvider<BrokerPublisher> brokers,
|
|
@Value("${app.job.max-attempts:5}") int maxAttempts,
|
|
@Value("${app.job.timeout:60s}") Duration timeout,
|
|
@Value("${app.job.backoff-base-ms:1000}") long backoffBaseMs,
|
|
@Value("${app.job.backoff-cap-ms:30000}") long backoffCapMs,
|
|
@Value("${app.upload.max-bytes:10485760}") int maxBytes,
|
|
@Value("${app.upload.max-pixels:40000000}") long maxPixels,
|
|
@Value("${app.image.jpeg-quality:0.85}") float jpegQuality,
|
|
@Value("${app.image.webp-quality:0.8}") float webpQuality) {
|
|
this.attachments = attachments;
|
|
this.variants = variants;
|
|
this.outbox = outbox;
|
|
this.failedJobs = failedJobs;
|
|
this.storage = storage;
|
|
this.json = json;
|
|
this.transactions = new TransactionTemplate(transactionManager);
|
|
this.brokers = brokers;
|
|
this.maxAttempts = maxAttempts;
|
|
this.timeout = timeout;
|
|
this.backoffBaseMs = backoffBaseMs;
|
|
this.backoffCapMs = backoffCapMs;
|
|
this.maxBytes = maxBytes;
|
|
this.maxPixels = maxPixels;
|
|
this.jpegQuality = jpegQuality;
|
|
this.webpQuality = webpQuality;
|
|
}
|
|
|
|
public void enqueue(AttachmentEntity attachment, int attempt, Instant availableAt) {
|
|
JobPayload payload = new JobPayload(1, attachment.getJobId(), "attachment.process.v1", attachment.getId(),
|
|
attachment.getPipelineVersion(), attempt, attachment.getCreatedAt());
|
|
outbox.save(new OutboxMessage(UUID.randomUUID(), attachment.getJobId(), attachment.getId(), attempt,
|
|
json.writeValueAsString(payload), availableAt, now()));
|
|
}
|
|
|
|
public void handle(String body, boolean redelivered) {
|
|
JobPayload job = parse(body);
|
|
Claim claim = transactions.execute(status -> claim(job, redelivered));
|
|
if (claim == null || !claim.process()) {
|
|
if (claim != null && claim.dlq()) {
|
|
publishFailed(body);
|
|
}
|
|
return;
|
|
}
|
|
try {
|
|
try {
|
|
Processed processed = process(claim);
|
|
Boolean saved = transactions.execute(status -> finishReady(claim, processed));
|
|
if (Boolean.TRUE.equals(saved)) {
|
|
storage.deleteQuietly(claim.stagingKey());
|
|
}
|
|
} catch (PermanentJobException exception) {
|
|
Boolean saved = transactions.execute(status -> finishFailed(claim, exception.errorCode()));
|
|
if (Boolean.TRUE.equals(saved)) {
|
|
publishFailed(body);
|
|
}
|
|
} catch (TransientJobException exception) {
|
|
log.info("transient job failure attachment={} type={}", claim.attachmentId(), exception.getClass().getSimpleName());
|
|
transactions.executeWithoutResult(status -> finishRetry(claim));
|
|
}
|
|
} finally {
|
|
localLeases.remove(claim.leaseToken());
|
|
}
|
|
}
|
|
|
|
public void rejectPoison(String body) {
|
|
log.warn("poison job");
|
|
publishFailed(body);
|
|
}
|
|
|
|
public AttachmentEntity replay(UUID attachmentId) {
|
|
return transactions.execute(status -> {
|
|
AttachmentEntity attachment = attachments.lockById(attachmentId).orElseThrow(NotFoundException::new);
|
|
if ("ready".equals(attachment.getStatus())) {
|
|
return attachment;
|
|
}
|
|
if (!"failed".equals(attachment.getStatus())) {
|
|
throw new ConflictException("Задание ещё выполняется");
|
|
}
|
|
Instant now = now();
|
|
attachment.prepareReplay(now);
|
|
for (FailedJobEntity failed : failedJobs.findByAttachmentIdAndReplayedAtIsNull(attachmentId)) {
|
|
failed.markReplayed(now);
|
|
}
|
|
enqueue(attachment, 1, now);
|
|
log.info("replay attachment={}", attachmentId);
|
|
return attachment;
|
|
});
|
|
}
|
|
|
|
public int reapExpired() {
|
|
int recovered = 0;
|
|
for (UUID id : attachments.findExpiredLeaseIds(Instant.now())) {
|
|
Boolean changed = transactions.execute(status -> recoverOne(id));
|
|
if (Boolean.TRUE.equals(changed)) {
|
|
recovered++;
|
|
}
|
|
}
|
|
return recovered;
|
|
}
|
|
|
|
private boolean recoverOne(UUID id) {
|
|
AttachmentEntity attachment = attachments.lockById(id).orElse(null);
|
|
Instant now = now();
|
|
if (attachment == null || !"processing".equals(attachment.getStatus()) || attachment.leaseHeld(now)) {
|
|
return false;
|
|
}
|
|
if (attachment.getAttempt() >= maxAttempts) {
|
|
markFailed(attachment, "retry_exhausted", now);
|
|
return true;
|
|
}
|
|
attachment.markQueued(now);
|
|
enqueue(attachment, attachment.getAttempt() + 1, now);
|
|
log.info("lease expired attachment={} attempt={}", id, attachment.getAttempt());
|
|
return true;
|
|
}
|
|
|
|
private Claim claim(JobPayload job, boolean redelivered) {
|
|
AttachmentEntity attachment = attachments.lockById(job.attachmentId()).orElse(null);
|
|
if (attachment == null || attachment.getPipelineVersion() > job.pipelineVersion()) {
|
|
return Claim.done(false);
|
|
}
|
|
if ("ready".equals(attachment.getStatus()) || "failed".equals(attachment.getStatus())) {
|
|
return Claim.done(false);
|
|
}
|
|
Instant now = now();
|
|
boolean ownedHere = attachment.getLeaseToken() != null && localLeases.contains(attachment.getLeaseToken());
|
|
if ("processing".equals(attachment.getStatus()) && attachment.leaseHeld(now) && ownedHere && !redelivered) {
|
|
return Claim.done(false);
|
|
}
|
|
int next = attachment.getAttempt() + 1;
|
|
if (next > maxAttempts) {
|
|
markFailed(attachment, "retry_exhausted", now);
|
|
return Claim.done(true);
|
|
}
|
|
if (attachment.getStagingKey() == null || attachment.getJobId() == null) {
|
|
markFailed(attachment, "processing_failed", now);
|
|
return Claim.done(true);
|
|
}
|
|
UUID token = UUID.randomUUID();
|
|
attachment.markProcessing(next, token, now.plus(timeout), now);
|
|
localLeases.add(token);
|
|
return new Claim(true, false, attachment.getId(), attachment.getJobId(), token, next, attachment.getStagingKey(),
|
|
attachment.getObjectKey(), attachment.getContentType(), attachment.getSizeBytes(), attachment.getSha256(),
|
|
attachment.getPipelineVersion());
|
|
}
|
|
|
|
private Processed process(Claim claim) {
|
|
long started = System.nanoTime();
|
|
byte[] original = readStaging(claim);
|
|
checkTime(started);
|
|
if (original.length != claim.sizeBytes() || !sha256(original).equals(claim.sha256())) {
|
|
throw new PermanentJobException("invalid_image");
|
|
}
|
|
try {
|
|
ImageFiles.inspect(original, claim.contentType(), maxPixels);
|
|
} catch (UnsupportedMediaException | PayloadTooLargeException exception) {
|
|
throw new PermanentJobException("invalid_image");
|
|
}
|
|
if (failPermanentRequested()) {
|
|
throw new PermanentJobException("processing_failed");
|
|
}
|
|
try {
|
|
checkTime(started);
|
|
storage.put(claim.objectKey(), original, claim.contentType());
|
|
var rendered = ImageVariants.render(original, jpegQuality, webpQuality);
|
|
checkTime(started);
|
|
for (ImageVariants.VariantBytes variant : rendered) {
|
|
storage.put(variantKey(claim.attachmentId(), variant.name()), variant.bytes(), variant.contentType());
|
|
}
|
|
if (crashAfterWriteRequested()) {
|
|
log.warn("fault injection: exit after object write before result commit");
|
|
System.exit(1);
|
|
}
|
|
return new Processed(rendered);
|
|
} catch (PermanentJobException | TransientJobException exception) {
|
|
throw exception;
|
|
} catch (UnavailableException exception) {
|
|
throw new TransientJobException(exception);
|
|
} catch (IOException exception) {
|
|
throw new PermanentJobException("invalid_image");
|
|
} catch (RuntimeException exception) {
|
|
log.warn("transient processing failure type={}", exception.getClass().getSimpleName());
|
|
throw new TransientJobException(exception);
|
|
}
|
|
}
|
|
|
|
private byte[] readStaging(Claim claim) {
|
|
try (InputStream input = storage.get(claim.stagingKey())) {
|
|
return ImageFiles.readLimited(input, maxBytes);
|
|
} catch (PayloadTooLargeException exception) {
|
|
throw new PermanentJobException("invalid_image");
|
|
} catch (NotFoundException exception) {
|
|
throw new PermanentJobException("processing_failed");
|
|
} catch (UnavailableException exception) {
|
|
throw new TransientJobException(exception);
|
|
} catch (IOException exception) {
|
|
throw new TransientJobException(exception);
|
|
}
|
|
}
|
|
|
|
private boolean finishReady(Claim claim, Processed processed) {
|
|
AttachmentEntity attachment = attachments.lockById(claim.attachmentId()).orElse(null);
|
|
if (attachment == null || lost(claim, attachment) || "ready".equals(attachment.getStatus())) {
|
|
return false;
|
|
}
|
|
variants.deleteForPipeline(attachment.getId(), attachment.getPipelineVersion());
|
|
for (ImageVariants.VariantBytes variant : processed.variants()) {
|
|
variants.save(new AttachmentVariantEntity(UUID.randomUUID(), attachment.getId(), attachment.getPipelineVersion(),
|
|
variant.name(), variant.contentType(), variant.bytes().length, variant.width(), variant.height(),
|
|
sha256(variant.bytes()), variantKey(attachment.getId(), variant.name())));
|
|
}
|
|
attachment.markReady(now());
|
|
log.info("attachment ready id={}", attachment.getId());
|
|
return true;
|
|
}
|
|
|
|
private boolean finishFailed(Claim claim, String code) {
|
|
AttachmentEntity attachment = attachments.lockById(claim.attachmentId()).orElse(null);
|
|
if (attachment == null || lost(claim, attachment) || "ready".equals(attachment.getStatus())
|
|
|| "failed".equals(attachment.getStatus())) {
|
|
return false;
|
|
}
|
|
markFailed(attachment, code, now());
|
|
return true;
|
|
}
|
|
|
|
private void publishFailed(String body) {
|
|
BrokerPublisher publisher = brokers.getIfAvailable();
|
|
if (publisher != null) {
|
|
publisher.failed(body);
|
|
}
|
|
}
|
|
|
|
private void finishRetry(Claim claim) {
|
|
AttachmentEntity attachment = attachments.lockById(claim.attachmentId()).orElse(null);
|
|
if (attachment == null || lost(claim, attachment) || "ready".equals(attachment.getStatus())
|
|
|| "failed".equals(attachment.getStatus())) {
|
|
return;
|
|
}
|
|
Instant now = now();
|
|
if (attachment.getAttempt() >= maxAttempts) {
|
|
markFailed(attachment, "retry_exhausted", now);
|
|
return;
|
|
}
|
|
attachment.markQueued(now);
|
|
long delay = RetryDelay.millis(attachment.getAttempt(), backoffBaseMs, backoffCapMs);
|
|
enqueue(attachment, attachment.getAttempt() + 1, now.plusMillis(delay));
|
|
}
|
|
|
|
private void markFailed(AttachmentEntity attachment, String code, Instant now) {
|
|
attachment.markFailed(code, now);
|
|
failedJobs.save(new FailedJobEntity(UUID.randomUUID(), attachment.getJobId(), attachment.getId(), code,
|
|
attachment.getAttempt(), now));
|
|
log.info("attachment failed id={} code={}", attachment.getId(), code);
|
|
}
|
|
|
|
private static boolean lost(Claim claim, AttachmentEntity attachment) {
|
|
return attachment.getLeaseToken() == null || !claim.leaseToken().equals(attachment.getLeaseToken());
|
|
}
|
|
|
|
private void checkTime(long started) {
|
|
if (System.nanoTime() - started > timeout.toNanos()) {
|
|
throw new TransientJobException("timeout");
|
|
}
|
|
}
|
|
|
|
private JobPayload parse(String body) {
|
|
try {
|
|
JobPayload job = json.readValue(body, JobPayload.class);
|
|
if (job == null || job.schemaVersion() != 1 || job.attachmentId() == null || job.jobId() == null
|
|
|| !"attachment.process.v1".equals(job.type()) || job.pipelineVersion() < 1 || job.attempt() < 1) {
|
|
throw new PoisonJobException();
|
|
}
|
|
return job;
|
|
} catch (PoisonJobException exception) {
|
|
throw exception;
|
|
} catch (RuntimeException exception) {
|
|
throw new PoisonJobException();
|
|
}
|
|
}
|
|
|
|
private static boolean crashAfterWriteRequested() {
|
|
try {
|
|
return Files.deleteIfExists(CRASH_AFTER_WRITE);
|
|
} catch (IOException exception) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
private static boolean failPermanentRequested() {
|
|
try {
|
|
return Files.deleteIfExists(FAIL_PERMANENT);
|
|
} catch (IOException exception) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
static String variantKey(UUID attachmentId, String name) {
|
|
return "variants/" + attachmentId + "/" + name;
|
|
}
|
|
|
|
private static String sha256(byte[] bytes) {
|
|
try {
|
|
return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(bytes));
|
|
} catch (NoSuchAlgorithmException exception) {
|
|
throw new IllegalStateException(exception);
|
|
}
|
|
}
|
|
|
|
private static Instant now() {
|
|
return Instant.now().truncatedTo(ChronoUnit.MICROS);
|
|
}
|
|
|
|
private record Claim(boolean process, boolean dlq, UUID attachmentId, UUID jobId, UUID leaseToken, int attempt,
|
|
String stagingKey, String objectKey, String contentType, long sizeBytes, String sha256,
|
|
int pipelineVersion) {
|
|
static Claim done(boolean dlq) {
|
|
return new Claim(false, dlq, null, null, null, 0, null, null, null, 0, null, 0);
|
|
}
|
|
}
|
|
|
|
private record Processed(java.util.List<ImageVariants.VariantBytes> variants) {
|
|
}
|
|
}
|