forked from mtusi2026/mtusi2026lab5
all for 4
This commit is contained in:
@@ -103,6 +103,17 @@
|
||||
<version>8.5.17</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-amqp</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.drewnoakes</groupId>
|
||||
<artifactId>metadata-extractor</artifactId>
|
||||
<version>2.19.0</version>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
package com.vnp.demo.config;
|
||||
|
||||
import org.springframework.amqp.core.Declarables;
|
||||
import org.springframework.amqp.core.Queue;
|
||||
import org.springframework.amqp.core.QueueBuilder;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
@Configuration
|
||||
public class RabbitConfig {
|
||||
|
||||
public static final String QUEUE = "attachments.v1";
|
||||
public static final String DLQ = "attachments.v1.dlq";
|
||||
|
||||
@Bean
|
||||
public Declarables attachmentQueues() {
|
||||
return new Declarables(
|
||||
new Queue(DLQ, true),
|
||||
|
||||
QueueBuilder.durable(QUEUE)
|
||||
.deadLetterExchange("")
|
||||
.deadLetterRoutingKey(DLQ)
|
||||
.build()
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -37,7 +37,7 @@ public class AttachmentController {
|
||||
}
|
||||
|
||||
@PostMapping("/chats/{chat_id}/attachments")
|
||||
@ResponseStatus(HttpStatus.CREATED)
|
||||
@ResponseStatus(HttpStatus.ACCEPTED)
|
||||
public AttachmentResponse upload(
|
||||
@AuthenticationPrincipal UUID userId,
|
||||
@PathVariable("chat_id") UUID chatId,
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
package com.vnp.demo.dto;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.Objects;
|
||||
import java.util.UUID;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
public record AttachmentJobMessage(
|
||||
@JsonProperty("schema_version")
|
||||
int schemaVersion,
|
||||
|
||||
@JsonProperty("job_id")
|
||||
UUID jobId,
|
||||
|
||||
String type,
|
||||
|
||||
@JsonProperty("attachment_id")
|
||||
UUID attachmentId,
|
||||
|
||||
@JsonProperty("pipeline_version")
|
||||
int pipelineVersion,
|
||||
|
||||
int attempt,
|
||||
|
||||
@JsonProperty("created_at")
|
||||
Instant createdAt
|
||||
) {
|
||||
public static final int SCHEMA_VERSION = 1;
|
||||
|
||||
public static final String TYPE =
|
||||
"attachment.process.v1";
|
||||
|
||||
public AttachmentJobMessage {
|
||||
if (schemaVersion != SCHEMA_VERSION) {
|
||||
throw new IllegalArgumentException(
|
||||
"Unsupported job schema version"
|
||||
);
|
||||
}
|
||||
|
||||
if (!TYPE.equals(type)) {
|
||||
throw new IllegalArgumentException(
|
||||
"Unsupported job type"
|
||||
);
|
||||
}
|
||||
|
||||
Objects.requireNonNull(jobId, "jobId");
|
||||
Objects.requireNonNull(attachmentId, "attachmentId");
|
||||
Objects.requireNonNull(createdAt, "createdAt");
|
||||
|
||||
if (pipelineVersion < 1) {
|
||||
throw new IllegalArgumentException(
|
||||
"pipelineVersion must be positive"
|
||||
);
|
||||
}
|
||||
|
||||
if (attempt < 1) {
|
||||
throw new IllegalArgumentException(
|
||||
"attempt must be positive"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
public static AttachmentJobMessage create(
|
||||
UUID jobId,
|
||||
UUID attachmentId,
|
||||
int pipelineVersion,
|
||||
int attempt,
|
||||
Instant createdAt
|
||||
) {
|
||||
return new AttachmentJobMessage(
|
||||
SCHEMA_VERSION,
|
||||
jobId,
|
||||
TYPE,
|
||||
attachmentId,
|
||||
pipelineVersion,
|
||||
attempt,
|
||||
createdAt
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,19 +1,11 @@
|
||||
package com.vnp.demo.service;
|
||||
|
||||
import java.time.Instant;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import com.vnp.demo.service.AttachmentPersistenceService.CleanupResult;
|
||||
|
||||
import io.minio.Result;
|
||||
import io.minio.messages.Item;
|
||||
|
||||
@Component
|
||||
@ConditionalOnProperty(
|
||||
name = "app.cleanup.enabled",
|
||||
@@ -25,94 +17,24 @@ public class AttachmentCleanupJob {
|
||||
private static final Logger log =
|
||||
LoggerFactory.getLogger(AttachmentCleanupJob.class);
|
||||
|
||||
private final S3StorageService storage;
|
||||
private final AttachmentPersistenceService persistence;
|
||||
private final long graceSeconds;
|
||||
private final AttachmentPipeline pipeline;
|
||||
|
||||
public AttachmentCleanupJob(
|
||||
S3StorageService storage,
|
||||
AttachmentPersistenceService persistence,
|
||||
@Value("${app.cleanup.grace-seconds:3600}")
|
||||
long graceSeconds
|
||||
) {
|
||||
if (graceSeconds < 60) {
|
||||
throw new IllegalArgumentException(
|
||||
"Grace period должен быть не меньше 60 секунд"
|
||||
);
|
||||
}
|
||||
|
||||
this.storage = storage;
|
||||
this.persistence = persistence;
|
||||
this.graceSeconds = graceSeconds;
|
||||
public AttachmentCleanupJob(AttachmentPipeline pipeline) {
|
||||
this.pipeline = pipeline;
|
||||
}
|
||||
|
||||
@Scheduled(
|
||||
initialDelayString =
|
||||
"${app.cleanup.initial-delay-ms:60000}",
|
||||
fixedDelayString =
|
||||
"${app.cleanup.interval-ms:300000}"
|
||||
initialDelayString = "${app.cleanup.initial-delay-ms:60000}",
|
||||
fixedDelayString = "${app.cleanup.interval-ms:300000}"
|
||||
)
|
||||
public void cleanup() {
|
||||
Instant cutoff = Instant.now()
|
||||
.minusSeconds(graceSeconds);
|
||||
|
||||
long scanned = 0;
|
||||
long candidates = 0;
|
||||
long deleted = 0;
|
||||
|
||||
try {
|
||||
for (Result<Item> result : storage.listOriginals()) {
|
||||
Item item = result.get();
|
||||
pipeline.cleanup();
|
||||
|
||||
if (item.isDir()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
scanned++;
|
||||
|
||||
Instant lastModified = item.lastModified()
|
||||
.toInstant();
|
||||
|
||||
if (!lastModified.isBefore(cutoff)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
candidates++;
|
||||
|
||||
CleanupResult outcome =
|
||||
persistence.removeIfOrphan(
|
||||
item.objectName(),
|
||||
cutoff
|
||||
);
|
||||
|
||||
if (outcome == CleanupResult.DELETED) {
|
||||
deleted++;
|
||||
|
||||
log.info(
|
||||
"Attachment cleanup removed orphan: {}",
|
||||
item.objectName()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
log.info(
|
||||
"Attachment cleanup completed: "
|
||||
+ "scanned={}, candidates={}, deleted={}",
|
||||
scanned,
|
||||
candidates,
|
||||
deleted
|
||||
);
|
||||
} catch (Exception e) {
|
||||
// Прерываем текущий проход при ошибке БД или S3.
|
||||
// Следующий запуск повторит проверку.
|
||||
// Не выводим credentials или содержимое ответа S3.
|
||||
} catch (Exception error) {
|
||||
log.warn(
|
||||
"Attachment cleanup interrupted: "
|
||||
+ "scanned={}, candidates={}, deleted={}, errorType={}",
|
||||
scanned,
|
||||
candidates,
|
||||
deleted,
|
||||
e.getClass().getSimpleName()
|
||||
"Attachment cleanup interrupted: {}",
|
||||
error.getClass().getSimpleName()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -13,101 +13,95 @@ import com.vnp.demo.models.Attachment;
|
||||
import com.vnp.demo.repositories.AttachmentRepository;
|
||||
import com.vnp.demo.repositories.ChatMemberRepository;
|
||||
import com.vnp.demo.validation.ImageValidator;
|
||||
import com.vnp.demo.validation.ImageValidator.ValidatedImage;
|
||||
|
||||
@Service
|
||||
public class AttachmentService {
|
||||
|
||||
private final AttachmentRepository attachmentRepository;
|
||||
private final ChatMemberRepository chatMemberRepository;
|
||||
private final ImageValidator imageValidator;
|
||||
private final AttachmentRepository attachments;
|
||||
private final ChatMemberRepository members;
|
||||
private final ImageValidator validator;
|
||||
private final UploadLimiter limiter;
|
||||
private final S3StorageService storage;
|
||||
private final UploadLimiter uploadLimiter;
|
||||
private final AttachmentPersistenceService persistence;
|
||||
private final AttachmentPipeline pipeline;
|
||||
|
||||
public AttachmentService(
|
||||
AttachmentRepository attachmentRepository,
|
||||
ChatMemberRepository chatMemberRepository,
|
||||
ImageValidator imageValidator,
|
||||
AttachmentRepository attachments,
|
||||
ChatMemberRepository members,
|
||||
ImageValidator validator,
|
||||
UploadLimiter limiter,
|
||||
S3StorageService storage,
|
||||
UploadLimiter uploadLimiter,
|
||||
AttachmentPersistenceService persistence
|
||||
AttachmentPipeline pipeline
|
||||
) {
|
||||
this.attachmentRepository = attachmentRepository;
|
||||
this.chatMemberRepository = chatMemberRepository;
|
||||
this.imageValidator = imageValidator;
|
||||
this.attachments = attachments;
|
||||
this.members = members;
|
||||
this.validator = validator;
|
||||
this.limiter = limiter;
|
||||
this.storage = storage;
|
||||
this.uploadLimiter = uploadLimiter;
|
||||
this.persistence = persistence;
|
||||
this.pipeline = pipeline;
|
||||
}
|
||||
|
||||
public AttachmentResponse upload(
|
||||
UUID userId,
|
||||
UUID chatId,
|
||||
String filename,
|
||||
String contentType,
|
||||
long contentLength,
|
||||
InputStream input
|
||||
) {
|
||||
checkChatAccess(userId, chatId);
|
||||
UUID userId,
|
||||
UUID chatId,
|
||||
String filename,
|
||||
String contentType,
|
||||
long contentLength,
|
||||
InputStream input
|
||||
) {
|
||||
checkChatAccess(userId, chatId);
|
||||
validator.validateFilename(filename);
|
||||
|
||||
imageValidator.validateFilename(filename);
|
||||
try (var permit = limiter.acquire();
|
||||
var image = validator.validate(
|
||||
input,
|
||||
contentLength,
|
||||
contentType
|
||||
)) {
|
||||
|
||||
try (UploadLimiter.Permit permit = uploadLimiter.acquire();
|
||||
ValidatedImage image = imageValidator.validate(
|
||||
input,
|
||||
contentLength,
|
||||
contentType
|
||||
)) {
|
||||
UUID id = UUID.randomUUID();
|
||||
|
||||
UUID attachmentId = UUID.randomUUID();
|
||||
Instant now = Instant.now()
|
||||
.truncatedTo(ChronoUnit.MICROS);
|
||||
|
||||
String objectKey = "originals/"
|
||||
+ chatId
|
||||
+ "/"
|
||||
+ attachmentId;
|
||||
Attachment attachment = new Attachment(
|
||||
id,
|
||||
chatId,
|
||||
userId,
|
||||
filename,
|
||||
image.contentType(),
|
||||
image.sizeBytes(),
|
||||
image.sha256(),
|
||||
|
||||
Instant now = Instant.now()
|
||||
.truncatedTo(ChronoUnit.MICROS);
|
||||
"originals/"
|
||||
+ chatId
|
||||
+ "/"
|
||||
+ id
|
||||
+ "/pending",
|
||||
|
||||
Attachment attachment = new Attachment(
|
||||
attachmentId,
|
||||
chatId,
|
||||
userId,
|
||||
filename,
|
||||
image.contentType(),
|
||||
image.sizeBytes(),
|
||||
image.sha256(),
|
||||
objectKey,
|
||||
"ready",
|
||||
now,
|
||||
now
|
||||
);
|
||||
"queued",
|
||||
now,
|
||||
now
|
||||
);
|
||||
|
||||
persistence.save(attachment, image.path());
|
||||
pipeline.enqueue(attachment, image.path());
|
||||
|
||||
return AttachmentResponse.from(attachment);
|
||||
}
|
||||
return AttachmentResponse.from(attachment);
|
||||
}
|
||||
}
|
||||
|
||||
public AttachmentResponse getMetadata(
|
||||
UUID userId,
|
||||
UUID attachmentId
|
||||
UUID id
|
||||
) {
|
||||
return AttachmentResponse.from(
|
||||
getAccessibleAttachment(userId, attachmentId)
|
||||
);
|
||||
return pipeline.metadata(accessible(userId, id));
|
||||
}
|
||||
|
||||
public Download download(
|
||||
UUID userId,
|
||||
UUID attachmentId,
|
||||
UUID id,
|
||||
String variant
|
||||
) {
|
||||
Attachment attachment = getAccessibleAttachment(
|
||||
userId,
|
||||
attachmentId
|
||||
);
|
||||
Attachment attachment = accessible(userId, id);
|
||||
|
||||
if (!"ready".equals(attachment.status())) {
|
||||
throw BusinessException.conflict(
|
||||
@@ -115,29 +109,36 @@ public class AttachmentService {
|
||||
);
|
||||
}
|
||||
|
||||
if (!"original".equals(variant)) {
|
||||
throw BusinessException.notFound();
|
||||
if ("original".equals(variant)) {
|
||||
return new Download(
|
||||
attachment.filename(),
|
||||
attachment.contentType(),
|
||||
storage.get(
|
||||
attachment.objectKey(),
|
||||
attachment.sizeBytes(),
|
||||
attachment.sha256()
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
byte[] bytes = storage.get(
|
||||
attachment.objectKey(),
|
||||
attachment.sizeBytes(),
|
||||
attachment.sha256()
|
||||
);
|
||||
var selected = pipeline.variants(id).stream()
|
||||
.filter(item -> item.name().equals(variant))
|
||||
.findFirst()
|
||||
.orElseThrow(BusinessException::notFound);
|
||||
|
||||
return new Download(
|
||||
attachment.filename(),
|
||||
attachment.contentType(),
|
||||
bytes
|
||||
selected.name() + ".jpg",
|
||||
selected.contentType(),
|
||||
storage.get(
|
||||
selected.objectKey(),
|
||||
selected.sizeBytes(),
|
||||
selected.sha256()
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
private Attachment getAccessibleAttachment(
|
||||
UUID userId,
|
||||
UUID attachmentId
|
||||
) {
|
||||
Attachment attachment = attachmentRepository
|
||||
.findById(attachmentId)
|
||||
private Attachment accessible(UUID userId, UUID id) {
|
||||
Attachment attachment = attachments.findById(id)
|
||||
.orElseThrow(BusinessException::notFound);
|
||||
|
||||
checkChatAccess(userId, attachment.chatId());
|
||||
@@ -146,10 +147,7 @@ public class AttachmentService {
|
||||
}
|
||||
|
||||
private void checkChatAccess(UUID userId, UUID chatId) {
|
||||
boolean member = chatMemberRepository
|
||||
.existsByIdChatIdAndIdUserId(chatId, userId);
|
||||
|
||||
if (!member) {
|
||||
if (!members.existsByIdChatIdAndIdUserId(chatId, userId)) {
|
||||
throw BusinessException.notFound();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
package com.vnp.demo.service;
|
||||
|
||||
import java.util.UUID;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.transaction.PlatformTransactionManager;
|
||||
import org.springframework.transaction.TransactionDefinition;
|
||||
import org.springframework.transaction.support.TransactionTemplate;
|
||||
|
||||
@Component
|
||||
public class FaultGate {
|
||||
|
||||
private final boolean enabled;
|
||||
private final JdbcTemplate jdbc;
|
||||
private final TransactionTemplate tx;
|
||||
|
||||
public FaultGate(
|
||||
JdbcTemplate jdbc,
|
||||
PlatformTransactionManager manager,
|
||||
|
||||
@Value("${FAULT_INJECTION_ENABLED:false}")
|
||||
boolean enabled
|
||||
) {
|
||||
this.jdbc = jdbc;
|
||||
this.enabled = enabled;
|
||||
|
||||
tx = new TransactionTemplate(manager);
|
||||
|
||||
tx.setPropagationBehavior(
|
||||
TransactionDefinition.PROPAGATION_REQUIRES_NEW
|
||||
);
|
||||
|
||||
tx.setTimeout(5);
|
||||
}
|
||||
|
||||
public boolean paused() {
|
||||
if (!enabled) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return Boolean.TRUE.equals(
|
||||
jdbc.queryForObject(
|
||||
"""
|
||||
SELECT EXISTS (
|
||||
SELECT 1
|
||||
FROM attachment_test_faults
|
||||
WHERE point = 'hold_publish'
|
||||
AND remaining > 0
|
||||
)
|
||||
""",
|
||||
Boolean.class
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
public boolean hit(UUID job, String point) {
|
||||
if (!enabled) {
|
||||
return false;
|
||||
}
|
||||
|
||||
boolean hit = Boolean.TRUE.equals(
|
||||
tx.execute(ignored ->
|
||||
jdbc.update(
|
||||
"""
|
||||
UPDATE attachment_test_faults
|
||||
SET remaining = remaining - 1,
|
||||
hits = hits + 1,
|
||||
job_id = ?,
|
||||
hit_at = clock_timestamp()
|
||||
WHERE point = ?
|
||||
AND remaining > 0
|
||||
""",
|
||||
job,
|
||||
point
|
||||
) == 1
|
||||
)
|
||||
);
|
||||
|
||||
if (!hit) {
|
||||
return false;
|
||||
}
|
||||
|
||||
switch (point) {
|
||||
case "before_publish",
|
||||
"after_objects",
|
||||
"before_ack" ->
|
||||
Runtime.getRuntime().halt(137);
|
||||
|
||||
case "permanent_error" ->
|
||||
throw new ImageProcessor.PermanentFailure();
|
||||
|
||||
case "transient_error" ->
|
||||
throw new IllegalStateException(
|
||||
"injected_transient_error"
|
||||
);
|
||||
|
||||
default -> {
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,274 @@
|
||||
package com.vnp.demo.service;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.Semaphore;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
@Component
|
||||
public class ImageProcessor {
|
||||
|
||||
public static class PermanentFailure extends RuntimeException {
|
||||
public PermanentFailure() {
|
||||
super("invalid_image");
|
||||
}
|
||||
}
|
||||
|
||||
public static class TransientFailure extends RuntimeException {
|
||||
private final String code;
|
||||
|
||||
public TransientFailure(String code) {
|
||||
super(code);
|
||||
this.code = code;
|
||||
}
|
||||
|
||||
public String code() {
|
||||
return code;
|
||||
}
|
||||
}
|
||||
|
||||
public record Info(
|
||||
String name,
|
||||
|
||||
@JsonProperty("content_type")
|
||||
String contentType,
|
||||
|
||||
int width,
|
||||
int height
|
||||
) {
|
||||
}
|
||||
|
||||
public record Generated(
|
||||
Info info,
|
||||
byte[] bytes
|
||||
) {
|
||||
}
|
||||
|
||||
private final ObjectMapper mapper;
|
||||
|
||||
private final Set<Path> active =
|
||||
ConcurrentHashMap.newKeySet();
|
||||
|
||||
private final Semaphore slots = new Semaphore(1);
|
||||
private final Path root = Path.of("/tmp/image-jobs");
|
||||
|
||||
public ImageProcessor(ObjectMapper mapper) {
|
||||
this.mapper = mapper;
|
||||
}
|
||||
|
||||
public List<Generated> render(
|
||||
byte[] source,
|
||||
boolean testHang
|
||||
) {
|
||||
if (source.length == 0 || source.length > 10485760) {
|
||||
throw new PermanentFailure();
|
||||
}
|
||||
|
||||
if (!slots.tryAcquire()) {
|
||||
throw new TransientFailure("processor_busy");
|
||||
}
|
||||
|
||||
Path directory = null;
|
||||
Process process = null;
|
||||
|
||||
try {
|
||||
Files.createDirectories(root);
|
||||
|
||||
directory = Files.createTempDirectory(root, "job-");
|
||||
active.add(directory);
|
||||
|
||||
Path input = directory.resolve("input.bin");
|
||||
Files.write(input, source);
|
||||
|
||||
ProcessBuilder builder = new ProcessBuilder(
|
||||
"/opt/images/bin/python",
|
||||
"/app/process_image.py",
|
||||
input.toString(),
|
||||
directory.toString()
|
||||
);
|
||||
|
||||
builder.redirectOutput(
|
||||
ProcessBuilder.Redirect.DISCARD
|
||||
);
|
||||
|
||||
builder.redirectError(
|
||||
ProcessBuilder.Redirect.DISCARD
|
||||
);
|
||||
|
||||
builder.environment().remove("IMAGE_TEST_SLEEP");
|
||||
|
||||
if (testHang) {
|
||||
builder.environment().put(
|
||||
"IMAGE_TEST_SLEEP",
|
||||
"1"
|
||||
);
|
||||
}
|
||||
|
||||
process = builder.start();
|
||||
|
||||
if (!process.waitFor(25, TimeUnit.SECONDS)) {
|
||||
throw new TransientFailure(
|
||||
"processing_timeout"
|
||||
);
|
||||
}
|
||||
|
||||
if (process.exitValue() == 10) {
|
||||
throw new PermanentFailure();
|
||||
}
|
||||
|
||||
if (process.exitValue() != 0) {
|
||||
throw new TransientFailure(
|
||||
"processing_resource_limit"
|
||||
);
|
||||
}
|
||||
|
||||
Path manifest = directory.resolve("manifest.json");
|
||||
|
||||
if (Files.size(manifest) > 4096) {
|
||||
throw new TransientFailure("invalid_manifest");
|
||||
}
|
||||
|
||||
Info[] infos = mapper.readValue(
|
||||
Files.readAllBytes(manifest),
|
||||
Info[].class
|
||||
);
|
||||
|
||||
Set<String> names = new HashSet<>();
|
||||
List<Generated> result = new ArrayList<>();
|
||||
|
||||
for (Info info : infos) {
|
||||
String mime = switch (info.name()) {
|
||||
case "thumbnail", "crop" -> "image/jpeg";
|
||||
case "optimized" -> "image/webp";
|
||||
default -> throw new TransientFailure(
|
||||
"invalid_variant"
|
||||
);
|
||||
};
|
||||
|
||||
if (!names.add(info.name())
|
||||
|| !mime.equals(info.contentType())
|
||||
|| info.width() < 1
|
||||
|| info.height() < 1
|
||||
|| info.width() > 1280
|
||||
|| info.height() > 1280) {
|
||||
throw new TransientFailure(
|
||||
"invalid_manifest"
|
||||
);
|
||||
}
|
||||
|
||||
String extension = mime.equals("image/webp")
|
||||
? ".webp"
|
||||
: ".jpg";
|
||||
|
||||
Path file = directory.resolve(
|
||||
info.name() + extension
|
||||
);
|
||||
|
||||
long size = Files.size(file);
|
||||
|
||||
if (size < 1 || size > 10485760) {
|
||||
throw new TransientFailure(
|
||||
"invalid_variant_size"
|
||||
);
|
||||
}
|
||||
|
||||
result.add(
|
||||
new Generated(
|
||||
info,
|
||||
Files.readAllBytes(file)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
if (!names.equals(
|
||||
Set.of("thumbnail", "crop", "optimized")
|
||||
)) {
|
||||
throw new TransientFailure(
|
||||
"incomplete_variants"
|
||||
);
|
||||
}
|
||||
|
||||
return result;
|
||||
|
||||
} catch (PermanentFailure | TransientFailure error) {
|
||||
throw error;
|
||||
|
||||
} catch (Exception error) {
|
||||
if (error instanceof InterruptedException) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
|
||||
throw new TransientFailure(
|
||||
"processing_unavailable"
|
||||
);
|
||||
|
||||
} finally {
|
||||
if (process != null && process.isAlive()) {
|
||||
process.destroyForcibly();
|
||||
|
||||
try {
|
||||
process.waitFor(5, TimeUnit.SECONDS);
|
||||
} catch (InterruptedException error) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
|
||||
if (directory != null) {
|
||||
active.remove(directory);
|
||||
remove(directory);
|
||||
}
|
||||
|
||||
slots.release();
|
||||
}
|
||||
}
|
||||
|
||||
private void remove(Path directory) {
|
||||
try (var paths = Files.walk(directory)) {
|
||||
for (Path path : paths
|
||||
.sorted(Comparator.reverseOrder())
|
||||
.toList()) {
|
||||
|
||||
Files.deleteIfExists(path);
|
||||
}
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
}
|
||||
|
||||
@Scheduled(
|
||||
initialDelay = 60000,
|
||||
fixedDelay = 300000
|
||||
)
|
||||
public void cleanTemporaryFiles() {
|
||||
if (!Files.isDirectory(root)) {
|
||||
return;
|
||||
}
|
||||
|
||||
try (var children = Files.list(root)) {
|
||||
for (Path path : children.toList()) {
|
||||
if (Files.isDirectory(path)
|
||||
&& !active.contains(path)
|
||||
&& Files.getLastModifiedTime(path)
|
||||
.toInstant()
|
||||
.isBefore(
|
||||
Instant.now().minusSeconds(3600)
|
||||
)) {
|
||||
remove(path);
|
||||
}
|
||||
}
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -219,25 +219,25 @@ public class MessageService {
|
||||
}
|
||||
|
||||
private void validateAttachments(
|
||||
UUID chatId,
|
||||
List<UUID> attachmentIds
|
||||
) {
|
||||
UUID chatId,
|
||||
List<UUID> attachmentIds
|
||||
) {
|
||||
for (UUID attachmentId : attachmentIds) {
|
||||
Attachment attachment = attachmentRepository
|
||||
.findById(attachmentId)
|
||||
.orElseThrow(BusinessException::notFound);
|
||||
Attachment attachment = attachmentRepository
|
||||
.findById(attachmentId)
|
||||
.orElseThrow(BusinessException::notFound);
|
||||
|
||||
if (!chatId.equals(attachment.chatId())) {
|
||||
if (!chatId.equals(attachment.chatId())) {
|
||||
throw BusinessException.notFound();
|
||||
}
|
||||
}
|
||||
|
||||
if (!"ready".equals(attachment.status())) {
|
||||
if ("failed".equals(attachment.status())) {
|
||||
throw BusinessException.conflict(
|
||||
"Вложение не готово"
|
||||
"Обработка вложения завершилась ошибкой"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Transactional(readOnly = true)
|
||||
public MessagePageResponse list(
|
||||
|
||||
@@ -0,0 +1,261 @@
|
||||
package com.vnp.demo.service;
|
||||
|
||||
import com.vnp.demo.repositories.AttachmentRepository;
|
||||
|
||||
import org.springframework.boot.ApplicationArguments;
|
||||
import org.springframework.boot.ApplicationRunner;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.context.ConfigurableApplicationContext;
|
||||
import org.springframework.core.env.Environment;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.transaction.PlatformTransactionManager;
|
||||
import org.springframework.transaction.support.TransactionTemplate;
|
||||
|
||||
import java.util.UUID;
|
||||
|
||||
@Component
|
||||
@ConditionalOnProperty(
|
||||
name = "replay.enabled",
|
||||
havingValue = "true"
|
||||
)
|
||||
public class ReplayRunner implements ApplicationRunner {
|
||||
|
||||
private final JdbcTemplate jdbc;
|
||||
private final AttachmentRepository attachments;
|
||||
private final S3StorageService storage;
|
||||
private final AttachmentPipeline pipeline;
|
||||
private final TransactionTemplate tx;
|
||||
private final Environment env;
|
||||
private final ConfigurableApplicationContext context;
|
||||
|
||||
public ReplayRunner(
|
||||
JdbcTemplate jdbc,
|
||||
AttachmentRepository attachments,
|
||||
S3StorageService storage,
|
||||
AttachmentPipeline pipeline,
|
||||
PlatformTransactionManager manager,
|
||||
Environment env,
|
||||
ConfigurableApplicationContext context
|
||||
) {
|
||||
this.jdbc = jdbc;
|
||||
this.attachments = attachments;
|
||||
this.storage = storage;
|
||||
this.pipeline = pipeline;
|
||||
this.env = env;
|
||||
this.context = context;
|
||||
|
||||
tx = new TransactionTemplate(manager);
|
||||
tx.setTimeout(60);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run(ApplicationArguments ignored) {
|
||||
int code = 0;
|
||||
|
||||
try {
|
||||
UUID id = UUID.fromString(
|
||||
env.getRequiredProperty("REPLAY_JOB_ID")
|
||||
);
|
||||
|
||||
String actor = env.getRequiredProperty(
|
||||
"REPLAY_ACTOR"
|
||||
);
|
||||
|
||||
String reason = env.getRequiredProperty(
|
||||
"REPLAY_REASON"
|
||||
);
|
||||
|
||||
int budget = env.getProperty(
|
||||
"app.jobs.max-attempts",
|
||||
Integer.class,
|
||||
3
|
||||
);
|
||||
|
||||
if (!actor.matches("[A-Za-z0-9_.-]{1,64}")
|
||||
|| reason.isBlank()
|
||||
|| reason.length() > 255
|
||||
|| budget < 1
|
||||
|| budget > 100) {
|
||||
throw new IllegalArgumentException(
|
||||
"invalid_replay_request"
|
||||
);
|
||||
}
|
||||
|
||||
String result = tx.execute(status -> {
|
||||
var job = jdbc.queryForMap(
|
||||
"""
|
||||
SELECT *
|
||||
FROM attachment_jobs
|
||||
WHERE id = ?
|
||||
FOR UPDATE
|
||||
""",
|
||||
id
|
||||
);
|
||||
|
||||
UUID attachmentId =
|
||||
(UUID) job.get("attachment_id");
|
||||
|
||||
var attachment = attachments
|
||||
.findById(attachmentId)
|
||||
.orElseThrow();
|
||||
|
||||
int version = ((Number) job.get(
|
||||
"pipeline_version"
|
||||
)).intValue();
|
||||
|
||||
int attempts = ((Number) job.get(
|
||||
"attempts"
|
||||
)).intValue();
|
||||
|
||||
Integer activeVersion = jdbc.queryForObject(
|
||||
"""
|
||||
SELECT pipeline_version
|
||||
FROM attachments
|
||||
WHERE id = ?
|
||||
""",
|
||||
Integer.class,
|
||||
attachmentId
|
||||
);
|
||||
|
||||
if (version != 1
|
||||
|| activeVersion == null
|
||||
|| activeVersion != version) {
|
||||
throw new IllegalStateException(
|
||||
"obsolete_pipeline"
|
||||
);
|
||||
}
|
||||
|
||||
String action;
|
||||
|
||||
if ("ready".equals(job.get("status"))) {
|
||||
pipeline.verifyReady(attachment);
|
||||
action = "already_ready";
|
||||
|
||||
} else {
|
||||
if (!"failed".equals(job.get("status"))
|
||||
|| !"failed".equals(
|
||||
attachment.status()
|
||||
)) {
|
||||
throw new IllegalStateException(
|
||||
"job_is_not_failed"
|
||||
);
|
||||
}
|
||||
|
||||
String staging = jdbc.queryForObject(
|
||||
"""
|
||||
SELECT staging_key
|
||||
FROM attachments
|
||||
WHERE id = ?
|
||||
""",
|
||||
String.class,
|
||||
attachmentId
|
||||
);
|
||||
|
||||
if (staging == null) {
|
||||
throw new IllegalStateException(
|
||||
"staging_expired"
|
||||
);
|
||||
}
|
||||
|
||||
/*
|
||||
* Та же блокировка используется очисткой.
|
||||
* Staging нельзя удалить во время replay.
|
||||
*/
|
||||
attachments.lockObject(staging);
|
||||
|
||||
storage.get(
|
||||
staging,
|
||||
attachment.sizeBytes(),
|
||||
attachment.sha256()
|
||||
);
|
||||
|
||||
jdbc.update(
|
||||
"""
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'queued',
|
||||
max_attempts = ?,
|
||||
available_at = clock_timestamp(),
|
||||
lease_owner = NULL,
|
||||
lease_until = NULL,
|
||||
last_error_code = NULL,
|
||||
updated_at = clock_timestamp()
|
||||
WHERE id = ?
|
||||
""",
|
||||
Math.addExact(attempts, budget),
|
||||
id
|
||||
);
|
||||
|
||||
jdbc.update(
|
||||
"""
|
||||
UPDATE attachments
|
||||
SET status = 'queued',
|
||||
error_code = NULL,
|
||||
updated_at = clock_timestamp()
|
||||
WHERE id = ?
|
||||
""",
|
||||
attachmentId
|
||||
);
|
||||
|
||||
jdbc.update(
|
||||
"""
|
||||
INSERT INTO attachment_outbox(
|
||||
id,
|
||||
job_id,
|
||||
attempt
|
||||
)
|
||||
VALUES (?, ?, ?)
|
||||
""",
|
||||
UUID.randomUUID(),
|
||||
id,
|
||||
attempts + 1
|
||||
);
|
||||
|
||||
action = "enqueued";
|
||||
}
|
||||
|
||||
jdbc.update(
|
||||
"""
|
||||
INSERT INTO attachment_replay_audit(
|
||||
job_id,
|
||||
actor,
|
||||
reason,
|
||||
action,
|
||||
previous_attempts
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
""",
|
||||
id,
|
||||
actor,
|
||||
reason,
|
||||
action,
|
||||
attempts
|
||||
);
|
||||
|
||||
return action;
|
||||
});
|
||||
|
||||
System.out.println(
|
||||
"REPLAY_RESULT=" + result
|
||||
);
|
||||
|
||||
} catch (Exception error) {
|
||||
code = 1;
|
||||
|
||||
System.err.println(
|
||||
"REPLAY_FAILED: "
|
||||
+ error.getClass().getSimpleName()
|
||||
);
|
||||
}
|
||||
|
||||
final int exitCode = code;
|
||||
|
||||
System.exit(
|
||||
SpringApplication.exit(
|
||||
context,
|
||||
() -> exitCode
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -184,6 +184,44 @@ public class S3StorageService {
|
||||
}
|
||||
}
|
||||
|
||||
public void putBytes(
|
||||
String objectKey,
|
||||
byte[] bytes,
|
||||
String contentType
|
||||
) {
|
||||
if (bytes.length < 1
|
||||
|| bytes.length > ImageValidator.MAX_BYTES) {
|
||||
throw new IllegalArgumentException("Invalid object size");
|
||||
}
|
||||
|
||||
try (var input = new java.io.ByteArrayInputStream(bytes)) {
|
||||
client.putObject(
|
||||
PutObjectArgs.builder()
|
||||
.bucket(bucket)
|
||||
.object(objectKey)
|
||||
.contentType(contentType)
|
||||
.stream(
|
||||
input,
|
||||
bytes.length,
|
||||
ImageValidator.MAX_BYTES
|
||||
)
|
||||
.build()
|
||||
);
|
||||
} catch (Exception error) {
|
||||
throw unavailable();
|
||||
}
|
||||
}
|
||||
|
||||
public Iterable<Result<Item>> listPrefix(String prefix) {
|
||||
return client.listObjects(
|
||||
ListObjectsArgs.builder()
|
||||
.bucket(bucket)
|
||||
.prefix(prefix)
|
||||
.recursive(true)
|
||||
.build()
|
||||
);
|
||||
}
|
||||
|
||||
private static BusinessException unavailable() {
|
||||
return new BusinessException(
|
||||
"unavailable",
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
package com.vnp.demo.service;
|
||||
|
||||
import com.drew.imaging.ImageMetadataReader;
|
||||
import com.drew.metadata.exif.ExifIFD0Directory;
|
||||
import com.vnp.demo.validation.ImageValidator;
|
||||
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import javax.imageio.ImageIO;
|
||||
import javax.imageio.ImageReader;
|
||||
import javax.imageio.stream.MemoryCacheImageInputStream;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.awt.Graphics2D;
|
||||
import java.awt.RenderingHints;
|
||||
import java.awt.geom.AffineTransform;
|
||||
import java.awt.image.BufferedImage;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
|
||||
@Component
|
||||
public class Thumbnailer {
|
||||
|
||||
public record Thumbnail(
|
||||
byte[] bytes,
|
||||
int width,
|
||||
int height
|
||||
) {
|
||||
}
|
||||
|
||||
public Thumbnail render(byte[] source) {
|
||||
BufferedImage original = null;
|
||||
BufferedImage target = null;
|
||||
ImageReader reader = null;
|
||||
|
||||
try (var input = new MemoryCacheImageInputStream(
|
||||
new ByteArrayInputStream(source)
|
||||
)) {
|
||||
var readers = ImageIO.getImageReaders(input);
|
||||
|
||||
if (!readers.hasNext()) {
|
||||
throw new IllegalArgumentException("invalid_image");
|
||||
}
|
||||
|
||||
reader = readers.next();
|
||||
reader.setInput(input, true, true);
|
||||
|
||||
int width = reader.getWidth(0);
|
||||
int height = reader.getHeight(0);
|
||||
|
||||
if (width < 1
|
||||
|| height < 1
|
||||
|| (long) width * height > ImageValidator.MAX_PIXELS) {
|
||||
throw new IllegalArgumentException("invalid_image");
|
||||
}
|
||||
|
||||
int orientation = 1;
|
||||
|
||||
var metadata = ImageMetadataReader.readMetadata(
|
||||
new ByteArrayInputStream(source)
|
||||
);
|
||||
|
||||
var exif = metadata.getFirstDirectoryOfType(
|
||||
ExifIFD0Directory.class
|
||||
);
|
||||
|
||||
if (exif != null) {
|
||||
Integer value = exif.getInteger(
|
||||
ExifIFD0Directory.TAG_ORIENTATION
|
||||
);
|
||||
|
||||
if (value != null && value >= 1 && value <= 8) {
|
||||
orientation = value;
|
||||
}
|
||||
}
|
||||
|
||||
original = reader.read(0);
|
||||
|
||||
int orientedWidth = orientation >= 5 ? height : width;
|
||||
int orientedHeight = orientation >= 5 ? width : height;
|
||||
|
||||
int denominator = Math.max(
|
||||
256,
|
||||
Math.max(orientedWidth, orientedHeight)
|
||||
);
|
||||
|
||||
int targetWidth = Math.max(
|
||||
1,
|
||||
(int) ((long) orientedWidth * 256 / denominator)
|
||||
);
|
||||
|
||||
int targetHeight = Math.max(
|
||||
1,
|
||||
(int) ((long) orientedHeight * 256 / denominator)
|
||||
);
|
||||
|
||||
target = new BufferedImage(
|
||||
targetWidth,
|
||||
targetHeight,
|
||||
BufferedImage.TYPE_INT_RGB
|
||||
);
|
||||
|
||||
Graphics2D graphics = target.createGraphics();
|
||||
|
||||
try {
|
||||
graphics.setColor(Color.WHITE);
|
||||
graphics.fillRect(0, 0, targetWidth, targetHeight);
|
||||
|
||||
graphics.setRenderingHint(
|
||||
RenderingHints.KEY_INTERPOLATION,
|
||||
RenderingHints.VALUE_INTERPOLATION_BICUBIC
|
||||
);
|
||||
|
||||
AffineTransform transform =
|
||||
AffineTransform.getScaleInstance(
|
||||
(double) targetWidth / orientedWidth,
|
||||
(double) targetHeight / orientedHeight
|
||||
);
|
||||
|
||||
transform.concatenate(
|
||||
switch (orientation) {
|
||||
case 2 -> new AffineTransform(
|
||||
-1, 0, 0, 1, width, 0
|
||||
);
|
||||
case 3 -> new AffineTransform(
|
||||
-1, 0, 0, -1, width, height
|
||||
);
|
||||
case 4 -> new AffineTransform(
|
||||
1, 0, 0, -1, 0, height
|
||||
);
|
||||
case 5 -> new AffineTransform(
|
||||
0, 1, 1, 0, 0, 0
|
||||
);
|
||||
case 6 -> new AffineTransform(
|
||||
0, 1, -1, 0, height, 0
|
||||
);
|
||||
case 7 -> new AffineTransform(
|
||||
0, -1, -1, 0, height, width
|
||||
);
|
||||
case 8 -> new AffineTransform(
|
||||
0, -1, 1, 0, 0, width
|
||||
);
|
||||
default -> new AffineTransform();
|
||||
}
|
||||
);
|
||||
|
||||
graphics.drawImage(original, transform, null);
|
||||
} finally {
|
||||
graphics.dispose();
|
||||
}
|
||||
|
||||
var output = new ByteArrayOutputStream();
|
||||
|
||||
if (!ImageIO.write(target, "jpeg", output)) {
|
||||
throw new IllegalArgumentException(
|
||||
"jpeg_encoder_unavailable"
|
||||
);
|
||||
}
|
||||
|
||||
return new Thumbnail(
|
||||
output.toByteArray(),
|
||||
targetWidth,
|
||||
targetHeight
|
||||
);
|
||||
} catch (Exception error) {
|
||||
throw new IllegalArgumentException("invalid_image", error);
|
||||
} finally {
|
||||
if (reader != null) {
|
||||
reader.dispose();
|
||||
}
|
||||
|
||||
if (original != null) {
|
||||
original.flush();
|
||||
}
|
||||
|
||||
if (target != null) {
|
||||
target.flush();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,11 +4,14 @@ server:
|
||||
threads:
|
||||
max: 64
|
||||
connection-timeout: 2s
|
||||
|
||||
spring:
|
||||
application:
|
||||
name: messenger
|
||||
|
||||
lifecycle:
|
||||
timeout-per-shutdown-phase: 15s
|
||||
|
||||
jackson:
|
||||
default-property-inclusion: always
|
||||
deserialization:
|
||||
@@ -16,10 +19,12 @@ spring:
|
||||
fail-on-trailing-tokens: true
|
||||
mapper:
|
||||
allow-coercion-of-scalars: false
|
||||
|
||||
datasource:
|
||||
url: ${DB_URL:jdbc:postgresql://localhost:5432/messenger}
|
||||
username: ${DB_USERNAME:messenger}
|
||||
password: ${DB_PASSWORD:messenger}
|
||||
|
||||
hikari:
|
||||
maximum-pool-size: ${DB_POOL_SIZE:8}
|
||||
minimum-idle: 1
|
||||
@@ -29,6 +34,7 @@ spring:
|
||||
connectTimeout: 2
|
||||
socketTimeout: 8
|
||||
options: "-c statement_timeout=5000 -c lock_timeout=3000"
|
||||
|
||||
data:
|
||||
redis:
|
||||
host: ${REDIS_HOST:localhost}
|
||||
@@ -36,17 +42,49 @@ spring:
|
||||
password: ${REDIS_PASSWORD:}
|
||||
timeout: 1s
|
||||
connect-timeout: 1s
|
||||
|
||||
rabbitmq:
|
||||
host: ${RABBITMQ_HOST:localhost}
|
||||
port: 5672
|
||||
username: ${RABBITMQ_USER:messenger_jobs}
|
||||
password: ${RABBITMQ_PASSWORD:}
|
||||
|
||||
connection-timeout: 3s
|
||||
requested-heartbeat: 15s
|
||||
|
||||
publisher-confirm-type: correlated
|
||||
publisher-returns: true
|
||||
|
||||
template:
|
||||
mandatory: true
|
||||
|
||||
listener:
|
||||
simple:
|
||||
acknowledge-mode: manual
|
||||
prefetch: 1
|
||||
concurrency: 1
|
||||
max-concurrency: 1
|
||||
default-requeue-rejected: true
|
||||
|
||||
task:
|
||||
scheduling:
|
||||
pool:
|
||||
size: 3
|
||||
|
||||
flyway:
|
||||
enabled: ${FLYWAY_ENABLED:true}
|
||||
|
||||
jpa:
|
||||
hibernate:
|
||||
ddl-auto: validate
|
||||
open-in-view: false
|
||||
|
||||
transaction:
|
||||
default-timeout: 8s
|
||||
|
||||
app:
|
||||
session:
|
||||
ttl-seconds: 86400
|
||||
ttl-seconds: ${SESSION_TTL_SECONDS:3600}
|
||||
|
||||
s3:
|
||||
endpoint: ${S3_ENDPOINT:http://localhost:9000}
|
||||
@@ -57,7 +95,15 @@ app:
|
||||
uploads:
|
||||
max-concurrent: ${UPLOAD_MAX_CONCURRENT:1}
|
||||
temp-dir: ${UPLOAD_TEMP_DIR:/tmp/messenger-uploads}
|
||||
|
||||
|
||||
worker:
|
||||
enabled: ${WORKER_ENABLED:false}
|
||||
|
||||
jobs:
|
||||
max-attempts: ${JOB_MAX_ATTEMPTS:3}
|
||||
lease-seconds: ${JOB_TIMEOUT_SECONDS:120}
|
||||
failed-retention-seconds: ${STAGING_RETENTION_SECONDS:604800}
|
||||
|
||||
cleanup:
|
||||
enabled: ${ATTACHMENT_CLEANUP_ENABLED:true}
|
||||
grace-seconds: ${ATTACHMENT_CLEANUP_GRACE_SECONDS:3600}
|
||||
|
||||
@@ -0,0 +1,220 @@
|
||||
-- Расширение метаданных вложения для асинхронной обработки.
|
||||
|
||||
ALTER TABLE attachments
|
||||
DROP CONSTRAINT chk_attachment_status;
|
||||
|
||||
ALTER TABLE attachments
|
||||
ADD CONSTRAINT chk_attachment_status
|
||||
CHECK (
|
||||
status IN (
|
||||
'queued',
|
||||
'processing',
|
||||
'ready',
|
||||
'failed'
|
||||
)
|
||||
);
|
||||
|
||||
ALTER TABLE attachments
|
||||
ADD COLUMN staging_key TEXT,
|
||||
ADD COLUMN pipeline_version INTEGER NOT NULL DEFAULT 1,
|
||||
ADD COLUMN error_code VARCHAR(64);
|
||||
|
||||
ALTER TABLE attachments
|
||||
ADD CONSTRAINT uq_attachment_staging_key
|
||||
UNIQUE (staging_key),
|
||||
|
||||
ADD CONSTRAINT chk_attachment_pipeline_version
|
||||
CHECK (pipeline_version >= 1),
|
||||
|
||||
ADD CONSTRAINT chk_attachment_error
|
||||
CHECK (
|
||||
(
|
||||
status = 'failed'
|
||||
AND error_code IS NOT NULL
|
||||
AND length(error_code) > 0
|
||||
)
|
||||
OR
|
||||
(
|
||||
status <> 'failed'
|
||||
AND error_code IS NULL
|
||||
)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_attachments_status
|
||||
ON attachments(status, updated_at);
|
||||
|
||||
|
||||
-- Готовые варианты изображения.
|
||||
-- object_key используется внутри приложения и не выдаётся клиенту.
|
||||
-- Ключ эффекта соответствует контракту:
|
||||
-- attachment_id + pipeline_version + variant.
|
||||
|
||||
CREATE TABLE attachment_variants (
|
||||
attachment_id UUID NOT NULL
|
||||
REFERENCES attachments(id) ON DELETE CASCADE,
|
||||
|
||||
pipeline_version INTEGER NOT NULL,
|
||||
name VARCHAR(32) NOT NULL,
|
||||
|
||||
object_key TEXT NOT NULL UNIQUE,
|
||||
content_type VARCHAR(32) NOT NULL,
|
||||
|
||||
size_bytes BIGINT NOT NULL,
|
||||
width INTEGER NOT NULL,
|
||||
height INTEGER NOT NULL,
|
||||
sha256 CHAR(64) NOT NULL,
|
||||
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
|
||||
PRIMARY KEY (
|
||||
attachment_id,
|
||||
pipeline_version,
|
||||
name
|
||||
),
|
||||
|
||||
CONSTRAINT chk_variant_pipeline_version
|
||||
CHECK (pipeline_version >= 1),
|
||||
|
||||
CONSTRAINT chk_variant_name
|
||||
CHECK (
|
||||
name IN (
|
||||
'thumbnail',
|
||||
'crop',
|
||||
'optimized'
|
||||
)
|
||||
),
|
||||
|
||||
CONSTRAINT chk_variant_content_type
|
||||
CHECK (
|
||||
content_type IN (
|
||||
'image/jpeg',
|
||||
'image/png',
|
||||
'image/webp'
|
||||
)
|
||||
),
|
||||
|
||||
CONSTRAINT chk_variant_size
|
||||
CHECK (size_bytes > 0),
|
||||
|
||||
CONSTRAINT chk_variant_dimensions
|
||||
CHECK (width > 0 AND height > 0),
|
||||
|
||||
CONSTRAINT chk_variant_sha256
|
||||
CHECK (sha256 ~ '^[a-f0-9]{64}$')
|
||||
);
|
||||
|
||||
|
||||
-- Одна логическая работа над вложением в рамках версии pipeline.
|
||||
-- job_id не меняется при повторной доставке.
|
||||
-- attempts увеличивается при начале управляемой попытки,
|
||||
-- а не при каждом получении дубликата сообщения.
|
||||
|
||||
CREATE TABLE attachment_jobs (
|
||||
id UUID PRIMARY KEY,
|
||||
|
||||
attachment_id UUID NOT NULL
|
||||
REFERENCES attachments(id) ON DELETE CASCADE,
|
||||
|
||||
pipeline_version INTEGER NOT NULL DEFAULT 1,
|
||||
|
||||
status VARCHAR(16) NOT NULL DEFAULT 'queued',
|
||||
|
||||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
max_attempts INTEGER NOT NULL DEFAULT 3,
|
||||
|
||||
available_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
|
||||
lease_owner UUID,
|
||||
lease_until TIMESTAMPTZ,
|
||||
|
||||
last_error_code VARCHAR(64),
|
||||
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
|
||||
CONSTRAINT uq_attachment_job_pipeline
|
||||
UNIQUE (
|
||||
attachment_id,
|
||||
pipeline_version
|
||||
),
|
||||
|
||||
CONSTRAINT chk_job_pipeline_version
|
||||
CHECK (pipeline_version >= 1),
|
||||
|
||||
CONSTRAINT chk_job_status
|
||||
CHECK (
|
||||
status IN (
|
||||
'queued',
|
||||
'processing',
|
||||
'ready',
|
||||
'failed'
|
||||
)
|
||||
),
|
||||
|
||||
CONSTRAINT chk_job_attempts
|
||||
CHECK (
|
||||
max_attempts >= 1
|
||||
AND attempts >= 0
|
||||
AND attempts <= max_attempts
|
||||
),
|
||||
|
||||
CONSTRAINT chk_job_lease
|
||||
CHECK (
|
||||
(
|
||||
status = 'processing'
|
||||
AND lease_owner IS NOT NULL
|
||||
AND lease_until IS NOT NULL
|
||||
)
|
||||
OR
|
||||
(
|
||||
status <> 'processing'
|
||||
AND lease_owner IS NULL
|
||||
AND lease_until IS NULL
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_attachment_jobs_available
|
||||
ON attachment_jobs(available_at, id)
|
||||
WHERE status = 'queued';
|
||||
|
||||
CREATE INDEX idx_attachment_jobs_expired_lease
|
||||
ON attachment_jobs(lease_until, id)
|
||||
WHERE status = 'processing';
|
||||
|
||||
|
||||
-- Намерение публикации задания в брокер.
|
||||
-- Записывается в той же транзакции, что и метаданные задания.
|
||||
-- published_at устанавливается после подтверждения брокера.
|
||||
-- Потеря подтверждения может привести к повторной публикации:
|
||||
-- worker обязан обрабатывать её идемпотентно.
|
||||
|
||||
CREATE TABLE attachment_outbox (
|
||||
id UUID PRIMARY KEY,
|
||||
|
||||
job_id UUID NOT NULL
|
||||
REFERENCES attachment_jobs(id) ON DELETE CASCADE,
|
||||
|
||||
attempt INTEGER NOT NULL,
|
||||
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
next_publish_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
|
||||
published_at TIMESTAMPTZ,
|
||||
|
||||
publish_attempts INTEGER NOT NULL DEFAULT 0,
|
||||
last_error_code VARCHAR(64),
|
||||
|
||||
CONSTRAINT uq_attachment_outbox_attempt
|
||||
UNIQUE (job_id, attempt),
|
||||
|
||||
CONSTRAINT chk_outbox_attempt
|
||||
CHECK (attempt >= 1),
|
||||
|
||||
CONSTRAINT chk_outbox_publish_attempts
|
||||
CHECK (publish_attempts >= 0)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_attachment_outbox_pending
|
||||
ON attachment_outbox(next_publish_at, id)
|
||||
WHERE published_at IS NULL;
|
||||
@@ -0,0 +1,94 @@
|
||||
CREATE TABLE attachment_replay_audit (
|
||||
id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
||||
|
||||
job_id UUID NOT NULL REFERENCES attachment_jobs(id),
|
||||
actor VARCHAR(64) NOT NULL,
|
||||
reason VARCHAR(255) NOT NULL,
|
||||
|
||||
action VARCHAR(32) NOT NULL
|
||||
CHECK (action IN ('enqueued', 'already_ready')),
|
||||
|
||||
previous_attempts INTEGER NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
CREATE TABLE attachment_attempt_events (
|
||||
id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
||||
|
||||
job_id UUID NOT NULL REFERENCES attachment_jobs(id),
|
||||
attempt INTEGER NOT NULL,
|
||||
state VARCHAR(16) NOT NULL,
|
||||
error_code VARCHAR(64) NOT NULL,
|
||||
delay_seconds INTEGER NOT NULL,
|
||||
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
-- Используется только при FAULT_INJECTION_ENABLED=true.
|
||||
-- Управления через публичный HTTP API нет.
|
||||
CREATE TABLE attachment_test_faults (
|
||||
point VARCHAR(32) PRIMARY KEY,
|
||||
remaining INTEGER NOT NULL CHECK (remaining >= 0),
|
||||
hits INTEGER NOT NULL DEFAULT 0,
|
||||
|
||||
job_id UUID REFERENCES attachment_jobs(id),
|
||||
hit_at TIMESTAMPTZ
|
||||
);
|
||||
|
||||
-- Существующие ready-вложения с неполным набором вариантов
|
||||
-- отправляются на дообработку.
|
||||
INSERT INTO attachment_jobs (
|
||||
id,
|
||||
attachment_id,
|
||||
pipeline_version,
|
||||
status,
|
||||
max_attempts
|
||||
)
|
||||
SELECT
|
||||
gen_random_uuid(),
|
||||
a.id,
|
||||
a.pipeline_version,
|
||||
'queued',
|
||||
3
|
||||
FROM attachments a
|
||||
WHERE a.status = 'ready'
|
||||
AND (
|
||||
SELECT count(*)
|
||||
FROM attachment_variants v
|
||||
WHERE v.attachment_id = a.id
|
||||
AND v.pipeline_version = a.pipeline_version
|
||||
AND v.name IN ('thumbnail', 'crop', 'optimized')
|
||||
) < 3
|
||||
ON CONFLICT (attachment_id, pipeline_version)
|
||||
DO UPDATE SET
|
||||
status = 'queued',
|
||||
max_attempts = greatest(
|
||||
attachment_jobs.max_attempts,
|
||||
attachment_jobs.attempts + 3
|
||||
),
|
||||
available_at = clock_timestamp(),
|
||||
lease_owner = NULL,
|
||||
lease_until = NULL,
|
||||
last_error_code = NULL,
|
||||
updated_at = clock_timestamp()
|
||||
WHERE attachment_jobs.status = 'ready';
|
||||
|
||||
UPDATE attachments a
|
||||
SET status = 'queued',
|
||||
staging_key = a.object_key,
|
||||
error_code = NULL,
|
||||
updated_at = clock_timestamp()
|
||||
WHERE a.status = 'ready'
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM attachment_jobs j
|
||||
WHERE j.attachment_id = a.id
|
||||
AND j.pipeline_version = a.pipeline_version
|
||||
AND j.status = 'queued'
|
||||
);
|
||||
|
||||
INSERT INTO attachment_outbox(id, job_id, attempt)
|
||||
SELECT gen_random_uuid(), id, attempts + 1
|
||||
FROM attachment_jobs
|
||||
WHERE status = 'queued'
|
||||
ON CONFLICT (job_id, attempt) DO NOTHING;
|
||||
Reference in New Issue
Block a user