all for 4

This commit is contained in:
2026-11-11 23:05:16 +03:00
parent 1ec85a2402
commit a2e143caea
26 changed files with 5103 additions and 281 deletions
+11
View File
@@ -103,6 +103,17 @@
<version>8.5.17</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-amqp</artifactId>
</dependency>
<dependency>
<groupId>com.drewnoakes</groupId>
<artifactId>metadata-extractor</artifactId>
<version>2.19.0</version>
</dependency>
</dependencies>
<build>
@@ -0,0 +1,26 @@
package com.vnp.demo.config;
import org.springframework.amqp.core.Declarables;
import org.springframework.amqp.core.Queue;
import org.springframework.amqp.core.QueueBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class RabbitConfig {
public static final String QUEUE = "attachments.v1";
public static final String DLQ = "attachments.v1.dlq";
@Bean
public Declarables attachmentQueues() {
return new Declarables(
new Queue(DLQ, true),
QueueBuilder.durable(QUEUE)
.deadLetterExchange("")
.deadLetterRoutingKey(DLQ)
.build()
);
}
}
@@ -37,7 +37,7 @@ public class AttachmentController {
}
@PostMapping("/chats/{chat_id}/attachments")
@ResponseStatus(HttpStatus.CREATED)
@ResponseStatus(HttpStatus.ACCEPTED)
public AttachmentResponse upload(
@AuthenticationPrincipal UUID userId,
@PathVariable("chat_id") UUID chatId,
@@ -0,0 +1,81 @@
package com.vnp.demo.dto;
import java.time.Instant;
import java.util.Objects;
import java.util.UUID;
import com.fasterxml.jackson.annotation.JsonProperty;
public record AttachmentJobMessage(
@JsonProperty("schema_version")
int schemaVersion,
@JsonProperty("job_id")
UUID jobId,
String type,
@JsonProperty("attachment_id")
UUID attachmentId,
@JsonProperty("pipeline_version")
int pipelineVersion,
int attempt,
@JsonProperty("created_at")
Instant createdAt
) {
public static final int SCHEMA_VERSION = 1;
public static final String TYPE =
"attachment.process.v1";
public AttachmentJobMessage {
if (schemaVersion != SCHEMA_VERSION) {
throw new IllegalArgumentException(
"Unsupported job schema version"
);
}
if (!TYPE.equals(type)) {
throw new IllegalArgumentException(
"Unsupported job type"
);
}
Objects.requireNonNull(jobId, "jobId");
Objects.requireNonNull(attachmentId, "attachmentId");
Objects.requireNonNull(createdAt, "createdAt");
if (pipelineVersion < 1) {
throw new IllegalArgumentException(
"pipelineVersion must be positive"
);
}
if (attempt < 1) {
throw new IllegalArgumentException(
"attempt must be positive"
);
}
}
public static AttachmentJobMessage create(
UUID jobId,
UUID attachmentId,
int pipelineVersion,
int attempt,
Instant createdAt
) {
return new AttachmentJobMessage(
SCHEMA_VERSION,
jobId,
TYPE,
attachmentId,
pipelineVersion,
attempt,
createdAt
);
}
}
@@ -1,19 +1,11 @@
package com.vnp.demo.service;
import java.time.Instant;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import com.vnp.demo.service.AttachmentPersistenceService.CleanupResult;
import io.minio.Result;
import io.minio.messages.Item;
@Component
@ConditionalOnProperty(
name = "app.cleanup.enabled",
@@ -25,94 +17,24 @@ public class AttachmentCleanupJob {
private static final Logger log =
LoggerFactory.getLogger(AttachmentCleanupJob.class);
private final S3StorageService storage;
private final AttachmentPersistenceService persistence;
private final long graceSeconds;
private final AttachmentPipeline pipeline;
public AttachmentCleanupJob(
S3StorageService storage,
AttachmentPersistenceService persistence,
@Value("${app.cleanup.grace-seconds:3600}")
long graceSeconds
) {
if (graceSeconds < 60) {
throw new IllegalArgumentException(
"Grace period должен быть не меньше 60 секунд"
);
}
this.storage = storage;
this.persistence = persistence;
this.graceSeconds = graceSeconds;
public AttachmentCleanupJob(AttachmentPipeline pipeline) {
this.pipeline = pipeline;
}
@Scheduled(
initialDelayString =
"${app.cleanup.initial-delay-ms:60000}",
fixedDelayString =
"${app.cleanup.interval-ms:300000}"
initialDelayString = "${app.cleanup.initial-delay-ms:60000}",
fixedDelayString = "${app.cleanup.interval-ms:300000}"
)
public void cleanup() {
Instant cutoff = Instant.now()
.minusSeconds(graceSeconds);
long scanned = 0;
long candidates = 0;
long deleted = 0;
try {
for (Result<Item> result : storage.listOriginals()) {
Item item = result.get();
pipeline.cleanup();
if (item.isDir()) {
continue;
}
scanned++;
Instant lastModified = item.lastModified()
.toInstant();
if (!lastModified.isBefore(cutoff)) {
continue;
}
candidates++;
CleanupResult outcome =
persistence.removeIfOrphan(
item.objectName(),
cutoff
);
if (outcome == CleanupResult.DELETED) {
deleted++;
log.info(
"Attachment cleanup removed orphan: {}",
item.objectName()
);
}
}
log.info(
"Attachment cleanup completed: "
+ "scanned={}, candidates={}, deleted={}",
scanned,
candidates,
deleted
);
} catch (Exception e) {
// Прерываем текущий проход при ошибке БД или S3.
// Следующий запуск повторит проверку.
// Не выводим credentials или содержимое ответа S3.
} catch (Exception error) {
log.warn(
"Attachment cleanup interrupted: "
+ "scanned={}, candidates={}, deleted={}, errorType={}",
scanned,
candidates,
deleted,
e.getClass().getSimpleName()
"Attachment cleanup interrupted: {}",
error.getClass().getSimpleName()
);
}
}
File diff suppressed because it is too large Load Diff
@@ -13,101 +13,95 @@ import com.vnp.demo.models.Attachment;
import com.vnp.demo.repositories.AttachmentRepository;
import com.vnp.demo.repositories.ChatMemberRepository;
import com.vnp.demo.validation.ImageValidator;
import com.vnp.demo.validation.ImageValidator.ValidatedImage;
@Service
public class AttachmentService {
private final AttachmentRepository attachmentRepository;
private final ChatMemberRepository chatMemberRepository;
private final ImageValidator imageValidator;
private final AttachmentRepository attachments;
private final ChatMemberRepository members;
private final ImageValidator validator;
private final UploadLimiter limiter;
private final S3StorageService storage;
private final UploadLimiter uploadLimiter;
private final AttachmentPersistenceService persistence;
private final AttachmentPipeline pipeline;
public AttachmentService(
AttachmentRepository attachmentRepository,
ChatMemberRepository chatMemberRepository,
ImageValidator imageValidator,
AttachmentRepository attachments,
ChatMemberRepository members,
ImageValidator validator,
UploadLimiter limiter,
S3StorageService storage,
UploadLimiter uploadLimiter,
AttachmentPersistenceService persistence
AttachmentPipeline pipeline
) {
this.attachmentRepository = attachmentRepository;
this.chatMemberRepository = chatMemberRepository;
this.imageValidator = imageValidator;
this.attachments = attachments;
this.members = members;
this.validator = validator;
this.limiter = limiter;
this.storage = storage;
this.uploadLimiter = uploadLimiter;
this.persistence = persistence;
this.pipeline = pipeline;
}
public AttachmentResponse upload(
UUID userId,
UUID chatId,
String filename,
String contentType,
long contentLength,
InputStream input
) {
checkChatAccess(userId, chatId);
UUID userId,
UUID chatId,
String filename,
String contentType,
long contentLength,
InputStream input
) {
checkChatAccess(userId, chatId);
validator.validateFilename(filename);
imageValidator.validateFilename(filename);
try (var permit = limiter.acquire();
var image = validator.validate(
input,
contentLength,
contentType
)) {
try (UploadLimiter.Permit permit = uploadLimiter.acquire();
ValidatedImage image = imageValidator.validate(
input,
contentLength,
contentType
)) {
UUID id = UUID.randomUUID();
UUID attachmentId = UUID.randomUUID();
Instant now = Instant.now()
.truncatedTo(ChronoUnit.MICROS);
String objectKey = "originals/"
+ chatId
+ "/"
+ attachmentId;
Attachment attachment = new Attachment(
id,
chatId,
userId,
filename,
image.contentType(),
image.sizeBytes(),
image.sha256(),
Instant now = Instant.now()
.truncatedTo(ChronoUnit.MICROS);
"originals/"
+ chatId
+ "/"
+ id
+ "/pending",
Attachment attachment = new Attachment(
attachmentId,
chatId,
userId,
filename,
image.contentType(),
image.sizeBytes(),
image.sha256(),
objectKey,
"ready",
now,
now
);
"queued",
now,
now
);
persistence.save(attachment, image.path());
pipeline.enqueue(attachment, image.path());
return AttachmentResponse.from(attachment);
}
return AttachmentResponse.from(attachment);
}
}
public AttachmentResponse getMetadata(
UUID userId,
UUID attachmentId
UUID id
) {
return AttachmentResponse.from(
getAccessibleAttachment(userId, attachmentId)
);
return pipeline.metadata(accessible(userId, id));
}
public Download download(
UUID userId,
UUID attachmentId,
UUID id,
String variant
) {
Attachment attachment = getAccessibleAttachment(
userId,
attachmentId
);
Attachment attachment = accessible(userId, id);
if (!"ready".equals(attachment.status())) {
throw BusinessException.conflict(
@@ -115,29 +109,36 @@ public class AttachmentService {
);
}
if (!"original".equals(variant)) {
throw BusinessException.notFound();
if ("original".equals(variant)) {
return new Download(
attachment.filename(),
attachment.contentType(),
storage.get(
attachment.objectKey(),
attachment.sizeBytes(),
attachment.sha256()
)
);
}
byte[] bytes = storage.get(
attachment.objectKey(),
attachment.sizeBytes(),
attachment.sha256()
);
var selected = pipeline.variants(id).stream()
.filter(item -> item.name().equals(variant))
.findFirst()
.orElseThrow(BusinessException::notFound);
return new Download(
attachment.filename(),
attachment.contentType(),
bytes
selected.name() + ".jpg",
selected.contentType(),
storage.get(
selected.objectKey(),
selected.sizeBytes(),
selected.sha256()
)
);
}
private Attachment getAccessibleAttachment(
UUID userId,
UUID attachmentId
) {
Attachment attachment = attachmentRepository
.findById(attachmentId)
private Attachment accessible(UUID userId, UUID id) {
Attachment attachment = attachments.findById(id)
.orElseThrow(BusinessException::notFound);
checkChatAccess(userId, attachment.chatId());
@@ -146,10 +147,7 @@ public class AttachmentService {
}
private void checkChatAccess(UUID userId, UUID chatId) {
boolean member = chatMemberRepository
.existsByIdChatIdAndIdUserId(chatId, userId);
if (!member) {
if (!members.existsByIdChatIdAndIdUserId(chatId, userId)) {
throw BusinessException.notFound();
}
}
@@ -0,0 +1,105 @@
package com.vnp.demo.service;
import java.util.UUID;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.support.TransactionTemplate;
@Component
public class FaultGate {
private final boolean enabled;
private final JdbcTemplate jdbc;
private final TransactionTemplate tx;
public FaultGate(
JdbcTemplate jdbc,
PlatformTransactionManager manager,
@Value("${FAULT_INJECTION_ENABLED:false}")
boolean enabled
) {
this.jdbc = jdbc;
this.enabled = enabled;
tx = new TransactionTemplate(manager);
tx.setPropagationBehavior(
TransactionDefinition.PROPAGATION_REQUIRES_NEW
);
tx.setTimeout(5);
}
public boolean paused() {
if (!enabled) {
return false;
}
return Boolean.TRUE.equals(
jdbc.queryForObject(
"""
SELECT EXISTS (
SELECT 1
FROM attachment_test_faults
WHERE point = 'hold_publish'
AND remaining > 0
)
""",
Boolean.class
)
);
}
public boolean hit(UUID job, String point) {
if (!enabled) {
return false;
}
boolean hit = Boolean.TRUE.equals(
tx.execute(ignored ->
jdbc.update(
"""
UPDATE attachment_test_faults
SET remaining = remaining - 1,
hits = hits + 1,
job_id = ?,
hit_at = clock_timestamp()
WHERE point = ?
AND remaining > 0
""",
job,
point
) == 1
)
);
if (!hit) {
return false;
}
switch (point) {
case "before_publish",
"after_objects",
"before_ack" ->
Runtime.getRuntime().halt(137);
case "permanent_error" ->
throw new ImageProcessor.PermanentFailure();
case "transient_error" ->
throw new IllegalStateException(
"injected_transient_error"
);
default -> {
}
}
return true;
}
}
@@ -0,0 +1,274 @@
package com.vnp.demo.service;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.ObjectMapper;
@Component
public class ImageProcessor {
public static class PermanentFailure extends RuntimeException {
public PermanentFailure() {
super("invalid_image");
}
}
public static class TransientFailure extends RuntimeException {
private final String code;
public TransientFailure(String code) {
super(code);
this.code = code;
}
public String code() {
return code;
}
}
public record Info(
String name,
@JsonProperty("content_type")
String contentType,
int width,
int height
) {
}
public record Generated(
Info info,
byte[] bytes
) {
}
private final ObjectMapper mapper;
private final Set<Path> active =
ConcurrentHashMap.newKeySet();
private final Semaphore slots = new Semaphore(1);
private final Path root = Path.of("/tmp/image-jobs");
public ImageProcessor(ObjectMapper mapper) {
this.mapper = mapper;
}
public List<Generated> render(
byte[] source,
boolean testHang
) {
if (source.length == 0 || source.length > 10485760) {
throw new PermanentFailure();
}
if (!slots.tryAcquire()) {
throw new TransientFailure("processor_busy");
}
Path directory = null;
Process process = null;
try {
Files.createDirectories(root);
directory = Files.createTempDirectory(root, "job-");
active.add(directory);
Path input = directory.resolve("input.bin");
Files.write(input, source);
ProcessBuilder builder = new ProcessBuilder(
"/opt/images/bin/python",
"/app/process_image.py",
input.toString(),
directory.toString()
);
builder.redirectOutput(
ProcessBuilder.Redirect.DISCARD
);
builder.redirectError(
ProcessBuilder.Redirect.DISCARD
);
builder.environment().remove("IMAGE_TEST_SLEEP");
if (testHang) {
builder.environment().put(
"IMAGE_TEST_SLEEP",
"1"
);
}
process = builder.start();
if (!process.waitFor(25, TimeUnit.SECONDS)) {
throw new TransientFailure(
"processing_timeout"
);
}
if (process.exitValue() == 10) {
throw new PermanentFailure();
}
if (process.exitValue() != 0) {
throw new TransientFailure(
"processing_resource_limit"
);
}
Path manifest = directory.resolve("manifest.json");
if (Files.size(manifest) > 4096) {
throw new TransientFailure("invalid_manifest");
}
Info[] infos = mapper.readValue(
Files.readAllBytes(manifest),
Info[].class
);
Set<String> names = new HashSet<>();
List<Generated> result = new ArrayList<>();
for (Info info : infos) {
String mime = switch (info.name()) {
case "thumbnail", "crop" -> "image/jpeg";
case "optimized" -> "image/webp";
default -> throw new TransientFailure(
"invalid_variant"
);
};
if (!names.add(info.name())
|| !mime.equals(info.contentType())
|| info.width() < 1
|| info.height() < 1
|| info.width() > 1280
|| info.height() > 1280) {
throw new TransientFailure(
"invalid_manifest"
);
}
String extension = mime.equals("image/webp")
? ".webp"
: ".jpg";
Path file = directory.resolve(
info.name() + extension
);
long size = Files.size(file);
if (size < 1 || size > 10485760) {
throw new TransientFailure(
"invalid_variant_size"
);
}
result.add(
new Generated(
info,
Files.readAllBytes(file)
)
);
}
if (!names.equals(
Set.of("thumbnail", "crop", "optimized")
)) {
throw new TransientFailure(
"incomplete_variants"
);
}
return result;
} catch (PermanentFailure | TransientFailure error) {
throw error;
} catch (Exception error) {
if (error instanceof InterruptedException) {
Thread.currentThread().interrupt();
}
throw new TransientFailure(
"processing_unavailable"
);
} finally {
if (process != null && process.isAlive()) {
process.destroyForcibly();
try {
process.waitFor(5, TimeUnit.SECONDS);
} catch (InterruptedException error) {
Thread.currentThread().interrupt();
}
}
if (directory != null) {
active.remove(directory);
remove(directory);
}
slots.release();
}
}
private void remove(Path directory) {
try (var paths = Files.walk(directory)) {
for (Path path : paths
.sorted(Comparator.reverseOrder())
.toList()) {
Files.deleteIfExists(path);
}
} catch (Exception ignored) {
}
}
@Scheduled(
initialDelay = 60000,
fixedDelay = 300000
)
public void cleanTemporaryFiles() {
if (!Files.isDirectory(root)) {
return;
}
try (var children = Files.list(root)) {
for (Path path : children.toList()) {
if (Files.isDirectory(path)
&& !active.contains(path)
&& Files.getLastModifiedTime(path)
.toInstant()
.isBefore(
Instant.now().minusSeconds(3600)
)) {
remove(path);
}
}
} catch (Exception ignored) {
}
}
}
@@ -219,25 +219,25 @@ public class MessageService {
}
private void validateAttachments(
UUID chatId,
List<UUID> attachmentIds
) {
UUID chatId,
List<UUID> attachmentIds
) {
for (UUID attachmentId : attachmentIds) {
Attachment attachment = attachmentRepository
.findById(attachmentId)
.orElseThrow(BusinessException::notFound);
Attachment attachment = attachmentRepository
.findById(attachmentId)
.orElseThrow(BusinessException::notFound);
if (!chatId.equals(attachment.chatId())) {
if (!chatId.equals(attachment.chatId())) {
throw BusinessException.notFound();
}
}
if (!"ready".equals(attachment.status())) {
if ("failed".equals(attachment.status())) {
throw BusinessException.conflict(
"Вложение не готово"
"Обработка вложения завершилась ошибкой"
);
}
}
}
}
}
@Transactional(readOnly = true)
public MessagePageResponse list(
@@ -0,0 +1,261 @@
package com.vnp.demo.service;
import com.vnp.demo.repositories.AttachmentRepository;
import org.springframework.boot.ApplicationArguments;
import org.springframework.boot.ApplicationRunner;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.core.env.Environment;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.support.TransactionTemplate;
import java.util.UUID;
@Component
@ConditionalOnProperty(
name = "replay.enabled",
havingValue = "true"
)
public class ReplayRunner implements ApplicationRunner {
private final JdbcTemplate jdbc;
private final AttachmentRepository attachments;
private final S3StorageService storage;
private final AttachmentPipeline pipeline;
private final TransactionTemplate tx;
private final Environment env;
private final ConfigurableApplicationContext context;
public ReplayRunner(
JdbcTemplate jdbc,
AttachmentRepository attachments,
S3StorageService storage,
AttachmentPipeline pipeline,
PlatformTransactionManager manager,
Environment env,
ConfigurableApplicationContext context
) {
this.jdbc = jdbc;
this.attachments = attachments;
this.storage = storage;
this.pipeline = pipeline;
this.env = env;
this.context = context;
tx = new TransactionTemplate(manager);
tx.setTimeout(60);
}
@Override
public void run(ApplicationArguments ignored) {
int code = 0;
try {
UUID id = UUID.fromString(
env.getRequiredProperty("REPLAY_JOB_ID")
);
String actor = env.getRequiredProperty(
"REPLAY_ACTOR"
);
String reason = env.getRequiredProperty(
"REPLAY_REASON"
);
int budget = env.getProperty(
"app.jobs.max-attempts",
Integer.class,
3
);
if (!actor.matches("[A-Za-z0-9_.-]{1,64}")
|| reason.isBlank()
|| reason.length() > 255
|| budget < 1
|| budget > 100) {
throw new IllegalArgumentException(
"invalid_replay_request"
);
}
String result = tx.execute(status -> {
var job = jdbc.queryForMap(
"""
SELECT *
FROM attachment_jobs
WHERE id = ?
FOR UPDATE
""",
id
);
UUID attachmentId =
(UUID) job.get("attachment_id");
var attachment = attachments
.findById(attachmentId)
.orElseThrow();
int version = ((Number) job.get(
"pipeline_version"
)).intValue();
int attempts = ((Number) job.get(
"attempts"
)).intValue();
Integer activeVersion = jdbc.queryForObject(
"""
SELECT pipeline_version
FROM attachments
WHERE id = ?
""",
Integer.class,
attachmentId
);
if (version != 1
|| activeVersion == null
|| activeVersion != version) {
throw new IllegalStateException(
"obsolete_pipeline"
);
}
String action;
if ("ready".equals(job.get("status"))) {
pipeline.verifyReady(attachment);
action = "already_ready";
} else {
if (!"failed".equals(job.get("status"))
|| !"failed".equals(
attachment.status()
)) {
throw new IllegalStateException(
"job_is_not_failed"
);
}
String staging = jdbc.queryForObject(
"""
SELECT staging_key
FROM attachments
WHERE id = ?
""",
String.class,
attachmentId
);
if (staging == null) {
throw new IllegalStateException(
"staging_expired"
);
}
/*
* Та же блокировка используется очисткой.
* Staging нельзя удалить во время replay.
*/
attachments.lockObject(staging);
storage.get(
staging,
attachment.sizeBytes(),
attachment.sha256()
);
jdbc.update(
"""
UPDATE attachment_jobs
SET status = 'queued',
max_attempts = ?,
available_at = clock_timestamp(),
lease_owner = NULL,
lease_until = NULL,
last_error_code = NULL,
updated_at = clock_timestamp()
WHERE id = ?
""",
Math.addExact(attempts, budget),
id
);
jdbc.update(
"""
UPDATE attachments
SET status = 'queued',
error_code = NULL,
updated_at = clock_timestamp()
WHERE id = ?
""",
attachmentId
);
jdbc.update(
"""
INSERT INTO attachment_outbox(
id,
job_id,
attempt
)
VALUES (?, ?, ?)
""",
UUID.randomUUID(),
id,
attempts + 1
);
action = "enqueued";
}
jdbc.update(
"""
INSERT INTO attachment_replay_audit(
job_id,
actor,
reason,
action,
previous_attempts
)
VALUES (?, ?, ?, ?, ?)
""",
id,
actor,
reason,
action,
attempts
);
return action;
});
System.out.println(
"REPLAY_RESULT=" + result
);
} catch (Exception error) {
code = 1;
System.err.println(
"REPLAY_FAILED: "
+ error.getClass().getSimpleName()
);
}
final int exitCode = code;
System.exit(
SpringApplication.exit(
context,
() -> exitCode
)
);
}
}
@@ -184,6 +184,44 @@ public class S3StorageService {
}
}
public void putBytes(
String objectKey,
byte[] bytes,
String contentType
) {
if (bytes.length < 1
|| bytes.length > ImageValidator.MAX_BYTES) {
throw new IllegalArgumentException("Invalid object size");
}
try (var input = new java.io.ByteArrayInputStream(bytes)) {
client.putObject(
PutObjectArgs.builder()
.bucket(bucket)
.object(objectKey)
.contentType(contentType)
.stream(
input,
bytes.length,
ImageValidator.MAX_BYTES
)
.build()
);
} catch (Exception error) {
throw unavailable();
}
}
public Iterable<Result<Item>> listPrefix(String prefix) {
return client.listObjects(
ListObjectsArgs.builder()
.bucket(bucket)
.prefix(prefix)
.recursive(true)
.build()
);
}
private static BusinessException unavailable() {
return new BusinessException(
"unavailable",
@@ -0,0 +1,182 @@
package com.vnp.demo.service;
import com.drew.imaging.ImageMetadataReader;
import com.drew.metadata.exif.ExifIFD0Directory;
import com.vnp.demo.validation.ImageValidator;
import org.springframework.stereotype.Component;
import javax.imageio.ImageIO;
import javax.imageio.ImageReader;
import javax.imageio.stream.MemoryCacheImageInputStream;
import java.awt.Color;
import java.awt.Graphics2D;
import java.awt.RenderingHints;
import java.awt.geom.AffineTransform;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
@Component
public class Thumbnailer {
public record Thumbnail(
byte[] bytes,
int width,
int height
) {
}
public Thumbnail render(byte[] source) {
BufferedImage original = null;
BufferedImage target = null;
ImageReader reader = null;
try (var input = new MemoryCacheImageInputStream(
new ByteArrayInputStream(source)
)) {
var readers = ImageIO.getImageReaders(input);
if (!readers.hasNext()) {
throw new IllegalArgumentException("invalid_image");
}
reader = readers.next();
reader.setInput(input, true, true);
int width = reader.getWidth(0);
int height = reader.getHeight(0);
if (width < 1
|| height < 1
|| (long) width * height > ImageValidator.MAX_PIXELS) {
throw new IllegalArgumentException("invalid_image");
}
int orientation = 1;
var metadata = ImageMetadataReader.readMetadata(
new ByteArrayInputStream(source)
);
var exif = metadata.getFirstDirectoryOfType(
ExifIFD0Directory.class
);
if (exif != null) {
Integer value = exif.getInteger(
ExifIFD0Directory.TAG_ORIENTATION
);
if (value != null && value >= 1 && value <= 8) {
orientation = value;
}
}
original = reader.read(0);
int orientedWidth = orientation >= 5 ? height : width;
int orientedHeight = orientation >= 5 ? width : height;
int denominator = Math.max(
256,
Math.max(orientedWidth, orientedHeight)
);
int targetWidth = Math.max(
1,
(int) ((long) orientedWidth * 256 / denominator)
);
int targetHeight = Math.max(
1,
(int) ((long) orientedHeight * 256 / denominator)
);
target = new BufferedImage(
targetWidth,
targetHeight,
BufferedImage.TYPE_INT_RGB
);
Graphics2D graphics = target.createGraphics();
try {
graphics.setColor(Color.WHITE);
graphics.fillRect(0, 0, targetWidth, targetHeight);
graphics.setRenderingHint(
RenderingHints.KEY_INTERPOLATION,
RenderingHints.VALUE_INTERPOLATION_BICUBIC
);
AffineTransform transform =
AffineTransform.getScaleInstance(
(double) targetWidth / orientedWidth,
(double) targetHeight / orientedHeight
);
transform.concatenate(
switch (orientation) {
case 2 -> new AffineTransform(
-1, 0, 0, 1, width, 0
);
case 3 -> new AffineTransform(
-1, 0, 0, -1, width, height
);
case 4 -> new AffineTransform(
1, 0, 0, -1, 0, height
);
case 5 -> new AffineTransform(
0, 1, 1, 0, 0, 0
);
case 6 -> new AffineTransform(
0, 1, -1, 0, height, 0
);
case 7 -> new AffineTransform(
0, -1, -1, 0, height, width
);
case 8 -> new AffineTransform(
0, -1, 1, 0, 0, width
);
default -> new AffineTransform();
}
);
graphics.drawImage(original, transform, null);
} finally {
graphics.dispose();
}
var output = new ByteArrayOutputStream();
if (!ImageIO.write(target, "jpeg", output)) {
throw new IllegalArgumentException(
"jpeg_encoder_unavailable"
);
}
return new Thumbnail(
output.toByteArray(),
targetWidth,
targetHeight
);
} catch (Exception error) {
throw new IllegalArgumentException("invalid_image", error);
} finally {
if (reader != null) {
reader.dispose();
}
if (original != null) {
original.flush();
}
if (target != null) {
target.flush();
}
}
}
}
+48 -2
View File
@@ -4,11 +4,14 @@ server:
threads:
max: 64
connection-timeout: 2s
spring:
application:
name: messenger
lifecycle:
timeout-per-shutdown-phase: 15s
jackson:
default-property-inclusion: always
deserialization:
@@ -16,10 +19,12 @@ spring:
fail-on-trailing-tokens: true
mapper:
allow-coercion-of-scalars: false
datasource:
url: ${DB_URL:jdbc:postgresql://localhost:5432/messenger}
username: ${DB_USERNAME:messenger}
password: ${DB_PASSWORD:messenger}
hikari:
maximum-pool-size: ${DB_POOL_SIZE:8}
minimum-idle: 1
@@ -29,6 +34,7 @@ spring:
connectTimeout: 2
socketTimeout: 8
options: "-c statement_timeout=5000 -c lock_timeout=3000"
data:
redis:
host: ${REDIS_HOST:localhost}
@@ -36,17 +42,49 @@ spring:
password: ${REDIS_PASSWORD:}
timeout: 1s
connect-timeout: 1s
rabbitmq:
host: ${RABBITMQ_HOST:localhost}
port: 5672
username: ${RABBITMQ_USER:messenger_jobs}
password: ${RABBITMQ_PASSWORD:}
connection-timeout: 3s
requested-heartbeat: 15s
publisher-confirm-type: correlated
publisher-returns: true
template:
mandatory: true
listener:
simple:
acknowledge-mode: manual
prefetch: 1
concurrency: 1
max-concurrency: 1
default-requeue-rejected: true
task:
scheduling:
pool:
size: 3
flyway:
enabled: ${FLYWAY_ENABLED:true}
jpa:
hibernate:
ddl-auto: validate
open-in-view: false
transaction:
default-timeout: 8s
app:
session:
ttl-seconds: 86400
ttl-seconds: ${SESSION_TTL_SECONDS:3600}
s3:
endpoint: ${S3_ENDPOINT:http://localhost:9000}
@@ -57,7 +95,15 @@ app:
uploads:
max-concurrent: ${UPLOAD_MAX_CONCURRENT:1}
temp-dir: ${UPLOAD_TEMP_DIR:/tmp/messenger-uploads}
worker:
enabled: ${WORKER_ENABLED:false}
jobs:
max-attempts: ${JOB_MAX_ATTEMPTS:3}
lease-seconds: ${JOB_TIMEOUT_SECONDS:120}
failed-retention-seconds: ${STAGING_RETENTION_SECONDS:604800}
cleanup:
enabled: ${ATTACHMENT_CLEANUP_ENABLED:true}
grace-seconds: ${ATTACHMENT_CLEANUP_GRACE_SECONDS:3600}
@@ -0,0 +1,220 @@
-- Расширение метаданных вложения для асинхронной обработки.
ALTER TABLE attachments
DROP CONSTRAINT chk_attachment_status;
ALTER TABLE attachments
ADD CONSTRAINT chk_attachment_status
CHECK (
status IN (
'queued',
'processing',
'ready',
'failed'
)
);
ALTER TABLE attachments
ADD COLUMN staging_key TEXT,
ADD COLUMN pipeline_version INTEGER NOT NULL DEFAULT 1,
ADD COLUMN error_code VARCHAR(64);
ALTER TABLE attachments
ADD CONSTRAINT uq_attachment_staging_key
UNIQUE (staging_key),
ADD CONSTRAINT chk_attachment_pipeline_version
CHECK (pipeline_version >= 1),
ADD CONSTRAINT chk_attachment_error
CHECK (
(
status = 'failed'
AND error_code IS NOT NULL
AND length(error_code) > 0
)
OR
(
status <> 'failed'
AND error_code IS NULL
)
);
CREATE INDEX idx_attachments_status
ON attachments(status, updated_at);
-- Готовые варианты изображения.
-- object_key используется внутри приложения и не выдаётся клиенту.
-- Ключ эффекта соответствует контракту:
-- attachment_id + pipeline_version + variant.
CREATE TABLE attachment_variants (
attachment_id UUID NOT NULL
REFERENCES attachments(id) ON DELETE CASCADE,
pipeline_version INTEGER NOT NULL,
name VARCHAR(32) NOT NULL,
object_key TEXT NOT NULL UNIQUE,
content_type VARCHAR(32) NOT NULL,
size_bytes BIGINT NOT NULL,
width INTEGER NOT NULL,
height INTEGER NOT NULL,
sha256 CHAR(64) NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (
attachment_id,
pipeline_version,
name
),
CONSTRAINT chk_variant_pipeline_version
CHECK (pipeline_version >= 1),
CONSTRAINT chk_variant_name
CHECK (
name IN (
'thumbnail',
'crop',
'optimized'
)
),
CONSTRAINT chk_variant_content_type
CHECK (
content_type IN (
'image/jpeg',
'image/png',
'image/webp'
)
),
CONSTRAINT chk_variant_size
CHECK (size_bytes > 0),
CONSTRAINT chk_variant_dimensions
CHECK (width > 0 AND height > 0),
CONSTRAINT chk_variant_sha256
CHECK (sha256 ~ '^[a-f0-9]{64}$')
);
-- Одна логическая работа над вложением в рамках версии pipeline.
-- job_id не меняется при повторной доставке.
-- attempts увеличивается при начале управляемой попытки,
-- а не при каждом получении дубликата сообщения.
CREATE TABLE attachment_jobs (
id UUID PRIMARY KEY,
attachment_id UUID NOT NULL
REFERENCES attachments(id) ON DELETE CASCADE,
pipeline_version INTEGER NOT NULL DEFAULT 1,
status VARCHAR(16) NOT NULL DEFAULT 'queued',
attempts INTEGER NOT NULL DEFAULT 0,
max_attempts INTEGER NOT NULL DEFAULT 3,
available_at TIMESTAMPTZ NOT NULL DEFAULT now(),
lease_owner UUID,
lease_until TIMESTAMPTZ,
last_error_code VARCHAR(64),
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
CONSTRAINT uq_attachment_job_pipeline
UNIQUE (
attachment_id,
pipeline_version
),
CONSTRAINT chk_job_pipeline_version
CHECK (pipeline_version >= 1),
CONSTRAINT chk_job_status
CHECK (
status IN (
'queued',
'processing',
'ready',
'failed'
)
),
CONSTRAINT chk_job_attempts
CHECK (
max_attempts >= 1
AND attempts >= 0
AND attempts <= max_attempts
),
CONSTRAINT chk_job_lease
CHECK (
(
status = 'processing'
AND lease_owner IS NOT NULL
AND lease_until IS NOT NULL
)
OR
(
status <> 'processing'
AND lease_owner IS NULL
AND lease_until IS NULL
)
)
);
CREATE INDEX idx_attachment_jobs_available
ON attachment_jobs(available_at, id)
WHERE status = 'queued';
CREATE INDEX idx_attachment_jobs_expired_lease
ON attachment_jobs(lease_until, id)
WHERE status = 'processing';
-- Намерение публикации задания в брокер.
-- Записывается в той же транзакции, что и метаданные задания.
-- published_at устанавливается после подтверждения брокера.
-- Потеря подтверждения может привести к повторной публикации:
-- worker обязан обрабатывать её идемпотентно.
CREATE TABLE attachment_outbox (
id UUID PRIMARY KEY,
job_id UUID NOT NULL
REFERENCES attachment_jobs(id) ON DELETE CASCADE,
attempt INTEGER NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
next_publish_at TIMESTAMPTZ NOT NULL DEFAULT now(),
published_at TIMESTAMPTZ,
publish_attempts INTEGER NOT NULL DEFAULT 0,
last_error_code VARCHAR(64),
CONSTRAINT uq_attachment_outbox_attempt
UNIQUE (job_id, attempt),
CONSTRAINT chk_outbox_attempt
CHECK (attempt >= 1),
CONSTRAINT chk_outbox_publish_attempts
CHECK (publish_attempts >= 0)
);
CREATE INDEX idx_attachment_outbox_pending
ON attachment_outbox(next_publish_at, id)
WHERE published_at IS NULL;
@@ -0,0 +1,94 @@
CREATE TABLE attachment_replay_audit (
id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
job_id UUID NOT NULL REFERENCES attachment_jobs(id),
actor VARCHAR(64) NOT NULL,
reason VARCHAR(255) NOT NULL,
action VARCHAR(32) NOT NULL
CHECK (action IN ('enqueued', 'already_ready')),
previous_attempts INTEGER NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE attachment_attempt_events (
id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
job_id UUID NOT NULL REFERENCES attachment_jobs(id),
attempt INTEGER NOT NULL,
state VARCHAR(16) NOT NULL,
error_code VARCHAR(64) NOT NULL,
delay_seconds INTEGER NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
);
-- Используется только при FAULT_INJECTION_ENABLED=true.
-- Управления через публичный HTTP API нет.
CREATE TABLE attachment_test_faults (
point VARCHAR(32) PRIMARY KEY,
remaining INTEGER NOT NULL CHECK (remaining >= 0),
hits INTEGER NOT NULL DEFAULT 0,
job_id UUID REFERENCES attachment_jobs(id),
hit_at TIMESTAMPTZ
);
-- Существующие ready-вложения с неполным набором вариантов
-- отправляются на дообработку.
INSERT INTO attachment_jobs (
id,
attachment_id,
pipeline_version,
status,
max_attempts
)
SELECT
gen_random_uuid(),
a.id,
a.pipeline_version,
'queued',
3
FROM attachments a
WHERE a.status = 'ready'
AND (
SELECT count(*)
FROM attachment_variants v
WHERE v.attachment_id = a.id
AND v.pipeline_version = a.pipeline_version
AND v.name IN ('thumbnail', 'crop', 'optimized')
) < 3
ON CONFLICT (attachment_id, pipeline_version)
DO UPDATE SET
status = 'queued',
max_attempts = greatest(
attachment_jobs.max_attempts,
attachment_jobs.attempts + 3
),
available_at = clock_timestamp(),
lease_owner = NULL,
lease_until = NULL,
last_error_code = NULL,
updated_at = clock_timestamp()
WHERE attachment_jobs.status = 'ready';
UPDATE attachments a
SET status = 'queued',
staging_key = a.object_key,
error_code = NULL,
updated_at = clock_timestamp()
WHERE a.status = 'ready'
AND EXISTS (
SELECT 1
FROM attachment_jobs j
WHERE j.attachment_id = a.id
AND j.pipeline_version = a.pipeline_version
AND j.status = 'queued'
);
INSERT INTO attachment_outbox(id, job_id, attempt)
SELECT gen_random_uuid(), id, attempts + 1
FROM attachment_jobs
WHERE status = 'queued'
ON CONFLICT (job_id, attempt) DO NOTHING;