Files
mtusi2026lab5/src/main/java/ru/mtusi/messenger/Jobs.java
T

437 lines
26 KiB
Java

package ru.mtusi.messenger;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import java.io.IOException;
import java.sql.Timestamp;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ThreadLocalRandom;
import javax.imageio.ImageReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.support.TransactionTemplate;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
import software.amazon.awssdk.services.s3.model.S3Object;
// Задания обработки вложений. Источник истины — таблица jobs в PostgreSQL; сообщение брокера —
// только сигнал «попытка attempt задания job_id готова к запуску» (contracts/jobs.schema.json).
//
// - вложение, job и запись outbox пишутся одной транзакцией с приёмом файла, публикует relay;
// - attempt — fencing token: захват, продление аренды и фиксация результата сверяют его в БД;
// - эффект идемпотентен по (attachment_id, pipeline_version, variant): детерминированные ключи S3 и PK.
@Component
class Jobs {
static final String TYPE = "attachment.process.v1";
static final int MAX_ATTEMPTS = env("JOB_MAX_ATTEMPTS", 5);
static final int TIMEOUT_SECONDS = env("JOB_TIMEOUT_SECONDS", 60);
static final int LEASE_SECONDS = env("JOB_LEASE_SECONDS", 15);
static final int PIPELINE_VERSION = env("PIPELINE_VERSION", 1);
static final long BACKOFF_BASE_MS = env("JOB_BACKOFF_BASE_MS", 1000), BACKOFF_MAX_MS = env("JOB_BACKOFF_MAX_MS", 30000);
static final long STAGING_RETENTION_SECONDS = env("STAGING_RETENTION_SECONDS", 86400);
// Внедрение сбоев только при FAULT_INJECTION=1; точку выбирает префикс имени файла fault-<point>.
static final boolean FAULTS = "1".equals(System.getenv("FAULT_INJECTION"));
static final String ACTOR = System.getenv().getOrDefault("HOSTNAME", "local");
static final ObjectMapper JSON = new ObjectMapper();
static final Logger log = LoggerFactory.getLogger(Jobs.class);
final JdbcTemplate db;
final TransactionTemplate tx, separate;
final Broker broker;
final boolean relayEnabled;
long relayPausedUntil, lastWarning;
Jobs(JdbcTemplate db, PlatformTransactionManager tm, Broker broker, @Value("${outbox.enabled:true}") boolean relayEnabled) {
this.db = db;
this.broker = broker;
this.relayEnabled = relayEnabled;
tx = new TransactionTemplate(tm);
separate = new TransactionTemplate(tm);
separate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
}
static int env(String name, int fallback) {
return Integer.parseInt(System.getenv().getOrDefault(name, String.valueOf(fallback)));
}
@JsonInclude(JsonInclude.Include.NON_NULL)
record Msg(Integer schema_version, UUID job_id, String type, UUID attachment_id, Integer pipeline_version, Integer attempt,
String created_at, String traceparent, String tracestate) {}
record Variant(String name, String contentType, long size, int width, int height, String sha256, String key) {}
record Source(String contentType, long size, String sha256, String key) {}
// Приём: метаданные, задание и outbox — одна транзакция. После COMMIT работа не теряется
// ни при падении API, ни при недоступном брокере: relay опубликует позже.
void accept(UUID id, UUID chat, UUID owner, String filename, String type, long size, String sha256, int width, int height, Instant at) {
Timestamp ts = Timestamp.from(at);
tx.executeWithoutResult(s -> {
db.update("insert into attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,width,height,state,object_key,created_at,updated_at)"
+ " values(?,?,?,?,?,?,?,?,?,'queued',?,?,?)", id, chat, owner, filename, type, size, sha256, width, height, Storage.ORIGINALS + id, ts, ts);
UUID job = UUID.randomUUID();
db.update("insert into jobs(job_id,attachment_id,pipeline_version,state,attempt,attempts_limit,created_at,updated_at) values(?,?,?,'queued',1,?,?,?)",
job, id, PIPELINE_VERSION, MAX_ATTEMPTS, ts, ts);
db.update("insert into outbox(job_id,attempt) values(?,1)", job);
audit(job, 1, "accepted", null);
});
}
// Outbox relay работает в API-репликах и в worker; FOR UPDATE SKIP LOCKED делит записи между ними.
// Падение после publish до DELETE даёт повторную публикацию — её гасит идемпотентный consumer.
// ponytail: опрос раз в секунду (+≤1 с к времени до ready); LISTEN/NOTIFY — если задержка станет важна.
@Scheduled(fixedDelayString = "${outbox.poll:PT1S}")
void relay() {
if (!relayEnabled || System.currentTimeMillis() < relayPausedUntil) return;
try {
while (Boolean.TRUE.equals(tx.execute(s -> relayBatch()))) {
// полная пачка: сразу следующая
}
} catch (RuntimeException e) {
relayPausedUntil = System.currentTimeMillis() + 2000;
if (System.currentTimeMillis() - lastWarning > 30000) {
lastWarning = System.currentTimeMillis();
log.warn("outbox relay paused: {}", e.toString());
}
}
}
record Pending(long id, UUID job, int attempt, UUID attachment, int version, Instant created, String filename) {}
boolean relayBatch() {
List<Pending> rows = db.query("select o.id,o.job_id,o.attempt,j.attachment_id,j.pipeline_version,j.created_at,a.filename from outbox o"
+ " join jobs j on j.job_id=o.job_id join attachments a on a.id=j.attachment_id where o.available_at<=now() order by o.id limit 100 for update of o skip locked",
(rs, n) -> new Pending(rs.getLong(1), rs.getObject(2, UUID.class), rs.getInt(3), rs.getObject(4, UUID.class), rs.getInt(5), rs.getTimestamp(6).toInstant(), rs.getString(7)));
if (rows.isEmpty()) return false;
List<Broker.Message> messages = new ArrayList<>();
for (Pending r : rows) {
crash("crash-before-publish", r.attachment(), r.filename());
messages.add(new Broker.Message(r.job() + "/" + r.attempt(), message(r.job(), r.attempt(), r.attachment(), r.version(), r.created())));
}
try {
broker.publish(messages);
} catch (Exception e) {
throw new IllegalStateException("publish failed: " + e, e);
}
for (Pending r : rows) crash("crash-after-publish", r.attachment(), r.filename());
db.update("delete from outbox where id=any(?)", (Object) rows.stream().map(Pending::id).toArray(Long[]::new));
return rows.size() == 100;
}
static byte[] message(UUID job, int attempt, UUID attachment, int version, Instant created) {
try {
return JSON.writeValueAsBytes(new Msg(1, job, TYPE, attachment, version, attempt, created.toString(), null, null));
} catch (IOException e) {
throw new IllegalStateException(e);
}
}
// Строгая проверка по jobs.schema.json; лишние поля — ошибка разбора (сообщение уйдёт в DLQ).
static Msg parse(byte[] body) throws IOException {
Msg m = JSON.readValue(body, Msg.class);
if (!Integer.valueOf(1).equals(m.schema_version()) || !TYPE.equals(m.type()) || m.job_id() == null || m.attachment_id() == null
|| m.pipeline_version() == null || m.pipeline_version() < 1 || m.attempt() == null || m.attempt() < 1 || m.created_at() == null)
throw new IOException("message does not match jobs.schema.json");
Instant.parse(m.created_at());
return m;
}
// Захват попытки: только queued с тем же attempt. Иначе дубль или устаревшее сообщение — no-op и ack.
Claim claim(Msg m) {
return tx.execute(s -> {
List<Claim> claimed = db.query("update jobs j set state='processing',lease_owner=?,lease_until=now()+make_interval(secs=>?),started_at=now(),updated_at=now()"
+ " from attachments a where j.job_id=? and j.state='queued' and j.attempt=? and j.attachment_id=? and j.pipeline_version=? and a.id=j.attachment_id"
+ " returning a.filename", (rs, n) -> new Claim(m.job_id(), m.attachment_id(), m.attempt(), m.pipeline_version(), rs.getString(1)),
ACTOR, LEASE_SECONDS, m.job_id(), m.attempt(), m.attachment_id(), m.pipeline_version());
if (claimed.isEmpty()) {
// Причина — в audit: по нему эксперимент считает погашенные дубли.
String why = db.query("select state,attempt from jobs where job_id=?", rs -> !rs.next() ? "unknown_job"
: rs.getString(1).equals("done") ? "duplicate_done" : rs.getInt(2) > m.attempt() ? "stale_attempt"
: rs.getString(1).equals("processing") ? "duplicate_in_progress" : rs.getString(1).equals("failed") ? "dropped_failed" : "mismatch", m.job_id());
if (!why.equals("unknown_job")) audit(m.job_id(), m.attempt(), why, null);
log.info("job {} attempt {}: {}, ack without processing", m.job_id(), m.attempt(), why);
return null;
}
// ready-вложение (новая версия pipeline) остаётся ready: старый результат отдаётся до новой фиксации.
db.update("update attachments set state='processing',updated_at=now() where id=? and state='queued'", m.attachment_id());
audit(m.job_id(), m.attempt(), "claimed", null);
return claimed.get(0);
});
}
Source source(UUID attachment) {
return db.queryForObject("select content_type,size_bytes,sha256,object_key from attachments where id=?",
(rs, n) -> new Source(rs.getString(1), rs.getLong(2), rs.getString(3), rs.getString(4)), attachment);
}
// Heartbeat: продлевает аренду, но не дальше JOB_TIMEOUT_SECONDS от начала попытки.
boolean extend(Claim c) {
return db.update("update jobs set lease_until=now()+make_interval(secs=>?) where job_id=? and state='processing' and attempt=? and started_at>now()-make_interval(secs=>?)",
LEASE_SECONDS, c.job, c.attempt, TIMEOUT_SECONDS) == 1;
}
// Фиксация результата: варианты, указатель версии и done — атомарно и только владельцем попытки.
boolean complete(Claim c, List<Variant> variants) {
return Boolean.TRUE.equals(tx.execute(s -> {
if (lock(c) == null) return false;
for (Variant v : variants)
db.update("insert into attachment_variants(attachment_id,pipeline_version,name,content_type,size_bytes,width,height,sha256,object_key,job_id)"
+ " values(?,?,?,?,?,?,?,?,?,?) on conflict do nothing", c.attachment, c.version, v.name(), v.contentType(), v.size(), v.width(), v.height(), v.sha256(), v.key(), c.job);
// Указатель текущей версии только растёт: старая job после новой не перезапишет результат.
boolean current = db.update("update attachments set state='ready',error_code=null,pipeline_version=?,updated_at=now() where id=? and state<>'lost'"
+ " and coalesce(pipeline_version,0)<=?", c.version, c.attachment, c.version) == 1;
db.update("update jobs set state='done',lease_owner=null,lease_until=null,error_code=null,updated_at=now() where job_id=?", c.job);
audit(c.job, c.attempt, "done", current ? "current" : "superseded");
return true;
}));
}
// Временная ошибка: следующая попытка через backoff с jitter, после исчерпания — failed/retry_exhausted.
void retry(Claim c, Exception error) {
tx.executeWithoutResult(s -> {
Integer limit = lock(c);
if (limit == null) return;
if (c.attempt >= limit) {
failLocked(c.job, c.attempt, c.attachment, "retry_exhausted", error.toString());
return;
}
long cap = Math.min(BACKOFF_MAX_MS, BACKOFF_BASE_MS << Math.min(20, c.attempt - 1));
long delay = cap / 2 + ThreadLocalRandom.current().nextLong(cap / 2 + 1);
requeueLocked(c.job, c.attempt, c.attachment, delay, "retry_scheduled", error.toString());
});
}
void fail(Claim c, String code, String error) {
tx.executeWithoutResult(s -> {
if (lock(c) != null) failLocked(c.job, c.attempt, c.attachment, code, error);
});
}
// Fencing: попытка ещё владеет заданием; строка заблокирована до конца транзакции. null — не владеет.
Integer lock(Claim c) {
Integer limit = db.query("select attempts_limit from jobs where job_id=? and state='processing' and attempt=? for update",
rs -> rs.next() ? rs.getInt(1) : null, c.job, c.attempt);
if (limit == null) log.warn("job {} attempt {} no longer owns the lease, result discarded", c.job, c.attempt);
return limit;
}
void requeueLocked(UUID job, int attempt, UUID attachment, long delayMs, String event, String error) {
db.update("update jobs set state='queued',attempt=attempt+1,lease_owner=null,lease_until=null,next_attempt_at=now()+make_interval(secs=>?),last_error=?,updated_at=now() where job_id=?",
delayMs / 1000.0, truncate(error), job);
db.update("update attachments set state='queued',updated_at=now() where id=? and state='processing'", attachment);
db.update("insert into outbox(job_id,attempt,available_at) values(?,?,now()+make_interval(secs=>?))", job, attempt + 1, delayMs / 1000.0);
audit(job, attempt, event, "next_attempt=" + (attempt + 1) + " delay_ms=" + delayMs + " error=" + truncate(error));
}
void failLocked(UUID job, int attempt, UUID attachment, String code, String error) {
db.update("update jobs set state='failed',error_code=?,last_error=?,lease_owner=null,lease_until=null,updated_at=now() where job_id=?", code, truncate(error), job);
// failed только у вложения без готовой версии: ready (другая версия pipeline) не откатывается.
db.update("update attachments set state='failed',error_code=?,updated_at=now() where id=? and state in ('queued','processing') and pipeline_version is null", code, attachment);
audit(job, attempt, "failed", code);
}
// Истёкшая аренда: worker умер или завис (heartbeat не продлевал) — следующая попытка, без второго эффекта.
void sweep() {
tx.executeWithoutResult(s -> {
List<Object[]> expired = db.query("select job_id,attempt,attempts_limit,attachment_id from jobs where state='processing' and lease_until<now()"
+ " order by lease_until limit 100 for update skip locked", (rs, n) -> new Object[]{rs.getObject(1, UUID.class), rs.getInt(2), rs.getInt(3), rs.getObject(4, UUID.class)});
for (Object[] j : expired) {
UUID job = (UUID) j[0], attachment = (UUID) j[3];
int attempt = (int) j[1];
if (attempt >= (int) j[2]) failLocked(job, attempt, attachment, "retry_exhausted", "lease expired");
else requeueLocked(job, attempt, attachment, 0, "lease_expired", "lease expired");
}
});
}
// Staging удаляется после безопасной фиксации (ready); непривязанные и failed — после retention.
int sweepStaging(Storage storage) {
int deleted = 0;
Instant cutoff = Instant.now().minusSeconds(STAGING_RETENTION_SECONDS);
for (ListObjectsV2Response page : storage.list(Storage.STAGING)) {
Map<UUID, S3Object> objects = new HashMap<>();
for (S3Object o : page.contents()) {
try { objects.put(UUID.fromString(o.key().substring(Storage.STAGING.length())), o); } catch (IllegalArgumentException ignored) {}
}
if (objects.isEmpty()) continue;
Map<UUID, Object[]> rows = new HashMap<>();
db.query("select id,state,updated_at from attachments where id=any(?)", rs -> {
rows.put(rs.getObject(1, UUID.class), new Object[]{rs.getString(2), rs.getTimestamp(3).toInstant()});
}, (Object) objects.keySet().toArray(new UUID[0]));
for (Map.Entry<UUID, S3Object> e : objects.entrySet()) {
Object[] r = rows.get(e.getKey());
boolean delete = r == null ? e.getValue().lastModified().isBefore(cutoff)
: r[0].equals("ready") || (r[0].equals("failed") && ((Instant) r[1]).isBefore(cutoff));
if (delete) {
storage.delete(e.getValue().key());
deleted++;
}
}
}
if (deleted > 0) log.info("staging sweep deleted {} objects", deleted);
return deleted;
}
// Операторские команды: docker compose run --rm jobctl <command>.
int cli(String[] args) throws IOException {
String command = args.length > 0 ? args[0] : "";
Object result;
switch (command) {
case "status" -> {
Map<String, Object> r = new LinkedHashMap<>();
r.put("jobs", db.queryForList("select pipeline_version,state,count(*) from jobs group by 1,2 order by 1,2"));
r.put("attachments", db.queryForList("select state,count(*) from attachments group by 1 order by 1"));
r.put("outbox_pending", db.queryForObject("select count(*) from outbox", Long.class));
r.put("failed", db.queryForList("select job_id::text,attachment_id::text,pipeline_version,attempt,error_code,updated_at::text from jobs where state='failed' order by updated_at desc limit 50"));
result = r;
}
case "replay" -> {
List<UUID> ids = args.length > 1 && args[1].equals("--all-failed")
? db.queryForList("select job_id from jobs where state='failed'", UUID.class) : ids(args, 1);
Map<String, String> r = new LinkedHashMap<>();
for (UUID id : ids) r.put(id.toString(), replay(id));
result = r;
}
case "reprocess" -> {
if (args.length < 3 || !args[1].startsWith("--pipeline-version=")) return usage();
int version = Integer.parseInt(args[1].substring(19));
boolean all = args[2].equals("--all");
result = Map.of("jobs_created", reprocess(version, all ? List.of() : ids(args, 2), all));
}
case "redispatch" -> {
long older = args.length > 1 && args[1].startsWith("--older-than=") ? Long.parseLong(args[1].substring(13)) : 300;
result = Map.of("republished", redispatch(older));
}
default -> { return usage(); }
}
System.out.println(JSON.copy().enable(SerializationFeature.INDENT_OUTPUT).writeValueAsString(result));
return 0;
}
static int usage() {
System.err.println("usage: jobctl status | replay <job_id>...|--all-failed | reprocess --pipeline-version=N <attachment_id>...|--all | redispatch [--older-than=SECONDS]");
return 2;
}
static List<UUID> ids(String[] args, int from) {
List<UUID> ids = new ArrayList<>();
for (int i = from; i < args.length; i++) ids.add(UUID.fromString(args[i]));
return ids;
}
// Явный replay только failed: done-задание не исполняется повторно, прежний эффект защищён PK и fencing.
String replay(UUID job) {
return tx.execute(s -> {
Object[] j = db.query("select state,attempt,attachment_id from jobs where job_id=? for update",
rs -> rs.next() ? new Object[]{rs.getString(1), rs.getInt(2), rs.getObject(3, UUID.class)} : null, job);
if (j == null) return "unknown job";
if (!j[0].equals("failed")) return "skipped: job is " + j[0];
int attempt = (int) j[1] + 1;
db.update("update jobs set state='queued',attempt=?,attempts_limit=?,error_code=null,next_attempt_at=now(),updated_at=now() where job_id=?",
attempt, attempt + MAX_ATTEMPTS - 1, job);
db.update("update attachments set state='queued',error_code=null,updated_at=now() where id=? and state='failed'", j[2]);
db.update("insert into outbox(job_id,attempt) values(?,?)", job, attempt);
audit(job, attempt, "replayed", "operator");
return "queued as attempt " + attempt;
});
}
// Новая версия pipeline — новая логическая работа (новый job_id); на одну версию не больше одной job.
int reprocess(int version, List<UUID> ids, boolean all) {
return db.update("with j as (insert into jobs(job_id,attachment_id,pipeline_version,state,attempt,attempts_limit,created_at,updated_at)"
+ " select gen_random_uuid(),id,?,'queued',1,?,now(),now() from attachments where state in ('queued','processing','ready') and (? or id=any(?))"
+ " on conflict (attachment_id,pipeline_version) do nothing returning job_id) insert into outbox(job_id,attempt) select job_id,1 from j",
version, MAX_ATTEMPTS, all, ids.toArray(new UUID[0]));
}
// Ручное восстановление queued-заданий, чьё сообщение потеряно (например, ушло в DLQ по delivery-limit).
int redispatch(long olderThanSeconds) {
return db.update("with r as (update jobs j set next_attempt_at=now(),updated_at=now() where state='queued' and next_attempt_at<now()-make_interval(secs=>?)"
+ " and not exists(select 1 from outbox o where o.job_id=j.job_id) returning job_id,attempt) insert into outbox(job_id,attempt) select job_id,attempt from r", olderThanSeconds);
}
void audit(UUID job, int attempt, String event, String detail) {
db.update("insert into job_audit(job_id,attempt,event,actor,detail) values(?,?,?,?,?)", job, attempt, event, ACTOR, detail);
}
static String truncate(String s) {
return s == null || s.length() <= 500 ? s : s.substring(0, 500);
}
// Аварийная остановка процесса в заданной точке, один раз на вложение (отметка коммитится до halt).
void crash(String point, UUID attachment, String filename) {
if (!FAULTS || filename == null || !filename.startsWith("fault-" + point)) return;
Integer fired = separate.execute(s -> db.update("insert into fault_marks(point,attachment_id) values(?,?) on conflict do nothing", point, attachment));
if (fired == null || fired == 0) return;
log.warn("FAULT INJECTION {}: halting process (attachment {})", point, attachment);
// Пауза, чтобы уже отправляемый HTTP-ответ этого процесса дошёл: падение — именно в фоновой точке.
try { Thread.sleep(1000); } catch (InterruptedException ignored) {}
Runtime.getRuntime().halt(137);
}
// Захваченная попытка в worker: флаги для heartbeat/таймаута и ссылки, которые прерывает abort().
static final class Claim {
final UUID job, attachment;
final int attempt, version;
final String filename;
final long started = System.nanoTime();
volatile boolean fenced;
volatile ImageReader reader;
volatile Process process;
Claim(UUID job, UUID attachment, int attempt, int version, String filename) {
this.job = job;
this.attachment = attachment;
this.attempt = attempt;
this.version = version;
this.filename = filename;
}
long elapsedSeconds() {
return (System.nanoTime() - started) / 1_000_000_000L;
}
// Перед каждым побочным эффектом: аренда потеряна — стоп без записи; превышен таймаут — временная ошибка.
void check() {
if (fenced) throw new Fenced();
if (elapsedSeconds() >= TIMEOUT_SECONDS) throw new TimedOut();
}
void abort() {
ImageReader r = reader;
if (r != null) r.abort();
Process p = process;
if (p != null) p.destroyForcibly();
}
}
static final class Fenced extends RuntimeException {
Fenced() { super("lease lost"); }
}
static final class TimedOut extends RuntimeException {
TimedOut() { super("job timeout exceeded"); }
}
static final class Permanent extends RuntimeException {
final String code;
Permanent(String code, String message) {
super(message);
this.code = code;
}
}
}