Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
437 lines
26 KiB
Java
437 lines
26 KiB
Java
package ru.mtusi.messenger;
|
|
|
|
import com.fasterxml.jackson.annotation.JsonInclude;
|
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
import com.fasterxml.jackson.databind.SerializationFeature;
|
|
import java.io.IOException;
|
|
import java.sql.Timestamp;
|
|
import java.time.Instant;
|
|
import java.util.ArrayList;
|
|
import java.util.HashMap;
|
|
import java.util.LinkedHashMap;
|
|
import java.util.List;
|
|
import java.util.Map;
|
|
import java.util.UUID;
|
|
import java.util.concurrent.ThreadLocalRandom;
|
|
import javax.imageio.ImageReader;
|
|
import org.slf4j.Logger;
|
|
import org.slf4j.LoggerFactory;
|
|
import org.springframework.beans.factory.annotation.Value;
|
|
import org.springframework.jdbc.core.JdbcTemplate;
|
|
import org.springframework.scheduling.annotation.Scheduled;
|
|
import org.springframework.stereotype.Component;
|
|
import org.springframework.transaction.PlatformTransactionManager;
|
|
import org.springframework.transaction.TransactionDefinition;
|
|
import org.springframework.transaction.support.TransactionTemplate;
|
|
import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
|
|
import software.amazon.awssdk.services.s3.model.S3Object;
|
|
|
|
// Задания обработки вложений. Источник истины — таблица jobs в PostgreSQL; сообщение брокера —
|
|
// только сигнал «попытка attempt задания job_id готова к запуску» (contracts/jobs.schema.json).
|
|
//
|
|
// - вложение, job и запись outbox пишутся одной транзакцией с приёмом файла, публикует relay;
|
|
// - attempt — fencing token: захват, продление аренды и фиксация результата сверяют его в БД;
|
|
// - эффект идемпотентен по (attachment_id, pipeline_version, variant): детерминированные ключи S3 и PK.
|
|
@Component
|
|
class Jobs {
|
|
static final String TYPE = "attachment.process.v1";
|
|
static final int MAX_ATTEMPTS = env("JOB_MAX_ATTEMPTS", 5);
|
|
static final int TIMEOUT_SECONDS = env("JOB_TIMEOUT_SECONDS", 60);
|
|
static final int LEASE_SECONDS = env("JOB_LEASE_SECONDS", 15);
|
|
static final int PIPELINE_VERSION = env("PIPELINE_VERSION", 1);
|
|
static final long BACKOFF_BASE_MS = env("JOB_BACKOFF_BASE_MS", 1000), BACKOFF_MAX_MS = env("JOB_BACKOFF_MAX_MS", 30000);
|
|
static final long STAGING_RETENTION_SECONDS = env("STAGING_RETENTION_SECONDS", 86400);
|
|
// Внедрение сбоев только при FAULT_INJECTION=1; точку выбирает префикс имени файла fault-<point>.
|
|
static final boolean FAULTS = "1".equals(System.getenv("FAULT_INJECTION"));
|
|
static final String ACTOR = System.getenv().getOrDefault("HOSTNAME", "local");
|
|
static final ObjectMapper JSON = new ObjectMapper();
|
|
static final Logger log = LoggerFactory.getLogger(Jobs.class);
|
|
|
|
final JdbcTemplate db;
|
|
final TransactionTemplate tx, separate;
|
|
final Broker broker;
|
|
final boolean relayEnabled;
|
|
long relayPausedUntil, lastWarning;
|
|
|
|
Jobs(JdbcTemplate db, PlatformTransactionManager tm, Broker broker, @Value("${outbox.enabled:true}") boolean relayEnabled) {
|
|
this.db = db;
|
|
this.broker = broker;
|
|
this.relayEnabled = relayEnabled;
|
|
tx = new TransactionTemplate(tm);
|
|
separate = new TransactionTemplate(tm);
|
|
separate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
|
|
}
|
|
|
|
static int env(String name, int fallback) {
|
|
return Integer.parseInt(System.getenv().getOrDefault(name, String.valueOf(fallback)));
|
|
}
|
|
|
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
|
record Msg(Integer schema_version, UUID job_id, String type, UUID attachment_id, Integer pipeline_version, Integer attempt,
|
|
String created_at, String traceparent, String tracestate) {}
|
|
|
|
record Variant(String name, String contentType, long size, int width, int height, String sha256, String key) {}
|
|
|
|
record Source(String contentType, long size, String sha256, String key) {}
|
|
|
|
// Приём: метаданные, задание и outbox — одна транзакция. После COMMIT работа не теряется
|
|
// ни при падении API, ни при недоступном брокере: relay опубликует позже.
|
|
void accept(UUID id, UUID chat, UUID owner, String filename, String type, long size, String sha256, int width, int height, Instant at) {
|
|
Timestamp ts = Timestamp.from(at);
|
|
tx.executeWithoutResult(s -> {
|
|
db.update("insert into attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,width,height,state,object_key,created_at,updated_at)"
|
|
+ " values(?,?,?,?,?,?,?,?,?,'queued',?,?,?)", id, chat, owner, filename, type, size, sha256, width, height, Storage.ORIGINALS + id, ts, ts);
|
|
UUID job = UUID.randomUUID();
|
|
db.update("insert into jobs(job_id,attachment_id,pipeline_version,state,attempt,attempts_limit,created_at,updated_at) values(?,?,?,'queued',1,?,?,?)",
|
|
job, id, PIPELINE_VERSION, MAX_ATTEMPTS, ts, ts);
|
|
db.update("insert into outbox(job_id,attempt) values(?,1)", job);
|
|
audit(job, 1, "accepted", null);
|
|
});
|
|
}
|
|
|
|
// Outbox relay работает в API-репликах и в worker; FOR UPDATE SKIP LOCKED делит записи между ними.
|
|
// Падение после publish до DELETE даёт повторную публикацию — её гасит идемпотентный consumer.
|
|
// ponytail: опрос раз в секунду (+≤1 с к времени до ready); LISTEN/NOTIFY — если задержка станет важна.
|
|
@Scheduled(fixedDelayString = "${outbox.poll:PT1S}")
|
|
void relay() {
|
|
if (!relayEnabled || System.currentTimeMillis() < relayPausedUntil) return;
|
|
try {
|
|
while (Boolean.TRUE.equals(tx.execute(s -> relayBatch()))) {
|
|
// полная пачка: сразу следующая
|
|
}
|
|
} catch (RuntimeException e) {
|
|
relayPausedUntil = System.currentTimeMillis() + 2000;
|
|
if (System.currentTimeMillis() - lastWarning > 30000) {
|
|
lastWarning = System.currentTimeMillis();
|
|
log.warn("outbox relay paused: {}", e.toString());
|
|
}
|
|
}
|
|
}
|
|
|
|
record Pending(long id, UUID job, int attempt, UUID attachment, int version, Instant created, String filename) {}
|
|
|
|
boolean relayBatch() {
|
|
List<Pending> rows = db.query("select o.id,o.job_id,o.attempt,j.attachment_id,j.pipeline_version,j.created_at,a.filename from outbox o"
|
|
+ " join jobs j on j.job_id=o.job_id join attachments a on a.id=j.attachment_id where o.available_at<=now() order by o.id limit 100 for update of o skip locked",
|
|
(rs, n) -> new Pending(rs.getLong(1), rs.getObject(2, UUID.class), rs.getInt(3), rs.getObject(4, UUID.class), rs.getInt(5), rs.getTimestamp(6).toInstant(), rs.getString(7)));
|
|
if (rows.isEmpty()) return false;
|
|
List<Broker.Message> messages = new ArrayList<>();
|
|
for (Pending r : rows) {
|
|
crash("crash-before-publish", r.attachment(), r.filename());
|
|
messages.add(new Broker.Message(r.job() + "/" + r.attempt(), message(r.job(), r.attempt(), r.attachment(), r.version(), r.created())));
|
|
}
|
|
try {
|
|
broker.publish(messages);
|
|
} catch (Exception e) {
|
|
throw new IllegalStateException("publish failed: " + e, e);
|
|
}
|
|
for (Pending r : rows) crash("crash-after-publish", r.attachment(), r.filename());
|
|
db.update("delete from outbox where id=any(?)", (Object) rows.stream().map(Pending::id).toArray(Long[]::new));
|
|
return rows.size() == 100;
|
|
}
|
|
|
|
static byte[] message(UUID job, int attempt, UUID attachment, int version, Instant created) {
|
|
try {
|
|
return JSON.writeValueAsBytes(new Msg(1, job, TYPE, attachment, version, attempt, created.toString(), null, null));
|
|
} catch (IOException e) {
|
|
throw new IllegalStateException(e);
|
|
}
|
|
}
|
|
|
|
// Строгая проверка по jobs.schema.json; лишние поля — ошибка разбора (сообщение уйдёт в DLQ).
|
|
static Msg parse(byte[] body) throws IOException {
|
|
Msg m = JSON.readValue(body, Msg.class);
|
|
if (!Integer.valueOf(1).equals(m.schema_version()) || !TYPE.equals(m.type()) || m.job_id() == null || m.attachment_id() == null
|
|
|| m.pipeline_version() == null || m.pipeline_version() < 1 || m.attempt() == null || m.attempt() < 1 || m.created_at() == null)
|
|
throw new IOException("message does not match jobs.schema.json");
|
|
Instant.parse(m.created_at());
|
|
return m;
|
|
}
|
|
|
|
// Захват попытки: только queued с тем же attempt. Иначе дубль или устаревшее сообщение — no-op и ack.
|
|
Claim claim(Msg m) {
|
|
return tx.execute(s -> {
|
|
List<Claim> claimed = db.query("update jobs j set state='processing',lease_owner=?,lease_until=now()+make_interval(secs=>?),started_at=now(),updated_at=now()"
|
|
+ " from attachments a where j.job_id=? and j.state='queued' and j.attempt=? and j.attachment_id=? and j.pipeline_version=? and a.id=j.attachment_id"
|
|
+ " returning a.filename", (rs, n) -> new Claim(m.job_id(), m.attachment_id(), m.attempt(), m.pipeline_version(), rs.getString(1)),
|
|
ACTOR, LEASE_SECONDS, m.job_id(), m.attempt(), m.attachment_id(), m.pipeline_version());
|
|
if (claimed.isEmpty()) {
|
|
// Причина — в audit: по нему эксперимент считает погашенные дубли.
|
|
String why = db.query("select state,attempt from jobs where job_id=?", rs -> !rs.next() ? "unknown_job"
|
|
: rs.getString(1).equals("done") ? "duplicate_done" : rs.getInt(2) > m.attempt() ? "stale_attempt"
|
|
: rs.getString(1).equals("processing") ? "duplicate_in_progress" : rs.getString(1).equals("failed") ? "dropped_failed" : "mismatch", m.job_id());
|
|
if (!why.equals("unknown_job")) audit(m.job_id(), m.attempt(), why, null);
|
|
log.info("job {} attempt {}: {}, ack without processing", m.job_id(), m.attempt(), why);
|
|
return null;
|
|
}
|
|
// ready-вложение (новая версия pipeline) остаётся ready: старый результат отдаётся до новой фиксации.
|
|
db.update("update attachments set state='processing',updated_at=now() where id=? and state='queued'", m.attachment_id());
|
|
audit(m.job_id(), m.attempt(), "claimed", null);
|
|
return claimed.get(0);
|
|
});
|
|
}
|
|
|
|
Source source(UUID attachment) {
|
|
return db.queryForObject("select content_type,size_bytes,sha256,object_key from attachments where id=?",
|
|
(rs, n) -> new Source(rs.getString(1), rs.getLong(2), rs.getString(3), rs.getString(4)), attachment);
|
|
}
|
|
|
|
// Heartbeat: продлевает аренду, но не дальше JOB_TIMEOUT_SECONDS от начала попытки.
|
|
boolean extend(Claim c) {
|
|
return db.update("update jobs set lease_until=now()+make_interval(secs=>?) where job_id=? and state='processing' and attempt=? and started_at>now()-make_interval(secs=>?)",
|
|
LEASE_SECONDS, c.job, c.attempt, TIMEOUT_SECONDS) == 1;
|
|
}
|
|
|
|
// Фиксация результата: варианты, указатель версии и done — атомарно и только владельцем попытки.
|
|
boolean complete(Claim c, List<Variant> variants) {
|
|
return Boolean.TRUE.equals(tx.execute(s -> {
|
|
if (lock(c) == null) return false;
|
|
for (Variant v : variants)
|
|
db.update("insert into attachment_variants(attachment_id,pipeline_version,name,content_type,size_bytes,width,height,sha256,object_key,job_id)"
|
|
+ " values(?,?,?,?,?,?,?,?,?,?) on conflict do nothing", c.attachment, c.version, v.name(), v.contentType(), v.size(), v.width(), v.height(), v.sha256(), v.key(), c.job);
|
|
// Указатель текущей версии только растёт: старая job после новой не перезапишет результат.
|
|
boolean current = db.update("update attachments set state='ready',error_code=null,pipeline_version=?,updated_at=now() where id=? and state<>'lost'"
|
|
+ " and coalesce(pipeline_version,0)<=?", c.version, c.attachment, c.version) == 1;
|
|
db.update("update jobs set state='done',lease_owner=null,lease_until=null,error_code=null,updated_at=now() where job_id=?", c.job);
|
|
audit(c.job, c.attempt, "done", current ? "current" : "superseded");
|
|
return true;
|
|
}));
|
|
}
|
|
|
|
// Временная ошибка: следующая попытка через backoff с jitter, после исчерпания — failed/retry_exhausted.
|
|
void retry(Claim c, Exception error) {
|
|
tx.executeWithoutResult(s -> {
|
|
Integer limit = lock(c);
|
|
if (limit == null) return;
|
|
if (c.attempt >= limit) {
|
|
failLocked(c.job, c.attempt, c.attachment, "retry_exhausted", error.toString());
|
|
return;
|
|
}
|
|
long cap = Math.min(BACKOFF_MAX_MS, BACKOFF_BASE_MS << Math.min(20, c.attempt - 1));
|
|
long delay = cap / 2 + ThreadLocalRandom.current().nextLong(cap / 2 + 1);
|
|
requeueLocked(c.job, c.attempt, c.attachment, delay, "retry_scheduled", error.toString());
|
|
});
|
|
}
|
|
|
|
void fail(Claim c, String code, String error) {
|
|
tx.executeWithoutResult(s -> {
|
|
if (lock(c) != null) failLocked(c.job, c.attempt, c.attachment, code, error);
|
|
});
|
|
}
|
|
|
|
// Fencing: попытка ещё владеет заданием; строка заблокирована до конца транзакции. null — не владеет.
|
|
Integer lock(Claim c) {
|
|
Integer limit = db.query("select attempts_limit from jobs where job_id=? and state='processing' and attempt=? for update",
|
|
rs -> rs.next() ? rs.getInt(1) : null, c.job, c.attempt);
|
|
if (limit == null) log.warn("job {} attempt {} no longer owns the lease, result discarded", c.job, c.attempt);
|
|
return limit;
|
|
}
|
|
|
|
void requeueLocked(UUID job, int attempt, UUID attachment, long delayMs, String event, String error) {
|
|
db.update("update jobs set state='queued',attempt=attempt+1,lease_owner=null,lease_until=null,next_attempt_at=now()+make_interval(secs=>?),last_error=?,updated_at=now() where job_id=?",
|
|
delayMs / 1000.0, truncate(error), job);
|
|
db.update("update attachments set state='queued',updated_at=now() where id=? and state='processing'", attachment);
|
|
db.update("insert into outbox(job_id,attempt,available_at) values(?,?,now()+make_interval(secs=>?))", job, attempt + 1, delayMs / 1000.0);
|
|
audit(job, attempt, event, "next_attempt=" + (attempt + 1) + " delay_ms=" + delayMs + " error=" + truncate(error));
|
|
}
|
|
|
|
void failLocked(UUID job, int attempt, UUID attachment, String code, String error) {
|
|
db.update("update jobs set state='failed',error_code=?,last_error=?,lease_owner=null,lease_until=null,updated_at=now() where job_id=?", code, truncate(error), job);
|
|
// failed только у вложения без готовой версии: ready (другая версия pipeline) не откатывается.
|
|
db.update("update attachments set state='failed',error_code=?,updated_at=now() where id=? and state in ('queued','processing') and pipeline_version is null", code, attachment);
|
|
audit(job, attempt, "failed", code);
|
|
}
|
|
|
|
// Истёкшая аренда: worker умер или завис (heartbeat не продлевал) — следующая попытка, без второго эффекта.
|
|
void sweep() {
|
|
tx.executeWithoutResult(s -> {
|
|
List<Object[]> expired = db.query("select job_id,attempt,attempts_limit,attachment_id from jobs where state='processing' and lease_until<now()"
|
|
+ " order by lease_until limit 100 for update skip locked", (rs, n) -> new Object[]{rs.getObject(1, UUID.class), rs.getInt(2), rs.getInt(3), rs.getObject(4, UUID.class)});
|
|
for (Object[] j : expired) {
|
|
UUID job = (UUID) j[0], attachment = (UUID) j[3];
|
|
int attempt = (int) j[1];
|
|
if (attempt >= (int) j[2]) failLocked(job, attempt, attachment, "retry_exhausted", "lease expired");
|
|
else requeueLocked(job, attempt, attachment, 0, "lease_expired", "lease expired");
|
|
}
|
|
});
|
|
}
|
|
|
|
// Staging удаляется после безопасной фиксации (ready); непривязанные и failed — после retention.
|
|
int sweepStaging(Storage storage) {
|
|
int deleted = 0;
|
|
Instant cutoff = Instant.now().minusSeconds(STAGING_RETENTION_SECONDS);
|
|
for (ListObjectsV2Response page : storage.list(Storage.STAGING)) {
|
|
Map<UUID, S3Object> objects = new HashMap<>();
|
|
for (S3Object o : page.contents()) {
|
|
try { objects.put(UUID.fromString(o.key().substring(Storage.STAGING.length())), o); } catch (IllegalArgumentException ignored) {}
|
|
}
|
|
if (objects.isEmpty()) continue;
|
|
Map<UUID, Object[]> rows = new HashMap<>();
|
|
db.query("select id,state,updated_at from attachments where id=any(?)", rs -> {
|
|
rows.put(rs.getObject(1, UUID.class), new Object[]{rs.getString(2), rs.getTimestamp(3).toInstant()});
|
|
}, (Object) objects.keySet().toArray(new UUID[0]));
|
|
for (Map.Entry<UUID, S3Object> e : objects.entrySet()) {
|
|
Object[] r = rows.get(e.getKey());
|
|
boolean delete = r == null ? e.getValue().lastModified().isBefore(cutoff)
|
|
: r[0].equals("ready") || (r[0].equals("failed") && ((Instant) r[1]).isBefore(cutoff));
|
|
if (delete) {
|
|
storage.delete(e.getValue().key());
|
|
deleted++;
|
|
}
|
|
}
|
|
}
|
|
if (deleted > 0) log.info("staging sweep deleted {} objects", deleted);
|
|
return deleted;
|
|
}
|
|
|
|
// Операторские команды: docker compose run --rm jobctl <command>.
|
|
int cli(String[] args) throws IOException {
|
|
String command = args.length > 0 ? args[0] : "";
|
|
Object result;
|
|
switch (command) {
|
|
case "status" -> {
|
|
Map<String, Object> r = new LinkedHashMap<>();
|
|
r.put("jobs", db.queryForList("select pipeline_version,state,count(*) from jobs group by 1,2 order by 1,2"));
|
|
r.put("attachments", db.queryForList("select state,count(*) from attachments group by 1 order by 1"));
|
|
r.put("outbox_pending", db.queryForObject("select count(*) from outbox", Long.class));
|
|
r.put("failed", db.queryForList("select job_id::text,attachment_id::text,pipeline_version,attempt,error_code,updated_at::text from jobs where state='failed' order by updated_at desc limit 50"));
|
|
result = r;
|
|
}
|
|
case "replay" -> {
|
|
List<UUID> ids = args.length > 1 && args[1].equals("--all-failed")
|
|
? db.queryForList("select job_id from jobs where state='failed'", UUID.class) : ids(args, 1);
|
|
Map<String, String> r = new LinkedHashMap<>();
|
|
for (UUID id : ids) r.put(id.toString(), replay(id));
|
|
result = r;
|
|
}
|
|
case "reprocess" -> {
|
|
if (args.length < 3 || !args[1].startsWith("--pipeline-version=")) return usage();
|
|
int version = Integer.parseInt(args[1].substring(19));
|
|
boolean all = args[2].equals("--all");
|
|
result = Map.of("jobs_created", reprocess(version, all ? List.of() : ids(args, 2), all));
|
|
}
|
|
case "redispatch" -> {
|
|
long older = args.length > 1 && args[1].startsWith("--older-than=") ? Long.parseLong(args[1].substring(13)) : 300;
|
|
result = Map.of("republished", redispatch(older));
|
|
}
|
|
default -> { return usage(); }
|
|
}
|
|
System.out.println(JSON.copy().enable(SerializationFeature.INDENT_OUTPUT).writeValueAsString(result));
|
|
return 0;
|
|
}
|
|
|
|
static int usage() {
|
|
System.err.println("usage: jobctl status | replay <job_id>...|--all-failed | reprocess --pipeline-version=N <attachment_id>...|--all | redispatch [--older-than=SECONDS]");
|
|
return 2;
|
|
}
|
|
|
|
static List<UUID> ids(String[] args, int from) {
|
|
List<UUID> ids = new ArrayList<>();
|
|
for (int i = from; i < args.length; i++) ids.add(UUID.fromString(args[i]));
|
|
return ids;
|
|
}
|
|
|
|
// Явный replay только failed: done-задание не исполняется повторно, прежний эффект защищён PK и fencing.
|
|
String replay(UUID job) {
|
|
return tx.execute(s -> {
|
|
Object[] j = db.query("select state,attempt,attachment_id from jobs where job_id=? for update",
|
|
rs -> rs.next() ? new Object[]{rs.getString(1), rs.getInt(2), rs.getObject(3, UUID.class)} : null, job);
|
|
if (j == null) return "unknown job";
|
|
if (!j[0].equals("failed")) return "skipped: job is " + j[0];
|
|
int attempt = (int) j[1] + 1;
|
|
db.update("update jobs set state='queued',attempt=?,attempts_limit=?,error_code=null,next_attempt_at=now(),updated_at=now() where job_id=?",
|
|
attempt, attempt + MAX_ATTEMPTS - 1, job);
|
|
db.update("update attachments set state='queued',error_code=null,updated_at=now() where id=? and state='failed'", j[2]);
|
|
db.update("insert into outbox(job_id,attempt) values(?,?)", job, attempt);
|
|
audit(job, attempt, "replayed", "operator");
|
|
return "queued as attempt " + attempt;
|
|
});
|
|
}
|
|
|
|
// Новая версия pipeline — новая логическая работа (новый job_id); на одну версию не больше одной job.
|
|
int reprocess(int version, List<UUID> ids, boolean all) {
|
|
return db.update("with j as (insert into jobs(job_id,attachment_id,pipeline_version,state,attempt,attempts_limit,created_at,updated_at)"
|
|
+ " select gen_random_uuid(),id,?,'queued',1,?,now(),now() from attachments where state in ('queued','processing','ready') and (? or id=any(?))"
|
|
+ " on conflict (attachment_id,pipeline_version) do nothing returning job_id) insert into outbox(job_id,attempt) select job_id,1 from j",
|
|
version, MAX_ATTEMPTS, all, ids.toArray(new UUID[0]));
|
|
}
|
|
|
|
// Ручное восстановление queued-заданий, чьё сообщение потеряно (например, ушло в DLQ по delivery-limit).
|
|
int redispatch(long olderThanSeconds) {
|
|
return db.update("with r as (update jobs j set next_attempt_at=now(),updated_at=now() where state='queued' and next_attempt_at<now()-make_interval(secs=>?)"
|
|
+ " and not exists(select 1 from outbox o where o.job_id=j.job_id) returning job_id,attempt) insert into outbox(job_id,attempt) select job_id,attempt from r", olderThanSeconds);
|
|
}
|
|
|
|
void audit(UUID job, int attempt, String event, String detail) {
|
|
db.update("insert into job_audit(job_id,attempt,event,actor,detail) values(?,?,?,?,?)", job, attempt, event, ACTOR, detail);
|
|
}
|
|
|
|
static String truncate(String s) {
|
|
return s == null || s.length() <= 500 ? s : s.substring(0, 500);
|
|
}
|
|
|
|
// Аварийная остановка процесса в заданной точке, один раз на вложение (отметка коммитится до halt).
|
|
void crash(String point, UUID attachment, String filename) {
|
|
if (!FAULTS || filename == null || !filename.startsWith("fault-" + point)) return;
|
|
Integer fired = separate.execute(s -> db.update("insert into fault_marks(point,attachment_id) values(?,?) on conflict do nothing", point, attachment));
|
|
if (fired == null || fired == 0) return;
|
|
log.warn("FAULT INJECTION {}: halting process (attachment {})", point, attachment);
|
|
// Пауза, чтобы уже отправляемый HTTP-ответ этого процесса дошёл: падение — именно в фоновой точке.
|
|
try { Thread.sleep(1000); } catch (InterruptedException ignored) {}
|
|
Runtime.getRuntime().halt(137);
|
|
}
|
|
|
|
// Захваченная попытка в worker: флаги для heartbeat/таймаута и ссылки, которые прерывает abort().
|
|
static final class Claim {
|
|
final UUID job, attachment;
|
|
final int attempt, version;
|
|
final String filename;
|
|
final long started = System.nanoTime();
|
|
volatile boolean fenced;
|
|
volatile ImageReader reader;
|
|
volatile Process process;
|
|
|
|
Claim(UUID job, UUID attachment, int attempt, int version, String filename) {
|
|
this.job = job;
|
|
this.attachment = attachment;
|
|
this.attempt = attempt;
|
|
this.version = version;
|
|
this.filename = filename;
|
|
}
|
|
|
|
long elapsedSeconds() {
|
|
return (System.nanoTime() - started) / 1_000_000_000L;
|
|
}
|
|
|
|
// Перед каждым побочным эффектом: аренда потеряна — стоп без записи; превышен таймаут — временная ошибка.
|
|
void check() {
|
|
if (fenced) throw new Fenced();
|
|
if (elapsedSeconds() >= TIMEOUT_SECONDS) throw new TimedOut();
|
|
}
|
|
|
|
void abort() {
|
|
ImageReader r = reader;
|
|
if (r != null) r.abort();
|
|
Process p = process;
|
|
if (p != null) p.destroyForcibly();
|
|
}
|
|
}
|
|
|
|
static final class Fenced extends RuntimeException {
|
|
Fenced() { super("lease lost"); }
|
|
}
|
|
|
|
static final class TimedOut extends RuntimeException {
|
|
TimedOut() { super("job timeout exceeded"); }
|
|
}
|
|
|
|
static final class Permanent extends RuntimeException {
|
|
final String code;
|
|
|
|
Permanent(String code, String message) {
|
|
super(message);
|
|
this.code = code;
|
|
}
|
|
}
|
|
}
|