138 lines
7.7 KiB
C++
138 lines
7.7 KiB
C++
#include "job_repo.hpp"
|
|
#include "image_pipeline.hpp"
|
|
#include "outbox.hpp"
|
|
#include "fault_injection.hpp"
|
|
|
|
#include <iostream>
|
|
#include <optional>
|
|
#include <algorithm>
|
|
#include <random>
|
|
|
|
JobRepo::Result JobRepo::run(const std::string& job_id, const std::string& attachment_id,
|
|
int version, int attempt, const std::function<JobResult(const JobInput&)>& process) {
|
|
auto lease = pool_.acquire();
|
|
{
|
|
pqxx::nontransaction tx(*lease);
|
|
if (!tx.exec("SELECT pg_try_advisory_lock(hashtextextended($1::uuid::text, 0))",
|
|
pqxx::params{job_id})[0][0].as<bool>()) return Result::Busy;
|
|
}
|
|
struct Unlock {
|
|
pqxx::connection& conn;
|
|
const std::string& id;
|
|
~Unlock() {
|
|
try {
|
|
pqxx::nontransaction tx(conn);
|
|
tx.exec("SELECT pg_advisory_unlock(hashtextextended($1::uuid::text, 0))", pqxx::params{id});
|
|
} catch (...) {}
|
|
}
|
|
} unlock{*lease, job_id};
|
|
JobInput input;
|
|
{
|
|
pqxx::work tx(*lease);
|
|
auto rows = tx.exec(
|
|
"SELECT j.status, j.retries, j.retry_base, j.source_key, j.pipeline_version AS job_version, "
|
|
"a.pipeline_version AS desired_version, j.next_attempt_at<=now() AS due, "
|
|
"a.status AS attachment_status FROM jobs j "
|
|
"JOIN attachments a ON a.id=j.attachment_id "
|
|
"WHERE j.id=$1::uuid AND j.attachment_id=$2::uuid FOR UPDATE OF j, a",
|
|
pqxx::params{job_id, attachment_id});
|
|
if (rows.empty()) return Result::Missing;
|
|
const auto& row = rows[0];
|
|
if (row["job_version"].as<int>() != version) return Result::Missing;
|
|
const std::string status = row["status"].as<std::string>();
|
|
if (version < row["desired_version"].as<int>() || status == "superseded") {
|
|
tx.exec("UPDATE jobs SET status='superseded', updated_at=now() "
|
|
"WHERE id=$1::uuid AND status IN ('queued','processing')", pqxx::params{job_id});
|
|
tx.commit();
|
|
return Result::Superseded;
|
|
}
|
|
if (version > row["desired_version"].as<int>()) return Result::Missing;
|
|
if (status == "ready" || status == "failed") return Result::Done;
|
|
input = JobInput{version, row["source_key"].as<std::string>()};
|
|
const int retries = row["retries"].as<int>();
|
|
// A previous processing state with no advisory lock means its worker died.
|
|
if (status == "processing") {
|
|
return fail(tx, job_id, attachment_id, version, retries + 1, row["retry_base"].as<int>(), "processing_failed", false);
|
|
}
|
|
// A newer attempt is already durable in outbox/the stream; discard an old delivery.
|
|
if (attempt < retries + 1) return Result::Scheduled;
|
|
if (attempt > retries + 1) return Result::Missing;
|
|
if (!row["due"].as<bool>()) return Result::Busy;
|
|
tx.exec("UPDATE jobs SET status='processing', error_code=NULL, updated_at=now() "
|
|
"WHERE id=$1::uuid", pqxx::params{job_id});
|
|
tx.exec("UPDATE attachments SET status='processing', error_code=NULL, updated_at=now() "
|
|
"WHERE id=$1::uuid AND pipeline_version=$2 AND published_version IS NULL",
|
|
pqxx::params{attachment_id, version});
|
|
tx.commit();
|
|
}
|
|
try {
|
|
lab_worker_fault("worker_before_effect", job_id, version);
|
|
const JobResult result = process(input);
|
|
pqxx::work tx(*lease);
|
|
const auto current = tx.exec(
|
|
"SELECT a.pipeline_version FROM jobs j JOIN attachments a ON a.id=j.attachment_id "
|
|
"WHERE j.id=$1::uuid FOR UPDATE OF j,a", pqxx::params{job_id});
|
|
tx.exec("INSERT INTO attachment_results(attachment_id,pipeline_version,job_id,original_key,variants) "
|
|
"VALUES($1::uuid,$2,$3::uuid,$4,$5::jsonb) ON CONFLICT DO NOTHING",
|
|
pqxx::params{attachment_id, version, job_id, result.original_key, result.variants_json});
|
|
const bool publish = current[0][0].as<int>() == version;
|
|
if (publish) {
|
|
// The pointer and complete result become visible in one transaction.
|
|
tx.exec("UPDATE attachments a SET status='ready', published_version=$2, "
|
|
"s3_key=r.original_key, variants=r.variants, error_code=NULL, updated_at=now() "
|
|
"FROM attachment_results r WHERE a.id=$1::uuid AND a.pipeline_version=$2 "
|
|
"AND r.attachment_id=a.id AND r.pipeline_version=$2",
|
|
pqxx::params{attachment_id, version});
|
|
}
|
|
tx.exec("UPDATE jobs SET status=$2, error_code=NULL, updated_at=now() WHERE id=$1::uuid",
|
|
pqxx::params{job_id, publish ? "ready" : "superseded"});
|
|
tx.commit();
|
|
lab_worker_fault("worker_after_commit", job_id, version);
|
|
return publish ? Result::Done : Result::Superseded;
|
|
} catch (const std::exception& e) {
|
|
const auto* invalid = dynamic_cast<const InvalidImage*>(&e);
|
|
const std::string code = invalid ? invalid->what() : "processing_failed";
|
|
pqxx::work tx(*lease);
|
|
auto rows = tx.exec("SELECT j.status,j.retries,j.retry_base,a.pipeline_version FROM jobs j "
|
|
"JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1::uuid FOR UPDATE OF j,a",
|
|
pqxx::params{job_id});
|
|
if (rows.empty()) return Result::Missing;
|
|
if (rows[0]["pipeline_version"].as<int>() != version) {
|
|
tx.exec("UPDATE jobs SET status='superseded',updated_at=now() WHERE id=$1::uuid "
|
|
"AND status IN ('queued','processing')", pqxx::params{job_id});
|
|
tx.commit();
|
|
return Result::Superseded;
|
|
}
|
|
const std::string status = rows[0]["status"].as<std::string>();
|
|
// A lost COMMIT reply must not roll back an already published ready result.
|
|
if (status == "ready" || status == "failed") return Result::Done;
|
|
return fail(tx, job_id, attachment_id, version, rows[0]["retries"].as<int>() + 1,
|
|
rows[0]["retry_base"].as<int>(), code, invalid != nullptr);
|
|
}
|
|
}
|
|
|
|
JobRepo::Result JobRepo::fail(pqxx::work& tx, const std::string& job_id,
|
|
const std::string& attachment_id, int version, int retries, int retry_base,
|
|
const std::string& code, bool permanent) {
|
|
const bool failed = permanent || retries - retry_base >= policy_.max_attempts;
|
|
long long ceiling = policy_.base_delay_ms;
|
|
for (int i = 1; i < retries - retry_base; ++i) ceiling = std::min<long long>(policy_.max_delay_ms, ceiling * 2);
|
|
ceiling = std::min<long long>(policy_.max_delay_ms, ceiling);
|
|
thread_local std::mt19937 rng(std::random_device{}());
|
|
const int delay = failed ? 0 : std::uniform_int_distribution<int>(
|
|
static_cast<int>((ceiling + 1) / 2), static_cast<int>(ceiling))(rng);
|
|
tx.exec("UPDATE jobs SET status=$2, retries=$3, error_code=$4, "
|
|
"next_attempt_at=now()+($5::bigint * interval '1 millisecond'), updated_at=now() "
|
|
"WHERE id=$1::uuid",
|
|
pqxx::params{job_id, failed ? "failed" : "queued", retries, code, delay});
|
|
tx.exec("UPDATE attachments SET status=$2, error_code=$3, updated_at=now() "
|
|
"WHERE id=$1::uuid AND status='processing' AND pipeline_version=$4 AND published_version IS NULL",
|
|
pqxx::params{attachment_id, failed ? "failed" : "queued",
|
|
failed ? std::optional<std::string>(code) : std::nullopt, version});
|
|
if (!failed) enqueue_attachment_job(tx, job_id);
|
|
tx.commit();
|
|
std::clog << "job=" << job_id << " retries=" << retries << " error_code=" << code
|
|
<< " retry_delay_ms=" << delay << '\n';
|
|
return failed ? Result::Done : Result::Scheduled;
|
|
}
|