Files
mtusi2026lab5/src/job_repo.cpp
T
2026-11-16 09:28:38 +03:00

138 lines
7.7 KiB
C++

#include "job_repo.hpp"
#include "image_pipeline.hpp"
#include "outbox.hpp"
#include "fault_injection.hpp"
#include <iostream>
#include <optional>
#include <algorithm>
#include <random>
JobRepo::Result JobRepo::run(const std::string& job_id, const std::string& attachment_id,
int version, int attempt, const std::function<JobResult(const JobInput&)>& process) {
auto lease = pool_.acquire();
{
pqxx::nontransaction tx(*lease);
if (!tx.exec("SELECT pg_try_advisory_lock(hashtextextended($1::uuid::text, 0))",
pqxx::params{job_id})[0][0].as<bool>()) return Result::Busy;
}
struct Unlock {
pqxx::connection& conn;
const std::string& id;
~Unlock() {
try {
pqxx::nontransaction tx(conn);
tx.exec("SELECT pg_advisory_unlock(hashtextextended($1::uuid::text, 0))", pqxx::params{id});
} catch (...) {}
}
} unlock{*lease, job_id};
JobInput input;
{
pqxx::work tx(*lease);
auto rows = tx.exec(
"SELECT j.status, j.retries, j.retry_base, j.source_key, j.pipeline_version AS job_version, "
"a.pipeline_version AS desired_version, j.next_attempt_at<=now() AS due, "
"a.status AS attachment_status FROM jobs j "
"JOIN attachments a ON a.id=j.attachment_id "
"WHERE j.id=$1::uuid AND j.attachment_id=$2::uuid FOR UPDATE OF j, a",
pqxx::params{job_id, attachment_id});
if (rows.empty()) return Result::Missing;
const auto& row = rows[0];
if (row["job_version"].as<int>() != version) return Result::Missing;
const std::string status = row["status"].as<std::string>();
if (version < row["desired_version"].as<int>() || status == "superseded") {
tx.exec("UPDATE jobs SET status='superseded', updated_at=now() "
"WHERE id=$1::uuid AND status IN ('queued','processing')", pqxx::params{job_id});
tx.commit();
return Result::Superseded;
}
if (version > row["desired_version"].as<int>()) return Result::Missing;
if (status == "ready" || status == "failed") return Result::Done;
input = JobInput{version, row["source_key"].as<std::string>()};
const int retries = row["retries"].as<int>();
// A previous processing state with no advisory lock means its worker died.
if (status == "processing") {
return fail(tx, job_id, attachment_id, version, retries + 1, row["retry_base"].as<int>(), "processing_failed", false);
}
// A newer attempt is already durable in outbox/the stream; discard an old delivery.
if (attempt < retries + 1) return Result::Scheduled;
if (attempt > retries + 1) return Result::Missing;
if (!row["due"].as<bool>()) return Result::Busy;
tx.exec("UPDATE jobs SET status='processing', error_code=NULL, updated_at=now() "
"WHERE id=$1::uuid", pqxx::params{job_id});
tx.exec("UPDATE attachments SET status='processing', error_code=NULL, updated_at=now() "
"WHERE id=$1::uuid AND pipeline_version=$2 AND published_version IS NULL",
pqxx::params{attachment_id, version});
tx.commit();
}
try {
lab_worker_fault("worker_before_effect", job_id, version);
const JobResult result = process(input);
pqxx::work tx(*lease);
const auto current = tx.exec(
"SELECT a.pipeline_version FROM jobs j JOIN attachments a ON a.id=j.attachment_id "
"WHERE j.id=$1::uuid FOR UPDATE OF j,a", pqxx::params{job_id});
tx.exec("INSERT INTO attachment_results(attachment_id,pipeline_version,job_id,original_key,variants) "
"VALUES($1::uuid,$2,$3::uuid,$4,$5::jsonb) ON CONFLICT DO NOTHING",
pqxx::params{attachment_id, version, job_id, result.original_key, result.variants_json});
const bool publish = current[0][0].as<int>() == version;
if (publish) {
// The pointer and complete result become visible in one transaction.
tx.exec("UPDATE attachments a SET status='ready', published_version=$2, "
"s3_key=r.original_key, variants=r.variants, error_code=NULL, updated_at=now() "
"FROM attachment_results r WHERE a.id=$1::uuid AND a.pipeline_version=$2 "
"AND r.attachment_id=a.id AND r.pipeline_version=$2",
pqxx::params{attachment_id, version});
}
tx.exec("UPDATE jobs SET status=$2, error_code=NULL, updated_at=now() WHERE id=$1::uuid",
pqxx::params{job_id, publish ? "ready" : "superseded"});
tx.commit();
lab_worker_fault("worker_after_commit", job_id, version);
return publish ? Result::Done : Result::Superseded;
} catch (const std::exception& e) {
const auto* invalid = dynamic_cast<const InvalidImage*>(&e);
const std::string code = invalid ? invalid->what() : "processing_failed";
pqxx::work tx(*lease);
auto rows = tx.exec("SELECT j.status,j.retries,j.retry_base,a.pipeline_version FROM jobs j "
"JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1::uuid FOR UPDATE OF j,a",
pqxx::params{job_id});
if (rows.empty()) return Result::Missing;
if (rows[0]["pipeline_version"].as<int>() != version) {
tx.exec("UPDATE jobs SET status='superseded',updated_at=now() WHERE id=$1::uuid "
"AND status IN ('queued','processing')", pqxx::params{job_id});
tx.commit();
return Result::Superseded;
}
const std::string status = rows[0]["status"].as<std::string>();
// A lost COMMIT reply must not roll back an already published ready result.
if (status == "ready" || status == "failed") return Result::Done;
return fail(tx, job_id, attachment_id, version, rows[0]["retries"].as<int>() + 1,
rows[0]["retry_base"].as<int>(), code, invalid != nullptr);
}
}
JobRepo::Result JobRepo::fail(pqxx::work& tx, const std::string& job_id,
const std::string& attachment_id, int version, int retries, int retry_base,
const std::string& code, bool permanent) {
const bool failed = permanent || retries - retry_base >= policy_.max_attempts;
long long ceiling = policy_.base_delay_ms;
for (int i = 1; i < retries - retry_base; ++i) ceiling = std::min<long long>(policy_.max_delay_ms, ceiling * 2);
ceiling = std::min<long long>(policy_.max_delay_ms, ceiling);
thread_local std::mt19937 rng(std::random_device{}());
const int delay = failed ? 0 : std::uniform_int_distribution<int>(
static_cast<int>((ceiling + 1) / 2), static_cast<int>(ceiling))(rng);
tx.exec("UPDATE jobs SET status=$2, retries=$3, error_code=$4, "
"next_attempt_at=now()+($5::bigint * interval '1 millisecond'), updated_at=now() "
"WHERE id=$1::uuid",
pqxx::params{job_id, failed ? "failed" : "queued", retries, code, delay});
tx.exec("UPDATE attachments SET status=$2, error_code=$3, updated_at=now() "
"WHERE id=$1::uuid AND status='processing' AND pipeline_version=$4 AND published_version IS NULL",
pqxx::params{attachment_id, failed ? "failed" : "queued",
failed ? std::optional<std::string>(code) : std::nullopt, version});
if (!failed) enqueue_attachment_job(tx, job_id);
tx.commit();
std::clog << "job=" << job_id << " retries=" << retries << " error_code=" << code
<< " retry_delay_ms=" << delay << '\n';
return failed ? Result::Done : Result::Scheduled;
}