lab6
This commit is contained in:
@@ -0,0 +1,290 @@
|
||||
#include "telemetry.hpp"
|
||||
#include "s3_client.hpp"
|
||||
|
||||
#include <miniocpp/args.h>
|
||||
#include <miniocpp/client.h>
|
||||
#include <miniocpp/providers.h>
|
||||
#include <miniocpp/request.h>
|
||||
#include <miniocpp/response.h>
|
||||
|
||||
#include <cstdlib>
|
||||
#include <fstream>
|
||||
#include <stdexcept>
|
||||
|
||||
namespace {
|
||||
|
||||
std::string read_file_trim(const std::string& path) {
|
||||
std::ifstream in(path);
|
||||
if (!in) {
|
||||
throw std::runtime_error("cannot open secret/file: " + path);
|
||||
}
|
||||
std::string s((std::istreambuf_iterator<char>(in)),
|
||||
std::istreambuf_iterator<char>());
|
||||
while (!s.empty() && (s.back() == '\n' || s.back() == '\r' || s.back() == ' ')) {
|
||||
s.pop_back();
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
std::string env_or(const char* key, const char* fallback = "") {
|
||||
const char* v = std::getenv(key);
|
||||
return (v && *v) ? std::string(v) : std::string(fallback);
|
||||
}
|
||||
|
||||
void parse_endpoint(const std::string& endpoint, std::string& host, unsigned& port,
|
||||
bool& https) {
|
||||
https = true;
|
||||
host = endpoint;
|
||||
port = 0;
|
||||
if (host.rfind("http://", 0) == 0) {
|
||||
https = false;
|
||||
host = host.substr(7);
|
||||
} else if (host.rfind("https://", 0) == 0) {
|
||||
https = true;
|
||||
host = host.substr(8);
|
||||
}
|
||||
// strip path
|
||||
const auto slash = host.find('/');
|
||||
if (slash != std::string::npos) {
|
||||
host = host.substr(0, slash);
|
||||
}
|
||||
const auto colon = host.rfind(':');
|
||||
if (colon != std::string::npos) {
|
||||
port = static_cast<unsigned>(std::stoul(host.substr(colon + 1)));
|
||||
host = host.substr(0, colon);
|
||||
}
|
||||
if (port == 0) {
|
||||
port = https ? 443u : 80u;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
struct S3Client::Impl {
|
||||
minio::s3::BaseUrl base_url;
|
||||
minio::s3::BaseUrl presign_url;
|
||||
std::unique_ptr<minio::creds::StaticProvider> provider;
|
||||
std::unique_ptr<minio::s3::Client> client;
|
||||
std::unique_ptr<minio::s3::Client> presign_client; // null → use client
|
||||
};
|
||||
|
||||
S3Client::Config S3Client::from_env() {
|
||||
Config c;
|
||||
// Ops endpoint (Stat/Get/Put/Delete). Presign may use S3_PRESIGN_ENDPOINT
|
||||
// (read in ctor) so signed Host matches the client-reachable address.
|
||||
c.endpoint = env_or("S3_ENDPOINT", "http://minio:9000");
|
||||
c.region = env_or("S3_REGION", "us-east-1");
|
||||
c.bucket = env_or("S3_BUCKET");
|
||||
if (c.bucket.empty()) {
|
||||
throw std::runtime_error("S3_BUCKET is required");
|
||||
}
|
||||
|
||||
const std::string key_file = env_or("S3_ACCESS_KEY_ID_FILE");
|
||||
const std::string secret_file = env_or("S3_SECRET_ACCESS_KEY_FILE");
|
||||
if (key_file.empty() || secret_file.empty()) {
|
||||
throw std::runtime_error(
|
||||
"S3_ACCESS_KEY_ID_FILE and S3_SECRET_ACCESS_KEY_FILE are required");
|
||||
}
|
||||
c.access_key = read_file_trim(key_file);
|
||||
c.secret_key = read_file_trim(secret_file);
|
||||
if (c.access_key.empty() || c.secret_key.empty()) {
|
||||
throw std::runtime_error("S3 credentials files are empty");
|
||||
}
|
||||
return c;
|
||||
}
|
||||
|
||||
S3Client::S3Client(Config cfg) : cfg_(std::move(cfg)), impl_(std::make_unique<Impl>()) {
|
||||
if (cfg_.endpoint.empty() || cfg_.bucket.empty() ||
|
||||
cfg_.access_key.empty() || cfg_.secret_key.empty()) {
|
||||
throw std::runtime_error("S3Client: incomplete config");
|
||||
}
|
||||
|
||||
std::string host;
|
||||
unsigned port = 0;
|
||||
bool https = false;
|
||||
parse_endpoint(cfg_.endpoint, host, port, https);
|
||||
|
||||
impl_->base_url = minio::s3::BaseUrl(host, https, cfg_.region);
|
||||
impl_->base_url.port = port;
|
||||
// MinIO requires path-style addressing.
|
||||
impl_->base_url.virtual_style = false;
|
||||
|
||||
impl_->provider = std::make_unique<minio::creds::StaticProvider>(
|
||||
cfg_.access_key, cfg_.secret_key);
|
||||
impl_->client =
|
||||
std::make_unique<minio::s3::Client>(impl_->base_url, impl_->provider.get());
|
||||
|
||||
// Separate BaseUrl for presigning when S3_PRESIGN_ENDPOINT differs.
|
||||
const std::string presign_ep = env_or("S3_PRESIGN_ENDPOINT");
|
||||
if (!presign_ep.empty() && presign_ep != cfg_.endpoint) {
|
||||
std::string ph;
|
||||
unsigned pp = 0;
|
||||
bool phttps = false;
|
||||
parse_endpoint(presign_ep, ph, pp, phttps);
|
||||
impl_->presign_url = minio::s3::BaseUrl(ph, phttps, cfg_.region);
|
||||
impl_->presign_url.port = pp;
|
||||
impl_->presign_url.virtual_style = false;
|
||||
impl_->presign_client = std::make_unique<minio::s3::Client>(
|
||||
impl_->presign_url, impl_->provider.get());
|
||||
}
|
||||
}
|
||||
|
||||
S3Client::~S3Client() = default;
|
||||
|
||||
void S3Client::put_object(const std::string& key,
|
||||
std::istream& body,
|
||||
std::int64_t size_bytes,
|
||||
const std::string& content_type) {
|
||||
telemetry::Span operation("s3.put_object", opentelemetry::trace::SpanKind::kClient);
|
||||
if (size_bytes < 0) {
|
||||
throw std::invalid_argument("put_object: negative size");
|
||||
}
|
||||
|
||||
// part_size=0 → library chooses multipart sizing; streams from `body`.
|
||||
minio::s3::PutObjectArgs args(body, static_cast<long>(size_bytes), 0);
|
||||
args.bucket = cfg_.bucket;
|
||||
args.object = key;
|
||||
if (!content_type.empty()) {
|
||||
args.content_type = content_type;
|
||||
}
|
||||
|
||||
minio::s3::PutObjectResponse resp = impl_->client->PutObject(args);
|
||||
if (!resp) {
|
||||
throw std::runtime_error("S3 PutObject failed: " + resp.Error().String());
|
||||
}
|
||||
}
|
||||
|
||||
void S3Client::get_object(const std::string& key, std::ostream& out) {
|
||||
telemetry::Span operation("s3.get_object", opentelemetry::trace::SpanKind::kClient);
|
||||
get_object(key, [&](const char* data, std::size_t n) {
|
||||
out.write(data, static_cast<std::streamsize>(n));
|
||||
if (!out) {
|
||||
throw std::runtime_error("get_object: write failed");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
std::int64_t S3Client::get_object(
|
||||
const std::string& key,
|
||||
const std::function<void(const char*, std::size_t)>& on_chunk) {
|
||||
telemetry::Span operation("s3.get_object", opentelemetry::trace::SpanKind::kClient);
|
||||
std::int64_t total = 0;
|
||||
minio::s3::GetObjectArgs args;
|
||||
args.bucket = cfg_.bucket;
|
||||
args.object = key;
|
||||
args.datafunc = [&](minio::http::DataFunctionArgs da) -> bool {
|
||||
on_chunk(da.datachunk.data(), da.datachunk.size());
|
||||
total += static_cast<std::int64_t>(da.datachunk.size());
|
||||
return true;
|
||||
};
|
||||
|
||||
minio::s3::GetObjectResponse resp = impl_->client->GetObject(args);
|
||||
if (!resp) {
|
||||
throw std::runtime_error("S3 GetObject failed: " + resp.Error().String());
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
void S3Client::copy_object(const std::string& src_key,
|
||||
const std::string& dst_key) {
|
||||
telemetry::Span operation("s3.copy_object", opentelemetry::trace::SpanKind::kClient);
|
||||
if (src_key.empty() || dst_key.empty() || src_key == dst_key) {
|
||||
throw std::invalid_argument("copy_object: bad keys");
|
||||
}
|
||||
minio::s3::CopyObjectArgs args;
|
||||
args.bucket = cfg_.bucket;
|
||||
args.object = dst_key;
|
||||
args.source.bucket = cfg_.bucket;
|
||||
args.source.object = src_key;
|
||||
minio::s3::CopyObjectResponse resp = impl_->client->CopyObject(args);
|
||||
if (!resp) {
|
||||
throw std::runtime_error("S3 CopyObject failed: " + resp.Error().String());
|
||||
}
|
||||
}
|
||||
|
||||
void S3Client::delete_object(const std::string& key) {
|
||||
telemetry::Span operation("s3.delete_object", opentelemetry::trace::SpanKind::kClient);
|
||||
minio::s3::RemoveObjectArgs args;
|
||||
args.bucket = cfg_.bucket;
|
||||
args.object = key;
|
||||
minio::s3::RemoveObjectResponse resp = impl_->client->RemoveObject(args);
|
||||
if (!resp) {
|
||||
throw std::runtime_error("S3 DeleteObject failed: " + resp.Error().String());
|
||||
}
|
||||
}
|
||||
|
||||
bool S3Client::stat_object(const std::string& key, std::int64_t& size_bytes,
|
||||
std::string& etag) {
|
||||
telemetry::Span operation("s3.stat_object", opentelemetry::trace::SpanKind::kClient);
|
||||
minio::s3::StatObjectArgs args;
|
||||
args.bucket = cfg_.bucket;
|
||||
args.object = key;
|
||||
minio::s3::StatObjectResponse resp = impl_->client->StatObject(args);
|
||||
if (!resp) {
|
||||
// NoSuchKey / Not Found → absent
|
||||
const std::string err = resp.Error().String();
|
||||
if (resp.code == "NoSuchKey" || resp.status_code == 404) {
|
||||
return false;
|
||||
}
|
||||
throw std::runtime_error("S3 StatObject failed: " + err);
|
||||
}
|
||||
size_bytes = static_cast<std::int64_t>(resp.size);
|
||||
etag = resp.etag;
|
||||
return true;
|
||||
}
|
||||
|
||||
std::string S3Client::presign_put(const std::string& key,
|
||||
unsigned expiry_seconds,
|
||||
const std::string& content_type) {
|
||||
telemetry::Span operation("s3.presign_put", opentelemetry::trace::SpanKind::kClient);
|
||||
if (expiry_seconds < 1 || expiry_seconds > 7 * 24 * 3600u) {
|
||||
throw std::invalid_argument("presign_put: bad expiry");
|
||||
}
|
||||
minio::s3::Client* c =
|
||||
impl_->presign_client ? impl_->presign_client.get() : impl_->client.get();
|
||||
|
||||
// Note: minio-cpp query-presign signs method/host/path/query only — it does
|
||||
// not bind Content-Type into the signature. Clients should still send
|
||||
// Content-Type so the object metadata is correct; complete() re-validates
|
||||
// magic/sha256 against the session.
|
||||
(void)content_type;
|
||||
minio::s3::GetPresignedObjectUrlArgs args;
|
||||
args.bucket = cfg_.bucket;
|
||||
args.object = key;
|
||||
args.method = minio::http::Method::kPut;
|
||||
args.expiry_seconds = expiry_seconds;
|
||||
|
||||
minio::s3::GetPresignedObjectUrlResponse resp = c->GetPresignedObjectUrl(args);
|
||||
if (!resp) {
|
||||
throw std::runtime_error("S3 PresignPut failed: " + resp.Error().String());
|
||||
}
|
||||
return resp.url;
|
||||
}
|
||||
|
||||
std::vector<std::string> S3Client::old_staging(const std::string& cursor,
|
||||
int retention_seconds,
|
||||
std::string& next_cursor) {
|
||||
telemetry::Span operation("s3.old_staging", opentelemetry::trace::SpanKind::kClient);
|
||||
minio::s3::ListObjectsArgs args;
|
||||
args.bucket = cfg_.bucket;
|
||||
args.prefix = "staging/";
|
||||
args.recursive = true;
|
||||
args.max_keys = 100;
|
||||
args.start_after = cursor;
|
||||
auto cutoff = minio::utils::UtcTime::Now();
|
||||
cutoff.Add(-retention_seconds);
|
||||
auto result = impl_->client->ListObjects(args);
|
||||
std::vector<std::string> keys;
|
||||
next_cursor.clear();
|
||||
for (int count = 0; result && count < 100; ++count) {
|
||||
const auto& item = *result;
|
||||
if (!item) throw std::runtime_error("S3 staging listing failed");
|
||||
if (!item.is_prefix && item.last_modified <= cutoff) keys.push_back(item.name);
|
||||
if (count == 99) {
|
||||
next_cursor = item.name;
|
||||
break;
|
||||
}
|
||||
++result;
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
Reference in New Issue
Block a user