forked from mtusi2026/mtusi2026lab7
242 lines
6.8 KiB
Go
242 lines
6.8 KiB
Go
package objectstore
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/minio/minio-go/v7"
|
|
"github.com/minio/minio-go/v7/pkg/credentials"
|
|
|
|
"go.opentelemetry.io/otel"
|
|
"go.opentelemetry.io/otel/attribute"
|
|
"go.opentelemetry.io/otel/codes"
|
|
"go.opentelemetry.io/otel/trace"
|
|
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab7/internal/model"
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab7/internal/security/tlsconfig"
|
|
)
|
|
|
|
var ErrNotFound = errors.New("object not found")
|
|
|
|
type Config struct {
|
|
Endpoint string
|
|
Region string
|
|
Bucket string
|
|
AccessKey string
|
|
SecretKey string
|
|
UseTLS bool
|
|
CAPath string
|
|
Timeout time.Duration
|
|
}
|
|
|
|
type Store struct {
|
|
client *minio.Client
|
|
bucket string
|
|
timeout time.Duration
|
|
}
|
|
|
|
func New(config Config) (*Store, error) {
|
|
endpoint, err := endpointHost(config.Endpoint, config.UseTLS)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if strings.TrimSpace(config.Bucket) == "" || config.Timeout <= 0 {
|
|
return nil, errors.New("invalid object store configuration")
|
|
}
|
|
options := &minio.Options{
|
|
Creds: credentials.NewStaticV4(config.AccessKey, config.SecretKey, ""),
|
|
Secure: config.UseTLS,
|
|
Region: config.Region,
|
|
}
|
|
if config.UseTLS {
|
|
host, _, err := net.SplitHostPort(endpoint)
|
|
if err != nil {
|
|
return nil, errors.New("S3 endpoint must include a port")
|
|
}
|
|
tlsConfiguration, err := tlsconfig.LoadClient(tlsconfig.ClientConfig{
|
|
CAPath: config.CAPath, ServerName: host,
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("load S3 TLS configuration: %w", err)
|
|
}
|
|
transport := http.DefaultTransport.(*http.Transport).Clone()
|
|
transport.TLSClientConfig = tlsConfiguration
|
|
options.Transport = transport
|
|
}
|
|
client, err := minio.New(endpoint, options)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("create S3 client: %w", err)
|
|
}
|
|
return &Store{client: client, bucket: config.Bucket, timeout: config.Timeout}, nil
|
|
}
|
|
|
|
func endpointHost(value string, useTLS bool) (string, error) {
|
|
value = strings.TrimSpace(value)
|
|
if !strings.Contains(value, "://") {
|
|
if value == "" {
|
|
return "", errors.New("S3 endpoint is required")
|
|
}
|
|
return value, nil
|
|
}
|
|
parsed, err := url.Parse(value)
|
|
if err != nil || parsed.Host == "" || parsed.Path != "" && parsed.Path != "/" {
|
|
return "", errors.New("invalid S3 endpoint")
|
|
}
|
|
if (parsed.Scheme == "https") != useTLS || parsed.Scheme != "http" && parsed.Scheme != "https" {
|
|
return "", errors.New("S3 endpoint scheme conflicts with S3_USE_TLS")
|
|
}
|
|
return parsed.Host, nil
|
|
}
|
|
|
|
func (s *Store) Ping(ctx context.Context) error {
|
|
ctx, cancel := context.WithTimeout(ctx, s.timeout)
|
|
defer cancel()
|
|
if _, err := s.client.GetBucketLocation(ctx, s.bucket); err != nil {
|
|
return fmt.Errorf("check S3 bucket location: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Store) Put(ctx context.Context, key, contentType string, data []byte) error {
|
|
ctx, span := s.startSpan(ctx, "PUT")
|
|
defer span.End()
|
|
ctx, cancel := context.WithTimeout(ctx, s.timeout)
|
|
defer cancel()
|
|
_, err := s.client.PutObject(
|
|
ctx, s.bucket, key, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{ContentType: contentType},
|
|
)
|
|
if err != nil {
|
|
recordSpanError(span, err)
|
|
return fmt.Errorf("put S3 object: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Store) Get(ctx context.Context, key string) (io.ReadCloser, error) {
|
|
ctx, span := s.startSpan(ctx, "GET")
|
|
ctx, cancel := context.WithTimeout(ctx, s.timeout)
|
|
object, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{})
|
|
if err != nil {
|
|
cancel()
|
|
recordSpanError(span, err)
|
|
span.End()
|
|
return nil, objectError("get S3 object", err)
|
|
}
|
|
if _, err := object.Stat(); err != nil {
|
|
cancel()
|
|
_ = object.Close()
|
|
recordSpanError(span, err)
|
|
span.End()
|
|
return nil, objectError("stat S3 object", err)
|
|
}
|
|
return &tracedReader{ReadCloser: &timeoutReader{ReadCloser: object, cancel: cancel}, span: span}, nil
|
|
}
|
|
|
|
func (s *Store) Stat(ctx context.Context, key string) (model.StoredObject, error) {
|
|
ctx, cancel := context.WithTimeout(ctx, s.timeout)
|
|
defer cancel()
|
|
info, err := s.client.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{})
|
|
if err != nil {
|
|
return model.StoredObject{}, objectError("stat S3 object", err)
|
|
}
|
|
return model.StoredObject{Key: info.Key, Size: info.Size, LastModified: info.LastModified}, nil
|
|
}
|
|
|
|
func (s *Store) Delete(ctx context.Context, key string) error {
|
|
ctx, span := s.startSpan(ctx, "DELETE")
|
|
defer span.End()
|
|
ctx, cancel := context.WithTimeout(ctx, s.timeout)
|
|
defer cancel()
|
|
if err := s.client.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{}); err != nil {
|
|
recordSpanError(span, err)
|
|
return objectError("delete S3 object", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Store) List(ctx context.Context, prefix, after string, limit int) ([]model.StoredObject, string, error) {
|
|
if limit < 1 || limit > 1000 {
|
|
return nil, "", errors.New("object list limit must be between 1 and 1000")
|
|
}
|
|
ctx, cancel := context.WithTimeout(ctx, s.timeout)
|
|
defer cancel()
|
|
objects := make([]model.StoredObject, 0, limit+1)
|
|
options := minio.ListObjectsOptions{Prefix: prefix, Recursive: true, StartAfter: after, MaxKeys: limit + 1}
|
|
for info := range s.client.ListObjects(ctx, s.bucket, options) {
|
|
if info.Err != nil {
|
|
return nil, "", fmt.Errorf("list S3 objects: %w", info.Err)
|
|
}
|
|
objects = append(objects, model.StoredObject{Key: info.Key, Size: info.Size, LastModified: info.LastModified})
|
|
if len(objects) == limit+1 {
|
|
cancel()
|
|
break
|
|
}
|
|
}
|
|
var next string
|
|
if len(objects) > limit {
|
|
objects = objects[:limit]
|
|
next = objects[len(objects)-1].Key
|
|
}
|
|
return objects, next, nil
|
|
}
|
|
|
|
type timeoutReader struct {
|
|
io.ReadCloser
|
|
cancel context.CancelFunc
|
|
}
|
|
|
|
type tracedReader struct {
|
|
io.ReadCloser
|
|
span trace.Span
|
|
}
|
|
|
|
func (r *tracedReader) Read(buffer []byte) (int, error) {
|
|
read, err := r.ReadCloser.Read(buffer)
|
|
if err != nil && !errors.Is(err, io.EOF) {
|
|
recordSpanError(r.span, err)
|
|
}
|
|
return read, err
|
|
}
|
|
|
|
func (r *tracedReader) Close() error {
|
|
err := r.ReadCloser.Close()
|
|
if err != nil {
|
|
recordSpanError(r.span, err)
|
|
}
|
|
r.span.End()
|
|
return err
|
|
}
|
|
|
|
func (r *timeoutReader) Close() error {
|
|
err := r.ReadCloser.Close()
|
|
r.cancel()
|
|
return err
|
|
}
|
|
|
|
func objectError(action string, err error) error {
|
|
response := minio.ToErrorResponse(err)
|
|
if response.Code == "NoSuchKey" || response.Code == "NoSuchObject" {
|
|
return ErrNotFound
|
|
}
|
|
return fmt.Errorf("%s: %w", action, err)
|
|
}
|
|
|
|
func (s *Store) startSpan(ctx context.Context, operation string) (context.Context, trace.Span) {
|
|
return otel.Tracer("objectstore").Start(ctx, "s3 "+operation, trace.WithSpanKind(trace.SpanKindClient),
|
|
trace.WithAttributes(attribute.String("storage.system", "s3"), attribute.String("storage.operation.name", operation),
|
|
attribute.String("s3.bucket.name", s.bucket)))
|
|
}
|
|
|
|
func recordSpanError(span trace.Span, err error) {
|
|
span.RecordError(err)
|
|
span.SetStatus(codes.Error, "S3 operation failed")
|
|
}
|