// Package objectstore is the only place that knows how to talk to the private S3 // bucket. PostgreSQL remains the source of truth for metadata; this package // stores and returns opaque bytes only. package objectstore import ( "context" "errors" "io" "net/http" "strings" "time" "github.com/aws/aws-sdk-go-v2/aws" awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/smithy-go" ) var ErrNotFound = errors.New("object not found") type Config struct { Endpoint string Region string Bucket string AccessKey string SecretKey string } type Client struct { s3 *s3.Client bucket string } // Object describes one key returned by List. It intentionally exposes only the // fields the reconciler needs. type Object struct { Key string Size int64 LastModified time.Time } func New(cfg Config) (*Client, error) { awsCfg, err := awsconfig.LoadDefaultConfig(context.Background(), awsconfig.WithRegion(cfg.Region), awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( cfg.AccessKey, cfg.SecretKey, "")), ) if err != nil { return nil, err } // Bounded retry count keeps a broken S3 dependency from turning an // HTTP upload into an unbounded wait. Context deadlines in the handlers set // the overall request budget. awsCfg.RetryMaxAttempts = 2 transport := http.DefaultTransport.(*http.Transport).Clone() transport.MaxIdleConnsPerHost = 20 transport.MaxIdleConns = 40 client := s3.NewFromConfig(awsCfg, func(options *s3.Options) { // RustFS is addressed by its Compose service name, so virtual-host // addressing cannot work without extra DNS entries. Path-style is the // portable choice for every S3-compatible server used in this stand. options.UsePathStyle = true options.BaseEndpoint = aws.String(strings.TrimRight(cfg.Endpoint, "/")) options.HTTPClient = &http.Client{Transport: transport} }) return &Client{s3: client, bucket: cfg.Bucket}, nil } func (c *Client) Bucket() string { return c.bucket } // Ping is used by /health/ready. HeadBucket is the cheapest authenticated call // that proves both the endpoint and the scoped credentials work. func (c *Client) Ping(ctx context.Context) error { pingCtx, cancel := context.WithTimeout(ctx, 2*time.Second) defer cancel() _, err := c.s3.HeadBucket(pingCtx, &s3.HeadBucketInput{Bucket: aws.String(c.bucket)}) return mapError(err) } func (c *Client) Put(ctx context.Context, key string, body io.ReadSeeker, size int64, contentType string) error { _, err := c.s3.PutObject(ctx, &s3.PutObjectInput{ Bucket: aws.String(c.bucket), Key: aws.String(key), Body: body, ContentLength: aws.Int64(size), ContentType: aws.String(contentType), }) return mapError(err) } func (c *Client) Get(ctx context.Context, key string) (io.ReadCloser, int64, error) { output, err := c.s3.GetObject(ctx, &s3.GetObjectInput{ Bucket: aws.String(c.bucket), Key: aws.String(key), }) if err != nil { return nil, 0, mapError(err) } size := int64(0) if output.ContentLength != nil { size = *output.ContentLength } return output.Body, size, nil } func (c *Client) Delete(ctx context.Context, key string) error { _, err := c.s3.DeleteObject(ctx, &s3.DeleteObjectInput{ Bucket: aws.String(c.bucket), Key: aws.String(key), }) return mapError(err) } func (c *Client) List(ctx context.Context, prefix string) ([]Object, error) { var objects []Object var continuation *string for { output, err := c.s3.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ Bucket: aws.String(c.bucket), Prefix: aws.String(prefix), ContinuationToken: continuation, }) if err != nil { return nil, mapError(err) } for _, item := range output.Contents { if item.Key == nil { continue } object := Object{Key: *item.Key} if item.Size != nil { object.Size = *item.Size } if item.LastModified != nil { object.LastModified = *item.LastModified } objects = append(objects, object) } if output.IsTruncated == nil || !*output.IsTruncated { return objects, nil } continuation = output.NextContinuationToken if continuation == nil { return objects, nil } } } // mapError turns a missing object into a sentinel the HTTP layer can map to // 404, while every other failure stays a dependency error (503). func mapError(err error) error { if err == nil { return nil } var apiErr smithy.APIError if errors.As(err, &apiErr) { switch apiErr.ErrorCode() { case "NoSuchKey", "NoSuchBucket", "NotFound", "XMinioInvalidObjectName": return ErrNotFound } } return err }