207 lines
6.8 KiB
Go
207 lines
6.8 KiB
Go
package testenv
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/docker/docker/api/types/container"
|
|
"github.com/docker/go-connections/nat"
|
|
"github.com/google/uuid"
|
|
"github.com/testcontainers/testcontainers-go"
|
|
"github.com/testcontainers/testcontainers-go/wait"
|
|
|
|
"messenger/internal/platform/s3store"
|
|
)
|
|
|
|
const S3Image = "chrislusf/seaweedfs:4.53@sha256:ccad23e18639c8b98cee17e7100b95830a06077b56d8e5b006d6dcf8ec5e94a0"
|
|
|
|
// S3 keeps independent API, reconciler and bootstrap identities. Store uses the
|
|
// reconciler identity so tests can inspect and clean up their objects.
|
|
type S3 struct {
|
|
Store, APIStore, Admin *s3store.Store
|
|
Endpoint, Region, Bucket string
|
|
GRPCAddress string
|
|
AccessKey, SecretKey string
|
|
APIAccessKey, APISecretKey string
|
|
AdminAccessKey, AdminSecretKey string
|
|
container testcontainers.Container
|
|
volume string
|
|
identities, options []byte
|
|
}
|
|
|
|
func StartS3(ctx context.Context) (*S3, error) {
|
|
useDockerDesktopPipe()
|
|
s := &S3{Region: "us-east-1", Bucket: "messenger-attachments", volume: "lab4-s3-test-" + uuid.NewString()}
|
|
keys := []*string{&s.AccessKey, &s.SecretKey, &s.APIAccessKey, &s.APISecretKey, &s.AdminAccessKey, &s.AdminSecretKey}
|
|
for _, key := range keys {
|
|
b := make([]byte, 24)
|
|
if _, err := rand.Read(b); err != nil {
|
|
return nil, err
|
|
}
|
|
*key = hex.EncodeToString(b)
|
|
}
|
|
type credential struct {
|
|
AccessKey string `json:"accessKey"`
|
|
SecretKey string `json:"secretKey"`
|
|
}
|
|
type identity struct {
|
|
Name string `json:"name"`
|
|
Credentials []credential `json:"credentials"`
|
|
Actions []string `json:"actions"`
|
|
}
|
|
config := struct {
|
|
Identities []identity `json:"identities"`
|
|
}{Identities: []identity{
|
|
{"bootstrap", []credential{{s.AdminAccessKey, s.AdminSecretKey}}, []string{"Admin", "Read", "Write", "List"}},
|
|
{"api", []credential{{s.APIAccessKey, s.APISecretKey}}, []string{"Read:" + s.Bucket, "Write:" + s.Bucket}},
|
|
{"reconciler", []credential{{s.AccessKey, s.SecretKey}}, []string{"List:" + s.Bucket, "Read:" + s.Bucket, "Write:" + s.Bucket}},
|
|
}}
|
|
var err error
|
|
s.identities, err = json.Marshal(config)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
_, source, _, _ := runtime.Caller(0)
|
|
s.options, err = os.ReadFile(filepath.Join(filepath.Dir(source), "../../infra/s3/seaweedfs.options"))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read S3 options: %w", err)
|
|
}
|
|
startCtx, cancel := context.WithTimeout(ctx, 90*time.Second)
|
|
defer cancel()
|
|
if err := s.launch(startCtx); err != nil {
|
|
s.Close()
|
|
return nil, err
|
|
}
|
|
if err := s.Admin.Bootstrap(startCtx); err != nil {
|
|
s.Close()
|
|
return nil, err
|
|
}
|
|
return s, nil
|
|
}
|
|
|
|
func (s *S3) launch(ctx context.Context) error {
|
|
pids := int64(256)
|
|
firstStatus := 0
|
|
c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
|
|
ContainerRequest: testcontainers.ContainerRequest{
|
|
Image: S3Image,
|
|
Cmd: []string{"server", "-options=/etc/seaweedfs/lab.options"},
|
|
Env: map[string]string{"GOMEMLIMIT": "460MiB"},
|
|
ExposedPorts: []string{"8333/tcp", "18333/tcp"},
|
|
Mounts: testcontainers.Mounts(testcontainers.VolumeMount(s.volume, "/data")),
|
|
Files: []testcontainers.ContainerFile{
|
|
{Reader: bytes.NewReader(s.identities), ContainerFilePath: "/run/secrets/s3_identities", FileMode: 0444},
|
|
{Reader: bytes.NewReader(s.options), ContainerFilePath: "/etc/seaweedfs/lab.options", FileMode: 0444},
|
|
},
|
|
HostConfigModifier: func(h *container.HostConfig) {
|
|
h.Resources.Memory = 512 * 1024 * 1024
|
|
h.Resources.NanoCPUs = 1_000_000_000
|
|
h.Resources.PidsLimit = &pids
|
|
h.PortBindings = nat.PortMap{"8333/tcp": {{HostIP: "127.0.0.1"}}, "18333/tcp": {{HostIP: "127.0.0.1"}}}
|
|
},
|
|
// The first reachable S3 response must already reject anonymous access.
|
|
WaitingFor: wait.ForHTTP("/").WithPort("8333/tcp").WithStatusCodeMatcher(func(code int) bool { firstStatus = code; return true }).WithStartupTimeout(60 * time.Second),
|
|
},
|
|
Started: true,
|
|
})
|
|
s.container = c
|
|
if err != nil {
|
|
return fmt.Errorf("start S3: %w", err)
|
|
}
|
|
if firstStatus != http.StatusForbidden {
|
|
return fmt.Errorf("first anonymous S3 response was %d, expected 403", firstStatus)
|
|
}
|
|
s.Endpoint, err = c.PortEndpoint(ctx, "8333/tcp", "http")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
s.GRPCAddress, err = c.PortEndpoint(ctx, "18333/tcp", "")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
s.Store, err = s3store.New(s.Endpoint, s.Region, s.Bucket, s.AccessKey, s.SecretKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
s.APIStore, err = s3store.New(s.Endpoint, s.Region, s.Bucket, s.APIAccessKey, s.APISecretKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
s.Admin, err = s3store.New(s.Endpoint, s.Region, s.Bucket, s.AdminAccessKey, s.AdminSecretKey)
|
|
return err
|
|
}
|
|
|
|
func (s *S3) Stop(ctx context.Context) error {
|
|
timeout := 5 * time.Second
|
|
return s.container.Stop(ctx, &timeout)
|
|
}
|
|
|
|
func (s *S3) Start(ctx context.Context) error {
|
|
if err := s.container.Start(ctx); err != nil {
|
|
return err
|
|
}
|
|
return s.Admin.WaitReady(ctx)
|
|
}
|
|
|
|
// Recreate replaces the container while keeping only its named /data volume.
|
|
// Bootstrap is deliberately not called: bucket metadata must survive as well.
|
|
func (s *S3) Recreate(ctx context.Context) error {
|
|
if err := testcontainers.TerminateContainer(s.container); err != nil {
|
|
return err
|
|
}
|
|
s.container = nil
|
|
if err := s.launch(ctx); err != nil {
|
|
return err
|
|
}
|
|
return s.Admin.WaitReady(ctx)
|
|
}
|
|
|
|
// Diagnostics retains bounded startup evidence without exposing fixture keys.
|
|
func (s *S3) Diagnostics(ctx context.Context) string {
|
|
if s.container == nil {
|
|
return "no container"
|
|
}
|
|
r, err := s.container.Logs(ctx)
|
|
if err != nil {
|
|
return err.Error()
|
|
}
|
|
defer r.Close()
|
|
redact := strings.NewReplacer(s.AccessKey, "<redacted>", s.SecretKey, "<redacted>", s.APIAccessKey, "<redacted>", s.APISecretKey, "<redacted>", s.AdminAccessKey, "<redacted>", s.AdminSecretKey, "<redacted>")
|
|
var lines []string
|
|
scan := bufio.NewScanner(io.LimitReader(r, 256*1024))
|
|
for scan.Scan() {
|
|
line := scan.Text()
|
|
lower := strings.ToLower(line)
|
|
if strings.Contains(lower, "volume") || strings.Contains(lower, "master") || strings.Contains(lower, "leader") || strings.Contains(lower, "error") || strings.Contains(lower, "lookup") {
|
|
lines = append(lines, redact.Replace(line))
|
|
if len(lines) > 60 {
|
|
lines = lines[1:]
|
|
}
|
|
}
|
|
}
|
|
return strings.Join(lines, "\n")
|
|
}
|
|
|
|
func (s *S3) Close() {
|
|
if s.container != nil {
|
|
_ = testcontainers.TerminateContainer(s.container)
|
|
}
|
|
if strings.HasPrefix(s.volume, "lab4-s3-test-") {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
_ = withDocker(ctx, func(cli *testcontainers.DockerClient) error { return cli.VolumeRemove(ctx, s.volume, true) })
|
|
}
|
|
}
|