forked from mtusi2026/mtusi2026lab7
479 lines
18 KiB
Go
479 lines
18 KiB
Go
package tests
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/binary"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"hash/crc32"
|
|
"image"
|
|
"image/jpeg"
|
|
"io"
|
|
"mime/multipart"
|
|
"net/http"
|
|
"os"
|
|
"os/exec"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"messenger/src/backend/repositories"
|
|
"messenger/src/backend/services"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
)
|
|
|
|
func directInput(data []byte, mime string) services.DirectUploadInput {
|
|
sum := sha256.Sum256(data)
|
|
return services.DirectUploadInput{Filename: "direct.png", ContentType: mime, SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(sum[:])}
|
|
}
|
|
|
|
func beginDirect(t *testing.T, f *testFixture, store *repositories.AttachmentStore, objects *repositories.ObjectStore, user, chat string, data []byte, mime string) (services.DirectUploadResponse, *services.AttachmentService) {
|
|
t.Helper()
|
|
t.Setenv("S3_PUBLIC_ENDPOINT", os.Getenv("S3_ENDPOINT"))
|
|
service := services.NewAttachmentService(store, objects)
|
|
result, err := service.InitDirect(context.Background(), user, chat, directInput(data, mime))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
rememberDirect(t, store, objects, result.ID, chat, user)
|
|
return result, service
|
|
}
|
|
|
|
func rememberDirect(t *testing.T, store *repositories.AttachmentStore, objects *repositories.ObjectStore, id, chat, user string) {
|
|
t.Helper()
|
|
u, err := store.GetDirect(context.Background(), id, chat, user)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() {
|
|
objects.Delete(context.Background(), u.StagingKey)
|
|
objects.Delete(context.Background(), u.FinalKey)
|
|
})
|
|
}
|
|
|
|
func postDirect(t *testing.T, p repositories.PostUpload, data []byte, change func(map[string]string)) int {
|
|
t.Helper()
|
|
fields := map[string]string{}
|
|
for k, v := range p.Fields {
|
|
fields[k] = v
|
|
}
|
|
if change != nil {
|
|
change(fields)
|
|
}
|
|
var body bytes.Buffer
|
|
writer := multipart.NewWriter(&body)
|
|
for k, v := range fields {
|
|
if err := writer.WriteField(k, v); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
file, err := writer.CreateFormFile("file", "image.png")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err = file.Write(data); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = writer.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
req, err := http.NewRequest("POST", p.URL, &body)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
req.Header.Set("Content-Type", writer.FormDataContentType())
|
|
client := http.Client{Timeout: 10 * time.Second}
|
|
response, err := client.Do(req)
|
|
if err != nil {
|
|
t.Fatal("S3 form request failed")
|
|
}
|
|
defer response.Body.Close()
|
|
io.Copy(io.Discard, response.Body)
|
|
return response.StatusCode
|
|
}
|
|
|
|
func requireDirectError(t *testing.T, err error, status int, code string) {
|
|
t.Helper()
|
|
var e repositories.UploadError
|
|
if !errors.As(err, &e) || e.Status != status || e.Code != code {
|
|
t.Fatalf("expected %d %s, got %v", status, code, err)
|
|
}
|
|
}
|
|
|
|
func TestDirectUploadFlow(t *testing.T) {
|
|
f, store, objects, user, chat := uploadFixture(t)
|
|
png := append(uploadPNG(t), []byte("aaaa")...)
|
|
var jpg bytes.Buffer
|
|
if err := jpeg.Encode(&jpg, image.NewRGBA(image.Rect(0, 0, 8, 8)), nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, sample := range []struct {
|
|
data []byte
|
|
mime string
|
|
}{{png, "image/png"}, {jpg.Bytes(), "image/jpeg"}} {
|
|
result, service := beginDirect(t, f, store, objects, user, chat, sample.data, sample.mime)
|
|
if result.ExpiresAt.Sub(time.Now()) > 121*time.Second || result.CompleteBefore.Sub(result.ExpiresAt) < 599*time.Second {
|
|
t.Fatal("wrong upload deadlines")
|
|
}
|
|
if status := postDirect(t, result.Upload, sample.data, nil); status != 204 {
|
|
t.Fatalf("POST: %d", status)
|
|
}
|
|
a, err := service.CompleteDirect(context.Background(), user, chat, result.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
router := attachmentRouter(f, objects)
|
|
attachmentCall(t, f, router, "GET", "/api/v1/attachments/"+a.ID+"/content", user, "", nil, 409)
|
|
if a.Status != "queued" {
|
|
t.Fatal("direct complete must queue processing")
|
|
}
|
|
var jobs int
|
|
if err = f.db.QueryRow(context.Background(), "SELECT count(*) FROM attachment_jobs WHERE attachment_id=$1", a.ID).Scan(&jobs); err != nil || jobs != 1 {
|
|
t.Fatal("durable job missing", err)
|
|
}
|
|
message := []byte(`{"text":"","attachment_ids":["` + a.ID + `"]}`)
|
|
attachmentCall(t, f, router, "POST", "/api/v1/chats/"+chat+"/messages", user, "application/json", message, 201)
|
|
// Lost success response is represented by ignoring a, then asking another service.
|
|
again, err := services.NewAttachmentService(store, objects).CompleteDirect(context.Background(), user, chat, result.ID)
|
|
if err != nil || again.ID != a.ID || !again.CreatedAt.Equal(a.CreatedAt) {
|
|
t.Fatal("complete is not idempotent", err)
|
|
}
|
|
replacement := append([]byte(nil), sample.data...)
|
|
replacement[len(replacement)-1] ^= 1
|
|
if status := postDirect(t, result.Upload, replacement, nil); status != 204 {
|
|
t.Fatal("expected valid staging replay")
|
|
}
|
|
final, err := objects.Get(context.Background(), a.ObjectKey)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
original, err := io.ReadAll(final)
|
|
final.Close()
|
|
if err != nil || !bytes.Equal(original, sample.data) {
|
|
t.Fatal("staging replay modified final")
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestDirectUploadValidationHTTP(t *testing.T) {
|
|
f, _, objects, user, chat := uploadFixture(t)
|
|
t.Setenv("S3_PUBLIC_ENDPOINT", os.Getenv("S3_ENDPOINT"))
|
|
router := attachmentRouter(f, objects)
|
|
path := "/api/v1/chats/" + chat + "/attachment-uploads"
|
|
input := directInput(uploadPNG(t), "image/png")
|
|
valid, _ := json.Marshal(input)
|
|
for _, sample := range []struct {
|
|
body []byte
|
|
status int
|
|
}{
|
|
{[]byte(`null`), 400}, {[]byte(`{}`), 400}, {append(append([]byte(nil), valid...), []byte(` {}`)...), 400},
|
|
{[]byte(strings.Repeat("x", 4097)), 400}, {[]byte(`{"filename":"x","size_bytes":1.5}`), 400},
|
|
} {
|
|
attachmentCall(t, f, router, "POST", path, user, "application/json", sample.body, sample.status)
|
|
}
|
|
for _, sample := range []struct {
|
|
mutate func(*services.DirectUploadInput)
|
|
status int
|
|
}{
|
|
{func(i *services.DirectUploadInput) { i.Filename = "../image.png" }, 400},
|
|
{func(i *services.DirectUploadInput) { i.SHA256 = "invalid" }, 400},
|
|
{func(i *services.DirectUploadInput) { i.SizeBytes = 0 }, 400},
|
|
{func(i *services.DirectUploadInput) { i.SizeBytes = 10485761 }, 413},
|
|
{func(i *services.DirectUploadInput) { i.ContentType = "text/plain" }, 415},
|
|
} {
|
|
i := input
|
|
sample.mutate(&i)
|
|
body, _ := json.Marshal(i)
|
|
attachmentCall(t, f, router, "POST", path, user, "application/json", body, sample.status)
|
|
}
|
|
store := repositories.NewAttachmentStore(f.db)
|
|
w := attachmentCall(t, f, router, "POST", path, user, "application/json", valid, 201)
|
|
var result services.DirectUploadResponse
|
|
if err := json.Unmarshal(w.Body.Bytes(), &result); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
rememberDirect(t, store, objects, result.ID, chat, user)
|
|
if status := postDirect(t, result.Upload, uploadPNG(t), nil); status != 204 {
|
|
t.Fatal("HTTP-created form failed")
|
|
}
|
|
attachmentCall(t, f, router, "POST", path+"/"+result.ID+"/complete", user, "", nil, 202)
|
|
}
|
|
|
|
func TestDirectUploadValidationContents(t *testing.T) {
|
|
f, store, objects, user, chat := uploadFixture(t)
|
|
png := uploadPNG(t)
|
|
huge := append([]byte(nil), png...)
|
|
binary.BigEndian.PutUint32(huge[16:20], 10001)
|
|
binary.BigEndian.PutUint32(huge[20:24], 4000)
|
|
binary.BigEndian.PutUint32(huge[29:33], crc32.ChecksumIEEE(huge[12:29]))
|
|
for _, sample := range []struct {
|
|
data []byte
|
|
mime, code string
|
|
status int
|
|
}{
|
|
{png[:40], "image/png", "unsupported_media_type", 415},
|
|
{png, "image/jpeg", "unsupported_media_type", 415},
|
|
{huge, "image/png", "payload_too_large", 413},
|
|
} {
|
|
result, service := beginDirect(t, f, store, objects, user, chat, sample.data, sample.mime)
|
|
if postDirect(t, result.Upload, sample.data, nil) != 204 {
|
|
t.Fatal("S3 refused matching declared policy")
|
|
}
|
|
_, err := service.CompleteDirect(context.Background(), user, chat, result.ID)
|
|
requireDirectError(t, err, sample.status, sample.code)
|
|
_, err = service.CompleteDirect(context.Background(), user, chat, result.ID)
|
|
requireDirectError(t, err, sample.status, sample.code)
|
|
}
|
|
result, service := beginDirect(t, f, store, objects, user, chat, png, "image/png")
|
|
wrong := append([]byte(nil), png...)
|
|
wrong[len(wrong)-1] ^= 1
|
|
if postDirect(t, result.Upload, wrong, nil) != 204 {
|
|
t.Fatal("form POST failed")
|
|
}
|
|
_, err := service.CompleteDirect(context.Background(), user, chat, result.ID)
|
|
requireDirectError(t, err, 400, "upload_checksum_mismatch")
|
|
result, service = beginDirect(t, f, store, objects, user, chat, png, "image/png")
|
|
if postDirect(t, result.Upload, append(png, 1), nil) < 400 {
|
|
t.Fatal("S3 did not enforce exact policy size")
|
|
}
|
|
u, _ := store.GetDirect(context.Background(), result.ID, chat, user)
|
|
// Simulate a trusted-side fault that bypassed policy; complete must still check.
|
|
if err = objects.Put(context.Background(), u.StagingKey, bytes.NewReader(png[:40]), 40, "image/png"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, err = service.CompleteDirect(context.Background(), user, chat, result.ID)
|
|
requireDirectError(t, err, 400, "upload_size_mismatch")
|
|
}
|
|
|
|
func TestDirectUploadAccess(t *testing.T) {
|
|
f, store, objects, user, chat := uploadFixture(t)
|
|
png := uploadPNG(t)
|
|
result, service := beginDirect(t, f, store, objects, user, chat, png, "image/png")
|
|
eve, _ := createTestUser(t, f.users, "eve", "Eve")
|
|
bob, _ := createTestUser(t, f.users, "bob", "Bob")
|
|
if _, err := f.db.Exec(context.Background(), `INSERT INTO chat_members(chat_id,user_id) VALUES($1,$2)`, chat, bob.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
other := createTestChat(t, f.chats, "other", []string{user})
|
|
router := attachmentRouter(f, objects)
|
|
path := "/api/v1/chats/" + chat + "/attachment-uploads"
|
|
for _, actor := range []string{eve.ID, bob.ID} {
|
|
attachmentCall(t, f, router, "POST", path+"/"+result.ID+"/complete", actor, "", nil, 404)
|
|
}
|
|
attachmentCall(t, f, router, "POST", path+"/"+result.ID+"/complete", "", "", nil, 401)
|
|
attachmentCall(t, f, router, "POST", "/api/v1/chats/"+other.ID+"/attachment-uploads/"+result.ID+"/complete", user, "", nil, 404)
|
|
body, _ := json.Marshal(directInput(png, "image/png"))
|
|
attachmentCall(t, f, router, "POST", path, eve.ID, "application/json", body, 404)
|
|
u, _ := store.GetDirect(context.Background(), result.ID, chat, user)
|
|
attachmentCall(t, f, router, "POST", "/api/v1/chats/"+chat+"/messages", user, "application/json", []byte(`{"text":"","attachment_ids":["`+u.AttachmentID+`"]}`), 404)
|
|
_, err := service.CompleteDirect(context.Background(), user, chat, result.ID)
|
|
requireDirectError(t, err, 409, "upload_not_uploaded")
|
|
for _, change := range []func(map[string]string){
|
|
func(fields map[string]string) { fields["key"] = "originals/" + uuid.NewString() },
|
|
func(fields map[string]string) { fields["Content-Type"] = "image/jpeg" },
|
|
func(fields map[string]string) { fields["x-amz-signature"] = strings.Repeat("0", 64) },
|
|
} {
|
|
if postDirect(t, result.Upload, png, change) < 400 {
|
|
t.Fatal("S3 accepted tampered form")
|
|
}
|
|
}
|
|
expired, err := objects.PresignDirect(context.Background(), u.StagingKey, "image/png", int64(len(png)), time.Now().Add(-time.Minute))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if postDirect(t, expired, png, nil) < 400 {
|
|
t.Fatal("S3 accepted expired form")
|
|
}
|
|
if _, err = f.db.Exec(context.Background(), `UPDATE direct_uploads SET complete_before=clock_timestamp()-interval '1 second' WHERE id=$1`, result.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, err = service.CompleteDirect(context.Background(), user, chat, result.ID)
|
|
requireDirectError(t, err, 409, "upload_expired")
|
|
}
|
|
|
|
type directFault struct {
|
|
*repositories.ObjectStore
|
|
afterCopy func() error
|
|
failGet bool
|
|
}
|
|
|
|
func (f directFault) Copy(ctx context.Context, source, dest, etag string) error {
|
|
if err := f.ObjectStore.Copy(ctx, source, dest, etag); err != nil {
|
|
return err
|
|
}
|
|
if f.afterCopy != nil {
|
|
return f.afterCopy()
|
|
}
|
|
return nil
|
|
}
|
|
func (f directFault) Get(ctx context.Context, key string) (io.ReadCloser, error) {
|
|
if f.failGet {
|
|
return nil, errors.New("injected GET failure")
|
|
}
|
|
return f.ObjectStore.Get(ctx, key)
|
|
}
|
|
|
|
func TestDirectUploadRecoveryConcurrentComplete(t *testing.T) {
|
|
f, store, objects, user, chat := uploadFixture(t)
|
|
data := uploadPNG(t)
|
|
result, _ := beginDirect(t, f, store, objects, user, chat, data, "image/png")
|
|
if postDirect(t, result.Upload, data, nil) != 204 {
|
|
t.Fatal("POST failed")
|
|
}
|
|
var wg sync.WaitGroup
|
|
ids := make(chan string, 2)
|
|
errs := make(chan error, 2)
|
|
for i := 0; i < 2; i++ {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
a, err := services.NewAttachmentService(store, objects).CompleteDirect(context.Background(), user, chat, result.ID)
|
|
ids <- a.ID
|
|
errs <- err
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
close(ids)
|
|
close(errs)
|
|
for err := range errs {
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
first := <-ids
|
|
if second := <-ids; first == "" || first != second {
|
|
t.Fatal("concurrent complete produced different attachments")
|
|
}
|
|
var n int
|
|
if err := f.db.QueryRow(context.Background(), `SELECT count(*) FROM attachments`).Scan(&n); err != nil || n != 1 {
|
|
t.Fatal("unexpected metadata count", n, err)
|
|
}
|
|
}
|
|
|
|
func TestDirectUploadRecoveryFailures(t *testing.T) {
|
|
for _, name := range []string{"copy-result-loss", "get-failure", "metadata-rejection", "database-disconnect"} {
|
|
t.Run(name, func(t *testing.T) {
|
|
f, store, objects, user, chat := uploadFixture(t)
|
|
data := uploadPNG(t)
|
|
result, normal := beginDirect(t, f, store, objects, user, chat, data, "image/png")
|
|
if postDirect(t, result.Upload, data, nil) != 204 {
|
|
t.Fatal("POST failed")
|
|
}
|
|
fault := directFault{ObjectStore: objects}
|
|
switch name {
|
|
case "copy-result-loss":
|
|
fault.afterCopy = func() error { return errors.New("COPY acknowledgement lost") }
|
|
case "get-failure":
|
|
fault.failGet = true
|
|
case "metadata-rejection":
|
|
if _, err := f.db.Exec(context.Background(), `ALTER TABLE attachments ADD CONSTRAINT reject_direct CHECK (filename <> 'direct.png')`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
case "database-disconnect":
|
|
fault.afterCopy = func() error {
|
|
_, err := f.db.Exec(context.Background(), `SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE application_name=$1 AND state='idle in transaction' AND pid<>pg_backend_pid()`, f.db.Config().ConnConfig.RuntimeParams["application_name"])
|
|
return err
|
|
}
|
|
}
|
|
_, err := services.NewAttachmentService(store, fault).CompleteDirect(context.Background(), user, chat, result.ID)
|
|
if err == nil {
|
|
t.Fatal("expected injected failure")
|
|
}
|
|
var count int
|
|
if err = f.db.QueryRow(context.Background(), `SELECT count(*) FROM upload_attempts`).Scan(&count); err != nil || count != 1 {
|
|
t.Fatal("durable journal missing", err)
|
|
}
|
|
if err = f.db.QueryRow(context.Background(), `SELECT count(*) FROM attachments`).Scan(&count); err != nil || count != 0 {
|
|
t.Fatal("published incomplete attachment", err)
|
|
}
|
|
if name == "metadata-rejection" {
|
|
if _, err = f.db.Exec(context.Background(), `ALTER TABLE attachments DROP CONSTRAINT reject_direct`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if _, err = normal.CompleteDirect(context.Background(), user, chat, result.ID); err != nil {
|
|
t.Fatal("retry failed", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDirectUploadRecoveryProcessExit(t *testing.T) {
|
|
f, store, objects, user, chat := uploadFixture(t)
|
|
data := uploadPNG(t)
|
|
result, _ := beginDirect(t, f, store, objects, user, chat, data, "image/png")
|
|
if postDirect(t, result.Upload, data, nil) != 204 {
|
|
t.Fatal("POST failed")
|
|
}
|
|
child := exec.Command(os.Args[0], "-test.run=^TestDirectUploadCrashHelper$")
|
|
child.Env = append(os.Environ(), "DIRECT_CRASH_HELPER=1", "UPLOAD_TEST_DATABASE="+f.db.Config().ConnString(), "UPLOAD_TEST_SCHEMA="+f.db.Config().ConnConfig.RuntimeParams["search_path"], "UPLOAD_TEST_USER="+user, "UPLOAD_TEST_CHAT="+chat, "DIRECT_UPLOAD_ID="+result.ID)
|
|
output, err := child.CombinedOutput()
|
|
var exit *exec.ExitError
|
|
if !errors.As(err, &exit) || exit.ExitCode() != 23 {
|
|
t.Fatalf("helper did not crash after COPY: %v %s", err, output)
|
|
}
|
|
ageAndClean(t, f, store, objects)
|
|
}
|
|
|
|
func TestDirectUploadCrashHelper(t *testing.T) {
|
|
if os.Getenv("DIRECT_CRASH_HELPER") != "1" {
|
|
return
|
|
}
|
|
cfg, err := pgxpool.ParseConfig(os.Getenv("UPLOAD_TEST_DATABASE"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cfg.ConnConfig.RuntimeParams["search_path"] = os.Getenv("UPLOAD_TEST_SCHEMA")
|
|
cfg.ConnConfig.RuntimeParams["application_name"] = os.Getenv("UPLOAD_TEST_SCHEMA")
|
|
db, err := pgxpool.NewWithConfig(context.Background(), cfg)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer db.Close()
|
|
objects := uploadObjects(t)
|
|
fault := directFault{ObjectStore: objects, afterCopy: func() error { os.Exit(23); return nil }}
|
|
_, err = services.NewAttachmentService(repositories.NewAttachmentStore(db), fault).CompleteDirect(context.Background(), os.Getenv("UPLOAD_TEST_USER"), os.Getenv("UPLOAD_TEST_CHAT"), os.Getenv("DIRECT_UPLOAD_ID"))
|
|
t.Fatalf("helper failed before exit: %v", err)
|
|
}
|
|
|
|
func TestDirectUploadRecoverySharedSlotAndCancellation(t *testing.T) {
|
|
f, store, objects, user, chat := uploadFixture(t)
|
|
data := uploadPNG(t)
|
|
result, _ := beginDirect(t, f, store, objects, user, chat, data, "image/png")
|
|
if postDirect(t, result.Upload, data, nil) != 204 {
|
|
t.Fatal("POST failed")
|
|
}
|
|
entered, release := make(chan struct{}), make(chan struct{})
|
|
fault := directFault{ObjectStore: objects, afterCopy: func() error {
|
|
close(entered)
|
|
<-release
|
|
return errors.New("injected operation cancellation")
|
|
}}
|
|
service := services.NewAttachmentService(store, fault)
|
|
done := make(chan error, 1)
|
|
go func() { _, err := service.CompleteDirect(context.Background(), user, chat, result.ID); done <- err }()
|
|
select {
|
|
case <-entered:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("COPY not reached")
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
_, proxyErr := service.Upload(ctx, user, chat, "waiting.png", "image/png", bytes.NewReader(data))
|
|
_, directErr := service.CompleteDirect(ctx, user, chat, result.ID)
|
|
close(release)
|
|
if err := <-done; err == nil {
|
|
t.Fatal("injected failure ignored")
|
|
}
|
|
if !errors.Is(proxyErr, context.Canceled) || !errors.Is(directErr, context.Canceled) {
|
|
t.Fatalf("waiting requests ignored cancellation: proxy=%v direct=%v", proxyErr, directErr)
|
|
}
|
|
ageAndClean(t, f, store, objects)
|
|
}
|